[AIR-195] Extend config with isogen options

Change-Id: Ibde769336b955d450105c928e2be707327273879
This commit is contained in:
Dmitry Ukov 2019-10-16 12:22:37 +00:00
parent 34cca34796
commit 48e14c3b55
20 changed files with 305 additions and 162 deletions

View File

@ -9,16 +9,13 @@ import (
// NewISOGenCommand creates a new command for ISO image creation
func NewISOGenCommand(parent *cobra.Command, rootSettings *environment.AirshipCTLSettings) *cobra.Command {
settings := &isogen.Settings{AirshipCTLSettings: rootSettings}
imageGen := &cobra.Command{
Use: "isogen",
Short: "Generate bootstrap ISO image",
RunE: func(cmd *cobra.Command, args []string) error {
return isogen.GenerateBootstrapIso(settings, args)
return isogen.GenerateBootstrapIso(rootSettings, args)
},
}
settings.InitFlags(imageGen)
return imageGen
}

View File

@ -4,5 +4,4 @@ Usage:
bootstrap isogen [flags]
Flags:
-c, --config string Configuration file path for ISO builder container.
-h, --help help for isogen
-h, --help help for isogen

View File

@ -8,8 +8,10 @@ import (
"strings"
"opendev.org/airship/airshipctl/pkg/bootstrap/cloudinit"
"opendev.org/airship/airshipctl/pkg/config"
"opendev.org/airship/airshipctl/pkg/container"
"opendev.org/airship/airshipctl/pkg/document"
"opendev.org/airship/airshipctl/pkg/environment"
"opendev.org/airship/airshipctl/pkg/errors"
"opendev.org/airship/airshipctl/pkg/log"
"opendev.org/airship/airshipctl/pkg/util"
@ -22,24 +24,37 @@ const (
)
// GenerateBootstrapIso will generate data for cloud init and start ISO builder container
func GenerateBootstrapIso(settings *Settings, args []string) error {
if settings.IsogenConfigFile == "" {
log.Print("Reading config file location from global settings is not supported")
return errors.ErrNotImplemented{}
}
func GenerateBootstrapIso(settings *environment.AirshipCTLSettings, args []string) error {
ctx := context.Background()
cfg := &Config{}
if err := util.ReadYAMLFile(settings.IsogenConfigFile, &cfg); err != nil {
globalConf := settings.Config()
if err := globalConf.EnsureComplete(); err != nil {
return err
}
if err := verifyInputs(cfg, args); err != nil {
cfg, err := globalConf.CurrentContextBootstrapInfo()
if err != nil {
return err
}
docBundle, err := document.NewBundle(fs.MakeRealFS(), args[0], "")
var manifest *config.Manifest
manifest, err = globalConf.CurrentContextManifest()
if err != nil {
return err
}
// TODO (dukov) This check should be implemented as part of the config module
if manifest == nil {
return errors.ErrMissingConfig{What: "manifest for currnet context not found"}
}
if err = verifyInputs(cfg); err != nil {
return err
}
// TODO (dukov) replace with the appropriate function once it's available
// in doncument module
docBundle, err := document.NewBundle(fs.MakeRealFS(), manifest.TargetPath, "")
if err != nil {
return err
}
@ -60,12 +75,7 @@ func GenerateBootstrapIso(settings *Settings, args []string) error {
return verifyArtifacts(cfg)
}
func verifyInputs(cfg *Config, args []string) error {
if len(args) == 0 {
log.Print("Specify path to document model. Config param from global settings is not supported")
return errors.ErrNotImplemented{}
}
func verifyInputs(cfg *config.Bootstrap) error {
if cfg.Container.Volume == "" {
log.Print("Specify volume bind for ISO builder container")
return errors.ErrWrongConfig{}
@ -87,21 +97,19 @@ func verifyInputs(cfg *Config, args []string) error {
return nil
}
func getContainerCfg(cfg *Config, userData []byte, netConf []byte) (map[string][]byte, error) {
func getContainerCfg(cfg *config.Bootstrap, userData []byte, netConf []byte) map[string][]byte {
hostVol := strings.Split(cfg.Container.Volume, ":")[0]
fls := make(map[string][]byte)
fls[filepath.Join(hostVol, cfg.Builder.UserDataFileName)] = userData
fls[filepath.Join(hostVol, cfg.Builder.NetworkConfigFileName)] = netConf
builderData, err := cfg.ToYAML()
if err != nil {
return nil, err
}
// TODO (dukov) Get rid of this ugly conversion byte -> string -> byte
builderData := []byte(cfg.String())
fls[filepath.Join(hostVol, builderConfigFileName)] = builderData
return fls, nil
return fls
}
func verifyArtifacts(cfg *Config) error {
func verifyArtifacts(cfg *config.Bootstrap) error {
hostVol := strings.Split(cfg.Container.Volume, ":")[0]
metadataPath := filepath.Join(hostVol, cfg.Builder.OutputMetadataFileName)
_, err := os.Stat(metadataPath)
@ -111,22 +119,17 @@ func verifyArtifacts(cfg *Config) error {
func generateBootstrapIso(
docBubdle document.Bundle,
builder container.Container,
cfg *Config,
cfg *config.Bootstrap,
debug bool,
) error {
cntVol := strings.Split(cfg.Container.Volume, ":")[1]
log.Print("Creating cloud-init for ephemeral K8s")
userData, netConf, err := cloudinit.GetCloudData(docBubdle, EphemeralClusterAnnotation)
if err != nil {
return err
}
var fls map[string][]byte
fls, err = getContainerCfg(cfg, userData, netConf)
userData, netConf, err := cloudinit.GetCloudData(docBubdle, document.EphemeralClusterMarker)
if err != nil {
return err
}
fls := getContainerCfg(cfg, userData, netConf)
if err = util.WriteFiles(fls, 0600); err != nil {
return err
}

View File

@ -10,6 +10,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"opendev.org/airship/airshipctl/pkg/config"
"opendev.org/airship/airshipctl/pkg/document"
"opendev.org/airship/airshipctl/pkg/errors"
"opendev.org/airship/airshipctl/pkg/log"
@ -51,12 +52,12 @@ func TestBootstrapIso(t *testing.T) {
volBind := "/tmp:/dst"
testErr := fmt.Errorf("TestErr")
testCfg := &Config{
Container: Container{
testCfg := &config.Bootstrap{
Container: &config.Container{
Volume: volBind,
ContainerRuntime: "docker",
},
Builder: Builder{
Builder: &config.Builder{
UserDataFileName: "user-data",
NetworkConfigFileName: "net-conf",
},
@ -71,7 +72,7 @@ func TestBootstrapIso(t *testing.T) {
tests := []struct {
builder *mockContainer
cfg *Config
cfg *config.Bootstrap
debug bool
expectedOut []string
expectdErr error
@ -124,59 +125,53 @@ func TestBootstrapIso(t *testing.T) {
func TestVerifyInputs(t *testing.T) {
tests := []struct {
cfg *Config
cfg *config.Bootstrap
args []string
expectedErr error
}{
{
cfg: &Config{},
args: []string{},
expectedErr: errors.ErrNotImplemented{},
},
{
cfg: &Config{},
args: []string{"."},
cfg: &config.Bootstrap{
Container: &config.Container{},
},
expectedErr: errors.ErrWrongConfig{},
},
{
cfg: &Config{
Container: Container{
cfg: &config.Bootstrap{
Container: &config.Container{
Volume: "/tmp:/dst",
},
Builder: &config.Builder{},
},
args: []string{"."},
expectedErr: errors.ErrWrongConfig{},
},
{
cfg: &Config{
Container: Container{
cfg: &config.Bootstrap{
Container: &config.Container{
Volume: "/tmp",
},
Builder: Builder{
Builder: &config.Builder{
UserDataFileName: "user-data",
NetworkConfigFileName: "net-conf",
},
},
args: []string{"."},
expectedErr: nil,
},
{
cfg: &Config{
Container: Container{
cfg: &config.Bootstrap{
Container: &config.Container{
Volume: "/tmp:/dst:/dst1",
},
Builder: Builder{
Builder: &config.Builder{
UserDataFileName: "user-data",
NetworkConfigFileName: "net-conf",
},
},
args: []string{"."},
expectedErr: errors.ErrWrongConfig{},
},
}
for _, tt := range tests {
actualErr := verifyInputs(tt.cfg, tt.args)
actualErr := verifyInputs(tt.cfg)
assert.Equal(t, tt.expectedErr, actualErr)
}
}

View File

@ -1,61 +0,0 @@
package isogen
import (
"github.com/spf13/cobra"
"sigs.k8s.io/yaml"
"opendev.org/airship/airshipctl/pkg/environment"
)
const (
// TODO this should be part of a airshipctl config
EphemeralClusterAnnotation = "airshipit.org/clustertype=ephemeral"
)
// Settings settings for isogen command
type Settings struct {
*environment.AirshipCTLSettings
// Configuration file (YAML-formatted) path for ISO builder container.
IsogenConfigFile string
}
// InitFlags adds falgs and their default settings for isogen command
func (i *Settings) InitFlags(cmd *cobra.Command) {
flags := cmd.Flags()
flags.StringVarP(&i.IsogenConfigFile, "config", "c", "", "Configuration file path for ISO builder container.")
}
// Config ISO builder container configuration
type Config struct {
// Configuration parameters for container
Container Container `json:"container,omitempty"`
// Configuration parameters for ISO builder
Builder Builder `json:"builder,omitempty"`
}
// Container parameters
type Container struct {
// Container volume directory binding.
Volume string `json:"volume,omitempty"`
// ISO generator container image URL
Image string `json:"image,omitempty"`
// Container Runtime Interface driver
ContainerRuntime string `json:"containerRuntime,omitempty"`
}
// Builder parameters
type Builder struct {
// Cloud Init user-data file name placed to the container volume root
UserDataFileName string `json:"userDataFileName,omitempty"`
// Cloud Init network-config file name placed to the container volume root
NetworkConfigFileName string `json:"networkConfigFileName,omitempty"`
// File name for output metadata
OutputMetadataFileName string `json:"outputMetadataFileName,omitempty"`
}
// ToYAML serializes confid to YAML
func (c *Config) ToYAML() ([]byte, error) {
return yaml.Marshal(c)
}

View File

@ -1,24 +0,0 @@
package isogen
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestToYaml(t *testing.T) {
expectedBytes := []byte(`builder: {}
container:
containerRuntime: docker
`)
cnf := &Config{
Container: Container{
ContainerRuntime: "docker",
},
}
actualBytes, err := cnf.ToYAML()
require.NoError(t, err)
assert.Equal(t, actualBytes, expectedBytes)
}

View File

@ -400,6 +400,7 @@ func (c *Config) GetCluster(cName, cType string) (*Cluster, error) {
}
return cluster, nil
}
func (c *Config) AddCluster(theCluster *ClusterOptions) (*Cluster, error) {
// Need to create new cluster placeholder
// Get list of ClusterPurposes that match the theCluster.name
@ -589,6 +590,20 @@ func (c *Config) CurrentContextManifest() (*Manifest, error) {
return c.Manifests[currentContext.Manifest], nil
}
// CurrentContextBootstrapInfo returns bootstrap info for current context
func (c *Config) CurrentContextBootstrapInfo() (*Bootstrap, error) {
currentCluster, err := c.CurrentContextCluster()
if err != nil {
return nil, err
}
bootstrap, exists := c.ModulesConfig.BootstrapInfo[currentCluster.Bootstrap]
if !exists {
return nil, ErrBootstrapInfoNotFound{Name: currentCluster.Bootstrap}
}
return bootstrap, nil
}
// Purge removes the config file
func (c *Config) Purge() error {
return os.Remove(c.loadedConfigPath)
@ -602,10 +617,10 @@ func (c *Config) Equal(d *Config) bool {
authInfoEq := reflect.DeepEqual(c.AuthInfos, d.AuthInfos)
contextEq := reflect.DeepEqual(c.Contexts, d.Contexts)
manifestEq := reflect.DeepEqual(c.Manifests, d.Manifests)
modulesEq := reflect.DeepEqual(c.ModulesConfig, d.ModulesConfig)
return c.Kind == d.Kind &&
c.APIVersion == d.APIVersion &&
clusterEq && authInfoEq && contextEq && manifestEq &&
c.ModulesConfig.Equal(d.ModulesConfig)
clusterEq && authInfoEq && contextEq && manifestEq && modulesEq
}
// Cluster functions
@ -753,7 +768,7 @@ func (m *Modules) Equal(n *Modules) bool {
if n == nil {
return n == m
}
return m.Dummy == n.Dummy
return reflect.DeepEqual(m.BootstrapInfo, n.BootstrapInfo)
}
func (m *Modules) String() string {
yaml, err := yaml.Marshal(&m)
@ -763,6 +778,60 @@ func (m *Modules) String() string {
return string(yaml)
}
// Bootstrap functions
func (b *Bootstrap) Equal(c *Bootstrap) bool {
if c == nil {
return b == c
}
contEq := reflect.DeepEqual(b.Container, c.Container)
bldrEq := reflect.DeepEqual(b.Builder, c.Builder)
return contEq && bldrEq
}
func (b *Bootstrap) String() string {
yaml, err := yaml.Marshal(&b)
if err != nil {
return ""
}
return string(yaml)
}
// Container functions
func (c *Container) Equal(d *Container) bool {
if d == nil {
return d == c
}
return c.Volume == d.Volume &&
c.Image == d.Image &&
c.ContainerRuntime == d.ContainerRuntime
}
func (c *Container) String() string {
yaml, err := yaml.Marshal(&c)
if err != nil {
return ""
}
return string(yaml)
}
// Builder functions
func (b *Builder) Equal(c *Builder) bool {
if c == nil {
return b == c
}
return b.UserDataFileName == c.UserDataFileName &&
b.NetworkConfigFileName == c.NetworkConfigFileName &&
b.OutputMetadataFileName == c.OutputMetadataFileName
}
func (b *Builder) String() string {
yaml, err := yaml.Marshal(&b)
if err != nil {
return ""
}
return string(yaml)
}
// ClusterComplexName functions
func (c *ClusterComplexName) validName() bool {
err := ValidClusterType(c.clusterType)

View File

@ -67,6 +67,30 @@ func TestString(t *testing.T) {
name: "repository",
stringer: DummyRepository(),
},
{
name: "bootstrap",
stringer: DummyBootstrap(),
},
{
name: "bootstrap",
stringer: DummyBootstrap(),
},
{
name: "builder",
stringer: &Builder{
UserDataFileName: "user-data",
NetworkConfigFileName: "netconfig",
OutputMetadataFileName: "output-metadata.yaml",
},
},
{
name: "container",
stringer: &Container{
Volume: "/dummy:dummy",
Image: "dummy_image:dummy_tag",
ContainerRuntime: "docker",
},
},
}
for _, tt := range tests {
@ -139,14 +163,48 @@ func TestEqual(t *testing.T) {
assert.False(t, testRepository1.Equal(nil))
})
// TODO(howell): this needs to be fleshed out when the Modules type is finished
t.Run("modules-equal", func(t *testing.T) {
testModules1 := &Modules{Dummy: "same"}
testModules2 := &Modules{Dummy: "different"}
testModules1 := NewModules()
testModules2 := NewModules()
testModules2.BootstrapInfo["different"] = &Bootstrap{
Container: &Container{Volume: "different"},
}
assert.True(t, testModules1.Equal(testModules1))
assert.False(t, testModules1.Equal(testModules2))
assert.False(t, testModules1.Equal(nil))
})
t.Run("bootstrap-equal", func(t *testing.T) {
testBootstrap1 := &Bootstrap{
Container: &Container{
Image: "same",
},
}
testBootstrap2 := &Bootstrap{
Container: &Container{
Image: "different",
},
}
assert.True(t, testBootstrap1.Equal(testBootstrap1))
assert.False(t, testBootstrap1.Equal(testBootstrap2))
assert.False(t, testBootstrap1.Equal(nil))
})
t.Run("container-equal", func(t *testing.T) {
testContainer1 := &Container{Image: "same"}
testContainer2 := &Container{Image: "different"}
assert.True(t, testContainer1.Equal(testContainer1))
assert.False(t, testContainer1.Equal(testContainer2))
assert.False(t, testContainer1.Equal(nil))
})
t.Run("builder-equal", func(t *testing.T) {
testBuilder1 := &Builder{UserDataFileName: "same"}
testBuilder2 := &Builder{UserDataFileName: "different"}
assert.True(t, testBuilder1.Equal(testBuilder1))
assert.False(t, testBuilder1.Equal(testBuilder2))
assert.False(t, testBuilder1.Equal(nil))
})
}
func TestLoadConfig(t *testing.T) {

15
pkg/config/errors.go Normal file
View File

@ -0,0 +1,15 @@
package config
import (
"fmt"
)
// ErrBootstrapInfoNotFound returned if bootstrap
// information is not found for cluster
type ErrBootstrapInfoNotFound struct {
Name string
}
func (e ErrBootstrapInfoNotFound) Error() string {
return fmt.Sprintf("Bootstrap info %s not found", e.Name)
}

View File

@ -79,7 +79,7 @@ func DummyCluster() *Cluster {
cluster.CertificateAuthority = "dummy_ca"
c.SetKubeCluster(cluster)
c.NameInKubeconf = "dummycluster_target"
c.Bootstrap = "dummy_bootstrap"
c.Bootstrap = "dummy_bootstrap_config"
return c
}
@ -108,7 +108,9 @@ func DummyAuthInfo() *AuthInfo {
}
func DummyModules() *Modules {
return &Modules{Dummy: "dummy-module"}
m := NewModules()
m.BootstrapInfo["dummy_bootstrap_config"] = DummyBootstrap()
return m
}
// DummyClusterPurpose , utility function used for tests
@ -166,6 +168,25 @@ func DummyContextOptions() *ContextOptions {
return co
}
func DummyBootstrap() *Bootstrap {
bs := &Bootstrap{}
cont := Container{
Volume: "/dummy:dummy",
Image: "dummy_image:dummy_tag",
ContainerRuntime: "docker",
}
builder := Builder{
UserDataFileName: "user-data",
NetworkConfigFileName: "netconfig",
OutputMetadataFileName: "output-metadata.yaml",
}
bs.Container = &cont
bs.Builder = &builder
return bs
}
const (
testConfigYAML = `apiVersion: airshipit.org/v1alpha1
clusters:

View File

@ -0,0 +1,8 @@
builder:
networkConfigFileName: netconfig
outputMetadataFileName: output-metadata.yaml
userDataFileName: user-data
container:
containerRuntime: docker
image: dummy_image:dummy_tag
volume: /dummy:dummy

View File

@ -0,0 +1,3 @@
networkConfigFileName: netconfig
outputMetadataFileName: output-metadata.yaml
userDataFileName: user-data

View File

@ -1,4 +1,4 @@
bootstrap-info: dummy_bootstrap
bootstrap-info: dummy_bootstrap_config
cluster-kubeconf: dummycluster_target
LocationOfOrigin: ""

View File

@ -3,10 +3,10 @@ clusters:
dummy_cluster:
cluster-type:
ephemeral:
bootstrap-info: dummy_bootstrap
bootstrap-info: dummy_bootstrap_config
cluster-kubeconf: dummycluster_ephemeral
target:
bootstrap-info: dummy_bootstrap
bootstrap-info: dummy_bootstrap_config
cluster-kubeconf: dummycluster_target
contexts:
dummy_context:
@ -32,6 +32,15 @@ manifests:
username: dummy_user
target-path: /var/tmp/
modules-config:
dummy-for-tests: dummy-module
bootstrapInfo:
dummy_bootstrap_config:
builder:
networkConfigFileName: netconfig
outputMetadataFileName: output-metadata.yaml
userDataFileName: user-data
container:
containerRuntime: docker
image: dummy_image:dummy_tag
volume: /dummy:dummy
users:
dummy_user: {}

View File

@ -0,0 +1,3 @@
containerRuntime: docker
image: dummy_image:dummy_tag
volume: /dummy:dummy

View File

@ -1 +1,10 @@
dummy-for-tests: dummy-module
bootstrapInfo:
dummy_bootstrap_config:
builder:
networkConfigFileName: netconfig
outputMetadataFileName: output-metadata.yaml
userDataFileName: user-data
container:
containerRuntime: docker
image: dummy_image:dummy_tag
volume: /dummy:dummy

View File

@ -1,6 +1,6 @@
Cluster: dummycluster
target:
bootstrap-info: dummy_bootstrap
bootstrap-info: dummy_bootstrap_config
cluster-kubeconf: dummycluster_target
LocationOfOrigin: ""

View File

@ -94,7 +94,7 @@ type Cluster struct {
// Configuration that the Document Module would need
// Configuration that the Workflows Module would need
type Modules struct {
Dummy string `json:"dummy-for-tests"`
BootstrapInfo map[string]*Bootstrap `json:"bootstrapInfo"`
}
// Context is a tuple of references to a cluster (how do I communicate with a kubernetes context),
@ -148,3 +148,31 @@ type ClusterComplexName struct {
clusterName string
clusterType string
}
// Bootstrap holds configurations for bootstrap steps
type Bootstrap struct {
// Configuration parameters for container
Container *Container `json:"container,omitempty"`
// Configuration parameters for ISO builder
Builder *Builder `json:"builder,omitempty"`
}
// Container parameters
type Container struct {
// Container volume directory binding.
Volume string `json:"volume,omitempty"`
// ISO generator container image URL
Image string `json:"image,omitempty"`
// Container Runtime Interface driver
ContainerRuntime string `json:"containerRuntime,omitempty"`
}
// Builder parameters
type Builder struct {
// Cloud Init user-data file name placed to the container volume root
UserDataFileName string `json:"userDataFileName,omitempty"`
// Cloud Init network-config file name placed to the container volume root
NetworkConfigFileName string `json:"networkConfigFileName,omitempty"`
// File name for output metadata
OutputMetadataFileName string `json:"outputMetadataFileName,omitempty"`
}

View File

@ -56,7 +56,9 @@ func NewAuthInfo() *AuthInfo {
}
func NewModules() *Modules {
return &Modules{}
return &Modules{
BootstrapInfo: make(map[string]*Bootstrap),
}
}
// NewClusterPurpose is a convenience function that returns a new ClusterPurpose

View File

@ -0,0 +1,9 @@
package document
// Document lables and annotations
const (
ClusterType = "clustertype"
// TODO (dukov) Replace with constants defined in config module once
// module dependency loop has been resolved
EphemeralClusterMarker = "airshipit.org/clustertype=ephemeral"
)