stackube/pkg/openstack/client.go

/*
Copyright (c) 2017 OpenStack Foundation.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package openstack

import (
	"errors"
	"fmt"
	"os"

	crv1 "git.openstack.org/openstack/stackube/pkg/apis/v1"
	crdClient "git.openstack.org/openstack/stackube/pkg/kubecrd"
	drivertypes "git.openstack.org/openstack/stackube/pkg/openstack/types"
	"git.openstack.org/openstack/stackube/pkg/util"

	"github.com/docker/distribution/uuid"
	"github.com/golang/glog"
	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack"
	"github.com/gophercloud/gophercloud/openstack/identity/v2/tenants"
	"github.com/gophercloud/gophercloud/openstack/identity/v2/users"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/routers"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/portsbinding"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/groups"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/rules"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/networks"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/ports"
	"github.com/gophercloud/gophercloud/openstack/networking/v2/subnets"
	"github.com/gophercloud/gophercloud/pagination"

	gcfg "gopkg.in/gcfg.v1"
)

const (
	StatusCodeAlreadyExists int = 409

	podNamePrefix     = "kube"
	securitygroupName = "kube-securitygroup-default"
	HostnameMaxLen    = 63

	// Service affinities
	ServiceAffinityNone     = "None"
	ServiceAffinityClientIP = "ClientIP"
)

var (
	adminStateUp = true

	ErrNotFound        = errors.New("NotFound")
	ErrMultipleResults = errors.New("MultipleResults")
)

type Client struct {
	Identity          *gophercloud.ServiceClient
	Provider          *gophercloud.ProviderClient
	Network           *gophercloud.ServiceClient
	Region            string
	ExtNetID          string
	PluginName        string
	IntegrationBridge string
	CRDClient         *crdClient.CRDClient
}

type PluginOpts struct {
	PluginName        string `gcfg:"plugin-name"`
	IntegrationBridge string `gcfg:"integration-bridge"`
}

type Config struct {
	Global struct {
		AuthUrl    string `gcfg:"auth-url"`
		Username   string `gcfg:"username"`
		Password   string `gcfg:"password"`
		TenantName string `gcfg:"tenant-name"`
		Region     string `gcfg:"region"`
		ExtNetID   string `gcfg:"ext-net-id"`
	}
	Plugin PluginOpts
}

func toAuthOptions(cfg Config) gophercloud.AuthOptions {
	return gophercloud.AuthOptions{
		IdentityEndpoint: cfg.Global.AuthUrl,
		Username:         cfg.Global.Username,
		Password:         cfg.Global.Password,
		TenantName:       cfg.Global.TenantName,
		AllowReauth:      true,
	}
}

func NewClient(config string, kubeConfig string) (*Client, error) {
	var opts gophercloud.AuthOptions
	cfg, err := readConfig(config)
	if err != nil {
		return nil, fmt.Errorf("Failed read cloudconfig: %v", err)
	}
	glog.V(1).Infof("Initializing openstack client with config %v", cfg)

	if cfg.Global.ExtNetID == "" {
		return nil, fmt.Errorf("external network ID not set")
	}

	opts = toAuthOptions(cfg)
	provider, err := openstack.AuthenticatedClient(opts)
	if err != nil {
		return nil, err
	}

	identity, err := openstack.NewIdentityV2(provider, gophercloud.EndpointOpts{
		Availability: gophercloud.AvailabilityAdmin,
	})
	if err != nil {
		return nil, err
	}

	network, err := openstack.NewNetworkV2(provider, gophercloud.EndpointOpts{
		Region: cfg.Global.Region,
	})
	if err != nil {
		glog.Warning("Failed to find neutron endpoint: %v", err)
		return nil, err
	}

	// Create CRD client
	k8sConfig, err := util.NewClusterConfig(kubeConfig)
	if err != nil {
		return nil, fmt.Errorf("failed to build kubeconfig: %v", err)
	}
	kubeCRDClient, err := crdClient.NewCRDClient(k8sConfig)
	if err != nil {
		return nil, fmt.Errorf("failed to create client for CRD: %v", err)
	}

	client := &Client{
		Identity:          identity,
		Provider:          provider,
		Network:           network,
		Region:            cfg.Global.Region,
		ExtNetID:          cfg.Global.ExtNetID,
		PluginName:        cfg.Plugin.PluginName,
		IntegrationBridge: cfg.Plugin.IntegrationBridge,
		CRDClient:         kubeCRDClient,
	}
	return client, nil
}

func readConfig(config string) (Config, error) {
	conf, err := os.Open(config)
	if err != nil {
		return Config{}, err
	}
	var cfg Config
	err = gcfg.ReadInto(&cfg, conf)
	if err != nil {
		return Config{}, err
	}
	return cfg, nil
}

func (c *Client) GetTenantIDFromName(tenantName string) (string, error) {
	if util.IsSystemNamespace(tenantName) {
		tenantName = util.SystemTenant
	}

	// If tenantID is specified, return it directly
	var (
		tenant *crv1.Tenant
		err    error
	)
	if tenant, err = c.CRDClient.GetTenant(tenantName); err != nil {
		return "", err
	}
	if tenant.Spec.TenantID != "" {
		return tenant.Spec.TenantID, nil
	}

	// Otherwise, fetch tenantID from OpenStack
	var tenantID string
	err = tenants.List(c.Identity, nil).EachPage(func(page pagination.Page) (bool, error) {
		tenantList, err1 := tenants.ExtractTenants(page)
		if err1 != nil {
			return false, err1
		}
		for _, t := range tenantList {
			if t.Name == tenantName {
				tenantID = t.ID
				break
			}
		}
		return true, nil
	})
	if err != nil {
		return "", err
	}

	glog.V(3).Infof("Got tenantID: %v for tenantName: %v", tenantID, tenantName)

	return tenantID, nil
}

func (c *Client) CreateTenant(tenantName string) (string, error) {
	createOpts := tenants.CreateOpts{
		Name:        tenantName,
		Description: "stackube",
		Enabled:     gophercloud.Enabled,
	}

	_, err := tenants.Create(c.Identity, createOpts).Extract()
	if err != nil && !IsAlreadyExists(err) {
		glog.Errorf("Failed to create tenant %s: %v", tenantName, err)
		return "", err
	}
	glog.V(4).Infof("Tenant %s created", tenantName)
	tenantID, err := c.GetTenantIDFromName(tenantName)
	if err != nil {
		return "", err
	}
	return tenantID, nil
}

func (c *Client) DeleteTenant(tenantName string) error {
	return tenants.List(c.Identity, nil).EachPage(func(page pagination.Page) (bool, error) {
		tenantList, err := tenants.ExtractTenants(page)
		if err != nil {
			return false, err
		}
		for _, t := range tenantList {
			if t.Name == tenantName {
				err := tenants.Delete(c.Identity, t.ID).ExtractErr()
				if err != nil {
					glog.Errorf("Delete openstack tenant %s error: %v", tenantName, err)
					return false, err
				}
				glog.V(4).Infof("Tenant %s deleted", tenantName)
				break
			}
		}
		return true, nil
	})
}

func (c *Client) CreateUser(username, password, tenantID string) error {
	opts := users.CreateOpts{
		Name:     username,
		TenantID: tenantID,
		Enabled:  gophercloud.Enabled,
		Password: password,
	}
	_, err := users.Create(c.Identity, opts).Extract()
	if err != nil && !IsAlreadyExists(err) {
		glog.Errorf("Failed to create user %s: %v", username, err)
		return err
	}
	glog.V(4).Infof("User %s created", username)
	return nil
}

func (c *Client) DeleteAllUsersOnTenant(tenantName string) error {
	tenantID, err := c.GetTenantIDFromName(tenantName)
	if err != nil {
		return nil
	}
	return users.ListUsers(c.Identity, tenantID).EachPage(func(page pagination.Page) (bool, error) {
		usersList, err := users.ExtractUsers(page)
		if err != nil {
			return false, err
		}
		for _, u := range usersList {
			res := users.Delete(c.Identity, u.ID)
			if res.Err != nil {
				glog.Errorf("Delete openstack user %s error: %v", u.Name, err)
				return false, err
			}
			glog.V(4).Infof("User %s deleted", u.Name)
		}
		return true, nil
	})
}

// IsAlreadyExists determines if the err is an error which indicates that a specified resource already exists.
func IsAlreadyExists(err error) bool {
	return reasonForError(err) == StatusCodeAlreadyExists
}

func reasonForError(err error) int {
	switch t := err.(type) {
	case gophercloud.ErrUnexpectedResponseCode:
		return t.Actual
	}
	return 0
}

// Get tenant's network by tenantID(tenant and network are one to one mapping in stackube)
func (os *Client) GetOpenStackNetworkByTenantID(tenantID string) (*networks.Network, error) {
	opts := networks.ListOpts{TenantID: tenantID}
	return os.getOpenStackNetwork(&opts)
}

// Get openstack network by id
func (os *Client) getOpenStackNetworkByID(id string) (*networks.Network, error) {
	opts := networks.ListOpts{ID: id}
	return os.getOpenStackNetwork(&opts)
}

// Get openstack network by name
func (os *Client) getOpenStackNetworkByName(name string) (*networks.Network, error) {
	opts := networks.ListOpts{Name: name}
	return os.getOpenStackNetwork(&opts)
}

// Get openstack network
func (os *Client) getOpenStackNetwork(opts *networks.ListOpts) (*networks.Network, error) {
	var osNetwork *networks.Network
	pager := networks.List(os.Network, *opts)
	err := pager.EachPage(func(page pagination.Page) (bool, error) {
		networkList, e := networks.ExtractNetworks(page)
		if len(networkList) > 1 {
			return false, ErrMultipleResults
		}

		if len(networkList) == 1 {
			osNetwork = &networkList[0]
		}

		return true, e
	})

	if err == nil && osNetwork == nil {
		return nil, ErrNotFound
	}

	return osNetwork, err
}

// Get provider subnet by id
func (os *Client) GetProviderSubnet(osSubnetID string) (*drivertypes.Subnet, error) {
	s, err := subnets.Get(os.Network, osSubnetID).Extract()
	if err != nil {
		glog.Errorf("Get openstack subnet failed: %v", err)
		return nil, err
	}

	var routes []*drivertypes.Route
	for _, r := range s.HostRoutes {
		route := drivertypes.Route{
			Nexthop:         r.NextHop,
			DestinationCIDR: r.DestinationCIDR,
		}
		routes = append(routes, &route)
	}

	providerSubnet := drivertypes.Subnet{
		Uid:        s.ID,
		Cidr:       s.CIDR,
		Gateway:    s.GatewayIP,
		Name:       s.Name,
		Dnsservers: s.DNSNameservers,
		Routes:     routes,
	}

	return &providerSubnet, nil
}

// Get network by networkID
func (os *Client) GetNetworkByID(networkID string) (*drivertypes.Network, error) {
	osNetwork, err := os.getOpenStackNetworkByID(networkID)
	if err != nil {
		glog.Errorf("failed to fetch openstack network by iD: %v, failure: %v", networkID, err)
		return nil, err
	}

	return os.OSNetworktoProviderNetwork(osNetwork)
}

// Get network by networkName
func (os *Client) GetNetwork(networkName string) (*drivertypes.Network, error) {
	osNetwork, err := os.getOpenStackNetworkByName(networkName)
	if err != nil {
		glog.Warningf("try to fetch openstack network by name: %v but failed: %v", networkName, err)
		return nil, err
	}

	return os.OSNetworktoProviderNetwork(osNetwork)
}

func (os *Client) OSNetworktoProviderNetwork(osNetwork *networks.Network) (*drivertypes.Network, error) {
	var providerNetwork drivertypes.Network
	var providerSubnets []*drivertypes.Subnet
	providerNetwork.Name = osNetwork.Name
	providerNetwork.Uid = osNetwork.ID
	providerNetwork.Status = os.ToProviderStatus(osNetwork.Status)
	providerNetwork.TenantID = osNetwork.TenantID

	for _, subnetID := range osNetwork.Subnets {
		s, err := os.GetProviderSubnet(subnetID)
		if err != nil {
			return nil, err
		}
		providerSubnets = append(providerSubnets, s)
	}

	providerNetwork.Subnets = providerSubnets

	return &providerNetwork, nil
}

func (os *Client) ToProviderStatus(status string) string {
	switch status {
	case "ACTIVE":
		return "Active"
	case "BUILD":
		return "Pending"
	case "DOWN", "ERROR":
		return "Failed"
	default:
		return "Failed"
	}
}

// Create network
func (os *Client) CreateNetwork(network *drivertypes.Network) error {
	if len(network.Subnets) == 0 {
		return errors.New("Subnets is null")
	}

	// create network
	opts := networks.CreateOpts{
		Name:         network.Name,
		AdminStateUp: &adminStateUp,
		TenantID:     network.TenantID,
	}
	osNet, err := networks.Create(os.Network, opts).Extract()
	if err != nil {
		glog.Errorf("Create openstack network %s failed: %v", network.Name, err)
		return err
	}

	// create router
	routerOpts := routers.CreateOpts{
		// use network name as router name for convenience
		Name:        network.Name,
		TenantID:    network.TenantID,
		GatewayInfo: &routers.GatewayInfo{NetworkID: os.ExtNetID},
	}
	osRouter, err := routers.Create(os.Network, routerOpts).Extract()
	if err != nil {
		glog.Errorf("Create openstack router %s failed: %v", network.Name, err)
		delErr := os.DeleteNetwork(network.Name)
		if delErr != nil {
			glog.Errorf("Delete openstack network %s failed: %v", network.Name, delErr)
		}
		return err
	}

	// create subnets and connect them to router
	networkID := osNet.ID
	network.Status = os.ToProviderStatus(osNet.Status)
	network.Uid = osNet.ID
	for _, sub := range network.Subnets {
		// create subnet
		subnetOpts := subnets.CreateOpts{
			NetworkID:      networkID,
			CIDR:           sub.Cidr,
			Name:           sub.Name,
			IPVersion:      gophercloud.IPv4,
			TenantID:       network.TenantID,
			GatewayIP:      &sub.Gateway,
			DNSNameservers: sub.Dnsservers,
		}
		s, err := subnets.Create(os.Network, subnetOpts).Extract()
		if err != nil {
			glog.Errorf("Create openstack subnet %s failed: %v", sub.Name, err)
			delErr := os.DeleteNetwork(network.Name)
			if delErr != nil {
				glog.Errorf("Delete openstack network %s failed: %v", network.Name, delErr)
			}
			return err
		}

		// add subnet to router
		opts := routers.AddInterfaceOpts{
			SubnetID: s.ID,
		}
		_, err = routers.AddInterface(os.Network, osRouter.ID, opts).Extract()
		if err != nil {
			glog.Errorf("Create openstack subnet %s failed: %v", sub.Name, err)
			delErr := os.DeleteNetwork(network.Name)
			if delErr != nil {
				glog.Errorf("Delete openstack network %s failed: %v", network.Name, delErr)
			}
			return err
		}
	}

	return nil
}

// Update network
func (os *Client) UpdateNetwork(network *drivertypes.Network) error {
	// TODO: update network subnets
	return nil
}

func (os *Client) getRouterByName(name string) (*routers.Router, error) {
	var result *routers.Router

	opts := routers.ListOpts{Name: name}
	pager := routers.List(os.Network, opts)
	err := pager.EachPage(func(page pagination.Page) (bool, error) {
		routerList, e := routers.ExtractRouters(page)
		if len(routerList) > 1 {
			return false, ErrMultipleResults
		} else if len(routerList) == 1 {
			result = &routerList[0]
		}

		return true, e
	})
	if err != nil {
		return nil, err
	}

	return result, nil
}

// Delete network by networkName
func (os *Client) DeleteNetwork(networkName string) error {
	osNetwork, err := os.getOpenStackNetworkByName(networkName)
	if err != nil {
		glog.Errorf("Get openstack network failed: %v", err)
		return err
	}

	if osNetwork != nil {
		// Delete ports
		opts := ports.ListOpts{NetworkID: osNetwork.ID}
		pager := ports.List(os.Network, opts)
		err := pager.EachPage(func(page pagination.Page) (bool, error) {
			portList, err := ports.ExtractPorts(page)
			if err != nil {
				glog.Errorf("Get openstack ports error: %v", err)
				return false, err
			}

			for _, port := range portList {
				if port.DeviceOwner == "network:router_interface" {
					continue
				}

				err = ports.Delete(os.Network, port.ID).ExtractErr()
				if err != nil {
					glog.Warningf("Delete port %v failed: %v", port.ID, err)
				}
			}

			return true, nil
		})
		if err != nil {
			glog.Errorf("Delete ports error: %v", err)
		}

		router, err := os.getRouterByName(networkName)
		if err != nil {
			glog.Errorf("Get openstack router %s error: %v", networkName, err)
			return err
		}

		// delete all subnets
		for _, subnet := range osNetwork.Subnets {
			if router != nil {
				opts := routers.RemoveInterfaceOpts{SubnetID: subnet}
				_, err := routers.RemoveInterface(os.Network, router.ID, opts).Extract()
				if err != nil {
					glog.Errorf("Get openstack router %s error: %v", networkName, err)
					return err
				}
			}

			err = subnets.Delete(os.Network, subnet).ExtractErr()
			if err != nil {
				glog.Errorf("Delete openstack subnet %s error: %v", subnet, err)
				return err
			}
		}

		// delete router
		if router != nil {
			err = routers.Delete(os.Network, router.ID).ExtractErr()
			if err != nil {
				glog.Errorf("Delete openstack router %s error: %v", router.ID, err)
				return err
			}
		}

		// delete network
		err = networks.Delete(os.Network, osNetwork.ID).ExtractErr()
		if err != nil {
			glog.Errorf("Delete openstack network %s error: %v", osNetwork.ID, err)
			return err
		}
	}

	return nil
}

// Check the tenant id exist
func (os *Client) CheckTenantID(tenantID string) (bool, error) {
	opts := tenants.ListOpts{}
	pager := tenants.List(os.Identity, &opts)

	var found bool
	err := pager.EachPage(func(page pagination.Page) (bool, error) {

		tenantList, err := tenants.ExtractTenants(page)
		if err != nil {
			return false, err
		}

		if len(tenantList) == 0 {
			return false, ErrNotFound
		}

		for _, t := range tenantList {
			if t.ID == tenantID || t.Name == tenantID {
				found = true
			}
		}

		return true, nil
	})

	return found, err
}

func (os *Client) GetPort(name string) (*ports.Port, error) {
	opts := ports.ListOpts{Name: name}
	pager := ports.List(os.Network, opts)

	var port *ports.Port
	err := pager.EachPage(func(page pagination.Page) (bool, error) {
		portList, err := ports.ExtractPorts(page)
		if err != nil {
			glog.Errorf("Get openstack ports error: %v", err)
			return false, err
		}

		if len(portList) > 1 {
			return false, ErrMultipleResults
		}

		if len(portList) == 0 {
			return false, ErrNotFound
		}

		port = &portList[0]

		return true, err
	})

	return port, err
}

func getHostName() string {
	host, err := os.Hostname()
	if err != nil {
		return ""
	}

	return host
}

func (os *Client) ensureSecurityGroup(tenantID string) (string, error) {
	var securitygroup *groups.SecGroup

	opts := groups.ListOpts{
		TenantID: tenantID,
		Name:     securitygroupName,
	}
	pager := groups.List(os.Network, opts)
	err := pager.EachPage(func(page pagination.Page) (bool, error) {
		sg, err := groups.ExtractGroups(page)
		if err != nil {
			glog.Errorf("Get openstack securitygroups error: %v", err)
			return false, err
		}

		if len(sg) > 0 {
			securitygroup = &sg[0]
		}

		return true, err
	})
	if err != nil {
		return "", err
	}

	// If securitygroup doesn't exist, create a new one
	if securitygroup == nil {
		securitygroup, err = groups.Create(os.Network, groups.CreateOpts{
			Name:     securitygroupName,
			TenantID: tenantID,
		}).Extract()

		if err != nil {
			return "", err
		}
	}

	var secGroupsRules int
	listopts := rules.ListOpts{
		TenantID:   tenantID,
		Direction:  string(rules.DirIngress),
		SecGroupID: securitygroup.ID,
	}
	rulesPager := rules.List(os.Network, listopts)
	err = rulesPager.EachPage(func(page pagination.Page) (bool, error) {
		r, err := rules.ExtractRules(page)
		if err != nil {
			glog.Errorf("Get openstack securitygroup rules error: %v", err)
			return false, err
		}

		secGroupsRules = len(r)

		return true, err
	})
	if err != nil {
		return "", err
	}

	// create new rules
	if secGroupsRules == 0 {
		// create egress rule
		_, err = rules.Create(os.Network, rules.CreateOpts{
			TenantID:   tenantID,
			SecGroupID: securitygroup.ID,
			Direction:  rules.DirEgress,
			EtherType:  rules.EtherType4,
		}).Extract()

		// create ingress rule
		_, err := rules.Create(os.Network, rules.CreateOpts{
			TenantID:   tenantID,
			SecGroupID: securitygroup.ID,
			Direction:  rules.DirIngress,
			EtherType:  rules.EtherType4,
		}).Extract()
		if err != nil {
			return "", err
		}
	}

	return securitygroup.ID, nil
}

// Create an port
func (os *Client) CreatePort(networkID, tenantID, portName string) (*portsbinding.Port, error) {
	securitygroup, err := os.ensureSecurityGroup(tenantID)
	if err != nil {
		glog.Errorf("EnsureSecurityGroup failed: %v", err)
		return nil, err
	}

	opts := portsbinding.CreateOpts{
		HostID: getHostName(),
		CreateOptsBuilder: ports.CreateOpts{
			NetworkID:      networkID,
			Name:           portName,
			AdminStateUp:   &adminStateUp,
			TenantID:       tenantID,
			DeviceID:       uuid.Generate().String(),
			DeviceOwner:    fmt.Sprintf("compute:%s", getHostName()),
			SecurityGroups: []string{securitygroup},
		},
	}

	port, err := portsbinding.Create(os.Network, opts).Extract()
	if err != nil {
		glog.Errorf("Create port %s failed: %v", portName, err)
		return nil, err
	}
	return port, nil
}

// List all ports in the network
func (os *Client) ListPorts(networkID, deviceOwner string) ([]ports.Port, error) {
	var results []ports.Port
	opts := ports.ListOpts{
		NetworkID:   networkID,
		DeviceOwner: deviceOwner,
	}
	pager := ports.List(os.Network, opts)
	err := pager.EachPage(func(page pagination.Page) (bool, error) {
		portList, err := ports.ExtractPorts(page)
		if err != nil {
			glog.Errorf("Get openstack ports error: %v", err)
			return false, err
		}

		for _, port := range portList {
			results = append(results, port)
		}

		return true, err
	})

	if err != nil {
		return nil, err
	}

	return results, nil
}

// Delete port by portName
func (os *Client) DeletePort(portName string) error {
	port, err := os.GetPort(portName)
	if err == util.ErrNotFound {
		glog.V(4).Infof("Port %s already deleted", portName)
		return nil
	} else if err != nil {
		glog.Errorf("Get openstack port %s failed: %v", portName, err)
		return err
	}

	if port != nil {
		err := ports.Delete(os.Network, port.ID).ExtractErr()
		if err != nil {
			glog.Errorf("Delete openstack port %s failed: %v", portName, err)
			return err
		}
	}

	return nil
}