From 291127f0d1f74aa5510f1cb56c67e5d498de375d Mon Sep 17 00:00:00 2001
From: Pengfei Ni <feiskyer@gmail.com>
Date: Sat, 27 May 2017 19:14:05 +0800
Subject: [PATCH] Add devstack plugin for stackube

The plugin install components including

* OpenStack Keystone, Neutron and Cinder with Ceph backend
* Kubernetes with frakti container runtime
* Kubernetes auth via keystone

Implements: blueprint kubernetes-deploy
Change-Id: Ib2ad60b71bc51bdedaefd7014c83ed7ba56414e3
Signed-off-by: Pengfei Ni <feiskyer@gmail.com>
---
 devstack/README.md              |  50 ++++++++
 devstack/kubeadm.yaml           |  13 ++
 devstack/local.conf.node.sample |  29 +++++
 devstack/local.conf.sample      |  33 +++++
 devstack/plugin.sh              | 215 ++++++++++++++++++++++++++++++++
 devstack/settings               |  15 +++
 6 files changed, 355 insertions(+)
 create mode 100644 devstack/README.md
 create mode 100644 devstack/kubeadm.yaml
 create mode 100644 devstack/local.conf.node.sample
 create mode 100644 devstack/local.conf.sample
 create mode 100644 devstack/plugin.sh
 create mode 100644 devstack/settings

diff --git a/devstack/README.md b/devstack/README.md
new file mode 100644
index 0000000..af4d1c5
--- /dev/null
+++ b/devstack/README.md
@@ -0,0 +1,50 @@
+# devstack plugin
+
+devstack plugin for stackube.
+
+## All-in-one
+
+```sh
+# create stack user
+sudo useradd -s /bin/bash -d /opt/stack -m stack
+echo "stack ALL=(ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/stack
+sudo su - stack
+
+git clone https://git.openstack.org/openstack-dev/devstack -b stable/ocata
+cd devstack
+```
+
+Create `local.conf` from [local.conf.sample](local.conf.sample) and then run `./stack.sh` to install.
+
+```sh
+./stack.sh
+```
+
+Wait a while for installation compelete, then setup kubernetes and OpenStack client:
+
+```sh
+# Kubernetes
+export KUBECONFIG=$HOME/admin.conf
+kubectl cluster-info
+
+# OpenStack
+source openrc admin admin
+openstack service list
+```
+
+## Add a node
+
+```sh
+# create stack user
+sudo useradd -s /bin/bash -d /opt/stack -m stack
+echo "stack ALL=(ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/stack
+sudo su - stack
+
+git clone https://git.openstack.org/openstack-dev/devstack -b stable/ocata
+cd devstack
+```
+
+Create `local.conf` from [local.conf.node.sample](local.conf.node.sample), set `HOST_IP` to local host's IP, set `SERVICE_HOST` to master's IP and set `KUBEADM_TOKEN` to kubeadm token (could be got by `kubeadm token list`).
+
+Then run `./stack.sh` to install.
+
diff --git a/devstack/kubeadm.yaml b/devstack/kubeadm.yaml
new file mode 100644
index 0000000..1f58269
--- /dev/null
+++ b/devstack/kubeadm.yaml
@@ -0,0 +1,13 @@
+kind: MasterConfiguration
+apiVersion: kubeadm.k8s.io/v1alpha1
+kubernetesVersion: "stable"
+controllerManagerExtraArgs:
+  horizontal-pod-autoscaler-use-rest-clients: "true"
+  horizontal-pod-autoscaler-sync-period: "10s"
+  node-monitor-grace-period: "10s"
+  feature-gates: "AllAlpha=true"
+  enable-dynamic-provisioning: "true"
+apiServerExtraArgs:
+  runtime-config: "api/all=true"
+  feature-gates: "AllAlpha=true"
+  experimental-keystone-url: "https://KEYSTONE_HOST:5000/v2.0"
diff --git a/devstack/local.conf.node.sample b/devstack/local.conf.node.sample
new file mode 100644
index 0000000..a8fc6d6
--- /dev/null
+++ b/devstack/local.conf.node.sample
@@ -0,0 +1,29 @@
+[[local|localrc]]
+# Set to local host's IP
+HOST_IP=10.0.0.3
+# Set to master node's IP
+SERVICE_HOST="10.0.0.2"
+MYSQL_HOST=${SERVICE_HOST}
+RABBIT_HOST=${SERVICE_HOST}
+ADMIN_PASSWORD=password
+DATABASE_PASSWORD=$ADMIN_PASSWORD
+RABBIT_PASSWORD=$ADMIN_PASSWORD
+SERVICE_PASSWORD=$ADMIN_PASSWORD
+
+ENABLED_SERVICES=""
+
+# Neutron
+enable_service q-agt
+
+# Kubernetes
+STACKUBE_HOME=${STACKUBE_HOME:-$DEST/stackube}
+# Kubernetes master IP, should set to master IP for minions
+KUBERNETES_MASTER_IP=${SERVICE_HOST}
+# Get token by running following command on the master node
+# sudo kubeadm token list | awk '/kubeadm/{print $1}'
+KUBEADM_TOKEN="token"
+CONTAINER_CIDR="10.244.2.0/24"
+enable_plugin stackube git://git.openstack.org/openstack/stackube
+enable_service stackube
+# Enable kubernetes node components
+enable_service kubernetes_node
diff --git a/devstack/local.conf.sample b/devstack/local.conf.sample
new file mode 100644
index 0000000..f17189a
--- /dev/null
+++ b/devstack/local.conf.sample
@@ -0,0 +1,33 @@
+[[local|localrc]]
+ADMIN_PASSWORD=password
+DATABASE_PASSWORD=$ADMIN_PASSWORD
+RABBIT_PASSWORD=$ADMIN_PASSWORD
+SERVICE_PASSWORD=$ADMIN_PASSWORD
+
+ENABLED_SERVICES="tls-proxy"
+
+# Keystone
+enable_service key
+enable_service mysql
+enable_service rabbit
+
+# Neutron
+enable_service neutron
+enable_service q-agt
+enable_service q-dhcp
+enable_service q-l3
+enable_service q-svc
+enable_plugin neutron-lbaas git://git.openstack.org/openstack/neutron-lbaas stable/ocata
+enable_service q-lbaasv2
+NEUTRON_LBAAS_SERVICE_PROVIDERV2="LOADBALANCERV2:Haproxy:neutron_lbaas.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default"
+
+# Kubernetes
+enable_plugin stackube git://git.openstack.org/openstack/stackube
+enable_service stackube
+# Enable kubernetes master components
+enable_service kubernetes_master
+
+# Cinder
+ENABLE_CEPH_CINDER=True
+enable_service cinder c-api c-vol c-sch c-bak
+enable_plugin devstack-plugin-ceph git://git.openstack.org/openstack/devstack-plugin-ceph
diff --git a/devstack/plugin.sh b/devstack/plugin.sh
new file mode 100644
index 0000000..fce1735
--- /dev/null
+++ b/devstack/plugin.sh
@@ -0,0 +1,215 @@
+#!/bin/bash
+STACKUBE_ROOT=$(dirname "${BASH_SOURCE}")
+
+function configure_cni {
+    sudo mkdir -p /etc/cni/net.d
+    sudo sh -c "cat >/etc/cni/net.d/10-mynet.conf <<EOF
+{
+    \"cniVersion\": \"0.3.0\",
+    \"name\": \"mynet\",
+    \"type\": \"bridge\",
+    \"bridge\": \"cni0\",
+    \"isGateway\": true,
+    \"ipMasq\": true,
+    \"ipam\": {
+        \"type\": \"host-local\",
+        \"subnet\": \"${CONTAINER_CIDR}\",
+        \"routes\": [
+            { \"dst\": \"0.0.0.0/0\"  }
+        ]
+    }
+}
+EOF"
+    sudo sh -c 'cat >/etc/cni/net.d/99-loopback.conf <<EOF
+{
+    "cniVersion": "0.3.0",
+    "type": "loopback"
+}
+EOF'
+}
+
+function install_docker {
+    if is_ubuntu; then
+        sudo apt-get update
+        sudo apt-get install -y docker.io
+    elif is_fedora; then
+        sudo yum install -y docker
+    else
+        exit_distro_not_supported
+    fi
+ 
+    sudo systemctl start docker
+}
+
+function install_hyper {
+    if is_ubuntu; then
+        sudo apt-get update && sudo apt-get install -y qemu libvirt-bin
+    elif is_fedora; then
+        sudo yum install -y libvirt
+    fi
+
+    sudo systemctl restart libvirtd
+
+    if command -v /usr/bin/hyperd > /dev/null 2>&1; then
+        echo "hyperd already installed on this host, using it instead"
+    else
+        curl -sSL https://hypercontainer.io/install | sudo bash
+    fi
+    sudo sh -c 'cat>/etc/hyper/config <<EOF
+Kernel=/var/lib/hyper/kernel
+Initrd=/var/lib/hyper/hyper-initrd.img
+Hypervisor=qemu
+StorageDriver=overlay
+gRPCHost=127.0.0.1:22318
+EOF'
+}
+
+function install_frakti {
+    if command -v /usr/bin/frakti > /dev/null 2>&1; then
+        sudo rm -f /usr/bin/frakti
+    fi
+    sudo curl -sSL https://github.com/kubernetes/frakti/releases/download/${FRAKTI_VERSION}/frakti -o /usr/bin/frakti
+    sudo chmod +x /usr/bin/frakti
+    cgroup_driver=$(sudo docker info | awk '/Cgroup Driver/{print $3}')
+    sudo sh -c "cat > /lib/systemd/system/frakti.service <<EOF
+[Unit]
+Description=Hypervisor-based container runtime for Kubernetes
+Documentation=https://github.com/kubernetes/frakti
+After=network.target
+[Service]
+ExecStart=/usr/bin/frakti --v=3 \
+          --log-dir=/var/log/frakti \
+          --logtostderr=false \
+          --cgroup-driver=${cgroup_driver} \
+          --listen=/var/run/frakti.sock \
+          --streaming-server-addr=%H \
+          --hyper-endpoint=127.0.0.1:22318
+MountFlags=shared
+TasksMax=8192
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitCORE=infinity
+TimeoutStartSec=0
+Restart=on-abnormal
+[Install]
+WantedBy=multi-user.target
+EOF"
+}
+
+function install_kubelet {
+    if is_fedora; then
+        sudo sh -c 'cat > /etc/yum.repos.d/kubernetes.repo <<EOF
+[kubernetes]
+name=Kubernetes
+baseurl=http://yum.kubernetes.io/repos/kubernetes-el7-x86_64
+enabled=1
+gpgcheck=1
+repo_gpgcheck=1
+gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg
+       https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
+EOF'
+        sudo setenforce 0
+        sudo sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
+        sudo yum install -y kubernetes-cni kubelet kubeadm kubectl
+    elif is_ubuntu; then
+        sudo apt-get update && sudo apt-get install -y apt-transport-https
+        sudo curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
+        sudo sh -c 'cat > /etc/apt/sources.list.d/kubernetes.list <<EOF 
+deb http://apt.kubernetes.io/ kubernetes-xenial main
+EOF'
+        sudo apt-get update
+        sudo apt-get install -y kubernetes-cni kubelet kubeadm kubectl
+    else
+        exit_distro_not_supported
+    fi
+}
+
+function install_master {
+    sed -i "s/KEYSTONE_HOST/${SERVICE_HOST}/g" ${STACKUBE_ROOT}/kubeadm.yaml
+    sudo kubeadm init kubeadm init --pod-network-cidr ${CLUSTER_CIDR} --config ${STACKUBE_ROOT}/kubeadm.yaml
+    # Enable schedule pods on the master for testing.
+    sudo cp /etc/kubernetes/admin.conf $HOME/
+    sudo chown $(id -u):$(id -g) $HOME/admin.conf
+    export KUBECONFIG=$HOME/admin.conf
+    kubectl taint nodes --all node-role.kubernetes.io/master-
+}
+
+function install_node {
+    if [ "${KUBEADM_TOKEN}" = "" ]; then
+        echo "KUBEADM_TOKEN must be set for node"
+        exit 1
+    fi
+    sudo kubeadm join --token "${KUBEADM_TOKEN}" ${KUBERNETES_MASTER_IP}:${KUBERNETES_MASTER_PORT}
+}
+
+function configure_kubelet {
+    sudo sed -i '2 i\Environment="KUBELET_EXTRA_ARGS=--container-runtime=remote --container-runtime-endpoint=/var/run/frakti.sock --feature-gates=AllAlpha=true"' /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+    sudo systemctl daemon-reload
+}
+
+function remove_kubernetes {
+    sudo kubeadm reset
+    sudo systemctl stop kubelet
+
+    if is_fedora; then
+        sudo yum remove -y qemu-hyper hyperstart hyper-container libvirt
+        sudo yum remove -y kubernetes-cni kubelet kubeadm kubectl docker
+    elif is_ubuntu; then
+        sudo apt-get remove -y hyperstart hyper-container qemu libvirt-bin
+        sudo apt-get remove -y kubernetes-cni kubelet kubeadm kubectl docker
+    fi
+
+    sudo rm -rf /usr/bin/frakti /etc/cni/net.d /lib/systemd/system/frakti.service
+}
+
+function install_stackube {
+    install_docker
+    install_hyper
+    install_frakti
+    install_kubelet
+}
+
+function init_stackube {
+    sudo systemctl daemon-reload
+    sudo systemctl restart docker
+    sudo systemctl restart libvirtd
+    sudo systemctl restart hyperd
+    sudo systemctl restart frakti
+
+    if is_service_enabled kubernetes_master; then
+        install_master
+    elif is_service_enabled kubernetes_node; then
+        install_node
+    fi
+}
+
+function configure_stackube {
+    configure_cni
+    configure_kubelet
+}
+
+# check for service enabled
+if is_service_enabled stackube; then
+
+    if [[ "$1" == "stack" && "$2" == "install" ]]; then
+        echo_summary "Installing stackube"
+        install_stackube
+
+    elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
+        echo_summary "Configuring stackube"
+        configure_stackube
+
+    elif [[ "$1" == "stack" && "$2" == "extra" ]]; then
+        # Initialize and start the stackube service
+        echo_summary "Initializing stackube"
+        init_stackube
+    fi
+
+    if [[ "$1" == "unstack" ]]; then
+        remove_kubernetes
+    fi
+
+    if [[ "$1" == "clean" ]]; then
+        echo ''
+    fi
+fi
diff --git a/devstack/settings b/devstack/settings
new file mode 100644
index 0000000..678f63a
--- /dev/null
+++ b/devstack/settings
@@ -0,0 +1,15 @@
+STACKUBE_HOME=${STACKUBE_HOME:-$DEST/stackube}
+# Kubernetes master IP, should set to master IP for minions
+KUBERNETES_MASTER_IP=${KUBERNETES_MASTER_IP:-${SERVICE_HOST}}
+KUBERNETES_MASTER_PORT=${KUBERNETES_MASTER_PORT:-6443}
+# Kubeadm token, only for minions
+KUBEADM_TOKEN=${KUBEADM_TOKEN:-""}
+# The version of frakti
+FRAKTI_VERSION=${FRAKTI_VERSION:-"v0.2"}
+# Kubernetes CLUSTER CIDR
+CLUSTER_CIDR=${CLUSTER_CIDR:-"10.244.0.0/16"}
+# CNI Configurations
+# Brige network plugin is used now
+# TODO: replace network plugin with stackube-cni
+CONTAINER_CIDR=${CONTAINER_CIDR:-"10.244.1.0/24"}
+CNI_VERSION=${CNI_VERSION:-"v0.5.2"}