From bc5264df05787c281ac6b5a21077fe654cd7bfa2 Mon Sep 17 00:00:00 2001 From: Pete Birley Date: Fri, 24 Mar 2017 07:20:56 -0400 Subject: [PATCH 1/5] Initial Vagrant/Kubeadm-aio Dev Env --- dev/Vagrantfile | 91 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 dev/Vagrantfile diff --git a/dev/Vagrantfile b/dev/Vagrantfile new file mode 100644 index 0000000000..6ce8d8429b --- /dev/null +++ b/dev/Vagrantfile @@ -0,0 +1,91 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# All Vagrant configuration is done below. The "2" in Vagrant.configure +# configures the configuration version (we support older styles for +# backwards compatibility). Please don't change it unless you know what +# you're doing. +Vagrant.configure("2") do |config| + # The most common configuration options are documented and commented below. + # For a complete reference, please see the online documentation at + # https://docs.vagrantup.com. + + # Every Vagrant development environment requires a box. You can search for + # boxes at https://atlas.hashicorp.com/search. + config.vm.box = "ubuntu/xenial64" + + # Disable automatic box update checking. If you disable this, then + # boxes will only be checked for updates when the user runs + # `vagrant box outdated`. This is not recommended. + # config.vm.box_check_update = false + + # Create a forwarded port mapping which allows access to a specific port + # within the machine from a port on the host machine. In the example below, + # accessing "localhost:8080" will access port 80 on the guest machine. + config.vm.network "forwarded_port", guest: 6443, host: 8443 + + # Create a private network, which allows host-only access to the machine + # using a specific IP. + # config.vm.network "private_network", ip: "192.168.33.10" + + # Create a public network, which generally matched to bridged network. + # Bridged networks make the machine appear as another physical device on + # your network. + # config.vm.network "public_network" + + # Share an additional folder to the guest VM. The first argument is + # the path on the host to the actual folder. The second argument is + # the path on the guest to mount the folder. And the optional third + # argument is a set of non-required options. + # config.vm.synced_folder "../data", "/vagrant_data" + + # Provider-specific configuration so you can fine-tune various + # backing providers for Vagrant. These expose provider-specific options. + # Example for VirtualBox: + # + config.vm.provider "virtualbox" do |vb| + # Display the VirtualBox GUI when booting the machine + vb.gui = true + + # Customize the amount of memory on the VM: + vb.memory = "8192" + end + # + # View the documentation for the provider you are using for more + # information on available options. + + # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies + # such as FTP and Heroku are also available. See the documentation at + # https://docs.vagrantup.com/v2/push/atlas.html for more information. + # config.push.define "atlas" do |push| + # push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME" + # end + + # Enable provisioning with a shell script. Additional provisioners such as + # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the + # documentation for more information about their specific syntax and use. + config.vm.provision "shell", inline: <<-SHELL + apt-get update + apt-get install -y \ + docker.io \ + nfs-common + mkdir -p /var/lib/kublet + mount --bind /var/lib/kublet /var/lib/kublet + mount --make-shared /var/lib/kublet + docker run \ + -dt \ + --name=kubeadm-aio \ + --net=host \ + --security-opt=seccomp:unconfined \ + --cap-add=SYS_ADMIN \ + --tmpfs=/run \ + --tmpfs=/run/lock \ + --volume=/etc/machine-id:/etc/machine-id:ro \ + --volume=/home:/home:rw \ + --volume=/etc/kubernetes:/etc/kubernetes:rw \ + --volume=/sys/fs/cgroup:/sys/fs/cgroup:ro \ + --volume=/var/run/docker.sock:/run/docker.sock \ + --env KUBELET_CONTAINER=docker.io/port/kubeadm-aio:latest \ + docker.io/port/kubeadm-aio:latest + SHELL +end From 1b1658a1ff14f8544a4db89017e81c959f633dd9 Mon Sep 17 00:00:00 2001 From: Pete Birley Date: Sun, 26 Mar 2017 11:09:05 -0500 Subject: [PATCH 2/5] Initial helm and kubectl support for Vagrant dev env --- .gitignore | 2 ++ dev/README.md | 43 ++++++++++++++++++++++++++++++++ dev/Vagrantfile | 58 ++++++++++++++++++++++--------------------- dev/config.rb | 4 +++ dev/setup-dev-host.sh | 17 +++++++++++++ 5 files changed, 96 insertions(+), 28 deletions(-) create mode 100644 dev/README.md create mode 100644 dev/config.rb create mode 100755 dev/setup-dev-host.sh diff --git a/.gitignore b/.gitignore index 1e69553255..5a9f2a3e8b 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ .idea/ **/_partials.tpl **/_globals.tpl +dev/.vagrant +dev/*.log diff --git a/dev/README.md b/dev/README.md new file mode 100644 index 0000000000..2e5c0acf45 --- /dev/null +++ b/dev/README.md @@ -0,0 +1,43 @@ +# Development Environment Setup + +## Requirements + + * Hardware + * 16GB RAM + * 32GB HDD Space + * Software + * Vagrant + * VirtualBox + * Kubectl + * Helm + * Git + +## Deploy + + * Make sure you are in the directory containing the Vagrantfile before running the following commands. + +### Create VM + +``` bash +vagrant up --provider virtualbox +``` + +### Deploy NFS Provisioner for development PVCs + +``` bash +vagrant ssh --command "sudo docker exec kubeadm-aio kubectl create -R -f /opt/nfs-provisioner/" +``` + +### Setup Clients and deploy Helm's tiller + +``` bash +./setup-dev-host.sh +``` + +### Label VM node(s) for OpenStack-Helm Deployment + +``` bash +kubectl label nodes openstack-control-plane=enabled --all --namespace=openstack +kubectl label nodes openvswitch=enabled --all --namespace=openstack +kubectl label nodes openstack-compute-node=enabled --all --namespace=openstack +``` diff --git a/dev/Vagrantfile b/dev/Vagrantfile index 6ce8d8429b..7ed0c604ac 100644 --- a/dev/Vagrantfile +++ b/dev/Vagrantfile @@ -1,5 +1,14 @@ # -*- mode: ruby -*- # vi: set ft=ruby : +# NOTE: Variable overrides are in ./config.rb +require "yaml" +require "fileutils" + +# Use a variable file for overrides: +CONFIG = File.expand_path("config.rb") +if File.exist?(CONFIG) + require CONFIG +end # All Vagrant configuration is done below. The "2" in Vagrant.configure # configures the configuration version (we support older styles for @@ -19,52 +28,44 @@ Vagrant.configure("2") do |config| # `vagrant box outdated`. This is not recommended. # config.vm.box_check_update = false - # Create a forwarded port mapping which allows access to a specific port - # within the machine from a port on the host machine. In the example below, - # accessing "localhost:8080" will access port 80 on the guest machine. - config.vm.network "forwarded_port", guest: 6443, host: 8443 - # Create a private network, which allows host-only access to the machine # using a specific IP. - # config.vm.network "private_network", ip: "192.168.33.10" - - # Create a public network, which generally matched to bridged network. - # Bridged networks make the machine appear as another physical device on - # your network. - # config.vm.network "public_network" + config.vm.network "private_network", ip: "192.168.33.10" # Share an additional folder to the guest VM. The first argument is # the path on the host to the actual folder. The second argument is # the path on the guest to mount the folder. And the optional third # argument is a set of non-required options. - # config.vm.synced_folder "../data", "/vagrant_data" + config.vm.synced_folder "../", "/opt/openstack-helm" # Provider-specific configuration so you can fine-tune various # backing providers for Vagrant. These expose provider-specific options. - # Example for VirtualBox: - # config.vm.provider "virtualbox" do |vb| # Display the VirtualBox GUI when booting the machine vb.gui = true # Customize the amount of memory on the VM: - vb.memory = "8192" + vb.memory = $ram + + # Customize the number of vCPUs in the VM: + vb.cpus = $vcpu_cores + + # Set the size of the VM's root disk: + unless File.exist?('.vagrant/machines/default/virtualbox/openstack-helm-storage.vdi') + vb.customize ['createhd', '--filename', '.vagrant/machines/default/virtualbox/openstack-helm-storage', '--size', $docker_disk] + end + vb.customize ['storageattach', :id, '--storagectl', 'SCSI', '--port', 2, '--device', 0, '--type', 'hdd', '--medium', '.vagrant/machines/default/virtualbox/openstack-helm-storage.vdi'] + end - # - # View the documentation for the provider you are using for more - # information on available options. - # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies - # such as FTP and Heroku are also available. See the documentation at - # https://docs.vagrantup.com/v2/push/atlas.html for more information. - # config.push.define "atlas" do |push| - # push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME" - # end - - # Enable provisioning with a shell script. Additional provisioners such as - # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the - # documentation for more information about their specific syntax and use. + # Enable provisioning with a shell script. config.vm.provision "shell", inline: <<-SHELL + # Setup docker storage + mkfs.xfs /dev/disk/by-path/pci-0000\:00\:14.0-scsi-0\:0\:2\:0 -f -L docker-srg + mkdir -p /var/lib/docker + echo "LABEL=docker-srg /var/lib/docker xfs defaults 0 0" >> /etc/fstab + mount -a + apt-get update apt-get install -y \ docker.io \ @@ -85,6 +86,7 @@ Vagrant.configure("2") do |config| --volume=/etc/kubernetes:/etc/kubernetes:rw \ --volume=/sys/fs/cgroup:/sys/fs/cgroup:ro \ --volume=/var/run/docker.sock:/run/docker.sock \ + --env KUBE_BIND_DEV=enp0s8 \ --env KUBELET_CONTAINER=docker.io/port/kubeadm-aio:latest \ docker.io/port/kubeadm-aio:latest SHELL diff --git a/dev/config.rb b/dev/config.rb new file mode 100644 index 0000000000..3547aca7b8 --- /dev/null +++ b/dev/config.rb @@ -0,0 +1,4 @@ +# VM Specs +$docker_disk = 20480 +$vcpu_cores = 4 +$ram = 8192 diff --git a/dev/setup-dev-host.sh b/dev/setup-dev-host.sh new file mode 100755 index 0000000000..29f9ec90d9 --- /dev/null +++ b/dev/setup-dev-host.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -e +# Setting up kubectl creds +mkdir -p ${HOME}/.kube +if [ -f ${HOME}/.kube/config ]; then + echo "Previous kube config found, backing it up" + mv -v ${HOME}/.kube/config ${HOME}/.kube/config.$(date "+%F-%T") +fi +echo "Getting kubeconfig from kube1" +vagrant ssh default -c "sudo cat /etc/kubernetes/admin.conf" > ${HOME}/.kube/config + +# Setting up helm client if present +if which helm 2>/dev/null; then + helm init +fi + +echo "clients should now be ready to access the Kubernetes cluster" From 13913cd6ac2133a3352b8476c26be22ea153e7fa Mon Sep 17 00:00:00 2001 From: Pete Birley Date: Fri, 31 Mar 2017 07:33:17 -0500 Subject: [PATCH 3/5] Update Vagrant based on PR Feedback --- dev/Vagrantfile | 10 +++++----- dev/config.rb | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/dev/Vagrantfile b/dev/Vagrantfile index 7ed0c604ac..55efb9eedb 100644 --- a/dev/Vagrantfile +++ b/dev/Vagrantfile @@ -21,7 +21,7 @@ Vagrant.configure("2") do |config| # Every Vagrant development environment requires a box. You can search for # boxes at https://atlas.hashicorp.com/search. - config.vm.box = "ubuntu/xenial64" + config.vm.box = $vm_image # Disable automatic box update checking. If you disable this, then # boxes will only be checked for updates when the user runs @@ -42,7 +42,7 @@ Vagrant.configure("2") do |config| # backing providers for Vagrant. These expose provider-specific options. config.vm.provider "virtualbox" do |vb| # Display the VirtualBox GUI when booting the machine - vb.gui = true + vb.gui = false # Customize the amount of memory on the VM: vb.memory = $ram @@ -51,10 +51,10 @@ Vagrant.configure("2") do |config| vb.cpus = $vcpu_cores # Set the size of the VM's root disk: - unless File.exist?('.vagrant/machines/default/virtualbox/openstack-helm-storage.vdi') - vb.customize ['createhd', '--filename', '.vagrant/machines/default/virtualbox/openstack-helm-storage', '--size', $docker_disk] + unless File.exist?('.vagrant/machines/default/openstack-helm-storage.vdi') + vb.customize ['createhd', '--filename', '.vagrant/machines/default/openstack-helm-storage', '--size', $docker_disk] end - vb.customize ['storageattach', :id, '--storagectl', 'SCSI', '--port', 2, '--device', 0, '--type', 'hdd', '--medium', '.vagrant/machines/default/virtualbox/openstack-helm-storage.vdi'] + vb.customize ['storageattach', :id, '--storagectl', 'SCSI', '--port', 2, '--device', 0, '--type', 'hdd', '--medium', '.vagrant/machines/default/openstack-helm-storage.vdi'] end diff --git a/dev/config.rb b/dev/config.rb index 3547aca7b8..6311f1a0ab 100644 --- a/dev/config.rb +++ b/dev/config.rb @@ -1,4 +1,5 @@ # VM Specs +$vm_image = "ubuntu/xenial64" $docker_disk = 20480 $vcpu_cores = 4 $ram = 8192 From 9c9eea4bd3e951e3d677156936a91d28501b7472 Mon Sep 17 00:00:00 2001 From: Pete Birley Date: Fri, 31 Mar 2017 10:53:31 -0500 Subject: [PATCH 4/5] Specify Vagrant and Virtualbox Min versions --- dev/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/README.md b/dev/README.md index 2e5c0acf45..1a0012ca11 100644 --- a/dev/README.md +++ b/dev/README.md @@ -6,8 +6,8 @@ * 16GB RAM * 32GB HDD Space * Software - * Vagrant - * VirtualBox + * Vagrant >= 1.8.0 + * VirtualBox >= 5.1.0 * Kubectl * Helm * Git From eff2eb8b593dd6a93e0ddef1311e84f03749cae3 Mon Sep 17 00:00:00 2001 From: Pete Birley Date: Mon, 3 Apr 2017 09:06:20 -0500 Subject: [PATCH 5/5] NFS PVC Volume support --- dev/Vagrantfile | 21 ++++++++++++++++++++- dev/config.rb | 1 + 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/dev/Vagrantfile b/dev/Vagrantfile index 55efb9eedb..15ab2752a6 100644 --- a/dev/Vagrantfile +++ b/dev/Vagrantfile @@ -50,12 +50,19 @@ Vagrant.configure("2") do |config| # Customize the number of vCPUs in the VM: vb.cpus = $vcpu_cores - # Set the size of the VM's root disk: + # Set the size of the VM's docker disk: unless File.exist?('.vagrant/machines/default/openstack-helm-storage.vdi') vb.customize ['createhd', '--filename', '.vagrant/machines/default/openstack-helm-storage', '--size', $docker_disk] end vb.customize ['storageattach', :id, '--storagectl', 'SCSI', '--port', 2, '--device', 0, '--type', 'hdd', '--medium', '.vagrant/machines/default/openstack-helm-storage.vdi'] + + # Set the size of the VM's PVC disk: + unless File.exist?('.vagrant/machines/default/openstack-helm-storage-kube-pvc.vdi') + vb.customize ['createhd', '--filename', '.vagrant/machines/default/openstack-helm-storage-kube-pvc', '--size', $pvc_disk] + end + vb.customize ['storageattach', :id, '--storagectl', 'SCSI', '--port', 3, '--device', 0, '--type', 'hdd', '--medium', '.vagrant/machines/default/openstack-helm-storage-kube-pvc.vdi'] + end # Enable provisioning with a shell script. @@ -64,15 +71,27 @@ Vagrant.configure("2") do |config| mkfs.xfs /dev/disk/by-path/pci-0000\:00\:14.0-scsi-0\:0\:2\:0 -f -L docker-srg mkdir -p /var/lib/docker echo "LABEL=docker-srg /var/lib/docker xfs defaults 0 0" >> /etc/fstab + + # Setup kubelet pvc storage + mkfs.xfs /dev/disk/by-path/pci-0000\:00\:14.0-scsi-0\:0\:3\:0 -f -L kube-srg + mkdir -p /var/lib/nfs-provisioner + echo "LABEL=kube-srg /var/lib/nfs-provisioner xfs defaults 0 0" >> /etc/fstab + + # Mount Storage mount -a + # Install requirements apt-get update apt-get install -y \ docker.io \ nfs-common + + # Setup kubelet lib as shared mount mkdir -p /var/lib/kublet mount --bind /var/lib/kublet /var/lib/kublet mount --make-shared /var/lib/kublet + + # Run AIO container docker run \ -dt \ --name=kubeadm-aio \ diff --git a/dev/config.rb b/dev/config.rb index 6311f1a0ab..98d3f78cad 100644 --- a/dev/config.rb +++ b/dev/config.rb @@ -1,5 +1,6 @@ # VM Specs $vm_image = "ubuntu/xenial64" $docker_disk = 20480 +$pvc_disk = 10240 $vcpu_cores = 4 $ram = 8192