Merge pull request #10 from hpcloud-mon/feature/devstack

Feature/devstack

Berksfile

@@ -5,6 +5,7 @@ if Gem::Specification::find_by_name('berkshelf').version.to_s[0] == '3'
 end
 
 cookbook 'mini-mon', path: './cookbooks/mini-mon'
+cookbook 'devstack', path: './cookbooks/devstack'
 cookbook 'mon_api', git: 'https://github.com/hpcloud-mon/cookbooks-mon_api'
 cookbook 'kafka', git: 'https://github.com/hpcloud-mon/cookbooks-kafka'
 cookbook 'mon_agent', git: 'https://github.com/hpcloud-mon/cookbooks-mon_agent'

README.md

@@ -43,6 +43,8 @@ The `vertica::console` recipe is not enabled by default, but if it is added, thi
 ## Install Vagrant
 
 ### Install VirtualBox and Vagrant
+Note: Vagrant version 1.5.0 or higher is required.
+
 #### MacOS
 The following steps assume you have [Homebrew](http://brew.sh/) installed.  Otherwise, install [VirtualBox](http://www.virtualbox.org) and [Vagrant](http://www.vagrantup.com) manually from their websites, then continue with Set Up Berkshelf below.
 
@@ -80,8 +82,9 @@ sudo gem install berkshelf
 
 - Your host OS home dir is synced to `/vagrant_home` on the VM.
 - The root dir of the mon-vagrant repo on your host OS is synced to `/vagrant` on the VM.
-- The VM will have an IP of 196.168.10.4 that can be access from other services running on the host.
-- Run `vagrant ssh` to log in
+- The main VM will have an IP of 192.168.10.4 that can be access from other services running on the host.
+- An additional VM running DevStack will be created at 192.168.10.5
+- Run `vagrant ssh <host>` to log in, where `<host>` is either `mini-mon` or `devstack`
 - Run `vagrant help` for more info
 
 ## Updating

Vagrantfile

@@ -20,21 +20,40 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     end
   end
 
-  config.vm.box = "precise64"
-  config.vm.box_url = "http://files.vagrantup.com/precise64.box"
   config.vm.synced_folder "~/", "/vagrant_home"
 
   # One vm running all the services
-  config.vm.hostname = 'mini-mon'
-  config.vm.network :private_network, ip: "192.168.10.4"
-  config.vm.provider "virtualbox" do |vb|
-    vb.memory = 6144
-    vb.cpus = 4
+  config.vm.define "mini-mon" do |mm|
+    mm.vm.hostname = 'mini-mon'
+    mm.vm.box = "precise64"
+    mm.vm.box_url = "http://files.vagrantup.com/precise64.box"
+    mm.vm.network :private_network, ip: "192.168.10.4"
+    mm.vm.provider "virtualbox" do |vb|
+      vb.memory = 6144
+      vb.cpus = 4
+    end
+    mm.vm.provision :chef_solo do |chef|
+      chef.roles_path = "roles"
+      chef.data_bags_path = "data_bags"
+      chef.add_role "Mini-Mon"
+    end
   end
-  config.vm.provision :chef_solo do |chef|
-    chef.roles_path = "roles"
-    chef.data_bags_path = "data_bags"
-    chef.add_role "Mini-Mon"
+
+  # One vm just for devstack (to access the UI)
+  config.vm.define "devstack" do |ds|
+    ds.vm.hostname = "devstack"
+    ds.vm.box = "cyrusbio/devstack"
+    ds.vm.network :private_network, ip: "192.168.10.5"
+    ds.vm.network :private_network, ip: "10.1.2.44"
+    ds.vm.provider "virtualbox" do |vb|
+      vb.memory = 5280
+      vb.cpus = 4
+    end
+    ds.vm.provision :chef_solo do |chef|
+      chef.roles_path = "roles"
+      chef.data_bags_path = "data_bags"
+      chef.add_role "Devstack"
+    end
   end
 
 end

cookbooks/devstack/README.md

@@ -0,0 +1,12 @@
+devstack cookbook
+=========================
+Various helper recipes for the devstack server setup.
+
+Requirements
+------------
+
+Recipes
+---------
+- default - configures base apt repostories needed for devstack as well as option apt cache
+- mon-ui - installs the Monitoring panel in the UI
+- keystone - simple keystone installation

cookbooks/devstack/files/default/autostack.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+IFS="
+"
+
+unpriv_user=$USER
+basedir=`dirname $0`
+
+if [ ! -e $basedir/stack-screenrc ]; then
+    echo "$basedir/stack-screenrc not found.  Did you run ./stack.sh?"
+    exit 1
+fi
+
+
+for line in `cat $basedir/stack-screenrc |tr -d "\r"`; do
+    if  [[ $line == stuff* ]]; then
+        # Extract the command line to run this service
+        command=`echo "$line" |sed 's/^stuff //;s/"//g'`
+        base_command=`echo $command |sed 's:.*bin/::;s/ .*//'`
+
+        # Skip screen sessions that are only a tail command
+        [[ $command == *tail* ]] && continue
+
+        # Determine an appropriate log directory
+        parent=`echo "$command" |cut -d' ' -f2 |sed 's/;//'`
+        logdir="/var/log/`basename $parent`"
+        logfile="$base_command.log"
+
+        echo "Creating /etc/init/$base_command.conf"
+
+        sudo tee "/etc/init/$base_command.conf" >/dev/null <<EOF
+description "$base_command server"
+author "David Schroeder <david.schroeder@hp.com>"
+
+start on (filesystem and net-device-up IFACE!=lo)
+stop on runlevel [016]
+
+pre-start script
+    mkdir -p $logdir
+    chown $unpriv_user:root $logdir
+end script
+
+respawn
+
+exec su -c "$command --log-dir=$logdir --log-file=$logfile" $unpriv_user
+EOF
+    # Fire up the service
+    service $base_command restart
+    fi
+done

cookbooks/devstack/metadata.rb

@@ -0,0 +1,7 @@
+name             "devstack"
+maintainer       "Mon Team"
+maintainer_email "hpcs-mon@hp.com"
+license          "All rights reserved"
+description      "Build customized devstack server"
+long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
+version          "0.0.1"

cookbooks/devstack/recipes/default.rb

@@ -0,0 +1,30 @@
+# Turn a devstack image (https://vagrantcloud.com/cyrusbio/devstack) into
+# something useful to mini-mon
+
+admin_token = 'ADMIN'
+my_ip = '192.168.10.5'
+ks_conf = '/etc/keystone/keystone.conf'
+
+# Modify keystone configuration to...
+## ...bind to all IPs
+execute "sed -i.bak1 's^\\(.*bind_host = \\).*^\\10.0.0.0^' #{ks_conf}"
+# ...set the default admin token
+execute "sed -i.bak2 's^\\(admin_token = \\).*^\\1#{admin_token}^' #{ks_conf}"
+# ...set endpoints that other mini-mon nodes can use
+execute "sed -i.bak3 's^\\(.*endpoint = http://\\).*\\(:.*\\)^\\1#{my_ip}\\2^' #{ks_conf}"
+
+
+# Regular devstack relies on a user manually running "rejoin-stack.sh" which
+# fires up a bunch of screen sessions, one for each process.  We don't really
+# want that here.  Instead autostack.sh will create upstart scripts for each
+# devstack process, sendnig its output to real log files under /var/log/. 
+cookbook_file "autostack.sh" do
+    mode 0755
+    owner "vagrant"
+    path "/home/vagrant/devstack/autostack.sh"
+    action :create_if_missing
+end
+
+execute "/home/vagrant/devstack/autostack.sh"
+
+

cookbooks/devstack/recipes/keystone.rb

@@ -1,15 +1,4 @@
-# Install a simple keystone just using sqlite, then setup a user mini-mon/password
-
-package 'keystone' do
-  action :install
-end
-
-service 'keystone' do
-  action [ :enable, :start ]
-end
-
-# The python-keystoneclient that comes with precise is broken by a newer python-prettytable needed by python-monclient, so cmdline interaction is messed
-# up, using the api directly is fine though
+# Sets up a user mini-mon username/password in keystone
 
 python 'make default keystone user' do
   action :run

cookbooks/devstack/recipes/mon-ui.rb

@@ -0,0 +1,21 @@
+# Installs the mon-ui panel
+
+# Grab the necessary packages
+include_recipe "python"
+['monitoring-plugin','python-monclient'].each do |pkg|
+    python_pip pkg do
+        action :install
+    end
+end
+
+# Set up symlinks
+# Use 'execute' resource because chef does not support symlinking directories
+execute "ln -sfv /usr/local/lib/python2.7/dist-packages/enabled/* /opt/stack/horizon/openstack_dashboard/local/enabled/"
+execute "ln -sv /usr/local/lib/python2.7/dist-packages/monitoring /opt/stack/horizon/monitoring"
+execute "ln -sv /usr/local/lib/python2.7/dist-packages/cosmos/overcloud  /opt/stack/horizon/openstack_dashboard/dashboards/overcloud"
+
+# Bounce the webserver
+service "apache2" do
+    action :restart
+end
+

roles/Devstack.json

@@ -0,0 +1,20 @@
+{
+  "name": "Devstack",
+  "description": "Sets up a devstack server for keystone and UI",
+  "json_class": "Chef::Role",
+  "default_attributes": {
+    "apt": {
+      "periodic_update_min_delay": 60
+    }
+  },
+  "override_attributes": {
+  },
+  "chef_type": "role",
+  "run_list": [
+    "recipe[devstack]",
+    "recipe[devstack::mon-ui]",
+    "recipe[devstack::keystone]"
+  ],
+  "env_run_lists": {
+  }
+}

roles/Mini-Mon.json

@@ -55,7 +55,6 @@
   "chef_type": "role",
   "run_list": [
     "recipe[mini-mon]",
-    "recipe[mini-mon::keystone]",
     "recipe[percona::cluster]",
     "recipe[mini-mon::mysql_schema]",
     "recipe[zookeeper]",

split/Vagrantfile

@@ -23,8 +23,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     end
   end
 
-  config.vm.box = "precise64"
-  config.vm.box_url = "http://files.vagrantup.com/precise64.box"
   config.vm.synced_folder "~/", "/vagrant_home"
   config.vm.provider "virtualbox" do |vb|
     vb.customize ["modifyvm", :id, "--memory", "768"]
@@ -33,6 +31,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   # VM specific settings, these machines come up in order they are specified.
   config.vm.define "mysql" do |mysql|
     mysql.vm.hostname = 'mysql'
+    mysql.vm.box = "precise64"
+    mysql.vm.box_url = "http://files.vagrantup.com/precise64.box"
     mysql.vm.network :private_network, ip: "192.168.10.6"
     mysql.vm.provision :chef_solo do |chef|
       chef.roles_path = "roles"
@@ -43,6 +43,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
   config.vm.define "kafka" do |kafka|
     kafka.vm.hostname = 'kafka'
+    kafka.vm.box = "precise64"
+    kafka.vm.box_url = "http://files.vagrantup.com/precise64.box"
     kafka.vm.network :private_network, ip: "192.168.10.10"
     kafka.vm.provision :chef_solo do |chef|
       chef.roles_path = "roles"
@@ -56,6 +58,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
   config.vm.define "vertica" do |vertica|
     vertica.vm.hostname = 'vertica'
+    vertica.vm.box = "precise64"
+    vertica.vm.box_url = "http://files.vagrantup.com/precise64.box"
     vertica.vm.network :private_network, ip: "192.168.10.8"
     vertica.vm.provision :chef_solo do |chef|
       chef.roles_path = "roles"
@@ -69,6 +73,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
   config.vm.define "api" do |api|
     api.vm.hostname = 'api'
+    api.vm.box = "precise64"
+    api.vm.box_url = "http://files.vagrantup.com/precise64.box"
     api.vm.network :private_network, ip: "192.168.10.4"
     api.vm.provision :chef_solo do |chef|
       chef.roles_path = "roles"
@@ -77,8 +83,26 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     end
   end
 
+  config.vm.define "devstack" do |devstack|
+    devstack.vm.hostname = 'devstack'
+    devstack.vm.box = "cyrusbio/devstack"
+    devstack.vm.network :private_network, ip: "192.168.10.5"
+    devstack.vm.network :private_network, ip: "10.1.2.44"
+    devstack.vm.provider "virtualbox" do |vb|
+      vb.memory = 5280
+      vb.cpus = 4
+    end
+    devstack.vm.provision :chef_solo do |chef|
+      chef.roles_path = "roles"
+      chef.data_bags_path = "data_bags"
+      chef.add_role "Devstack"
+    end
+  end
+
   config.vm.define "persister" do |persister|
     persister.vm.hostname = 'persister'
+    persister.vm.box = "precise64"
+    persister.vm.box_url = "http://files.vagrantup.com/precise64.box"
     persister.vm.network :private_network, ip: "192.168.10.12"
     persister.vm.provision :chef_solo do |chef|
       chef.roles_path = "roles"
@@ -92,6 +116,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
   config.vm.define "thresh" do |thresh|
     thresh.vm.hostname = 'thresh'
+    thresh.vm.box = "precise64"
+    thresh.vm.box_url = "http://files.vagrantup.com/precise64.box"
     thresh.vm.network :private_network, ip: "192.168.10.14"
     thresh.vm.provision :chef_solo do |chef|
       chef.roles_path = "roles"

split/roles/Devstack.json

@@ -0,0 +1,20 @@
+{
+  "name": "Devstack",
+  "description": "Sets up a devstack server for keystone and UI",
+  "json_class": "Chef::Role",
+  "default_attributes": {
+    "apt": {
+      "periodic_update_min_delay": 60
+    }
+  },
+  "override_attributes": {
+  },
+  "chef_type": "role",
+  "run_list": [
+    "recipe[devstack]",
+    "recipe[devstack::mon-ui]",
+    "recipe[devstack::keystone]"
+  ],
+  "env_run_lists": {
+  }
+}