Merge pull request #65 from Mirantis/cgenie/solar-bootstrap

Cgenie/solar bootstrap

.gitignore

@@ -26,3 +26,7 @@ celery*.log
 *.dot
 *.png
 resources_compiled.py
+
+# bootstrap
+bootstrap/trusty64
+bootstrap/solar-master.box

Vagrantfile

@@ -5,38 +5,31 @@
 VAGRANTFILE_API_VERSION = "2"
 SLAVES_COUNT = 2
 
-init_script = <<SCRIPT
-apt-get update
-apt-get -y install python-pip python-dev
-pip install ansible
-ansible-playbook -i "localhost," -c local /vagrant/main.yml /vagrant/docker.yml
+solar_script = <<SCRIPT
+ansible-playbook -i "localhost," -c local /vagrant/bootstrap/playbooks/solar.yml
 SCRIPT
 
 slave_script = <<SCRIPT
-apt-get update
-apt-get upgrade
-apt-get dist-upgrade
-apt-get -y install python-pip python-dev
-pip install ansible
-ansible-playbook -i "localhost," -c local /vagrant/main.yml /vagrant/docker.yml /vagrant/slave.yml /vagrant/slave_cinder.yml
+ansible-playbook -i "localhost," -c local /vagrant/bootstrap/playbooks/custom-configs.yml -e master_ip=10.0.0.2
 SCRIPT
 
 master_celery = <<SCRIPT
-ansible-playbook -i "localhost," -c local /vagrant/celery.yml --skip-tags slave
+ansible-playbook -i "localhost," -c local /vagrant/bootstrap/playbooks/celery.yml --skip-tags slave
 SCRIPT
 
 slave_celery = <<SCRIPT
-ansible-playbook -i "localhost," -c local /vagrant/celery.yml --skip-tags master
+ansible-playbook -i "localhost," -c local /vagrant/bootstrap/playbooks/celery.yml --skip-tags master
 SCRIPT
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
+  config.vm.define "solar-dev", primary: true do |config|
       #config.vm.box = "deb/jessie-amd64"
       #config.vm.box = "rustyrobot/deb-jessie-amd64"
-  config.vm.box = "ubuntu/trusty64"
+      #config.vm.box = "ubuntu/trusty64"
+      config.vm.box = "cgenie/solar-master"
 
-  config.vm.define "solar-dev", primary: true do |config|
-    config.vm.provision "shell", inline: init_script, privileged: true
+    config.vm.provision "shell", inline: solar_script, privileged: true
     config.vm.provision "shell", inline: master_celery, privileged: true
     config.vm.provision "file", source: "~/.vagrant.d/insecure_private_key", destination: "/vagrant/tmp/keys/ssh_private"
     config.vm.provision "file", source: "ansible.cfg", destination: "/home/vagrant/.ansible.cfg"
@@ -44,7 +37,11 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     config.vm.host_name = "solar-dev"
 
     config.vm.provider :virtualbox do |v|
-      v.customize ["modifyvm", :id, "--memory", 1024]
+      v.customize [
+        "modifyvm", :id,
+        "--memory", 1024,
+        "--paravirtprovider", "kvm" # for linux guest
+      ]
       v.name = "solar-dev"
     end
   end
@@ -53,14 +50,26 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     index = i + 1
     ip_index = i + 3
     config.vm.define "solar-dev#{index}" do |config|
-      config.vm.provision "shell", inline: init_script, privileged: true
+      # Box solar-dev3 is for 'solar_bootstrap' resource demo
+      if index == 3 then
+        config.vm.box = "ubuntu/trusty64"
+      else
+        # standard box with all stuff preinstalled
+        config.vm.box = "cgenie/solar-master"
+      end
+
       config.vm.provision "shell", inline: slave_script, privileged: true
+      config.vm.provision "shell", inline: solar_script, privileged: true
       config.vm.provision "shell", inline: slave_celery, privileged: true
       config.vm.network "private_network", ip: "10.0.0.#{ip_index}"
       config.vm.host_name = "solar-dev#{index}"
 
       config.vm.provider :virtualbox do |v|
-        v.customize ["modifyvm", :id, "--memory", 1024]
+        v.customize [
+            "modifyvm", :id,
+            "--memory", 1024,
+            "--paravirtprovider", "kvm" # for linux guest
+        ]
         v.name = "solar-dev#{index}"
       end
     end

bootstrap/README.md

@@ -0,0 +1,25 @@
+# Solar image building
+
+In `bootstrap/trusty64` directory there are `box.ovf` and `box-disk1.vmdk`
+files from the `trusty64` Vagrant box (normally found in
+ `~/.vagrant.d/boxes/trusty64/0/virtualbox`).
+
+To build, install Packer (https://www.packer.io/):
+```
+cd bootstrap
+packer build solar-master.json
+cp solar-master.box ../
+cd ..
+vagrant up
+```
+
+If Vagrant throws error about `vboxsf` try this:
+```
+vagrant plugin install vagrant-vbguest
+```
+(see https://github.com/shiguredo/packer-templates/issues/16).
+
+If you're rebuilding the same box, make sure Vagrant reimports it:
+```
+vagrant box remove solar-master
+```

bootstrap/build-image.yml

@@ -0,0 +1,8 @@
+---
+
+- hosts: all
+  sudo: yes
+  tasks:
+     #- shell: vagrant init ubuntu/trusty64
+
+     - shell: /usr/local/bin/packer build solar.json -var 'is_master=true'

bootstrap/packer.yml

@@ -0,0 +1,12 @@
+---
+
+- hosts: all
+  sudo: yes
+  tasks:
+    - shell: wget 'https://dl.bintray.com/mitchellh/packer/packer_0.8.2_linux_amd64.zip' -O /tmp/packer-0.8.2.zip
+      args:
+        creates: /tmp/packer-0.8.2.zip
+    - unarchive:
+       src: /tmp/packer-0.8.2.zip
+       dest: /usr/local/bin
+       copy: no

bootstrap/playbooks/build-main.yml

@@ -0,0 +1,11 @@
+---
+
+- name: Main build script
+  hosts: all
+  sudo: yes
+  tasks:
+    - include: tasks/base.yml
+    - include: tasks/puppet.yml
+    - include: tasks/docker.yml
+    #- include: celery.yml tags=['master'] celery_dir=/var/run/celery
+    - include: tasks/cinder.yml

bootstrap/playbooks/celery.yml

@@ -0,0 +1,21 @@
+---
+
+- name: Solar Celery config
+  hosts: all
+  sudo: yes
+  vars:
+    celery_dir: /var/run/celery
+  tasks:
+    - shell: mkdir -p {{ celery_dir }}
+    - shell: pip install celery
+    - shell: hostname
+      register: hostname
+    - shell: celery multi kill 2
+             chdir={{ celery_dir }}
+      tags: [stop]
+    - shell: celery multi start 2 -A solar.orchestration.runner -Q:1 scheduler,system_log -Q:2 celery,{{ hostname.stdout }}
+             chdir={{ celery_dir }}
+      tags: [master]
+    - shell: celery multi start 1 -A solar.orchestration.runner -Q:1  {{ hostname.stdout }}
+             chdir={{ celery_dir }}
+      tags: [slave]

bootstrap/playbooks/custom-configs.yml

@@ -0,0 +1,8 @@
+---
+
+- name: Custom Solar configs
+  hosts: all
+  sudo: yes
+  tasks:
+    - lineinfile: line='slaveof {{ master_ip }} 6379' dest=/etc/redis/redis.conf
+    - service: name=redis-server state=restarted

bootstrap/playbooks/files/hiera.yaml

@@ -0,0 +1,15 @@
+:backends:
+  - redis
+  #- yaml
+  #- json
+:yaml:
+  :datadir: /etc/puppet/hieradata
+:json:
+  :datadir: /etc/puppet/hieradata
+:redis:
+  :port: 6379
+  :db: 0
+  :host: localhost
+  :deserialize: :json
+:hierarchy:
+  - resource

bootstrap/playbooks/files/ubuntu-ansible.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+# TODO: maybe this is better:
+# http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-apt-ubuntu
+
+apt-get remove -f python-pip
+sudo apt-get install -y python-setuptools
+sudo easy_install pip
+sudo pip install -U pip
+sudo pip install ansible

bootstrap/playbooks/solar.yml

@@ -0,0 +1,7 @@
+---
+
+- hosts: all
+  sudo: yes
+  tasks:
+    # Setup development env for solar
+    - shell: python setup.py develop chdir=/vagrant/solar

bootstrap/playbooks/tasks/base.yml

@@ -0,0 +1,56 @@
+---
+
+- shell: apt-get update
+
+- name: Base packages
+  apt: name={{ item }} state=present
+  with_items:
+  - git
+  - python-mock
+  - python-keystoneclient
+  - python-mysqldb
+  - python-setuptools
+  - ruby-dev
+  - unzip
+  # Redis
+  - redis-server
+  # Graph drawing
+  - python-pygraphviz
+  # Other tools
+  - htop
+  - jq
+  - tmux
+  - vim
+  # Dev
+  - ipython
+  - python-pudb
+  # Required by packer
+  - build-essential
+
+# PIP
+#- apt: name=python-pip state=absent
+#- shell: easy_install pip
+#- shell: pip install -U pip
+#- shell: pip install -U setuptools
+- shell: pip install httpie
+- shell: pip install docker-py==1.1.0
+
+# Redis
+- shell: pip install redis
+- lineinfile: dest=/etc/redis/redis.conf regexp='^bind ' line='bind 0.0.0.0'
+- service: name=redis-server state=restarted
+
+# Ubuntu OpenStack packages
+#- apt: name=ubuntu-cloud-keyring state=present
+#- shell: echo "deb http://ubuntu-cloud.archive.canonical.com/ubuntu trusty-updates/kilo main" > /etc/apt/sources.list.d/cloudarchive-kilo.list
+#- shell: echo "deb http://osci-mirror-poz.infra.mirantis.net/pkgs/ubuntu-2015-06-25-194717 trusty-updates main" > /etc/apt/sources.list.d/fuel-kilo.list
+#- shell: echo "deb http://osci-mirror-poz.infra.mirantis.net/pkgs/ubuntu-latest trusty main" > /etc/apt/sources.list.d/fuel-kilo.list
+# cloudarchive key
+#- shell: apt-key adv --recv-key --keyserver keyserver.ubuntu.com 5EDB1B62EC4926EA
+# some other keys
+#- shell: apt-key adv --recv-key --keyserver keyserver.ubuntu.com 9D6D8F6BC857C906
+#- shell: apt-key adv --recv-key --keyserver keyserver.ubuntu.com 7638D0442B90D010
+# mirantis poznan
+#- shell: apt-key adv --recv-key --keyserver keyserver.ubuntu.com 40976EAF437D05B5
+#- shell: apt-key adv --recv-key --keyserver keyserver.ubuntu.com 3B4FE6ACC0B21F32
+#- shell: apt-get update

bootstrap/playbooks/tasks/cinder.yml

@@ -0,0 +1,15 @@
+---
+
+- apt: name=lvm2 state=present
+- command: sudo truncate -s 10G /root/cinder.img creates=/root/cinder.img
+- shell: sudo losetup -a|grep cinder
+  register: loop_created
+  ignore_errors: True
+- command: sudo losetup /dev/loop0 /root/cinder.img
+  when: loop_created|failed
+#  retries: 5
+#  delay: 1
+- lvg: vg=cinder-volumes pvs=/dev/loop0
+  when: loop_created|failed
+#  retries: 5
+#  delay: 1

bootstrap/playbooks/tasks/docker.yml

@@ -0,0 +1,17 @@
+---
+
+- shell: docker --version
+  ignore_errors: true
+  register: docker_version
+# This script is completely broken, it has so many sleeps...
+- shell: curl -sSL https://get.docker.com/ | sudo sh
+  when: docker_version | failed
+
+# Here's a raw paste of what the above script really does for Ubuntu
+#- shell: apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
+#- shell: mkdir -p /etc/apt/sources.list.d
+#- shell: echo deb https://apt.dockerproject.org/repo ubuntu-trusty main > /etc/apt/sources.list.d/docker.list
+#  args:
+#    creates: /etc/apt/sources.list.d/docker.list
+#- shell: apt-get update
+#- shell: apt-get install -y -q docker-engine

bootstrap/playbooks/tasks/puppet.yml

@@ -0,0 +1,24 @@
+---
+
+# Puppet
+- shell: wget https://apt.puppetlabs.com/puppetlabs-release-trusty.deb -O /root/puppetlabs-release-trusty.deb
+- shell: dpkg -i /root/puppetlabs-release-trusty.deb
+- shell: apt-get update
+
+- apt: name=puppet state=present
+- git: repo=https://github.com/CGenie/hiera-redis dest=/root/hiera-redis
+- shell: gem build hiera-redis.gemspec && gem install hiera-redis-3.0.0.gem chdir=/root/hiera-redis
+- template: src=files/hiera.yaml dest=/etc/puppet/hiera.yaml
+- file: path=/etc/puppet/hieradata state=directory
+# Make paths puppet 4 compatible
+- file: path=/etc/puppetlabs/code/ state=directory
+- file: src=/etc/puppet/hiera.yaml dest=/etc/puppetlabs/code/hiera.yaml state=link
+- file: path=/var/tmp/puppet/modules state=directory owner=puppet
+- file: path=/var/tmp/puppet/Puppetfile state=touch owner=puppet
+- file: path=/etc/puppet/modules state=absent
+- file: path=/etc/puppetlabs/code/modules state=absent
+- file: path=/etc/puppetlabs/code/hieradata state=absent
+- file: src=/var/tmp/puppet/modules dest=/etc/puppet/modules state=link
+- file: src=/var/tmp/puppet/modules dest=/etc/puppetlabs/code/modules state=link
+- file: src=/etc/puppet/hieradata dest=/etc/puppetlabs/code/hieradata state=link
+- shell: gem install librarian-puppet --no-ri --no-rdoc

bootstrap/solar-master.json

@@ -0,0 +1,22 @@
+{
+  "builders": [{
+    "type": "virtualbox-ovf",
+    "source_path": "trusty64/box.ovf",
+    "ssh_username": "vagrant",
+    "ssh_password": "vagrant",
+    "shutdown_command": "echo 'vagrant' | sudo -S shutdown -P now"
+  }],
+  "provisioners": [
+    {
+      "type": "shell",
+      "script": "playbooks/files/ubuntu-ansible.sh"
+    }, {
+      "type": "ansible-local",
+      "playbook_dir": "playbooks",
+      "playbook_file": "playbooks/build-main.yml"
+    }],
+  "post-processors": [{
+    "type": "vagrant",
+    "output": "solar-master.box"
+  }]
+}

example-bootstrap.py

@@ -0,0 +1,109 @@
+import click
+import sys
+import time
+
+from solar.core import actions
+from solar.core import resource
+from solar.core import signals
+from solar.core import validation
+from solar.core.resource import virtual_resource as vr
+from solar import errors
+
+from solar.interfaces.db import get_db
+
+
+GIT_PUPPET_LIBS_URL = 'https://github.com/CGenie/puppet-libs-resource'
+
+
+# TODO
+# Resource for repository OR puppet apt-module in run.pp
+# add-apt-repository cloud-archive:juno
+# To discuss: install stuff in Docker container
+
+# NOTE
+# No copy of manifests, pull from upstream (implemented in the puppet handler)
+# Official puppet manifests, not fuel-library
+
+
+db = get_db()
+
+
+@click.group()
+def main():
+    pass
+
+
+def setup_resources():
+    db.clear()
+
+    signals.Connections.clear()
+
+    node3 = vr.create('node3', 'resources/ro_node/', {
+        'ip': '10.0.0.5',
+        'ssh_key': '/vagrant/.vagrant/machines/solar-dev3/virtualbox/private_key',
+        'ssh_user': 'vagrant'
+    })[0]
+
+    solar_bootstrap3 = vr.create('solar_bootstrap3', 'resources/solar_bootstrap', {'master_ip': '10.0.0.2'})[0]
+
+    signals.connect(node3, solar_bootstrap3)
+
+    has_errors = False
+    for r in locals().values():
+        if not isinstance(r, resource.Resource):
+            continue
+
+        print 'Validating {}'.format(r.name)
+        errors = validation.validate_resource(r)
+        if errors:
+            has_errors = True
+            print 'ERROR: %s: %s' % (r.name, errors)
+
+    if has_errors:
+        sys.exit(1)
+
+resources_to_run = [
+    'solar_bootstrap3',
+]
+
+
+@click.command()
+def deploy():
+    setup_resources()
+
+    # run
+    resources = map(resource.wrap_resource, db.get_list(collection=db.COLLECTIONS.resource))
+    resources = {r.name: r for r in resources}
+
+    for name in resources_to_run:
+        try:
+            actions.resource_action(resources[name], 'run')
+        except errors.SolarError as e:
+            print 'WARNING: %s' % str(e)
+            raise
+
+    time.sleep(10)
+
+
+@click.command()
+def undeploy():
+    resources = map(resource.wrap_resource, db.get_list(collection=db.COLLECTIONS.resource))
+    resources = {r.name: r for r in resources}
+
+    for name in reversed(resources_to_run):
+        try:
+            actions.resource_action(resources[name], 'remove')
+        except errors.SolarError as e:
+            print 'WARNING: %s' % str(e)
+
+    db.clear()
+
+    signals.Connections.clear()
+
+
+main.add_command(deploy)
+main.add_command(undeploy)
+
+
+if __name__ == '__main__':
+    main()

example-puppet.py

@@ -657,7 +657,6 @@ resources_to_run = [
     'neutron_agents_ovs2',
 ]
 
-
 @click.command()
 def deploy():
     setup_resources()

main.yml

@@ -3,29 +3,26 @@
 - hosts: all
   sudo: yes
   tasks:
+    - apt: name=git state=present
+    - apt: name=python-mock state=present
+    - apt: name=python-keystoneclient state=present
+
+    # PIP
     - apt: name=python-pip state=absent
     - apt: name=python-six state=absent
     - shell: easy_install pip
     - shell: pip install -U pip
     - shell: pip install -U setuptools
     - shell: pip install httpie
-    #- apt: name=python-virtualenv state=present
-    #- apt: name=virtualenvwrapper state=present
-    - apt: name=python-mock state=present
-    - apt: name=ipython state=present
-    - apt: name=python-pudb state=present
-    #- apt: name=python-pip state=present
     - shell: pip install docker-py==1.1.0
-    - apt: name=python-keystoneclient state=present
 
+    # Redis
     - apt: name=redis-server state=present
-    #- apt: name=python-redis state=present
     - shell: pip install redis
     - lineinfile: dest=/etc/redis/redis.conf regexp='^bind ' line='bind 0.0.0.0'
     - service: name=redis-server state=restarted
 
-    - apt: name=git state=present
-
+    # Puppet
     - shell: wget https://apt.puppetlabs.com/puppetlabs-release-trusty.deb -O /root/puppetlabs-release-trusty.deb
     - shell: dpkg -i /root/puppetlabs-release-trusty.deb
     - shell: apt-get update
@@ -50,11 +47,16 @@
     - shell: gem install librarian-puppet --no-ri --no-rdoc
 
       # Setup additional development tools
-    - apt: name=vim state=present
-    - apt: name=tmux state=present
-    - apt: name=htop state=present
-    - apt: name=python-mysqldb state=present
-    - apt: name=jq state=present
+    - name: Additional development tools
+      apt: name={{ item }} state=present
+      with_items:
+      - vim
+      - tmux
+      - htop
+      - python-mysqldb
+      - jq
+      - ipython
+      - python-pudb
 
     # Graph drawing
     #- apt: name=python-matplotlib state=present
@@ -64,6 +66,7 @@
       # Setup development env for solar
     - shell: python setup.py develop chdir=/vagrant/solar
 
+    # Ubuntu OpenStack packages
     #- apt: name=ubuntu-cloud-keyring state=present
     #- shell: echo "deb http://ubuntu-cloud.archive.canonical.com/ubuntu trusty-updates/kilo main" > /etc/apt/sources.list.d/cloudarchive-kilo.list
     #- shell: echo "deb http://osci-mirror-poz.infra.mirantis.net/pkgs/ubuntu-2015-06-25-194717 trusty-updates main" > /etc/apt/sources.list.d/fuel-kilo.list

resources/solar_bootstrap/actions/run.yaml

@@ -0,0 +1,12 @@
+---
+
+# TODO: this shouldn't be outside of the resource directory
+- hosts: all
+  sudo: yes
+  tasks:
+    - script: /vagrant/bootstrap/playbooks/files/ubuntu-ansible.sh
+    - include: /vagrant/bootstrap/playbooks/tasks/cinder.yml
+#- include: celery.yml tags=['master'] celery_dir=/var/run/celery
+- include: /vagrant/bootstrap/playbooks/build-main.yml
+- include: /vagrant/bootstrap/playbooks/custom-configs.yml master_ip={{ master_ip }}
+- include: /vagrant/bootstrap/playbooks/celery.yml tags=slave

resources/solar_bootstrap/meta.yaml

@@ -0,0 +1,18 @@
+handler: ansible
+id: 'solar_bootstrap'
+input:
+  ip:
+    schema: str!
+    value:
+  ssh_key:
+    schema: str!
+    value:
+  ssh_user:
+    schema: str!
+    value:
+
+  master_ip:
+    schema: str!
+    value:
+tags: []
+version: 1.0.0