From 092a82be1aff36d771d839e9b3d026161018d892 Mon Sep 17 00:00:00 2001
From: Przemyslaw Kaminski <pkaminski@mirantis.com>
Date: Wed, 5 Aug 2015 09:56:05 +0200
Subject: [PATCH 1/9] main.yml cleanup

---
 main.yml | 33 ++++++++++++++++++---------------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/main.yml b/main.yml
index 00ced16d..34e5d1ee 100644
--- a/main.yml
+++ b/main.yml
@@ -3,28 +3,25 @@
 - hosts: all
   sudo: yes
   tasks:
+    - apt: name=git state=present
+    - apt: name=python-mock state=present
+    - apt: name=python-keystoneclient state=present
+
+    # PIP
     - apt: name=python-pip state=absent
     - shell: easy_install pip
     - shell: pip install -U pip
     - shell: pip install -U setuptools
     - shell: pip install httpie
-    #- apt: name=python-virtualenv state=present
-    #- apt: name=virtualenvwrapper state=present
-    - apt: name=python-mock state=present
-    - apt: name=ipython state=present
-    - apt: name=python-pudb state=present
-    #- apt: name=python-pip state=present
     - shell: pip install docker-py==1.1.0
-    - apt: name=python-keystoneclient state=present
 
+    # Redis
     - apt: name=redis-server state=present
-    #- apt: name=python-redis state=present
     - shell: pip install redis
     - lineinfile: dest=/etc/redis/redis.conf regexp='^bind ' line='bind 0.0.0.0'
     - service: name=redis-server state=restarted
 
-    - apt: name=git state=present
-
+    # Puppet
     - shell: wget https://apt.puppetlabs.com/puppetlabs-release-trusty.deb -O /root/puppetlabs-release-trusty.deb
     - shell: dpkg -i /root/puppetlabs-release-trusty.deb
     - shell: apt-get update
@@ -49,11 +46,16 @@
     - shell: gem install librarian-puppet --no-ri --no-rdoc
 
       # Setup additional development tools
-    - apt: name=vim state=present
-    - apt: name=tmux state=present
-    - apt: name=htop state=present
-    - apt: name=python-mysqldb state=present
-    - apt: name=jq state=present
+    - name: Additional development tools
+      apt: name={{ item }} state=present
+      with_items:
+      - vim
+      - tmux
+      - htop
+      - python-mysqldb
+      - jq
+      - ipython
+      - python-pudb
 
     # Graph drawing
     #- apt: name=python-matplotlib state=present
@@ -63,6 +65,7 @@
       # Setup development env for solar
     - shell: python setup.py develop chdir=/vagrant/solar
 
+    # Ubuntu OpenStack packages
     #- apt: name=ubuntu-cloud-keyring state=present
     #- shell: echo "deb http://ubuntu-cloud.archive.canonical.com/ubuntu trusty-updates/kilo main" > /etc/apt/sources.list.d/cloudarchive-kilo.list
     #- shell: echo "deb http://osci-mirror-poz.infra.mirantis.net/pkgs/ubuntu-2015-06-25-194717 trusty-updates main" > /etc/apt/sources.list.d/fuel-kilo.list

From 79d91f5be2edc088ac668d9b75c85ec09523d320 Mon Sep 17 00:00:00 2001
From: Przemyslaw Kaminski <pkaminski@mirantis.com>
Date: Thu, 6 Aug 2015 10:57:48 +0200
Subject: [PATCH 2/9] Solar bootstarpping playbooks & packer.io guide

---
 .gitignore                             |  4 ++
 Vagrantfile                            | 25 +++++-------
 bootstrap/README.md                    | 20 +++++++++
 bootstrap/build-image.yml              |  8 ++++
 bootstrap/packer.yml                   | 12 ++++++
 bootstrap/playbooks/base.yml           | 56 ++++++++++++++++++++++++++
 bootstrap/playbooks/build-main.yml     | 10 +++++
 bootstrap/playbooks/celery.yml         | 20 +++++++++
 bootstrap/playbooks/cinder.yml         | 11 +++++
 bootstrap/playbooks/custom-configs.yml |  7 ++++
 bootstrap/playbooks/docker.yml         |  7 ++++
 bootstrap/playbooks/files/hiera.yaml   | 15 +++++++
 bootstrap/playbooks/puppet.yml         | 24 +++++++++++
 bootstrap/playbooks/solar.yml          |  7 ++++
 bootstrap/solar-master.json            | 28 +++++++++++++
 15 files changed, 238 insertions(+), 16 deletions(-)
 create mode 100644 bootstrap/README.md
 create mode 100644 bootstrap/build-image.yml
 create mode 100644 bootstrap/packer.yml
 create mode 100644 bootstrap/playbooks/base.yml
 create mode 100644 bootstrap/playbooks/build-main.yml
 create mode 100644 bootstrap/playbooks/celery.yml
 create mode 100644 bootstrap/playbooks/cinder.yml
 create mode 100644 bootstrap/playbooks/custom-configs.yml
 create mode 100644 bootstrap/playbooks/docker.yml
 create mode 100644 bootstrap/playbooks/files/hiera.yaml
 create mode 100644 bootstrap/playbooks/puppet.yml
 create mode 100644 bootstrap/playbooks/solar.yml
 create mode 100644 bootstrap/solar-master.json

diff --git a/.gitignore b/.gitignore
index 18550fc1..1d9f0c69 100644
--- a/.gitignore
+++ b/.gitignore
@@ -26,3 +26,7 @@ celery*.log
 *.dot
 *.png
 resources_compiled.py
+
+# bootstrap
+bootstrap/trusty64
+bootstrap/solar-master.box
diff --git a/Vagrantfile b/Vagrantfile
index 539766f9..bac27f72 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -5,38 +5,31 @@
 VAGRANTFILE_API_VERSION = "2"
 SLAVES_COUNT = 2
 
-init_script = <<SCRIPT
-apt-get update
-apt-get -y install python-pip python-dev
-pip install ansible
-ansible-playbook -i "localhost," -c local /vagrant/main.yml /vagrant/docker.yml
+solar_script = <<SCRIPT
+ansible-playbook -i "localhost," -c local /vagrant/bootstrap/playbooks/solar.yml
 SCRIPT
 
 slave_script = <<SCRIPT
-apt-get update
-apt-get upgrade
-apt-get dist-upgrade
-apt-get -y install python-pip python-dev
-pip install ansible
-ansible-playbook -i "localhost," -c local /vagrant/main.yml /vagrant/docker.yml /vagrant/slave.yml /vagrant/slave_cinder.yml
+ansible-playbook -i "localhost," -c local /vagrant/bootstrap/playbooks/custom-configs.yml -e master_ip=10.0.0.2
 SCRIPT
 
 master_celery = <<SCRIPT
-ansible-playbook -i "localhost," -c local /vagrant/celery.yml --skip-tags slave
+ansible-playbook -i "localhost," -c local /vagrant/bootstrap/playbooks/celery.yml --skip-tags slave
 SCRIPT
 
 slave_celery = <<SCRIPT
-ansible-playbook -i "localhost," -c local /vagrant/celery.yml --skip-tags master
+ansible-playbook -i "localhost," -c local /vagrant/bootstrap/playbooks/celery.yml --skip-tags master
 SCRIPT
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
   #config.vm.box = "deb/jessie-amd64"
   #config.vm.box = "rustyrobot/deb-jessie-amd64"
-  config.vm.box = "ubuntu/trusty64"
+  #config.vm.box = "ubuntu/trusty64"
+  config.vm.box = "solar-master.box"
 
   config.vm.define "solar-dev", primary: true do |config|
-    config.vm.provision "shell", inline: init_script, privileged: true
+    config.vm.provision "shell", inline: solar_script, privileged: true
     config.vm.provision "shell", inline: master_celery, privileged: true
     config.vm.provision "file", source: "~/.vagrant.d/insecure_private_key", destination: "/vagrant/tmp/keys/ssh_private"
     config.vm.provision "file", source: "ansible.cfg", destination: "/home/vagrant/.ansible.cfg"
@@ -53,8 +46,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     index = i + 1
     ip_index = i + 3
     config.vm.define "solar-dev#{index}" do |config|
-      config.vm.provision "shell", inline: init_script, privileged: true
       config.vm.provision "shell", inline: slave_script, privileged: true
+      config.vm.provision "shell", inline: solar_script, privileged: true
       config.vm.provision "shell", inline: slave_celery, privileged: true
       config.vm.network "private_network", ip: "10.0.0.#{ip_index}"
       config.vm.host_name = "solar-dev#{index}"
diff --git a/bootstrap/README.md b/bootstrap/README.md
new file mode 100644
index 00000000..834df1e0
--- /dev/null
+++ b/bootstrap/README.md
@@ -0,0 +1,20 @@
+# Solar image building
+
+In `bootstrap/trusty64` directory there are `box.ovf` and `box-disk1.vmdk`
+files from the `trusty64` Vagrant box (normally found in
+ `~/.vagrant.d/boxes/trusty64/0/virtualbox`).
+
+To build, install Packer (https://www.packer.io/):
+```
+cd bootstrap
+packer build solar-master.json
+cp solar-master.box ../
+cd ..
+vagrant up
+```
+
+If Vagrant throws error about `vboxsf` try this:
+```
+vagrant plugin install vagrant-vbguest
+```
+(see https://github.com/shiguredo/packer-templates/issues/16).
\ No newline at end of file
diff --git a/bootstrap/build-image.yml b/bootstrap/build-image.yml
new file mode 100644
index 00000000..9dd5e139
--- /dev/null
+++ b/bootstrap/build-image.yml
@@ -0,0 +1,8 @@
+---
+
+- hosts: all
+  sudo: yes
+  tasks:
+     #- shell: vagrant init ubuntu/trusty64
+
+     - shell: /usr/local/bin/packer build solar.json -var 'is_master=true'
diff --git a/bootstrap/packer.yml b/bootstrap/packer.yml
new file mode 100644
index 00000000..e5c81d4f
--- /dev/null
+++ b/bootstrap/packer.yml
@@ -0,0 +1,12 @@
+---
+
+- hosts: all
+  sudo: yes
+  tasks:
+    - shell: wget 'https://dl.bintray.com/mitchellh/packer/packer_0.8.2_linux_amd64.zip' -O /tmp/packer-0.8.2.zip
+      args:
+        creates: /tmp/packer-0.8.2.zip
+    - unarchive:
+       src: /tmp/packer-0.8.2.zip
+       dest: /usr/local/bin
+       copy: no
diff --git a/bootstrap/playbooks/base.yml b/bootstrap/playbooks/base.yml
new file mode 100644
index 00000000..2debca36
--- /dev/null
+++ b/bootstrap/playbooks/base.yml
@@ -0,0 +1,56 @@
+---
+
+- shell: apt-get update
+
+- name: Base packages
+  apt: name={{ item }} state=present
+  with_items:
+  - git
+  - python-mock
+  - python-keystoneclient
+  - python-mysqldb
+  - python-setuptools
+  - ruby-dev
+  - unzip
+  # Redis
+  - redis-server
+  # Graph drawing
+  - python-pygraphviz
+  # Other tools
+  - htop
+  - jq
+  - tmux
+  - vim
+  # Dev
+  - ipython
+  - python-pudb
+  # Required by packer
+  - build-essential
+
+# PIP
+#- apt: name=python-pip state=absent
+#- shell: easy_install pip
+#- shell: pip install -U pip
+#- shell: pip install -U setuptools
+- shell: pip install httpie
+- shell: pip install docker-py==1.1.0
+
+# Redis
+- shell: pip install redis
+- lineinfile: dest=/etc/redis/redis.conf regexp='^bind ' line='bind 0.0.0.0'
+- service: name=redis-server state=restarted
+
+# Ubuntu OpenStack packages
+#- apt: name=ubuntu-cloud-keyring state=present
+#- shell: echo "deb http://ubuntu-cloud.archive.canonical.com/ubuntu trusty-updates/kilo main" > /etc/apt/sources.list.d/cloudarchive-kilo.list
+#- shell: echo "deb http://osci-mirror-poz.infra.mirantis.net/pkgs/ubuntu-2015-06-25-194717 trusty-updates main" > /etc/apt/sources.list.d/fuel-kilo.list
+#- shell: echo "deb http://osci-mirror-poz.infra.mirantis.net/pkgs/ubuntu-latest trusty main" > /etc/apt/sources.list.d/fuel-kilo.list
+# cloudarchive key
+#- shell: apt-key adv --recv-key --keyserver keyserver.ubuntu.com 5EDB1B62EC4926EA
+# some other keys
+#- shell: apt-key adv --recv-key --keyserver keyserver.ubuntu.com 9D6D8F6BC857C906
+#- shell: apt-key adv --recv-key --keyserver keyserver.ubuntu.com 7638D0442B90D010
+# mirantis poznan
+#- shell: apt-key adv --recv-key --keyserver keyserver.ubuntu.com 40976EAF437D05B5
+#- shell: apt-key adv --recv-key --keyserver keyserver.ubuntu.com 3B4FE6ACC0B21F32
+#- shell: apt-get update
diff --git a/bootstrap/playbooks/build-main.yml b/bootstrap/playbooks/build-main.yml
new file mode 100644
index 00000000..f71615a1
--- /dev/null
+++ b/bootstrap/playbooks/build-main.yml
@@ -0,0 +1,10 @@
+---
+
+- hosts: all
+  sudo: yes
+  tasks:
+    - include: base.yml
+    - include: puppet.yml
+    - include: docker.yml
+    #- include: celery.yml tags=['master'] celery_dir=/var/run/celery
+    - include: cinder.yml
diff --git a/bootstrap/playbooks/celery.yml b/bootstrap/playbooks/celery.yml
new file mode 100644
index 00000000..5bbf7651
--- /dev/null
+++ b/bootstrap/playbooks/celery.yml
@@ -0,0 +1,20 @@
+---
+
+- hosts: all
+  sudo: yes
+  vars:
+    celery_dir: /var/run/celery
+  tasks:
+    - shell: mkdir -p {{ celery_dir }}
+    - shell: pip install celery
+    - shell: hostname
+      register: hostname
+    - shell: celery multi kill 2
+             chdir={{ celery_dir }}
+      tags: [stop]
+    - shell: celery multi start 2 -A solar.orchestration.runner -Q:1 scheduler,system_log -Q:2 celery,{{ hostname.stdout }}
+             chdir={{ celery_dir }}
+      tags: [master]
+    - shell: celery multi start 1 -A solar.orchestration.runner -Q:1  {{ hostname.stdout }}
+             chdir={{ celery_dir }}
+      tags: [slave]
diff --git a/bootstrap/playbooks/cinder.yml b/bootstrap/playbooks/cinder.yml
new file mode 100644
index 00000000..8d95d2f3
--- /dev/null
+++ b/bootstrap/playbooks/cinder.yml
@@ -0,0 +1,11 @@
+---
+
+- apt: name=lvm2 state=present
+- command: sudo truncate -s 10G /root/cinder.img creates=/root/cinder.img
+- shell: sudo losetup -a|grep cinder
+  register: loop_created
+  ignore_errors: True
+- command: sudo losetup /dev/loop0 /root/cinder.img
+  when: loop_created|failed
+- lvg: vg=cinder-volumes pvs=/dev/loop0
+  when: loop_created|failed
diff --git a/bootstrap/playbooks/custom-configs.yml b/bootstrap/playbooks/custom-configs.yml
new file mode 100644
index 00000000..76636a56
--- /dev/null
+++ b/bootstrap/playbooks/custom-configs.yml
@@ -0,0 +1,7 @@
+---
+
+- hosts: all
+  sudo: yes
+  tasks:
+    - lineinfile: line='slaveof {{ master_ip }} 6379' dest=/etc/redis/redis.conf
+    - service: name=redis-server state=restarted
diff --git a/bootstrap/playbooks/docker.yml b/bootstrap/playbooks/docker.yml
new file mode 100644
index 00000000..e855cc53
--- /dev/null
+++ b/bootstrap/playbooks/docker.yml
@@ -0,0 +1,7 @@
+---
+
+- shell: docker --version
+  ignore_errors: true
+  register: docker_version
+- shell: curl -sSL https://get.docker.com/ | sudo sh
+  when: docker_version | failed
diff --git a/bootstrap/playbooks/files/hiera.yaml b/bootstrap/playbooks/files/hiera.yaml
new file mode 100644
index 00000000..ca27ee57
--- /dev/null
+++ b/bootstrap/playbooks/files/hiera.yaml
@@ -0,0 +1,15 @@
+:backends:
+  - redis
+  #- yaml
+  #- json
+:yaml:
+  :datadir: /etc/puppet/hieradata
+:json:
+  :datadir: /etc/puppet/hieradata
+:redis:
+  :port: 6379
+  :db: 0
+  :host: localhost
+  :deserialize: :json
+:hierarchy:
+  - resource
diff --git a/bootstrap/playbooks/puppet.yml b/bootstrap/playbooks/puppet.yml
new file mode 100644
index 00000000..91542c57
--- /dev/null
+++ b/bootstrap/playbooks/puppet.yml
@@ -0,0 +1,24 @@
+---
+
+# Puppet
+- shell: wget https://apt.puppetlabs.com/puppetlabs-release-trusty.deb -O /root/puppetlabs-release-trusty.deb
+- shell: dpkg -i /root/puppetlabs-release-trusty.deb
+- shell: apt-get update
+
+- apt: name=puppet state=present
+- git: repo=https://github.com/CGenie/hiera-redis dest=/root/hiera-redis
+- shell: gem build hiera-redis.gemspec && gem install hiera-redis-3.0.0.gem chdir=/root/hiera-redis
+- template: src=files/hiera.yaml dest=/etc/puppet/hiera.yaml
+- file: path=/etc/puppet/hieradata state=directory
+# Make paths puppet 4 compatible
+- file: path=/etc/puppetlabs/code/ state=directory
+- file: src=/etc/puppet/hiera.yaml dest=/etc/puppetlabs/code/hiera.yaml state=link
+- file: path=/var/tmp/puppet/modules state=directory owner=puppet
+- file: path=/var/tmp/puppet/Puppetfile state=touch owner=puppet
+- file: path=/etc/puppet/modules state=absent
+- file: path=/etc/puppetlabs/code/modules state=absent
+- file: path=/etc/puppetlabs/code/hieradata state=absent
+- file: src=/var/tmp/puppet/modules dest=/etc/puppet/modules state=link
+- file: src=/var/tmp/puppet/modules dest=/etc/puppetlabs/code/modules state=link
+- file: src=/etc/puppet/hieradata dest=/etc/puppetlabs/code/hieradata state=link
+- shell: gem install librarian-puppet --no-ri --no-rdoc
diff --git a/bootstrap/playbooks/solar.yml b/bootstrap/playbooks/solar.yml
new file mode 100644
index 00000000..e4c3ee6b
--- /dev/null
+++ b/bootstrap/playbooks/solar.yml
@@ -0,0 +1,7 @@
+---
+
+- hosts: all
+  sudo: yes
+  tasks:
+    # Setup development env for solar
+    - shell: python setup.py develop chdir=/vagrant/solar
diff --git a/bootstrap/solar-master.json b/bootstrap/solar-master.json
new file mode 100644
index 00000000..b799801e
--- /dev/null
+++ b/bootstrap/solar-master.json
@@ -0,0 +1,28 @@
+{
+  "builders": [{
+    "type": "virtualbox-ovf",
+    "source_path": "trusty64/box.ovf",
+    "ssh_username": "vagrant",
+    "ssh_password": "vagrant",
+    "shutdown_command": "echo 'vagrant' | sudo -S shutdown -P now"
+  }],
+  "provisioners": [
+    {
+      "type": "shell",
+      "inline": [
+        "sudo apt-get remove -f python-pip",
+        "sudo apt-get install -y python-setuptools",
+        "sudo easy_install pip",
+        "sudo pip install -U pip",
+        "sudo pip install ansible"
+      ]
+    }, {
+      "type": "ansible-local",
+      "playbook_dir": "playbooks",
+      "playbook_file": "playbooks/build-main.yml"
+    }],
+  "post-processors": [{
+    "type": "vagrant",
+    "output": "solar-master.box"
+  }]
+}
\ No newline at end of file

From b206fc0864fde8b81bd258eb959c8b6d2e997945 Mon Sep 17 00:00:00 2001
From: Przemyslaw Kaminski <pkaminski@mirantis.com>
Date: Thu, 6 Aug 2015 10:58:09 +0200
Subject: [PATCH 3/9] Fix resource.py when value is not specified in meta.yaml

---
 solar/solar/core/resource/resource.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/solar/solar/core/resource/resource.py b/solar/solar/core/resource/resource.py
index 1b4955ed..4a18927d 100644
--- a/solar/solar/core/resource/resource.py
+++ b/solar/solar/core/resource/resource.py
@@ -58,9 +58,7 @@ class Resource(object):
 
         self.metadata = raw_resource
 
-        args = self.metadata['input']
-
-        return {k: v['value'] for k, v in args.items()}
+        return Resource.get_raw_resource_args(raw_resource)
 
     def set_args_from_dict(self, new_args):
         args = self.args_dict()
@@ -167,10 +165,14 @@ class Resource(object):
         else:
             raise Exception('Uuups, action is not available')
 
+    @staticmethod
+    def get_raw_resource_args(raw_resource):
+        return {k: v.get('value') for k, v in raw_resource['input'].items()}
+
 
 def wrap_resource(raw_resource):
     name = raw_resource['id']
-    args = {k: v['value'] for k, v in raw_resource['input'].items()}
+    args = Resource.get_raw_resource_args(raw_resource)
     tags = raw_resource.get('tags', [])
     virtual_resource = raw_resource.get('virtual_resource', [])
 

From 3e0500a452784fb3ffd38d2751c22402e731a38d Mon Sep 17 00:00:00 2001
From: Przemyslaw Kaminski <pkaminski@mirantis.com>
Date: Thu, 6 Aug 2015 13:09:47 +0200
Subject: [PATCH 4/9] Moved pre-install stuff (ansible, pip) to separate script

---
 bootstrap/README.md                         | 7 ++++++-
 bootstrap/playbooks/files/ubuntu-ansible.sh | 7 +++++++
 bootstrap/solar-master.json                 | 8 +-------
 3 files changed, 14 insertions(+), 8 deletions(-)
 create mode 100644 bootstrap/playbooks/files/ubuntu-ansible.sh

diff --git a/bootstrap/README.md b/bootstrap/README.md
index 834df1e0..2ab6a83a 100644
--- a/bootstrap/README.md
+++ b/bootstrap/README.md
@@ -17,4 +17,9 @@ If Vagrant throws error about `vboxsf` try this:
 ```
 vagrant plugin install vagrant-vbguest
 ```
-(see https://github.com/shiguredo/packer-templates/issues/16).
\ No newline at end of file
+(see https://github.com/shiguredo/packer-templates/issues/16).
+
+If you're rebuilding the same box, make sure Vagrant reimports it:
+```
+vagrant box remove solar-master
+```
\ No newline at end of file
diff --git a/bootstrap/playbooks/files/ubuntu-ansible.sh b/bootstrap/playbooks/files/ubuntu-ansible.sh
new file mode 100644
index 00000000..4eb6b306
--- /dev/null
+++ b/bootstrap/playbooks/files/ubuntu-ansible.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+apt-get remove -f python-pip
+sudo apt-get install -y python-setuptools
+sudo easy_install pip
+sudo pip install -U pip
+sudo pip install ansible
diff --git a/bootstrap/solar-master.json b/bootstrap/solar-master.json
index b799801e..9a9117f3 100644
--- a/bootstrap/solar-master.json
+++ b/bootstrap/solar-master.json
@@ -9,13 +9,7 @@
   "provisioners": [
     {
       "type": "shell",
-      "inline": [
-        "sudo apt-get remove -f python-pip",
-        "sudo apt-get install -y python-setuptools",
-        "sudo easy_install pip",
-        "sudo pip install -U pip",
-        "sudo pip install ansible"
-      ]
+      "script": "playbooks/files/ubuntu-ansible.sh"
     }, {
       "type": "ansible-local",
       "playbook_dir": "playbooks",

From beb6b8536f03b180a93f63106a4f13059b83d35e Mon Sep 17 00:00:00 2001
From: Przemyslaw Kaminski <pkaminski@mirantis.com>
Date: Fri, 7 Aug 2015 09:10:39 +0200
Subject: [PATCH 5/9] Bootstrap resource added

- added example-bootstrap.py
- moved ansible tasks to separate dir
---
 Vagrantfile                                 |  18 ++--
 bootstrap/playbooks/build-main.yml          |  11 +-
 bootstrap/playbooks/celery.yml              |   3 +-
 bootstrap/playbooks/custom-configs.yml      |   3 +-
 bootstrap/playbooks/docker.yml              |   7 --
 bootstrap/playbooks/files/ubuntu-ansible.sh |   3 +
 bootstrap/playbooks/{ => tasks}/base.yml    |   0
 bootstrap/playbooks/{ => tasks}/cinder.yml  |   4 +
 bootstrap/playbooks/tasks/docker.yml        |  17 +++
 bootstrap/playbooks/{ => tasks}/puppet.yml  |   0
 example-bootstrap.py                        | 108 ++++++++++++++++++++
 example-puppet.py                           |   1 -
 resources/solar_bootstrap/actions/run.yaml  |  12 +++
 resources/solar_bootstrap/meta.yaml         |  18 ++++
 14 files changed, 182 insertions(+), 23 deletions(-)
 delete mode 100644 bootstrap/playbooks/docker.yml
 rename bootstrap/playbooks/{ => tasks}/base.yml (100%)
 rename bootstrap/playbooks/{ => tasks}/cinder.yml (86%)
 create mode 100644 bootstrap/playbooks/tasks/docker.yml
 rename bootstrap/playbooks/{ => tasks}/puppet.yml (100%)
 create mode 100644 example-bootstrap.py
 create mode 100644 resources/solar_bootstrap/actions/run.yaml
 create mode 100644 resources/solar_bootstrap/meta.yaml

diff --git a/Vagrantfile b/Vagrantfile
index bac27f72..060dd7e2 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -23,12 +23,12 @@ SCRIPT
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
-  #config.vm.box = "deb/jessie-amd64"
-  #config.vm.box = "rustyrobot/deb-jessie-amd64"
-  #config.vm.box = "ubuntu/trusty64"
-  config.vm.box = "solar-master.box"
-
   config.vm.define "solar-dev", primary: true do |config|
+      #config.vm.box = "deb/jessie-amd64"
+      #config.vm.box = "rustyrobot/deb-jessie-amd64"
+      #config.vm.box = "ubuntu/trusty64"
+      config.vm.box = "solar-master.box"
+
     config.vm.provision "shell", inline: solar_script, privileged: true
     config.vm.provision "shell", inline: master_celery, privileged: true
     config.vm.provision "file", source: "~/.vagrant.d/insecure_private_key", destination: "/vagrant/tmp/keys/ssh_private"
@@ -46,9 +46,11 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     index = i + 1
     ip_index = i + 3
     config.vm.define "solar-dev#{index}" do |config|
-      config.vm.provision "shell", inline: slave_script, privileged: true
-      config.vm.provision "shell", inline: solar_script, privileged: true
-      config.vm.provision "shell", inline: slave_celery, privileged: true
+      config.vm.box = "ubuntu/trusty64"
+
+      #config.vm.provision "shell", inline: slave_script, privileged: true
+      #config.vm.provision "shell", inline: solar_script, privileged: true
+      #config.vm.provision "shell", inline: slave_celery, privileged: true
       config.vm.network "private_network", ip: "10.0.0.#{ip_index}"
       config.vm.host_name = "solar-dev#{index}"
 
diff --git a/bootstrap/playbooks/build-main.yml b/bootstrap/playbooks/build-main.yml
index f71615a1..d785a743 100644
--- a/bootstrap/playbooks/build-main.yml
+++ b/bootstrap/playbooks/build-main.yml
@@ -1,10 +1,11 @@
 ---
 
-- hosts: all
+- name: Main build script
+  hosts: all
   sudo: yes
   tasks:
-    - include: base.yml
-    - include: puppet.yml
-    - include: docker.yml
+    - include: tasks/base.yml
+    - include: tasks/puppet.yml
+    - include: tasks/docker.yml
     #- include: celery.yml tags=['master'] celery_dir=/var/run/celery
-    - include: cinder.yml
+    - include: tasks/cinder.yml
diff --git a/bootstrap/playbooks/celery.yml b/bootstrap/playbooks/celery.yml
index 5bbf7651..10944246 100644
--- a/bootstrap/playbooks/celery.yml
+++ b/bootstrap/playbooks/celery.yml
@@ -1,6 +1,7 @@
 ---
 
-- hosts: all
+- name: Solar Celery config
+  hosts: all
   sudo: yes
   vars:
     celery_dir: /var/run/celery
diff --git a/bootstrap/playbooks/custom-configs.yml b/bootstrap/playbooks/custom-configs.yml
index 76636a56..05bc1827 100644
--- a/bootstrap/playbooks/custom-configs.yml
+++ b/bootstrap/playbooks/custom-configs.yml
@@ -1,6 +1,7 @@
 ---
 
-- hosts: all
+- name: Custom Solar configs
+  hosts: all
   sudo: yes
   tasks:
     - lineinfile: line='slaveof {{ master_ip }} 6379' dest=/etc/redis/redis.conf
diff --git a/bootstrap/playbooks/docker.yml b/bootstrap/playbooks/docker.yml
deleted file mode 100644
index e855cc53..00000000
--- a/bootstrap/playbooks/docker.yml
+++ /dev/null
@@ -1,7 +0,0 @@
----
-
-- shell: docker --version
-  ignore_errors: true
-  register: docker_version
-- shell: curl -sSL https://get.docker.com/ | sudo sh
-  when: docker_version | failed
diff --git a/bootstrap/playbooks/files/ubuntu-ansible.sh b/bootstrap/playbooks/files/ubuntu-ansible.sh
index 4eb6b306..11c4a206 100644
--- a/bootstrap/playbooks/files/ubuntu-ansible.sh
+++ b/bootstrap/playbooks/files/ubuntu-ansible.sh
@@ -1,5 +1,8 @@
 #!/bin/sh
 
+# TODO: maybe this is better:
+# http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-apt-ubuntu
+
 apt-get remove -f python-pip
 sudo apt-get install -y python-setuptools
 sudo easy_install pip
diff --git a/bootstrap/playbooks/base.yml b/bootstrap/playbooks/tasks/base.yml
similarity index 100%
rename from bootstrap/playbooks/base.yml
rename to bootstrap/playbooks/tasks/base.yml
diff --git a/bootstrap/playbooks/cinder.yml b/bootstrap/playbooks/tasks/cinder.yml
similarity index 86%
rename from bootstrap/playbooks/cinder.yml
rename to bootstrap/playbooks/tasks/cinder.yml
index 8d95d2f3..f34b1438 100644
--- a/bootstrap/playbooks/cinder.yml
+++ b/bootstrap/playbooks/tasks/cinder.yml
@@ -7,5 +7,9 @@
   ignore_errors: True
 - command: sudo losetup /dev/loop0 /root/cinder.img
   when: loop_created|failed
+#  retries: 5
+#  delay: 1
 - lvg: vg=cinder-volumes pvs=/dev/loop0
   when: loop_created|failed
+#  retries: 5
+#  delay: 1
diff --git a/bootstrap/playbooks/tasks/docker.yml b/bootstrap/playbooks/tasks/docker.yml
new file mode 100644
index 00000000..3203e643
--- /dev/null
+++ b/bootstrap/playbooks/tasks/docker.yml
@@ -0,0 +1,17 @@
+---
+
+- shell: docker --version
+  ignore_errors: true
+  register: docker_version
+# This script is completely broken, it has so many sleeps...
+- shell: curl -sSL https://get.docker.com/ | sudo sh
+  when: docker_version | failed
+
+# Here's a raw paste of what the above script really does for Ubuntu
+#- shell: apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
+#- shell: mkdir -p /etc/apt/sources.list.d
+#- shell: echo deb https://apt.dockerproject.org/repo ubuntu-trusty main > /etc/apt/sources.list.d/docker.list
+#  args:
+#    creates: /etc/apt/sources.list.d/docker.list
+#- shell: apt-get update
+#- shell: apt-get install -y -q docker-engine
diff --git a/bootstrap/playbooks/puppet.yml b/bootstrap/playbooks/tasks/puppet.yml
similarity index 100%
rename from bootstrap/playbooks/puppet.yml
rename to bootstrap/playbooks/tasks/puppet.yml
diff --git a/example-bootstrap.py b/example-bootstrap.py
new file mode 100644
index 00000000..955601b4
--- /dev/null
+++ b/example-bootstrap.py
@@ -0,0 +1,108 @@
+import click
+import sys
+import time
+
+from solar.core import actions
+from solar.core import resource
+from solar.core import signals
+from solar.core import validation
+from solar.core.resource import virtual_resource as vr
+from solar import errors
+
+from solar.interfaces.db import get_db
+
+
+GIT_PUPPET_LIBS_URL = 'https://github.com/CGenie/puppet-libs-resource'
+
+
+# TODO
+# Resource for repository OR puppet apt-module in run.pp
+# add-apt-repository cloud-archive:juno
+# To discuss: install stuff in Docker container
+
+# NOTE
+# No copy of manifests, pull from upstream (implemented in the puppet handler)
+# Official puppet manifests, not fuel-library
+
+
+db = get_db()
+
+
+@click.group()
+def main():
+    pass
+
+
+def setup_resources():
+    db.clear()
+
+    signals.Connections.clear()
+
+    node1, node2 = vr.create('nodes', 'templates/nodes.yml', {})
+
+    solar_bootstrap1 = vr.create('solar_bootstrap1', 'resources/solar_bootstrap', {'master_ip': '10.0.0.2'})[0]
+    solar_bootstrap2 = vr.create('solar_bootstrap2', 'resources/solar_bootstrap', {'master_ip': '10.0.0.2'})[0]
+
+    signals.connect(node1, solar_bootstrap1)
+    signals.connect(node2, solar_bootstrap2)
+
+    has_errors = False
+    for r in locals().values():
+        if not isinstance(r, resource.Resource):
+            continue
+
+        print 'Validating {}'.format(r.name)
+        errors = validation.validate_resource(r)
+        if errors:
+            has_errors = True
+            print 'ERROR: %s: %s' % (r.name, errors)
+
+    if has_errors:
+        sys.exit(1)
+
+resources_to_run = [
+    'solar_bootstrap1',
+    'solar_bootstrap2',
+]
+
+
+@click.command()
+def deploy():
+    setup_resources()
+
+    # run
+    resources = map(resource.wrap_resource, db.get_list(collection=db.COLLECTIONS.resource))
+    resources = {r.name: r for r in resources}
+
+    for name in resources_to_run:
+        try:
+            actions.resource_action(resources[name], 'run')
+        except errors.SolarError as e:
+            print 'WARNING: %s' % str(e)
+            raise
+
+    time.sleep(10)
+
+
+@click.command()
+def undeploy():
+    resources = map(resource.wrap_resource, db.get_list(collection=db.COLLECTIONS.resource))
+    resources = {r.name: r for r in resources}
+
+    for name in reversed(resources_to_run):
+        try:
+            actions.resource_action(resources[name], 'remove')
+        except errors.SolarError as e:
+            print 'WARNING: %s' % str(e)
+
+    db.clear()
+
+    signals.Connections.clear()
+
+
+main.add_command(deploy)
+main.add_command(undeploy)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/example-puppet.py b/example-puppet.py
index 7c555748..fc883234 100644
--- a/example-puppet.py
+++ b/example-puppet.py
@@ -548,7 +548,6 @@ resources_to_run = [
 ]
 
 
-
 @click.command()
 def deploy():
     setup_resources()
diff --git a/resources/solar_bootstrap/actions/run.yaml b/resources/solar_bootstrap/actions/run.yaml
new file mode 100644
index 00000000..24edaf65
--- /dev/null
+++ b/resources/solar_bootstrap/actions/run.yaml
@@ -0,0 +1,12 @@
+---
+
+# TODO: this shouldn't be outside of the resource directory
+- hosts: all
+  sudo: yes
+  tasks:
+    - script: /vagrant/bootstrap/playbooks/files/ubuntu-ansible.sh
+    - include: /vagrant/bootstrap/playbooks/tasks/cinder.yml
+#- include: celery.yml tags=['master'] celery_dir=/var/run/celery
+- include: /vagrant/bootstrap/playbooks/build-main.yml
+- include: /vagrant/bootstrap/playbooks/custom-configs.yml master_ip={{ master_ip }}
+- include: /vagrant/bootstrap/playbooks/celery.yml tags=slave
diff --git a/resources/solar_bootstrap/meta.yaml b/resources/solar_bootstrap/meta.yaml
new file mode 100644
index 00000000..d89f224f
--- /dev/null
+++ b/resources/solar_bootstrap/meta.yaml
@@ -0,0 +1,18 @@
+handler: ansible
+id: 'solar_bootstrap'
+input:
+  ip:
+    schema: str!
+    value:
+  ssh_key:
+    schema: str!
+    value:
+  ssh_user:
+    schema: str!
+    value:
+
+  master_ip:
+    schema: str!
+    value:
+tags: []
+version: 1.0.0

From c7ea19c8c6fa2bcba0431ea42d72941c4fad72f7 Mon Sep 17 00:00:00 2001
From: Przemyslaw Kaminski <pkaminski@mirantis.com>
Date: Fri, 7 Aug 2015 11:55:21 +0200
Subject: [PATCH 6/9] Add solar-dev3 node for testing

---
 Vagrantfile          | 10 ++++++++--
 example-bootstrap.py | 15 ++++++++-------
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/Vagrantfile b/Vagrantfile
index 060dd7e2..87324cf5 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -3,7 +3,7 @@
 
 # Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
 VAGRANTFILE_API_VERSION = "2"
-SLAVES_COUNT = 2
+SLAVES_COUNT = 3
 
 solar_script = <<SCRIPT
 ansible-playbook -i "localhost," -c local /vagrant/bootstrap/playbooks/solar.yml
@@ -46,7 +46,13 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     index = i + 1
     ip_index = i + 3
     config.vm.define "solar-dev#{index}" do |config|
-      config.vm.box = "ubuntu/trusty64"
+      # Box solar-dev3 is for 'solar_bootstrap' resource demo
+      if index == 3 then
+        config.vm.box = "ubuntu/trusty64"
+      else
+        # standard box with all stuff preinstalled
+        config.vm.box = "solar-master.box"
+      end
 
       #config.vm.provision "shell", inline: slave_script, privileged: true
       #config.vm.provision "shell", inline: solar_script, privileged: true
diff --git a/example-bootstrap.py b/example-bootstrap.py
index 955601b4..93867fd0 100644
--- a/example-bootstrap.py
+++ b/example-bootstrap.py
@@ -38,13 +38,15 @@ def setup_resources():
 
     signals.Connections.clear()
 
-    node1, node2 = vr.create('nodes', 'templates/nodes.yml', {})
+    node3 = vr.create('node3', 'resources/ro_node/', {
+        'ip': '10.0.0.5',
+        'ssh_key': '/vagrant/.vagrant/machines/solar-dev3/virtualbox/private_key',
+        'ssh_user': 'vagrant'
+    })[0]
 
-    solar_bootstrap1 = vr.create('solar_bootstrap1', 'resources/solar_bootstrap', {'master_ip': '10.0.0.2'})[0]
-    solar_bootstrap2 = vr.create('solar_bootstrap2', 'resources/solar_bootstrap', {'master_ip': '10.0.0.2'})[0]
+    solar_bootstrap3 = vr.create('solar_bootstrap3', 'resources/solar_bootstrap', {'master_ip': '10.0.0.2'})[0]
 
-    signals.connect(node1, solar_bootstrap1)
-    signals.connect(node2, solar_bootstrap2)
+    signals.connect(node3, solar_bootstrap3)
 
     has_errors = False
     for r in locals().values():
@@ -61,8 +63,7 @@ def setup_resources():
         sys.exit(1)
 
 resources_to_run = [
-    'solar_bootstrap1',
-    'solar_bootstrap2',
+    'solar_bootstrap3',
 ]
 
 

From 103462b190b2c73f2cd26378c3335badbf7a5ed9 Mon Sep 17 00:00:00 2001
From: Przemyslaw Kaminski <pkaminski@mirantis.com>
Date: Mon, 31 Aug 2015 14:03:35 +0200
Subject: [PATCH 7/9] Fix solar-master box

---
 Vagrantfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Vagrantfile b/Vagrantfile
index 87324cf5..bf58d36a 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -3,7 +3,7 @@
 
 # Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
 VAGRANTFILE_API_VERSION = "2"
-SLAVES_COUNT = 3
+SLAVES_COUNT = 2
 
 solar_script = <<SCRIPT
 ansible-playbook -i "localhost," -c local /vagrant/bootstrap/playbooks/solar.yml
@@ -27,7 +27,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
       #config.vm.box = "deb/jessie-amd64"
       #config.vm.box = "rustyrobot/deb-jessie-amd64"
       #config.vm.box = "ubuntu/trusty64"
-      config.vm.box = "solar-master.box"
+      config.vm.box = "cgenie/solar-master"
 
     config.vm.provision "shell", inline: solar_script, privileged: true
     config.vm.provision "shell", inline: master_celery, privileged: true
@@ -51,7 +51,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
         config.vm.box = "ubuntu/trusty64"
       else
         # standard box with all stuff preinstalled
-        config.vm.box = "solar-master.box"
+        config.vm.box = "cgenie/solar-master"
       end
 
       #config.vm.provision "shell", inline: slave_script, privileged: true

From 9c9862d06a852b3a6059e97232b06979c1e209d5 Mon Sep 17 00:00:00 2001
From: Przemyslaw Kaminski <pkaminski@mirantis.com>
Date: Mon, 31 Aug 2015 14:16:53 +0200
Subject: [PATCH 8/9] Vagrantfile: add libvirt paravirtualization by default
 (works for VB >= 5.0)

---
 Vagrantfile | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/Vagrantfile b/Vagrantfile
index bf58d36a..de623255 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -37,7 +37,11 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     config.vm.host_name = "solar-dev"
 
     config.vm.provider :virtualbox do |v|
-      v.customize ["modifyvm", :id, "--memory", 1024]
+      v.customize [
+        "modifyvm", :id,
+        "--memory", 1024,
+        "--paravirtprovider", "kvm" # for linux guest
+      ]
       v.name = "solar-dev"
     end
   end
@@ -61,7 +65,11 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
       config.vm.host_name = "solar-dev#{index}"
 
       config.vm.provider :virtualbox do |v|
-        v.customize ["modifyvm", :id, "--memory", 1024]
+        v.customize [
+            "modifyvm", :id,
+            "--memory", 1024,
+            "--paravirtprovider", "kvm" # for linux guest
+        ]
         v.name = "solar-dev#{index}"
       end
     end

From e7fc69669d8e8af14139df41e905cc0af47e550c Mon Sep 17 00:00:00 2001
From: Przemyslaw Kaminski <pkaminski@mirantis.com>
Date: Mon, 31 Aug 2015 14:37:21 +0200
Subject: [PATCH 9/9] Vagrantfile: add slave custom setups

---
 Vagrantfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Vagrantfile b/Vagrantfile
index de623255..756a1e2b 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -58,9 +58,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
         config.vm.box = "cgenie/solar-master"
       end
 
-      #config.vm.provision "shell", inline: slave_script, privileged: true
-      #config.vm.provision "shell", inline: solar_script, privileged: true
-      #config.vm.provision "shell", inline: slave_celery, privileged: true
+      config.vm.provision "shell", inline: slave_script, privileged: true
+      config.vm.provision "shell", inline: solar_script, privileged: true
+      config.vm.provision "shell", inline: slave_celery, privileged: true
       config.vm.network "private_network", ip: "10.0.0.#{ip_index}"
       config.vm.host_name = "solar-dev#{index}"