PG support in Vagrant and in test env

Changes included:
- non standard docker image (alpine-postgres)
- new setting in vagrant-settings.yaml_defaults (solar_db_backend)
- multidb logic in playbooks (databases.yaml)
- possibility to set backend by SOLAR_DB_BACKEND in jenkins run.sh
- packer images version 0.3.0.pre0 (pre builded docker image)

Change-Id: Id48236aa778597ed787f8455ec4d4c869d0ef1cf
Closes-bug: #1546205

Vagrantfile

@@ -44,6 +44,7 @@ MASTER_CPUS = cfg["master_cpus"]
 SLAVES_CPUS = cfg["slaves_cpus"]
 PARAVIRT_PROVIDER = cfg.fetch('paravirtprovider', false)
 PREPROVISIONED = cfg.fetch('preprovisioned', true)
+SOLAR_DB_BACKEND = cfg.fetch('solar_db_backend', 'riak')
 
 # Initialize noop plugins only in case of PXE boot
 require_relative 'bootstrap/vagrant_plugins/noop' unless PREPROVISIONED
@@ -57,14 +58,13 @@ solar_agent_script = ansible_playbook_command("solar-agent.yaml")
 
 master_pxe = ansible_playbook_command("pxe.yaml")
 
-
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
   config.vm.define "solar-dev", primary: true do |config|
     config.vm.box = MASTER_IMAGE
     config.vm.box_version = MASTER_IMAGE_VERSION
 
-    config.vm.provision "shell", inline: solar_script, privileged: true
+    config.vm.provision "shell", inline: solar_script, privileged: true, env: {"SOLAR_DB_BACKEND": SOLAR_DB_BACKEND}
     config.vm.provision "shell", inline: master_pxe, privileged: true unless PREPROVISIONED
     config.vm.provision "file", source: "~/.vagrant.d/insecure_private_key", destination: "/vagrant/tmp/keys/ssh_private"
     config.vm.host_name = "solar-dev"

bootstrap/playbooks/databases.yaml

@@ -0,0 +1,7 @@
+- include: riak.yaml
+  when: lookup('env', 'SOLAR_DB_BACKEND') == "riak"
+- include: postgres.yaml
+  when: lookup('env', 'SOLAR_DB_BACKEND') == "postgres"
+- fail: msg="Invalid SOLAR_DB_BACKEND set"
+  when: (lookup('env', 'SOLAR_DB_BACKEND') != "riak") and
+        (lookup('env', 'SOLAR_DB_BACKEND') != "postgres")

bootstrap/playbooks/postgres.yaml

@@ -0,0 +1,18 @@
+- name: start pg container
+  docker:
+    name: vagrant_pg_1
+    image: alpine-postgres
+    ports:
+      5432:5432
+    env:
+      POSTGRES_USER: solar
+      POSTGRES_PASSWORD: solar
+      POSTGRES_DATABASE: solar
+- wait_for: host=127.0.0.1 port=5432 timeout=15 delay=1
+- shell: docker exec vagrant_pg_1 gosu postgres psql -c "DROP DATABASE solar;"
+- shell: docker exec vagrant_pg_1 gosu postgres psql -c "CREATE DATABASE solar WITH owner=postgres LC_COLLATE='C' TEMPLATE template0;"
+- lineinfile:
+    dest: /.solar_config_override
+    line: "solar_db: postgresql://solar:solar@127.0.0.1:5432/solar"
+    state: present
+    create: yes

bootstrap/playbooks/riak.yaml

@@ -0,0 +1,20 @@
+- name: start riak container
+  docker:
+    name: vagrant_riak_1
+    image: solarproject/riak
+    volumes:
+        - /vagrant/utils/riak/app.config:/etc/riak/app.config
+    ports:
+      - 8087:8087
+      - 8098:8098
+# preconfigure docker container
+# add counters datatype etc.
+- shell: timeout 60 docker exec vagrant_riak_1 riak-admin wait_for_service riak_kv
+- shell: timeout 10 docker exec vagrant_riak_1 riak-admin bucket-type create counters '{"props":{"datatype":"counter"}}'
+  ignore_errors: yes
+- shell: timeout 10 docker exec vagrant_riak_1 riak-admin bucket-type activate counters
+- lineinfile:
+    dest: /.solar_config_override
+    line: "solar_db: riak://127.0.0.1:8087"
+    state: present
+    create: yes

bootstrap/playbooks/solar.yaml

@@ -33,11 +33,6 @@
 
     # create custom config file /vagrant/.solar_config_override for vagrant env
     - file: path=/.solar_config_override state=touch mode=0644
-    - lineinfile:
-        dest: /.solar_config_override
-        line: "solar_db: riak://127.0.0.1:8087"
-        state: present
-        create: yes
     - lineinfile:
         dest: /.solar_config_override
         line: "log_file: /var/log/solar/solar.log"
@@ -66,20 +61,7 @@
     - file: src=/vagrant/solar-resources/resources dest=/var/lib/solar/repositories/resources state=link owner=vagrant
     - file: src=/vagrant/solar-resources/templates dest=/var/lib/solar/repositories/templates state=link owner=vagrant
 
-    - name: start riak container
+    # shared stuff for all databases
-      docker:
+    - include: databases.yaml
-        name: vagrant_riak_1
-        image: solarproject/riak
-        volumes:
-            - /vagrant/utils/riak/app.config:/etc/riak/app.config
-        ports:
-          - 8087:8087
-          - 8098:8098
-    # preconfigure docker container
-    # add counters datatype etc.
-    - shell: timeout 60 docker exec vagrant_riak_1 riak-admin wait_for_service riak_kv
-    - shell: timeout 10 docker exec vagrant_riak_1 riak-admin bucket-type create counters '{"props":{"datatype":"counter"}}'
-      ignore_errors: yes
-    - shell: timeout 10 docker exec vagrant_riak_1 riak-admin bucket-type activate counters
 
     - include: tasks/worker_upstart.yaml

bootstrap/playbooks/tasks/docker.yaml

@@ -18,3 +18,10 @@
 
 # pre download riak image
 - shell: docker pull solarproject/riak
+
+# prebuild pg container
+- git: repo=https://github.com/kiasaki/docker-alpine-postgres.git dest=/tmp/docker-alpine-postgres update=yes
+- shell: make build
+  args:
+    chdir: /tmp/docker-alpine-postgres
+- shell: rm -fr /tmp/docker-alpine-postgres

utils/jenkins/run.sh

@@ -14,6 +14,8 @@ IMAGE_PATH=${IMAGE_PATH:-bootstrap/output-qemu/ubuntu1404}
 TEST_SCRIPT=${TEST_SCRIPT:-/vagrant/examples/hosts_file/hosts.py}
 DEPLOY_TIMEOUT=${DEPLOY_TIMEOUT:-60}
 
+SOLAR_DB_BACKEND=${SOLAR_DB_BACKEND:-riak}
+
 SSH_OPTIONS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
 
 dos.py erase ${ENV_NAME} || true
@@ -57,12 +59,20 @@ sudo rm -rf /vagrant
 sudo mv /home/vagrant/solar /vagrant
 
 sudo chown -R ${ADMIN_USER} ${INSTALL_DIR}
-sudo ansible-playbook -v -i \"localhost,\" -c local ${INSTALL_DIR}/bootstrap/playbooks/solar.yaml
+sudo SOLAR_DB_BACKEND=${SOLAR_DB_BACKEND} ansible-playbook -v -i \"localhost,\" -c local ${INSTALL_DIR}/bootstrap/playbooks/solar.yaml
 
 set -e
 
 # wait for riak
-sudo docker exec vagrant_riak_1 riak-admin wait_for_service riak_kv
+
+if [ $SOLAR_DB_BACKEND == "riak" ]
+then
+   sudo docker exec vagrant_riak_1 riak-admin wait_for_service riak_kv;
+elif [ $SOLAR_DB_BACKEND == "postgres" ]
+then
+   # TODO: Should be replaced with something smarter
+   sleep 5
+fi
 
 export SOLAR_CONFIG_OVERRIDE="/.solar_config_override"
 

vagrant-settings.yaml_defaults

@@ -4,9 +4,9 @@
 slaves_count: 2
 slaves_ram: 1024
 master_image: solar-project/solar-master
-master_image_version: <0.3.0
+master_image_version: ">= 0.3.0.pre0"
 slaves_image: solar-project/solar-master
-slaves_image_version: <0.3.0
+slaves_image_version: ">= 0.3.0.pre0"
 master_ram: 1024
 master_cpus: 1
 master_ips:
@@ -35,3 +35,8 @@ preprovisioned: true
 # it should be used in order to provision nodes
 # by solar
 # preprovisioned: false
+
+
+# needed for starting / preparing db server
+# you can choose from riak | postgresql
+solar_db_backend: riak