Remove redis and celery container, use init script for celery

- Removed all packages and container related to redis
- docker-compose will start only riak container
- add init script isntallation for solar-celery
- cleanedup some of the commented code in bootstrap scripts
- disable host key verification for vagrant user

Change-Id: I9b10d07fe6ab527bc029518da407c5165f9b3d22

bootstrap/playbooks/celery.yaml

@@ -1,23 +0,0 @@
----
-
-- name: Solar Celery config
-  hosts: all
-  sudo: yes
-  vars:
-    celery_dir: /var/run/celery
-  environment:
-    SOLAR_CONFIG_OVERRIDE: /.solar_config_override
-  tasks:
-    - shell: mkdir -p {{ celery_dir }}
-      tags: [install]
-    - shell: pip install celery
-      tags: [install]
-    - shell: hostname
-      register: hostname
-    - shell: celery multi stopwait 2  -A solar.orchestration.runner
-             chdir={{ celery_dir }}
-      tags: [stop]
-    - shell: celery multi start 2 -A solar.orchestration.runner -P:2 prefork -c:1 1 -c:2 3 -Q:1 scheduler,system_log -Q:2 celery,{{ hostname.stdout }}
-             chdir={{ celery_dir }}
-      tags: [master]
-

bootstrap/playbooks/solar.yaml

@@ -14,10 +14,13 @@
 
 - hosts: all
   tasks:
+    # add application related directories
+    - file: path=/var/log/solar state=directory owner=vagrant group=vagrant
+    - file: path=/var/run/celery state=directory owner=vagrant group=vagrant
     #  setup solar-resources
     # change to openstack/solar-resources later
     - git: repo=https://github.com/Mirantis/solar-resources.git dest=/vagrant/solar-resources update=no
-
+    - shell: pip install gevent
     # set default config location
     - lineinfile:
         dest: /home/vagrant/.bashrc
@@ -33,10 +36,9 @@
         create: yes
     - lineinfile:
         dest: /.solar_config_override
-        line: "redis: redis://10.0.0.2:6379/1"
+        line: "log_file: /var/log/solar/solar.log"
         state: present
         create: yes
-
     - lineinfile:
         dest: /home/vagrant/.bashrc
         line: export PYTHONWARNINGS="ignore"
@@ -55,7 +57,8 @@
     - file: src=/vagrant/solar-resources/resources dest=/var/lib/solar/repositories/resources state=link owner=vagrant
     - file: src=/vagrant/solar-resources/templates dest=/var/lib/solar/repositories/templates state=link owner=vagrant
 
-    - name: Starting docker containers
-      shell: docker-compose up -d chdir=/vagrant
+    - name: start riak container
+      shell: docker-compose up -d riak chdir=/vagrant
     - name: configuring Ansible
       copy: src=/vagrant/bootstrap/ansible.cfg dest=/home/vagrant/.ansible.cfg
+    - include: tasks/celery_init.yaml

bootstrap/playbooks/tasks/base.yaml

@@ -41,36 +41,14 @@
 
 
 # PIP
-#- apt: name=python-pip state=absent
-#- shell: easy_install pip
-#- shell: pip install -U pip
-#- shell: pip install -U setuptools
 - shell: sudo pip install httpie
 - shell: sudo pip install docker-py==1.1.0
 
 # faster json
 - shell: pip install ujson
 
-# Redis
-- shell: sudo pip install redis
-
 # fresh tox
 - shell: sudo pip install tox
 
 # install riak package
 - shell: sudo pip install riak
-
-# Ubuntu OpenStack packages
-#- apt: name=ubuntu-cloud-keyring state=present
-#- shell: echo "deb http://ubuntu-cloud.archive.canonical.com/ubuntu trusty-updates/kilo main" > /etc/apt/sources.list.d/cloudarchive-kilo.list
-#- shell: echo "deb http://osci-mirror-poz.infra.mirantis.net/pkgs/ubuntu-2015-06-25-194717 trusty-updates main" > /etc/apt/sources.list.d/fuel-kilo.list
-#- shell: echo "deb http://osci-mirror-poz.infra.mirantis.net/pkgs/ubuntu-latest trusty main" > /etc/apt/sources.list.d/fuel-kilo.list
-# cloudarchive key
-#- shell: apt-key adv --recv-key --keyserver keyserver.ubuntu.com 5EDB1B62EC4926EA
-# some other keys
-#- shell: apt-key adv --recv-key --keyserver keyserver.ubuntu.com 9D6D8F6BC857C906
-#- shell: apt-key adv --recv-key --keyserver keyserver.ubuntu.com 7638D0442B90D010
-# mirantis poznan
-#- shell: apt-key adv --recv-key --keyserver keyserver.ubuntu.com 40976EAF437D05B5
-#- shell: apt-key adv --recv-key --keyserver keyserver.ubuntu.com 3B4FE6ACC0B21F32
-#- shell: apt-get update

bootstrap/playbooks/tasks/celery_init.yaml

@@ -0,0 +1,8 @@
+---
+
+- copy: src=/vagrant/utils/solar-celery dest=/etc/init.d/solar-celery
+- shell: chmod +x /etc/init.d/solar-celery
+- shell: chown vagrant:vagrant /etc/init.d/solar-celery
+- shell: update-rc.d solar-celery defaults
+- shell: update-rc.d solar-celery enable
+- shell: su vagrant service solar-celery start

bootstrap/playbooks/tasks/ssh_conf.yaml

@@ -2,3 +2,5 @@
 
 - file: path=/root/.ssh state=directory mode=0700
 - template: src=files/ssh_conf dest=/root/.ssh/config
+- file: path=/home/vagrant/.ssh state=directory mode=0700
+- template: src=files/ssh_conf dest=/home/vagrant/.ssh/config

docker-compose.yml

@@ -1,31 +1,3 @@
-solar-celery:
-  image: solarproject/solar-celery
-  # path inside of the container should be exactly the same as outside
-  # because solar uses absolute path to find resoruce actions files
-  volumes:
-    - /vagrant/.vagrant:/vagrant/.vagrant
-    - /vagrant:/solar
-    - /vagrant/solar-resources:/vagrant/solar-resources
-    - /root/.ssh:/root/.ssh
-    - ./bootstrap/playbooks/celery.yaml:/celery.yaml
-    - /var/lib/solar/repositories:/var/lib/solar/repositories
-    - /.solar_config_override:/.solar_config_override
-    - ./run.sh:/run.sh
-    - ./.config:~/.config
-  environment:
-    - REDIS_HOST=redis
-    - REDIS_PORT=6379
-    - RIAK_HOST=riak
-    - RIAK_PORT=8087
-  # links are not used for configuration because we can rely on non-container
-  # based datastores
-  links:
-    - riak
-    - redis
-  # for torrent protocol
-  ports:
-    - 6881-6981:6881-6981
-
 riak:
   image: solarproject/riak
   volumes:
@@ -33,9 +5,3 @@ riak:
   ports:
     - 8087:8087
     - 8098:8098
-redis:
-  image: tutum/redis
-  ports:
-    - 6379:6379
-  environment:
-    - REDIS_PASS=**None**

solar/config.py

@@ -28,6 +28,7 @@ C.celery_broker = 'sqla+sqlite:////tmp/celery.db'
 C.celery_backend = 'db+sqlite:////tmp/celery.db'
 C.riak_ensemble = False
 C.lock_bucket_type = None
+C.log_file = 'solar.log'
 
 
 def _lookup_vals(setter, config, prefix=None):

solar/core/log.py

@@ -15,12 +15,13 @@
 import logging
 import sys
 
+from solar.config import C
 
 log = logging.getLogger('solar')
 
 
 def setup_logger():
-    handler = logging.FileHandler('solar.log')
+    handler = logging.FileHandler(C.log_file)
     handler.setLevel(logging.DEBUG)
     formatter = logging.Formatter(
         '%(asctime)s %(levelname)s %(funcName)s'

utils/solar-celery

@@ -9,7 +9,6 @@
 # Default-Stop:     0 1 6
 ### END INIT INFO
 
-mkdir -p /var/run/celery
 export SOLAR_CONFIG_OVERRIDE=/.solar_config_override
 START="$(/usr/local/bin/celery multi start 1 -A solar.orchestration.runner -P gevent -c 1000 -Q system_log,celery,scheduler --pidfile=/var/run/celery/%N.pid --logfile=/var/run/celery/%N.log)"
 STOP="$(/usr/local/bin/celery multi stopwait 1 --pidfile=/var/run/celery/%N.pid)"
@@ -23,7 +22,6 @@ case "$1" in
   stop)
     "{$STOP}"
     ;;
-
   restart|force-reload)
     ${0} stop
     ${0} start