From 6cc4690ff22242a3c315da6cf7a005b26b9a1355 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Wed, 13 Jan 2016 11:33:02 +0200 Subject: [PATCH] Remove redis and celery container, use init script for celery - Removed all packages and container related to redis - docker-compose will start only riak container - add init script isntallation for solar-celery - cleanedup some of the commented code in bootstrap scripts - disable host key verification for vagrant user Change-Id: I9b10d07fe6ab527bc029518da407c5165f9b3d22 --- bootstrap/playbooks/celery.yaml | 23 --------------- bootstrap/playbooks/solar.yaml | 13 +++++---- bootstrap/playbooks/tasks/base.yaml | 22 -------------- bootstrap/playbooks/tasks/celery_init.yaml | 8 +++++ bootstrap/playbooks/tasks/ssh_conf.yaml | 2 ++ docker-compose.yml | 34 ---------------------- solar/config.py | 1 + solar/core/log.py | 3 +- utils/solar-celery | 2 -- 9 files changed, 21 insertions(+), 87 deletions(-) delete mode 100644 bootstrap/playbooks/celery.yaml create mode 100644 bootstrap/playbooks/tasks/celery_init.yaml diff --git a/bootstrap/playbooks/celery.yaml b/bootstrap/playbooks/celery.yaml deleted file mode 100644 index e6fa9d5c..00000000 --- a/bootstrap/playbooks/celery.yaml +++ /dev/null @@ -1,23 +0,0 @@ ---- - -- name: Solar Celery config - hosts: all - sudo: yes - vars: - celery_dir: /var/run/celery - environment: - SOLAR_CONFIG_OVERRIDE: /.solar_config_override - tasks: - - shell: mkdir -p {{ celery_dir }} - tags: [install] - - shell: pip install celery - tags: [install] - - shell: hostname - register: hostname - - shell: celery multi stopwait 2 -A solar.orchestration.runner - chdir={{ celery_dir }} - tags: [stop] - - shell: celery multi start 2 -A solar.orchestration.runner -P:2 prefork -c:1 1 -c:2 3 -Q:1 scheduler,system_log -Q:2 celery,{{ hostname.stdout }} - chdir={{ celery_dir }} - tags: [master] - diff --git a/bootstrap/playbooks/solar.yaml b/bootstrap/playbooks/solar.yaml index 2a695bc5..3b9cd90b 100644 --- a/bootstrap/playbooks/solar.yaml +++ b/bootstrap/playbooks/solar.yaml @@ -14,10 +14,13 @@ - hosts: all tasks: + # add application related directories + - file: path=/var/log/solar state=directory owner=vagrant group=vagrant + - file: path=/var/run/celery state=directory owner=vagrant group=vagrant # setup solar-resources # change to openstack/solar-resources later - git: repo=https://github.com/Mirantis/solar-resources.git dest=/vagrant/solar-resources update=no - + - shell: pip install gevent # set default config location - lineinfile: dest: /home/vagrant/.bashrc @@ -33,10 +36,9 @@ create: yes - lineinfile: dest: /.solar_config_override - line: "redis: redis://10.0.0.2:6379/1" + line: "log_file: /var/log/solar/solar.log" state: present create: yes - - lineinfile: dest: /home/vagrant/.bashrc line: export PYTHONWARNINGS="ignore" @@ -55,7 +57,8 @@ - file: src=/vagrant/solar-resources/resources dest=/var/lib/solar/repositories/resources state=link owner=vagrant - file: src=/vagrant/solar-resources/templates dest=/var/lib/solar/repositories/templates state=link owner=vagrant - - name: Starting docker containers - shell: docker-compose up -d chdir=/vagrant + - name: start riak container + shell: docker-compose up -d riak chdir=/vagrant - name: configuring Ansible copy: src=/vagrant/bootstrap/ansible.cfg dest=/home/vagrant/.ansible.cfg + - include: tasks/celery_init.yaml diff --git a/bootstrap/playbooks/tasks/base.yaml b/bootstrap/playbooks/tasks/base.yaml index e8d2b954..3b0226e7 100644 --- a/bootstrap/playbooks/tasks/base.yaml +++ b/bootstrap/playbooks/tasks/base.yaml @@ -41,36 +41,14 @@ # PIP -#- apt: name=python-pip state=absent -#- shell: easy_install pip -#- shell: pip install -U pip -#- shell: pip install -U setuptools - shell: sudo pip install httpie - shell: sudo pip install docker-py==1.1.0 # faster json - shell: pip install ujson -# Redis -- shell: sudo pip install redis - # fresh tox - shell: sudo pip install tox # install riak package - shell: sudo pip install riak - -# Ubuntu OpenStack packages -#- apt: name=ubuntu-cloud-keyring state=present -#- shell: echo "deb http://ubuntu-cloud.archive.canonical.com/ubuntu trusty-updates/kilo main" > /etc/apt/sources.list.d/cloudarchive-kilo.list -#- shell: echo "deb http://osci-mirror-poz.infra.mirantis.net/pkgs/ubuntu-2015-06-25-194717 trusty-updates main" > /etc/apt/sources.list.d/fuel-kilo.list -#- shell: echo "deb http://osci-mirror-poz.infra.mirantis.net/pkgs/ubuntu-latest trusty main" > /etc/apt/sources.list.d/fuel-kilo.list -# cloudarchive key -#- shell: apt-key adv --recv-key --keyserver keyserver.ubuntu.com 5EDB1B62EC4926EA -# some other keys -#- shell: apt-key adv --recv-key --keyserver keyserver.ubuntu.com 9D6D8F6BC857C906 -#- shell: apt-key adv --recv-key --keyserver keyserver.ubuntu.com 7638D0442B90D010 -# mirantis poznan -#- shell: apt-key adv --recv-key --keyserver keyserver.ubuntu.com 40976EAF437D05B5 -#- shell: apt-key adv --recv-key --keyserver keyserver.ubuntu.com 3B4FE6ACC0B21F32 -#- shell: apt-get update diff --git a/bootstrap/playbooks/tasks/celery_init.yaml b/bootstrap/playbooks/tasks/celery_init.yaml new file mode 100644 index 00000000..89f9ab84 --- /dev/null +++ b/bootstrap/playbooks/tasks/celery_init.yaml @@ -0,0 +1,8 @@ +--- + +- copy: src=/vagrant/utils/solar-celery dest=/etc/init.d/solar-celery +- shell: chmod +x /etc/init.d/solar-celery +- shell: chown vagrant:vagrant /etc/init.d/solar-celery +- shell: update-rc.d solar-celery defaults +- shell: update-rc.d solar-celery enable +- shell: su vagrant service solar-celery start diff --git a/bootstrap/playbooks/tasks/ssh_conf.yaml b/bootstrap/playbooks/tasks/ssh_conf.yaml index 6dd03225..4e6dfe58 100644 --- a/bootstrap/playbooks/tasks/ssh_conf.yaml +++ b/bootstrap/playbooks/tasks/ssh_conf.yaml @@ -2,3 +2,5 @@ - file: path=/root/.ssh state=directory mode=0700 - template: src=files/ssh_conf dest=/root/.ssh/config +- file: path=/home/vagrant/.ssh state=directory mode=0700 +- template: src=files/ssh_conf dest=/home/vagrant/.ssh/config diff --git a/docker-compose.yml b/docker-compose.yml index d01c8d6e..95983f4e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,31 +1,3 @@ -solar-celery: - image: solarproject/solar-celery - # path inside of the container should be exactly the same as outside - # because solar uses absolute path to find resoruce actions files - volumes: - - /vagrant/.vagrant:/vagrant/.vagrant - - /vagrant:/solar - - /vagrant/solar-resources:/vagrant/solar-resources - - /root/.ssh:/root/.ssh - - ./bootstrap/playbooks/celery.yaml:/celery.yaml - - /var/lib/solar/repositories:/var/lib/solar/repositories - - /.solar_config_override:/.solar_config_override - - ./run.sh:/run.sh - - ./.config:~/.config - environment: - - REDIS_HOST=redis - - REDIS_PORT=6379 - - RIAK_HOST=riak - - RIAK_PORT=8087 - # links are not used for configuration because we can rely on non-container - # based datastores - links: - - riak - - redis - # for torrent protocol - ports: - - 6881-6981:6881-6981 - riak: image: solarproject/riak volumes: @@ -33,9 +5,3 @@ riak: ports: - 8087:8087 - 8098:8098 -redis: - image: tutum/redis - ports: - - 6379:6379 - environment: - - REDIS_PASS=**None** diff --git a/solar/config.py b/solar/config.py index 988c8401..281521b8 100644 --- a/solar/config.py +++ b/solar/config.py @@ -28,6 +28,7 @@ C.celery_broker = 'sqla+sqlite:////tmp/celery.db' C.celery_backend = 'db+sqlite:////tmp/celery.db' C.riak_ensemble = False C.lock_bucket_type = None +C.log_file = 'solar.log' def _lookup_vals(setter, config, prefix=None): diff --git a/solar/core/log.py b/solar/core/log.py index c89ec2a9..f24caf5c 100644 --- a/solar/core/log.py +++ b/solar/core/log.py @@ -15,12 +15,13 @@ import logging import sys +from solar.config import C log = logging.getLogger('solar') def setup_logger(): - handler = logging.FileHandler('solar.log') + handler = logging.FileHandler(C.log_file) handler.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s %(levelname)s %(funcName)s' diff --git a/utils/solar-celery b/utils/solar-celery index f932ea2c..53f91865 100755 --- a/utils/solar-celery +++ b/utils/solar-celery @@ -9,7 +9,6 @@ # Default-Stop: 0 1 6 ### END INIT INFO -mkdir -p /var/run/celery export SOLAR_CONFIG_OVERRIDE=/.solar_config_override START="$(/usr/local/bin/celery multi start 1 -A solar.orchestration.runner -P gevent -c 1000 -Q system_log,celery,scheduler --pidfile=/var/run/celery/%N.pid --logfile=/var/run/celery/%N.log)" STOP="$(/usr/local/bin/celery multi stopwait 1 --pidfile=/var/run/celery/%N.pid)" @@ -23,7 +22,6 @@ case "$1" in stop) "{$STOP}" ;; - restart|force-reload) ${0} stop ${0} start