Role cleanup/changes for newton OSA

- The database create and user creates have been removed from the
  role. These tasks have been relocated to the playbook.

- Tags cleanup.

- The "Attempt venv download" task now uses the "checksum" parameter
  from the get_url module to check if it is necessary to download
  the venv.

- The playbook and its variables (on extras folder) have been updated
  to reflect these chages.

Change-Id: If15f32d51a4f0265322abbcf25d96ba5c77fe1a6

extras/env.d/sahara.yml

@@ -22,12 +22,17 @@ component_skel:
 container_skel:
   sahara_container:
     belongs_to:
-      - infra_containers
-      - os-infra_containers
+      - sahara-infra_containers
     contains:
       - sahara_api
       - sahara_engine
     properties:
       service_name: sahara
-      container_release: trusty
-      container_fs_size: 12G
+
+physical_skel:
+  sahara-infra_containers:
+    belongs_to:
+      - all_containers
+  sahara-infra_hosts:
+    belongs_to:
+      - hosts

extras/group_vars_sahara.yml

@@ -1,13 +1,20 @@
-## Sahara
-sahara_service_port: 8386
-sahara_service_proto: http
-sahara_service_adminuri_proto: "{{ openstack_service_adminuri_proto | default(sahara_service_proto) }}"
-sahara_service_user_name: sahara
-sahara_service_project_name: service
-sahara_service_project_domain_id: default
-sahara_service_user_domain_id: default
-sahara_service_adminuri: "{{ sahara_service_adminuri_proto }}://{{ internal_lb_vip_address }}:{{ sahara_service_port }}"
-sahara_service_adminurl: "{{ sahara_service_adminuri }}/v1.1/%(tenant_id)s"
+---
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 sahara_service_region: "{{ service_region }}"
-sahara_rabbitmq_userid: sahara
-sahara_rabbitmq_vhost: /sahara
+sahara_service_in_ldap: "{{ service_ldap_backend_enabled }}"
+
+sahara_galera_address: "{{ galera_address }}"
+
+# If there are any Ceilometer hosts in the environment, then enable its usage
+sahara_ceilometer_enabled: "{{ (groups['ceilometer_all'] is defined) and (groups['ceilometer_all'] | length > 0) }}"

extras/haproxy_sahara.yml

@@ -2,9 +2,9 @@ haproxy_service_configs:
   - service:
       haproxy_service_name: sahara_api
       haproxy_backend_nodes: "{{ groups['sahara_api'] | default([]) }}"
+      haproxy_ssl: "{{ haproxy_ssl }}"
+      haproxy_balance_alg: source
       haproxy_port: 8386
       haproxy_balance_type: http
       haproxy_backend_options:
-        - "forwardfor"
-        - "httpchk /versions"
-        - "httplog"
+        - "httpchk /healthcheck"

extras/os-sahara-install.yml

@@ -13,96 +13,39 @@
 
 - name: Install sahara server
   hosts: sahara_all
+  gather_facts: "{{ gather_facts | default(True) }}"
   max_fail_percentage: 20
   user: root
   pre_tasks:
-    - name: Use the lxc-openstack aa profile
-      lxc_container:
-        name: "{{ container_name }}"
-        container_config:
-          - "lxc.aa_profile=lxc-openstack"
-      delegate_to: "{{ physical_host }}"
-      when: not is_metal | bool
-      register: container_config
-      tags:
-        - lxc-aa-profile
-    - name: Wait for container ssh
-      wait_for:
-        port: "22"
-        delay: "{{ ssh_delay }}"
-        search_regex: "OpenSSH"
-        host: "{{ ansible_ssh_host }}"
-      delegate_to: "{{ physical_host }}"
-      when: >
-            (container_config is defined and container_config | changed) or
-            (container_extra_config is defined and container_config | changed)
-      register: ssh_wait_check
-      until: ssh_wait_check | success
-      retries: 3
-      tags:
-        - ssh-wait
-    - name: Sort the rabbitmq servers
-      dist_sort:
-        value_to_lookup: "{{ container_name }}"
-        ref_list: "{{ groups['sahara_all'] }}"
-        src_list: "{{ rabbitmq_servers }}"
-      register: servers
-    - name: Set rabbitmq servers
-      set_fact:
-        rabbitmq_servers: "{{ servers.sorted_list }}"
-    - name: Create log dir
-      file:
-        path: "{{ item.path }}"
-        state: directory
-      with_items:
-        - { path: "/openstack/log/{{ inventory_hostname }}-sahara" }
-      when: is_metal | bool
-      tags:
-        - sahara-logs
-        - sahara-log-dirs
-    - name: Create log aggregation links
-      file:
-        src: "{{ item.src }}"
-        dest: "{{ item.dest }}"
-        state: "{{ item.state }}"
-        force: "yes"
-      with_items:
-        - { src: "/openstack/log/{{ inventory_hostname }}-sahara", dest: "/var/log/sahara", state: "link" }
-      when: is_metal | bool
-      tags:
-        - sahara-logs
-    - name: Ensure Rabbitmq vhost
-      rabbitmq_vhost:
-        name: "{{ sahara_rabbitmq_vhost }}"
-        state: "present"
-      delegate_to: "{{ groups['rabbitmq_all'][0] }}"
-      when:
-        - inventory_hostname == groups['sahara_all'][0]
-        - groups['rabbitmq_all']|length > 0
-      tags:
-        - sahara-rabbitmq
-        - sahara-rabbitmq-vhost
-    - name: Ensure rabbitmq user
-      rabbitmq_user:
+    - include: common-tasks/rabbitmq-servers-sort.yml
+      vars:
+        sort_group_name: "sahara_all"
+    - include: common-tasks/os-lxc-container-setup.yml
+    - include: common-tasks/rabbitmq-vhost-user.yml
+      vars:
         user: "{{ sahara_rabbitmq_userid }}"
         password: "{{ sahara_rabbitmq_password }}"
         vhost: "{{ sahara_rabbitmq_vhost }}"
-        configure_priv: ".*"
-        read_priv: ".*"
-        write_priv: ".*"
-        state: "present"
-      delegate_to: "{{ groups['rabbitmq_all'][0] }}"
       when:
         - inventory_hostname == groups['sahara_all'][0]
-        - groups['rabbitmq_all']|length > 0
-      tags:
-        - sahara-rabbitmq
-        - sahara-rabbitmq-user
+        - groups['rabbitmq_all'] | length > 0
+    - include: common-tasks/os-log-dir-setup.yml
+      vars:
+        log_dirs:
+          - src: "/openstack/log/{{ inventory_hostname }}-sahara"
+            dest: "/var/log/sahara"
+    - include: common-tasks/mysql-db-user.yml
+      vars:
+        user_name: "{{ sahara_galera_user }}"
+        password: "{{ sahara_container_mysql_password }}"
+        login_host: "{{ sahara_galera_address }}"
+        db_name: "{{ sahara_galera_database }}"
+      when: inventory_hostname == groups['sahara_all'][0]
+    - include: common-tasks/package-cache-proxy.yml
   roles:
     - role: "os_sahara"
-      sahara_galera_address: "{{ galera_address }}"
       sahara_venv_tag: "{{ openstack_release }}"
-      sahara_venv_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}/sahara-{{ openstack_release }}.tgz"
+      sahara_venv_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}/sahara-{{ openstack_release }}-{{ ansible_architecture | lower }}.tgz"
       tags:
         - "os-sahara"
     - { role: "openstack_openrc", tags: [ "openstack-openrc" ] }
@@ -117,10 +60,5 @@
       tags:
         - "system-crontab-coordination"
   vars:
-    galera_address: "{{ internal_lb_vip_address }}"
     is_metal: "{{ properties.is_metal|default(false) }}"
-    sahara_rabbitmq_userid: sahara
-    sahara_rabbitmq_vhost: /sahara
-    sahara_rabbitmq_servers: "{{ rabbitmq_servers }}"
-    sahara_rabbitmq_port: "{{ rabbitmq_port }}"
-    sahara_rabbitmq_use_ssl: "{{ rabbitmq_use_ssl }}"
+    pip_lock_to_internal_repo: "{{ (pip_links | length) >= 1 }}"

extras/repo-sahara.yml

@@ -2,3 +2,4 @@
 sahara_git_repo: https://git.openstack.org/openstack/sahara
 sahara_git_install_branch: master
 sahara_git_dest: "/opt/sahara_{{ sahara_git_install_branch | replace('/', '_') }}"
+sahara_git_project_group: sahara_all

tasks/main.yml

@@ -33,17 +33,33 @@
     - always
 
 - include: sahara_pre_install.yml
+  tags:
+    - sahara-install
+
 - include: sahara_install.yml
+  tags:
+    - sahara-install
+
 - include: sahara_post_install.yml
+  tags:
+    - sahara-install
+    - sahara-config
+
 - include: sahara_init.yml
+  tags:
+    - sahara-install
 
 - include: sahara_db_setup.yml
   when: >
     inventory_hostname == groups['sahara_all'][0]
+  tags:
+    - sahara-install
 
 - include: sahara_service_setup.yml
   when: >
     inventory_hostname == groups['sahara_all'][0]
+  tags:
+    - sahara-install
 
 - name: Flush handlers
   meta: flush_handlers

tasks/sahara_db_setup.yml

@@ -11,38 +11,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Create DB for service
-  mysql_db:
-    login_user: "{{ galera_root_user }}"
-    login_password: "{{ galera_root_password }}"
-    login_host: "{{ sahara_galera_address }}"
-    name: "{{ sahara_galera_database }}"
-    state: "present"
-  tags:
-    - sahara-db-setup
-
-- name: Grant access to the DB for the service
-  mysql_user:
-    login_user: "{{ galera_root_user }}"
-    login_password: "{{ galera_root_password }}"
-    login_host: "{{ sahara_galera_address }}"
-    name: "{{ sahara_galera_user }}"
-    password: "{{ sahara_container_mysql_password }}"
-    host: "{{ item }}"
-    state: "present"
-    priv: "{{ sahara_galera_database }}.*:ALL"
-  with_items:
-    - "localhost"
-    - "%"
-  tags:
-    - sahara-db-setup
-
 - name: Perform a Sahara DB sync
   command: "{{ sahara_bin }}/sahara-db-manage
               --config-file {{ sahara_db_config }} upgrade head"
   become: yes
   become_user: "{{ sahara_system_user_name }}"
-  tags:
-    - sahara-db-sync
-    - sahara-setup
-    - sahara-command-bin

tasks/sahara_init_common.yml

@@ -13,13 +13,9 @@
 
 - include: sahara_init_upstart.yml
   when: pid1_name == "init"
-  tags:
-    - sahara-init
 
 - include: sahara_init_systemd.yml
   when: pid1_name == "systemd"
-  tags:
-    - sahara-init
 
 - name: Load service
   service:

tasks/sahara_install.yml

@@ -14,9 +14,6 @@
 - include: sahara_install_apt.yml
   when:
     - ansible_pkg_mgr == 'apt'
-  tags:
-    - sahara-install
-    - sahara-apt-packages
 
 - name: Create developer mode constraint file
   copy:
@@ -27,9 +24,6 @@
       {% endfor %}
   when:
     - sahara_developer_mode | bool
-  tags:
-    - sahara-install
-    - sahara-pip-packages
 
 - name: Clone requirements git repository
   git:
@@ -40,27 +34,18 @@
     version: "{{ sahara_requirements_git_install_branch }}"
   when:
     - sahara_developer_mode | bool
-  tags:
-    - sahara-install
-    - sahara-pip-packages
 
 - name: Add constraints to pip_install_options fact for developer mode
   set_fact:
     pip_install_options_fact: "{{ pip_install_options|default('') }} --constraint /opt/developer-pip-constraints.txt --constraint /opt/requirements/upper-constraints.txt"
   when:
     - sahara_developer_mode | bool
-  tags:
-    - sahara-install
-    - sahara-pip-packages
 
 - name: Set pip_install_options_fact when not in developer mode
   set_fact:
     pip_install_options_fact: "{{ pip_install_options|default('') }}"
   when:
     - not sahara_developer_mode | bool
-  tags:
-    - sahara-install
-    - sahara-pip-packages
 
 - name: Install required pip packages
   pip:
@@ -71,20 +56,6 @@
   until: install_packages|success
   retries: 5
   delay: 2
-  tags:
-    - sahara-install
-    - sahara-pip-packages
-
-- name: Get local venv checksum
-  stat:
-    path: "/var/cache/{{ sahara_venv_download_url | basename }}"
-    get_md5: False
-  when:
-    - not sahara_developer_mode | bool
-  register: local_venv_stat
-  tags:
-    - sahara-install
-    - sahara-pip-packages
 
 - name: Get remote venv checksum
   uri:
@@ -93,36 +64,20 @@
   when:
     - not sahara_developer_mode | bool
   register: remote_venv_checksum
-  tags:
-    - sahara-install
-    - sahara-pip-packages
 
-# TODO: When project moves to ansible 2 we can pass this a sha256sum which will:
-#       a) allow us to remove force: yes
-#       b) allow the module to calculate the checksum of dest file which would
-#          result in file being downloaded only if provided and dest sha256sum
-#          checksums differ
 - name: Attempt venv download
   get_url:
     url: "{{ sahara_venv_download_url }}"
     dest: "/var/cache/{{ sahara_venv_download_url | basename }}"
-    force: yes
+    checksum: "sha1:{{ remote_venv_checksum.content }}"
   ignore_errors: true
   register: get_venv
   when:
     - not sahara_developer_mode | bool
-    - (local_venv_stat.stat.exists == False or
-        {{ local_venv_stat.stat.checksum is defined and local_venv_stat.stat.checksum != remote_venv_checksum.content | trim }})
-  tags:
-    - sahara-install
-    - sahara-pip-packages
 
 - name: Set sahara get_venv fact
   set_fact:
     sahara_get_venv: "{{ get_venv }}"
-  tags:
-    - sahara-install
-    - sahara-pip-packages
 
 - name: Remove existing venv
   file:
@@ -130,18 +85,12 @@
     state: absent
   when:
     - sahara_get_venv | changed
-  tags:
-    - sahara-install
-    - sahara-pip-packages
 
 - name: Create sahara venv dir
   file:
     path: "{{ sahara_bin | dirname }}"
     state: directory
   register: sahara_venv_dir
-  tags:
-    - sahara-install
-    - sahara-pip-packages
 
 - name: Unarchive pre-built venv
   unarchive:
@@ -153,14 +102,11 @@
     - sahara_get_venv | changed or sahara_venv_dir | changed
   notify:
     - Restart sahara services
-  tags:
-    - sahara-install
-    - sahara-pip-packages
 
 - name: Install pip packages (venv)
   pip:
     name: "{{ sahara_pip_packages | join(' ') }}"
-    state: present
+    state: latest
     virtualenv: "{{ sahara_bin | dirname }}"
     virtualenv_site_packages: "no"
     extra_args: "{{ pip_install_options_fact }}"
@@ -172,9 +118,6 @@
     - sahara_get_venv | failed or sahara_developer_mode | bool
   notify:
     - Restart sahara services
-  tags:
-    - sahara-install
-    - sahara-pip-packages
 
 - name: Update virtualenv path
   command: >
@@ -182,7 +125,3 @@
   when:
     - not sahara_developer_mode | bool
     - sahara_get_venv | success
-  tags:
-    - sahara-install
-    - sahara-pip-packages
-

tasks/sahara_post_install.yml

@@ -36,6 +36,3 @@
   notify:
     - Restart sahara api
     - Restart sahara engine
-  tags:
-    - sahara-config
-

tasks/sahara_pre_install.yml

@@ -16,8 +16,6 @@
     name: "{{ sahara_system_group_name }}"
     state: "present"
     system: "yes"
-  tags:
-    - sahara-group
 
 - name: Create the sahara system user
   user:
@@ -28,8 +26,6 @@
     system: "yes"
     createhome: "yes"
     home: "{{ sahara_system_user_home }}"
-  tags:
-    - sahara-user
 
 - name: Create sahara dir
   file:
@@ -43,8 +39,6 @@
     - { path: "/etc/sahara" }
     - { path: "/var/cache/sahara" }
     - { path: "{{ sahara_system_user_home }}" }
-  tags:
-    - sahara-dirs
 
 - name: Test for log directory or link
   shell: |
@@ -57,9 +51,6 @@
   register: log_dir
   failed_when: false
   changed_when: log_dir.rc != 0
-  tags:
-    - sahara-dirs
-    - sahara-logs
 
 - name: Create sahara log dir
   file:
@@ -69,6 +60,3 @@
     group: "{{ sahara_system_group_name }}"
     mode: "0755"
   when: log_dir.rc != 0
-  tags:
-    - sahara-dirs
-    - sahara-logs

tasks/sahara_service_setup.yml

@@ -27,10 +27,6 @@
   until: add_service|success
   retries: 5
   delay: 2
-  tags:
-    - sahara-api-setup
-    - sahara-service-add
-    - sahara-setup
 
 # Create an admin user
 - name: Ensure sahara user
@@ -49,10 +45,6 @@
   until: add_service|success
   retries: 5
   delay: 10
-  tags:
-    - sahara-api-setup
-    - sahara-service-add
-    - sahara-setup
 
 # Add a role to the user
 - name: Ensure sahara user to admin role
@@ -71,10 +63,6 @@
   until: add_service|success
   retries: 5
   delay: 10
-  tags:
-    - sahara-api-setup
-    - sahara-service-add
-    - sahara-setup
 
 # Create an endpoint
 - name: Ensure sahara endpoint
@@ -99,10 +87,6 @@
   until: add_service|success
   retries: 5
   delay: 10
-  tags:
-    - sahara-api-setup
-    - sahara-service-add
-    - sahara-setup
 
 # Create proxy domain
 - name: Ensure sahara_proxy domain
@@ -119,6 +103,3 @@
   retries: 5
   delay: 10
   when: sahara_use_domain_for_proxy_users|bool
-  tags:
-    - sahara-proxy-domain-add
-    - sahara-setup