Merge "Implement container bind mount for all logs"

playbooks/common-tasks/os-lxc-container-setup.yml

@@ -27,39 +27,97 @@
 #  If extra container configurations are desirable set the
 #  "extra_container_config" list to strings containing the options needed.
 
+- name: Set default bind mounts
+  set_fact:
+    lxc_default_bind_mounts:
+      - bind_dir_path: "/var/log"
+        mount_path: "/openstack/log/{{ inventory_hostname }}"
+  when: lxc_default_bind_mounts is undefined
+
 - name: Ensure mount directories exists
   file:
     path: "{{ item['mount_path'] }}"
     state: "directory"
-  with_items: "{{ list_of_bind_mounts | default([]) }}"
+  with_items:
+    - "{{ list_of_bind_mounts | default([]) }}"
+    - "{{ lxc_default_bind_mounts }}"
   delegate_to: "{{ physical_host }}"
   when:
     - list_of_bind_mounts is defined
     - not is_metal | bool
 
-- name: LXC Directory bind mount
+- name: LXC bind mount directories
   lxc_container:
     name: "{{ inventory_hostname }}"
     container_command: |
       [[ ! -d "{{ item['bind_dir_path'] }}" ]] && mkdir -p "{{ item['bind_dir_path'] }}"
-    container_config:
+  with_items:
-      - "lxc.mount.entry={{ item['mount_path'] }} {{ item['bind_dir_path'].lstrip('/') }} none bind 0 0"
+    - "{{ list_of_bind_mounts | default([]) }}"
-  with_items: "{{ list_of_bind_mounts | default([]) }}"
+    - "{{ lxc_default_bind_mounts }}"
   delegate_to: "{{ physical_host }}"
   register: _bm
   when:
     - list_of_bind_mounts is defined
     - not is_metal | bool
 
-- name: Extra lxc config
+- name: Add bind mount configuration to container
-  lxc_container:
+  lineinfile:
-    name: "{{ inventory_hostname }}"
+    dest: "/var/lib/lxc/{{ inventory_hostname }}/config"
-    container_config: "{{ extra_container_config }}"
+    line: "lxc.mount.entry = {{ item['mount_path'] }} {{ item['bind_dir_path'].lstrip('/') }} none bind 0 0"
+    backup: "true"
+  with_items:
+    - "{{ list_of_bind_mounts | default([]) }}"
+    - "{{ lxc_default_bind_mounts }}"
   delegate_to: "{{ physical_host }}"
+  when:
+    - list_of_bind_mounts is defined
+    - not is_metal | bool
+  register: _mc
+
+- name: Extra lxc config
+  lineinfile:
+    dest: "/var/lib/lxc/{{ inventory_hostname }}/config"
+    line: "{{ item.split('=')[0] }} = {{ item.split('=', 1)[1] }}"
+    insertafter: "^{{ item.split('=')[0] }}"
+    backup: "true"
+  with_items: "{{ extra_container_config }}"
+  delegate_to: "{{ physical_host }}"
+  register: _ec
   when:
     - extra_container_config is defined
     - not is_metal | bool
-  register: _ec
+
+# Due to https://github.com/ansible/ansible-modules-extras/issues/2691
+# this uses the LXC CLI tools to ensure that we get logging.
+# TODO(odyssey4me): revisit this once the bug is fixed and released
+- name: Lxc container restart
+  command: >
+    lxc-stop --name {{ inventory_hostname }}
+    --logfile {{ lxc_container_log_path }}/lxc-{{ inventory_hostname }}.log
+    --logpriority {{ (debug | bool) | ternary('DEBUG', 'INFO') }}
+  delegate_to: "{{ physical_host }}"
+  register: container_stop
+  until: container_stop | success
+  retries: 3
+  when:
+    - not is_metal | bool
+    - (_mc is defined and _mc | changed) or (_ec is defined and _ec | changed)
+
+# Due to https://github.com/ansible/ansible-modules-extras/issues/2691
+# this uses the LXC CLI tools to ensure that we get logging.
+# TODO(odyssey4me): revisit this once the bug is fixed and released
+- name: Start Container
+  command: >
+    lxc-start --daemon --name {{ inventory_hostname }}
+    --logfile {{ lxc_container_log_path }}/lxc-{{ inventory_hostname }}.log
+    --logpriority {{ (debug | bool) | ternary('DEBUG', 'INFO') }}
+  delegate_to: "{{ physical_host }}"
+  register: container_start
+  until: container_start | success
+  retries: 3
+  when:
+    - not is_metal | bool
+    - (_mc is defined and _mc | changed) or (_ec is defined and _ec | changed)
 
 - name: Wait for container ssh
   wait_for:
@@ -68,11 +126,9 @@
     search_regex: "OpenSSH"
     host: "{{ ansible_ssh_host }}"
   delegate_to: "{{ physical_host }}"
-  when:
-    - >
-      (_bm is defined and _bm | changed) or
-      (_ec is defined and _ec | changed)
-    - not is_metal | bool
   register: ssh_wait_check
   until: ssh_wait_check | success
   retries: 3
+  when:
+    - (_bm is defined and _bm | changed) or (_ec is defined and _ec | changed)
+    - not is_metal | bool

playbooks/galera-install.yml

@@ -19,6 +19,11 @@
   gather_facts: "{{ gather_facts | default(True) }}"
   user: root
   tasks:
+    - include: common-tasks/os-log-dir-setup.yml
+      vars:
+        log_dirs:
+          - src: "/openstack/log/{{ inventory_hostname }}-mysql_logs"
+            dest: "/var/log/mysql_logs"
     - include: common-tasks/os-lxc-container-setup.yml
       vars:
         list_of_bind_mounts:

playbooks/inventory/group_vars/repo_all.yml

@@ -17,11 +17,6 @@
 repo_server_package_state: "{{ package_state }}"
 repo_build_package_state: "{{ package_state }}"
 
-# The default bind mount to hold the repo data
-repo_all_lxc_container_bind_mounts:
-  - mount_path: "/openstack/{{ inventory_hostname }}"
-    bind_dir_path: "/var/www"
-
 # Optionally set this variable to the location on the deployment
 # host where a set of git clones may be sourced to stage the repo
 # server.

playbooks/repo-server.yml

@@ -29,7 +29,9 @@
 
     - include: common-tasks/os-lxc-container-setup.yml
       vars:
-        list_of_bind_mounts: "{{ repo_all_lxc_container_bind_mounts }}"
+        list_of_bind_mounts:
+          - mount_path: "/openstack/{{ inventory_hostname }}"
+            bind_dir_path: "/var/www"
       when: repo_build_git_cache is not defined or not _local_git_cache.stat.exists
 
     - include: common-tasks/os-lxc-container-setup.yml
@@ -37,11 +39,14 @@
         repo_build_git_cache_bind_mount:
           - mount_path: "{{ repo_build_git_cache }}"
             bind_dir_path: "{{ repo_build_git_cache }}"
-        list_of_bind_mounts: "{{ repo_all_lxc_container_bind_mounts + repo_build_git_cache_bind_mount }}"
+        list_of_bind_mounts:
+          - mount_path: "/openstack/{{ inventory_hostname }}"
+            bind_dir_path: "/var/www"
+          - mount_path: "{{ repo_build_git_cache }}"
+            bind_dir_path: "{{ repo_build_git_cache }}"
       when:
         - repo_build_git_cache is defined
         - _local_git_cache.stat.exists
-
   roles:
     - { role: "repo_server", tags: [ "repo-server" ] }
     - role: "rsyslog_client"

playbooks/rsyslog-install.yml

@@ -22,7 +22,7 @@
     - include: common-tasks/os-lxc-container-setup.yml
       vars:
         list_of_bind_mounts:
-          - bind_dir_path: "{{ storage_directory }}"
+          - bind_dir_path: "{{ rsyslog_server_storage_directory }}"
             mount_path: "/openstack/{{ inventory_hostname }}/log-storage"
     - include: common-tasks/package-cache-proxy.yml
   roles:
@@ -31,5 +31,4 @@
       tags:
         - "system-crontab-coordination"
   vars:
-    storage_directory: "{{ rsyslog_server_storage_directory }}"
     is_metal: "{{ properties.is_metal|default(false) }}"

releasenotes/notes/bindmount-logs-3c23aab5b5ed3440.yaml

@@ -0,0 +1,25 @@
+---
+features:
+  - Containers will now bind mount all logs to the physical host
+    machine in the "/openstack/log/{{ inventory_hostname }}"
+    location. This change will ensure containers using a block
+    backed file system (lvm, zfs, bfrfs) do not run into issues
+    with full file systems due to logging.
+upgrade:
+  - When upgrading deployers will need to ensure they have a
+    backup of all logging from within the container prior to
+    running the playbooks. If the logging node is present within
+    the deployment all logs should already be sync'd with the
+    logging server and no action is required. As a pre-step it's
+    recommended that deployers clean up logging directories from
+    within containers prior to running the playbooks. After the
+    playbooks have run the bind mount will be in effect at
+    "/var/log" which will mount over all previous log files and
+    directories.
+  - Due to a new bind mount at "/var/log" all containers will be
+    restarted. This is a required restart. It is recommended that
+    deployers run the container restarts in serial to not impact
+    production workloads.
+fixes:
+  - Logging within the container has been bind mounted to the hosts
+    this reslves issue `1588051 <https://bugs.launchpad.net/openstack-ansible/+bug/1588051>_`

scripts/gate-check-commit.sh

@@ -79,9 +79,6 @@ popd
 # Implement the log directory
 mkdir -p /openstack/log
 
-# Implement the log directory link for openstack-infra log publishing
-ln -sf /openstack/log "$(dirname "${0}")/../logs"
-
 pushd "$(dirname "${0}")/../playbooks"
   # Disable Ansible color output
   export ANSIBLE_NOCOLOR=1

scripts/scripts-library.sh

@@ -132,7 +132,15 @@ function exit_fail {
 }
 
 function gate_job_exit_tasks {
-  [[ -d "/openstack/log" ]] && chmod -R 0777 /openstack/log
+  # If this is a gate node from OpenStack-Infra Store all logs into the
+  #  execution directory after gate run.
+  if [[ -d "/etc/nodepool" ]];then
+    GATE_LOG_DIR="$(dirname "${0}")/../logs"
+    mkdir -p "${GATE_LOG_DIR}/host" "${GATE_LOG_DIR}/openstack"
+    rsync -av --ignore-errors /var/log/ "${GATE_LOG_DIR}/host" || true
+    rsync -av --ignore-errors /openstack/log/ "${GATE_LOG_DIR}/openstack" || true
+    chmod -R 0777 "${GATE_LOG_DIR}"
+  fi
 }
 
 function print_info {