diff --git a/playbooks/common-tasks/os-lxc-container-setup.yml b/playbooks/common-tasks/os-lxc-container-setup.yml index 1c64fc5737..2e3523840a 100644 --- a/playbooks/common-tasks/os-lxc-container-setup.yml +++ b/playbooks/common-tasks/os-lxc-container-setup.yml @@ -27,39 +27,97 @@ # If extra container configurations are desirable set the # "extra_container_config" list to strings containing the options needed. +- name: Set default bind mounts + set_fact: + lxc_default_bind_mounts: + - bind_dir_path: "/var/log" + mount_path: "/openstack/log/{{ inventory_hostname }}" + when: lxc_default_bind_mounts is undefined + - name: Ensure mount directories exists file: path: "{{ item['mount_path'] }}" state: "directory" - with_items: "{{ list_of_bind_mounts | default([]) }}" + with_items: + - "{{ list_of_bind_mounts | default([]) }}" + - "{{ lxc_default_bind_mounts }}" delegate_to: "{{ physical_host }}" when: - list_of_bind_mounts is defined - not is_metal | bool -- name: LXC Directory bind mount +- name: LXC bind mount directories lxc_container: name: "{{ inventory_hostname }}" container_command: | [[ ! -d "{{ item['bind_dir_path'] }}" ]] && mkdir -p "{{ item['bind_dir_path'] }}" - container_config: - - "lxc.mount.entry={{ item['mount_path'] }} {{ item['bind_dir_path'].lstrip('/') }} none bind 0 0" - with_items: "{{ list_of_bind_mounts | default([]) }}" + with_items: + - "{{ list_of_bind_mounts | default([]) }}" + - "{{ lxc_default_bind_mounts }}" delegate_to: "{{ physical_host }}" register: _bm when: - list_of_bind_mounts is defined - not is_metal | bool -- name: Extra lxc config - lxc_container: - name: "{{ inventory_hostname }}" - container_config: "{{ extra_container_config }}" +- name: Add bind mount configuration to container + lineinfile: + dest: "/var/lib/lxc/{{ inventory_hostname }}/config" + line: "lxc.mount.entry = {{ item['mount_path'] }} {{ item['bind_dir_path'].lstrip('/') }} none bind 0 0" + backup: "true" + with_items: + - "{{ list_of_bind_mounts | default([]) }}" + - "{{ lxc_default_bind_mounts }}" delegate_to: "{{ physical_host }}" + when: + - list_of_bind_mounts is defined + - not is_metal | bool + register: _mc + +- name: Extra lxc config + lineinfile: + dest: "/var/lib/lxc/{{ inventory_hostname }}/config" + line: "{{ item.split('=')[0] }} = {{ item.split('=', 1)[1] }}" + insertafter: "^{{ item.split('=')[0] }}" + backup: "true" + with_items: "{{ extra_container_config }}" + delegate_to: "{{ physical_host }}" + register: _ec when: - extra_container_config is defined - not is_metal | bool - register: _ec + +# Due to https://github.com/ansible/ansible-modules-extras/issues/2691 +# this uses the LXC CLI tools to ensure that we get logging. +# TODO(odyssey4me): revisit this once the bug is fixed and released +- name: Lxc container restart + command: > + lxc-stop --name {{ inventory_hostname }} + --logfile {{ lxc_container_log_path }}/lxc-{{ inventory_hostname }}.log + --logpriority {{ (debug | bool) | ternary('DEBUG', 'INFO') }} + delegate_to: "{{ physical_host }}" + register: container_stop + until: container_stop | success + retries: 3 + when: + - not is_metal | bool + - (_mc is defined and _mc | changed) or (_ec is defined and _ec | changed) + +# Due to https://github.com/ansible/ansible-modules-extras/issues/2691 +# this uses the LXC CLI tools to ensure that we get logging. +# TODO(odyssey4me): revisit this once the bug is fixed and released +- name: Start Container + command: > + lxc-start --daemon --name {{ inventory_hostname }} + --logfile {{ lxc_container_log_path }}/lxc-{{ inventory_hostname }}.log + --logpriority {{ (debug | bool) | ternary('DEBUG', 'INFO') }} + delegate_to: "{{ physical_host }}" + register: container_start + until: container_start | success + retries: 3 + when: + - not is_metal | bool + - (_mc is defined and _mc | changed) or (_ec is defined and _ec | changed) - name: Wait for container ssh wait_for: @@ -68,11 +126,9 @@ search_regex: "OpenSSH" host: "{{ ansible_ssh_host }}" delegate_to: "{{ physical_host }}" - when: - - > - (_bm is defined and _bm | changed) or - (_ec is defined and _ec | changed) - - not is_metal | bool register: ssh_wait_check until: ssh_wait_check | success retries: 3 + when: + - (_bm is defined and _bm | changed) or (_ec is defined and _ec | changed) + - not is_metal | bool diff --git a/playbooks/galera-install.yml b/playbooks/galera-install.yml index c8abff840e..5811c30992 100644 --- a/playbooks/galera-install.yml +++ b/playbooks/galera-install.yml @@ -19,6 +19,11 @@ gather_facts: "{{ gather_facts | default(True) }}" user: root tasks: + - include: common-tasks/os-log-dir-setup.yml + vars: + log_dirs: + - src: "/openstack/log/{{ inventory_hostname }}-mysql_logs" + dest: "/var/log/mysql_logs" - include: common-tasks/os-lxc-container-setup.yml vars: list_of_bind_mounts: diff --git a/playbooks/inventory/group_vars/repo_all.yml b/playbooks/inventory/group_vars/repo_all.yml index b13fc265ff..2ad20fce5d 100644 --- a/playbooks/inventory/group_vars/repo_all.yml +++ b/playbooks/inventory/group_vars/repo_all.yml @@ -17,11 +17,6 @@ repo_server_package_state: "{{ package_state }}" repo_build_package_state: "{{ package_state }}" -# The default bind mount to hold the repo data -repo_all_lxc_container_bind_mounts: - - mount_path: "/openstack/{{ inventory_hostname }}" - bind_dir_path: "/var/www" - # Optionally set this variable to the location on the deployment # host where a set of git clones may be sourced to stage the repo # server. diff --git a/playbooks/repo-server.yml b/playbooks/repo-server.yml index e65c7c6c91..e6c8d09dd0 100644 --- a/playbooks/repo-server.yml +++ b/playbooks/repo-server.yml @@ -29,7 +29,9 @@ - include: common-tasks/os-lxc-container-setup.yml vars: - list_of_bind_mounts: "{{ repo_all_lxc_container_bind_mounts }}" + list_of_bind_mounts: + - mount_path: "/openstack/{{ inventory_hostname }}" + bind_dir_path: "/var/www" when: repo_build_git_cache is not defined or not _local_git_cache.stat.exists - include: common-tasks/os-lxc-container-setup.yml @@ -37,11 +39,14 @@ repo_build_git_cache_bind_mount: - mount_path: "{{ repo_build_git_cache }}" bind_dir_path: "{{ repo_build_git_cache }}" - list_of_bind_mounts: "{{ repo_all_lxc_container_bind_mounts + repo_build_git_cache_bind_mount }}" + list_of_bind_mounts: + - mount_path: "/openstack/{{ inventory_hostname }}" + bind_dir_path: "/var/www" + - mount_path: "{{ repo_build_git_cache }}" + bind_dir_path: "{{ repo_build_git_cache }}" when: - repo_build_git_cache is defined - _local_git_cache.stat.exists - roles: - { role: "repo_server", tags: [ "repo-server" ] } - role: "rsyslog_client" diff --git a/playbooks/rsyslog-install.yml b/playbooks/rsyslog-install.yml index e0a9ee1e80..dd9b1e9662 100644 --- a/playbooks/rsyslog-install.yml +++ b/playbooks/rsyslog-install.yml @@ -22,7 +22,7 @@ - include: common-tasks/os-lxc-container-setup.yml vars: list_of_bind_mounts: - - bind_dir_path: "{{ storage_directory }}" + - bind_dir_path: "{{ rsyslog_server_storage_directory }}" mount_path: "/openstack/{{ inventory_hostname }}/log-storage" - include: common-tasks/package-cache-proxy.yml roles: @@ -31,5 +31,4 @@ tags: - "system-crontab-coordination" vars: - storage_directory: "{{ rsyslog_server_storage_directory }}" is_metal: "{{ properties.is_metal|default(false) }}" diff --git a/releasenotes/notes/bindmount-logs-3c23aab5b5ed3440.yaml b/releasenotes/notes/bindmount-logs-3c23aab5b5ed3440.yaml new file mode 100644 index 0000000000..1ceea65fa1 --- /dev/null +++ b/releasenotes/notes/bindmount-logs-3c23aab5b5ed3440.yaml @@ -0,0 +1,25 @@ +--- +features: + - Containers will now bind mount all logs to the physical host + machine in the "/openstack/log/{{ inventory_hostname }}" + location. This change will ensure containers using a block + backed file system (lvm, zfs, bfrfs) do not run into issues + with full file systems due to logging. +upgrade: + - When upgrading deployers will need to ensure they have a + backup of all logging from within the container prior to + running the playbooks. If the logging node is present within + the deployment all logs should already be sync'd with the + logging server and no action is required. As a pre-step it's + recommended that deployers clean up logging directories from + within containers prior to running the playbooks. After the + playbooks have run the bind mount will be in effect at + "/var/log" which will mount over all previous log files and + directories. + - Due to a new bind mount at "/var/log" all containers will be + restarted. This is a required restart. It is recommended that + deployers run the container restarts in serial to not impact + production workloads. +fixes: + - Logging within the container has been bind mounted to the hosts + this reslves issue `1588051 _` diff --git a/scripts/gate-check-commit.sh b/scripts/gate-check-commit.sh index db820e3437..8a62d2b448 100755 --- a/scripts/gate-check-commit.sh +++ b/scripts/gate-check-commit.sh @@ -79,9 +79,6 @@ popd # Implement the log directory mkdir -p /openstack/log -# Implement the log directory link for openstack-infra log publishing -ln -sf /openstack/log "$(dirname "${0}")/../logs" - pushd "$(dirname "${0}")/../playbooks" # Disable Ansible color output export ANSIBLE_NOCOLOR=1 diff --git a/scripts/scripts-library.sh b/scripts/scripts-library.sh index b09974eb51..77f6538a5b 100755 --- a/scripts/scripts-library.sh +++ b/scripts/scripts-library.sh @@ -132,7 +132,15 @@ function exit_fail { } function gate_job_exit_tasks { - [[ -d "/openstack/log" ]] && chmod -R 0777 /openstack/log + # If this is a gate node from OpenStack-Infra Store all logs into the + # execution directory after gate run. + if [[ -d "/etc/nodepool" ]];then + GATE_LOG_DIR="$(dirname "${0}")/../logs" + mkdir -p "${GATE_LOG_DIR}/host" "${GATE_LOG_DIR}/openstack" + rsync -av --ignore-errors /var/log/ "${GATE_LOG_DIR}/host" || true + rsync -av --ignore-errors /openstack/log/ "${GATE_LOG_DIR}/openstack" || true + chmod -R 0777 "${GATE_LOG_DIR}" + fi } function print_info {