Update cached LXC image in place

The LXC container creation playbook is one of the longest-running playbooks
in the repository.  It generally takes 15-17 minutes to run during the gate
jobs.  Much of this time is spent updating each container with the latest
packages.

This patch causes the LXC cached image to be updated one time before that
image is used for all of the containers.  It reduces the amount of times the
updates actually run and this shortens the time it takes to complete the
playbook.

The updates to the cached image will only occur if a new cache image has
just been downloaded.

Partial-bug: 1489169

Change-Id: Iba64f9a3aeb999907088f2a99e8904700074550b

playbooks/inventory/group_vars/hosts.yml

@@ -43,6 +43,7 @@ lxc_container_caches:
   - url: "{{ repo_pip_default_index | netorigin }}/container_images/rpc-trusty-container.tgz"
     name: "trusty.tgz"
     sha256sum: "56c6a6e132ea7d10be2f3e8104f47136ccf408b30e362133f0dc4a0a9adb4d0c"
+    chroot_path: trusty/rootfs-amd64
 
 
 ## RabbitMQ

playbooks/roles/lxc_container_create/tasks/container_create.yml

@@ -13,13 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Obtain the Systems SSH-Key
-  set_fact:
-    lxc_container_ssh_key: "{{ lookup('file', '/root/.ssh/id_rsa.pub') }}"
-  when: >
-    lxc_container_ssh_key is not defined
-  delegate_to: "{{ physical_host }}"
-
 - name: Check for lxc volume group
   shell: "(which vgs > /dev/null && vgs | grep -o '{{ lxc_container_vg_name }}') || false"
   register: vg_result
@@ -106,60 +99,6 @@
   tags:
     - lxc-container-service-config
 
-- name: Setup basic container ssh
-  lxc_container:
-    name: "{{ inventory_hostname }}"
-    container_command: |
-      # Enable root ssh login
-      if grep -q "^PermitRootLogin" /etc/ssh/sshd_config;then
-        sed -i 's/PermitRootLogin.*/PermitRootLogin\ yes/g' /etc/ssh/sshd_config
-      else
-        echo 'PermitRootLogin yes' | tee -a /etc/ssh/sshd_config
-      fi
-      # Disable ssh password auth
-      if grep -q "^PasswordAuthentication" /etc/ssh/sshd_config;then
-        sed -i 's/PasswordAuthentication.*/PasswordAuthentication\ no/g' /etc/ssh/sshd_config
-      else
-        echo 'PasswordAuthentication no' | tee -a /etc/ssh/sshd_config
-      fi
-      # Disable UseDNS in ssh
-      if grep -q "^UseDNS" /etc/ssh/sshd_config;then
-        sed -i 's/UseDNS.*/UseDNS\ no/g' /etc/ssh/sshd_config
-      else
-        echo 'UseDNS no' | tee -a /etc/ssh/sshd_config
-      fi
-      # Disable x11 forwarding in ssh
-      if grep -q "^X11Forwarding" /etc/ssh/sshd_config;then
-        sed -i 's/X11Forwarding.*/X11Forwarding\ no/g' /etc/ssh/sshd_config
-      else
-        echo 'X11Forwarding no' | tee -a /etc/ssh/sshd_config
-      fi
-      # Enable tcp keepalive in ssh
-      if grep -q "^TCPKeepAlive" /etc/ssh/sshd_config;then
-        sed -i 's/TCPKeepAlive.*/TCPKeepAlive\ yes/g' /etc/ssh/sshd_config
-      else
-        echo 'TCPKeepAlive yes' | tee -a /etc/ssh/sshd_config
-      fi
-      service ssh restart
-  with_dict: container_networks
-  delegate_to: "{{ physical_host }}"
-  tags:
-    - lxc-container-ssh-config
-
-- name: Create ssh key entry
-  lxc_container:
-    name: "{{ inventory_hostname }}"
-    container_command: |
-      mkdir -p ~/.ssh/
-      if [ ! -f "~/.ssh/authorized_keys" ];then
-        touch ~/.ssh/authorized_keys
-      fi
-      grep '{{ lxc_container_ssh_key }}' ~/.ssh/authorized_keys || echo '{{ lxc_container_ssh_key }}' | tee -a ~/.ssh/authorized_keys
-  with_dict: container_networks
-  delegate_to: "{{ physical_host }}"
-  tags:
-    - lxc-container-key
-
 - name: Container network interfaces
   lxc_container:
     name: "{{ inventory_hostname }}"
@@ -273,58 +212,3 @@
   delegate_to: "{{ physical_host }}"
   tags:
     - lxc-container-proxy
-
-# Uses lxc_container because the repos need to be available before python2.7 is installed
-#  and python2.7 may not be installed at this point.
-- name: Create main apt repos
-  lxc_container:
-    name: "{{ inventory_hostname }}"
-    container_command: |
-      # Configure defined apt-repos
-      rm /etc/apt/sources.list
-      echo '# Sources created by the ansible' | tee /etc/apt/sources.list
-      echo 'deb {{ lxc_container_template_main_apt_repo }} {{ lxc_container_release }} main restricted universe multiverse' | tee -a /etc/apt/sources.list
-      echo 'deb {{ lxc_container_template_main_apt_repo }} {{ lxc_container_release }}-updates main restricted universe multiverse' | tee -a /etc/apt/sources.list
-      echo 'deb {{ lxc_container_template_main_apt_repo }} {{ lxc_container_release }}-backports main restricted universe multiverse' | tee -a /etc/apt/sources.list
-      echo 'deb {{ lxc_container_template_security_apt_repo }} {{ lxc_container_release }}-security main restricted universe multiverse' | tee -a /etc/apt/sources.list
-      for i in {1..3};do
-        timeout 60 sh -c "/usr/bin/apt-get update && /usr/bin/apt-key update"
-        if [ "$?" == 0 ];then
-          break
-        else
-          if [ ! "$i" == "3" ];then
-            echo "Failure to update on attempt $i retrying..."
-            /usr/bin/apt-get clean
-            sleep 2
-          else
-            echo 'Failed to update'
-            exit 99
-          fi
-        fi
-      done
-  delegate_to: "{{ physical_host }}"
-  tags:
-    - lxc-container-sources
-
-# Update the container and ensure that its all patched. This is using lxc_container
-#  because python2.7 may not be installed at this point.
-- name: Ensure container is updated
-  lxc_container:
-    name: "{{ inventory_hostname }}"
-    container_command: |
-      apt-get -y upgrade
-  delegate_to: "{{ physical_host }}"
-  tags:
-    - lxc-container-upgrade
-
-# Uses lxc_container because python2.7 may not be installed within the container at this point.
-- name: Ensure python is installed and is default 2.7
-  lxc_container:
-    name: "{{ inventory_hostname }}"
-    container_command: |
-      apt-get -y install python2.7
-      rm /usr/bin/python
-      ln -s /usr/bin/python2.7 /usr/bin/python
-  delegate_to: "{{ physical_host }}"
-  tags:
-    - lxc-container-python

playbooks/roles/lxc_hosts/defaults/main.yml

@@ -44,6 +44,15 @@ lxc_kernel_options:
   - { key: 'fs.inotify.max_user_instances', value: 1024 }
   - { key: 'vm.swappiness', value: 10 }
 
+# Default image to build from
+lxc_container_release: trusty
+lxc_container_user_name: ubuntu
+lxc_container_user_password: "{{ lookup('pipe', 'date --rfc-3339=ns | sha512sum | base64 | head -c 32') }}"
+lxc_container_template_options: >
+  --release {{ lxc_container_release }}
+  --user {{ lxc_container_user_name }}
+  --password {{ lxc_container_user_password }}
+
 lxc_container_template_main_apt_repo: "https://mirror.rackspace.com/ubuntu"
 lxc_container_template_security_apt_repo: "https://mirror.rackspace.com/ubuntu"
 
@@ -64,8 +73,24 @@ lxc_apt_packages:
   - python-dev
   - python3-lxc
 
+# Commands to run against cached LXC image
+lxc_cache_commands:
+  - apt-get update
+  - apt-get -y upgrade
+  - apt-get -y install python2.7
+  - rm -f /usr/bin/python
+  - ln -s /usr/bin/python2.7 /usr/bin/python
+
+lxc_cache_sshd_configuration:
+    - { regexp: "^PermitRootLogin",         line: "PermitRootLogin yes" }
+    - { regexp: "^TCPKeepAlive",            line: "TCPKeepAlive yes" }
+    - { regexp: "^UseDNS",                  line: "UseDNS no" }
+    - { regexp: "^X11Forwarding",           line: "X11Forwarding no" }
+    - { regexp: "^PasswordAuthentication",  line: "PasswordAuthentication no" }
+
 # Prebuilt images to deploy onto hosts for use in containers.
 # lxc_container_caches:
 #   - url: "https://rpc-repo.rackspace.com/container_images/rpc-trusty-container.tgz"
 #     name: "trusty.tgz"
 #     sha256sum: "56c6a6e132ea7d10be2f3e8104f47136ccf408b30e362133f0dc4a0a9adb4d0c"
+#     chroot_path: trusty/rootfs-amd64

playbooks/roles/lxc_hosts/tasks/lxc_cache.yml

@@ -39,4 +39,3 @@
   tags:
     - lxc-cache
     - lxc-cache-unarchive
-

playbooks/roles/lxc_hosts/tasks/lxc_cache_preparation.yml

@@ -0,0 +1,66 @@
+---
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Create apt repos in the cached container
+  template:
+    src: sources.list.j2
+    dest: "{{ lxc_container_cache_path }}/{{ item.chroot_path }}/etc/apt/sources.list"
+  with_items: lxc_container_caches
+  tags:
+    - lxc-cache
+    - lxc-cache-update
+
+# This task runs several commands against the cached image to speed up the
+# lxc_container_create playbook.
+- name: Prepare cached image
+  command: "chroot {{ lxc_container_cache_path }}/{{ item[0].chroot_path }} {{ item[1] }}"
+  with_nested:
+    - lxc_container_caches
+    - lxc_cache_commands
+  when: cache_download|changed
+  tags:
+    - lxc-cache
+    - lxc-cache-update
+
+- name: Adjust sshd configuration in container
+  lineinfile:
+    dest: "{{ lxc_container_cache_path }}/{{ item[0].chroot_path }}/etc/ssh/sshd_config"
+    regexp: "{{ item[1].regexp }}"
+    line: "{{ item[1].line }}"
+    state: present
+  with_nested:
+    - lxc_container_caches
+    - lxc_cache_sshd_configuration
+  tags:
+    - lxc-cache
+    - lxc-cache-update
+
+- name: Obtain the system's ssh public key
+  set_fact:
+    lxc_container_ssh_key: "{{ lookup('file', '/root/.ssh/id_rsa.pub') }}"
+  when: lxc_container_ssh_key is not defined
+  delegate_to: "{{ physical_host }}"
+  tags:
+    - lxc-cache
+    - lxc-cache-update
+
+- name: Deploy ssh public key into the cached image
+  lineinfile:
+    dest: "{{ lxc_container_cache_path }}/{{ item.chroot_path }}/root/.ssh/authorized_keys"
+    line: "{{ lxc_container_ssh_key }}"
+  with_items: lxc_container_caches
+  tags:
+    - lxc-cache
+    - lxc-cache-update

playbooks/roles/lxc_hosts/tasks/main.yml

@@ -19,6 +19,7 @@
 - include: lxc_install.yml
 - include: lxc_dnsmasq_cleanup.yml
 - include: lxc_cache.yml
+- include: lxc_cache_preparation.yml
   when: lxc_container_caches is defined
 - name: Flush handlers
   meta: flush_handlers

playbooks/roles/lxc_hosts/templates/sources.list.j2

@@ -0,0 +1,5 @@
+# Sources created by the ansible
+deb {{ lxc_container_template_main_apt_repo }} {{ lxc_container_release }} main restricted universe multiverse
+deb {{ lxc_container_template_main_apt_repo }} {{ lxc_container_release }}-updates main restricted universe multiverse
+deb {{ lxc_container_template_main_apt_repo }} {{ lxc_container_release }}-backports main restricted universe multiverse
+deb {{ lxc_container_template_security_apt_repo }} {{ lxc_container_release }}-security main restricted universe multiverse