add acng to the host system and preseeds

This change ensures that the VMs and host systems cache apt packges
locally which will speed up the boot and deployment process.

Change-Id: I234e338b9f1b9f11ff1e81ede8c5717e033fdad8
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

multi-node-aio/playbooks/pxe/acng.conf.j2

@@ -0,0 +1,34 @@
+# {{ ansible_managed }}
+
+CacheDir: /var/www/pkg-cache
+LogDir: /var/log/apt-cacher-ng
+Port: 3142
+BindAddress: 10.0.2.1
+Remap-debrep: file:deb_mirror*.gz /debian ; file:backends_debian # Debian Archives
+Remap-uburep: file:ubuntu_mirrors /ubuntu ; file:backends_ubuntu # Ubuntu Archives
+Remap-debvol: file:debvol_mirror*.gz /debian-volatile ; file:backends_debvol # Debian Volatile Archives
+Remap-cygwin: file:cygwin_mirrors /cygwin # ; file:backends_cygwin # incomplete, please create this file or specify preferred mirrors here
+Remap-sfnet:  file:sfnet_mirrors # ; file:backends_sfnet # incomplete, please create this file or specify preferred mirrors here
+Remap-alxrep: file:archlx_mirrors /archlinux # ; file:backend_archlx # Arch Linux
+Remap-fedora: file:fedora_mirrors # Fedora Linux
+Remap-epel:   file:epel_mirrors # Fedora EPEL
+Remap-slrep:  file:sl_mirrors # Scientific Linux
+Remap-centos: file:centos_mirrors /centos #centos
+ReportPage: acng-report.html
+PidFile: /var/run/apt-cacher-ng
+ExTreshold: 4
+LocalDirs: acng-doc /usr/share/doc/apt-cacher-ng
+PassThroughPattern: .*
+{% if http_proxy is defined and http_proxy %}
+Proxy: {{ http_proxy }}
+{% endif %}
+{% if ansible_distribution_release | lower != 'trusty' %}
+VfilePatternEx: ^/\?release=[0-9]+&arch=
+{% endif %}
+# NOTE(mhayden): Caching the CentOS mirror list causes yum to throw
+# 503 errors intermittently since the remote file is dynamic. Also,
+# yum has issues with retrieving the mariadb.org repodata bz2 and
+# that causes more intermittent 503 errors. This DontCache line
+# tells apt-cacher-ng to allow requests for these to pass through
+# without being cached.
+DontCache: (mirrorlist\.centos\.org)|(mariadb\.org.*\.bz2$)

multi-node-aio/playbooks/pxe/debian/mnaio-post-install-script.sh.j2

@@ -11,5 +11,5 @@ mkdir -p /root/.ssh
 chmod 0700 /root/.ssh
 echo "{{ tftp_ssh_key }}" >> /root/.ssh/authorized_keys
 
-wget {{ tftp_server }}/networking/mnaio-bridges.cfg -O /etc/network/interfaces.d/mnaio-bridges.cfg
-wget {{ tftp_server }}/networking/basic-interface.cfg -O /etc/network/interfaces
+wget --no-proxy http://{{ tftp_server }}/networking/mnaio-bridges.cfg -O /etc/network/interfaces.d/mnaio-bridges.cfg
+wget --no-proxy http://{{ tftp_server }}/networking/basic-interface.cfg -O /etc/network/interfaces

multi-node-aio/playbooks/pxe/debian/mnaio.preseed.j2

@@ -247,7 +247,7 @@ d-i grub-installer/only_debian boolean true
 # Post Install
 
 d-i preseed/late_command string \
-  in-target bash -c "wget {{ tftp_server }}/scripts/mnaio-post-install-script.sh -O /opt/mnaio-post-install-script.sh"; \
+  in-target bash -c "wget --no-proxy http://{{ tftp_server }}/scripts/mnaio-post-install-script.sh -O /opt/mnaio-post-install-script.sh"; \
   in-target bash -c 'chmod +x /opt/mnaio-post-install-script.sh'; \
   in-target bash -c '/opt/mnaio-post-install-script.sh'
 # Finish

multi-node-aio/playbooks/pxe/debian/vm-post-install-script.sh.j2

@@ -11,8 +11,8 @@ mkdir -p /root/.ssh
 chmod 0700 /root/.ssh
 echo "{{ tftp_ssh_key }}" >> /root/.ssh/authorized_keys
 
-wget {{ tftp_server }}/networking/$(cat /etc/hostname)-bridges.cfg -O /etc/network/interfaces.d/vm-bridges.cfg
-wget {{ tftp_server }}/networking/basic-interface.cfg -O /etc/network/interfaces
+wget --no-proxy http://{{ tftp_server }}/networking/$(cat /etc/hostname)-bridges.cfg -O /etc/network/interfaces.d/vm-bridges.cfg
+wget --no-proxy http://{{ tftp_server }}/networking/basic-interface.cfg -O /etc/network/interfaces
 
 cat > /etc/network/if-up.d/post-up-rules <<EOF
 #!/usr/bin/env bash

multi-node-aio/playbooks/pxe/debian/vm.preseed.j2

@@ -219,7 +219,7 @@ d-i grub-installer/only_debian boolean true
 # Post Install
 
 d-i preseed/late_command string \
-  in-target bash -c "wget {{ tftp_server }}/scripts/vm-post-install-script.sh -O /opt/vm-post-install-script.sh"; \
+  in-target bash -c "wget --no-proxy http://{{ tftp_server }}/scripts/vm-post-install-script.sh -O /opt/vm-post-install-script.sh"; \
   in-target bash -c 'chmod +x /opt/vm-post-install-script.sh'; \
   in-target bash -c '/opt/vm-post-install-script.sh'
 # Finish

multi-node-aio/playbooks/setup-host.yml

@@ -217,6 +217,66 @@
           fi
         done
 
+    - name: Install repo caching server packages
+      package:
+        name: "{{ item }}"
+        state: "latest"
+      with_items: "{{ mnaio_pkg_cache_server_distro_packages }}"
+
+    - name: Create cache directory
+      file:
+        path: "/var/www/pkg-cache"
+        state: "directory"
+        owner: "apt-cacher-ng"
+        group: "www-data"
+        mode: "02775"
+
+    - name: Stat the cache path
+      stat:
+        path: /var/cache/apt-cacher-ng
+      register: acs
+
+    - name: Remove cacher directory if its a directory
+      file:
+        path: "/var/cache/apt-cacher-ng"
+        state: "absent"
+      when:
+        - acs.stat.isdir is defined and acs.stat.isdir
+
+    - name: Link cacher to the repo path
+      file:
+        src: "/var/www/pkg-cache"
+        dest: "/var/cache/apt-cacher-ng"
+        state: "link"
+
+    - name: create yum merged mirror list
+      shell: |
+        curl https://www.centos.org/download/full-mirrorlist.csv | sed 's/^.*"http:/http:/' | sed 's/".*$//' | grep ^http >/etc/apt-cacher-ng/centos_mirrors
+        echo "http://mirror.centos.org/centos/" >>/etc/apt-cacher-ng/centos_mirrors
+
+    - name: Drop acng.conf
+      template:
+        src: "pxe/acng.conf.j2"
+        dest: "/etc/apt-cacher-ng/acng.conf"
+      notify:
+        - reload acng
+
+    - name: Drop apt package manager proxy
+      copy:
+        content: 'Acquire::http { Proxy "{{ default_ubuntu_mirror_proxy }}"; };'
+        dest: "/etc/apt/apt.conf.d/00apt-cacher-proxy"
+
+    - name: Update apt when proxy is added
+      apt:
+        update_cache: yes
+
+  handlers:
+    - name: reload acng
+      service:
+        name: "apt-cacher-ng"
+        state: restarted
+        enabled: yes
+
   vars_files:
     - vars/main.yml
   tags:

multi-node-aio/playbooks/vars/main.yml

@@ -4,10 +4,9 @@ default_interface: "{{ default_network | default('eth0') }}"
 default_vm_image: "{{ default_image | default('ubuntu-16.04-amd64') }}"
 default_vm_storage: "{{ vm_disk_size | default(61440) }}"
 
-default_ubuntu_mirror_proxy: ''
+default_ubuntu_mirror_proxy: 'http://10.0.2.1:3142/'
 default_ubuntu_mirror_hostname: archive.ubuntu.com
 default_ubuntu_mirror_directory: /ubuntu
-
 ############### DEFAULTS ###############
 
 

multi-node-aio/playbooks/vars/ubuntu.yml

@@ -40,4 +40,7 @@ mnaio_pxe_distro_packages:
 mnaio_dhcp_distro_packages:
   - isc-dhcp-server
 
+mnaio_pkg_cache_server_distro_packages:
+  - apt-cacher-ng
+
 mnaio_host_iptables_service: "{{ (ansible_distribution | lower + '-' + ansible_distribution_version | lower == 'ubuntu-14.04') | ternary('iptables-persistent', 'netfilter-persistent') }}"