[#112] Add doc pull in deployment scripts

Added the optional use of doc pull in gate deployment scripts. When the
remote_work_dir is defined, the new pull doc role will invoke airshipctl
document pull command to download the manifests to the remote_work_dir.
This role is not included in zuul job so the zuul job will continue to use
the already checked out manifests in the zuul.project.src directory.

Fixes #112

Change-Id: I754c08661ad0b635819cb3730e05a6986d90f5ca
Signed-off-by: James Gu <james.gu@att.com>

playbooks/airship-airshipctl-test-runner.yaml

@@ -16,9 +16,9 @@
     - name: set default roles
       set_fact:
         test_roles_default:
-          - sync-manifests
-          - airshipctl-systemwide-executable
           - airshipctl-test-configs
+          - airshipctl-pull-docs
+          - airshipctl-systemwide-executable
           - airshipctl-build-ephemeral-iso
           - install-kubectl
           - airshipctl-deploy-ephemeral-node

playbooks/vars/local-dev.yaml

@@ -10,9 +10,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-remote_work_dir: "/tmp/airship"
 proxy:
   enabled: "{{ lookup('env', 'USE_PROXY', default='false') }}"
   http: "{{ lookup('env', 'HTTP_PROXY', default='') }}"
   https: "{{ lookup('env', 'HTTPS_PROXY', default='') }}"
   noproxy: "{{ lookup('env', 'NO_PROXY', default='') }}"
+
+# If remote_work_dir is defined, the deployment script will invoke airshipctl
+# document pull command to download the manifests to the remote_work_dir.
+# Otherwise, the current working dir where the script is launched is used and
+# document pull is skipped.
+remote_work_dir: "/tmp/airship"

playbooks/vars/test-config.yaml

@@ -12,7 +12,7 @@
 
 airship_config_action: generate
 airship_config_iso_gen_target_path: "{{ serve_dir }}"
-airship_config_manifest_directory: "{{ remote_work_dir | default(zuul.project.src_dir) }}"
+airship_config_manifest_directory: "{{ remote_work_dir | default(zuul.project.src_dir) | default(local_src_dir) }}"
 airship_config_ephemeral_ip: "{{ airship_gate_ipam.nat_network.ephemeral_ip }}"
 airship_config_iso_builder_docker_image: "quay.io/airshipit/isogen:latest-debian_stable"
 airship_config_ca_data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRFNU1USXlOakE0TWpneU5Gb1hEVEk1TVRJeU16QTRNamd5TkZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTTFSClM0d3lnajNpU0JBZjlCR0JUS1p5VTFwYmdDaGQ2WTdJektaZWRoakM2K3k1ZEJpWm81ZUx6Z2tEc2gzOC9YQ1MKenFPS2V5cE5RcDN5QVlLdmJKSHg3ODZxSFZZNjg1ZDVYVDNaOHNyVVRzVDR5WmNzZHAzV3lHdDM0eXYzNi9BSQoxK1NlUFErdU5JemN6bzNEdWhXR0ZoQjk3VjZwRitFUTBlVWN5bk05c2hkL3AwWVFzWDR1ZlhxaENENVpzZnZUCnBka3UvTWkyWnVGUldUUUtNeGpqczV3Z2RBWnBsNnN0L2ZkbmZwd1Q5cC9WTjRuaXJnMEsxOURTSFFJTHVrU2MKb013bXNBeDJrZmxITWhPazg5S3FpMEloL2cyczRFYTRvWURZemt0Y2JRZ24wd0lqZ2dmdnVzM3pRbEczN2lwYQo4cVRzS2VmVGdkUjhnZkJDNUZNQ0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFJek9BL00xWmRGUElzd2VoWjFuemJ0VFNURG4KRHMyVnhSV0VnclFFYzNSYmV3a1NkbTlBS3MwVGR0ZHdEbnBEL2tRYkNyS2xEeFF3RWg3NFZNSFZYYkFadDdsVwpCSm90T21xdXgxYThKYklDRTljR0FHRzFvS0g5R29jWERZY0JzOTA3ckxIdStpVzFnL0xVdG5hN1dSampqZnBLCnFGelFmOGdJUHZIM09BZ3B1RVVncUx5QU8ya0VnelZwTjZwQVJxSnZVRks2TUQ0YzFmMnlxWGxwNXhrN2dFSnIKUzQ4WmF6d0RmWUVmV3Jrdld1YWdvZ1M2SktvbjVEZ0Z1ZHhINXM2Snl6R3lPVnZ0eG1TY2FvOHNxaCs3UXkybgoyLzFVcU5ZK0hlN0x4d04rYkhwYkIxNUtIMTU5ZHNuS3BRbjRORG1jSTZrVnJ3MDVJMUg5ZGRBbGF0bz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=

roles/airshipctl-pull-docs/tasks/main.yaml

@@ -10,4 +10,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-remote_work_dir: "/tmp/airship-manifests"
+# The role is used to pull the manifests on remote host.
+# Durning tests in zuul, zuul copies git repository to target node, however
+# When running outside zuul/opendev ci env, we want to pull manifests from
+# commit to remote machine as well.
+
+
+- name: pull documents
+  command: airshipctl document pull --debug
+  when: remote_work_dir is defined

roles/airshipctl-systemwide-executable/tasks/main.yaml

@@ -12,7 +12,7 @@
 
 - name: "make docker-image"
   make:
-    chdir: "{{ remote_work_dir | default(zuul.project.src_dir) }}"
+    chdir: "{{ remote_work_dir + '/airshipctl' if remote_work_dir is defined else zuul.project.src_dir }}"
     target: docker-image
     params:
       USE_PROXY: "{{ proxy.enabled }}"
@@ -27,7 +27,7 @@
     sudo docker cp "${CONTAINER}:/usr/local/bin/airshipctl" "/usr/local/bin/airshipctl"
     docker rm "${CONTAINER}"
   args:
-    chdir: "{{ remote_work_dir | default(zuul.project.src_dir) }}"
+    chdir: "{{ remote_work_dir + '/airshipctl' if remote_work_dir is defined else zuul.project.src_dir }}"
 
 - name: "make sure airshipctl is executable"
-  command: airshipctl version
+  command: airshipctl version

roles/airshipctl-test-configs/defaults/main.yaml

@@ -12,7 +12,9 @@
 
 airship_config_iso_gen_target_path: /srv/iso
 airship_config_manifest_directory: /tmp/airship
-airship_config_primary_repo_url: dummy.url.com
+# the git repo url or local file system path to a cloned repo, e.g., /home/stack/airshipctl
+airship_config_primary_repo_url: "https://review.opendev.org/airship/airshipctl"
+airship_config_primary_repo_branch: "master"
 airship_config_ephemeral_ip: "10.23.25.101"
 airship_config_iso_builder_docker_image: quay.io/airshipit/isogen:latest
 airship_config_iso_port: 8099

roles/airshipctl-test-configs/templates/airshipconfig.j2

@@ -39,15 +39,12 @@ manifests:
     repositories:
       primary:
         checkout:
-          branch: "master"
+          branch: {{ airship_config_primary_repo_branch }}
           force: false
           remoteRef: ""
           tag: ""
         url: {{ airship_config_primary_repo_url }}
-    ## this is temporary hack, as soon as we use `document pull` command in gate process
-    ## this will subpath will be airshipctl/manifests/site/test-site, as airshipctl
-    ## will be primary repository
-    subPath: {{ airship_site_name }}
+    subPath: "{{ 'airshipctl/' if remote_work_dir is defined else '' }}{{ airship_site_name }}"
     targetPath: {{ airship_config_manifest_directory }}
 users:
   dummy_user: {}

roles/sync-manifests/tasks/main.yaml

@@ -1,28 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# The role is used to copy manifest directory to remote host
-# Durning tests in zuul, zuul copies git repository to target node, however
-# When running outside zuul/opendev ci env, we want to copy manifests from
-# commit to remote machine as well. `local_src_dir` should be set from command
-# line in anisble-playbook -e option,
-# eg: ansible-playbook -e local_src_dir="/home/ubuntu/airshipctl"
-
-
-- name: sync repository
-  synchronize:
-    delete: true
-    dest: "{{ remote_work_dir }}"
-    recursive: true
-# trailing slash "/" is needed to copy contents of the directory not directory itself.
-    src: "{{ local_src_dir }}/"
-  no_log: true

tools/gate/22_test_configs.sh

@@ -22,8 +22,9 @@ set -xe
 TMP_DIR=${TMP_DIR:-"$(dirname $(mktemp -u))"}
 ANSIBLE_HOSTS=${ANSIBLE_HOSTS:-"${TMP_DIR}/ansible_hosts"}
 PLAYBOOK_CONFIG=${PLAYBOOK_CONFIG:-"${TMP_DIR}/config.yaml"}
+export AIRSHIPCTL_WS=${AIRSHIPCTL_WS:-$PWD}
 
-sudo ansible-playbook -i "$ANSIBLE_HOSTS" \
+sudo --preserve-env=AIRSHIPCTL_WS ansible-playbook -i "$ANSIBLE_HOSTS" \
 	playbooks/airship-airshipctl-test-runner.yaml \
 	-e @tools/gate/config_test_configs.yaml \
 	-e @"$PLAYBOOK_CONFIG"

tools/gate/config_template.yaml

@@ -16,7 +16,6 @@ serve_port: 8099
 local_src_dir: "$AIRSHIPCTL_WS"
 ansible_user: root
 site_name: test-site
-remote_work_dir: "$AIRSHIPCTL_WS"
 sphinx_build_dir: docs/build
 python_version: 3.7
 zuul: