Merge "Terraform roles and jobs."

doc/source/hashicorp-jobs.rst

@@ -0,0 +1,7 @@
+Packer Jobs
+===========
+
+.. zuul:autojob:: packer
+.. zuul:autojob:: terraform-apply
+.. zuul:autojob:: terraform-base
+.. zuul:autojob:: terraform-plan

doc/source/hashicorp-roles.rst

@@ -2,4 +2,6 @@ Packer Roles
 ============
 
 .. zuul:autorole:: ensure-packer
+.. zuul:autorole:: ensure-terraform
 .. zuul:autorole:: packer
+.. zuul:autorole:: terraform

doc/source/jobs.rst

@@ -9,8 +9,8 @@ Jobs
    js-jobs
    docker-jobs
    go-jobs
+   hashicorp-jobs
    haskell-jobs
    helm-jobs
-   packer-jobs
    system-jobs
    deprecated-jobs

doc/source/packer-jobs.rst

@@ -1,4 +0,0 @@
-Packer Jobs
-===========
-
-.. zuul:autojob:: packer

doc/source/roles.rst

@@ -16,13 +16,13 @@ Roles
    deprecated-roles
    galaxy-roles
    go-roles
+   hashicorp-roles
    haskell-roles
    helm-roles
    java-roles
    js-roles
    kubernetes-roles
    launchpad-roles
-   packer-roles
    puppet-roles
    python-roles
    system-roles

playbooks/terraform/pre.yaml

@@ -0,0 +1,5 @@
+- hosts: all
+  roles:
+    - ensure-terraform
+    - revoke-sudo
+

playbooks/terraform/run.yaml

@@ -0,0 +1,4 @@
+- hosts: all
+  roles:
+    - terraform
+

roles/ensure-packer/tasks/install-packer.yaml

@@ -56,4 +56,3 @@
 
 - name: Output packer version
   command: "{{ packer_executable }} version"
-

roles/ensure-terraform/README.rst

@@ -0,0 +1,24 @@
+Install terraform
+
+**Role Variables**
+
+.. zuul:rolevar:: terraform_install_dir
+   :default: {{ ansible_user_dir }}/.local/bin/
+
+   Directory to install terraform in.
+
+.. zuul:rolevar:: terraform_version
+   :default: 0.12.26
+
+   Version of terraform to install.
+   Zuul will skip the installation if this matches an already installed version of terraform.
+
+.. zuul:rolevar:: terraform_os
+   :default: {{ ansible_system | lower }}
+
+   OS target of package to install.
+
+.. zuul:rolevar:: terraform_arch
+   :default: amd64 / 386
+
+   Architecture target of package to install.

roles/ensure-terraform/defaults/main.yaml

@@ -0,0 +1,7 @@
+hashicorp_releases_fqdn: https://releases.hashicorp.com
+terraform_version: 0.12.26
+terraform_install_dir: "{{ ansible_user_dir  }}/.local/bin/"
+terraform_os: "{{ ansible_system | lower }}"
+terraform_arch: "{{ terraform_arch_translation[ansible_architecture] }}"
+terraform_package: "terraform_{{ terraform_version }}_{{ terraform_os }}_{{ terraform_arch }}"
+terraform_executable: "{{ ansible_user_dir }}/.local/bin/terraform"

roles/ensure-terraform/tasks/install-terraform.yaml

@@ -0,0 +1,74 @@
+- name: Check if unzip is installed
+  command: unzip -v  # noqa 303
+  failed_when: false
+  register: _unzip_probe
+
+- name: Install unzip
+  when: _unzip_probe.rc != 0
+  package:
+    name: unzip
+  become: yes
+
+- name: Get terraform checksums
+  uri:
+    url: "{{ hashicorp_releases_fqdn }}/\
+          terraform/{{ terraform_version }}/terraform_{{ terraform_version }}_SHA256SUMS"
+    return_content: true
+  register: terraform_version_checksums
+
+- name: Set terraform checksum
+  set_fact:
+    terraform_checksum: "{{\
+        terraform_version_checksums.content |\
+        regex_search( '[a-z0-9]+  ' + terraform_package) |\
+        regex_replace( '(?P<checksum>[a-z0-9]+)  ' + terraform_package, '\\g<checksum>')
+      }}"
+
+- name: Create temp directory
+  tempfile:
+    state: directory
+  register: terraform_install_tempdir
+
+- name: Download terraform archive
+  get_url:
+    url: "{{ hashicorp_releases_fqdn }}/\
+      terraform/{{ terraform_version }}/{{ terraform_package }}.zip"
+    dest: "{{ terraform_install_tempdir.path }}/{{ terraform_package }}.zip"
+    checksum: "sha256:{{ terraform_checksum }}"
+
+- name: Create terraform package directory
+  file:
+    path: "{{ terraform_install_tempdir.path }}/{{ terraform_package }}"
+    state: directory
+
+- name: Unarchive terraform
+  unarchive:
+    src: "{{ terraform_install_tempdir.path }}/{{ terraform_package }}.zip"
+    dest: "{{ terraform_install_tempdir.path }}/{{ terraform_package }}"
+    remote_src: yes
+
+- name: Make sure installation directory exists
+  file:
+    path: "{{ terraform_install_dir }}"
+    state: directory
+
+- name: Install terraform
+  copy:
+    src: "{{ terraform_install_tempdir.path }}/{{ terraform_package }}/terraform"
+    dest: "{{ terraform_install_dir }}/terraform"
+    mode: '0755'
+    owner: "{{ ansible_user }}"
+    remote_src: yes
+
+- name: Remove tempdir
+  file:
+    path: "{{ terraform_install_tempdir }}"
+    state: absent
+
+- name: Set terraform executable fact
+  set_fact:
+    terraform_executable: "{{ terraform_install_dir }}/terraform"
+    cacheable: true
+
+- name: Output terraform version
+  command: "{{ terraform_executable }} version"

roles/ensure-terraform/tasks/main.yaml

@@ -0,0 +1,12 @@
+- name: Check installed terraform version
+  command: "{{ terraform_executable }} version"
+  register: terraform_installed_version
+  failed_when: false
+
+- name: Install terraform
+  include_tasks: install-terraform.yaml
+  when:
+    - terraform_installed_version.rc != 0 or
+      "terraform_version != (terraform_installed_version.msg | \
+       regex_replace(terraform_version_pattern, '\\g<version>'))"
+

roles/ensure-terraform/vars/main.yaml

@@ -0,0 +1,6 @@
+terraform_arch_translation:
+  amd64: amd64
+  x86_64: amd64
+  i386: 386
+
+terraform_version_pattern: ^Terraform v(?P<version>\d+\.\d+\.\d+).*$

roles/packer/README.rst

@@ -26,7 +26,7 @@ Run packer command. Assumes the appropriate version of packer has been installed
 
    Environment variables to set in packer command.
 
-.. zuul:rolevar:: zuul_workdir
+.. zuul:rolevar:: packer_workdir
    :default: {{ zuul.project.src_dir }}
 
    Directory to run packer in.

roles/terraform/README.rst

@@ -0,0 +1,53 @@
+Run terraform command. Assumes the appropriate version of terraform has been installed.
+
+**Role Variables**
+
+.. zuul:rolevar:: terraform_executable
+   :default: {{ ansible_user_dir }}/.local/bin/terraform
+
+   Path to terraform executable to use.
+
+.. zuul:rolevar:: terraform_command
+   :default: build
+
+   Terraform command to run.
+   Examples are "plan", "apply"
+
+.. zuul:rolevar:: terraform_extra_args
+
+   String of extra command line options to pass to terraform.
+
+.. zuul:rolevar:: terraform_workspace
+
+   Name of the workspace to operate against.
+   By default this will not be created if it does not exist.
+
+.. zuul:rolevar:: terraform_create_workspace
+   :default: false
+
+   Set to true if the workspace should automatically be created if
+   doesn't already exist.
+
+.. zuul:rolevar:: terraform_comment
+   :default: true
+
+   Set to false to stop zuul from leaving a comment with the execution plan.
+
+.. zuul:rolevar:: terraform_overrides
+
+   List of override.tf files to create before initializing terraform.
+   This is useful if a module should use the source from a required project
+   that has been checked out by zuul instead of using a remote git repository.
+
+   .. zuul:rolevar:: dir
+
+      Directory to put override.tf
+
+   .. zuul:rolevar:: content
+
+      Free form content of the override.tf file.
+
+.. zuul:rolevar:: zuul_work_dir
+   :default: {{ zuul.project.src_dir }}
+
+   Directory to run terraform in.

roles/terraform/defaults/main.yaml

@@ -0,0 +1,5 @@
+terraform_executable: "{{ ansible_user_dir }}/.local/bin/terraform"
+terraform_extra_args: ""
+terraform_create_workspace: false
+terraform_comment: true
+zuul_work_dir: "{{ zuul.project.src_dir }}"

roles/terraform/tasks/main.yaml

@@ -0,0 +1,79 @@
+- name: Fail if no terraform command is given
+  fail:
+    msg: "No terraform command given."
+  when: terraform_command is not defined
+
+- name: Create terrafrom overrides
+  when: terraform_overrides is defined
+  copy:
+    content: "{{ zj_override.content }}"
+    dest: "{{ zj_override.dir }}/override.tf"
+  loop: "{{ terraform_overrides }}"
+  loop_control:
+    loop_var: zj_override
+
+- name: Initialize terraform
+  command: "{{ terraform_executable }} init -no-color -input=false"
+  args:
+    chdir: "{{ zuul_work_dir }}"
+  environment:
+    TF_IN_AUTOMATION: 1
+
+- name: List workspaces
+  when: terraform_workspace is defined
+  shell: |
+    set -o pipefail
+    {{ terraform_executable }} workspace list -no-color | sed 's/* //'
+  register: _terraform_workspace_list
+  args:
+    executable: /bin/bash
+  environment:
+    TF_IN_AUTOMATION: 1
+
+- name: Create workspace if it doesn't exist
+  when:
+    - terraform_workspace is defined
+    - terraform_workspace not in _terraform_workspace_list.stdout_lines
+    - terraform_create_workspace
+  command: "{{ terraform_executable }} workspace new -no-color {{ terraform_workspace }}"
+  environment:
+    TF_IN_AUTOMATION: 1
+
+- name: Select workspace
+  when:
+    - terraform_workspace is defined
+  command: "{{ terraform_executable }} workspace select -no-color {{ terraform_workspace }}"
+  environment:
+    TF_IN_AUTOMATION: 1
+
+- name: Run terraform
+  register: terraform_state_change
+  command: >-
+    {{ terraform_executable }} {{ terraform_command }}
+    -no-color
+    -input=false
+    {{ terraform_extra_args }}
+    {% if terraform_command == 'apply' or terraform_command == 'destroy' %}-auto-approve{% endif %}
+  args:
+    chdir: "{{ zuul_work_dir }}"
+  environment:
+    TF_IN_AUTOMATION: 1
+
+- name: Get path to main.tf relative to the repo root
+  when: terraform_command == "plan"
+  register: main_file_location
+  command: "git ls-files --full-name main.tf"  # noqa 303
+  args:
+    chdir: "{{ zuul_work_dir }}"
+
+- name: Return output if command is plan
+  when:
+    - terraform_command == "plan"
+    - terraform_comment
+  zuul_return:
+    data:
+      zuul:
+        file_comments: >
+          {% set file_comments = {} -%}
+          {% set _ = file_comments.update({main_file_location.stdout: [{'message': terraform_state_change.stdout }]}) %}
+          {{- file_comments -}}

test-playbooks/terraform/main.tf

@@ -0,0 +1,8 @@
+resource "local_file" "test_file" {
+  content = "test-content"
+  filename = "${path.module}/test-file"
+}
+
+module "test_module" {
+  source = "git::https://example.org/this/module/does/not/exist.git//module"
+}

test-playbooks/terraform/other-module/main.tf

@@ -0,0 +1,4 @@
+resource "local_file" "test_file" {
+  content = "test-content"
+  filename = "${path.module}/module-test-file"
+}

zuul-tests.d/packer-jobs.yaml

@@ -9,6 +9,7 @@
       - roles/packer/.*
       - test-playbooks/packer/.*
       - zuul.d/packer-jobs.yaml
+      - playbooks/packer/.*
     vars:
       packer_install_dir: '{{ ansible_user_dir }}/packer'
       packer_template: test-playbooks/packer/packer.json

zuul-tests.d/terraform-jobs.yaml

@@ -0,0 +1,158 @@
+- job:
+    name: zuul-jobs-test-terraform
+    parent: terraform-plan
+    description: Test terraform job
+    tags: all-platforms
+    files:
+      - roles/ensure-terraform/.*
+      - roles/terraform/.*
+      - test-playbooks/terraform/.*
+      - zuul.d/terraform-jobs.yaml
+      - playbooks/terraform/.*
+    vars:
+      zuul_work_dir: '{{ zuul.project.src_dir }}/test-playbooks/terraform'
+      terraform_workspace: testing
+      terraform_create_workspace: true
+      terraform_overrides:
+        - dir: '{{ zuul.project.src_dir }}/test-playbooks/terraform'
+          content: |
+            module "test_module" {
+              source = "./other-module"
+            }
+
+- job:
+    name: zuul-jobs-test-terraform-centos-7
+    description: Test terraform job on centos-7
+    parent: zuul-jobs-test-terraform
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: centos-7
+          label: centos-7
+
+- job:
+    name: zuul-jobs-test-terraform-centos-8
+    description: Test terraform job on centos-8
+    parent: zuul-jobs-test-terraform
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: centos-8
+          label: centos-8
+
+- job:
+    name: zuul-jobs-test-terraform-debian-stretch
+    description: Test terraform job on debian-stretch
+    parent: zuul-jobs-test-terraform
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: debian-stretch
+          label: debian-stretch
+
+- job:
+    name: zuul-jobs-test-terraform-fedora-31
+    description: Test terraform job on fedora-31
+    parent: zuul-jobs-test-terraform
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: fedora-31
+          label: fedora-31
+
+- job:
+    name: zuul-jobs-test-terraform-gentoo-17-0-systemd
+    description: Test terraform job on gentoo-17-0-systemd
+    parent: zuul-jobs-test-terraform
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: gentoo-17-0-systemd
+          label: gentoo-17-0-systemd
+
+- job:
+    name: zuul-jobs-test-terraform-opensuse-15
+    description: Test terraform job on opensuse-15
+    parent: zuul-jobs-test-terraform
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: opensuse-15
+          label: opensuse-15
+
+- job:
+    name: zuul-jobs-test-terraform-opensuse-tumbleweed-nv
+    voting: false
+    description: Test terraform job on opensuse-tumbleweed
+    parent: zuul-jobs-test-terraform
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: opensuse-tumbleweed
+          label: opensuse-tumbleweed
+
+- job:
+    name: zuul-jobs-test-terraform-ubuntu-bionic
+    description: Test terraform job on ubuntu-bionic
+    parent: zuul-jobs-test-terraform
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: ubuntu-bionic
+          label: ubuntu-bionic
+
+- job:
+    name: zuul-jobs-test-terraform-ubuntu-xenial
+    description: Test terraform job on ubuntu-xenial
+    parent: zuul-jobs-test-terraform
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: ubuntu-xenial
+          label: ubuntu-xenial
+
+- job:
+    name: zuul-jobs-test-terraform-ubuntu-bionic-plain
+    description: Test terraform job on ubuntu-bionic-plain
+    parent: zuul-jobs-test-terraform
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: ubuntu-bionic-plain
+          label: ubuntu-bionic-plain
+
+- job:
+    name: zuul-jobs-test-terraform-ubuntu-xenial-plain
+    description: Test terraform job on ubuntu-xenial-plain
+    parent: zuul-jobs-test-terraform
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: ubuntu-xenial-plain
+          label: ubuntu-xenial-plain
+
+- job:
+    name: zuul-jobs-test-terraform-centos-8-plain
+    description: Test terraform job on centos-8-plain
+    parent: zuul-jobs-test-terraform
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: centos-8-plain
+          label: centos-8-plain
+
+- project:
+    check: &id001
+      jobs:
+        - zuul-jobs-test-terraform-centos-7
+        - zuul-jobs-test-terraform-centos-8
+        - zuul-jobs-test-terraform-debian-stretch
+        - zuul-jobs-test-terraform-fedora-31
+        - zuul-jobs-test-terraform-gentoo-17-0-systemd
+        - zuul-jobs-test-terraform-opensuse-15
+        - zuul-jobs-test-terraform-ubuntu-bionic
+        - zuul-jobs-test-terraform-ubuntu-xenial
+        - zuul-jobs-test-terraform-ubuntu-bionic-plain
+        - zuul-jobs-test-terraform-ubuntu-xenial-plain
+        - zuul-jobs-test-terraform-centos-8-plain
+    gate: *id001

zuul.d/packer-jobs.yaml

@@ -1,7 +1,7 @@
 - job:
     name: packer
     description: |
-      Base job for packaer operations
+      Base job for packer operations
 
       Responds to these variables:
 

zuul.d/terraform-jobs.yaml

@@ -0,0 +1,104 @@
+- job:
+    name: terraform-base
+    description: |
+      Base job for terraform operations
+
+      Responds to these variables:
+
+      .. zuul:jobvar:: terraform_command
+
+         Command to pass to terraform.
+
+      .. zuul:jobvar:: terraform_workspace
+
+         Name of the workspace to operate against.
+         By default this will not be created if it does not exist.
+
+      .. zuul:rolevar:: terraform_create_workspace
+         :default: false
+
+         Set to true if the workspace should automatically be created if
+         doesn't already exist.
+
+      .. zuul:rolevar:: terraform_comment
+         :default: true
+
+         Set to false to stop zuul from leaving a comment with the execution plan.
+
+      .. zuul:rolevar:: terraform_overrides
+
+         List of override.tf files to create before initializing terraform.
+         This is useful if a module should use the source from a required project
+         that has been checked out by zuul instead of using a remote git repository.
+
+         .. zuul:rolevar:: dir
+
+            Directory to put override.tf
+
+         .. zuul:rolevar:: content
+
+            Free form content of the override.tf file.
+
+      .. zuul:jobvar:: terraform_extra_args
+         :default: ""
+
+         String containing extra arguments to append to the terraform command line.
+
+      .. zuul:jobvar:: terraform_install_dir
+         :default: {{ ansible_user_dir }}/terraform/
+
+         Path to install terraform in.
+
+      .. zuul:rolevar:: terraform_executable
+         :default: {{ ansible_user_dir }}/.local/bin/terraform
+
+         Path to terraform executable to use.
+
+      .. zuul:jobvar:: terraform_version
+         :default: 0.12.26
+
+         The version of terraform to use.
+
+      .. zuul:jobvar:: terraform_os
+         :default: {{ ansible_system | lower }}
+
+          OS to use when choosing terraform version.
+
+      .. zuul:jobvar:: terraform_arch
+         :default: amd64 / 386
+
+          Architecture to use when choosing terraform version
+
+      .. zuul:jobvar:: zuul_work_dir
+         :default: {{ zuul.project.src_dir }}
+
+         Path to operate in.
+
+    pre-run: playbooks/terraform/pre.yaml
+    run: playbooks/terraform/run.yaml
+
+- job:
+    name: terraform-plan
+    parent: terraform-base
+    description: |
+      Extends terraform-base.
+
+      .. zuul:jobvar:: terraform_command
+         :default: plan
+
+         Command to pass to terraform.
+    vars:
+      terraform_command: plan
+
+- job:
+    name: terraform-apply
+    parent: terraform-base
+    description: |
+      Extends terraform-base.
+
+      .. zuul:jobvar:: terraform_command
+         :default: apply
+
+         Command to pass to terraform.
+    vars:
+      terraform_command: apply