Terraform roles and jobs.

Adds terraform roles to install and execute terraform.

Supports adding an override.tf file to override configuration in CI
which is useful to let zuul handle module reposity authentication
instead of setting up credentials on the remote during the job.

Also returns the execution plan back as a comment for 'terraform plan'
to make it easy for reviewers.

Change-Id: I3b4f2bac7f055a0c0f9cb7888b4146ac9c007d25

doc/source/hashicorp-jobs.rst

@@ -0,0 +1,7 @@
+Packer Jobs
+===========
+
+.. zuul:autojob:: packer
+.. zuul:autojob:: terraform-apply
+.. zuul:autojob:: terraform-base
+.. zuul:autojob:: terraform-plan

doc/source/hashicorp-roles.rst

@@ -2,4 +2,6 @@ Packer Roles
 ============
 
 .. zuul:autorole:: ensure-packer
+.. zuul:autorole:: ensure-terraform
 .. zuul:autorole:: packer
+.. zuul:autorole:: terraform

doc/source/jobs.rst

@@ -9,8 +9,8 @@ Jobs
    js-jobs
    docker-jobs
    go-jobs
+   hashicorp-jobs
    haskell-jobs
    helm-jobs
-   packer-jobs
    system-jobs
    deprecated-jobs

doc/source/packer-jobs.rst

@@ -1,4 +0,0 @@
-Packer Jobs
-===========
-
-.. zuul:autojob:: packer

doc/source/roles.rst

@@ -16,13 +16,13 @@ Roles
    deprecated-roles
    galaxy-roles
    go-roles
+   hashicorp-roles
    haskell-roles
    helm-roles
    java-roles
    js-roles
    kubernetes-roles
    launchpad-roles
-   packer-roles
    puppet-roles
    python-roles
    system-roles

playbooks/terraform/pre.yaml

@@ -0,0 +1,5 @@
+- hosts: all
+  roles:
+    - ensure-terraform
+    - revoke-sudo
+

playbooks/terraform/run.yaml

@@ -0,0 +1,4 @@
+- hosts: all
+  roles:
+    - terraform
+

roles/ensure-packer/tasks/install-packer.yaml

@@ -56,4 +56,3 @@
 
 - name: Output packer version
   command: "{{ packer_executable }} version"
-

roles/ensure-terraform/README.rst

@@ -0,0 +1,24 @@
+Install terraform
+
+**Role Variables**
+
+.. zuul:rolevar:: terraform_install_dir
+   :default: {{ ansible_user_dir }}/.local/bin/
+
+   Directory to install terraform in.
+
+.. zuul:rolevar:: terraform_version
+   :default: 0.12.26
+
+   Version of terraform to install.
+   Zuul will skip the installation if this matches an already installed version of terraform.
+
+.. zuul:rolevar:: terraform_os
+   :default: {{ ansible_system | lower }}
+
+   OS target of package to install.
+
+.. zuul:rolevar:: terraform_arch
+   :default: amd64 / 386
+
+   Architecture target of package to install.

roles/ensure-terraform/defaults/main.yaml

@@ -0,0 +1,7 @@
+hashicorp_releases_fqdn: https://releases.hashicorp.com
+terraform_version: 0.12.26
+terraform_install_dir: "{{ ansible_user_dir  }}/.local/bin/"
+terraform_os: "{{ ansible_system | lower }}"
+terraform_arch: "{{ terraform_arch_translation[ansible_architecture] }}"
+terraform_package: "terraform_{{ terraform_version }}_{{ terraform_os }}_{{ terraform_arch }}"
+terraform_executable: "{{ ansible_user_dir }}/.local/bin/terraform"

roles/ensure-terraform/tasks/install-terraform.yaml

@@ -0,0 +1,74 @@
+- name: Check if unzip is installed
+  command: unzip -v  # noqa 303
+  failed_when: false
+  register: _unzip_probe
+
+- name: Install unzip
+  when: _unzip_probe.rc != 0
+  package:
+    name: unzip
+  become: yes
+
+- name: Get terraform checksums
+  uri:
+    url: "{{ hashicorp_releases_fqdn }}/\
+          terraform/{{ terraform_version }}/terraform_{{ terraform_version }}_SHA256SUMS"
+    return_content: true
+  register: terraform_version_checksums
+
+- name: Set terraform checksum
+  set_fact:
+    terraform_checksum: "{{\
+        terraform_version_checksums.content |\
+        regex_search( '[a-z0-9]+  ' + terraform_package) |\
+        regex_replace( '(?P<checksum>[a-z0-9]+)  ' + terraform_package, '\\g<checksum>')
+      }}"
+
+- name: Create temp directory
+  tempfile:
+    state: directory
+  register: terraform_install_tempdir
+
+- name: Download terraform archive
+  get_url:
+    url: "{{ hashicorp_releases_fqdn }}/\
+      terraform/{{ terraform_version }}/{{ terraform_package }}.zip"
+    dest: "{{ terraform_install_tempdir.path }}/{{ terraform_package }}.zip"
+    checksum: "sha256:{{ terraform_checksum }}"
+
+- name: Create terraform package directory
+  file:
+    path: "{{ terraform_install_tempdir.path }}/{{ terraform_package }}"
+    state: directory
+
+- name: Unarchive terraform
+  unarchive:
+    src: "{{ terraform_install_tempdir.path }}/{{ terraform_package }}.zip"
+    dest: "{{ terraform_install_tempdir.path }}/{{ terraform_package }}"
+    remote_src: yes
+
+- name: Make sure installation directory exists
+  file:
+    path: "{{ terraform_install_dir }}"
+    state: directory
+
+- name: Install terraform
+  copy:
+    src: "{{ terraform_install_tempdir.path }}/{{ terraform_package }}/terraform"
+    dest: "{{ terraform_install_dir }}/terraform"
+    mode: '0755'
+    owner: "{{ ansible_user }}"
+    remote_src: yes
+
+- name: Remove tempdir
+  file:
+    path: "{{ terraform_install_tempdir }}"
+    state: absent
+
+- name: Set terraform executable fact
+  set_fact:
+    terraform_executable: "{{ terraform_install_dir }}/terraform"
+    cacheable: true
+
+- name: Output terraform version
+  command: "{{ terraform_executable }} version"

roles/ensure-terraform/tasks/main.yaml

@@ -0,0 +1,12 @@
+- name: Check installed terraform version
+  command: "{{ terraform_executable }} version"
+  register: terraform_installed_version
+  failed_when: false
+
+- name: Install terraform
+  include_tasks: install-terraform.yaml
+  when:
+    - terraform_installed_version.rc != 0 or
+      "terraform_version != (terraform_installed_version.msg | \
+       regex_replace(terraform_version_pattern, '\\g<version>'))"
+

roles/ensure-terraform/vars/main.yaml

@@ -0,0 +1,6 @@
+terraform_arch_translation:
+  amd64: amd64
+  x86_64: amd64
+  i386: 386
+
+terraform_version_pattern: ^Terraform v(?P<version>\d+\.\d+\.\d+).*$

roles/packer/README.rst

@@ -26,7 +26,7 @@ Run packer command. Assumes the appropriate version of packer has been installed
 
    Environment variables to set in packer command.
 
-.. zuul:rolevar:: zuul_workdir
+.. zuul:rolevar:: packer_workdir
    :default: {{ zuul.project.src_dir }}
 
    Directory to run packer in.

roles/terraform/README.rst

@@ -0,0 +1,53 @@
+Run terraform command. Assumes the appropriate version of terraform has been installed.
+
+**Role Variables**
+
+.. zuul:rolevar:: terraform_executable
+   :default: {{ ansible_user_dir }}/.local/bin/terraform
+
+   Path to terraform executable to use.
+
+.. zuul:rolevar:: terraform_command
+   :default: build
+
+   Terraform command to run.
+   Examples are "plan", "apply"
+
+.. zuul:rolevar:: terraform_extra_args
+
+   String of extra command line options to pass to terraform.
+
+.. zuul:rolevar:: terraform_workspace
+
+   Name of the workspace to operate against.
+   By default this will not be created if it does not exist.
+
+.. zuul:rolevar:: terraform_create_workspace
+   :default: false
+
+   Set to true if the workspace should automatically be created if
+   doesn't already exist.
+
+.. zuul:rolevar:: terraform_comment
+   :default: true
+
+   Set to false to stop zuul from leaving a comment with the execution plan.
+
+.. zuul:rolevar:: terraform_overrides
+
+   List of override.tf files to create before initializing terraform.
+   This is useful if a module should use the source from a required project
+   that has been checked out by zuul instead of using a remote git repository.
+
+   .. zuul:rolevar:: dir
+
+      Directory to put override.tf
+
+   .. zuul:rolevar:: content
+
+      Free form content of the override.tf file.
+
+.. zuul:rolevar:: zuul_work_dir
+   :default: {{ zuul.project.src_dir }}
+
+   Directory to run terraform in.

roles/terraform/defaults/main.yaml

@@ -0,0 +1,5 @@
+terraform_executable: "{{ ansible_user_dir }}/.local/bin/terraform"
+terraform_extra_args: ""
+terraform_create_workspace: false
+terraform_comment: true
+zuul_work_dir: "{{ zuul.project.src_dir }}"

roles/terraform/tasks/main.yaml

@@ -0,0 +1,79 @@
+- name: Fail if no terraform command is given
+  fail:
+    msg: "No terraform command given."
+  when: terraform_command is not defined
+
+- name: Create terrafrom overrides
+  when: terraform_overrides is defined
+  copy:
+    content: "{{ zj_override.content }}"
+    dest: "{{ zj_override.dir }}/override.tf"
+  loop: "{{ terraform_overrides }}"
+  loop_control:
+    loop_var: zj_override
+
+- name: Initialize terraform
+  command: "{{ terraform_executable }} init -no-color -input=false"
+  args:
+    chdir: "{{ zuul_work_dir }}"
+  environment:
+    TF_IN_AUTOMATION: 1
+
+- name: List workspaces
+  when: terraform_workspace is defined
+  shell: |
+    set -o pipefail
+    {{ terraform_executable }} workspace list -no-color | sed 's/* //'
+  register: _terraform_workspace_list
+  args:
+    executable: /bin/bash
+  environment:
+    TF_IN_AUTOMATION: 1
+
+- name: Create workspace if it doesn't exist
+  when:
+    - terraform_workspace is defined
+    - terraform_workspace not in _terraform_workspace_list.stdout_lines
+    - terraform_create_workspace
+  command: "{{ terraform_executable }} workspace new -no-color {{ terraform_workspace }}"
+  environment:
+    TF_IN_AUTOMATION: 1
+
+- name: Select workspace
+  when:
+    - terraform_workspace is defined
+  command: "{{ terraform_executable }} workspace select -no-color {{ terraform_workspace }}"
+  environment:
+    TF_IN_AUTOMATION: 1
+
+- name: Run terraform
+  register: terraform_state_change
+  command: >-
+    {{ terraform_executable }} {{ terraform_command }}
+    -no-color
+    -input=false
+    {{ terraform_extra_args }}
+    {% if terraform_command == 'apply' or terraform_command == 'destroy' %}-auto-approve{% endif %}
+  args:
+    chdir: "{{ zuul_work_dir }}"
+  environment:
+    TF_IN_AUTOMATION: 1
+
+- name: Get path to main.tf relative to the repo root
+  when: terraform_command == "plan"
+  register: main_file_location
+  command: "git ls-files --full-name main.tf"  # noqa 303
+  args:
+    chdir: "{{ zuul_work_dir }}"
+
+- name: Return output if command is plan
+  when:
+    - terraform_command == "plan"
+    - terraform_comment
+  zuul_return:
+    data:
+      zuul:
+        file_comments: >
+          {% set file_comments = {} -%}
+          {% set _ = file_comments.update({main_file_location.stdout: [{'message': terraform_state_change.stdout }]}) %}
+          {{- file_comments -}}

test-playbooks/terraform/main.tf

@@ -0,0 +1,8 @@
+resource "local_file" "test_file" {
+  content = "test-content"
+  filename = "${path.module}/test-file"
+}
+
+module "test_module" {
+  source = "git::https://example.org/this/module/does/not/exist.git//module"
+}

test-playbooks/terraform/other-module/main.tf

@@ -0,0 +1,4 @@
+resource "local_file" "test_file" {
+  content = "test-content"
+  filename = "${path.module}/module-test-file"
+}

zuul-tests.d/packer-jobs.yaml

@@ -9,6 +9,7 @@
       - roles/packer/.*
       - test-playbooks/packer/.*
       - zuul.d/packer-jobs.yaml
+      - playbooks/packer/.*
     vars:
       packer_install_dir: '{{ ansible_user_dir }}/packer'
       packer_template: test-playbooks/packer/packer.json

zuul-tests.d/terraform-jobs.yaml

@@ -0,0 +1,158 @@
+- job:
+    name: zuul-jobs-test-terraform
+    parent: terraform-plan
+    description: Test terraform job
+    tags: all-platforms
+    files:
+      - roles/ensure-terraform/.*
+      - roles/terraform/.*
+      - test-playbooks/terraform/.*
+      - zuul.d/terraform-jobs.yaml
+      - playbooks/terraform/.*
+    vars:
+      zuul_work_dir: '{{ zuul.project.src_dir }}/test-playbooks/terraform'
+      terraform_workspace: testing
+      terraform_create_workspace: true
+      terraform_overrides:
+        - dir: '{{ zuul.project.src_dir }}/test-playbooks/terraform'
+          content: |
+            module "test_module" {
+              source = "./other-module"
+            }
+
+- job:
+    name: zuul-jobs-test-terraform-centos-7
+    description: Test terraform job on centos-7
+    parent: zuul-jobs-test-terraform
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: centos-7
+          label: centos-7
+
+- job:
+    name: zuul-jobs-test-terraform-centos-8
+    description: Test terraform job on centos-8
+    parent: zuul-jobs-test-terraform
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: centos-8
+          label: centos-8
+
+- job:
+    name: zuul-jobs-test-terraform-debian-stretch
+    description: Test terraform job on debian-stretch
+    parent: zuul-jobs-test-terraform
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: debian-stretch
+          label: debian-stretch
+
+- job:
+    name: zuul-jobs-test-terraform-fedora-31
+    description: Test terraform job on fedora-31
+    parent: zuul-jobs-test-terraform
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: fedora-31
+          label: fedora-31
+
+- job:
+    name: zuul-jobs-test-terraform-gentoo-17-0-systemd
+    description: Test terraform job on gentoo-17-0-systemd
+    parent: zuul-jobs-test-terraform
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: gentoo-17-0-systemd
+          label: gentoo-17-0-systemd
+
+- job:
+    name: zuul-jobs-test-terraform-opensuse-15
+    description: Test terraform job on opensuse-15
+    parent: zuul-jobs-test-terraform
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: opensuse-15
+          label: opensuse-15
+
+- job:
+    name: zuul-jobs-test-terraform-opensuse-tumbleweed-nv
+    voting: false
+    description: Test terraform job on opensuse-tumbleweed
+    parent: zuul-jobs-test-terraform
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: opensuse-tumbleweed
+          label: opensuse-tumbleweed
+
+- job:
+    name: zuul-jobs-test-terraform-ubuntu-bionic
+    description: Test terraform job on ubuntu-bionic
+    parent: zuul-jobs-test-terraform
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: ubuntu-bionic
+          label: ubuntu-bionic
+
+- job:
+    name: zuul-jobs-test-terraform-ubuntu-xenial
+    description: Test terraform job on ubuntu-xenial
+    parent: zuul-jobs-test-terraform
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: ubuntu-xenial
+          label: ubuntu-xenial
+
+- job:
+    name: zuul-jobs-test-terraform-ubuntu-bionic-plain
+    description: Test terraform job on ubuntu-bionic-plain
+    parent: zuul-jobs-test-terraform
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: ubuntu-bionic-plain
+          label: ubuntu-bionic-plain
+
+- job:
+    name: zuul-jobs-test-terraform-ubuntu-xenial-plain
+    description: Test terraform job on ubuntu-xenial-plain
+    parent: zuul-jobs-test-terraform
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: ubuntu-xenial-plain
+          label: ubuntu-xenial-plain
+
+- job:
+    name: zuul-jobs-test-terraform-centos-8-plain
+    description: Test terraform job on centos-8-plain
+    parent: zuul-jobs-test-terraform
+    tags: auto-generated
+    nodeset:
+      nodes:
+        - name: centos-8-plain
+          label: centos-8-plain
+
+- project:
+    check: &id001
+      jobs:
+        - zuul-jobs-test-terraform-centos-7
+        - zuul-jobs-test-terraform-centos-8
+        - zuul-jobs-test-terraform-debian-stretch
+        - zuul-jobs-test-terraform-fedora-31
+        - zuul-jobs-test-terraform-gentoo-17-0-systemd
+        - zuul-jobs-test-terraform-opensuse-15
+        - zuul-jobs-test-terraform-ubuntu-bionic
+        - zuul-jobs-test-terraform-ubuntu-xenial
+        - zuul-jobs-test-terraform-ubuntu-bionic-plain
+        - zuul-jobs-test-terraform-ubuntu-xenial-plain
+        - zuul-jobs-test-terraform-centos-8-plain
+    gate: *id001

zuul.d/packer-jobs.yaml

@@ -1,7 +1,7 @@
 - job:
     name: packer
     description: |
-      Base job for packaer operations
+      Base job for packer operations
 
       Responds to these variables:
 

zuul.d/terraform-jobs.yaml

@@ -0,0 +1,104 @@
+- job:
+    name: terraform-base
+    description: |
+      Base job for terraform operations
+
+      Responds to these variables:
+
+      .. zuul:jobvar:: terraform_command
+
+         Command to pass to terraform.
+
+      .. zuul:jobvar:: terraform_workspace
+
+         Name of the workspace to operate against.
+         By default this will not be created if it does not exist.
+
+      .. zuul:rolevar:: terraform_create_workspace
+         :default: false
+
+         Set to true if the workspace should automatically be created if
+         doesn't already exist.
+
+      .. zuul:rolevar:: terraform_comment
+         :default: true
+
+         Set to false to stop zuul from leaving a comment with the execution plan.
+
+      .. zuul:rolevar:: terraform_overrides
+
+         List of override.tf files to create before initializing terraform.
+         This is useful if a module should use the source from a required project
+         that has been checked out by zuul instead of using a remote git repository.
+
+         .. zuul:rolevar:: dir
+
+            Directory to put override.tf
+
+         .. zuul:rolevar:: content
+
+            Free form content of the override.tf file.
+
+      .. zuul:jobvar:: terraform_extra_args
+         :default: ""
+
+         String containing extra arguments to append to the terraform command line.
+
+      .. zuul:jobvar:: terraform_install_dir
+         :default: {{ ansible_user_dir }}/terraform/
+
+         Path to install terraform in.
+
+      .. zuul:rolevar:: terraform_executable
+         :default: {{ ansible_user_dir }}/.local/bin/terraform
+
+         Path to terraform executable to use.
+
+      .. zuul:jobvar:: terraform_version
+         :default: 0.12.26
+
+         The version of terraform to use.
+
+      .. zuul:jobvar:: terraform_os
+         :default: {{ ansible_system | lower }}
+
+          OS to use when choosing terraform version.
+
+      .. zuul:jobvar:: terraform_arch
+         :default: amd64 / 386
+
+          Architecture to use when choosing terraform version
+
+      .. zuul:jobvar:: zuul_work_dir
+         :default: {{ zuul.project.src_dir }}
+
+         Path to operate in.
+
+    pre-run: playbooks/terraform/pre.yaml
+    run: playbooks/terraform/run.yaml
+
+- job:
+    name: terraform-plan
+    parent: terraform-base
+    description: |
+      Extends terraform-base.
+
+      .. zuul:jobvar:: terraform_command
+         :default: plan
+
+         Command to pass to terraform.
+    vars:
+      terraform_command: plan
+
+- job:
+    name: terraform-apply
+    parent: terraform-base
+    description: |
+      Extends terraform-base.
+
+      .. zuul:jobvar:: terraform_command
+         :default: apply
+
+         Command to pass to terraform.
+    vars:
+      terraform_command: apply