diff --git a/doc/source/hashicorp-jobs.rst b/doc/source/hashicorp-jobs.rst new file mode 100644 index 000000000..ae610de54 --- /dev/null +++ b/doc/source/hashicorp-jobs.rst @@ -0,0 +1,7 @@ +Packer Jobs +=========== + +.. zuul:autojob:: packer +.. zuul:autojob:: terraform-apply +.. zuul:autojob:: terraform-base +.. zuul:autojob:: terraform-plan diff --git a/doc/source/packer-roles.rst b/doc/source/hashicorp-roles.rst similarity index 56% rename from doc/source/packer-roles.rst rename to doc/source/hashicorp-roles.rst index 57068faef..06a604284 100644 --- a/doc/source/packer-roles.rst +++ b/doc/source/hashicorp-roles.rst @@ -2,4 +2,6 @@ Packer Roles ============ .. zuul:autorole:: ensure-packer +.. zuul:autorole:: ensure-terraform .. zuul:autorole:: packer +.. zuul:autorole:: terraform diff --git a/doc/source/jobs.rst b/doc/source/jobs.rst index 3f61a29a4..4180cee0b 100644 --- a/doc/source/jobs.rst +++ b/doc/source/jobs.rst @@ -9,8 +9,8 @@ Jobs js-jobs docker-jobs go-jobs + hashicorp-jobs haskell-jobs helm-jobs - packer-jobs system-jobs deprecated-jobs diff --git a/doc/source/packer-jobs.rst b/doc/source/packer-jobs.rst deleted file mode 100644 index 24dfc788a..000000000 --- a/doc/source/packer-jobs.rst +++ /dev/null @@ -1,4 +0,0 @@ -Packer Jobs -=========== - -.. zuul:autojob:: packer diff --git a/doc/source/roles.rst b/doc/source/roles.rst index fda09d100..4492d0931 100644 --- a/doc/source/roles.rst +++ b/doc/source/roles.rst @@ -16,13 +16,13 @@ Roles deprecated-roles galaxy-roles go-roles + hashicorp-roles haskell-roles helm-roles java-roles js-roles kubernetes-roles launchpad-roles - packer-roles puppet-roles python-roles system-roles diff --git a/playbooks/terraform/pre.yaml b/playbooks/terraform/pre.yaml new file mode 100644 index 000000000..97620b4de --- /dev/null +++ b/playbooks/terraform/pre.yaml @@ -0,0 +1,5 @@ +- hosts: all + roles: + - ensure-terraform + - revoke-sudo + diff --git a/playbooks/terraform/run.yaml b/playbooks/terraform/run.yaml new file mode 100644 index 000000000..5719c84d2 --- /dev/null +++ b/playbooks/terraform/run.yaml @@ -0,0 +1,4 @@ +- hosts: all + roles: + - terraform + diff --git a/roles/ensure-packer/tasks/install-packer.yaml b/roles/ensure-packer/tasks/install-packer.yaml index 1553939a9..1b4fcf21f 100644 --- a/roles/ensure-packer/tasks/install-packer.yaml +++ b/roles/ensure-packer/tasks/install-packer.yaml @@ -56,4 +56,3 @@ - name: Output packer version command: "{{ packer_executable }} version" - diff --git a/roles/ensure-terraform/README.rst b/roles/ensure-terraform/README.rst new file mode 100644 index 000000000..e7d61a2b0 --- /dev/null +++ b/roles/ensure-terraform/README.rst @@ -0,0 +1,24 @@ +Install terraform + +**Role Variables** + +.. zuul:rolevar:: terraform_install_dir + :default: {{ ansible_user_dir }}/.local/bin/ + + Directory to install terraform in. + +.. zuul:rolevar:: terraform_version + :default: 0.12.26 + + Version of terraform to install. + Zuul will skip the installation if this matches an already installed version of terraform. + +.. zuul:rolevar:: terraform_os + :default: {{ ansible_system | lower }} + + OS target of package to install. + +.. zuul:rolevar:: terraform_arch + :default: amd64 / 386 + + Architecture target of package to install. diff --git a/roles/ensure-terraform/defaults/main.yaml b/roles/ensure-terraform/defaults/main.yaml new file mode 100644 index 000000000..add77921c --- /dev/null +++ b/roles/ensure-terraform/defaults/main.yaml @@ -0,0 +1,7 @@ +hashicorp_releases_fqdn: https://releases.hashicorp.com +terraform_version: 0.12.26 +terraform_install_dir: "{{ ansible_user_dir }}/.local/bin/" +terraform_os: "{{ ansible_system | lower }}" +terraform_arch: "{{ terraform_arch_translation[ansible_architecture] }}" +terraform_package: "terraform_{{ terraform_version }}_{{ terraform_os }}_{{ terraform_arch }}" +terraform_executable: "{{ ansible_user_dir }}/.local/bin/terraform" diff --git a/roles/ensure-terraform/tasks/install-terraform.yaml b/roles/ensure-terraform/tasks/install-terraform.yaml new file mode 100644 index 000000000..90a702017 --- /dev/null +++ b/roles/ensure-terraform/tasks/install-terraform.yaml @@ -0,0 +1,74 @@ +- name: Check if unzip is installed + command: unzip -v # noqa 303 + failed_when: false + register: _unzip_probe + +- name: Install unzip + when: _unzip_probe.rc != 0 + package: + name: unzip + become: yes + +- name: Get terraform checksums + uri: + url: "{{ hashicorp_releases_fqdn }}/\ + terraform/{{ terraform_version }}/terraform_{{ terraform_version }}_SHA256SUMS" + return_content: true + register: terraform_version_checksums + +- name: Set terraform checksum + set_fact: + terraform_checksum: "{{\ + terraform_version_checksums.content |\ + regex_search( '[a-z0-9]+ ' + terraform_package) |\ + regex_replace( '(?P[a-z0-9]+) ' + terraform_package, '\\g') + }}" + +- name: Create temp directory + tempfile: + state: directory + register: terraform_install_tempdir + +- name: Download terraform archive + get_url: + url: "{{ hashicorp_releases_fqdn }}/\ + terraform/{{ terraform_version }}/{{ terraform_package }}.zip" + dest: "{{ terraform_install_tempdir.path }}/{{ terraform_package }}.zip" + checksum: "sha256:{{ terraform_checksum }}" + +- name: Create terraform package directory + file: + path: "{{ terraform_install_tempdir.path }}/{{ terraform_package }}" + state: directory + +- name: Unarchive terraform + unarchive: + src: "{{ terraform_install_tempdir.path }}/{{ terraform_package }}.zip" + dest: "{{ terraform_install_tempdir.path }}/{{ terraform_package }}" + remote_src: yes + +- name: Make sure installation directory exists + file: + path: "{{ terraform_install_dir }}" + state: directory + +- name: Install terraform + copy: + src: "{{ terraform_install_tempdir.path }}/{{ terraform_package }}/terraform" + dest: "{{ terraform_install_dir }}/terraform" + mode: '0755' + owner: "{{ ansible_user }}" + remote_src: yes + +- name: Remove tempdir + file: + path: "{{ terraform_install_tempdir }}" + state: absent + +- name: Set terraform executable fact + set_fact: + terraform_executable: "{{ terraform_install_dir }}/terraform" + cacheable: true + +- name: Output terraform version + command: "{{ terraform_executable }} version" diff --git a/roles/ensure-terraform/tasks/main.yaml b/roles/ensure-terraform/tasks/main.yaml new file mode 100644 index 000000000..8d79ad6aa --- /dev/null +++ b/roles/ensure-terraform/tasks/main.yaml @@ -0,0 +1,12 @@ +- name: Check installed terraform version + command: "{{ terraform_executable }} version" + register: terraform_installed_version + failed_when: false + +- name: Install terraform + include_tasks: install-terraform.yaml + when: + - terraform_installed_version.rc != 0 or + "terraform_version != (terraform_installed_version.msg | \ + regex_replace(terraform_version_pattern, '\\g'))" + diff --git a/roles/ensure-terraform/vars/main.yaml b/roles/ensure-terraform/vars/main.yaml new file mode 100644 index 000000000..63ef5dcaa --- /dev/null +++ b/roles/ensure-terraform/vars/main.yaml @@ -0,0 +1,6 @@ +terraform_arch_translation: + amd64: amd64 + x86_64: amd64 + i386: 386 + +terraform_version_pattern: ^Terraform v(?P\d+\.\d+\.\d+).*$ diff --git a/roles/packer/README.rst b/roles/packer/README.rst index 91159fdbb..c7c7966c5 100644 --- a/roles/packer/README.rst +++ b/roles/packer/README.rst @@ -26,7 +26,7 @@ Run packer command. Assumes the appropriate version of packer has been installed Environment variables to set in packer command. -.. zuul:rolevar:: zuul_workdir +.. zuul:rolevar:: packer_workdir :default: {{ zuul.project.src_dir }} Directory to run packer in. diff --git a/roles/terraform/README.rst b/roles/terraform/README.rst new file mode 100644 index 000000000..525c6f4b8 --- /dev/null +++ b/roles/terraform/README.rst @@ -0,0 +1,53 @@ +Run terraform command. Assumes the appropriate version of terraform has been installed. + +**Role Variables** + +.. zuul:rolevar:: terraform_executable + :default: {{ ansible_user_dir }}/.local/bin/terraform + + Path to terraform executable to use. + +.. zuul:rolevar:: terraform_command + :default: build + + Terraform command to run. + Examples are "plan", "apply" + +.. zuul:rolevar:: terraform_extra_args + + String of extra command line options to pass to terraform. + +.. zuul:rolevar:: terraform_workspace + + Name of the workspace to operate against. + By default this will not be created if it does not exist. + +.. zuul:rolevar:: terraform_create_workspace + :default: false + + Set to true if the workspace should automatically be created if + doesn't already exist. + +.. zuul:rolevar:: terraform_comment + :default: true + + Set to false to stop zuul from leaving a comment with the execution plan. + +.. zuul:rolevar:: terraform_overrides + + List of override.tf files to create before initializing terraform. + This is useful if a module should use the source from a required project + that has been checked out by zuul instead of using a remote git repository. + + .. zuul:rolevar:: dir + + Directory to put override.tf + + .. zuul:rolevar:: content + + Free form content of the override.tf file. + +.. zuul:rolevar:: zuul_work_dir + :default: {{ zuul.project.src_dir }} + + Directory to run terraform in. diff --git a/roles/terraform/defaults/main.yaml b/roles/terraform/defaults/main.yaml new file mode 100644 index 000000000..ad5ca5b94 --- /dev/null +++ b/roles/terraform/defaults/main.yaml @@ -0,0 +1,5 @@ +terraform_executable: "{{ ansible_user_dir }}/.local/bin/terraform" +terraform_extra_args: "" +terraform_create_workspace: false +terraform_comment: true +zuul_work_dir: "{{ zuul.project.src_dir }}" diff --git a/roles/terraform/tasks/main.yaml b/roles/terraform/tasks/main.yaml new file mode 100644 index 000000000..1468b9c7e --- /dev/null +++ b/roles/terraform/tasks/main.yaml @@ -0,0 +1,79 @@ +- name: Fail if no terraform command is given + fail: + msg: "No terraform command given." + when: terraform_command is not defined + +- name: Create terrafrom overrides + when: terraform_overrides is defined + copy: + content: "{{ zj_override.content }}" + dest: "{{ zj_override.dir }}/override.tf" + loop: "{{ terraform_overrides }}" + loop_control: + loop_var: zj_override + +- name: Initialize terraform + command: "{{ terraform_executable }} init -no-color -input=false" + args: + chdir: "{{ zuul_work_dir }}" + environment: + TF_IN_AUTOMATION: 1 + +- name: List workspaces + when: terraform_workspace is defined + shell: | + set -o pipefail + {{ terraform_executable }} workspace list -no-color | sed 's/* //' + register: _terraform_workspace_list + args: + executable: /bin/bash + environment: + TF_IN_AUTOMATION: 1 + +- name: Create workspace if it doesn't exist + when: + - terraform_workspace is defined + - terraform_workspace not in _terraform_workspace_list.stdout_lines + - terraform_create_workspace + command: "{{ terraform_executable }} workspace new -no-color {{ terraform_workspace }}" + environment: + TF_IN_AUTOMATION: 1 + +- name: Select workspace + when: + - terraform_workspace is defined + command: "{{ terraform_executable }} workspace select -no-color {{ terraform_workspace }}" + environment: + TF_IN_AUTOMATION: 1 + +- name: Run terraform + register: terraform_state_change + command: >- + {{ terraform_executable }} {{ terraform_command }} + -no-color + -input=false + {{ terraform_extra_args }} + {% if terraform_command == 'apply' or terraform_command == 'destroy' %}-auto-approve{% endif %} + args: + chdir: "{{ zuul_work_dir }}" + environment: + TF_IN_AUTOMATION: 1 + +- name: Get path to main.tf relative to the repo root + when: terraform_command == "plan" + register: main_file_location + command: "git ls-files --full-name main.tf" # noqa 303 + args: + chdir: "{{ zuul_work_dir }}" + +- name: Return output if command is plan + when: + - terraform_command == "plan" + - terraform_comment + zuul_return: + data: + zuul: + file_comments: > + {% set file_comments = {} -%} + {% set _ = file_comments.update({main_file_location.stdout: [{'message': terraform_state_change.stdout }]}) %} + {{- file_comments -}} diff --git a/test-playbooks/terraform/main.tf b/test-playbooks/terraform/main.tf new file mode 100644 index 000000000..fc19b64d3 --- /dev/null +++ b/test-playbooks/terraform/main.tf @@ -0,0 +1,8 @@ +resource "local_file" "test_file" { + content = "test-content" + filename = "${path.module}/test-file" +} + +module "test_module" { + source = "git::https://example.org/this/module/does/not/exist.git//module" +} diff --git a/test-playbooks/terraform/other-module/main.tf b/test-playbooks/terraform/other-module/main.tf new file mode 100644 index 000000000..829af7de1 --- /dev/null +++ b/test-playbooks/terraform/other-module/main.tf @@ -0,0 +1,4 @@ +resource "local_file" "test_file" { + content = "test-content" + filename = "${path.module}/module-test-file" +} diff --git a/zuul-tests.d/packer-jobs.yaml b/zuul-tests.d/packer-jobs.yaml index b6228f808..a048391cc 100644 --- a/zuul-tests.d/packer-jobs.yaml +++ b/zuul-tests.d/packer-jobs.yaml @@ -9,6 +9,7 @@ - roles/packer/.* - test-playbooks/packer/.* - zuul.d/packer-jobs.yaml + - playbooks/packer/.* vars: packer_install_dir: '{{ ansible_user_dir }}/packer' packer_template: test-playbooks/packer/packer.json diff --git a/zuul-tests.d/terraform-jobs.yaml b/zuul-tests.d/terraform-jobs.yaml new file mode 100644 index 000000000..f9eaba73f --- /dev/null +++ b/zuul-tests.d/terraform-jobs.yaml @@ -0,0 +1,158 @@ +- job: + name: zuul-jobs-test-terraform + parent: terraform-plan + description: Test terraform job + tags: all-platforms + files: + - roles/ensure-terraform/.* + - roles/terraform/.* + - test-playbooks/terraform/.* + - zuul.d/terraform-jobs.yaml + - playbooks/terraform/.* + vars: + zuul_work_dir: '{{ zuul.project.src_dir }}/test-playbooks/terraform' + terraform_workspace: testing + terraform_create_workspace: true + terraform_overrides: + - dir: '{{ zuul.project.src_dir }}/test-playbooks/terraform' + content: | + module "test_module" { + source = "./other-module" + } + +- job: + name: zuul-jobs-test-terraform-centos-7 + description: Test terraform job on centos-7 + parent: zuul-jobs-test-terraform + tags: auto-generated + nodeset: + nodes: + - name: centos-7 + label: centos-7 + +- job: + name: zuul-jobs-test-terraform-centos-8 + description: Test terraform job on centos-8 + parent: zuul-jobs-test-terraform + tags: auto-generated + nodeset: + nodes: + - name: centos-8 + label: centos-8 + +- job: + name: zuul-jobs-test-terraform-debian-stretch + description: Test terraform job on debian-stretch + parent: zuul-jobs-test-terraform + tags: auto-generated + nodeset: + nodes: + - name: debian-stretch + label: debian-stretch + +- job: + name: zuul-jobs-test-terraform-fedora-31 + description: Test terraform job on fedora-31 + parent: zuul-jobs-test-terraform + tags: auto-generated + nodeset: + nodes: + - name: fedora-31 + label: fedora-31 + +- job: + name: zuul-jobs-test-terraform-gentoo-17-0-systemd + description: Test terraform job on gentoo-17-0-systemd + parent: zuul-jobs-test-terraform + tags: auto-generated + nodeset: + nodes: + - name: gentoo-17-0-systemd + label: gentoo-17-0-systemd + +- job: + name: zuul-jobs-test-terraform-opensuse-15 + description: Test terraform job on opensuse-15 + parent: zuul-jobs-test-terraform + tags: auto-generated + nodeset: + nodes: + - name: opensuse-15 + label: opensuse-15 + +- job: + name: zuul-jobs-test-terraform-opensuse-tumbleweed-nv + voting: false + description: Test terraform job on opensuse-tumbleweed + parent: zuul-jobs-test-terraform + tags: auto-generated + nodeset: + nodes: + - name: opensuse-tumbleweed + label: opensuse-tumbleweed + +- job: + name: zuul-jobs-test-terraform-ubuntu-bionic + description: Test terraform job on ubuntu-bionic + parent: zuul-jobs-test-terraform + tags: auto-generated + nodeset: + nodes: + - name: ubuntu-bionic + label: ubuntu-bionic + +- job: + name: zuul-jobs-test-terraform-ubuntu-xenial + description: Test terraform job on ubuntu-xenial + parent: zuul-jobs-test-terraform + tags: auto-generated + nodeset: + nodes: + - name: ubuntu-xenial + label: ubuntu-xenial + +- job: + name: zuul-jobs-test-terraform-ubuntu-bionic-plain + description: Test terraform job on ubuntu-bionic-plain + parent: zuul-jobs-test-terraform + tags: auto-generated + nodeset: + nodes: + - name: ubuntu-bionic-plain + label: ubuntu-bionic-plain + +- job: + name: zuul-jobs-test-terraform-ubuntu-xenial-plain + description: Test terraform job on ubuntu-xenial-plain + parent: zuul-jobs-test-terraform + tags: auto-generated + nodeset: + nodes: + - name: ubuntu-xenial-plain + label: ubuntu-xenial-plain + +- job: + name: zuul-jobs-test-terraform-centos-8-plain + description: Test terraform job on centos-8-plain + parent: zuul-jobs-test-terraform + tags: auto-generated + nodeset: + nodes: + - name: centos-8-plain + label: centos-8-plain + +- project: + check: &id001 + jobs: + - zuul-jobs-test-terraform-centos-7 + - zuul-jobs-test-terraform-centos-8 + - zuul-jobs-test-terraform-debian-stretch + - zuul-jobs-test-terraform-fedora-31 + - zuul-jobs-test-terraform-gentoo-17-0-systemd + - zuul-jobs-test-terraform-opensuse-15 + - zuul-jobs-test-terraform-ubuntu-bionic + - zuul-jobs-test-terraform-ubuntu-xenial + - zuul-jobs-test-terraform-ubuntu-bionic-plain + - zuul-jobs-test-terraform-ubuntu-xenial-plain + - zuul-jobs-test-terraform-centos-8-plain + gate: *id001 diff --git a/zuul.d/packer-jobs.yaml b/zuul.d/packer-jobs.yaml index d84c42d3b..139e178e5 100644 --- a/zuul.d/packer-jobs.yaml +++ b/zuul.d/packer-jobs.yaml @@ -1,7 +1,7 @@ - job: name: packer description: | - Base job for packaer operations + Base job for packer operations Responds to these variables: diff --git a/zuul.d/terraform-jobs.yaml b/zuul.d/terraform-jobs.yaml new file mode 100644 index 000000000..9b8465ed8 --- /dev/null +++ b/zuul.d/terraform-jobs.yaml @@ -0,0 +1,104 @@ +- job: + name: terraform-base + description: | + Base job for terraform operations + + Responds to these variables: + + .. zuul:jobvar:: terraform_command + + Command to pass to terraform. + + .. zuul:jobvar:: terraform_workspace + + Name of the workspace to operate against. + By default this will not be created if it does not exist. + + .. zuul:rolevar:: terraform_create_workspace + :default: false + + Set to true if the workspace should automatically be created if + doesn't already exist. + + .. zuul:rolevar:: terraform_comment + :default: true + + Set to false to stop zuul from leaving a comment with the execution plan. + + .. zuul:rolevar:: terraform_overrides + + List of override.tf files to create before initializing terraform. + This is useful if a module should use the source from a required project + that has been checked out by zuul instead of using a remote git repository. + + .. zuul:rolevar:: dir + + Directory to put override.tf + + .. zuul:rolevar:: content + + Free form content of the override.tf file. + + .. zuul:jobvar:: terraform_extra_args + :default: "" + + String containing extra arguments to append to the terraform command line. + + .. zuul:jobvar:: terraform_install_dir + :default: {{ ansible_user_dir }}/terraform/ + + Path to install terraform in. + + .. zuul:rolevar:: terraform_executable + :default: {{ ansible_user_dir }}/.local/bin/terraform + + Path to terraform executable to use. + + .. zuul:jobvar:: terraform_version + :default: 0.12.26 + + The version of terraform to use. + + .. zuul:jobvar:: terraform_os + :default: {{ ansible_system | lower }} + + OS to use when choosing terraform version. + + .. zuul:jobvar:: terraform_arch + :default: amd64 / 386 + + Architecture to use when choosing terraform version + + .. zuul:jobvar:: zuul_work_dir + :default: {{ zuul.project.src_dir }} + + Path to operate in. + + pre-run: playbooks/terraform/pre.yaml + run: playbooks/terraform/run.yaml + +- job: + name: terraform-plan + parent: terraform-base + description: | + Extends terraform-base. + + .. zuul:jobvar:: terraform_command + :default: plan + + Command to pass to terraform. + vars: + terraform_command: plan + +- job: + name: terraform-apply + parent: terraform-base + description: | + Extends terraform-base. + + .. zuul:jobvar:: terraform_command + :default: apply + + Command to pass to terraform. + vars: + terraform_command: apply