From e32333930a236046115c82b45e6e89888a5bdb70 Mon Sep 17 00:00:00 2001 From: David Moreau Simard Date: Wed, 24 Apr 2019 21:05:04 -0400 Subject: [PATCH] role: Add support and integration tests for postgresql This adds support for specifying the "django.db.backends.postgresql" django database engine, installing the required dependencies and setting up the necessary configuration. We're also adding a new package extras [postgresql] which will take care of installing psycopg2 automatically. This is self tested by integration tests which creates a postgresql Docker container and runs the ara_api role configured to run against the postgresql container. The tests are designed to run either locally or on Zuul. Change-Id: I9bb7166d01c96d8b39361a55b1c9952de8fa9798 --- .zuul.d/zuul.yaml | 14 +++ .../django.db.backends.postgresql.yaml | 49 ++++++++ roles/ara_api/vars/Fedora.yaml | 5 + roles/ara_api/vars/Ubuntu.yaml | 5 + setup.cfg | 2 + tests/install_docker.yaml | 29 +++++ tests/test_tasks.yaml | 90 +++++++++++++++ tests/with_postgresql.yaml | 105 ++++++++++++++++++ 8 files changed, 299 insertions(+) create mode 100644 roles/ara_api/tasks/database_engine/django.db.backends.postgresql.yaml create mode 100644 tests/install_docker.yaml create mode 100644 tests/test_tasks.yaml create mode 100644 tests/with_postgresql.yaml diff --git a/.zuul.d/zuul.yaml b/.zuul.d/zuul.yaml index 7a2ba870..750dea89 100644 --- a/.zuul.d/zuul.yaml +++ b/.zuul.d/zuul.yaml @@ -64,6 +64,7 @@ vars: ara_tests_cleanup: false ara_api_source: "{{ ansible_user_dir }}/src/opendev.org/recordsansible/ara" + ara_api_version: HEAD ansible_python_interpreter: /usr/bin/python3 - job: @@ -76,6 +77,17 @@ parent: ansible-role-ara-api-base nodeset: fedora-latest +# TODO: the install-docker role assumes running on Ubuntu +- job: + name: ansible-role-ara-api-ubuntu-postgresql + parent: ansible-role-ara-api-base + nodeset: ubuntu-bionic + roles: + - zuul: zuul/zuul-jobs + pre-run: tests/install_docker.yaml + run: tests/with_postgresql.yaml + post-run: tests/ara_tests_post.yaml + - job: name: ansible-role-ara-web-base parent: base @@ -187,6 +199,7 @@ - ansible-role-ara-tests-ubuntu-2.6 - ansible-role-ara-api-ubuntu - ansible-role-ara-api-fedora + - ansible-role-ara-api-ubuntu-postgresql - ansible-role-ara-web-ubuntu - ansible-role-ara-web-fedora - ara-tox-linters @@ -197,6 +210,7 @@ - ansible-role-ara-tests-ubuntu-2.6 - ansible-role-ara-api-ubuntu - ansible-role-ara-api-fedora + - ansible-role-ara-api-ubuntu-postgresql - ansible-role-ara-web-ubuntu - ansible-role-ara-web-fedora - ara-tox-linters diff --git a/roles/ara_api/tasks/database_engine/django.db.backends.postgresql.yaml b/roles/ara_api/tasks/database_engine/django.db.backends.postgresql.yaml new file mode 100644 index 00000000..c5298d05 --- /dev/null +++ b/roles/ara_api/tasks/database_engine/django.db.backends.postgresql.yaml @@ -0,0 +1,49 @@ +--- +# Copyright (c) 2019 Red Hat, Inc. +# +# This file is part of ARA Records Ansible. +# +# ARA Records Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ARA Records Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with ARA Records Ansible. If not, see . + +- name: Retrieve list of missing required packages for postgresql + set_fact: + _postgresql_missing_packages: "{{ ara_api_postgresql_packages | difference(ansible_facts.packages.keys()) }}" + +# Only attempt to elevate privileges if there are any missing packages +- when: _postgresql_missing_packages | length > 0 + block: + - name: Install required packages + become: yes + package: + name: "{{ ara_api_postgresql_packages }}" + state: present + rescue: + - name: Fail due to missing packages + fail: + msg: "Failed to elevate privileges and install missing required packages. Install the following packages before running this role again: {{ _postgresql_missing_packages | join(' ') }}" + +# Django requires psycopg2 when using postgresql +# https://docs.djangoproject.com/en/2.2/ref/databases/#postgresql-notes +- name: Ensure psycopg2 is installed + pip: + name: psycopg2 + state: present + virtualenv: "{{ ara_api_venv | bool | ternary(ara_api_venv_path, omit) }}" + virtualenv_python: python3 + +- name: Run SQL migrations + environment: + ARA_SETTINGS: "{{ ara_api_settings }}" + PATH: "{{ path_with_virtualenv | default(omit) }}" + command: ara-manage migrate diff --git a/roles/ara_api/vars/Fedora.yaml b/roles/ara_api/vars/Fedora.yaml index e627d624..64c6b57e 100644 --- a/roles/ara_api/vars/Fedora.yaml +++ b/roles/ara_api/vars/Fedora.yaml @@ -21,3 +21,8 @@ ara_api_required_packages: - python3-virtualenv - python3-libselinux - policycoreutils-python-utils + +ara_api_postgresql_packages: + - postgresql + - postgresql-devel + - gcc diff --git a/roles/ara_api/vars/Ubuntu.yaml b/roles/ara_api/vars/Ubuntu.yaml index 83aa99b3..8268c76b 100644 --- a/roles/ara_api/vars/Ubuntu.yaml +++ b/roles/ara_api/vars/Ubuntu.yaml @@ -19,3 +19,8 @@ ara_api_required_packages: - git - python3-venv + +ara_api_postgresql_packages: + - postgresql + - postgresql-server-dev-10 + - gcc diff --git a/setup.cfg b/setup.cfg index 0ec7f9d9..4c02beaa 100644 --- a/setup.cfg +++ b/setup.cfg @@ -39,6 +39,8 @@ server= django-filter dynaconf[yaml] whitenoise +postgresql= + psycopg2 [build_sphinx] source-dir = doc/source diff --git a/tests/install_docker.yaml b/tests/install_docker.yaml new file mode 100644 index 00000000..565880be --- /dev/null +++ b/tests/install_docker.yaml @@ -0,0 +1,29 @@ +--- +# Copyright (c) 2019 Red Hat, Inc. +# +# This file is part of ARA Records Ansible. +# +# ARA Records Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ARA Records Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with ARA Records Ansible. If not, see . + +# Runs the install-docker role exposed by Zuul: +# - https://opendev.org/zuul/zuul-jobs/src/branch/master/roles/install-docker +# - https://zuul-ci.org/docs/zuul-jobs/container-roles.html#role-install-docker + +# Some tests (such as with postgresql) requires Docker to set up a test database +# server. You do not need to run this role if you have Docker installed already. + +- name: Install docker + hosts: all + roles: + - install-docker diff --git a/tests/test_tasks.yaml b/tests/test_tasks.yaml new file mode 100644 index 00000000..f0b9bbf7 --- /dev/null +++ b/tests/test_tasks.yaml @@ -0,0 +1,90 @@ +# Copyright (c) 2019 Red Hat, Inc. +# +# This file is part of ARA Records Ansible. +# +# ARA is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ARA is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with ARA. If not, see . + +# This is a list of tasks instead of a playbook. They are meant to be included +# as post_tasks inside the same play context as our roles so they have access +# to the play and role variables. + +- name: Set defaults for Ansible package name and version + set_fact: + ara_tests_ansible_name: "{{ ara_tests_ansible_name | default('ansible') }}" + ara_tests_ansible_version: "{{ ara_tests_ansible_version | default('latest') }}" + +# If a version is not explicitly set we want to make sure to +# completely omit the version argument to pip, as it will be coming +# from the long-form integration_ansible_name variable. Additionally, if +# the version is the special value "latest", then we also want to omit +# any version number, but also set the package state to "latest". +- name: Set Ansible version for installation + set_fact: + _install_ansible_version: "{{ ara_tests_ansible_version }}" + when: ara_tests_ansible_version not in ("", "latest") + +- name: Set Ansible package state for installation + set_fact: + _install_ansible_state: latest + when: ara_tests_ansible_version == "latest" + +- name: Install Ansible alongside ARA + pip: + name: "{{ ara_tests_ansible_name }}" + version: "{{ _install_ansible_version | default(omit, True) }}" + state: "{{ _install_ansible_state | default(omit, True) }}" + virtualenv: "{{ ara_api_venv | bool | ternary(ara_api_venv_path, omit) }}" + virtualenv_python: python3 + +- name: Get ARA plugins directory + environment: + PATH: "{{ path_with_virtualenv | default(omit) }}" + command: python -m ara.setup.plugins + changed_when: false + register: ara_setup_plugins + +# These aren't in the same task (i.e, with loop) so we can tell individual test +# runs apart easily rather than keeping all the output bundled in a single task. +- environment: + ANSIBLE_CALLBACK_PLUGINS: "{{ ara_setup_plugins.stdout }}/callback" + ANSIBLE_ACTION_PLUGINS: "{{ ara_setup_plugins.stdout }}/action" + ARA_SETTINGS: "{{ ara_api_settings }}" + vars: + _ansible_playbook: "{{ ara_api_venv_path }}/bin/ansible-playbook -vvv" + _test_root: "{{ ara_api_source_checkout }}/tests/integration" + block: + # smoke.yaml tests setting ara_playbook_name in one of three plays + - name: Run smoke.yaml integration test + command: "{{ _ansible_playbook }} {{ _test_root }}/smoke.yaml" + + - name: Run hosts.yaml integration test + command: "{{ _ansible_playbook }} {{ _test_root }}/hosts.yaml" + + - name: Run import.yaml integration test + command: "{{ _ansible_playbook }} {{ _test_root }}/import.yaml" + + # Tests setting ara_playbook_name as an extra var + - name: Run failed.yaml integration test + command: > + {{ _ansible_playbook }} {{ _test_root }}/failed.yaml -e ara_playbook_name="Failed playbook" + ignore_errors: yes + + - name: Run incomplete.yaml integration test + shell: | + {{ _ansible_playbook }} {{ _test_root }}/incomplete.yaml & + sleep 5 + kill $! + args: + executable: /bin/bash + ignore_errors: yes diff --git a/tests/with_postgresql.yaml b/tests/with_postgresql.yaml new file mode 100644 index 00000000..ccbff4e4 --- /dev/null +++ b/tests/with_postgresql.yaml @@ -0,0 +1,105 @@ +# Copyright (c) 2019 Red Hat, Inc. +# +# This file is part of ARA Records Ansible. +# +# ARA is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ARA is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with ARA. If not, see . + +- name: Test the ARA API with postgresql + hosts: all + gather_facts: yes + vars: + ara_tests_cleanup: true + ara_api_root_dir: /tmp/ara-integration-tests/.ara + ara_api_secret_key: testing + ara_api_debug: true + ara_api_log_level: DEBUG + ara_api_database_engine: django.db.backends.postgresql + ara_api_database_name: ara + ara_api_database_user: ara + ara_api_database_password: password + ara_api_database_host: 127.0.0.1 + ara_api_database_port: 5432 + pre_tasks: + # TODO: This fails if the docker python library isn't installed but we can + # recover by running the command manually. The Ansible provided by Zuul + # does not have the module installed. + - name: Start a postgresql container + docker_container: + name: ara_tests_postgresql + image: postgres:10 + state: started + ports: + - "{{ ara_api_database_port }}:{{ ara_api_database_port }}" + env: + POSTGRES_DB: "{{ ara_api_database_name }}" + POSTGRES_USER: "{{ ara_api_database_user }}" + POSTGRES_PASSWORD: "{{ ara_api_database_password }}" + ignore_errors: yes + register: _docker_run + + - name: Start a postgresql container without docker python library + command: | + docker run -d \ + --name ara_tests_postgresql \ + -p {{ ara_api_database_port }}:{{ ara_api_database_port }} \ + -e POSTGRES_DB={{ ara_api_database_name }} \ + -e POSTGRES_USER={{ ara_api_database_user }} \ + -e POSTGRES_PASSWORD={{ ara_api_database_password }} \ + postgres:10 + when: + - _docker_run is failed + - "'Failed to import docker or docker-py' in _docker_run.msg" + tasks: + - block: + - name: Set up the API with the ara_api role + import_role: + name: ara_api + + # These are tasks rather than a standalone playbook to give us an easy + # access to all the variables within the same play. + - include_tasks: test_tasks.yaml + + - name: Set up postgre password file for passwordless commands + lineinfile: + path: "{{ ansible_user_dir }}/.pgpass" + create: yes + line: "{{ ara_api_database_host }}:{{ ara_api_database_port }}:{{ ara_api_database_name }}:{{ ara_api_database_user }}:{{ ara_api_database_password }}" + mode: 0600 + + # Dump is suffixed by .txt so we don't need magic mimetypes when + # viewing on the web. + - name: Dump database file + command: | + pg_dump -w \ + --host={{ ara_api_database_host }} \ + --username={{ ara_api_database_user }} \ + --dbname={{ ara_api_database_name }} \ + --file={{ ara_api_base_dir }}/pg_dump.sql.txt + + always: + - when: ara_tests_cleanup | bool + block: + - name: Clean up the postgresql container + docker_container: + name: ara_tests_postgresql + state: absent + ignore_errors: yes + register: _docker_rm + + - name: Remove the postgresql container without the docker python library + command: docker rm -f ara_tests_postgresql + ignore_errors: yes + when: + - _docker_rm is failed + - "'Failed to import docker or docker-py' in _docker_rm.msg"