diff --git a/inventory/service/groups.yaml b/inventory/service/groups.yaml index 7b4d34bea7..a3146c6a62 100644 --- a/inventory/service/groups.yaml +++ b/inventory/service/groups.yaml @@ -94,6 +94,7 @@ groups: - mirror[0-9]*.opendev.org - nb[0-9]*.opendev.org - openstackid[0-9]*.openstack.org + - paste[0-9]*.opendev.org - refstack[0-9]*.openstack.org - review[0-9]*.open*.org - review-test.opendev.org @@ -129,7 +130,9 @@ groups: - openstackid.org - openstackid[0-9]*.openstack.org paste: - - paste[0-9]*.open*.org + - paste[0-9]*.openstack.org + paste_opendev: + - paste[0-1]*.opendev.org puppet: - cacti[0-9]*.open*.org - elasticsearch[0-9]*.open*.org @@ -140,7 +143,7 @@ groups: - openstackid-dev*.openstack.org - openstackid.org - openstackid[0-9]*.openstack.org - - paste[0-9]*.open*.org + - paste[0-9]*.openstack.org - status*.open*.org - storyboard-dev[0-9]*.opendev.org - storyboard[0-9]*.opendev.org @@ -158,7 +161,7 @@ groups: - logstash[0-9]*.open*.org - openstackid[0-9]*.openstack.org - openstackid-dev[0-9]*.openstack.org - - paste[0-9]*.open*.org + - paste[0-9]*.openstack.org - status*.open*.org - storyboard[0-9]*.opendev.org - storyboard-dev[0-9]*.opendev.org diff --git a/inventory/service/host_vars/paste01.opendev.org.yaml b/inventory/service/host_vars/paste01.opendev.org.yaml new file mode 100644 index 0000000000..b2ef6cff36 --- /dev/null +++ b/inventory/service/host_vars/paste01.opendev.org.yaml @@ -0,0 +1,5 @@ +letsencrypt_certs: + paste01-opendev-org-main: + - paste01.opendev.org + - paste.opendev.org + - paste.openstack.org diff --git a/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml b/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml index 304285958f..02fe4f43d6 100644 --- a/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml +++ b/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml @@ -153,6 +153,11 @@ - name: letsencrypt updated nb03-opendev-org-main include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml +# paste + +- name: letsencrypt updated paste01-opendev-org-main + include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml + # review - name: letsencrypt updated review01-opendev-org-main diff --git a/playbooks/roles/lodgeit/README.rst b/playbooks/roles/lodgeit/README.rst new file mode 100644 index 0000000000..f7ed441fb7 --- /dev/null +++ b/playbooks/roles/lodgeit/README.rst @@ -0,0 +1,25 @@ +lodgeit + +Paste service. Runs a mariadb container and lodgeit container. + +** Role Variables ** + +.. zuul:rolevar:: lodgeit_db_username + :default: lodgeit + + db username + +.. zuul:rolevar:: lodgeit_db_password + :default: + + lodgeit_db_password + +.. zuul:rolevar:: lodgeit_db_dbname + :default: lodgeit + + database to connect to + +.. zuul:rolevar:: lodgeit_secret_key + :default: + + secret key diff --git a/playbooks/roles/lodgeit/defaults/main.yaml b/playbooks/roles/lodgeit/defaults/main.yaml new file mode 100644 index 0000000000..821088c319 --- /dev/null +++ b/playbooks/roles/lodgeit/defaults/main.yaml @@ -0,0 +1,2 @@ +lodgeit_db_username: lodgeit +lodgeit_db_dbname: lodgeit diff --git a/playbooks/roles/lodgeit/handlers/main.yaml b/playbooks/roles/lodgeit/handlers/main.yaml new file mode 100644 index 0000000000..17b3f0a8e8 --- /dev/null +++ b/playbooks/roles/lodgeit/handlers/main.yaml @@ -0,0 +1,4 @@ +- name: paste Reload apache2 + service: + name: apache2 + state: reloaded diff --git a/playbooks/roles/lodgeit/tasks/main.yaml b/playbooks/roles/lodgeit/tasks/main.yaml new file mode 100644 index 0000000000..6577a1244c --- /dev/null +++ b/playbooks/roles/lodgeit/tasks/main.yaml @@ -0,0 +1,70 @@ +- name: Ensure /etc/lodgeit-compose directory + file: + state: directory + path: /etc/lodgeit-compose + mode: 0755 + +- name: Put docker-compose file in place + template: + src: docker-compose.yaml.j2 + dest: /etc/lodgeit-compose/docker-compose.yaml + mode: 0644 + +- name: Setup mariadb container + block: + - name: Setup db directory + file: + state: directory + path: /var/lib/lodgeit_db + owner: root + group: root + mode: 0755 + + - name: Set up root mariadb conf file + template: + src: root.my.cnf.mariadb_container.j2 + dest: /root/.lodgeit_db.cnf + mode: 0400 + +- name: Install apache2 + apt: + name: + - apache2 + - apache2-utils + state: present + +- name: Apache modules + apache2_module: + state: present + name: "{{ item }}" + loop: + - rewrite + - proxy + - proxy_http + - ssl + - headers + +- name: Copy apache config + template: + src: paste.vhost.j2 + dest: /etc/apache2/sites-enabled/000-default.conf + owner: root + group: root + mode: 0644 + notify: paste Reload apache2 + +- name: Run docker-compose pull + shell: + cmd: docker-compose pull + chdir: /etc/lodgeit-compose/ + +- name: Run docker-compose up + shell: + cmd: "docker-compose up -d" + chdir: /etc/lodgeit-compose/ + +- name: Run docker prune to cleanup unneeded images + shell: + cmd: docker image prune -f + +# TODO : db backups diff --git a/playbooks/roles/lodgeit/templates/docker-compose.yaml.j2 b/playbooks/roles/lodgeit/templates/docker-compose.yaml.j2 new file mode 100644 index 0000000000..7bca1bf2c8 --- /dev/null +++ b/playbooks/roles/lodgeit/templates/docker-compose.yaml.j2 @@ -0,0 +1,36 @@ +version: '2' + +services: + mariadb: + image: docker.io/library/mariadb:10.4 + network_mode: host + restart: always + environment: + MYSQL_ROOT_PASSWORD: "{{ lodgeit_db_password }}" + MYSQL_DATABASE: "{{ lodgeit_db_dbname }}" + MYSQL_USER: "{{ lodgeit_db_username }}" + MYSQL_PASSWORD: "{{ lodgeit_db_password }}" + command: [ + '--wait_timeout=28800', + ] + volumes: + - /var/lib/lodgeit_db:/var/lib/mysql + logging: + driver: syslog + options: + tag: "docker-mariadb" + lodgeit: + image: docker.io/opendevorg/lodgeit + depends_on: + - mariadb + network_mode: host + command: ['/bin/bash', '-c', 'echo "*** Starting"; sleep 30; /usr/local/bin/uwsgi'] + logging: + driver: syslog + options: + tag: "docker-lodgeit" + environment: + LODGEIT_DBURI: 'mysql+pymysql://{{ lodgeit_db_username }}:{{ lodgeit_db_password }}@127.0.0.1:3306/{{ lodgeit_db_dbname }}' + LODGEIT_SECRET_KEY: '{{ lodgeit_secret_key }}' + LODGEIT_TITLE_OVERRIDE: 'Opendev Pastebin' + diff --git a/playbooks/roles/lodgeit/templates/paste.vhost.j2 b/playbooks/roles/lodgeit/templates/paste.vhost.j2 new file mode 100644 index 0000000000..99ded2af40 --- /dev/null +++ b/playbooks/roles/lodgeit/templates/paste.vhost.j2 @@ -0,0 +1,41 @@ + + ServerName {{ inventory_hostname }} + ServerAdmin webmaster@openstack.org + + ErrorLog ${APACHE_LOG_DIR}/paste-error.log + + LogLevel warn + + CustomLog ${APACHE_LOG_DIR}/paste-access.log combined + + Redirect / https://paste.opendev.org/ + + + + + ServerName {{ inventory_hostname }} + ServerAdmin webmaster@openstack.org + + AllowEncodedSlashes On + + ErrorLog ${APACHE_LOG_DIR}/paste-ssl-error.log + + LogLevel warn + + CustomLog ${APACHE_LOG_DIR}/paste-ssl-access.log combined + + SSLEngine on + SSLProtocol All -SSLv2 -SSLv3 + # Note: this list should ensure ciphers that provide forward secrecy + SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP + SSLHonorCipherOrder on + + SSLCertificateFile /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.cer + SSLCertificateKeyFile /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.key + SSLCertificateChainFile /etc/letsencrypt-certs/{{ inventory_hostname }}/ca.cer + + ProxyPass / http://localhost:9000/ retry=0 + ProxyPassReverse / http://localhost:9000/ + + + diff --git a/playbooks/roles/lodgeit/templates/root.my.cnf.mariadb_container.j2 b/playbooks/roles/lodgeit/templates/root.my.cnf.mariadb_container.j2 new file mode 100644 index 0000000000..29313e2c86 --- /dev/null +++ b/playbooks/roles/lodgeit/templates/root.my.cnf.mariadb_container.j2 @@ -0,0 +1,7 @@ +[client] +host=127.0.0.1 +port=3306 +user={{ lodgeit_db_username }} +password={{ lodgeit_db_password }} +database={{ lodgeit_db_dbname }} +ssl-mode=disabled diff --git a/playbooks/service-paste.yaml b/playbooks/service-paste.yaml new file mode 100644 index 0000000000..e308d5e205 --- /dev/null +++ b/playbooks/service-paste.yaml @@ -0,0 +1,6 @@ +- hosts: "paste_opendev:!disabled" + name: "Base: configure paste" + roles: + - iptables + - install-docker + - lodgeit diff --git a/playbooks/test-paste.yaml b/playbooks/test-paste.yaml new file mode 100644 index 0000000000..5d4e7105d9 --- /dev/null +++ b/playbooks/test-paste.yaml @@ -0,0 +1,6 @@ +- hosts: "paste_opendev" + tasks: + + - name: Run selenium container + include_role: + name: run-selenium diff --git a/playbooks/zuul/run-base.yaml b/playbooks/zuul/run-base.yaml index 642627b181..e5b71b100d 100644 --- a/playbooks/zuul/run-base.yaml +++ b/playbooks/zuul/run-base.yaml @@ -85,6 +85,7 @@ - host_vars/mirror01.openafs.provider.opendev.org.yaml - host_vars/mirror02.openafs.provider.opendev.org.yaml - host_vars/mirror-update01.opendev.org.yaml + - host_vars/paste01.opendev.org.yaml - host_vars/refstack01.openstack.org.yaml - name: Display group membership command: ansible localhost -m debug -a 'var=groups' diff --git a/playbooks/zuul/templates/host_vars/paste01.opendev.org.yaml.j2 b/playbooks/zuul/templates/host_vars/paste01.opendev.org.yaml.j2 new file mode 100644 index 0000000000..58a6dbe079 --- /dev/null +++ b/playbooks/zuul/templates/host_vars/paste01.opendev.org.yaml.j2 @@ -0,0 +1,2 @@ +lodgeit_secret_key: secretkey +lodgeit_db_password: password diff --git a/testinfra/test_paste.py b/testinfra/test_paste.py new file mode 100644 index 0000000000..be4fe0c90b --- /dev/null +++ b/testinfra/test_paste.py @@ -0,0 +1,51 @@ +# Copyright 2020 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from selenium import webdriver +from selenium.webdriver.support.ui import WebDriverWait +from selenium.common.exceptions import TimeoutException +import time + +testinfra_hosts = ['paste01.opendev.org'] + + +def test_lodgeit_container_web_listening(host): + paste_http = host.socket("tcp://127.0.0.1:80") + assert paste_http.is_listening + + paste_https = host.socket("tcp://127.0.0.1:443") + assert paste_https.is_listening + +def test_paste(host): + cmd = host.run('curl --insecure ' + '--resolve paste.opendev.org:443:127.0.0.1 ' + 'https://paste.opendev.org') + assert 'New Paste' in cmd.stdout + +def test_paste_screenshots(host): + driver = webdriver.Remote( + command_executor='http://%s:4444/wd/hub' % (host.backend.get_hostname()), + desired_capabilities=webdriver.DesiredCapabilities.FIREFOX) + + try: + driver.get("https://localhost") + WebDriverWait(driver, 30).until(lambda driver: driver.execute_script( + 'return document.readyState') == 'complete') + time.sleep(5) + driver.save_screenshot("/var/log/screenshots/paste-main-page.png") + + except TimeoutException as e: + raise e + finally: + driver.quit() diff --git a/zuul.d/infra-prod.yaml b/zuul.d/infra-prod.yaml index c68c5f65de..2534644304 100644 --- a/zuul.d/infra-prod.yaml +++ b/zuul.d/infra-prod.yaml @@ -256,6 +256,22 @@ - playbooks/roles/logrotate/ - roles/openafs-client/ +- job: + name: infra-prod-service-paste + parent: infra-prod-service-base + description: Run service-paste.yaml playbook. + vars: + playbook_name: service-paste.yaml + files: + - inventory/ + - playbooks/service-paste.yaml + - inventory/service/host_vars/paste01.opendev.org.yaml + - inventory/service/group_vars/paste + - playbooks/roles/install-docker/ + - playbooks/roles/pip3/ + - playbooks/roles/lodgeit/ + - playbooks/roles/iptables/ + - job: name: infra-prod-service-static parent: infra-prod-service-base diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 24c10445ae..42ceff608c 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -36,6 +36,9 @@ - system-config-run-meetpad - system-config-run-mirror-x86 - system-config-run-mirror-update + - system-config-run-paste: + dependencies: + - name: opendev-buildset-registry - system-config-run-static - system-config-run-docker-registry - system-config-run-etherpad: @@ -165,6 +168,9 @@ - system-config-run-meetpad - system-config-run-mirror-x86 - system-config-run-mirror-update + - system-config-run-paste: + dependencies: + - name: opendev-buildset-registry - system-config-run-static - system-config-run-docker-registry - system-config-run-etherpad: @@ -374,6 +380,7 @@ - infra-prod-service-kerberos - infra-prod-service-mirror-update - infra-prod-service-mirror + - infra-prod-service-paste - infra-prod-service-static - infra-prod-service-borg-backup - infra-prod-service-zookeeper diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml index 5240255426..cd4dabb027 100644 --- a/zuul.d/system-config-run.yaml +++ b/zuul.d/system-config-run.yaml @@ -687,6 +687,37 @@ - playbooks/zuul/templates/group_vars/meetpad.yaml.j2 - testinfra/test_meetpad.py +- job: + name: system-config-run-paste + parent: system-config-run-containers + description: | + Run the playbook for the paste server. + timeout: 3600 + required-projects: + - opendev/system-config + requires: + - lodgeit-container-image + nodeset: + nodes: + - name: bridge.openstack.org + label: ubuntu-bionic + - name: paste01.opendev.org + label: ubuntu-focal + vars: + run_playbooks: + - playbooks/letsencrypt.yaml + - playbooks/service-paste.yaml + run_test_playbook: playbooks/test-paste.yaml + files: + - playbooks/bridge.yaml + - playbooks/letsencrypt.yaml + - playbooks/service-paste.yaml + - playbooks/roles/lodgeit + - playbooks/roles/install-docker/ + - playbooks/roles/pip3/ + - playbooks/test-paste.yaml + - testinfra/test_paste.py + - job: name: system-config-run-zookeeper parent: system-config-run