Merge "Add paste service"

inventory/service/groups.yaml

@@ -94,6 +94,7 @@ groups:
     - mirror[0-9]*.opendev.org
     - nb[0-9]*.opendev.org
     - openstackid[0-9]*.openstack.org
+    - paste[0-9]*.opendev.org
     - refstack[0-9]*.openstack.org
     - review[0-9]*.open*.org
     - review-test.opendev.org
@@ -129,7 +130,9 @@ groups:
     - openstackid.org
     - openstackid[0-9]*.openstack.org
   paste:
-    - paste[0-9]*.open*.org
+    - paste[0-9]*.openstack.org
+  paste_opendev:
+    - paste[0-1]*.opendev.org
   puppet:
     - cacti[0-9]*.open*.org
     - elasticsearch[0-9]*.open*.org
@@ -140,7 +143,7 @@ groups:
     - openstackid-dev*.openstack.org
     - openstackid.org
     - openstackid[0-9]*.openstack.org
-    - paste[0-9]*.open*.org
+    - paste[0-9]*.openstack.org
     - status*.open*.org
     - storyboard-dev[0-9]*.opendev.org
     - storyboard[0-9]*.opendev.org
@@ -158,7 +161,7 @@ groups:
     - logstash[0-9]*.open*.org
     - openstackid[0-9]*.openstack.org
     - openstackid-dev[0-9]*.openstack.org
-    - paste[0-9]*.open*.org
+    - paste[0-9]*.openstack.org
     - status*.open*.org
     - storyboard[0-9]*.opendev.org
     - storyboard-dev[0-9]*.opendev.org

inventory/service/host_vars/paste01.opendev.org.yaml

@@ -0,0 +1,5 @@
+letsencrypt_certs:
+  paste01-opendev-org-main:
+    - paste01.opendev.org
+    - paste.opendev.org
+    - paste.openstack.org

playbooks/roles/letsencrypt-create-certs/handlers/main.yaml

@@ -153,6 +153,11 @@
 - name: letsencrypt updated nb03-opendev-org-main
   include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
 
+# paste
+
+- name: letsencrypt updated paste01-opendev-org-main
+  include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
+
 # review
 
 - name: letsencrypt updated review01-opendev-org-main

playbooks/roles/lodgeit/README.rst

@@ -0,0 +1,25 @@
+lodgeit
+
+Paste service.  Runs a mariadb container and lodgeit container.
+
+** Role Variables **
+
+.. zuul:rolevar:: lodgeit_db_username
+   :default: lodgeit
+
+   db username
+
+.. zuul:rolevar:: lodgeit_db_password
+   :default: <unset>
+
+   lodgeit_db_password
+
+.. zuul:rolevar:: lodgeit_db_dbname
+   :default: lodgeit
+
+   database to connect to
+
+.. zuul:rolevar:: lodgeit_secret_key
+   :default: <unset>
+
+   secret key

playbooks/roles/lodgeit/defaults/main.yaml

@@ -0,0 +1,2 @@
+lodgeit_db_username: lodgeit
+lodgeit_db_dbname: lodgeit

playbooks/roles/lodgeit/handlers/main.yaml

@@ -0,0 +1,4 @@
+- name: paste Reload apache2
+  service:
+    name: apache2
+    state: reloaded

playbooks/roles/lodgeit/tasks/main.yaml

@@ -0,0 +1,70 @@
+- name: Ensure /etc/lodgeit-compose directory
+  file:
+    state: directory
+    path: /etc/lodgeit-compose
+    mode: 0755
+
+- name: Put docker-compose file in place
+  template:
+    src: docker-compose.yaml.j2
+    dest: /etc/lodgeit-compose/docker-compose.yaml
+    mode: 0644
+
+- name: Setup mariadb container
+  block:
+    - name: Setup db directory
+      file:
+        state: directory
+        path: /var/lib/lodgeit_db
+        owner: root
+        group: root
+        mode: 0755
+
+    - name: Set up root mariadb conf file
+      template:
+        src: root.my.cnf.mariadb_container.j2
+        dest: /root/.lodgeit_db.cnf
+        mode: 0400
+
+- name: Install apache2
+  apt:
+    name:
+      - apache2
+      - apache2-utils
+    state: present
+
+- name: Apache modules
+  apache2_module:
+    state: present
+    name: "{{ item }}"
+  loop:
+    - rewrite
+    - proxy
+    - proxy_http
+    - ssl
+    - headers
+
+- name: Copy apache config
+  template:
+    src: paste.vhost.j2
+    dest: /etc/apache2/sites-enabled/000-default.conf
+    owner: root
+    group: root
+    mode: 0644
+  notify: paste Reload apache2
+
+- name: Run docker-compose pull
+  shell:
+    cmd: docker-compose pull
+    chdir: /etc/lodgeit-compose/
+
+- name: Run docker-compose up
+  shell:
+    cmd: "docker-compose up -d"
+    chdir: /etc/lodgeit-compose/
+
+- name: Run docker prune to cleanup unneeded images
+  shell:
+    cmd: docker image prune -f
+
+# TODO : db backups

playbooks/roles/lodgeit/templates/docker-compose.yaml.j2

@@ -0,0 +1,36 @@
+version: '2'
+
+services:
+  mariadb:
+    image: docker.io/library/mariadb:10.4
+    network_mode: host
+    restart: always
+    environment:
+      MYSQL_ROOT_PASSWORD: "{{ lodgeit_db_password }}"
+      MYSQL_DATABASE: "{{ lodgeit_db_dbname }}"
+      MYSQL_USER: "{{ lodgeit_db_username }}"
+      MYSQL_PASSWORD: "{{ lodgeit_db_password }}"
+    command: [
+      '--wait_timeout=28800',
+    ]
+    volumes:
+      - /var/lib/lodgeit_db:/var/lib/mysql
+    logging:
+      driver: syslog
+      options:
+        tag: "docker-mariadb"
+  lodgeit:
+    image: docker.io/opendevorg/lodgeit
+    depends_on:
+      - mariadb
+    network_mode: host
+    command: ['/bin/bash', '-c', 'echo "*** Starting"; sleep 30; /usr/local/bin/uwsgi']
+    logging:
+      driver: syslog
+      options:
+        tag: "docker-lodgeit"
+    environment:
+      LODGEIT_DBURI: 'mysql+pymysql://{{ lodgeit_db_username }}:{{ lodgeit_db_password }}@127.0.0.1:3306/{{ lodgeit_db_dbname }}'
+      LODGEIT_SECRET_KEY: '{{ lodgeit_secret_key }}'
+      LODGEIT_TITLE_OVERRIDE: '<img src="https://opendev.org/img/opendev.svg" style="width: 100px; padding-bottom:10px; margin-left:20px;" alt="Opendev Pastebin">'
+

playbooks/roles/lodgeit/templates/paste.vhost.j2

@@ -0,0 +1,41 @@
+<VirtualHost *:80>
+  ServerName {{ inventory_hostname }}
+  ServerAdmin webmaster@openstack.org
+
+  ErrorLog ${APACHE_LOG_DIR}/paste-error.log
+
+  LogLevel warn
+
+  CustomLog ${APACHE_LOG_DIR}/paste-access.log combined
+
+  Redirect / https://paste.opendev.org/
+
+</VirtualHost>
+
+<VirtualHost *:443>
+  ServerName {{ inventory_hostname }}
+  ServerAdmin webmaster@openstack.org
+
+  AllowEncodedSlashes On
+
+  ErrorLog ${APACHE_LOG_DIR}/paste-ssl-error.log
+
+  LogLevel warn
+
+  CustomLog ${APACHE_LOG_DIR}/paste-ssl-access.log combined
+
+  SSLEngine on
+  SSLProtocol All -SSLv2 -SSLv3
+  # Note: this list should ensure ciphers that provide forward secrecy
+  SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP
+  SSLHonorCipherOrder on
+
+  SSLCertificateFile /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.cer
+  SSLCertificateKeyFile /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.key
+  SSLCertificateChainFile /etc/letsencrypt-certs/{{ inventory_hostname }}/ca.cer
+
+  ProxyPass  / http://localhost:9000/ retry=0
+  ProxyPassReverse / http://localhost:9000/
+
+</VirtualHost>
+

playbooks/roles/lodgeit/templates/root.my.cnf.mariadb_container.j2

@@ -0,0 +1,7 @@
+[client]
+host=127.0.0.1
+port=3306
+user={{ lodgeit_db_username }}
+password={{ lodgeit_db_password }}
+database={{ lodgeit_db_dbname }}
+ssl-mode=disabled

playbooks/service-paste.yaml

@@ -0,0 +1,6 @@
+- hosts: "paste_opendev:!disabled"
+  name: "Base: configure paste"
+  roles:
+    - iptables
+    - install-docker
+    - lodgeit

playbooks/test-paste.yaml

@@ -0,0 +1,6 @@
+- hosts: "paste_opendev"
+  tasks:
+
+    - name: Run selenium container
+      include_role:
+        name: run-selenium

playbooks/zuul/run-base.yaml

@@ -85,6 +85,7 @@
         - host_vars/mirror01.openafs.provider.opendev.org.yaml
         - host_vars/mirror02.openafs.provider.opendev.org.yaml
         - host_vars/mirror-update01.opendev.org.yaml
+        - host_vars/paste01.opendev.org.yaml
         - host_vars/refstack01.openstack.org.yaml
     - name: Display group membership
       command: ansible localhost -m debug -a 'var=groups'

playbooks/zuul/templates/host_vars/paste01.opendev.org.yaml.j2

@@ -0,0 +1,2 @@
+lodgeit_secret_key: secretkey
+lodgeit_db_password: password

testinfra/test_paste.py

@@ -0,0 +1,51 @@
+# Copyright 2020 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from selenium import webdriver
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.common.exceptions import TimeoutException
+import time
+
+testinfra_hosts = ['paste01.opendev.org']
+
+
+def test_lodgeit_container_web_listening(host):
+    paste_http = host.socket("tcp://127.0.0.1:80")
+    assert paste_http.is_listening
+
+    paste_https = host.socket("tcp://127.0.0.1:443")
+    assert paste_https.is_listening
+
+def test_paste(host):
+    cmd = host.run('curl --insecure '
+                   '--resolve paste.opendev.org:443:127.0.0.1 '
+                   'https://paste.opendev.org')
+    assert 'New Paste' in cmd.stdout
+
+def test_paste_screenshots(host):
+    driver = webdriver.Remote(
+        command_executor='http://%s:4444/wd/hub' % (host.backend.get_hostname()),
+        desired_capabilities=webdriver.DesiredCapabilities.FIREFOX)
+
+    try:
+        driver.get("https://localhost")
+        WebDriverWait(driver, 30).until(lambda driver: driver.execute_script(
+            'return document.readyState') == 'complete')
+        time.sleep(5)
+        driver.save_screenshot("/var/log/screenshots/paste-main-page.png")
+
+    except TimeoutException as e:
+        raise e
+    finally:
+        driver.quit()

zuul.d/infra-prod.yaml

@@ -256,6 +256,22 @@
       - playbooks/roles/logrotate/
       - roles/openafs-client/
 
+- job:
+    name: infra-prod-service-paste
+    parent: infra-prod-service-base
+    description: Run service-paste.yaml playbook.
+    vars:
+      playbook_name: service-paste.yaml
+    files:
+      - inventory/
+      - playbooks/service-paste.yaml
+      - inventory/service/host_vars/paste01.opendev.org.yaml
+      - inventory/service/group_vars/paste
+      - playbooks/roles/install-docker/
+      - playbooks/roles/pip3/
+      - playbooks/roles/lodgeit/
+      - playbooks/roles/iptables/
+
 - job:
     name: infra-prod-service-static
     parent: infra-prod-service-base

zuul.d/project.yaml

@@ -36,6 +36,9 @@
         - system-config-run-meetpad
         - system-config-run-mirror-x86
         - system-config-run-mirror-update
+        - system-config-run-paste:
+            dependencies:
+              - name: opendev-buildset-registry
         - system-config-run-static
         - system-config-run-docker-registry
         - system-config-run-etherpad:
@@ -165,6 +168,9 @@
         - system-config-run-meetpad
         - system-config-run-mirror-x86
         - system-config-run-mirror-update
+        - system-config-run-paste:
+            dependencies:
+              - name: opendev-buildset-registry
         - system-config-run-static
         - system-config-run-docker-registry
         - system-config-run-etherpad:
@@ -374,6 +380,7 @@
         - infra-prod-service-kerberos
         - infra-prod-service-mirror-update
         - infra-prod-service-mirror
+        - infra-prod-service-paste
         - infra-prod-service-static
         - infra-prod-service-borg-backup
         - infra-prod-service-zookeeper

zuul.d/system-config-run.yaml

@@ -687,6 +687,37 @@
       - playbooks/zuul/templates/group_vars/meetpad.yaml.j2
       - testinfra/test_meetpad.py
 
+- job:
+    name: system-config-run-paste
+    parent: system-config-run-containers
+    description: |
+      Run the playbook for the paste server.
+    timeout: 3600
+    required-projects:
+      - opendev/system-config
+    requires:
+      - lodgeit-container-image
+    nodeset:
+      nodes:
+        - name: bridge.openstack.org
+          label: ubuntu-bionic
+        - name: paste01.opendev.org
+          label: ubuntu-focal
+    vars:
+      run_playbooks:
+        - playbooks/letsencrypt.yaml
+        - playbooks/service-paste.yaml
+      run_test_playbook: playbooks/test-paste.yaml
+    files:
+      - playbooks/bridge.yaml
+      - playbooks/letsencrypt.yaml
+      - playbooks/service-paste.yaml
+      - playbooks/roles/lodgeit
+      - playbooks/roles/install-docker/
+      - playbooks/roles/pip3/
+      - playbooks/test-paste.yaml
+      - testinfra/test_paste.py
+
 - job:
     name: system-config-run-zookeeper
     parent: system-config-run