opendev/system-config
Code Issues Proposed changes

Merge "Add paste service"

Browse Source
This commit is contained in:
Zuul 2021-07-13 00:07:03 +00:00 committed by Gerrit Code Review
commit 51480ca77e
18 changed files with 321 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
inventory/service/groups.yaml
View File

@ -94,6 +94,7 @@ groups:
- mirror[0-9]*.opendev.org
- nb[0-9]*.opendev.org
- openstackid[0-9]*.openstack.org
- paste[0-9]*.opendev.org
- refstack[0-9]*.openstack.org
- review[0-9]*.open*.org
- review-test.opendev.org
@ -129,7 +130,9 @@ groups:
- openstackid.org
- openstackid[0-9]*.openstack.org
paste:
- paste[0-9]*.open*.org
- paste[0-9]*.openstack.org
paste_opendev:
- paste[0-1]*.opendev.org
puppet:
- cacti[0-9]*.open*.org
- elasticsearch[0-9]*.open*.org
@ -140,7 +143,7 @@ groups:
- openstackid-dev*.openstack.org
- openstackid.org
- openstackid[0-9]*.openstack.org
- paste[0-9]*.open*.org
- paste[0-9]*.openstack.org
- status*.open*.org
- storyboard-dev[0-9]*.opendev.org
- storyboard[0-9]*.opendev.org
@ -158,7 +161,7 @@ groups:
- logstash[0-9]*.open*.org
- openstackid[0-9]*.openstack.org
- openstackid-dev[0-9]*.openstack.org
- paste[0-9]*.open*.org
- paste[0-9]*.openstack.org
- status*.open*.org
- storyboard[0-9]*.opendev.org
- storyboard-dev[0-9]*.opendev.org

5
inventory/service/host_vars/paste01.opendev.org.yaml Normal file
View File

@ -0,0 +1,5 @@
letsencrypt_certs:
paste01-opendev-org-main:
- paste01.opendev.org
- paste.opendev.org
- paste.openstack.org

5
playbooks/roles/letsencrypt-create-certs/handlers/main.yaml
View File

@ -153,6 +153,11 @@
- name: letsencrypt updated nb03-opendev-org-main
include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
# paste
- name: letsencrypt updated paste01-opendev-org-main
include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
# review
- name: letsencrypt updated review01-opendev-org-main

25
playbooks/roles/lodgeit/README.rst Normal file
View File

@ -0,0 +1,25 @@
lodgeit
Paste service. Runs a mariadb container and lodgeit container.
** Role Variables **
.. zuul:rolevar:: lodgeit_db_username
:default: lodgeit
db username
.. zuul:rolevar:: lodgeit_db_password
:default: <unset>
lodgeit_db_password
.. zuul:rolevar:: lodgeit_db_dbname
:default: lodgeit
database to connect to
.. zuul:rolevar:: lodgeit_secret_key
:default: <unset>
secret key

2
playbooks/roles/lodgeit/defaults/main.yaml Normal file
View File

@ -0,0 +1,2 @@
lodgeit_db_username: lodgeit
lodgeit_db_dbname: lodgeit

4
playbooks/roles/lodgeit/handlers/main.yaml Normal file
View File

@ -0,0 +1,4 @@
- name: paste Reload apache2
service:
name: apache2
state: reloaded

70
playbooks/roles/lodgeit/tasks/main.yaml Normal file
View File

@ -0,0 +1,70 @@
- name: Ensure /etc/lodgeit-compose directory
file:
state: directory
path: /etc/lodgeit-compose
mode: 0755
- name: Put docker-compose file in place
template:
src: docker-compose.yaml.j2
dest: /etc/lodgeit-compose/docker-compose.yaml
mode: 0644
- name: Setup mariadb container
block:
- name: Setup db directory
file:
state: directory
path: /var/lib/lodgeit_db
owner: root
group: root
mode: 0755
- name: Set up root mariadb conf file
template:
src: root.my.cnf.mariadb_container.j2
dest: /root/.lodgeit_db.cnf
mode: 0400
- name: Install apache2
apt:
name:
- apache2
- apache2-utils
state: present
- name: Apache modules
apache2_module:
state: present
name: "{{ item }}"
loop:
- rewrite
- proxy
- proxy_http
- ssl
- headers
- name: Copy apache config
template:
src: paste.vhost.j2
dest: /etc/apache2/sites-enabled/000-default.conf
owner: root
group: root
mode: 0644
notify: paste Reload apache2
- name: Run docker-compose pull
shell:
cmd: docker-compose pull
chdir: /etc/lodgeit-compose/
- name: Run docker-compose up
shell:
cmd: "docker-compose up -d"
chdir: /etc/lodgeit-compose/
- name: Run docker prune to cleanup unneeded images
shell:
cmd: docker image prune -f
# TODO : db backups

36
playbooks/roles/lodgeit/templates/docker-compose.yaml.j2 Normal file
View File

@ -0,0 +1,36 @@
version: '2'
services:
mariadb:
image: docker.io/library/mariadb:10.4
network_mode: host
restart: always
environment:
MYSQL_ROOT_PASSWORD: "{{ lodgeit_db_password }}"
MYSQL_DATABASE: "{{ lodgeit_db_dbname }}"
MYSQL_USER: "{{ lodgeit_db_username }}"
MYSQL_PASSWORD: "{{ lodgeit_db_password }}"
command: [
'--wait_timeout=28800',
]
volumes:
- /var/lib/lodgeit_db:/var/lib/mysql
logging:
driver: syslog
options:
tag: "docker-mariadb"
lodgeit:
image: docker.io/opendevorg/lodgeit
depends_on:
- mariadb
network_mode: host
command: ['/bin/bash', '-c', 'echo "*** Starting"; sleep 30; /usr/local/bin/uwsgi']
logging:
driver: syslog
options:
tag: "docker-lodgeit"
environment:
LODGEIT_DBURI: 'mysql+pymysql://{{ lodgeit_db_username }}:{{ lodgeit_db_password }}@127.0.0.1:3306/{{ lodgeit_db_dbname }}'
LODGEIT_SECRET_KEY: '{{ lodgeit_secret_key }}'
LODGEIT_TITLE_OVERRIDE: '<img src="https://opendev.org/img/opendev.svg" style="width: 100px; padding-bottom:10px; margin-left:20px;" alt="Opendev Pastebin">'

41
playbooks/roles/lodgeit/templates/paste.vhost.j2 Normal file
View File

@ -0,0 +1,41 @@
<VirtualHost *:80>
ServerName {{ inventory_hostname }}
ServerAdmin webmaster@openstack.org
ErrorLog ${APACHE_LOG_DIR}/paste-error.log
LogLevel warn
CustomLog ${APACHE_LOG_DIR}/paste-access.log combined
Redirect / https://paste.opendev.org/
</VirtualHost>
<VirtualHost *:443>
ServerName {{ inventory_hostname }}
ServerAdmin webmaster@openstack.org
AllowEncodedSlashes On
ErrorLog ${APACHE_LOG_DIR}/paste-ssl-error.log
LogLevel warn
CustomLog ${APACHE_LOG_DIR}/paste-ssl-access.log combined
SSLEngine on
SSLProtocol All -SSLv2 -SSLv3
# Note: this list should ensure ciphers that provide forward secrecy
SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP
SSLHonorCipherOrder on
SSLCertificateFile /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.cer
SSLCertificateKeyFile /etc/letsencrypt-certs/{{ inventory_hostname }}/{{ inventory_hostname }}.key
SSLCertificateChainFile /etc/letsencrypt-certs/{{ inventory_hostname }}/ca.cer
ProxyPass / http://localhost:9000/ retry=0
ProxyPassReverse / http://localhost:9000/
</VirtualHost>

7
playbooks/roles/lodgeit/templates/root.my.cnf.mariadb_container.j2 Normal file
View File

@ -0,0 +1,7 @@
[client]
host=127.0.0.1
port=3306
user={{ lodgeit_db_username }}
password={{ lodgeit_db_password }}
database={{ lodgeit_db_dbname }}
ssl-mode=disabled

6
playbooks/service-paste.yaml Normal file
View File

@ -0,0 +1,6 @@
- hosts: "paste_opendev:!disabled"
name: "Base: configure paste"
roles:
- iptables
- install-docker
- lodgeit

6
playbooks/test-paste.yaml Normal file
View File

@ -0,0 +1,6 @@
- hosts: "paste_opendev"
tasks:
- name: Run selenium container
include_role:
name: run-selenium

1
playbooks/zuul/run-base.yaml
View File

@ -85,6 +85,7 @@
- host_vars/mirror01.openafs.provider.opendev.org.yaml
- host_vars/mirror02.openafs.provider.opendev.org.yaml
- host_vars/mirror-update01.opendev.org.yaml
- host_vars/paste01.opendev.org.yaml
- host_vars/refstack01.openstack.org.yaml
- name: Display group membership
command: ansible localhost -m debug -a 'var=groups'

2
playbooks/zuul/templates/host_vars/paste01.opendev.org.yaml.j2 Normal file
View File

@ -0,0 +1,2 @@
lodgeit_secret_key: secretkey
lodgeit_db_password: password

51
testinfra/test_paste.py Normal file
View File

@ -0,0 +1,51 @@
# Copyright 2020 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import TimeoutException
import time
testinfra_hosts = ['paste01.opendev.org']
def test_lodgeit_container_web_listening(host):
paste_http = host.socket("tcp://127.0.0.1:80")
assert paste_http.is_listening
paste_https = host.socket("tcp://127.0.0.1:443")
assert paste_https.is_listening
def test_paste(host):
cmd = host.run('curl --insecure '
'--resolve paste.opendev.org:443:127.0.0.1 '
'https://paste.opendev.org')
assert 'New Paste' in cmd.stdout
def test_paste_screenshots(host):
driver = webdriver.Remote(
command_executor='http://%s:4444/wd/hub' % (host.backend.get_hostname()),
desired_capabilities=webdriver.DesiredCapabilities.FIREFOX)
try:
driver.get("https://localhost")
WebDriverWait(driver, 30).until(lambda driver: driver.execute_script(
'return document.readyState') == 'complete')
time.sleep(5)
driver.save_screenshot("/var/log/screenshots/paste-main-page.png")
except TimeoutException as e:
raise e
finally:
driver.quit()

16
zuul.d/infra-prod.yaml
View File

@ -256,6 +256,22 @@
- playbooks/roles/logrotate/
- roles/openafs-client/
- job:
name: infra-prod-service-paste
parent: infra-prod-service-base
description: Run service-paste.yaml playbook.
vars:
playbook_name: service-paste.yaml
files:
- inventory/
- playbooks/service-paste.yaml
- inventory/service/host_vars/paste01.opendev.org.yaml
- inventory/service/group_vars/paste
- playbooks/roles/install-docker/
- playbooks/roles/pip3/
- playbooks/roles/lodgeit/
- playbooks/roles/iptables/
- job:
name: infra-prod-service-static
parent: infra-prod-service-base

7
zuul.d/project.yaml
View File

@ -36,6 +36,9 @@
- system-config-run-meetpad
- system-config-run-mirror-x86
- system-config-run-mirror-update
- system-config-run-paste:
dependencies:
- name: opendev-buildset-registry
- system-config-run-static
- system-config-run-docker-registry
- system-config-run-etherpad:
@ -165,6 +168,9 @@
- system-config-run-meetpad
- system-config-run-mirror-x86
- system-config-run-mirror-update
- system-config-run-paste:
dependencies:
- name: opendev-buildset-registry
- system-config-run-static
- system-config-run-docker-registry
- system-config-run-etherpad:
@ -374,6 +380,7 @@
- infra-prod-service-kerberos
- infra-prod-service-mirror-update
- infra-prod-service-mirror
- infra-prod-service-paste
- infra-prod-service-static
- infra-prod-service-borg-backup
- infra-prod-service-zookeeper

31
zuul.d/system-config-run.yaml
View File

@ -687,6 +687,37 @@
- playbooks/zuul/templates/group_vars/meetpad.yaml.j2
- testinfra/test_meetpad.py
- job:
name: system-config-run-paste
parent: system-config-run-containers
description: |
Run the playbook for the paste server.
timeout: 3600
required-projects:
- opendev/system-config
requires:
- lodgeit-container-image
nodeset:
nodes:
- name: bridge.openstack.org
label: ubuntu-bionic
- name: paste01.opendev.org
label: ubuntu-focal
vars:
run_playbooks:
- playbooks/letsencrypt.yaml
- playbooks/service-paste.yaml
run_test_playbook: playbooks/test-paste.yaml
files:
- playbooks/bridge.yaml
- playbooks/letsencrypt.yaml
- playbooks/service-paste.yaml
- playbooks/roles/lodgeit
- playbooks/roles/install-docker/
- playbooks/roles/pip3/
- playbooks/test-paste.yaml
- testinfra/test_paste.py
- job:
name: system-config-run-zookeeper
parent: system-config-run