diff --git a/.zuul.yaml b/.zuul.yaml index 8536103661..9c1ef1e26e 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1213,6 +1213,29 @@ - playbooks/roles/registry/ - testinfra/test_registry.py +- job: + name: system-config-run-etherpad + parent: system-config-run + description: | + Run the playbook for the etherpad servers. + timeout: 3600 + nodeset: + nodes: + - name: bridge.openstack.org + label: ubuntu-bionic + - name: etherpad01.opendev.org + label: ubuntu-bionic + vars: + run_playbooks: + - playbooks/service-letsencrypt.yaml + - playbooks/service-etherpad.yaml + files: + - playbooks/bridge.yaml + - playbooks/service-letsencrypt.yaml + - playbooks/service-etherpad.yaml + - playbooks/roles/etherpad/.* + - docker/etherpad/.* + - job: name: system-config-run-gitea parent: system-config-run-containers @@ -1593,6 +1616,21 @@ - playbooks/roles/nodepool-.* - playbooks/templates/clouds/nodepool_.* +- job: + name: infra-prod-service-etherpad + parent: infra-prod-service-base + description: Run service-etherpad.yaml playbook + vars: + playbook_name: service-etherpad.yaml + files: + - inventory/.* + - playbooks/service-etherpad.yaml + - playbooks/host_vars/etherpad01.opendev.org.yaml + - playbooks/group_vars/etherpad.* + - playbooks/roles/install-docker/.* + - playbooks/roles/etherpad.* + - playbooks/roles/logrotate.* + - job: name: infra-prod-service-meetpad parent: infra-prod-service-base @@ -1908,6 +1946,7 @@ - system-config-run-mirror-update - system-config-run-static - system-config-run-docker-registry + - system-config-run-etherpad - system-config-run-gitea: dependencies: - name: opendev-buildset-registry @@ -1965,6 +2004,7 @@ - system-config-run-mirror-update - system-config-run-static - system-config-run-docker-registry + - system-config-run-etherpad - system-config-run-gitea: dependencies: - name: opendev-buildset-registry @@ -2035,6 +2075,7 @@ - infra-prod-service-gitea-lb - infra-prod-service-nameserver - infra-prod-service-nodepool + - infra-prod-service-etherpad - infra-prod-service-meetpad - infra-prod-service-mirror-update - infra-prod-service-mirror @@ -2062,6 +2103,7 @@ - infra-prod-service-gitea-lb - infra-prod-service-nameserver - infra-prod-service-nodepool + - infra-prod-service-etherpad - infra-prod-service-meetpad - infra-prod-service-mirror-update - infra-prod-service-mirror diff --git a/inventory/groups.yaml b/inventory/groups.yaml index 8605dc8a7c..ad38472b51 100644 --- a/inventory/groups.yaml +++ b/inventory/groups.yaml @@ -63,6 +63,7 @@ groups: kubernetes: - opendev-k8s*.opendev.org letsencrypt: + - etherpad[0-9]*.opendev.org - gitea[0-9]*.opendev.org - graphite01.opendev.org - insecure-ci-registry[0-9]*.opendev.org @@ -121,8 +122,8 @@ groups: - eavesdrop[0-9]*.open*.org - elasticsearch[0-9]*.open*.org - ethercalc[0-9]*.open*.org + - etherpad[0-9]*.openstack.org - etherpad-dev[0-9]*.open*.org - - etherpad[0-9]*.open*.org - firehose[0-9]*.open*.org - grafana[0-9]*.open*.org - graphite*.open*.org @@ -166,7 +167,7 @@ groups: - eavesdrop[0-9]*.open*.org - elasticsearch[0-9]*.open*.org - ethercalc[0-9]*.open*.org - - etherpad[0-9]*.open*.org + - etherpad[0-9]*.openstack.org - etherpad-dev[0-9]*.open*.org - firehose[0-9]*.open*.org - grafana[0-9]*.open*.org diff --git a/inventory/openstack.yaml b/inventory/openstack.yaml index a16c91f0ff..284963a6c9 100644 --- a/inventory/openstack.yaml +++ b/inventory/openstack.yaml @@ -147,6 +147,13 @@ all: region_name: DFW public_v4: 23.253.253.164 public_v6: 2001:4800:7817:104:be76:4eff:fe04:66e1 + etherpad01.opendev.org: + ansible_host: 2001:4800:7818:104:be76:4eff:fe02:b0ff + location: + cloud: openstackci-rax + region_name: DFW + public_v4: 104.130.124.120 + public_v6: 2001:4800:7818:104:be76:4eff:fe02:b0ff etherpad01.openstack.org: ansible_host: 2001:4800:7817:104:be76:4eff:fe04:8f0c location: diff --git a/playbooks/host_vars/etherpad01.opendev.org.yaml b/playbooks/host_vars/etherpad01.opendev.org.yaml new file mode 100644 index 0000000000..836990ce04 --- /dev/null +++ b/playbooks/host_vars/etherpad01.opendev.org.yaml @@ -0,0 +1,7 @@ +etherpad_vhost_name: etherpad.opendev.org +letsencrypt_certs: + etherpad01-opendev-org-main: + - etherpad.opendev.org + - etherpad01.opendev.org + - etherpad.openstack.org +etherpad_redirect_vhost: etherpad.openstack.org diff --git a/playbooks/roles/etherpad/README.rst b/playbooks/roles/etherpad/README.rst new file mode 100644 index 0000000000..4c88621d6d --- /dev/null +++ b/playbooks/roles/etherpad/README.rst @@ -0,0 +1 @@ +Run an Etherpad server. diff --git a/playbooks/roles/etherpad/defaults/main.yaml b/playbooks/roles/etherpad/defaults/main.yaml new file mode 100644 index 0000000000..270441336d --- /dev/null +++ b/playbooks/roles/etherpad/defaults/main.yaml @@ -0,0 +1,2 @@ +etherpad_db_user: username +etherpad_title: OpenDev Etherpad diff --git a/playbooks/roles/etherpad/files/my.cnf b/playbooks/roles/etherpad/files/my.cnf new file mode 100644 index 0000000000..1acadc95f2 --- /dev/null +++ b/playbooks/roles/etherpad/files/my.cnf @@ -0,0 +1,15 @@ +[client] +# Default is Latin1, if you need UTF-8 set this (also in server section) +default-character-set = utf8 + +[mysqld] +wait_timeout = 28800 +# +# * Character sets +# +# Default is Latin1, if you need UTF-8 set all this (also in client section) +# +character-set-server = utf8 +collation-server = utf8_bin +character_set_server = utf8 +collation_server = utf8_bin diff --git a/playbooks/roles/etherpad/files/robots.txt b/playbooks/roles/etherpad/files/robots.txt new file mode 100644 index 0000000000..1f53798bb4 --- /dev/null +++ b/playbooks/roles/etherpad/files/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / diff --git a/playbooks/roles/etherpad/handlers/main.yaml b/playbooks/roles/etherpad/handlers/main.yaml new file mode 100644 index 0000000000..0acbc4ebae --- /dev/null +++ b/playbooks/roles/etherpad/handlers/main.yaml @@ -0,0 +1,4 @@ +- name: etherpad Reload apache2 + service: + name: apache2 + state: reloaded diff --git a/playbooks/roles/etherpad/tasks/main.yaml b/playbooks/roles/etherpad/tasks/main.yaml new file mode 100644 index 0000000000..782fd6eae4 --- /dev/null +++ b/playbooks/roles/etherpad/tasks/main.yaml @@ -0,0 +1,123 @@ +- name: Ensure docker-compose directory exists + file: + state: directory + path: /etc/etherpad-docker + +- name: Write settings file + template: + src: docker-compose.yaml.j2 + dest: /etc/etherpad-docker/docker-compose.yaml + +- name: Ensure database volume exists + file: + state: directory + path: /var/etherpad/db + +- name: Ensure config directory exists + file: + state: directory + path: /etc/etherpad + +- name: Ensure db config directory exists + file: + state: directory + path: /etc/etherpad/mysql + +- name: Install database config settings + copy: + src: my.cnf + dest: /etc/etherpad/mysql/my.cnf + +- name: Install robots.txt + copy: + src: robots.txt + dest: /var/etherpad/robots.txt + +- name: Install apache2 + apt: + name: + - apache2 + - apache2-utils + state: present + +- name: Apache modules + apache2_module: + state: present + name: "{{ item }}" + loop: + - rewrite + - proxy + - proxy_http + - ssl + - headers + - proxy_wstunnel + +- name: Copy apache config + template: + src: etherpad.vhost.j2 + dest: /etc/apache2/sites-enabled/000-default.conf + owner: root + group: root + mode: 0644 + notify: etherpad Reload apache2 + +- name: Copy redirect config + template: + src: redirect.vhost.j2 + dest: "/etc/apache2/sites-enabled/010-{{ etherpad_redirect_vhost }}.conf" + owner: root + group: root + mode: 0644 + when: etherpad_redirect_vhost is defined + notify: etherpad Reload apache2 + +- name: Write settings file + template: + src: settings.json.j2 + dest: /etc/etherpad/settings.json + +- name: Install docker-compose + package: + name: + - docker-compose + state: present + +- name: Run docker-compose pull + shell: + cmd: docker-compose pull + chdir: /etc/etherpad-docker/ + +- name: Run docker-compose up + shell: + cmd: docker-compose up -d + chdir: /etc/etherpad-docker/ + +- name: Run docker prune to cleanup unneeded images + shell: + cmd: docker image prune -f + +- name: Create db backup dest + file: + state: directory + path: /var/backups/etherpad-mariadb + mode: 0700 + owner: root + group: root + +- name: Set up cron job to backup the database + cron: + name: etherpad-db-backup + state: present + user: root + job: > + /usr/bin/docker-compose -f /etc/etherpad-docker/docker-compose.yaml exec -T mariadb + bash -c '/usr/bin/mysqldump --opt --databases etherpad-lite --single-transaction -uroot -p"$MYSQL_ROOT_PASSWORD"' | + gzip -9 > /var/backups/etherpad-mariadb/etherpad-mariadb.sql.gz + minute: 42 + hour: 4 + +- name: Rotate db backups + include_role: + name: logrotate + vars: + logrotate_file_name: /var/backups/etherpad-mariadb/etherpad-mariadb.sql.gz diff --git a/playbooks/roles/etherpad/templates/docker-compose.yaml.j2 b/playbooks/roles/etherpad/templates/docker-compose.yaml.j2 new file mode 100644 index 0000000000..db35d37859 --- /dev/null +++ b/playbooks/roles/etherpad/templates/docker-compose.yaml.j2 @@ -0,0 +1,23 @@ +# Version 2 is the latest that is supported by docker-compose in +# Ubuntu Xenial. +version: '2' + +services: + mariadb: + image: docker.io/library/mariadb:10.4 + network_mode: host + restart: always + environment: + MYSQL_ROOT_PASSWORD: "{{ etherpad_db_root_password }}" + MYSQL_DATABASE: etherpad-lite + MYSQL_USER: "{{ etherpad_db_user }}" + MYSQL_PASSWORD: "{{ etherpad_db_password }}" + volumes: + - /var/etherpad/db:/var/lib/mysql + - /etc/etherpad/mysql:/etc/mysql/conf.d + etherpad: + restart: always + image: docker.io/etherpad/etherpad:1.8.0 + network_mode: host + volumes: + - /etc/etherpad/settings.json:/opt/etherpad-lite/settings.json diff --git a/playbooks/roles/etherpad/templates/etherpad.vhost.j2 b/playbooks/roles/etherpad/templates/etherpad.vhost.j2 new file mode 100644 index 0000000000..f2728deb82 --- /dev/null +++ b/playbooks/roles/etherpad/templates/etherpad.vhost.j2 @@ -0,0 +1,99 @@ + + ServerName {{ etherpad_vhost_name }} + ServerAdmin webmaster@openstack.org + + ErrorLog ${APACHE_LOG_DIR}/gerrit-error.log + + LogLevel warn + + CustomLog ${APACHE_LOG_DIR}/gerrit-access.log combined + + Redirect / https://{{ etherpad_vhost_name }}/ + + + + + + ServerName {{ etherpad_vhost_name }} + ServerAdmin webmaster@openstack.org + + AllowEncodedSlashes On + + ErrorLog ${APACHE_LOG_DIR}/etherpad-ssl-error.log + + LogLevel warn + + CustomLog ${APACHE_LOG_DIR}/etherpad-ssl-access.log combined + + SSLEngine on + SSLProtocol All -SSLv2 -SSLv3 + # Note: this list should ensure ciphers that provide forward secrecy + SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP + SSLHonorCipherOrder on + + SSLCertificateFile /etc/letsencrypt-certs/{{ etherpad_vhost_name }}/{{ etherpad_vhost_name }}.cer + SSLCertificateKeyFile /etc/letsencrypt-certs/{{ etherpad_vhost_name }}/{{ etherpad_vhost_name }}.key + SSLCertificateChainFile /etc/letsencrypt-certs/{{ etherpad_vhost_name }}/ca.cer + + BrowserMatch "MSIE [2-6]" \ + nokeepalive ssl-unclean-shutdown \ + downgrade-1.0 force-response-1.0 + # MSIE 7 and newer should be able to use keepalive + BrowserMatch "MSIE [17-9]" ssl-unclean-shutdown + + + # The following redirects "nice" urls such as https://etherpad.example.org/padname + # to https://etherpad.example.org/p/padname. It was problematic directly + # supporting "nice" urls as etherpad hardcodes /p/ in many places. + # Adapted from https://github.com/ether/etherpad-lite/wiki/How-to-put-Etherpad-Lite-behind-a-reverse-Proxy + RewriteEngine on + + # Do not rewrite the /server-status URL (though by default, this + # is only accessible from localhost). Connect to it with: + # ssh -L 8443:localhost:443 $HOSTNAME + # https://localhost:8443/server-status + RewriteRule ^/server-status$ /server-status [L] + + RewriteCond %{HTTP_HOST} !{{ etherpad_vhost_name }} + RewriteRule ^.*$ https://{{ etherpad_vhost_name }} [L,R=301] + + # Server robots.txt directly so that it does not affect + # etherpad-lite installation. + RewriteRule ^/robots.txt$ /var/etherpad/robots.txt [L] + + # Refuse external connections to the API through the proxy + RewriteRule ^/api/ - [F,L] + + RewriteCond %{REQUEST_URI} !^/p/ + RewriteCond %{REQUEST_URI} !^/locales/ + RewriteCond %{REQUEST_URI} !^/locales.json + RewriteCond %{REQUEST_URI} !^/admin + RewriteCond %{REQUEST_URI} !^/p/ + RewriteCond %{REQUEST_URI} !^/static/ + RewriteCond %{REQUEST_URI} !^/pluginfw/ + RewriteCond %{REQUEST_URI} !^/javascripts/ + RewriteCond %{REQUEST_URI} !^/socket.io/ + RewriteCond %{REQUEST_URI} !^/ep/ + RewriteCond %{REQUEST_URI} !^/minified/ + RewriteCond %{REQUEST_URI} !^/api/ + RewriteCond %{REQUEST_URI} !^/ro/ + RewriteCond %{REQUEST_URI} !^/error/ + RewriteCond %{REQUEST_URI} !^/jserror + RewriteCond %{REQUEST_URI} !/favicon.ico + RewriteCond %{REQUEST_URI} !/robots.txt + RewriteRule ^/+(.+)$ https://{{ etherpad_vhost_name }}/p/$1 [NC,L,R=301] + + + RewriteCond %{REQUEST_URI} ^/socket.io [NC] + RewriteCond %{QUERY_STRING} transport=websocket [NC] + RewriteRule /(.*) ws://localhost:9001/$1 [P,L] + ProxyPass /socket.io http://localhost:9001/socket.io retry=0 + ProxyPassReverse /socket.io http://localhost:9001/socket.io + + + ProxyPass / http://localhost:9001/ retry=0 + ProxyPassReverse / http://localhost:9001/ + + + + diff --git a/playbooks/roles/etherpad/templates/redirect.vhost.j2 b/playbooks/roles/etherpad/templates/redirect.vhost.j2 new file mode 100644 index 0000000000..c629608bec --- /dev/null +++ b/playbooks/roles/etherpad/templates/redirect.vhost.j2 @@ -0,0 +1,37 @@ +# ************************************ +# Managed by Ansible +# ************************************ + + + ServerName {{ etherpad_redirect_vhost }} + + LogLevel warn + ErrorLog /var/log/apache2/{{ etherpad_redirect_vhost }}_error.log + CustomLog /var/log/apache2/{{ etherpad_redirect_vhost }}_access.log combined + ServerSignature Off + + Redirect / https://{{ etherpad_vhost_name }}/ + + + + + ServerName {{ etherpad_redirect_vhost }} + + SSLEngine on + SSLProtocol All -SSLv2 -SSLv3 + # Note: this list should ensure ciphers that provide forward secrecy + SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP + SSLHonorCipherOrder on + + SSLCertificateFile /etc/letsencrypt-certs/{{ etherpad_vhost_name }}/{{ etherpad_vhost_name }}.cer + SSLCertificateKeyFile /etc/letsencrypt-certs/{{ etherpad_vhost_name }}/{{ etherpad_vhost_name }}.key + SSLCertificateChainFile /etc/letsencrypt-certs/{{ etherpad_vhost_name }}/ca.cer + + LogLevel warn + ErrorLog /var/log/apache2/{{ etherpad_redirect_vhost }}_error.log + CustomLog /var/log/apache2/{{ etherpad_redirect_vhost }}_access.log combined + ServerSignature Off + + Redirect / https://{{ etherpad_vhost_name }}/ + + diff --git a/playbooks/roles/etherpad/templates/settings.json.j2 b/playbooks/roles/etherpad/templates/settings.json.j2 new file mode 100644 index 0000000000..cfdbc01bb4 --- /dev/null +++ b/playbooks/roles/etherpad/templates/settings.json.j2 @@ -0,0 +1,120 @@ +/* + This file must be valid JSON. But comments are allowed + + Please edit settings.json, not settings.json.template +*/ +{ + // Name your instance! + "title": "{{ etherpad_title }}", + + //Ip and port which etherpad should bind at + "ip": "127.0.0.1", + "port" : 9001, + + // favicon default name + "favicon": "favicon.ico", + + // Session Key, used for reconnecting user sessions + // Set this to a secure string at least 10 characters long. Do not share this value. + "sessionKey" : "{{ etherpad_session_key }}", + + //The Type of the database. You can choose between dirty, sqlite and mysql + //You should use mysql or sqlite for anything else than testing or development + "dbType" : "mysql", + //the database specific settings + "dbSettings" : { + "user" : "{{ etherpad_db_user }}", + "host" : "localhost", + "password": "{{ etherpad_db_password }}", + "database": "etherpad-lite" + }, + //the default text of a pad + "defaultPadText" : "Welcome to Etherpad Lite!\n\nThis pad text is synchronized as you type, so that everyone viewing this page sees the same text. This allows you to collaborate seamlessly on documents!\n\nEtherpad Lite on Github: http:\/\/j.mp/ep-lite\n", + + /* Users must have a session to access pads. This effectively allows only group pads to be accessed. */ + "requireSession" : false, + + /* Users may edit pads but not create new ones. Pad creation is only via the API. This applies both to group pads and regular pads. */ + "editOnly" : false, + + /* if true, all css & js will be minified before sending to the client. This will improve the loading performance massivly, + but makes it impossible to debug the javascript/css */ + "minify" : true, + + /* How long may clients use served javascript code? Without versioning this + is may cause problems during deployment. */ + "maxAge" : 21600000, // 6 hours + + /* This is the path to the Abiword executable. Setting it to null, disables abiword. + Abiword is needed to enable the import/export of pads*/ + "abiword" : null, + + /* This setting is used if you require authentication of all users. + Note: /admin always requires authentication. */ + "requireAuthentication": false, + + /* Require authorization by a module, or a user with is_admin set, see below. */ + "requireAuthorization": false, + + /* Users for basic authentication. is_admin = true gives access to /admin. + If you do not uncomment this, /admin will not be available! */ + /* + "users": { + "admin": { + "password": "changeme1", + "is_admin": true + }, + "user": { + "password": "changeme1", + "is_admin": false + } + }, + */ + + // restrict socket.io transport methods + "socketTransportProtocols" : ["websocket", "xhr-polling", "jsonp-polling", "htmlfile"], + + /* The log level we are using, can be: DEBUG, INFO, WARN, ERROR */ + "loglevel": "INFO", + + //Logging configuration. See log4js documentation for further information + // https://github.com/nomiddlename/log4js-node + // You can add as many appenders as you want here: + "logconfig" : + { "appenders": [ + { "type": "console" + //, "category": "access"// only logs pad access + } + /* + , { "type": "file" + , "filename": "/var/log/eplite/etherpad-lite.log" + , "maxLogSize": 1024 + , "backups": 30 // how many log files there're gonna be at max + //, "category": "test" // only log a specific category + }*/ + /* + , { "type": "logLevelFilter" + , "level": "warn" // filters out all log messages that have a lower level than "error" + , "appender": + { Use whatever appender you want here } + }*/ + /* + , { "type": "logLevelFilter" + , "level": "error" // filters out all log messages that have a lower level than "error" + , "appender": + { "type": "smtp" + , "subject": "An error occured in your EPL instance!" + , "recipients": "bar@blurdybloop.com, baz@blurdybloop.com" + , "sendInterval": 60*5 // in secs -- will buffer log messages; set to 0 to send a mail for every message + , "transport": "SMTP", "SMTP": { // see https://github.com/andris9/Nodemailer#possible-transport-methods + "host": "smtp.example.com", "port": 465, + "secureConnection": true, + "auth": { + "user": "foo@example.com", + "pass": "bar_foo" + } + } + } + }*/ + ] } +} diff --git a/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml b/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml index c84e847705..ff2c9e8dad 100644 --- a/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml +++ b/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml @@ -185,6 +185,9 @@ - name: letsencrypt updated nb01-test-main include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml +- name: letsencrypt updated etherpad01-opendev-org-main + include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml + # We split out handlers for each gitea host as handlers should be run in order # This allows us to do a rolling restart of the gitea backends. - name: letsencrypt updated gitea01-main diff --git a/playbooks/service-etherpad.yaml b/playbooks/service-etherpad.yaml new file mode 100644 index 0000000000..3df4f10395 --- /dev/null +++ b/playbooks/service-etherpad.yaml @@ -0,0 +1,5 @@ +- hosts: "etherpad:!disabled" + name: "Base: configure etherpad" + roles: + - install-docker + - etherpad diff --git a/playbooks/zuul/run-base.yaml b/playbooks/zuul/run-base.yaml index 0641842c35..482f415461 100644 --- a/playbooks/zuul/run-base.yaml +++ b/playbooks/zuul/run-base.yaml @@ -85,6 +85,7 @@ - group_vars/control-plane-clouds.yaml - group_vars/afs-client.yaml - host_vars/bridge.openstack.org.yaml + - host_vars/etherpad01.opendev.org.yaml - host_vars/letsencrypt01.opendev.org.yaml - host_vars/letsencrypt02.opendev.org.yaml - host_vars/gitea99.opendev.org.yaml diff --git a/playbooks/zuul/templates/host_vars/etherpad01.opendev.org.yaml.j2 b/playbooks/zuul/templates/host_vars/etherpad01.opendev.org.yaml.j2 new file mode 100644 index 0000000000..c005479c88 --- /dev/null +++ b/playbooks/zuul/templates/host_vars/etherpad01.opendev.org.yaml.j2 @@ -0,0 +1,4 @@ +etherpad_db_root_password: rootpassword +etherpad_db_password: password +# Must be 14 characters +etherpad_session_key: 'R382T#^XDcNYmg'