From f477e35561e0e9f45503cfc0ca70624d7a9d2792 Mon Sep 17 00:00:00 2001 From: Jeremy Stanley Date: Mon, 29 Jan 2024 19:23:13 +0000 Subject: [PATCH] Upgrade to Keycloak 23.0 This includes a switch from the "legacy" style Wildfly-based image to a new setup using Quarkus. Because Keycloak maintainers consider H2 databases as a test/dev only option, there are no good migration and upgrade paths short of export/import data. Go ahead and change our deployment model to rely on a proper RDBMS, run locally from a container on the same server. Change-Id: I01f8045563e9f6db6168b92c5a868b8095c0d97b --- doc/source/keycloak.rst | 9 ++-- .../keycloak.yaml} | 4 +- .../roles/keycloak/files/99-bind-address.cnf | 3 ++ playbooks/roles/keycloak/handlers/main.yaml | 10 +++- .../keycloak/handlers/restart_keycloak.yaml | 15 ++++++ playbooks/roles/keycloak/tasks/main.yaml | 34 ++++++++----- .../keycloak/templates/docker-compose.yaml.j2 | 51 +++++++++++++++---- .../keycloak/templates/keycloak.vhost.j2 | 4 +- .../handlers/main.yaml | 2 +- .../templates/group_vars/keycloak.yaml.j2 | 2 + testinfra/test_keycloak.py | 37 ++++++++++---- zuul.d/infra-prod.yaml | 3 +- zuul.d/system-config-run.yaml | 6 +-- 13 files changed, 131 insertions(+), 49 deletions(-) rename inventory/service/{host_vars/keycloak01.opendev.org.yaml => group_vars/keycloak.yaml} (72%) create mode 100644 playbooks/roles/keycloak/files/99-bind-address.cnf create mode 100644 playbooks/roles/keycloak/handlers/restart_keycloak.yaml diff --git a/doc/source/keycloak.rst b/doc/source/keycloak.rst index b14ad60dde..f703cd3049 100644 --- a/doc/source/keycloak.rst +++ b/doc/source/keycloak.rst @@ -20,13 +20,14 @@ At a Glance * :git_file:`playbooks/service-keycloak.yaml` :Projects: * https://www.keycloak.org/ - * https://github.com/keycloak/keycloak-containers + * https://github.com/keycloak/keycloak + * https://github.com/keycloak/keycloak/tree/main/quarkus/container :Bugs: * https://storyboard.openstack.org/#!/project/748 - * https://issues.jboss.org/browse/KEYCLOAK + * https://github.com/keycloak/keycloak/issues Overview ======== -Apache is configured as a reverse proxy and there is an internal H2 -database stored at ``/var/keycloak/data``. +Apache is configured as a reverse proxy to ``[::1]:8080`` and there is +also a separate MariaDB database listening on ``[::1]:3306``. diff --git a/inventory/service/host_vars/keycloak01.opendev.org.yaml b/inventory/service/group_vars/keycloak.yaml similarity index 72% rename from inventory/service/host_vars/keycloak01.opendev.org.yaml rename to inventory/service/group_vars/keycloak.yaml index 608bd8edb5..a32eeb338c 100644 --- a/inventory/service/host_vars/keycloak01.opendev.org.yaml +++ b/inventory/service/group_vars/keycloak.yaml @@ -1,6 +1,6 @@ letsencrypt_certs: - keycloak01-opendev-org-main: + keycloak-opendev-org-main: # List the service name first since that determines the filename # and is referenced in the apache config. - keycloak.opendev.org - - keycloak01.opendev.org + - "{{ inventory_hostname }}" diff --git a/playbooks/roles/keycloak/files/99-bind-address.cnf b/playbooks/roles/keycloak/files/99-bind-address.cnf new file mode 100644 index 0000000000..cfb0dcabf5 --- /dev/null +++ b/playbooks/roles/keycloak/files/99-bind-address.cnf @@ -0,0 +1,3 @@ +[mysqld] +# Only listen on the loopback address, for added safety +bind-address=::1 diff --git a/playbooks/roles/keycloak/handlers/main.yaml b/playbooks/roles/keycloak/handlers/main.yaml index 9acec0bcc8..82f0a76e6c 100644 --- a/playbooks/roles/keycloak/handlers/main.yaml +++ b/playbooks/roles/keycloak/handlers/main.yaml @@ -1,4 +1,12 @@ -- name: keycloak Reload apache2 +- name: keycloak restart apache2 + service: + name: apache2 + state: restarted + +- name: keycloak reload apache2 service: name: apache2 state: reloaded + +- name: keycloak restart containers + include_tasks: roles/keycloak/handlers/restart_keycloak.yaml diff --git a/playbooks/roles/keycloak/handlers/restart_keycloak.yaml b/playbooks/roles/keycloak/handlers/restart_keycloak.yaml new file mode 100644 index 0000000000..b2aebff4a2 --- /dev/null +++ b/playbooks/roles/keycloak/handlers/restart_keycloak.yaml @@ -0,0 +1,15 @@ +- name: keycloak check for running containers + command: pgrep -f quarkus + ignore_errors: yes + register: quarkus_pids + +- name: keycloak restart containers if running + # Also makes sure the containers weren't just restarted by an image update + when: quarkus_pids.rc == 0 and "is up-to-date" in keycloak_dcup.stderr + block: + - name: down containers + shell: + cmd: docker-compose -f /etc/keycloak-compose/docker-compose.yaml down + - name: up containers + shell: + cmd: docker-compose -f /etc/keycloak-compose/docker-compose.yaml up -d diff --git a/playbooks/roles/keycloak/tasks/main.yaml b/playbooks/roles/keycloak/tasks/main.yaml index 8492d61a45..d71188fd1d 100644 --- a/playbooks/roles/keycloak/tasks/main.yaml +++ b/playbooks/roles/keycloak/tasks/main.yaml @@ -7,22 +7,27 @@ template: src: docker-compose.yaml.j2 dest: /etc/keycloak-docker/docker-compose.yaml + owner: root + group: root + mode: "0600" + notify: keycloak restart containers +# This deliberately does not set owner/group/mode, as the mariadb container +# chowns this directory to be owned by a container-internal user and drops +# root privileges. We don't want to reset this from outside the container. - name: Ensure data directory exists file: state: directory - path: /var/keycloak/data - owner: "1000" - group: "root" - mode: "0755" + path: /var/lib/keycloak/db -- name: Ensure log directory exists - file: - state: directory - path: /var/log/keycloak - owner: "1000" - group: "root" - mode: "0755" +- name: Copy our MariaDB config stub overriding bind-address + copy: + src: 99-bind-address.cnf + dest: /var/lib/keycloak/99-bind-address.cnf + owner: root + group: root + mode: "0644" + notify: keycloak restart containers - name: Install apache2 apt: @@ -42,6 +47,7 @@ - ssl - headers - proxy_wstunnel + notify: keycloak restart apache2 - name: Copy apache config template: @@ -50,7 +56,7 @@ owner: root group: root mode: 0644 - notify: keycloak Reload apache2 + notify: keycloak reload apache2 - name: Run docker-compose pull shell: @@ -61,11 +67,13 @@ shell: cmd: docker-compose up -d chdir: /etc/keycloak-docker/ + register: keycloak_dcup - name: Wait for keycloak to start wait_for: + host: "::1" port: 8080 - timeout: 60 + timeout: 300 - name: Run docker prune to cleanup unneeded images shell: diff --git a/playbooks/roles/keycloak/templates/docker-compose.yaml.j2 b/playbooks/roles/keycloak/templates/docker-compose.yaml.j2 index 303c5d9468..3070b154d1 100644 --- a/playbooks/roles/keycloak/templates/docker-compose.yaml.j2 +++ b/playbooks/roles/keycloak/templates/docker-compose.yaml.j2 @@ -3,18 +3,47 @@ version: '2' services: - keycloak: - image: quay.io/keycloak/keycloak:legacy + mariadb: + # 10.11 was synonymous with the "lts" tag when we brought up the service + image: docker.io/library/mariadb:10.11 network_mode: host restart: always environment: - - KEYCLOAK_USER=admin - - KEYCLOAK_PASSWORD="{{ keycloak_admin_password }}" - - DB_VENDOR=h2 - - PROXY_ADDRESS_FORWARDING=true - command: - -Djboss.bind.address.private=127.0.0.1 - -Djboss.bind.address=127.0.0.1 + MARIADB_ROOT_PASSWORD: "{{ keycloak_root_db_password }}" + MARIADB_DATABASE: keycloak + MARIADB_USER: keycloak + MARIADB_PASSWORD: "{{ keycloak_db_password }}" volumes: - - /var/keycloak/data:/opt/jboss/keycloak/standalone/data - - /var/log/keycloak:/opt/jboss/keycloak/standalone/log + - /var/lib/keycloak/db:/var/lib/mysql + - /var/lib/keycloak/99-bind-address.cnf:/etc/mysql/conf.d/99-bind-address.cnf:ro + logging: + driver: syslog + options: + tag: docker-mariadb + keycloak: + depends_on: + - mariadb + image: quay.io/keycloak/keycloak:23.0 + network_mode: host + restart: always + environment: + KC_DB_PASSWORD: "{{ keycloak_db_password }}" + KC_DB_USERNAME: keycloak + KEYCLOAK_ADMIN: admin + KEYCLOAK_ADMIN_PASSWORD: "{{ keycloak_admin_password }}" + command: + - 'start' + - '--hostname-strict=false' + - '--http-enabled=true' + - '--http-host=::1' + - '--proxy=edge' + - '--db=mariadb' + # Wrap the DB host address here because it ends up inserted into a + # colon-delimited JDBC URL internally. + - '--db-url-host=[::1]' + - '--db-url-port=3306' + - '--db-url-database=keycloak' + logging: + driver: syslog + options: + tag: docker-keycloak diff --git a/playbooks/roles/keycloak/templates/keycloak.vhost.j2 b/playbooks/roles/keycloak/templates/keycloak.vhost.j2 index f88d2b7b37..dd1247caed 100644 --- a/playbooks/roles/keycloak/templates/keycloak.vhost.j2 +++ b/playbooks/roles/keycloak/templates/keycloak.vhost.j2 @@ -48,8 +48,8 @@ # https://localhost:8443/server-status RewriteRule ^/server-status$ /server-status [L] - ProxyPass / http://localhost:8080/ retry=0 - ProxyPassReverse / http://localhost:8080/ + ProxyPass / http://[::1]:8080/ retry=0 + ProxyPassReverse / http://[::1]:8080/ ProxyPreserveHost on RequestHeader set "X-Forwarded-Proto" expr=%{REQUEST_SCHEME} diff --git a/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml b/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml index fc2ceb950b..284adc4f8d 100644 --- a/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml +++ b/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml @@ -247,7 +247,7 @@ - name: letsencrypt updated etherpad-opendev-org-main include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml -- name: letsencrypt updated keycloak01-opendev-org-main +- name: letsencrypt updated keycloak-opendev-org-main include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml - name: letsencrypt updated storyboard01-opendev-org-main diff --git a/playbooks/zuul/templates/group_vars/keycloak.yaml.j2 b/playbooks/zuul/templates/group_vars/keycloak.yaml.j2 index 293aef7a95..af394d4499 100644 --- a/playbooks/zuul/templates/group_vars/keycloak.yaml.j2 +++ b/playbooks/zuul/templates/group_vars/keycloak.yaml.j2 @@ -1 +1,3 @@ keycloak_admin_password: testpassword +keycloak_root_db_password: testdbrootpass +keycloak_db_password: testdbuserpass diff --git a/testinfra/test_keycloak.py b/testinfra/test_keycloak.py index 7e63228084..01eecf4ca3 100644 --- a/testinfra/test_keycloak.py +++ b/testinfra/test_keycloak.py @@ -17,21 +17,39 @@ import json -testinfra_hosts = ['keycloak01.opendev.org'] +testinfra_hosts = ['keycloak99.opendev.org'] +def test_rdbms_listening(host): + keycloak = host.socket("tcp://::1:3306") + assert keycloak.is_listening + def test_keycloak_listening(host): - keycloak = host.socket("tcp://127.0.0.1:8080") + keycloak = host.socket("tcp://::1:8080") assert keycloak.is_listening +def test_rdbms_used(host): + # This checks that keycloak created tables in the database, + # ensuring our intended database backend is actually used. + + # The nested quotes get really ornery, so try to defuse some + # of it with a raw string included via string formatting. + query = (r'select DESCRIPTION from keycloak.KEYCLOAK_ROLE ' + 'where NAME=\\"default-roles-master\\"') + cmd = host.run( + """docker-compose -f /etc/keycloak-docker/docker-compose.yaml \ + exec -T mariadb bash -c '/usr/bin/mysql -B -p$MARIADB_PASSWORD \ + -ukeycloak -e "%s"'""" % query) + assert ("role_default-roles" in cmd.stdout) + def test_keycloak_openid_config(host): # This tests the proxy config since the output is determined by # the proxy headers and is not hard-coded configuration. cmd = host.run('curl --insecure ' - '--resolve keycloak.opendev.org:443:127.0.0.1 ' - 'https://keycloak.opendev.org/auth/realms/master' + '--resolve keycloak.opendev.org:443:[::1] ' + 'https://keycloak.opendev.org/realms/master' '/.well-known/openid-configuration') - assert ('"issuer":"https://keycloak.opendev.org/auth/realms/master"' + assert ('"issuer":"https://keycloak.opendev.org/realms/master"' in cmd.stdout) def test_keycloak_admin_api(host): @@ -39,7 +57,7 @@ def test_keycloak_admin_api(host): # acquire an OIDC bearer token and then use it to check the # user count. cmd = host.run('curl --insecure ' - '--resolve keycloak.opendev.org:443:127.0.0.1 ' + '--resolve keycloak.opendev.org:443:[::1] ' '-X POST ' '-H "Content-Type: application/x-www-form-urlencoded" ' '-d "username=admin" ' @@ -47,14 +65,13 @@ def test_keycloak_admin_api(host): '-d "grant_type=password" ' '-d "client_id=admin-cli" ' 'https://keycloak.opendev.org' - '/auth/realms/master/protocol/openid-connect/token') + '/realms/master/protocol/openid-connect/token') token = json.loads(cmd.stdout) assert token["token_type"] == "Bearer" cmd = host.run('curl --insecure ' - '--resolve keycloak.opendev.org:443:127.0.0.1 ' + '--resolve keycloak.opendev.org:443:[::1] ' '-H "Authorization: Bearer %s" ' '-H "Content-Type: application/json" ' 'https://keycloak.opendev.org' - '/auth/admin/realms/master/users/count' - % token["access_token"]) + '/admin/realms/master/users/count' % token["access_token"]) assert cmd.stdout == "1" diff --git a/zuul.d/infra-prod.yaml b/zuul.d/infra-prod.yaml index 9f038c2a7f..114f6ed546 100644 --- a/zuul.d/infra-prod.yaml +++ b/zuul.d/infra-prod.yaml @@ -208,8 +208,7 @@ files: - inventory/base - playbooks/service-keycloak.yaml - - inventory/service/host_vars/keycloak01.opendev.org.yaml - - inventory/service/group_vars/keycloak + - inventory/service/group_vars/keycloak.yaml - playbooks/roles/keycloak/ - playbooks/roles/install-docker/ - playbooks/roles/iptables/ diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml index 393c29ad95..a8af48d9ab 100644 --- a/zuul.d/system-config-run.yaml +++ b/zuul.d/system-config-run.yaml @@ -785,8 +785,8 @@ nodeset: nodes: - <<: *bridge_node_x86 - - name: keycloak01.opendev.org - label: ubuntu-focal + - name: keycloak99.opendev.org + label: ubuntu-jammy groups: - <<: *bastion_group vars: @@ -794,7 +794,7 @@ - playbooks/letsencrypt.yaml - playbooks/service-keycloak.yaml files: - - inventory/service/host_vars/keycloak01.opendev.org.yaml + - inventory/service/group_vars/keycloak.yaml - playbooks/install-ansible.yaml - playbooks/letsencrypt.yaml - playbooks/service-keycloak.yaml