Merge "Update etherpad to v2.0.3"

doc/source/etherpad.rst

@@ -39,19 +39,34 @@ Manual Administrative Tasks
 The following sections describe tasks that individuals with root
 access may need to perform on rare occasions.
 
+All interaction with the Etherpad admin API requires retrieving a bearer
+token for authorization and authentication to the API. The secret used
+to authenticate when requesting a token is found in the Etherpad
+settings.json file under the ``client_secret`` key::
+
+  grep '"client_secret":' /etc/etherpad/settings.json | \
+    sed -e 's/\s\+"client_secret": "\(.*\)",/\1/'
+
+With this secret we can make a request to get a token::
+
+  curl --data grant_type=client_credentials \
+       --data client_id=api_admin \
+       --data client_secret="SECRETHERE" \
+       http://localhost:9001/oidc/token
+
+This will return a Bearer token that is valid for one hour. You will need
+this token to perform the actions described below.
+
 Deleting a Pad
 --------------
 
 On occasion it may be necessary to delete a pad, so as to redact
 sensitive or illegal data posted to it (the revision history it keeps
 makes this harder than just clearing the current contents through a
-browser). This is fairly easily accomplished via the `HTTP API`_, but
+browser). This is fairly easily accomplished via the `HTTP API`_::
-you need the key which is saved in a file on the server so it's easiest
-if done when SSH'd into it locally::
 
-  wget -qO- "http://localhost:9001/api/1/deletePad?apikey=$(sudo \
+  curl -H "Authorization: Bearer TOKENFROMABOVEPROCESS" \
-  docker-compose -f /etc/etherpad-docker/docker-compose.yaml exec etherpad \
+    "http://localhost:9001/api/1/deletePad?padID=XXXXXXXXXX" ; echo
-  cat /opt/etherpad-lite/APIKEY.txt)&padID=XXXXXXXXXX" ; echo
 
 ...where XXXXXXXXXX is the pad's name as it appears at the end of its
 URL (the trailing echo is just because the API response doesn't end with
@@ -75,6 +90,6 @@ on it, not being aware that this changes the document for everyone
 instead of just locally. Via the revision slider you can identify the
 last good version and then restore it via the API::
 
-  wget -qO- "http://localhost:9001/api/1.2.11/restoreRevision?apikey=$(sudo \
+  curl -H "Authorization: Bearer TOKENFROMABOVEPROCESS" \
-  docker-compose -f /etc/etherpad-docker/docker-compose.yaml exec etherpad \
+    "http://localhost:9001/api/1.2.11/restoreRevision?padID=XXXXXXXXXX&rev=NNN" \
-  cat /opt/etherpad-lite/APIKEY.txt)&padID=XXXXXXXXXX&rev=NNN" ; echo
+    ; echo

docker/etherpad/Dockerfile

@@ -22,15 +22,48 @@
 #
 # Author: muxator
 
-FROM node:20-bookworm-slim
+# We set defaults here so that we can make use of them in different
+# stages of the multi stage build.
+ARG EP_DIR=/opt/etherpad-lite
+ARG SETTINGS=./settings.json.docker
+ARG ETHERPAD_PLUGINS="ep_headings"
+
+FROM node:20-bookworm-slim as adminBuild
+ARG EP_DIR
+WORKDIR "${EP_DIR}"
+
+RUN export DEBIAN_FRONTEND=noninteractive; \
+    apt-get -qq update && \
+    apt-get -qq dist-upgrade && \
+    apt-get -qq --no-install-recommends install ca-certificates git && \
+    apt-get -qq clean && \
+    rm -rf /var/lib/apt/lists/*
+RUN git clone https://github.com/ether/etherpad-lite ${EP_DIR}
+RUN git checkout v2.0.3
+RUN cd ./admin && npm install -g pnpm && pnpm install && pnpm run build --outDir ./dist
+RUN cd ./ui && pnpm install && pnpm run build --outDir ./dist
+
+
+FROM node:20-bookworm-slim as build
 LABEL maintainer="infra-root@openstack.org"
 
+# Set these arguments when building the image from behind a proxy
+ARG http_proxy=
+ARG https_proxy=
+ARG no_proxy=
+
 ARG TIMEZONE=
+
 RUN \
   [ -z "${TIMEZONE}" ] || { \
     ln -sf /usr/share/zoneinfo/"${TIMEZONE#/usr/share/zoneinfo/}" /etc/localtime; \
     dpkg-reconfigure -f noninteractive tzdata; \
   }
+ENV TIMEZONE=${TIMEZONE}
+
+# Control the configuration file to be copied into the container.
+# We bind mount over this file
+ARG SETTINGS
 
 # plugins to install while building the container. By default no plugins are
 # installed.
@@ -38,7 +71,15 @@ RUN \
 #
 # EXAMPLE:
 #   ETHERPAD_PLUGINS="ep_codepad ep_author_neat"
-ARG ETHERPAD_PLUGINS="ep_headings"
+ARG ETHERPAD_PLUGINS
+
+# local plugins to install while building the container. By default no plugins are
+# installed.
+# If given a value, it has to be a space-separated, quoted list of plugin names.
+#
+# EXAMPLE:
+#   ETHERPAD_LOCAL_PLUGINS="../ep_my_plugin ../ep_another_plugin"
+ARG ETHERPAD_LOCAL_PLUGINS=
 
 # Control whether abiword will be installed, enabling exports to DOC/PDF/ODT formats.
 # By default, it is not installed.
@@ -56,12 +97,6 @@ ARG INSTALL_ABIWORD=
 #   INSTALL_LIBREOFFICE=true
 ARG INSTALL_SOFFICE=
 
-# By default, Etherpad container is built and run in "production" mode. This is
-# leaner (development dependencies are not installed) and runs faster (among
-# other things, assets are minified & compressed).
-ENV NODE_ENV=production
-ENV ETHERPAD_PRODUCTION=true
-
 # Follow the principle of least privilege: run as unprivileged user.
 #
 # Running as non-root enables running this image in platforms like OpenShift
@@ -73,25 +108,26 @@ ARG EP_HOME=
 ARG EP_UID=5001
 ARG EP_GID=0
 ARG EP_SHELL=
+
 RUN groupadd --system ${EP_GID:+--gid "${EP_GID}" --non-unique} etherpad && \
     useradd --system ${EP_UID:+--uid "${EP_UID}" --non-unique} --gid etherpad \
         ${EP_HOME:+--home-dir "${EP_HOME}"} --create-home \
         ${EP_SHELL:+--shell "${EP_SHELL}"} etherpad
 
-ARG EP_DIR=/opt/etherpad-lite
+ARG EP_DIR
 RUN mkdir -p "${EP_DIR}" && chown etherpad:etherpad "${EP_DIR}"
 
 # the mkdir is needed for configuration of openjdk-11-jre-headless, see
 # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199
 RUN export DEBIAN_FRONTEND=noninteractive; \
     mkdir -p /usr/share/man/man1 && \
-    npm install npm@6 -g  && \
+    npm install pnpm -g  && \
     apt-get -qq update && \
     apt-get -qq dist-upgrade && \
     apt-get -qq --no-install-recommends install \
         ca-certificates \
-        git \
         curl \
+        git \
         ${INSTALL_ABIWORD:+abiword} \
         ${INSTALL_SOFFICE:+libreoffice} \
         && \
@@ -102,28 +138,47 @@ USER etherpad
 
 RUN git clone https://github.com/ether/etherpad-lite ${EP_DIR}
 WORKDIR "${EP_DIR}"
-RUN git checkout v1.9.7
+RUN git checkout v2.0.3
+
+FROM build as development
+ARG ETHERPAD_PLUGINS
+
+COPY --chown=etherpad:etherpad --from=adminBuild /opt/etherpad-lite/admin/dist ./src/templates/admin
+COPY --chown=etherpad:etherpad --from=adminBuild /opt/etherpad-lite/ui/dist ./src/static/oidc
+
+RUN bin/installDeps.sh && \
+    if [ ! -z "${ETHERPAD_PLUGINS}" ] || [ ! -z "${ETHERPAD_LOCAL_PLUGINS}" ]; then \
+        pnpm run install-plugins ${ETHERPAD_PLUGINS} ${ETHERPAD_LOCAL_PLUGINS:+--path ${ETHERPAD_LOCAL_PLUGINS}}; \
+    fi
+
+
+FROM build as production
+ARG EP_DIR
+ARG SETTINGS
+ARG ETHERPAD_PLUGINS
+
+ENV NODE_ENV=production
+ENV ETHERPAD_PRODUCTION=true
+
+COPY --chown=etherpad:etherpad --from=adminBuild /opt/etherpad-lite/admin/dist ./src/templates/admin
+COPY --chown=etherpad:etherpad --from=adminBuild /opt/etherpad-lite/ui/dist ./src/static/oidc
+
+RUN bin/installDeps.sh && rm -rf ~/.npm && rm -rf ~/.local && rm -rf ~/.cache && \
+    if [ ! -z "${ETHERPAD_PLUGINS}" ] || [ ! -z "${ETHERPAD_LOCAL_PLUGINS}" ]; then \
+        pnpm run install-plugins ${ETHERPAD_PLUGINS} ${ETHERPAD_LOCAL_PLUGINS:+--path ${ETHERPAD_LOCAL_PLUGINS}}; \
+    fi
 
-# Plugins must be installed before installing Etherpad's dependencies, otherwise
-# npm will try to hoist common dependencies by removing them from
-# src/node_modules and installing them in the top-level node_modules. As of
-# v6.14.10, npm's hoist logic appears to be buggy, because it sometimes removes
-# dependencies from src/node_modules but fails to add them to the top-level
-# node_modules. Even if npm correctly hoists the dependencies, the hoisting
-# seems to confuse tools such as `npm outdated`, `npm update`, and some ESLint
-# rules.
-RUN { [ -z "${ETHERPAD_PLUGINS}" ] || \
-      npm install --no-save ${ETHERPAD_PLUGINS}; } && \
-    src/bin/installDeps.sh && \
-    rm -rf ~/.npm
 
 # Copy the configuration file.
-COPY --chown=etherpad:etherpad ./settings.json.docker "${EP_DIR}"/settings.json
+COPY --chown=etherpad:etherpad ${SETTINGS} "${EP_DIR}"/settings.json
 
 # Fix group permissions
 RUN chmod -R g=u .
 
-HEALTHCHECK --interval=20s --timeout=3s CMD curl -f http://localhost:9001 || exit 1
+USER etherpad
+
+HEALTHCHECK --interval=5s --timeout=3s \
+  CMD curl --silent http://localhost:9001/health | grep -E "pass|ok|up" > /dev/null || exit 1
 
 EXPOSE 9001
-CMD ["node", "src/node/server.js"]
+CMD ["pnpm", "run", "prod"]

docker/etherpad/settings.json.docker

@@ -527,7 +527,7 @@
   /*
    * Restrict socket.io transport methods
    */
-  "socketTransportProtocols" : ["xhr-polling", "jsonp-polling", "htmlfile"],
+  "socketTransportProtocols" : ["websocket", "polling"],
 
   "socketIo": {
     /*
@@ -650,5 +650,24 @@
   /*
    * Enable/Disable case-insensitive pad names.
    */
-  "lowerCasePadIds": "${LOWER_CASE_PAD_IDS:false}"
+  "lowerCasePadIds": "${LOWER_CASE_PAD_IDS:false}",
+  "sso": {
+    "issuer": "${SSO_ISSUER:http://localhost:9001}",
+      "clients": [
+        {
+          "client_id": "${ADMIN_CLIENT:admin_client}",
+          "client_secret": "${ADMIN_SECRET:admin}",
+          "grant_types": ["authorization_code"],
+          "response_types": ["code"],
+          "redirect_uris": ["${ADMIN_REDIRECT:http://localhost:9001/admin/}"]
+        },
+        {
+          "client_id": "${USER_CLIENT:user_client}",
+          "client_secret": "${USER_SECRET:user}",
+          "grant_types": ["authorization_code"],
+          "response_types": ["code"],
+          "redirect_uris": ["${USER_REDIRECT:http://localhost:9001/}"]
+        }
+      ]
+    }
 }

playbooks/roles/etherpad/templates/settings.json.j2

@@ -537,7 +537,7 @@
    * Restrict socket.io transport methods
    */
   // OpenDev: Add websocket for our Jitsi-Meet integration.
-  "socketTransportProtocols" : ["websocket", "xhr-polling", "jsonp-polling", "htmlfile"],
+  "socketTransportProtocols" : ["websocket", "polling"],
 
   "socketIo": {
     /*
@@ -660,5 +660,23 @@
   /*
    * Enable/Disable case-insensitive pad names.
    */
-  "lowerCasePadIds": false
+  "lowerCasePadIds": false,
+  "sso": {
+    "issuer": "${SSO_ISSUER:http://localhost:9001}",
+      "clients": [
+        {
+          "client_id": "api_admin",
+          "client_secret": "{{ etherpad_admin_api_secret }}",
+          "grant_types": ["client_credentials"],
+          "response_types": [],
+          "redirect_uris": [],
+          "extraParams": [
+            {
+              "name": "admin",
+              "value": "true"
+            }
+          ]
+        }
+      ]
+    }
 }

playbooks/zuul/templates/host_vars/etherpad99.opendev.org.yaml.j2

@@ -1,2 +1,3 @@
 etherpad_db_root_password: rootpassword
 etherpad_db_password: password
+etherpad_admin_api_secret: YH0FIYlSeTuMab3DKmfCs3Vlb5w9SuU63dGyJW5HZHE371YJ

testinfra/test_etherpad.py

@@ -12,6 +12,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import json
 import urllib.parse
 
 testinfra_hosts = ['etherpad99.opendev.org']
@@ -36,6 +37,19 @@ def test_etherpad_logs(host):
     assert mariadb_log_file.exists
     assert mariadb_log_file.contains('mariadbd: ready for connections.')
 
+def _get_bearer_token(host):
+    # Get a Beaer token via Oauth2 negotiation for API interaction
+    cmd = host.run('grep \'"client_secret":\' /etc/etherpad/settings.json |'
+                   ' sed -e \'s/\\s\\+"client_secret": "\\(.*\\)",/\\1/\'')
+    secret = cmd.stdout.strip()
+    cmd = host.run('curl --data grant_type=client_credentials '
+                   '--data client_id=api_admin --data client_secret="%s" '
+                   'http://localhost:9001/oidc/token' % secret)
+    ret = json.loads(cmd.stdout)
+    assert ret['token_type'] == 'Bearer'
+    token = ret['access_token']
+    return token
+
 def test_etherpad_4_byte_utf8(host):
     # The 🖖 utf8 character is a four byte character. This test ensures we
     # can set that value in a pad and get it back out again. This test both
@@ -43,18 +57,102 @@ def test_etherpad_4_byte_utf8(host):
     # utf8 chars
     teststr = '🖖 Live long and prosper 🖖'
     urlstr = urllib.parse.quote(teststr)
-    cmd = host.run('sudo docker-compose -f '
+
-                   '/etc/etherpad-docker/docker-compose.yaml '
+    token = _get_bearer_token(host)
-                   'exec -T etherpad cat /opt/etherpad-lite/APIKEY.txt')
+
-    token = cmd.stdout.strip()
+    # Test the API works and that 4 byte chars are happy.
-    cmd = host.run('wget -qO- "http://localhost:9001/api/1/createPad?'
+    cmd = host.run('curl -H "Authorization: Bearer %s" '
-                   'apikey=%s&padID=testing"' % token)
+                   '"http://localhost:9001/api/1/createPad?'
+                   'padID=testing"' % token)
     assert '"code":0' in cmd.stdout
     assert '"message":"ok"' in cmd.stdout
-    cmd = host.run('wget -qO- "http://localhost:9001/api/1/setText?'
+    cmd = host.run('curl -H "Authorization: Bearer %s" '
-                   'apikey=%s&padID=testing&text=%s"' % (token, urlstr))
+                   '"http://localhost:9001/api/1/setText?'
+                   'padID=testing&text=%s"' % (token, urlstr))
+    assert '"code":0' in cmd.stdout
+    assert '"message":"ok"' in cmd.stdout
+    cmd = host.run('curl -H "Authorization: Bearer %s" '
+                   '"http://localhost:9001/api/1/getText?'
+                   'padID=testing"' % token)
     assert '"code":0' in cmd.stdout
     assert '"message":"ok"' in cmd.stdout
-    cmd = host.run('wget -qO- "http://localhost:9001/api/1/getText?'
-                   'apikey=%s&padID=testing"' % token)
     assert teststr in cmd.stdout
+
+def test_etherpad_delete_pad(host):
+    urlstr = urllib.parse.quote('test pad delete string')
+    token = _get_bearer_token(host)
+
+    cmd = host.run('curl -H "Authorization: Bearer %s" '
+                   '"http://localhost:9001/api/1/createPad?'
+                   'padID=testing_delete"' % token)
+    assert '"code":0' in cmd.stdout
+    assert '"message":"ok"' in cmd.stdout
+    cmd = host.run('curl -H "Authorization: Bearer %s" '
+                   '"http://localhost:9001/api/1/setText?'
+                   'padID=testing_delete&text=%s"' % (token, urlstr))
+    assert '"code":0' in cmd.stdout
+    assert '"message":"ok"' in cmd.stdout
+    cmd = host.run('curl -H "Authorization: Bearer %s" '
+                   '"http://localhost:9001/api/1.2.1/listAllPads"' % token)
+    assert '"code":0' in cmd.stdout
+    assert '"message":"ok"' in cmd.stdout
+    pads = json.loads(cmd.stdout)['data']['padIDs']
+    assert 'testing_delete' in pads
+    cmd = host.run('curl -H "Authorization: Bearer %s" '
+                   '"http://localhost:9001/api/1/deletePad?'
+                   'padID=testing_delete"' % token)
+    assert '"code":0' in cmd.stdout
+    assert '"message":"ok"' in cmd.stdout
+    cmd = host.run('curl -H "Authorization: Bearer %s" '
+                   '"http://localhost:9001/api/1.2.1/listAllPads"' % token)
+    assert '"code":0' in cmd.stdout
+    assert '"message":"ok"' in cmd.stdout
+    pads = json.loads(cmd.stdout)['data']['padIDs']
+    assert 'testing_delete' not in pads
+
+def test_etherpad_restore_pad(host):
+    firststr = urllib.parse.quote('first')
+    secondstr = urllib.parse.quote('second')
+    token = _get_bearer_token(host)
+
+    cmd = host.run('curl -H "Authorization: Bearer %s" '
+                   '"http://localhost:9001/api/1/createPad?'
+                   'padID=testing_restore"' % token)
+    assert '"code":0' in cmd.stdout
+    assert '"message":"ok"' in cmd.stdout
+    cmd = host.run('curl -H "Authorization: Bearer %s" '
+                   '"http://localhost:9001/api/1/setText?'
+                   'padID=testing_restore&text=%s"' % (token, firststr))
+    assert '"code":0' in cmd.stdout
+    assert '"message":"ok"' in cmd.stdout
+    cmd = host.run('curl -H "Authorization: Bearer %s" '
+                   '"http://localhost:9001/api/1/setText?'
+                   'padID=testing_restore&text=%s"' % (token, secondstr))
+    assert '"code":0' in cmd.stdout
+    assert '"message":"ok"' in cmd.stdout
+    cmd = host.run('curl -H "Authorization: Bearer %s" '
+                   '"http://localhost:9001/api/1/getText?'
+                   'padID=testing_restore"' % token)
+    assert '"code":0' in cmd.stdout
+    assert '"message":"ok"' in cmd.stdout
+    assert firststr not in cmd.stdout
+    assert secondstr in cmd.stdout
+    cmd = host.run('curl -H "Authorization: Bearer %s" '
+                   '"http://localhost:9001/api/1/getRevisionsCount?'
+                   'padID=testing_restore"' % token)
+    assert '"code":0' in cmd.stdout
+    assert '"message":"ok"' in cmd.stdout
+    revisions = json.loads(cmd.stdout)['data']['revisions']
+    old_rev = urllib.parse.quote(str(revisions - 1))
+    cmd = host.run('curl -H "Authorization: Bearer %s" '
+                   '"http://localhost:9001/api/1.2.11/restoreRevision?'
+                   'padID=testing_restore&rev=%s"' % (token, old_rev))
+    assert '"code":0' in cmd.stdout
+    assert '"message":"ok"' in cmd.stdout
+    cmd = host.run('curl -H "Authorization: Bearer %s" '
+                   '"http://localhost:9001/api/1/getText?'
+                   'padID=testing_restore"' % token)
+    assert '"code":0' in cmd.stdout
+    assert '"message":"ok"' in cmd.stdout
+    assert firststr in cmd.stdout
+    assert secondstr not in cmd.stdout

zuul.d/docker-images/etherpad.yaml

@@ -7,6 +7,7 @@
     vars: &etherpad_vars
       docker_images:
         - context: docker/etherpad
+          target: production
           repository: opendevorg/etherpad
     files: &etherpad_files
       - docker/etherpad/