Fork the maxking/docker-mailman images

These images have a number of issues we've identified and worked
around. The current iteration of this change is essentially
identical to upstream but with a minor tweak to allow the latest
mailman version, and adjusts the paths for hyperkitty and postorius
URLs to match those in the upstream mailman-web codebase, but
doesn't try to address the other items. However, we should consider
moving our fixes from ansible into the docker images where possible
and upstream those updates.

Unfortunately upstream hasn't been super responsive so far hence this
fork. For tracking purposes here are the issues/PRs we've already filed
upstream:

  https://github.com/maxking/docker-mailman/pull/552
  https://github.com/maxking/docker-mailman/issues/548
  https://github.com/maxking/docker-mailman/issues/549
  https://github.com/maxking/docker-mailman/issues/550

Change-Id: I3314037d46c2ef2086a06dea0321d9f8cdd35c73

docker/mailman/core/Dockerfile

@@ -0,0 +1,37 @@
+# syntax = docker/dockerfile:1.3
+# Use 3.15 for Core since it has Python 3.9
+FROM alpine:3.15
+
+#Add startup script to container
+COPY docker-entrypoint.sh /usr/local/bin/
+
+# Add requirements file.
+COPY requirements.txt /tmp/
+
+#Install all required packages, add user for executing mailman and set execution rights for startup script
+RUN --mount=type=cache,target=/root/.cache \
+    apk update \
+    && apk add --virtual build-deps gcc python3-dev musl-dev postgresql-dev \
+       libffi-dev \
+  # Mailman html to plaintext conversion uses lynx.
+  # psutil needs linux-headers to compile on musl c library.
+    && apk add --no-cache bash su-exec postgresql-client mysql-client curl python3 py3-pip linux-headers py-cryptography mariadb-connector-c lynx \
+    && python3 -m pip install -U pip setuptools wheel \
+        && python3 -m pip install psycopg2 \
+                   gunicorn==19.9.0 \
+                   pymysql \
+                   sqlalchemy \
+                   -r /tmp/requirements.txt \
+    && apk del build-deps \
+    && adduser -S mailman
+
+# Change the working directory.
+WORKDIR /opt/mailman
+
+#Expose the ports for the api (8001) and lmtp (8024)
+EXPOSE 8001 8024
+
+ENV MAILMAN_CONFIG_FILE /etc/mailman.cfg
+
+ENTRYPOINT ["docker-entrypoint.sh"]
+CMD ["master", "--force"]

docker/mailman/core/README.md

@@ -0,0 +1,6 @@
+This is forked from https://github.com/maxking/docker-mailman/tree/main/core
+and you should refer to that repo for information. The upstream repo is MIT
+licensed.
+
+The contents in this dir are based on commit
+22b3b7d4024a00e9896837fa04883cbaeef38b20.

docker/mailman/core/docker-entrypoint.sh

@@ -0,0 +1,233 @@
+#! /bin/bash
+set -e
+
+function wait_for_postgres () {
+	# Check if the postgres database is up and accepting connections before
+	# moving forward.
+	# TODO: Use python3's psycopg2 module to do this in python3 instead of
+	# installing postgres-client in the image.
+	until psql $DATABASE_URL -c '\l'; do
+		>&2 echo "Postgres is unavailable - sleeping"
+		sleep 1
+	done
+	>&2 echo "Postgres is up - continuing"
+}
+
+function wait_for_mysql () {
+	# Check if MySQL is up and accepting connections.
+	readarray -d' ' -t ENDPOINT <<< $(python3 -c "from urllib.parse import urlparse; o = urlparse('$DATABASE_URL'); print('%s %s' % (o.hostname, o.port if o.port else '3306'));")
+	until mysqladmin ping --host ${ENDPOINT[0]} --port ${ENDPOINT[1]} --silent; do
+		>&2 echo "MySQL is unavailable - sleeping"
+		sleep 1
+	done
+	>&2 echo "MySQL is up - continuing"
+}
+
+# Empty the config file.
+echo "# This file is autogenerated at container startup." > /etc/mailman.cfg
+
+# Check if $MM_HOSTNAME is set, if not, set it to the value returned by
+# `hostname -i` command to set it to whatever IP address is assigned to the
+# container.
+if [[ ! -v MM_HOSTNAME ]]; then
+	export MM_HOSTNAME=`hostname -i`
+fi
+
+# SMTP_HOST defaults to the gateway
+if [[ ! -v SMTP_HOST ]]; then
+	export SMTP_HOST=$(/sbin/ip route | awk '/default/ { print $3 }')
+	echo "SMTP_HOST not specified, using the gateway ($SMTP_HOST) as default"
+fi
+
+if [[ ! -v SMTP_PORT ]]; then
+	export SMTP_PORT=25
+fi
+
+# Check if REST port, username, and password are set, if not, set them
+# to default values.
+if [[ ! -v MAILMAN_REST_PORT ]]; then
+	export MAILMAN_REST_PORT='8001'
+fi
+
+if [[ ! -v MAILMAN_REST_USER ]]; then
+	export MAILMAN_REST_USER='restadmin'
+fi
+
+if [[ ! -v MAILMAN_REST_PASSWORD ]]; then
+	export MAILMAN_REST_PASSWORD='restpass'
+fi
+
+function setup_database () {
+	if [[ ! -v DATABASE_URL ]]
+	then
+		echo "Environment variable DATABASE_URL should be defined..."
+		exit 1
+	fi
+
+	# Translate mysql:// urls to mysql+mysql:// backend:
+	if [[ "$DATABASE_URL" == mysql://* ]]; then
+		DATABASE_URL="mysql+pymysql://${DATABASE_URL:8}"
+		echo "Database URL was automatically rewritten to: $DATABASE_URL"
+	fi
+
+	# If DATABASE_CLASS is not set, guess it for common databases:
+	if [ -z "$DATABASE_CLASS" ]; then
+		if [[ ("$DATABASE_URL" == mysql:*) ||
+				("$DATABASE_URL" == mysql+*) ]]; then
+			DATABASE_CLASS=mailman.database.mysql.MySQLDatabase
+		fi
+		if [[ ("$DATABASE_URL" == postgres:*) ||
+				("$DATABASE_URL" == postgres+*) ]]; then
+			DATABASE_CLASS=mailman.database.postgresql.PostgreSQLDatabase
+		fi
+	fi
+
+	cat >> /etc/mailman.cfg <<EOF
+[database]
+class: $DATABASE_CLASS
+url: $DATABASE_URL
+EOF
+}
+
+
+# Check if $DATABASE_URL is defined, if not, use a standard sqlite database.
+#
+# If the $DATABASE_URL is defined and is postgres, check if it is available
+# yet. Do not start the container before the postgresql boots up.
+#
+# TODO: If the $DATABASE_URL is defined and is mysql, check if the database is
+# available before the container boots up.
+#
+# TODO: Check the database type and detect if it is up based on that. For now,
+# assume that postgres is being used if DATABASE_URL is defined.
+if [[ ! -v DATABASE_URL ]]; then
+	echo "DATABASE_URL is not defined. Using sqlite database..."
+else
+	setup_database
+fi
+
+
+if [[ "$DATABASE_TYPE" = 'postgres' ]]
+then
+	wait_for_postgres
+elif [[ "$DATABASE_TYPE" = 'mysql' ]]
+then
+	wait_for_mysql
+fi
+
+# Generate a basic mailman.cfg.
+cat >> /etc/mailman.cfg << EOF
+[runner.retry]
+sleep_time: 10s
+
+[webservice]
+hostname: $MM_HOSTNAME
+port: $MAILMAN_REST_PORT
+admin_user: $MAILMAN_REST_USER
+admin_pass: $MAILMAN_REST_PASSWORD
+configuration: /etc/gunicorn.cfg
+
+EOF
+
+# Generate a basic gunicorn.cfg.
+SITE_DIR=$(python3 -c 'import site; print(site.getsitepackages()[0])')
+cp "${SITE_DIR}/mailman/config/gunicorn.cfg" /etc/gunicorn.cfg
+
+# Generate a basic configuration to use exim
+cat > /tmp/exim-mailman.cfg <<EOF
+[mta]
+incoming: mailman.mta.exim4.LMTP
+outgoing: mailman.mta.deliver.deliver
+lmtp_host: $MM_HOSTNAME
+lmtp_port: 8024
+smtp_host: $SMTP_HOST
+smtp_port: $SMTP_PORT
+configuration: python:mailman.config.exim4
+
+EOF
+
+cat > /etc/postfix-mailman.cfg << EOF
+[postfix]
+transport_file_type: regex
+# While in regex mode, postmap_command is never used, a placeholder
+# is added here so that it doesn't break anything.
+postmap_command: true
+EOF
+
+# Generate a basic configuration to use postfix.
+cat > /tmp/postfix-mailman.cfg <<EOF
+[mta]
+incoming: mailman.mta.postfix.LMTP
+outgoing: mailman.mta.deliver.deliver
+lmtp_host: $MM_HOSTNAME
+lmtp_port: 8024
+smtp_host: $SMTP_HOST
+smtp_port: $SMTP_PORT
+configuration: /etc/postfix-mailman.cfg
+
+EOF
+
+if [ "$MTA" == "exim" ]
+then
+	echo "Using Exim configuration"
+	cat /tmp/exim-mailman.cfg >> /etc/mailman.cfg
+elif [ "$MTA" == "postfix" ]
+then
+	echo "Using Postfix configuration"
+	cat /tmp/postfix-mailman.cfg >> /etc/mailman.cfg
+else
+	echo "No MTA environment variable found, defaulting to Exim"
+	cat /tmp/exim-mailman.cfg >> /etc/mailman.cfg
+fi
+
+rm -f /tmp/{postfix,exim}-mailman.cfg
+
+if [[ -e /opt/mailman/mailman-extra.cfg ]]
+then
+	echo "Found configuration file at /opt/mailman/mailman-extra.cfg"
+	cat /opt/mailman/mailman-extra.cfg >> /etc/mailman.cfg
+fi
+
+if [[ -e /opt/mailman/gunicorn-extra.cfg ]]
+then
+       echo "Found [webserver] configuration file at /opt/mailman/gunicorn-extra.cfg"
+       cat /opt/mailman/gunicorn-extra.cfg > /etc/gunicorn.cfg
+fi
+
+if [[ -v HYPERKITTY_API_KEY ]]; then
+
+echo "HYPERKITTY_API_KEY found, setting up HyperKitty archiver..."
+
+cat >> /etc/mailman.cfg << EOF
+[archiver.hyperkitty]
+class: mailman_hyperkitty.Archiver
+enable: yes
+configuration: /etc/mailman-hyperkitty.cfg
+
+EOF
+
+if [[ ! -v HYPERKITTY_URL ]]; then
+	echo "HYPERKITTY_URL not set, using the default value of http://mailman-web:8000/hyperkitty"
+	export HYPERKITTY_URL="http://mailman-web:8000/hyperkitty/"
+fi
+
+# Generate a basic mailman-hyperkitty.cfg.
+cat > /etc/mailman-hyperkitty.cfg <<EOF
+[general]
+base_url: $HYPERKITTY_URL
+api_key: $HYPERKITTY_API_KEY
+EOF
+
+else
+
+echo "HYPERKITTY_API_KEY not defined, skipping HyperKitty setup..."
+
+fi
+
+# Now chown the places where mailman wants to write stuff.
+chown -R mailman /opt/mailman
+
+# Generate the LMTP files for postfix if needed.
+su-exec mailman mailman aliases
+
+exec su-exec mailman "$@"

docker/mailman/core/requirements.txt

@@ -0,0 +1,5 @@
+# This is a separate file from Dockerfile so that we can use dependabot
+# for version updates that isn't supported for contents inside the
+# Dockerfile.
+mailman==3.3.7
+mailman-hyperkitty==1.2.0

docker/mailman/web/Dockerfile

@@ -0,0 +1,52 @@
+# syntax = docker/dockerfile:1.3
+FROM alpine:3.16.2
+
+# Add needed files for uwsgi server + settings for django
+COPY mailman-web /opt/mailman-web
+# Add startup script to container
+COPY docker-entrypoint.sh /usr/local/bin/
+# Add requirements file.
+COPY requirements.txt /tmp/
+
+# Install packages and dependencies for postorius and hyperkitty Add user for
+# executing apps, change ownership for uwsgi+django files and set execution
+# rights for management script
+RUN --mount=type=cache,target=/root/.cache \
+    set -ex \
+	&& apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers \
+           postgresql-dev mariadb-dev mariadb-connector-c python3-dev libffi-dev openldap-dev cargo rust \
+	&& apk add --no-cache --virtual .mailman-rundeps bash sassc \
+	   postgresql-client mysql-client py3-mysqlclient curl mailcap gettext \
+	   python3 py3-pip xapian-core xapian-bindings-python3 libffi pcre-dev py-cryptography \
+	&& python3 -m pip install -U 'Django<4.1' pip setuptools wheel \
+	&& pip install -r /tmp/requirements.txt \
+		whoosh \
+		uwsgi \
+		'psycopg2<2.9' \
+		dj-database-url \
+		mysqlclient \
+		typing \
+		xapian-haystack \
+		django-auth-ldap \
+		python-memcached \
+		diskcache \
+		django-utils-six \
+		tzdata \
+	&& apk del .build-deps \
+	&& addgroup -S mailman \
+	&& adduser -S -G mailman mailman \
+	&& chown -R mailman /opt/mailman-web/ \
+	&& chmod u+x /opt/mailman-web/manage.py
+
+WORKDIR /opt/mailman-web
+
+# Expose port 8000 for http and port 8080 for uwsgi
+# (see web/mailman-web/uwsgi.ini#L2-L4)
+EXPOSE 8000 8080
+
+# Use stop signal for uwsgi server
+STOPSIGNAL SIGINT
+
+ENTRYPOINT ["docker-entrypoint.sh"]
+
+CMD ["uwsgi", "--ini", "/opt/mailman-web/uwsgi.ini"]

docker/mailman/web/README.md

@@ -0,0 +1,6 @@
+This is forked from https://github.com/maxking/docker-mailman/tree/main/web
+and you should refer to that repo for information. The upstream repo is MIT
+licensed.
+
+The contents in this dir are based on commit
+22b3b7d4024a00e9896837fa04883cbaeef38b20.

docker/mailman/web/docker-entrypoint.sh

@@ -0,0 +1,154 @@
+#! /bin/bash
+set -e
+
+
+function wait_for_postgres () {
+	# Check if the postgres database is up and accepting connections before
+	# moving forward.
+	# TODO: Use python's psycopg2 module to do this in python instead of
+	# installing postgres-client in the image.
+	until psql $DATABASE_URL -c '\l'; do
+		>&2 echo "Postgres is unavailable - sleeping"
+		sleep 1
+	done
+	>&2 echo "Postgres is up - continuing"
+}
+
+function wait_for_mysql () {
+	# Check if MySQL is up and accepting connections.
+	readarray -d' ' -t ENDPOINT <<< $(python3 -c "from urllib.parse import urlparse; o = urlparse('$DATABASE_URL'); print('%s %s' % (o.hostname, o.port if o.port else '3306'));")
+	until mysqladmin ping --host ${ENDPOINT[0]} --port ${ENDPOINT[1]} --silent; do
+		>&2 echo "MySQL is unavailable - sleeping"
+		sleep 1
+	done
+	>&2 echo "MySQL is up - continuing"
+}
+
+function check_or_create () {
+	# Check if the path exists, if not, create the directory.
+	if [[ ! -e dir ]]; then
+		echo "$1 does not exist, creating ..."
+		mkdir "$1"
+	fi
+}
+
+# function postgres_ready(){
+# python << END
+# import sys
+# import psycopg2
+# try:
+#     conn = psycopg2.connect(dbname="$POSTGRES_DB", user="$POSTGRES_USER", password="$POSTGRES_PASSWORD", host="postgres")
+# except psycopg2.OperationalError:
+#     sys.exit(-1)
+# sys.exit(0)
+# END
+# }
+
+# SMTP_HOST defaults to the gateway
+if [[ ! -v SMTP_HOST ]]; then
+	export SMTP_HOST=$(/sbin/ip route | awk '/default/ { print $3 }')
+fi
+
+# Check if $SECRET_KEY is defined, if not, bail out.
+if [[ ! -v SECRET_KEY ]]; then
+	echo "SECRET_KEY is not defined. Aborting."
+	exit 1
+fi
+
+# Check if $DATABASE_URL is defined, if not, use a standard sqlite database.
+#
+# If the $DATABASE_URL is defined and is postgres, check if it is available
+# yet. Do not start the container before the postgresql boots up.
+#
+# If the $DATABASE_URL is defined and is mysql, check if the database is
+# available before the container boots up.
+#
+# TODO: Check the database type and detect if it is up based on that. For now,
+# assume that postgres is being used if DATABASE_URL is defined.
+
+if [[ ! -v DATABASE_URL ]]; then
+	echo "DATABASE_URL is not defined. Using sqlite database..."
+	export DATABASE_URL=sqlite://mailmanweb.db
+	export DATABASE_TYPE='sqlite'
+fi
+
+if [[ "$DATABASE_TYPE" = 'postgres' ]]
+then
+	wait_for_postgres
+elif [[ "$DATABASE_TYPE" = 'mysql' ]]
+then
+	wait_for_mysql
+fi
+
+# Check if we are in the correct directory before running commands.
+if [[ ! $(pwd) == '/opt/mailman-web' ]]; then
+	echo "Running in the wrong directory...switching to /opt/mailman-web"
+	cd /opt/mailman-web
+fi
+
+# Check if the logs directory is setup.
+if [[ ! -e /opt/mailman-web-data/logs/mailmanweb.log ]]; then
+	echo "Creating log file for mailman web"
+	mkdir -p /opt/mailman-web-data/logs/
+	touch /opt/mailman-web-data/logs/mailmanweb.log
+fi
+
+if [[ ! -e /opt/mailman-web-data/logs/uwsgi.log ]]; then
+	echo "Creating log file for uwsgi.."
+	touch /opt/mailman-web-data/logs/uwsgi.log
+fi
+
+# Check if the settings_local.py file exists, if yes, copy it too.
+if [[ -e /opt/mailman-web-data/settings_local.py ]]; then
+	echo "Copying settings_local.py ..."
+	cp /opt/mailman-web-data/settings_local.py /opt/mailman-web/settings_local.py
+	chown mailman:mailman /opt/mailman-web/settings_local.py
+else
+	echo "settings_local.py not found, it is highly recommended that you provide one"
+	echo "Using default configuration to run."
+fi
+
+# Collect static for the django installation.
+python3 manage.py collectstatic --noinput --clear --verbosity 0
+
+
+# Compile all the installed po files to mo.
+SITE_DIR=$(python3 -c 'import site; print(site.getsitepackages()[0])')
+echo "Compiling locale files in $SITE_DIR"
+cd $SITE_DIR && /opt/mailman-web/manage.py compilemessages &&  cd -
+
+# Compress static files.
+python3 manage.py compress --force
+
+
+# Migrate all the data to the database if this is a new installation, otherwise
+# this command will upgrade the database.
+python3 manage.py migrate
+
+# If MAILMAN_ADMIN_USER and MAILMAN_ADMIN_EMAIL is defined create a new
+# superuser for Django. There is no password setup so it can't login yet unless
+# the password is reset.
+if [[ -v MAILMAN_ADMIN_USER ]] && [[ -v MAILMAN_ADMIN_EMAIL ]];
+then
+	echo "Creating admin user $MAILMAN_ADMIN_USER ..."
+	python3 manage.py createsuperuser --noinput --username "$MAILMAN_ADMIN_USER"\
+		   --email "$MAILMAN_ADMIN_EMAIL" 2> /dev/null || \
+		echo "Superuser $MAILMAN_ADMIN_USER already exists"
+fi
+
+# If SERVE_FROM_DOMAIN is defined then rename the default `example.com`
+# domain to the defined domain.
+if [[ -v SERVE_FROM_DOMAIN ]];
+then
+	echo "Setting $SERVE_FROM_DOMAIN as the default domain ..."
+	python3 manage.py shell -c \
+	"from django.contrib.sites.models import Site; Site.objects.filter(domain='example.com').update(domain='$SERVE_FROM_DOMAIN', name='$SERVE_FROM_DOMAIN')"
+fi
+
+# Create a mailman user with the specific UID and GID and do not create home
+# directory for it. Also chown the logs directory to write the files.
+chown mailman:mailman /opt/mailman-web-data -R
+
+[[ -v DISKCACHE_PATH ]] && chown mailman:mailman "${DISKCACHE_PATH}" -R
+
+exec $@

docker/mailman/web/mailman-web/manage.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+import os
+import sys
+
+if __name__ == "__main__":
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")
+
+    from django.core.management import execute_from_command_line
+
+    execute_from_command_line(sys.argv)

docker/mailman/web/mailman-web/settings.py

@@ -0,0 +1,414 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 1998-2016 by the Free Software Foundation, Inc.
+#
+# This file is part of Mailman Suite.
+#
+# Mailman Suite is free sofware: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# Mailman Suite is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+
+# You should have received a copy of the GNU General Public License along
+# with Mailman Suite.  If not, see <http://www.gnu.org/licenses/>.
+"""
+Django Settings for Mailman Suite (hyperkitty + postorius)
+
+For more information on this file, see
+https://docs.djangoproject.com/en/1.8/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/1.8/ref/settings/
+"""
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+import os
+import dj_database_url
+import sys
+from socket import gethostbyname
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = os.environ.get('SECRET_KEY')
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = False
+
+ADMINS = (
+     ('Mailman Suite Admin', 'root@localhost'),
+)
+
+SITE_ID = 1
+
+# Hosts/domain names that are valid for this site; required if DEBUG is False
+# See https://docs.djangoproject.com/en/3.1/ref/settings/#allowed-hosts
+ALLOWED_HOSTS = [
+    "localhost",  # Archiving API from Mailman, keep it.
+    # "lists.your-domain.org",
+    # Add here all production URLs you may have.
+    "mailman-web",
+    gethostbyname("mailman-web"),
+    os.environ.get('SERVE_FROM_DOMAIN'),
+    os.environ.get('DJANGO_ALLOWED_HOSTS'),
+]
+
+# Mailman API credentials
+MAILMAN_REST_API_URL = os.environ.get('MAILMAN_REST_URL', 'http://mailman-core:8001')
+MAILMAN_REST_API_USER = os.environ.get('MAILMAN_REST_USER', 'restadmin')
+MAILMAN_REST_API_PASS = os.environ.get('MAILMAN_REST_PASSWORD', 'restpass')
+MAILMAN_ARCHIVER_KEY = os.environ.get('HYPERKITTY_API_KEY')
+MAILMAN_ARCHIVER_FROM = (os.environ.get('MAILMAN_HOST_IP', gethostbyname(os.environ.get('MAILMAN_HOSTNAME', 'mailman-core'))),)
+
+# Application definition
+
+INSTALLED_APPS = []
+DEFAULT_APPS = [
+    'hyperkitty',
+    'postorius',
+    'django_mailman3',
+    # Uncomment the next line to enable the admin:
+    'django.contrib.admin',
+    # Uncomment the next line to enable admin documentation:
+    # 'django.contrib.admindocs',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.sites',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    'rest_framework',
+    'django_gravatar',
+    'compressor',
+    'haystack',
+    'django_extensions',
+    'django_q',
+    'allauth',
+    'allauth.account',
+    'allauth.socialaccount',
+]
+
+MAILMAN_WEB_SOCIAL_AUTH = [
+    'django_mailman3.lib.auth.fedora',
+    'allauth.socialaccount.providers.openid',
+    'allauth.socialaccount.providers.github',
+    'allauth.socialaccount.providers.gitlab',
+    'allauth.socialaccount.providers.google',
+]
+
+MIDDLEWARE = (
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.middleware.locale.LocaleMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    'django.middleware.security.SecurityMiddleware',
+    'django_mailman3.middleware.TimezoneMiddleware',
+    'postorius.middleware.PostoriusMiddleware',
+)
+
+ROOT_URLCONF = 'urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.i18n',
+                'django.template.context_processors.media',
+                'django.template.context_processors.static',
+                'django.template.context_processors.tz',
+                'django.template.context_processors.csrf',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+                'django_mailman3.context_processors.common',
+                'hyperkitty.context_processors.common',
+                'postorius.context_processors.postorius',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'wsgi.application'
+
+
+# Database
+# https://docs.djangoproject.com/en/1.8/ref/settings/#databases
+# dj_database_url uses $DATABASE_URL environment variable to create a
+# django-style-config-dict.
+# https://github.com/kennethreitz/dj-database-url
+DATABASES = {
+    'default': dj_database_url.config(conn_max_age=600)
+}
+
+# If you're behind a proxy, use the X-Forwarded-Host header
+# See https://docs.djangoproject.com/en/1.8/ref/settings/#use-x-forwarded-host
+USE_X_FORWARDED_HOST = True
+
+
+# Password validation
+# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+
+# Internationalization
+# https://docs.djangoproject.com/en/1.8/topics/i18n/
+
+LANGUAGE_CODE = 'en-us'
+
+TIME_ZONE = 'UTC'
+
+USE_I18N = True
+
+USE_L10N = True
+
+USE_TZ = True
+
+STATIC_ROOT = '/opt/mailman-web-data/static'
+
+STATIC_URL = '/static/'
+
+# Additional locations of static files
+
+
+# List of finder classes that know how to find static files in
+# various locations.
+STATICFILES_FINDERS = (
+    'django.contrib.staticfiles.finders.FileSystemFinder',
+    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
+    'compressor.finders.CompressorFinder',
+)
+
+
+SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer'
+
+LOGIN_URL = 'account_login'
+LOGIN_REDIRECT_URL = 'list_index'
+LOGOUT_URL = 'account_logout'
+
+
+# Use SERVE_FROM_DOMAIN as the default domain in the email.
+hostname = os.environ.get('SERVE_FROM_DOMAIN', 'localhost.local')
+DEFAULT_FROM_EMAIL = 'postorius@{}'.format(hostname)
+SERVER_EMAIL = 'root@{}'.format(hostname)
+
+# Change this when you have a real email backend
+EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
+EMAIL_HOST = os.environ.get('SMTP_HOST', '')
+EMAIL_PORT = os.environ.get('SMTP_PORT', 25)
+EMAIL_HOST_USER = os.environ.get('SMTP_HOST_USER', '')
+EMAIL_HOST_PASSWORD = os.environ.get('SMTP_HOST_PASSWORD', '')
+EMAIL_USE_TLS = os.environ.get('SMTP_USE_TLS', False)
+EMAIL_USE_SSL = os.environ.get('SMTP_USE_SSL', False)
+
+# Compatibility with Bootstrap 3
+from django.contrib.messages import constants as messages  # flake8: noqa
+MESSAGE_TAGS = {
+    messages.ERROR: 'danger'
+}
+
+
+#
+# Social auth
+#
+AUTHENTICATION_BACKENDS = (
+    'django.contrib.auth.backends.ModelBackend',
+    'allauth.account.auth_backends.AuthenticationBackend',
+)
+
+# Django Allauth
+ACCOUNT_AUTHENTICATION_METHOD = "username_email"
+ACCOUNT_EMAIL_REQUIRED = True
+ACCOUNT_EMAIL_VERIFICATION = "mandatory"
+# You probably want https in production, but this is a dev setup file
+ACCOUNT_DEFAULT_HTTP_PROTOCOL = "https"
+ACCOUNT_UNIQUE_EMAIL  = True
+
+SOCIALACCOUNT_PROVIDERS = {
+    'openid': {
+        'SERVERS': [
+            dict(id='yahoo',
+                 name='Yahoo',
+                 openid_url='http://me.yahoo.com'),
+        ],
+    },
+    'google': {
+        'SCOPE': ['profile', 'email'],
+        'AUTH_PARAMS': {'access_type': 'online'},
+    },
+    'facebook': {
+       'METHOD': 'oauth2',
+       'SCOPE': ['email'],
+       'FIELDS': [
+           'email',
+           'name',
+           'first_name',
+           'last_name',
+           'locale',
+           'timezone',
+           ],
+       'VERSION': 'v2.4',
+    },
+}
+
+
+# django-compressor
+# https://pypi.python.org/pypi/django_compressor
+#
+COMPRESS_PRECOMPILERS = (
+   ('text/less', 'lessc {infile} {outfile}'),
+   ('text/x-scss', 'sassc -t compressed {infile} {outfile}'),
+   ('text/x-sass', 'sassc -t compressed {infile} {outfile}'),
+)
+
+# On a production setup, setting COMPRESS_OFFLINE to True will bring a
+# significant performance improvement, as CSS files will not need to be
+# recompiled on each requests. It means running an additional "compress"
+# management command after each code upgrade.
+# http://django-compressor.readthedocs.io/en/latest/usage/#offline-compression
+# COMPRESS_OFFLINE = True
+
+#
+# Full-text search engine
+#
+HAYSTACK_CONNECTIONS = {
+    'default': {
+        'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine',
+        'PATH': "/opt/mailman-web-data/fulltext_index",
+        # You can also use the Xapian engine, it's faster and more accurate,
+        # but requires another library.
+        # http://django-haystack.readthedocs.io/en/v2.4.1/installing_search_engines.html#xapian
+        # Example configuration for Xapian:
+        #'ENGINE': 'xapian_backend.XapianEngine'
+    },
+}
+
+import sys
+# A sample logging configuration. The only tangible logging
+# performed by this configuration is to send an email to
+# the site admins on every HTTP 500 error when DEBUG=False.
+# See http://docs.djangoproject.com/en/dev/topics/logging for
+# more details on how to customize your logging configuration.
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'filters': {
+        'require_debug_false': {
+            '()': 'django.utils.log.RequireDebugFalse'
+        }
+    },
+    'handlers': {
+        'mail_admins': {
+            'level': 'ERROR',
+            'filters': ['require_debug_false'],
+            'class': 'django.utils.log.AdminEmailHandler'
+        },
+        'file':{
+            'level': 'INFO',
+            'class': 'logging.handlers.RotatingFileHandler',
+            #'class': 'logging.handlers.WatchedFileHandler',
+            'filename': os.environ.get('DJANGO_LOG_URL','/opt/mailman-web-data/logs/mailmanweb.log'),
+            'formatter': 'verbose',
+        },
+        'console': {
+            'class': 'logging.StreamHandler',
+            'formatter': 'simple',
+            'level': 'INFO',
+            'stream': sys.stdout,
+        },
+    },
+    'loggers': {
+        'django.request': {
+            'handlers': ['mail_admins', 'file'],
+            'level': 'INFO',
+            'propagate': True,
+        },
+        'django': {
+            'handlers': ['file'],
+            'level': 'INFO',
+            'propagate': True,
+        },
+        'hyperkitty': {
+            'handlers': ['file'],
+            'level': 'INFO',
+            'propagate': True,
+        },
+        'postorius': {
+            'handlers': ['file'],
+            'level': 'INFO',
+            'propagate': True
+        },
+    },
+    'formatters': {
+        'verbose': {
+            'format': '%(levelname)s %(asctime)s %(process)d %(name)s %(message)s'
+        },
+        'simple': {
+            'format': '%(levelname)s %(message)s'
+        },
+    },
+}
+
+
+if os.environ.get('LOG_TO_CONSOLE') == 'yes':
+    LOGGING['loggers']['django']['handlers'].append('console')
+    LOGGING['loggers']['django.request']['handlers'].append('console')
+
+# HyperKitty-specific
+#
+# Only display mailing-lists from the same virtual host as the webserver
+FILTER_VHOST = False
+
+
+Q_CLUSTER = {
+    'timeout': 300,
+    'retry': 300,
+    'save_limit': 100,
+    'orm': 'default',
+}
+
+POSTORIUS_TEMPLATE_BASE_URL =  os.environ.get('POSTORIUS_TEMPLATE_BASE_URL', 'http://mailman-web:8000')
+
+DISKCACHE_PATH = os.environ.get('DISKCACHE_PATH', '/opt/mailman-web-data/diskcache')
+DISKCACHE_SIZE = os.environ.get('DISKCACHE_SIZE', 2 ** 30) # 1 gigabyte
+
+CACHES = {
+    'default': {
+        'BACKEND': 'diskcache.DjangoCache',
+        'LOCATION': DISKCACHE_PATH,
+        'OPTIONS': {
+            'size_limit': DISKCACHE_SIZE,
+        },
+    },
+}
+
+try:
+    from settings_local import *
+except ImportError:
+    pass
+
+# Compatibility for older installs that override INSTALLED_APPS
+if not INSTALLED_APPS:
+    INSTALLED_APPS = DEFAULT_APPS + MAILMAN_WEB_SOCIAL_AUTH

docker/mailman/web/mailman-web/urls.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+# Copyright (C) 1998-2016 by the Free Software Foundation, Inc.
+#
+# This file is part of Postorius.
+#
+# Postorius is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# Postorius is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# Postorius.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.conf.urls import include
+from django.contrib import admin
+from django.urls import path, reverse_lazy
+from django.views.generic import RedirectView
+
+urlpatterns = [
+    path(r'', RedirectView.as_view(
+        url=reverse_lazy('list_index'),
+        permanent=True)),
+    path(r'mailman3/', include('postorius.urls')),
+    path(r'archives/', include('hyperkitty.urls')),
+    path(r'', include('django_mailman3.urls')),
+    path(r'accounts/', include('allauth.urls')),
+    # Django admin
+    path(r'admin/', admin.site.urls),
+]

docker/mailman/web/mailman-web/uwsgi.ini

@@ -0,0 +1,53 @@
+[uwsgi]
+# Port on which uwsgi will be listening.
+uwsgi-socket = 0.0.0.0:8080
+http-socket = 0.0.0.0:8000
+
+#Enable threading for python
+enable-threads = true
+
+# Setting uwsgi buffer size to what Apache2 supports.
+buffer-size = 8190
+
+# Move to the directory wher the django files are.
+chdir = /opt/mailman-web
+
+# Use the wsgi file provided with the django project.
+wsgi-file = wsgi.py
+
+# Setup default number of processes and threads per process.
+master = true
+processes = 2
+threads = 2
+
+# Drop privielges and don't run as root.
+uid = mailman
+gid = mailman
+
+# Setup the django_q related worker processes.
+attach-daemon = ./manage.py qcluster
+
+# Setup hyperkitty's cron jobs.
+# 'minutely' jobs are run hourly for perf reasons.
+# See https://github.com/maxking/docker-mailman/issues/327
+unique-cron = 0 -1 -1 -1 -1 ./manage.py runjobs minutely
+unique-cron = -15 -1 -1 -1 -1 ./manage.py runjobs quarter_hourly
+unique-cron = 0 -1 -1 -1 -1 ./manage.py runjobs hourly
+unique-cron = 0 0 -1 -1 -1 ./manage.py runjobs daily
+unique-cron = 0 0 1 -1 -1 ./manage.py runjobs monthly
+unique-cron = 0 0 -1 -1 0 ./manage.py runjobs weekly
+unique-cron = 0 0 1 1 -1 ./manage.py runjobs yearly
+
+# Setup the request log.
+req-logger = file:/opt/mailman-web-data/logs/uwsgi.log
+
+# Log cron seperately.
+logger = cron file:/opt/mailman-web-data/logs/uwsgi-cron.log
+log-route = cron uwsgi-cron
+
+# Log qcluster commands seperately.
+logger = qcluster file:/opt/mailman-web-data/logs/uwsgi-qcluster.log
+log-route = qcluster uwsgi-daemons
+
+# Last log and it logs the rest of the stuff.
+logger = file:/opt/mailman-web-data/logs/uwsgi-error.log

docker/mailman/web/mailman-web/wsgi.py

@@ -0,0 +1,38 @@
+"""
+WSGI config for HyperKitty project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/{{ docs_version }}/howto/deployment/wsgi/
+"""
+
+import os
+
+# import sys
+# import site
+
+# For some unknown reason, sometimes mod_wsgi fails to set the python paths to
+# the virtualenv, with the 'python-path' option. You can do it here too.
+#
+# # Remember original sys.path.
+# prev_sys_path = list(sys.path)
+# # Add here, for the settings module
+# site.addsitedir(os.path.abspath(os.path.dirname(__file__)))
+# # Add the virtualenv
+# venv = os.path.join(os.path.abspath(os.path.dirname(__file__)),
+#                     '..', 'lib', 'python2.6', 'site-packages')
+# site.addsitedir(venv)
+# # Reorder sys.path so new directories at the front.
+# new_sys_path = []
+# for item in list(sys.path):
+#     if item not in prev_sys_path:
+#         new_sys_path.append(item)
+#         sys.path.remove(item)
+#         sys.path[:0] = new_sys_path
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings")
+
+application = get_wsgi_application()

docker/mailman/web/requirements.txt

@@ -0,0 +1,4 @@
+mailmanclient==3.3.4
+postorius==1.3.7
+hyperkitty==1.3.6
+django-mailman3==1.3.8

playbooks/roles/mailman3/templates/docker-compose.yaml.j2

@@ -4,7 +4,7 @@
 version: '2'
 services:
   mailman-core:
-    image: docker.io/maxking/mailman-core:0.4
+    image: docker.io/opendevorg/mailman-core:latest
     restart: always
     container_name: mailman-core
     volumes:
@@ -27,7 +27,7 @@ services:
     #user: mailman
 
   mailman-web:
-    image: docker.io/maxking/mailman-web:0.4
+    image: docker.io/opendevorg/mailman-web:latest
     restart: always
     container_name: mailman-web
     depends_on:

playbooks/roles/mailman3/templates/mailman.vhost.j2

@@ -39,7 +39,7 @@
   SSLCertificateChainFile /etc/letsencrypt-certs/{{ inventory_hostname }}/ca.cer
 
   Alias /static /var/lib/mailman/web-data/static
-  Alias /favicon.ico /var/lib/mailman/web-data/static/hyperkitty/img/favicon.ico
+  Alias /favicon.ico /var/lib/mailman/web-data/static/archives/img/favicon.ico
 
   <Location "/admin">
     Require local
@@ -47,8 +47,8 @@
 
   RewriteEngine On
   RewriteRule "/pipermail/(.*)" "/var/lib/mailman/web-data/mm2archives/%{HTTP_HOST}/public/$1"
-  RewriteRule "/cgi-bin/mailman/listinfo/(.*)" "https://%{HTTP_HOST}/postorius/lists/$1.%{HTTP_HOST}/"
+  RewriteRule "/cgi-bin/mailman/listinfo/(.*)" "https://%{HTTP_HOST}/mailman3/lists/$1.%{HTTP_HOST}/"
-  RewriteRule "/cgi-bin/mailman/listinfo" "https://%{HTTP_HOST}/postorius/lists/"
+  RewriteRule "/cgi-bin/mailman/listinfo" "https://%{HTTP_HOST}/mailman3/lists/"
 
   ProxyPassMatch ^/static/ !
   ProxyPass "/" "uwsgi://localhost:8080/"

testinfra/test_lists_opendev_org.py

@@ -39,22 +39,22 @@ def test_apache2_listening(host):
 def test_mailman3_screenshots(host):
     shots = (
         ("https://lists.opendev.org:443", None, "mm3-opendev-main.png"),
-        ("https://lists.opendev.org:443/hyperkitty/",
+        ("https://lists.opendev.org:443/archives/",
          None, "mm3-opendev-archives.png"),
         ("https://lists.opendev.org:443"
-         "/hyperkitty/list/service-discuss@lists.opendev.org/",
+         "/archives/list/service-discuss@lists.opendev.org/",
          None, "mm3-opendev-list.png"),
         ("https://lists.opendev.org:443"
-         "/accounts/login/?next=/postorius/lists/",
+         "/accounts/login/?next=/mailman3/lists/",
          None, "mm3-opendev-login.png"),
         ("https://lists.openstack.org:443", None, "mm3-openstack-main.png"),
-        ("https://lists.openstack.org:443/hyperkitty/",
+        ("https://lists.openstack.org:443/archives/",
          None, "mm3-openstack-archives.png"),
         ("https://lists.openstack.org:443"
-         "/hyperkitty/list/openstack-discuss@lists.openstack.org/",
+         "/archives/list/openstack-discuss@lists.openstack.org/",
          None, "mm3-openstack-list.png"),
         ("https://lists.openstack.org:443"
-         "/accounts/login/?next=/postorius/lists/",
+         "/accounts/login/?next=/mailman3/lists/",
          None, "mm3-openstack-login.png"),
     )
 

tools/run-bashate.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 
 ROOT=$(readlink -fn $(dirname $0)/.. )
-find $ROOT -type f -not -wholename \*.tox/\* -and \( -name \*.sh \
+find $ROOT -type f -not -wholename \*.tox/\* -and \
-    -or -name \*rc -or -name functions\* \) -print0 \
+    \( -name \*.sh -or -name \*rc -or -name functions\* \) -and \
+    -not -path \*/docker/mailman/\* \
+    -print0 \
     | xargs -0 bashate -i E006,E010 -v

zuul.d/docker-images/mailman.yaml

@@ -0,0 +1,29 @@
+- job:
+    name: system-config-build-image-mailman
+    description: Build mailman docker images
+    provides: mailman-container-images
+    parent: system-config-build-image
+    vars: &mailman_vars
+      docker_images:
+        - context: docker/mailman/core
+          repository: opendevorg/mailman-core
+        - context: docker/mailman/web
+          repository: opendevorg/mailman-web
+      docker_use_buildkit: true
+    files: &mailman_files
+      - docker/mailman/
+
+- job:
+    name: system-config-upload-image-mailman
+    description: Build and upload mailman images.
+    provides: mailman-container-images
+    parent: system-config-upload-image
+    vars: *mailman_vars
+    files: *mailman_files
+
+- job:
+    name: system-config-promote-image-mailman
+    description: Promote previously published mailman images to latest.
+    parent: system-config-promote-image
+    vars: *mailman_vars
+    files: *mailman_files

zuul.d/project.yaml

@@ -30,7 +30,11 @@
                 soft: true
         - system-config-run-kerberos
         - system-config-run-lists
-        - system-config-run-lists3
+        - system-config-run-lists3:
+            dependencies:
+              - name: opendev-buildset-registry
+              - name: system-config-build-image-mailman
+                soft: true
         - system-config-run-nodepool:
             dependencies:
               - name: opendev-buildset-registry
@@ -113,6 +117,7 @@
               - name: system-config-build-image-python-base-3.9-bullseye
                 soft: true
         - system-config-build-image-etherpad
+        - system-config-build-image-mailman
         - system-config-build-image-gitea:
             dependencies:
               - name: opendev-buildset-registry
@@ -181,7 +186,11 @@
                 soft: true
         - system-config-run-kerberos
         - system-config-run-lists
-        - system-config-run-lists3
+        - system-config-run-lists3:
+            dependencies:
+              - name: opendev-buildset-registry
+              - name: system-config-upload-image-mailman
+                soft: true
         - system-config-run-nodepool:
             dependencies:
               - name: opendev-buildset-registry
@@ -252,6 +261,7 @@
         - system-config-upload-image-hound
         - system-config-upload-image-assets
         - system-config-upload-image-etherpad
+        - system-config-upload-image-mailman
         - system-config-upload-image-gitea:
             dependencies:
               - name: opendev-buildset-registry
@@ -303,6 +313,7 @@
         - system-config-promote-image-gitea-init
         - system-config-promote-image-gitea
         - system-config-promote-image-etherpad
+        - system-config-promote-image-mailman
         - system-config-promote-image-haproxy-statsd
         - system-config-promote-image-zookeeper-statsd
         - system-config-promote-image-accessbot
@@ -469,6 +480,8 @@
                 soft: true
               - name: infra-prod-letsencrypt
                 soft: true
+              - name: system-config-promote-image-mailman
+                soft: true
         - infra-prod-service-mirror: &infra-prod-service-mirror
             dependencies:
               - name: infra-prod-letsencrypt

zuul.d/system-config-run.yaml

@@ -290,7 +290,8 @@
     name: system-config-run-lists3
     # We don't use the system-config-run-containers base job because we
     # are consuming upstream containers only.
-    parent: system-config-run
+    parent: system-config-run-containers
+    requires: mailman-container-images
     description: |
       Run the playbook for a mailman3 list server.
     timeout: 3600