diff --git a/playbooks/roles/mailman3/files/robots.txt b/playbooks/roles/mailman3/files/robots.txt new file mode 100644 index 0000000000..1f62497f52 --- /dev/null +++ b/playbooks/roles/mailman3/files/robots.txt @@ -0,0 +1,7 @@ +User-agent: * + +Disallow: /accounts/* +Allow: /archives/* +Allow: /mailman3/lists/* + +Crawl-delay: 2 diff --git a/playbooks/roles/mailman3/tasks/main.yaml b/playbooks/roles/mailman3/tasks/main.yaml index 36eba320c1..d94c0560ec 100644 --- a/playbooks/roles/mailman3/tasks/main.yaml +++ b/playbooks/roles/mailman3/tasks/main.yaml @@ -141,6 +141,22 @@ shell: cmd: docker image prune -f +- name: Create robots.txt location dir + file: + path: /var/www/robots + state: directory + owner: root + group: root + mode: '0755' + +- name: Copy the robots.txt + copy: + src: robots.txt + dest: /var/www/robots/robots.txt + owner: root + group: root + mode: '0644' + - name: Install apache2 package: name: diff --git a/playbooks/roles/mailman3/templates/mailman.vhost.j2 b/playbooks/roles/mailman3/templates/mailman.vhost.j2 index e689af2f1a..0182ecf718 100644 --- a/playbooks/roles/mailman3/templates/mailman.vhost.j2 +++ b/playbooks/roles/mailman3/templates/mailman.vhost.j2 @@ -46,6 +46,7 @@ RewriteEngine On + RewriteRule ^/robots.txt$ /var/www/robots/robots.txt [L] RewriteRule "/pipermail/(.*)" "/var/lib/mailman/web-data/mm2archives/%{HTTP_HOST}/public/$1" RewriteRule "/cgi-bin/mailman/listinfo/(.*)" "https://%{HTTP_HOST}/mailman3/lists/$1.%{HTTP_HOST}/" RewriteRule "/cgi-bin/mailman/listinfo" "https://%{HTTP_HOST}/mailman3/lists/" @@ -66,4 +67,8 @@ Allow from all Require all granted + + + Require all granted + diff --git a/testinfra/test_lists_opendev_org.py b/testinfra/test_lists_opendev_org.py index 0dcc32834a..8199767e73 100644 --- a/testinfra/test_lists_opendev_org.py +++ b/testinfra/test_lists_opendev_org.py @@ -36,6 +36,13 @@ def test_apache2_listening(host): apache2_https = host.socket("tcp://0.0.0.0:443") assert apache2_https.is_listening +def test_robots(host): + cmd = host.run('curl --insecure ' + '--resolve lists.opendev.org:443:127.0.0.1 ' + 'https://lists.opendev.org/robots.txt') + assert 'Disallow: /accounts/*' in cmd.stdout + assert 'Allow: /archives/*' in cmd.stdout + def test_mailman3_screenshots(host): shots = ( ("https://lists.opendev.org:443", None, "mm3-opendev-main.png"),