From eeac5467c3a3d927cb108014fb22549332cab4bb Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Wed, 29 Apr 2020 08:24:50 -0700 Subject: [PATCH] Set up robots.txt on lists servers This sets up a robots.txt on our lists servers. To start this file prevents SEMrush bot from indexing our lists as that has been causing lists.openstack.org to OOM with many listinfo processes started by Apache. We've avoided this OOM by manually configuring this robots.txt. Other things we have ruled out are bup and input email causes qrunner's to grow unexpectedly large. Fairly confident this bot is the trigger. Note this fixes testing by adding 'hieradata' to set listpassword var. Depends-On: https://review.opendev.org/724389 Change-Id: Id4f6739a8cf6a01f9796fa54c86ba1af3e31fecf --- modules/openstack_project/files/mailman/robots.txt | 2 ++ modules/openstack_project/manifests/lists.pp | 9 +++++++++ playbooks/zuul/run-base.yaml | 1 + .../zuul/templates/host_vars/lists.openstack.org.yaml.j2 | 1 + 4 files changed, 13 insertions(+) create mode 100644 modules/openstack_project/files/mailman/robots.txt create mode 100644 playbooks/zuul/templates/host_vars/lists.openstack.org.yaml.j2 diff --git a/modules/openstack_project/files/mailman/robots.txt b/modules/openstack_project/files/mailman/robots.txt new file mode 100644 index 0000000000..a0ab15bd9c --- /dev/null +++ b/modules/openstack_project/files/mailman/robots.txt @@ -0,0 +1,2 @@ +User-agent: SemrushBot +Disallow: / diff --git a/modules/openstack_project/manifests/lists.pp b/modules/openstack_project/manifests/lists.pp index 5128d6c747..320a591712 100644 --- a/modules/openstack_project/manifests/lists.pp +++ b/modules/openstack_project/manifests/lists.pp @@ -8,6 +8,15 @@ class openstack_project::lists( multihost => true, } + file { '/var/www/robots.txt': + ensure => present, + source => 'puppet:///modules/openstack_project/mailman/robots.txt', + owner => 'root', + group => 'root', + replace => true, + mode => '0444', + } + # Disable inactive admins user::virtual::disable { 'oubiwann': } user::virtual::disable { 'rockstar': } diff --git a/playbooks/zuul/run-base.yaml b/playbooks/zuul/run-base.yaml index 560835af08..e6efaceb4e 100644 --- a/playbooks/zuul/run-base.yaml +++ b/playbooks/zuul/run-base.yaml @@ -67,6 +67,7 @@ - host_vars/etherpad01.opendev.org.yaml - host_vars/letsencrypt01.opendev.org.yaml - host_vars/letsencrypt02.opendev.org.yaml + - host_vars/lists.openstack.org.yaml - host_vars/gitea99.opendev.org.yaml - host_vars/mirror01.openafs.provider.opendev.org.yaml - host_vars/mirror-update01.opendev.org.yaml diff --git a/playbooks/zuul/templates/host_vars/lists.openstack.org.yaml.j2 b/playbooks/zuul/templates/host_vars/lists.openstack.org.yaml.j2 new file mode 100644 index 0000000000..2d38e6c9a3 --- /dev/null +++ b/playbooks/zuul/templates/host_vars/lists.openstack.org.yaml.j2 @@ -0,0 +1 @@ +listpassword: notarealpassword