From deaae4f66591d07c649ed846c12a12a7b38d84e6 Mon Sep 17 00:00:00 2001 From: Jeremy Stanley Date: Thu, 15 Oct 2020 19:58:34 +0000 Subject: [PATCH] Block restricted user agents for the tarballs site A copy of the filter used for our Gitea farm, this same activity has been showing up on our tarballs.opendev.org site as well which is consuming available connection slots for all vhosts on the static server. This is implemented as a macro so that it can be included into additional vhosts, and put into a separate role so that it can be added to all playbooks which need it. A subsequent change will add it to the Gitea servers, eliminating the redundant copy there. Change-Id: Ic2020b753076209f7708f76744fdf746bf933bd9 --- playbooks/roles/apache-ua-filter/README.rst | 6 +++ .../apache-ua-filter/files/ua-filter.conf | 52 +++++++++++++++++++ .../roles/apache-ua-filter/handlers/main.yaml | 9 ++++ .../roles/apache-ua-filter/tasks/main.yaml | 33 ++++++++++++ .../static/files/50-tarballs.opendev.org.conf | 2 + playbooks/service-static.yaml | 1 + 6 files changed, 103 insertions(+) create mode 100644 playbooks/roles/apache-ua-filter/README.rst create mode 100644 playbooks/roles/apache-ua-filter/files/ua-filter.conf create mode 100644 playbooks/roles/apache-ua-filter/handlers/main.yaml create mode 100644 playbooks/roles/apache-ua-filter/tasks/main.yaml diff --git a/playbooks/roles/apache-ua-filter/README.rst b/playbooks/roles/apache-ua-filter/README.rst new file mode 100644 index 0000000000..f9137ad268 --- /dev/null +++ b/playbooks/roles/apache-ua-filter/README.rst @@ -0,0 +1,6 @@ +Reject requests from problematic user agent strings + +This role installs and configures a filter macro called +UserAgentFilter which can be included in Apache vhosts + +**Role Variables** diff --git a/playbooks/roles/apache-ua-filter/files/ua-filter.conf b/playbooks/roles/apache-ua-filter/files/ua-filter.conf new file mode 100644 index 0000000000..5725d5e794 --- /dev/null +++ b/playbooks/roles/apache-ua-filter/files/ua-filter.conf @@ -0,0 +1,52 @@ +# Returns a 403 Not Authorized for these user agent strings + + RewriteEngine On + + # This interesting list is taken from + # + # https://github.com/mythsman/weiboCrawler/blob/master/opener.py + # + # which appears to be a crawler for a site "weibo" which is crawling + # gitea relentlessly. It seems to rotate though these to avoid + # detection. We are seeing these very specific user-agents appear + # suggesting this code has been repurposed or has otherwise gone mad + # and started going through opendev.org + + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Windows NT 10.0; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser)" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1" [OR] + RewriteCond %{HTTP_USER_AGENT} "=MQQBrowser/26 Mozilla/5.0 (Linux; U; Android 2.3.7; zh-cn; MB200 Build/GRJ22; CyanogenMod-7) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Opera/9.80 (Android 2.3.4; Linux; Opera Mobi/build-1107180945; U; en-GB) Presto/2.8.149 Version/11.10" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (BlackBerry; U; BlackBerry 9800; en) AppleWebKit/534.1+ (KHTML, like Gecko) Version/6.0.0.337 Mobile Safari/534.1+" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.0; U; en-US) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/233.70 Safari/534.6 TouchPad/1.0" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18124" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0; HTC; Titan)" [OR] + RewriteCond %{HTTP_USER_AGENT} "=UCWEB7.0.2.37/28/999" [OR] + RewriteCond %{HTTP_USER_AGENT} "=NOKIA5700/ UCWEB7.0.2.37/28/999" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Openwave/ UCWEB7.0.2.37/28/999" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/999" + RewriteRule . - [R=403,L] + + diff --git a/playbooks/roles/apache-ua-filter/handlers/main.yaml b/playbooks/roles/apache-ua-filter/handlers/main.yaml new file mode 100644 index 0000000000..fe996bf164 --- /dev/null +++ b/playbooks/roles/apache-ua-filter/handlers/main.yaml @@ -0,0 +1,9 @@ +- name: Reload apache2 + service: + name: apache2 + state: reloaded + +- name: Restart apache2 + service: + name: apache2 + state: restarted diff --git a/playbooks/roles/apache-ua-filter/tasks/main.yaml b/playbooks/roles/apache-ua-filter/tasks/main.yaml new file mode 100644 index 0000000000..8eae116e48 --- /dev/null +++ b/playbooks/roles/apache-ua-filter/tasks/main.yaml @@ -0,0 +1,33 @@ +- name: Install apache2 + apt: + name: + - apache2 + - apache2-utils + state: present + +- name: Apache rewrite module + apache2_module: + state: present + name: rewrite + notify: Restart apache2 + +- name: Apache macro module + apache2_module: + state: present + name: macro + notify: Restart apache2 + +- name: Apache headers module + apache2_module: + state: present + name: headers + notify: Restart apache2 + +- name: Copy apache UA filter + copy: + src: ua-filter.conf + dest: /etc/apache2/conf-enabled/ua-filter.conf + owner: root + group: root + mode: 0644 + notify: Reload apache2 diff --git a/playbooks/roles/static/files/50-tarballs.opendev.org.conf b/playbooks/roles/static/files/50-tarballs.opendev.org.conf index fb3b87b3a1..739eff1823 100644 --- a/playbooks/roles/static/files/50-tarballs.opendev.org.conf +++ b/playbooks/roles/static/files/50-tarballs.opendev.org.conf @@ -268,6 +268,8 @@ Define AFS_ROOT /afs/openstack.org/project/tarballs.opendev.org SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP SSLHonorCipherOrder on + Use UserAgentFilter + Options Indexes FollowSymLinks MultiViews AllowOverrideList Redirect RedirectMatch diff --git a/playbooks/service-static.yaml b/playbooks/service-static.yaml index 31da7b08e0..bacd9bf4da 100644 --- a/playbooks/service-static.yaml +++ b/playbooks/service-static.yaml @@ -11,4 +11,5 @@ - role: openafs-client openafs_client_cache_size: "{{ afs_client_cache_size | default(50000000) }}" # 50GiB openafs_client_cache_directory: '/opt/cache/openafs' + - role: apache-ua-filter - role: static