From eb61767d1ed590087084d5b461c5956e46adcef8 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Tue, 2 May 2023 09:26:13 -0700 Subject: [PATCH] Update user agent filter list We're see crawling on gitea impact service response time. The list of user agents was compiled looking for requests that were obvious crawling behavior, filtering out the UA strings, sorting by counts then removing obvious bots that seem to be more well behaved. What we are left with is a number of user agents making requests as older unsupported versions of mobile clients. Filter those. Change-Id: I6e774c83a66e25d1b1faefc7cacf39ffb77f509a --- .../apache-ua-filter/files/ua-filter.conf | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/apache-ua-filter/files/ua-filter.conf b/playbooks/roles/apache-ua-filter/files/ua-filter.conf index 7fb40aa205..672aa420ae 100644 --- a/playbooks/roles/apache-ua-filter/files/ua-filter.conf +++ b/playbooks/roles/apache-ua-filter/files/ua-filter.conf @@ -47,7 +47,36 @@ RewriteCond %{HTTP_USER_AGENT} "=NOKIA5700/ UCWEB7.0.2.37/28/999" [OR] RewriteCond %{HTTP_USER_AGENT} "=Openwave/ UCWEB7.0.2.37/28/999" [OR] RewriteCond %{HTTP_USER_AGENT} "=Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/999" [OR] - RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Mobile Safari/537.36" + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Mobile Safari/537.36" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; U; Android 2.3.7; zh-cn; MB200 Build/GRJ22; CyanogenMod-7) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; Android 7.1.1; OPPO R9sk Build/NMF26F; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/48.0.2564.116 Mobile Safari/537.36 T7/9.1 baidubrowser/7.19.13.0 (Baidu; P1 7.1.1)" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; Android 7.1.1; OPPO R9sk Build/NMF26F; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/62.0.3202.97 Mobile Safari/537.36" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; Android 7.1.1; OPPO R9sk Build/NMF26F; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/68.0.3440.106 Mobile Safari/537.36 AWP/2.0 SogouMSE,SogouMobileBrowser/5.22.8" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; Android 7.1.1; OPPO R9sk Build/NMF26F; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/70.0.3538.80 Mobile Safari/537.36 LieBaoFast/5.12.3" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; Android 7.1.1; OPPO R9sk Build/NMF26F) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/70.0.3538.80 Mobile Safari/537.36 360 Alitephone Browser (1.5.0.90/1.0.100.1078) mso_sdk(1.0.0)" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; Android 7.1.1; OPPO R9sk) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.90 Mobile Safari/537.36 EdgA/42.0.2.3819" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; Android 7.1.1; OPPO R9sk) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.111 Mobile Safari/537.36" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; Android 7.1.1; OPPO R9sk Build/NMF26F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Mobile Safari/537.36 OPR/53.0.2569.141117" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; U; Android 7.1.1; zh-cn; OPPO R9sk Build/NMF26F) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/70.0.3538.80 Mobile Safari/537.36 OppoBrowser/10.5.1.2" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; U; Android 7.1.1; zh-CN; OPPO R9sk Build/NMF26F) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.108 UCBrowser/12.6.0.1040 Mobile Safari/537.36" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; U; Android 7.1.1; zh-cn; OPPO R9sk Build/NMF26F) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.126 MQQBrowser/9.6 Mobile Safari/537.36" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; Android 8.0.0; SM-G955U Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Mobile Safari/537.36" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; Android 9; MI 9) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.105 Mobile Safari/537.36" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; Android 9; COL-AL10 Build/HUAWEICOL-AL10; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.126 MQQBrowser/6.2 TBS/044607 Mobile Safari/537.36 MMWEBID/7140 MicroMessenger/7.0.4.1420(0x27000437) Process/tools NetType/4G Language/zh_CN" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; Android 9; MI 9 Build/PKQ1.181121.001; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/66.0.3359.126 MQQBrowser/6.2 TBS/044704 Mobile Safari/537.36 V1_AND_SQ_8.0.8_1218_YYB_D QQ/8.0.8.4115 NetType/WIFI WebP/0.3.0 Pixel/1080 StatusBarHeight/110" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; Android 9; MI 9 Build/PKQ1.181121.001; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/68.0.3440.106 Mobile Safari/537.36 AWP/2.0 SogouMSE,SogouMobileBrowser/5.22.4" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; Android 9; MI 9 Build/PKQ1.181121.001; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/73.0.3683.90 Mobile Safari/537.36 LieBaoFast/5.10.1" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; Android 9; MI 9 Build/PKQ1.181121.001) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/73.0.3683.90 Mobile Safari/537.36 Maxthon/3248" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; Android 9; MI 9 Build/PKQ1.181121.001; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/62.0.3202.97 Mobile Safari/537.36" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; Android 9; V1924A Build/PKQ1.190714.001; wv) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36 VivoBrowser/6.7.12.0" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; U; Android 9; zh-CN; MI 9 Build/PKQ1.181121.001) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.108 UCBrowser/12.5.5.1035 Mobile Safari/537.36" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; U; Android 9; zh-CN; MI 9 Build/PKQ1.181121.001) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.108 Quark/3.4.0.113 Mobile Safari/537.36" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; Android 10; SM-G981B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.162 Mobile Safari/537.36" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; Android 11; Pixel 5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.91 Mobile Safari/537.36" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (iPhone; CPU iPhone OS 12_3_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 QQ/8.0.8.458 V1_IPH_SQ_8.0.8_1_APP_A Pixel/1242 Core/WKWebView Device/Apple(iPhone XS) NetType/WIFI QBWebViewType/1 WKType/1" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (iPhone; CPU iPhone OS 12_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 MicroMessenger/7.0.5(0x17000523) NetType/WIFI Language/zh_CN" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1" [OR] + RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (iPhone; CPU iPhone OS 13_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.4 Mobile/15E148 Safari/604.1" RewriteRule . - [R=403,L]