From a3c46a2de214b554d1db8cde55322bc5f69dedaf Mon Sep 17 00:00:00 2001 From: Sascha Leib Date: Sat, 20 Sep 2025 21:27:47 +0200 Subject: [PATCH] Improved IP-Ranges handling --- action.php | 2 +- config/default-config.json | 20 ++-- config/known-clients.json | 6 +- config/known-ipranges.json | 214 ++++++++++--------------------------- script.js | 157 ++++++++++++++++++++------- 5 files changed, 189 insertions(+), 210 deletions(-) diff --git a/action.php b/action.php index 982c9f5..7c68a3b 100644 --- a/action.php +++ b/action.php @@ -57,7 +57,7 @@ class action_plugin_botmon extends DokuWiki_Action_Plugin { } // add the deferred script loader:: - $code .= DOKU_TAB . "addEventListener('load', function(){" . NL; + $code .= DOKU_TAB . "addEventListener('DOMContentLoaded', function(){" . NL; $code .= DOKU_TAB . DOKU_TAB . "const e=document.createElement('script');" . NL; $code .= DOKU_TAB . DOKU_TAB . "e.async=true;e.defer=true;" . NL; $code .= DOKU_TAB . DOKU_TAB . "e.src='".DOKU_BASE."lib/plugins/botmon/client.js';" . NL; diff --git a/config/default-config.json b/config/default-config.json index 75bccc5..5898f80 100644 --- a/config/default-config.json +++ b/config/default-config.json @@ -5,7 +5,7 @@ "id": "botIpRange", "desc": "Common Bot IP range", "bot": 50 }, - {"func": "matchesClient", "params": ["aol", "msie", "chromeold","oldedge"], + {"func": "matchesClient", "params": ["aol", "msie", "chromeold","oldedge","operaold"], "id": "oldClient", "desc": "Obsolete browser version", "bot": 40 }, @@ -73,9 +73,12 @@ "ipRanges": [ {"from": "3.0.0.0", "to": "3.255.255.254", "label": "Amazon Data Services [US]"}, {"from": "8.127.0.0", "to": "8.223.255.254", "label": "Alibaba [CN]"}, + {"from": "14.160.0.0", "to": "14.191.255.255", "m": 11, "label": "VNPT [VN]"}, {"from": "24.240.0.0", "to": "24.247.255.255", "m": 13, "label": "Charter [US]"}, {"from": "27.106.0.0", "to": "27.106.127.254", "label": "Huawei [US]"}, {"from": "34.0.0.0", "to": "34.191.255.254", "label": "Google LLC"}, + {"from": "43.132.0.0", "to": "43.132.255.254", "m": 16, "label": "Tencent [CN]"}, + {"from": "43.133.0.0", "to": "43.133.255.254", "m": 16, "label": "Tencent [CN]"}, {"from": "45.0.0.0", "to": "45.255.255.254", "label": "Various small ISPs, mostly BR"}, {"from": "46.250.160.0", "to": "46.250.191.254", "label": "Huawei [MX]"}, {"from": "47.200.0.0", "to": "47.203.255.255", "m": 14, "label": "Frontier Communications [US]"}, @@ -88,10 +91,12 @@ {"from": "104.196.0.0", "to": "104.199.255.255", "m": 14, "label": "Google LLC"}, {"from": "110.238.80.0", "to": "110.238.127.254", "label": "Huawei [SG]"}, {"from": "111.119.192.0", "to": "111.119.255.254", "label": "Huawei [SG]"}, + {"from": "113.160.0.0", "to": "113.191.255.254", "m": 11, "label": "VNPT [VN]"}, {"from": "119.0.0.0", "to": "119.207.255.254", "label": "Unicom [CN]"}, {"from": "121.91.168.", "to": "121.91.175.254", "label": "Huawei [HK]"}, {"from": "122.8.0.0", "to": "122.8.255.254", "label": "CN-ISP [CN]"}, {"from": "122.9.0.0", "to": "122.9.255.254", "label": "Huawei [CN]"}, + {"from": "123.16.0.0", "to": "123.31.255.255", "m": 12, "label": "VNPT [VN]"}, {"from": "124.243.128.0", "to": "124.243.191.254", "label": "Huawei [SG]"}, {"from": "142.147.128.0", "to": "1142.147.255.254", "label": "Web2Objects LLC [US]"}, {"from": "150.40.128.0", "to": "150.40.255.254", "label": "Huawei [HK]"}, @@ -101,13 +106,14 @@ {"from": "177.0.0.0", "to": "177.255.255.254", "label": "BrasilNET [BR]"}, {"from": "179.0.0.0", "to": "179.255.255.254", "label": "BrasilNET [BR]"}, {"from": "183.87.32.0", "to": "183.87.159.254", "label": "Huawei [HK]"}, - {"from": "186.0.0.0", "to": "186.255.255.254", "label": "South-American ISPs (186.x)"}, - {"from": "187.0.0.0", "to": "187.255.255.254", "label": "South-American ISPs (187.x)"}, - {"from": "188.0.0.0", "to": "188.255.255.254", "label": "South-American ISPs (188.x)"}, - {"from": "189.0.0.0", "to": "189.255.255.254", "label": "South-American ISPs (189.x)"}, - {"from": "190.0.0.0", "to": "190.255.255.254", "label": "South-American ISPs (190.x)"}, + {"from": "186.0.0.0", "to": "186.255.255.254", "m": 8, "label": "South-American ISPs (186.x)"}, + {"from": "187.0.0.0", "to": "187.255.255.254", "m": 8, "label": "South-American ISPs (187.x)"}, + {"from": "188.0.0.0", "to": "188.255.255.254", "m": 8, "label": "South-American ISPs (188.x)"}, + {"from": "189.0.0.0", "to": "189.255.255.254", "m": 8, "label": "South-American ISPs (189.x)"}, + {"from": "190.0.0.0", "to": "190.255.255.254", "m": 8, "label": "South-American ISPs (190.x)"}, + {"from": "191.0.0.0", "to": "191.255.255.254", "m": 8, "label": "South-American ISPs (191.x)"}, {"from": "192.124.170.0", "to": "192.124.182.254", "label": "Relcom [CZ]"}, - {"from": "195.37.0.0", "to": "195.37.255.255", "label": "DFN [DE]"}, + {"from": "195.37.0.0", "to": "195.37.255.255", "m": 16, "label": "DFN [DE]"}, {"from": "2001:4860::::::", "to": "2001:4860:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Google LLC [US]"}, {"from": "2001:0ee0::::::", "to": "2001:ee3:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "label": "VNPT [VN]"}, {"from": "2408:8210::::::", "to": "2408:8210:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "label": "China Unicom [CN]"}, diff --git a/config/known-clients.json b/config/known-clients.json index 450cc74..2d30ce1 100644 --- a/config/known-clients.json +++ b/config/known-clients.json @@ -1,7 +1,11 @@ [ {"n": "Opera", "id": "opera", - "rx": [ "\\sOpera\\/.*?Version\\/(\\S+)", "Opera\\/(\\S+)", "\\sOPR\\/(\\d+)\\." ] + "rx": [ "\\sOpera\\/.*?Version\\/(1\\d\\d)\\.", "Opera\\/(1\\d\\d)\\.", "\\sOPR\\/(1\\d\\d)\\." ] + }, + {"n": "Opera Old", + "id": "operaold", + "rx": [ "\\sOpera\\/.*?Version\\/(\\d\\d)\\.", "Opera\\/(\\d\\d)\\.", "\\sOPR\\/(\\d\\d)\\." ] }, {"n": "Firefox on iOS", "id": "firefox", diff --git a/config/known-ipranges.json b/config/known-ipranges.json index 2542329..ca451b6 100644 --- a/config/known-ipranges.json +++ b/config/known-ipranges.json @@ -1,160 +1,54 @@ -{ - "isps": [ - {"i": "hetzner", "n": "Hetzner Online GmbH", "g": "DE"}, - {"i": "huawei", "n": "Huawei Cloud", "g": "HK"} - ], - "5": [ - {"9": {"i": "hetzner", "g": "FI"}}, - {"75": [ - {"128": {"m": 17, "i": "hetzner", "g": "DE"}} - ]} - ], - "23": [ - {"88": [ - {"0": {"m": 17, "i": "hetzner", "g": "DE"}} - ]} - ], - "37": [ - {"27": {"i": "hetzner", "g": "FI"}} - ], - "46": [ - {"4": {"i": "hetzner", "g": "DE"}}, - {"62": [ - {"128": {"m": 17, "i": "hetzner", "g": "FI"}} - ]} - ], - "49": [ - {"12": {"i": "hetzner", "g": "DE"}}, - {"13": {"i": "hetzner", "g": "DE"}} - ], - "65": [ - {"108": {"i": "hetzner", "g": "FI"}}, - {"109": {"i": "hetzner", "g": "FI"}}, - {"210": {"i": "hetzner", "g": "FI"}} - ], - "77": [ - {"42": [ - {"0": {"m": 17, "i": "hetzner", "g": "FI"}} - ]} - ], - "78": [ - {"46": {"i": "hetzner", "g": "DE"}}, - {"47": {"i": "hetzner", "g": "DE"}} - ], - "85": [ - {"10": [ - {"192": {"m": 18, "i": "hetzner", "g": "FI"}} - ]} - ], - "88": [ - {"99": {"i": "hetzner", "g": "DE"}}, - {"198": {"i": "hetzner", "g": "DE"}} - ], - "91": [ - {"98": {"i": "hetzner", "g": "DE"}}, - {"99": {"i": "hetzner", "g": "DE"}}, - {"107": [ - {"128": {"m": 17, "i": "hetzner", "g": "DE"}} - ]} - ], - "94": [ - {"130": {"i": "hetzner", "g": "DE"}} - ], - "95": [ - {"216": {"i": "hetzner", "g": "FI"}}, - {"217": {"i": "hetzner", "g": "FI"}} - ], - "110": [ - {"239": [ - {"64": {"m": 19, "i": "huawei", "g": "SG"}}, - {"96": {"m": 19, "i": "huawei", "g": "SG"}} - ]} - ], - "114": [ - {"119": [ - {"128": {"m": 19, "i": "huawei", "g": "SG"}} - ]} - ], - "116": [ - {"202": {"i": "hetzner", "g": "DE"}}, - {"203": {"i": "hetzner", "g": "DE"}} - ], - "119": [ - {"8": [ - {"32": {"m": 19, "i": "huawei", "g": "HK"}}, - {"96": {"m": 19, "i": "huawei", "g": "HK"}}, - {"160": {"m": 19, "i": "huawei", "g": "SG"}} - ]} - ], - "124": [ - {"243": [ - {"128": {"m": 18, "i": "huawei", "g": "SG"}} - ]} - ], - "128": [ - {"140": [ - {"0": {"m": 17, "i": "hetzner", "g": "DE"}} - ]} - ], - "135": [ - {"181": {"i": "hetzner", "g": "FI"}} - ], - "138": [ - {"199": [ - {"128": {"m": 17, "i": "hetzner", "g": "DE"}} - ]} - ], - "142": [ - {"132": [ - {"128": {"m": 17, "i": "hetzner", "g": "DE"}} - ]} - ], - "154": [ - {"220": [ - {"192": {"m": 19, "i": "huawei", "g": "HK"}} - ]} - ], - "157": [ - {"90": {"i": "hetzner", "g": "DE"}}, - {"180": [ - {"0": {"m": 17, "i": "hetzner", "g": "FI"}} - ]} - ], - "159": [ - {"69": {"i": "hetzner", "g": "DE"}} - ], - "162": [ - {"55": {"i": "hetzner", "g": "DE"}} - ], - "167": [ - {"233": {"i": "hetzner", "g": "DE"}}, - {"235": {"i": "hetzner", "g": "DE"}} - ], - "168": [ - {"119": {"i": "hetzner", "g": "DE"}} - ], - "176": [ - {"9": {"i": "hetzner", "g": "DE"}} - ], - "178": [ - {"63": {"i": "hetzner", "g": "DE"}} - ], - "188": [ - {"34": [ - {"128": {"m": 17, "i": "hetzner", "g": "DE"}} - ]}, - {"40": {"i": "hetzner", "g": "DE"}}, - {"245": {"i": "hetzner", "g": "DE"}} - ], - "195": [ - {"201": {"i": "hetzner", "g": "DE"}} - ], - "213": [ - {"133": [ - {"69": {"m": 19, "i": "hetzner", "g": "DE"}} - ]}, - {"239": [ - {"192": {"m": 18, "i": "hetzner", "g": "DE"}} - ]} - ] -} \ No newline at end of file +[ + {"from": "3.0.0.0", "to": "3.255.255.254", "label": "Amazon Data Services [US]"}, + {"from": "8.127.0.0", "to": "8.223.255.254", "label": "Alibaba [CN]"}, + {"from": "14.160.0.0", "to": "14.191.255.255", "m": 11, "label": "VNPT [VN]"}, + {"from": "24.240.0.0", "to": "24.247.255.255", "m": 13, "label": "Charter [US]"}, + {"from": "27.106.0.0", "to": "27.106.127.254", "label": "Huawei [US]"}, + {"from": "34.0.0.0", "to": "34.191.255.254", "label": "Google LLC"}, + {"from": "43.132.0.0", "to": "43.132.255.254", "m": 16, "label": "Tencent [CN]"}, + {"from": "43.133.0.0", "to": "43.133.255.254", "m": 16, "label": "Tencent [CN]"}, + {"from": "45.0.0.0", "to": "45.255.255.254", "label": "Various small ISPs, mostly BR"}, + {"from": "46.250.160.0", "to": "46.250.191.254", "label": "Huawei [MX]"}, + {"from": "47.200.0.0", "to": "47.203.255.255", "m": 14, "label": "Frontier Communications [US]"}, + {"from": "49.0.200.0", "to": "49.0.255.254", "label": "Huawei [SG]"}, + {"from": "52.220.0.0", "to": "52.222.255.254", "label": "Amazon Data Services"}, + {"from": "66.249.64.0", "to": "66.249.95.255", "m": 19, "label": "Google LLC"}, + {"from": "84.37.35.0", "to": "84.37.255.254", "label": "GTT.net [US]"}, + {"from": "94.74.64.0", "to": "94.74.127.254", "label": "Huawei [HK]"}, + {"from": "101.0.0.0", "to": "101.255.255.254", "label": "ChinaNet [CN]"}, + {"from": "104.196.0.0", "to": "104.199.255.255", "m": 14, "label": "Google LLC"}, + {"from": "110.238.80.0", "to": "110.238.127.254", "label": "Huawei [SG]"}, + {"from": "111.119.192.0", "to": "111.119.255.254", "label": "Huawei [SG]"}, + {"from": "113.160.0.0", "to": "113.191.255.254", "m": 11, "label": "VNPT [VN]"}, + {"from": "119.0.0.0", "to": "119.207.255.254", "label": "Unicom [CN]"}, + {"from": "121.91.168.", "to": "121.91.175.254", "label": "Huawei [HK]"}, + {"from": "122.8.0.0", "to": "122.8.255.254", "label": "CN-ISP [CN]"}, + {"from": "122.9.0.0", "to": "122.9.255.254", "label": "Huawei [CN]"}, + {"from": "123.16.0.0", "to": "123.31.255.255", "m": 12, "label": "VNPT [VN]"}, + {"from": "124.243.128.0", "to": "124.243.191.254", "label": "Huawei [SG]"}, + {"from": "142.147.128.0", "to": "1142.147.255.254", "label": "Web2Objects LLC [US]"}, + {"from": "150.40.128.0", "to": "150.40.255.254", "label": "Huawei [HK]"}, + {"from": "159.138.0.0", "to": "159.138.225.254", "label": "Huawei [TH]"}, + {"from": "162.128.0.0", "to": "162.128.255.254", "label": "Zenlayer [SG]"}, + {"from": "166.108.192.0", "to": "166.108.255.254", "label": "Huawei [SG]"}, + {"from": "177.0.0.0", "to": "177.255.255.254", "label": "BrasilNET [BR]"}, + {"from": "179.0.0.0", "to": "179.255.255.254", "label": "BrasilNET [BR]"}, + {"from": "183.87.32.0", "to": "183.87.159.254", "label": "Huawei [HK]"}, + {"from": "186.0.0.0", "to": "186.255.255.254", "m": 8, "label": "South-American ISPs (186.x)"}, + {"from": "187.0.0.0", "to": "187.255.255.254", "m": 8, "label": "South-American ISPs (187.x)"}, + {"from": "188.0.0.0", "to": "188.255.255.254", "m": 8, "label": "South-American ISPs (188.x)"}, + {"from": "189.0.0.0", "to": "189.255.255.254", "m": 8, "label": "South-American ISPs (189.x)"}, + {"from": "190.0.0.0", "to": "190.255.255.254", "m": 8, "label": "South-American ISPs (190.x)"}, + {"from": "191.0.0.0", "to": "191.255.255.254", "m": 8, "label": "South-American ISPs (191.x)"}, + {"from": "192.124.170.0", "to": "192.124.182.254", "label": "Relcom [CZ]"}, + {"from": "195.37.0.0", "to": "195.37.255.255", "m": 16, "label": "DFN [DE]"}, + {"from": "2001:4860::::::", "to": "2001:4860:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Google LLC [US]"}, + {"from": "2001:0ee0::::::", "to": "2001:ee3:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "label": "VNPT [VN]"}, + {"from": "2408:8210::::::", "to": "2408:8210:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "label": "China Unicom [CN]"}, + {"from": "2600:1f00::::::", "to": "2600:1fff:ffff:ffff:ffff:ffff:ffff:ffff", "m": "Amazon Cloud"}, + {"from": "2603:6010::::::", "to": "2603:6010:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Charter [US]"}, + {"from": "2603:8000::::::", "to": "2603:80ff:ffff:ffff:ffff:ffff:ffff:ffff", "m": 24, "label": "Charter [US]"}, + {"from": "2607:a400::::::", "to": "2607:a400:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Zenlayer Inc. [US]"}, + {"from": "2804:::::::", "to": "2804:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "m": 16, "label": "Inspire [BR]"}, + {"from": "2a0a:4cc0::::::", "to": "2a0a:4cc0:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "label": "Netcup [DE]"} +] \ No newline at end of file diff --git a/script.js b/script.js index a238ecd..1d49113 100644 --- a/script.js +++ b/script.js @@ -1104,61 +1104,96 @@ BotMon.live = { }, + ipRanges: { + + init: function() { + //console.log('BotMon.live.data.ipRanges.init()'); + // #TODO: Load from separate IP-Ranges file + }, + + // the actual bot list is stored here: + _list: [], + + add: function(data) { + //console.log('BotMon.live.data.ipRanges.add(',data,')'); + + const me = BotMon.live.data.ipRanges; + + // convert IP address to easier comparable form: + const ip2Num = BotMon.t._ip2Num; + + let item = { + 'from': ip2Num(data.from), + 'to': ip2Num(data.to), + 'label': data.label + }; + me._list.push(item); + + }, + + match: function(ip) { + //console.log('BotMon.live.data.ipRanges.match(',ip,')'); + + const me = BotMon.live.data.ipRanges; + + // convert IP address to easier comparable form: + const ipNum = BotMon.t._ip2Num(ip); + + for (let i=0; i < me._list.length; i++) { + const ipRange = me._list[i]; + + if (ipNum >= ipRange.from && ipNum <= ipRange.to) { + return ipRange; + } + + }; + return null; + } + }, + rules: { - // loads the list of rules and settings from a JSON file: + + /** + * Initializes the rules data. + * + * Loads the default config file and the user config file (if present). + * The default config file is used if the user config file does not have a certain setting. + * The user config file can override settings from the default config file. + * + * The rules are loaded from the `rules` property of the config files. + * The IP ranges are loaded from the `ipRanges` property of the config files. + * + * If an error occurs while loading the config file, it is displayed in the status bar. + * After the config file is loaded, the status bar is hidden. + */ init: async function() { //console.info('BotMon.live.data.rules.init()'); // Load the list of known bots: BotMon.live.gui.status.showBusy("Loading list of rules …"); - // relative file path to the rules file: - const filePath = 'config/default-config.json'; - // load the rules file: - this._loadrulesFile(BotMon._baseDir + filePath); - }, + const me = BotMon.live.data; - /** - * Loads the list of rules and settings from a JSON file. - * @param {String} url - the URL from which to load the rules file. - */ - - _loadrulesFile: async function(url) { - //console.info('BotMon.live.data.rules._loadrulesFile(',url,')');} - - const me = BotMon.live.data.rules; try { - const response = await fetch(url); - if (!response.ok) { - throw new Error(`${response.status} ${response.statusText}`); - } + BotMon.live.data._loadSettingsFile(['user-config', 'default-config'], + (json) => { - const json = await response.json(); + // override the threshold? + if (json.threshold) me._threshold = json.threshold; - if (json.rules) { - me._rulesList = json.rules; - } + // set the rules list: + if (json.rules) { + me.rules._rulesList = json.rules; + } - // override the threshold? - if (json.threshold) me._threshold = json.threshold; - - if (json.ipRanges) { - // clean up the IPs first: - let list = []; - json.ipRanges.forEach( it => { - let item = { - 'from': BotMon.t._ip2Num(it.from), - 'to': BotMon.t._ip2Num(it.to), - 'label': it.label + // load the IP ranges: + if (json.ipRanges) { + json.ipRanges.forEach( it => me.ipRanges.add(it)); }; - list.push(item); - }); - me._botIPs = list; - } - - me._ready = true; + } + ); } catch (error) { BotMon.live.gui.status.setError("Error while loading the config file: " + error.message); @@ -1166,6 +1201,7 @@ BotMon.live = { BotMon.live.gui.status.hideBusy("Status: Done."); BotMon.live.data._dispatch('rules') } + }, _rulesList: [], // list of rules to find out if a visitor is a bot @@ -1398,6 +1434,45 @@ BotMon.live = { }, + /** + * Loads a settings file from the specified list of filenames. + * If the file is successfully loaded, it will call the callback function + * with the loaded JSON data. + * If no file can be loaded, it will display an error message. + * + * @param {string[]} fns - list of filenames to load + * @param {function} callback - function to call with the loaded JSON data + */ + _loadSettingsFile: async function(fns, callback) { + //console.info('BotMon.live.data._loadSettingsFile()', fns); + + const kJsonExt = '.json'; + let loaded = false; // if successfully loaded file + + for (let i=0; i