From c31fb4d0a52b77db689869b5198247a94400b19a Mon Sep 17 00:00:00 2001 From: Sascha Leib Date: Fri, 3 Oct 2025 21:30:29 +0200 Subject: [PATCH] Web traffic monitoring --- action.php | 2 - admin.php | 10 +- client.js | 2 +- config/default-config.json | 69 ------------ config/known-bots.json | 12 ++ config/known-ipranges.json | 151 ++++++++++++++++--------- config/known-platforms.json | 2 +- script.js | 218 ++++++++++++++++++++++++++++++------ style.less | 22 +++- 9 files changed, 321 insertions(+), 167 deletions(-) diff --git a/action.php b/action.php index 7c68a3b..88c1ed6 100644 --- a/action.php +++ b/action.php @@ -81,8 +81,6 @@ class action_plugin_botmon extends DokuWiki_Action_Plugin { $username = ( !empty($INFO['userinfo']) && !empty($INFO['userinfo']['name']) ? $INFO['userinfo']['name'] : ''); - - // clean the page ID $pageId = preg_replace('/[\x00-\x1F]/', "\u{FFFD}", $INFO['id'] ?? ''); diff --git a/admin.php b/admin.php index 2e34c14..76222e5 100644 --- a/admin.php +++ b/admin.php @@ -42,7 +42,7 @@ class admin_plugin_botmon extends AdminPlugin {

Bot Monitoring Plugin

'; @@ -51,7 +51,7 @@ class admin_plugin_botmon extends AdminPlugin { } echo '
-

Today

+

Latest data

Loading …
@@ -68,6 +68,12 @@ class admin_plugin_botmon extends AdminPlugin {
+
+ +
+ Web traffic (humans only) +
+
diff --git a/client.js b/client.js index 925ffe0..432ae1b 100644 --- a/client.js +++ b/client.js @@ -73,7 +73,7 @@ console.error(err); } finally { /* send the next heartbeat signal after x seconds: */ - // setTimeout(this._onHeartbeat.bind(this, this._src.replace( this._scriptName, '/tick.php')),this._heartbeat * 1000); + setTimeout(this._onHeartbeat.bind(this, this._src.replace( this._scriptName, '/tick.php')),this._heartbeat * 1000); } } }).init(); \ No newline at end of file diff --git a/config/default-config.json b/config/default-config.json index 59ae26e..479f454 100644 --- a/config/default-config.json +++ b/config/default-config.json @@ -61,74 +61,5 @@ "id": "langMatch", "desc": "Client’s ‘Accept-Language’ header does not match the page language", "bot": 30 } - ], - "ipRanges": [ - {"from": "3.0.0.0", "to": "3.255.255.254", "label": "Amazon Data Services [US]"}, - {"from": "5.161.0.0", "to": "5.161.255.255", "m": 16, "label": "Hetzner [US]"}, - {"from": "8.127.0.0", "to": "8.223.255.254", "label": "Alibaba [CN]"}, - {"from": "14.160.0.0", "to": "14.191.255.254", "m": 11, "label": "VNPT [VN]"}, - {"from": "14.192.0.0", "to": "14.255.255.254", "m": 10, "label": "VNPT [VN]"}, - {"from": "14.224.0.0", "to": "24.255.255.254", "m": 11, "label": "Charter [US]"}, - {"from": "27.106.0.0", "to": "27.106.127.254", "m": 17, "label": "Huawei [US]"}, - {"from": "34.0.0.0", "to": "34.191.255.254", "label": "Google LLC"}, - {"from": "43.132.0.0", "to": "43.132.255.254", "m": 16, "label": "Tencent [CN]"}, - {"from": "43.133.0.0", "to": "43.133.255.254", "m": 16, "label": "Tencent [CN]"}, - {"from": "45.0.0.0", "to": "45.255.255.254", "label": "Various small ISPs, mostly BR"}, - {"from": "46.250.160.0", "to": "46.250.191.254", "m": 19, "label": "Huawei [MX]"}, - {"from": "47.200.0.0", "to": "47.203.255.254", "m": 14, "label": "Frontier Communications [US]"}, - {"from": "49.0.192.0", "to": "49.0.255.254", "m": 18, "label": "Huawei [SG]"}, - {"from": "52.220.0.0", "to": "52.222.255.254", "label": "Amazon Data Services"}, - {"from": "66.249.64.0", "to": "66.249.95.254", "m": 19, "label": "Google LLC"}, - {"from": "84.37.35.0", "to": "84.37.255.254", "label": "GTT.net [US]"}, - {"from": "94.74.64.0", "to": "94.74.127.254", "m": 18, "label": "Huawei [HK]"}, - {"from": "101.0.0.0", "to": "101.255.255.254", "m": 8,"label": "ChinaNet [CN]"}, - {"from": "104.196.0.0", "to": "104.199.255.254", "m": 14, "label": "Google LLC"}, - {"from": "110.238.64.0", "to": "110.238.127.254", "m": 18, "label": "Huawei [SG]"}, - {"from": "111.119.192.0", "to": "111.119.255.254", "m": 18, "label": "Huawei [SG]"}, - {"from": "113.160.0.0", "to": "113.191.255.254", "m": 11, "label": "VNPT [VN]"}, - {"from": "119.8.0.0", "to": "119.8.255.254", "m": 16, "label": "Huawei [HK]"}, - {"from": "119.13.0.0", "to": "119.13.255.254", "m": 16, "label": "Huawei [HK]"}, - {"from": "121.91.168.0", "to": "121.91.175.254", "m": 21, "label": "Huawei [HK]"}, - {"from": "122.8.0.0", "to": "122.8.255.254", "label": "CN-ISP [CN]"}, - {"from": "122.9.0.0", "to": "122.9.255.254", "m": 16, "label": "Huawei [CN]"}, - {"from": "123.16.0.0", "to": "123.31.255.254", "m": 12, "label": "VNPT [VN]"}, - {"from": "124.243.128.0", "to": "124.243.191.254", "m": 18, "label": "Huawei [SG]"}, - {"from": "138.59.0.0", "to": "138.59.225.254", "m": 16, "label": "South-American ISPs (138.59.x)"}, - {"from": "138.121.0.0", "to": "138.121.225.254", "m": 16, "label": "South-American ISPs (138.121.x)"}, - {"from": "142.147.128.0", "to": "1142.147.255.254", "m": 17, "label": "Web2Objects LLC [US]"}, - {"from": "146.174.128.0", "to": "146.174.191.254", "m": 18, "label": "Huawei [SG]"}, - {"from": "150.40.128.0", "to": "150.40.255.254", "m": 17, "label": "Huawei [HK]"}, - {"from": "159.138.0.0", "to": "159.138.225.254", "m": 16, "label": "Huawei [TH]"}, - {"from": "162.128.0.0", "to": "162.128.255.254", "m": 16, "label": "Zenlayer [SG]"}, - {"from": "166.108.192.0", "to": "166.108.255.254", "m": 18, "label": "Huawei [SG]"}, - {"from": "168.232.192.0", "to": "168.232.255.254", "m": 16, "label": "South-American ISPs (168.232.x)"}, - {"from": "170.82.0.0", "to": "170.82.255.254", "m": 16, "label": "South-American ISPs (170.254.x)"}, - {"from": "170.254.0.0", "to": "170.254.255.254", "m": 16, "label": "South-American ISPs (170.82.x)"}, - {"from": "171.224.0.0", "to": "171.239.255.254", "m": 12, "label": "Viettel [VN]"}, - {"from": "177.0.0.0", "to": "177.255.255.254", "m": 8, "label": "BrasilNET [BR]"}, - {"from": "179.0.0.0", "to": "179.255.255.254", "m": 8, "label": "BrasilNET [BR]"}, - {"from": "183.87.32.0", "to": "183.87.63.254", "label": "Huawei [HK]"}, - {"from": "186.0.0.0", "to": "186.255.255.254", "m": 8, "label": "South-American ISPs (186.x)"}, - {"from": "187.0.0.0", "to": "187.255.255.254", "m": 8, "label": "South-American ISPs (187.x)"}, - {"from": "188.0.0.0", "to": "188.255.255.254", "m": 8, "label": "South-American ISPs (188.x)"}, - {"from": "189.0.0.0", "to": "189.255.255.254", "m": 8, "label": "South-American ISPs (189.x)"}, - {"from": "190.0.0.0", "to": "190.255.255.254", "m": 8, "label": "South-American ISPs (190.x)"}, - {"from": "191.0.0.0", "to": "191.255.255.254", "m": 8, "label": "South-American ISPs (191.x)"}, - {"from": "192.124.170.0", "to": "192.124.182.254", "label": "Relcom [CZ]"}, - {"from": "195.37.0.0", "to": "195.37.255.254", "m": 16, "label": "DFN [DE]"}, - {"from": "200.0.0.0", "to": "200.255.255.254", "m": 8, "label": "South-American ISPs (200.x)"}, - {"from": "201.0.0.0", "to": "201.255.255.254", "m": 8, "label": "South-American ISPs (201.x)"}, - {"from": "212.95.128.0", "to": "212.95.159.254", "m": 19, "label": "Asiacell [IQ]"}, - {"from": "222.252.0.0", "to": "222.252.255.254", "m": 14, "label": "VNPT [VN]"}, - {"from": "2001:4860::::::", "to": "2001:4860:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Google LLC [US]"}, - {"from": "2001:0ee0::::::", "to": "2001:ee3:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "label": "VNPT [VN]"}, - {"from": "2408:8210::::::", "to": "2408:8210:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "label": "China Unicom [CN]"}, - {"from": "2600:1f00::::::", "to": "2600:1fff:ffff:ffff:ffff:ffff:ffff:ffff", "m": 24, "label": "Amazon Cloud"}, - {"from": "2603:6010::::::", "to": "2603:6010:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Charter [US]"}, - {"from": "2603:8000::::::", "to": "2603:80ff:ffff:ffff:ffff:ffff:ffff:ffff", "m": 24, "label": "Charter [US]"}, - {"from": "2607:a400::::::", "to": "2607:a400:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Zenlayer Inc. [US]"}, - {"from": "2804:::::::", "to": "2804:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "m": 16, "label": "Inspire [BR]"}, - {"from": "2a09:bac2::::::", "to": "2a09:bac2:0:ffff:ffff:ffff:ffff:ffff", "m": 48, "label": "Cloudflare"}, - {"from": "2a0a:4cc0::::::", "to": "2a0a:4cc0:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "label": "Netcup [DE]"} ] } \ No newline at end of file diff --git a/config/known-bots.json b/config/known-bots.json index 0fb0cd5..4e68292 100644 --- a/config/known-bots.json +++ b/config/known-bots.json @@ -297,5 +297,17 @@ "r": [], "rx": ["Go\\-http\\-client\\/(\\d+)", "quic\\-go\\-HTTP\\/(\\d+)"], "url": "https://github.com/golang/goen.wi" + }, + {"id": "bnl", + "n": "BnL Harvester", + "r": [], + "rx": ["NLUX_IAHarvester\\/(\\d+)"], + "url": "http://crawl.bnl.lu/" + }, + {"id": "turnitin", + "n": "TurnitinBot", + "r": [], + "rx": ["Turnitin\\s"], + "url": "https://www.turnitin.com/robot/crawlerinfo.html" } ] \ No newline at end of file diff --git a/config/known-ipranges.json b/config/known-ipranges.json index ff73d73..415b6cf 100644 --- a/config/known-ipranges.json +++ b/config/known-ipranges.json @@ -1,55 +1,96 @@ -[ - {"from": "3.0.0.0", "to": "3.255.255.254", "m": 8, "label": "Amazon Data Services [US]"}, - {"from": "8.127.0.0", "to": "8.223.255.254", "label": "Alibaba [CN]"}, - {"from": "14.160.0.0", "to": "14.191.255.255", "m": 11, "label": "VNPT [VN]"}, - {"from": "24.240.0.0", "to": "24.247.255.255", "m": 13, "label": "Charter [US]"}, - {"from": "27.106.0.0", "to": "27.106.127.254", "label": "Huawei [US]"}, - {"from": "34.0.0.0", "to": "34.191.255.254", "label": "Google LLC"}, - {"from": "43.132.0.0", "to": "43.132.255.254", "m": 16, "label": "Tencent [CN]"}, - {"from": "43.133.0.0", "to": "43.133.255.254", "m": 16, "label": "Tencent [CN]"}, - {"from": "45.0.0.0", "to": "45.255.255.254", "label": "Various small ISPs, mostly BR"}, - {"from": "46.250.160.0", "to": "46.250.191.254", "label": "Huawei [MX]"}, - {"from": "47.200.0.0", "to": "47.203.255.255", "m": 14, "label": "Frontier Communications [US]"}, - {"from": "49.0.200.0", "to": "49.0.255.254", "label": "Huawei [SG]"}, - {"from": "52.220.0.0", "to": "52.222.255.254", "label": "Amazon Data Services"}, - {"from": "66.249.64.0", "to": "66.249.95.255", "m": 19, "label": "Google LLC"}, - {"from": "84.37.35.0", "to": "84.37.255.254", "label": "GTT.net [US]"}, - {"from": "94.74.64.0", "to": "94.74.127.254", "label": "Huawei [HK]"}, - {"from": "101.0.0.0", "to": "101.255.255.254", "label": "ChinaNet [CN]"}, - {"from": "104.196.0.0", "to": "104.199.255.255", "m": 14, "label": "Google LLC"}, - {"from": "110.238.80.0", "to": "110.238.127.254", "label": "Huawei [SG]"}, - {"from": "111.119.192.0", "to": "111.119.255.254", "label": "Huawei [SG]"}, - {"from": "113.160.0.0", "to": "113.191.255.254", "m": 11, "label": "VNPT [VN]"}, - {"from": "119.0.0.0", "to": "119.207.255.254", "label": "Unicom [CN]"}, - {"from": "121.91.168.", "to": "121.91.175.254", "label": "Huawei [HK]"}, - {"from": "122.8.0.0", "to": "122.8.255.254", "label": "CN-ISP [CN]"}, - {"from": "122.9.0.0", "to": "122.9.255.254", "label": "Huawei [CN]"}, - {"from": "123.16.0.0", "to": "123.31.255.255", "m": 12, "label": "VNPT [VN]"}, - {"from": "124.243.128.0", "to": "124.243.191.254", "label": "Huawei [SG]"}, - {"from": "142.147.128.0", "to": "1142.147.255.254", "label": "Web2Objects LLC [US]"}, - {"from": "150.40.128.0", "to": "150.40.255.254", "label": "Huawei [HK]"}, - {"from": "159.138.0.0", "to": "159.138.225.254", "label": "Huawei [TH]"}, - {"from": "162.128.0.0", "to": "162.128.255.254", "label": "Zenlayer [SG]"}, - {"from": "166.108.192.0", "to": "166.108.255.254", "label": "Huawei [SG]"}, - {"from": "177.0.0.0", "to": "177.255.255.254", "label": "BrasilNET [BR]"}, - {"from": "179.0.0.0", "to": "179.255.255.254", "label": "BrasilNET [BR]"}, - {"from": "183.87.32.0", "to": "183.87.159.254", "label": "Huawei [HK]"}, - {"from": "186.0.0.0", "to": "186.255.255.254", "m": 8, "label": "South-American ISPs (186.x)"}, - {"from": "187.0.0.0", "to": "187.255.255.254", "m": 8, "label": "South-American ISPs (187.x)"}, - {"from": "188.0.0.0", "to": "188.255.255.254", "m": 8, "label": "South-American ISPs (188.x)"}, - {"from": "189.0.0.0", "to": "189.255.255.254", "m": 8, "label": "South-American ISPs (189.x)"}, - {"from": "190.0.0.0", "to": "190.255.255.254", "m": 8, "label": "South-American ISPs (190.x)"}, - {"from": "191.0.0.0", "to": "191.255.255.254", "m": 8, "label": "South-American ISPs (191.x)"}, - {"from": "192.124.170.0", "to": "192.124.182.254", "label": "Relcom [CZ]"}, - {"from": "195.37.0.0", "to": "195.37.255.255", "m": 16, "label": "DFN [DE]"}, - {"from": "2001:4860::::::", "to": "2001:4860:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Google LLC [US]"}, - {"from": "2001:0ee0::::::", "to": "2001:ee3:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "label": "VNPT [VN]"}, - {"from": "2408:8210::::::", "to": "2408:8210:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "label": "China Unicom [CN]"}, - {"from": "2600:1f00::::::", "to": "2600:1fff:ffff:ffff:ffff:ffff:ffff:ffff", "m": "Amazon Cloud"}, - {"from": "2603:6010::::::", "to": "2603:6010:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Charter [US]"}, - {"from": "2603:8000::::::", "to": "2603:80ff:ffff:ffff:ffff:ffff:ffff:ffff", "m": 24, "label": "Charter [US]"}, - {"from": "2607:a400::::::", "to": "2607:a400:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Zenlayer Inc. [US]"}, - {"from": "2804:::::::", "to": "2804:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "m": 16, "label": "Inspire [BR]"}, - {"from": "2a09:bac2::::::", "to": "2a09:bac2:0:ffff:ffff:ffff:ffff:ffff", "m": 48, "label": "Cloudflare"}, - {"from": "2a0a:4cc0::::::", "to": "2a0a:4cc0:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "label": "Netcup [DE]"} -] \ No newline at end of file +{ + "groups": [ + {"id": "alibaba", "name": "Alibaba"}, + {"id": "amazon", "name": "Amazon Data Services"}, + {"id": "amazon-microsoft", "name": "Amazon or Microsoft"}, + {"id": "brasilnet", "name": "BrasilNet"}, + {"id": "charter", "name": "Charter Communications Inc."}, + {"id": "cnisp", "name": "China ISP"}, + {"id": "cnmob", "name": "China Mobile Communications Corp."}, + {"id": "google", "name": "Google LLC"}, + {"id": "google-amazon", "name": "Google or Amazon"}, + {"id": "hetzner", "name": "Hetzner Hosting (US)"}, + {"id": "huawei", "name": "Huawei"}, + {"id": "misc_sa", "name": "Various South- and Central-American ISPs"}, + {"id": "tencent", "name": "Tencent"}, + {"id": "unicom", "name": "China Unicom"}, + {"id": "vnpt", "name": "Vietnam Telecom"}, + {"id": "vdsina", "name": "VDSina NL"}, + {"id": "zenlayer", "name": "Zenlayer"} + ], + "ranges": [ + {"from": "3.0.0.0", "to": "3.255.255.254", "m": 8, "g": "amazon"}, + {"from": "5.161.0.0", "to": "5.161.255.255", "m": 16, "g": "hetzner"}, + {"from": "8.128.0.0", "to": "8.191.255.254", "m": 10, "g": "alibaba"}, + {"from": "13.216.0.0", "to": "13.223.255.254", "m": 11, "g": "amazon"}, + {"from": "14.160.0.0", "to": "14.191.255.254", "m": 11, "g": "vnpt"}, + {"from": "14.192.0.0", "to": "14.255.255.254", "m": 10, "g": "vnpt"}, + {"from": "14.224.0.0", "to": "24.255.255.254", "m": 11, "g": "vnpt"}, + {"from": "27.106.0.0", "to": "27.106.127.254", "m": 17, "g": "huawei"}, + {"from": "34.0.0.0", "to": "34.255.255.254", "m": 8, "g": "google-amazon"}, + {"from": "39.64.0.0", "to": "39.95.255.254", "g": "cnmob"}, + {"from": "43.132.0.0", "to": "43.132.255.254", "m": 16, "g": "tencent"}, + {"from": "43.133.0.0", "to": "43.133.255.254", "m": 16, "g": "tencent"}, + {"from": "44.192.0.0", "to": "44.255.255.254", "m": 16, "g": "tencent"}, + {"from": "45.0.0.0", "to": "45.255.255.254", "m": 10, "g": "amazon"}, + {"from": "46.250.160.0", "to": "46.250.191.254", "m": 19, "g": "huawei"}, + {"from": "47.82.0.0", "to": "47.82.255.254", "m": 16, "g": "alibaba"}, + {"from": "47.200.0.0", "to": "47.203.255.254", "m": 14, "g": "frontier"}, + {"from": "49.0.192.0", "to": "49.0.255.254", "m": 18, "g": "huawei"}, + {"from": "52.0.0.0", "to": "52.255.255.254", "m": 8, "g": "amazon-microsoft"}, + {"from": "54.0.0.0", "to": "54.255.255.254", "m": 8, "g": "amazon"}, + {"from": "66.249.64.0", "to": "66.249.95.254", "m": 19, "g": "google"}, + {"from": "84.37.35.0", "to": "84.37.255.254", "g": "gtt"}, + {"from": "94.74.64.0", "to": "94.74.127.254", "m": 18, "g": "huawei"}, + {"from": "91.84.96.0", "to": "91.84.127.254", "m": 19, "g": "vdsina"}, + {"from": "101.0.0.0", "to": "101.255.255.254", "m": 8,"g": "chinanet"}, + {"from": "104.196.0.0", "to": "104.199.255.254", "m": 14, "g": "google"}, + {"from": "110.238.64.0", "to": "110.238.127.254", "m": 18, "g": "huawei"}, + {"from": "111.119.192.0", "to": "111.119.255.254", "m": 18, "g": "huawei"}, + {"from": "113.160.0.0", "to": "113.191.255.254", "m": 11, "g": "vnpt"}, + {"from": "114.208.0.0", "to": "114.223.255.254", "m": 12, "g": "unicom"}, + {"from": "114.224.0.0", "to": "114.255.255.254", "m": 11, "g": "unicom"}, + {"from": "119.8.0.0", "to": "119.8.255.254", "m": 16, "g": "huawei"}, + {"from": "119.13.0.0", "to": "119.13.255.254", "m": 16, "g": "huawei"}, + {"from": "121.91.168.0", "to": "121.91.175.254", "m": 21, "g": "huawei"}, + {"from": "122.8.0.0", "to": "122.8.255.254", "g": "cnisp"}, + {"from": "122.9.0.0", "to": "122.9.255.254", "m": 16, "g": "huawei"}, + {"from": "123.16.0.0", "to": "123.31.255.254", "m": 12, "g": "vnpt"}, + {"from": "124.243.128.0", "to": "124.243.191.254", "m": 18, "g": "huawei"}, + {"from": "138.59.0.0", "to": "138.59.225.254", "m": 16, "g": "misc_sa"}, + {"from": "138.121.0.0", "to": "138.121.225.254", "m": 16, "g": "misc_sa"}, + {"from": "142.147.128.0", "to": "1142.147.255.254", "m": 17, "g": "w2obj"}, + {"from": "146.174.128.0", "to": "146.174.191.254", "m": 18, "g": "huawei"}, + {"from": "150.40.128.0", "to": "150.40.255.254", "m": 17, "g": "huawei"}, + {"from": "159.138.0.0", "to": "159.138.225.254", "m": 16, "g": "huawei"}, + {"from": "162.128.0.0", "to": "162.128.255.254", "m": 16, "g": "zenlayer"}, + {"from": "166.108.192.0", "to": "166.108.255.254", "m": 18, "g": "huawei"}, + {"from": "168.232.192.0", "to": "168.232.255.254", "m": 16, "g": "misc_sa"}, + {"from": "170.82.0.0", "to": "170.82.255.254", "m": 16, "g": "misc_sa"}, + {"from": "170.254.0.0", "to": "170.254.255.254", "m": 16, "g": "misc_sa"}, + {"from": "171.224.0.0", "to": "171.239.255.254", "m": 12, "g": "viettel"}, + {"from": "177.0.0.0", "to": "177.255.255.254", "m": 8, "g": "brasilnet"}, + {"from": "179.0.0.0", "to": "179.255.255.254", "m": 8, "g": "brasilnet"}, + {"from": "183.87.32.0", "to": "183.87.63.254", "m": 19, "g": "huawei"}, + {"from": "186.0.0.0", "to": "186.255.255.254", "m": 8, "g": "misc_sa"}, + {"from": "187.0.0.0", "to": "187.255.255.254", "m": 8, "g": "misc_sa"}, + {"from": "188.0.0.0", "to": "188.255.255.254", "m": 8, "g": "misc_sa"}, + {"from": "189.0.0.0", "to": "189.255.255.254", "m": 8, "g": "misc_sa"}, + {"from": "190.0.0.0", "to": "190.255.255.254", "m": 8, "g": "misc_sa"}, + {"from": "191.0.0.0", "to": "191.255.255.254", "m": 8, "g": "misc_sa"}, + {"from": "192.124.170.0", "to": "192.124.182.254", "g": "relcom"}, + {"from": "200.0.0.0", "to": "200.255.255.254", "m": 8, "g": "misc_sa"}, + {"from": "201.0.0.0", "to": "201.255.255.254", "m": 8, "g": "misc_sa"}, + {"from": "212.95.128.0", "to": "212.95.159.254", "m": 19, "g": "asiacell"}, + {"from": "222.252.0.0", "to": "222.252.255.254", "m": 14, "g": "vnpt"}, + {"from": "2001:4860::::::", "to": "2001:4860:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "g": "google"}, + {"from": "2001:0ee0::::::", "to": "2001:ee3:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "g": "vnpt"}, + {"from": "2408:8210::::::", "to": "2408:8210:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "g": "unicom"}, + {"from": "2600:1f00::::::", "to": "2600:1fff:ffff:ffff:ffff:ffff:ffff:ffff", "m": 24, "g": "amazon"}, + {"from": "2603:6010::::::", "to": "2603:6010:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "g": "charter"}, + {"from": "2603:8000::::::", "to": "2603:80ff:ffff:ffff:ffff:ffff:ffff:ffff", "m": 24, "g": "charter"}, + {"from": "2607:a400::::::", "to": "2607:a400:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "g": "zenlayer"}, + {"from": "2804:::::::", "to": "2804:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "m": 16, "g": "misc_sa"}, + {"from": "2a0a:4cc0::::::", "to": "2a0a:4cc0:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "g": "netcup"} + ] +} \ No newline at end of file diff --git a/config/known-platforms.json b/config/known-platforms.json index 2e3da62..46e6c51 100644 --- a/config/known-platforms.json +++ b/config/known-platforms.json @@ -21,7 +21,7 @@ }, {"n": "Old Android", "id": "androidold", - "rx": [ "Android[\\s;\\/](\\d)\\." ] + "rx": [ "Android[\\s;\\/](\\d)[\\.;\\)]" ] }, {"n": "Android", "id": "android", diff --git a/script.js b/script.js index 02e481e..659c872 100644 --- a/script.js +++ b/script.js @@ -19,6 +19,12 @@ const BM_LOGTYPE = Object.freeze({ 'TICKER': 'tck' }); +// enumeration of IP versions: +const BM_IPVERSION = Object.freeze({ + 'IPv4': 4, + 'IPv6': 6 +}); + /* BotMon root object */ const BotMon = { @@ -35,13 +41,18 @@ const BotMon = { // get the time offset: this._timeDiff = BotMon.t._getTimeOffset(); + // get yesterday's date: + let d = new Date(); + d.setDate(d.getDate() - 1); + this._datestr = d.toISOString().slice(0, 10); + // init the sub-objects: BotMon.t._callInit(this); }, _baseDir: null, _lang: 'en', - _datestr: (new Date()).toISOString().slice(0, 10), + _datestr: '', _timeDiff: '', /* internal tools */ @@ -105,9 +116,9 @@ const BotMon = { if (!ip) { return 'null'; } else if (ip.indexOf(':') > 0) { /* IP6 */ - return (ip.split(':').map(d => ('0000'+d).slice(-4) ).join('')); + return (ip.split(':').map(d => ('0000'+d).slice(-4) ).join(':')); } else { /* IP4 */ - return Number(ip.split('.').map(d => ('000'+d).slice(-3) ).join('')); + return ip.split('.').map(d => ('000'+d).slice(-3) ).join('.'); } }, @@ -147,7 +158,7 @@ const BotMon = { } }; -/* everything specific to the "Today" tab is self-contained in the "live" object: */ +/* everything specific to the "Latest" tab is self-contained in the "live" object: */ BotMon.live = { init: function() { //console.info('BotMon.live.init()'); @@ -180,6 +191,9 @@ BotMon.live = { case 'rules': data._dispatchRulesLoaded = true; break; + case 'ipranges': + data._dispatchIPRangesLoaded = true; + break; case 'bots': data._dispatchBotsLoaded = true; break; @@ -194,7 +208,7 @@ BotMon.live = { } // are all the flags set? - if (data._dispatchBotsLoaded && data._dispatchClientsLoaded && data._dispatchPlatformsLoaded && data._dispatchRulesLoaded) { + if (data._dispatchBotsLoaded && data._dispatchClientsLoaded && data._dispatchPlatformsLoaded && data._dispatchRulesLoaded && data._dispatchIPRangesLoaded) { // chain the log files loading: BotMon.live.data.loadLogFile(BM_LOGTYPE.SERVER, BotMon.live.data._onServerLogLoaded); } @@ -203,6 +217,7 @@ BotMon.live = { _dispatchBotsLoaded: false, _dispatchClientsLoaded: false, _dispatchPlatformsLoaded: false, + _dispatchIPRangesLoaded: false, _dispatchRulesLoaded: false, // event callback, after the server log has been loaded: @@ -239,6 +254,7 @@ BotMon.live = { }, + // the data model: model: { // visitors storage: _visitors: [], @@ -486,8 +502,12 @@ BotMon.live = { } }, + // functions to analyse the data: analytics: { + /** + * Initializes the analytics data storage object: + */ init: function() { //console.info('BotMon.live.data.analytics.init()'); }, @@ -580,12 +600,15 @@ BotMon.live = { // add v._pageViews.forEach( pv => { me.addToRefererList(pv._ref); + me.addToPagesList(pv.pg); }); } }); BotMon.live.gui.status.hideBusy('Done.'); + + console.log(me._pagesList); }, // get a list of known bots: @@ -635,6 +658,49 @@ BotMon.live = { return rList; }, + // most visited pages list: + _pagesList: [], + + /** + * Add a page view to the list of most visited pages. + * @param {string} pageId - The page ID to add to the list. + * @example + * BotMon.live.data.analytics.addToPagesList('1234567890'); + */ + addToPagesList: function(pageId) { + //console.log('BotMon.live.data.analytics.addToPagesList', pageId); + + const me = BotMon.live.data.analytics; + + // already exists? + let pgObj = null; + for (let i = 0; i < me._pagesList.length; i++) { + if (me._pagesList[i].id == pageId) { + pgObj = me._pagesList[i]; + break; + } + } + + // if not exists, create it: + if (!pgObj) { + pgObj = { + id: pageId, + count: 1 + }; + me._pagesList.push(pgObj); + } else { + pgObj.count += 1; + } + }, + + getTopPages: function(max) { + //console.info('BotMon.live.data.analytics.getTopPages('+max+')'); + const me = BotMon.live.data.analytics; + return me._pagesList.toSorted( (a, b) => { + return b.count - a.count; + }).slice(0,max); + }, + // Referer List: _refererList: [], @@ -696,6 +762,14 @@ BotMon.live = { }; }, + /** + * Get a sorted list of the top referers. + * The list is sorted in descending order of count. + * If the array has more items than the given maximum, the rest of the items are added to an "other" item. + * Each item in the list has a "pct" property, which is the percentage of the total count. + * @param {number} max - The maximum number of items to return. + * @return {Array} The sorted list of top referers. + */ getTopReferers: function(max) { //console.info(('BotMon.live.data.analytics.getTopReferers(' + max + ')')); @@ -704,6 +778,15 @@ BotMon.live = { return me._makeTopList(me._refererList, max); }, + /** + * Create a sorted list of top items from a given array. + * The list is sorted in descending order of count. + * If the array has more items than the given maximum, the rest of the items are added to an "other" item. + * Each item in the list has a "pct" property, which is the percentage of the total count. + * @param {Array} arr - The array to sort and truncate. + * @param {number} max - The maximum number of items to return. + * @return {Array} The sorted list of top items. + */ _makeTopList: function(arr, max) { //console.info(('BotMon.live.data.analytics._makeTopList(arr,' + max + ')')); @@ -896,6 +979,7 @@ BotMon.live = { } }, + // information on "known bots": bots: { // loads the list of known bots from a JSON file: init: async function() { @@ -972,6 +1056,7 @@ BotMon.live = { _list: [] }, + // information on known clients (browsers): clients: { // loads the list of known clients from a JSON file: init: async function() { @@ -1038,6 +1123,7 @@ BotMon.live = { }, + // information on known platforms (operating systems): platforms: { // loads the list of known platforms from a JSON file: init: async function() { @@ -1104,15 +1190,43 @@ BotMon.live = { }, + // storage and functions for the known bot IP-Ranges: ipRanges: { init: function() { //console.log('BotMon.live.data.ipRanges.init()'); // #TODO: Load from separate IP-Ranges file + // load the rules file: + const me = BotMon.live.data; + + try { + BotMon.live.data._loadSettingsFile(['user-ipranges', 'known-ipranges'], + (json) => { + + // groups can be just saved in the data structure: + if (json.groups && json.groups.constructor.name == 'Array') { + me.ipRanges._groups = json.groups; + } + + // groups can be just saved in the data structure: + if (json.ranges && json.ranges.constructor.name == 'Array') { + json.ranges.forEach(range => { + me.ipRanges.add(range); + }) + } + + // finished loading + BotMon.live.gui.status.hideBusy("Status: Done."); + BotMon.live.data._dispatch('ipranges') + }); + } catch (error) { + BotMon.live.gui.status.setError("Error while loading the config file: " + error.message); + } }, // the actual bot list is stored here: _list: [], + _groups: [], add: function(data) { //console.log('BotMon.live.data.ipRanges.add(',data,')'); @@ -1123,14 +1237,29 @@ BotMon.live = { const ip2Num = BotMon.t._ip2Num; let item = { + 'cidr': data.from.replaceAll(/::+/g, '::') + '/' + ( data.m ? data.m : '??' ), 'from': ip2Num(data.from), 'to': ip2Num(data.to), - 'label': data.label + 'm': data.m, + 'g': data.g }; me._list.push(item); }, + getOwner: function(rangeInfo) { + + const me = BotMon.live.data.ipRanges; + + for (let i=0; i < me._groups.length; i++) { + const it = me._groups[i]; + if (it.id == rangeInfo.g) { + return it.name; + } + } + return `Unknown (“${rangeInfo.g})`; + }, + match: function(ip) { //console.log('BotMon.live.data.ipRanges.match(',ip,')'); @@ -1151,6 +1280,7 @@ BotMon.live = { } }, + // storage for the rules and related functions rules: { /** @@ -1183,25 +1313,17 @@ BotMon.live = { if (json.threshold) me._threshold = json.threshold; // set the rules list: - if (json.rules) { + if (json.rules && json.rules.constructor.name == 'Array') { me.rules._rulesList = json.rules; } - // load the IP ranges: - if (json.ipRanges) { - json.ipRanges.forEach( it => me.ipRanges.add(it)); - }; - + BotMon.live.gui.status.hideBusy("Status: Done."); + BotMon.live.data._dispatch('rules') } ); - } catch (error) { BotMon.live.gui.status.setError("Error while loading the config file: " + error.message); - } finally { - BotMon.live.gui.status.hideBusy("Status: Done."); - BotMon.live.data._dispatch('rules') } - }, _rulesList: [], // list of rules to find out if a visitor is a bot @@ -1717,6 +1839,31 @@ BotMon.live = { } } + // update the top pages; + const wmpages = document.getElementById('botmon__today__wm_pages'); + if (wmpages) { + + wmpages.appendChild(makeElement('dt', {}, "Top pages")); + + const pgList = BotMon.live.data.analytics.getTopPages(maxItemsPerList); + if (pgList) { + pgList.forEach( (pgInfo) => { + const pgDd = makeElement('dd'); + pgDd.appendChild(makeElement('a', { + 'class': 'page_icon', + 'href': DOKU_BASE + 'doku.php?id=' + encodeURIComponent(pgInfo.id), + 'target': 'preview', + 'title': "PageID: " + pgInfo.id + }, pgInfo.id)); + pgDd.appendChild(makeElement('span', { + 'class': 'count', + 'title': pgInfo.count + " page views" + }, pgInfo.count)); + wmpages.appendChild(pgDd); + }); + } + } + // update the top referrers; const wmreferers = document.getElementById('botmon__today__wm_referers'); if (wmreferers) { @@ -2014,27 +2161,27 @@ BotMon.live = { dl.appendChild(make('dd', {'class': 'has_icon platform pf_' + (data._platform ? data._platform.id : 'unknown')}, platformName + ( data._platform.v > 0 ? ' (' + data._platform.v + ')' : '' ) )); - dl.appendChild(make('dt', {}, "IP-Address:")); - const ipItem = make('dd', {'class': 'has_icon ipaddr ip' + ipType}); - ipItem.appendChild(make('span', {'class': 'address'} , data.ip)); - ipItem.appendChild(make('a', { - 'class': 'icon_only extlink dnscheck', - 'href': `https://dnschecker.org/ip-location.php?ip=${encodeURIComponent(data.ip)}`, - 'target': 'dnscheck', - 'title': "View this address on DNSChecker.org" - } , "Check Address")); - ipItem.appendChild(make('a', { - 'class': 'icon_only extlink ipinfo', - 'href': `https://ipinfo.io/${encodeURIComponent(data.ip)}`, - 'target': 'ipinfo', - 'title': "View this address on IPInfo.io" - } , "DNS Info")); - dl.appendChild(ipItem); - /*dl.appendChild(make('dt', {}, "ID:")); dl.appendChild(make('dd', {'class': 'has_icon ip' + data.typ}, data.id));*/ } + dl.appendChild(make('dt', {}, "IP-Address:")); + const ipItem = make('dd', {'class': 'has_icon ipaddr ip' + ipType}); + ipItem.appendChild(make('span', {'class': 'address'} , data.ip)); + ipItem.appendChild(make('a', { + 'class': 'icon_only extlink dnscheck', + 'href': `https://dnschecker.org/ip-location.php?ip=${encodeURIComponent(data.ip)}`, + 'target': 'dnscheck', + 'title': "View this address on DNSChecker.org" + } , "Check Address")); + ipItem.appendChild(make('a', { + 'class': 'icon_only extlink ipinfo', + 'href': `https://ipinfo.io/${encodeURIComponent(data.ip)}`, + 'target': 'ipinfo', + 'title': "View this address on IPInfo.io" + } , "DNS Info")); + dl.appendChild(ipItem); + if (Math.abs(data._lastSeen - data._firstSeen) < 100) { dl.appendChild(make('dt', {}, "Seen:")); dl.appendChild(make('dd', {'class': 'seen'}, data._firstSeen.toLocaleString())); @@ -2099,7 +2246,8 @@ BotMon.live = { if (tObj.func == 'fromKnownBotIP') { const rangeInfo = BotMon.live.data.ipRanges.match(data.ip); if (rangeInfo) { - tDesc += ' (' + (rangeInfo.label ? rangeInfo.label : 'Unknown') + ')'; + const owner = BotMon.live.data.ipRanges.getOwner(rangeInfo); + tDesc += ' (range: “' + rangeInfo.cidr + '”, ' + owner + ')'; } } diff --git a/style.less b/style.less index 6ef52ab..a148c28 100644 --- a/style.less +++ b/style.less @@ -373,8 +373,23 @@ &.ref_other::before { background-image: url('img/more.svg') } } + .page_icon { + display: inline-flex; + column-gap: .25em; + align-items: center; + } + .page_icon::before { + content: ''; + display: inline-block; + width: 20px; height: 20px; + background: transparent url('img/page.svg') center no-repeat; + background-position: 0 0; + background-size: 20px; + } + + /* grid layout for the overview: */ - .botmon_bots_grid, .botmon_webmetrics_grid { + .botmon_bots_grid, .botmon_webmetrics_grid, .botmon_traffic_grid { & { display: grid; grid-gap: 0 .33em; @@ -391,7 +406,10 @@ grid-template-columns: 1fr 1fr 1fr; } .botmon_webmetrics_grid { - grid-template-columns: 1fr 1fr 1fr 1fr; + grid-template-columns: 1fr 1fr 1fr; + } + .botmon_traffic_grid { + grid-template-columns: 2fr 1fr; } /* the "today" tab: */