Miko/dokuwiki-plugin-botmon
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Web traffic monitoring

Browse Source
This commit is contained in:
Sascha Leib
2025-10-03 21:30:29 +02:00
parent d02708c1ae
commit c31fb4d0a5
9 changed files with 321 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
action.php
View File

@@ -81,8 +81,6 @@ class action_plugin_botmon extends DokuWiki_Action_Plugin {
$username = ( !empty($INFO['userinfo']) && !empty($INFO['userinfo']['name']) $username = ( !empty($INFO['userinfo']) && !empty($INFO['userinfo']['name'])
? $INFO['userinfo']['name'] : ''); ? $INFO['userinfo']['name'] : '');
// clean the page ID // clean the page ID
$pageId = preg_replace('/[\x00-\x1F]/', "\u{FFFD}", $INFO['id'] ?? ''); $pageId = preg_replace('/[\x00-\x1F]/', "\u{FFFD}", $INFO['id'] ?? '');

10
admin.php
View File

@@ -42,7 +42,7 @@ class admin_plugin_botmon extends AdminPlugin {
<h1>Bot Monitoring Plugin</h1> <h1>Bot Monitoring Plugin</h1>
<nav id="botmon__tabs"> <nav id="botmon__tabs">
<ul class="tabs" role="tablist"> <ul class="tabs" role="tablist">
<li role="presentation" class="active"><a role="tab" href="#botmon__panel1" aria-controls="botmon__panel1" id="botmon__tab1" aria-selected="true">Today</a></li> <li role="presentation" class="active"><a role="tab" href="#botmon__panel1" aria-controls="botmon__panel1" id="botmon__tab1" aria-selected="true">Latest</a></li>
</ul> </ul>
</nav>'; </nav>';
@@ -51,7 +51,7 @@ class admin_plugin_botmon extends AdminPlugin {
} }
echo '<article role="tabpanel" id="botmon__today""> echo '<article role="tabpanel" id="botmon__today"">
<h2 class="a11y">Today</h2> <h2 class="a11y">Latest data</h2>
<header id="botmon__today__title">Loading&nbsp;&hellip;</header> <header id="botmon__today__title">Loading&nbsp;&hellip;</header>
<div id="botmon__today__content"> <div id="botmon__today__content">
<details id="botmon__today__overview" open> <details id="botmon__today__overview" open>
@@ -68,6 +68,12 @@ class admin_plugin_botmon extends AdminPlugin {
<dl id="botmon__today__wm_overview"></dl> <dl id="botmon__today__wm_overview"></dl>
<dl id="botmon__today__wm_clients"></dl> <dl id="botmon__today__wm_clients"></dl>
<dl id="botmon__today__wm_platforms"></dl> <dl id="botmon__today__wm_platforms"></dl>
</div>
</details>
<details id="botmon__today__traffic">
<summary>Web traffic (humans only)</summary>
<div class="botmon_traffic_grid">
<dl id="botmon__today__wm_pages"></dl>
<dl id="botmon__today__wm_referers"></dl> <dl id="botmon__today__wm_referers"></dl>
</div> </div>
</details> </details>

2
client.js
View File

@@ -73,7 +73,7 @@
console.error(err); console.error(err);
} finally { } finally {
/* send the next heartbeat signal after x seconds: */ /* send the next heartbeat signal after x seconds: */
// setTimeout(this._onHeartbeat.bind(this, this._src.replace( this._scriptName, '/tick.php')),this._heartbeat * 1000); setTimeout(this._onHeartbeat.bind(this, this._src.replace( this._scriptName, '/tick.php')),this._heartbeat * 1000);
} }
} }
}).init(); }).init();

69
config/default-config.json
View File

@@ -61,74 +61,5 @@
"id": "langMatch", "desc": "Client’s ‘Accept-Language’ header does not match the page language", "id": "langMatch", "desc": "Client’s ‘Accept-Language’ header does not match the page language",
"bot": 30 "bot": 30
} }
],
"ipRanges": [
{"from": "3.0.0.0", "to": "3.255.255.254", "label": "Amazon Data Services [US]"},
{"from": "5.161.0.0", "to": "5.161.255.255", "m": 16, "label": "Hetzner [US]"},
{"from": "8.127.0.0", "to": "8.223.255.254", "label": "Alibaba [CN]"},
{"from": "14.160.0.0", "to": "14.191.255.254", "m": 11, "label": "VNPT [VN]"},
{"from": "14.192.0.0", "to": "14.255.255.254", "m": 10, "label": "VNPT [VN]"},
{"from": "14.224.0.0", "to": "24.255.255.254", "m": 11, "label": "Charter [US]"},
{"from": "27.106.0.0", "to": "27.106.127.254", "m": 17, "label": "Huawei [US]"},
{"from": "34.0.0.0", "to": "34.191.255.254", "label": "Google LLC"},
{"from": "43.132.0.0", "to": "43.132.255.254", "m": 16, "label": "Tencent [CN]"},
{"from": "43.133.0.0", "to": "43.133.255.254", "m": 16, "label": "Tencent [CN]"},
{"from": "45.0.0.0", "to": "45.255.255.254", "label": "Various small ISPs, mostly BR"},
{"from": "46.250.160.0", "to": "46.250.191.254", "m": 19, "label": "Huawei [MX]"},
{"from": "47.200.0.0", "to": "47.203.255.254", "m": 14, "label": "Frontier Communications [US]"},
{"from": "49.0.192.0", "to": "49.0.255.254", "m": 18, "label": "Huawei [SG]"},
{"from": "52.220.0.0", "to": "52.222.255.254", "label": "Amazon Data Services"},
{"from": "66.249.64.0", "to": "66.249.95.254", "m": 19, "label": "Google LLC"},
{"from": "84.37.35.0", "to": "84.37.255.254", "label": "GTT.net [US]"},
{"from": "94.74.64.0", "to": "94.74.127.254", "m": 18, "label": "Huawei [HK]"},
{"from": "101.0.0.0", "to": "101.255.255.254", "m": 8,"label": "ChinaNet [CN]"},
{"from": "104.196.0.0", "to": "104.199.255.254", "m": 14, "label": "Google LLC"},
{"from": "110.238.64.0", "to": "110.238.127.254", "m": 18, "label": "Huawei [SG]"},
{"from": "111.119.192.0", "to": "111.119.255.254", "m": 18, "label": "Huawei [SG]"},
{"from": "113.160.0.0", "to": "113.191.255.254", "m": 11, "label": "VNPT [VN]"},
{"from": "119.8.0.0", "to": "119.8.255.254", "m": 16, "label": "Huawei [HK]"},
{"from": "119.13.0.0", "to": "119.13.255.254", "m": 16, "label": "Huawei [HK]"},
{"from": "121.91.168.0", "to": "121.91.175.254", "m": 21, "label": "Huawei [HK]"},
{"from": "122.8.0.0", "to": "122.8.255.254", "label": "CN-ISP [CN]"},
{"from": "122.9.0.0", "to": "122.9.255.254", "m": 16, "label": "Huawei [CN]"},
{"from": "123.16.0.0", "to": "123.31.255.254", "m": 12, "label": "VNPT [VN]"},
{"from": "124.243.128.0", "to": "124.243.191.254", "m": 18, "label": "Huawei [SG]"},
{"from": "138.59.0.0", "to": "138.59.225.254", "m": 16, "label": "South-American ISPs (138.59.x)"},
{"from": "138.121.0.0", "to": "138.121.225.254", "m": 16, "label": "South-American ISPs (138.121.x)"},
{"from": "142.147.128.0", "to": "1142.147.255.254", "m": 17, "label": "Web2Objects LLC [US]"},
{"from": "146.174.128.0", "to": "146.174.191.254", "m": 18, "label": "Huawei [SG]"},
{"from": "150.40.128.0", "to": "150.40.255.254", "m": 17, "label": "Huawei [HK]"},
{"from": "159.138.0.0", "to": "159.138.225.254", "m": 16, "label": "Huawei [TH]"},
{"from": "162.128.0.0", "to": "162.128.255.254", "m": 16, "label": "Zenlayer [SG]"},
{"from": "166.108.192.0", "to": "166.108.255.254", "m": 18, "label": "Huawei [SG]"},
{"from": "168.232.192.0", "to": "168.232.255.254", "m": 16, "label": "South-American ISPs (168.232.x)"},
{"from": "170.82.0.0", "to": "170.82.255.254", "m": 16, "label": "South-American ISPs (170.254.x)"},
{"from": "170.254.0.0", "to": "170.254.255.254", "m": 16, "label": "South-American ISPs (170.82.x)"},
{"from": "171.224.0.0", "to": "171.239.255.254", "m": 12, "label": "Viettel [VN]"},
{"from": "177.0.0.0", "to": "177.255.255.254", "m": 8, "label": "BrasilNET [BR]"},
{"from": "179.0.0.0", "to": "179.255.255.254", "m": 8, "label": "BrasilNET [BR]"},
{"from": "183.87.32.0", "to": "183.87.63.254", "label": "Huawei [HK]"},
{"from": "186.0.0.0", "to": "186.255.255.254", "m": 8, "label": "South-American ISPs (186.x)"},
{"from": "187.0.0.0", "to": "187.255.255.254", "m": 8, "label": "South-American ISPs (187.x)"},
{"from": "188.0.0.0", "to": "188.255.255.254", "m": 8, "label": "South-American ISPs (188.x)"},
{"from": "189.0.0.0", "to": "189.255.255.254", "m": 8, "label": "South-American ISPs (189.x)"},
{"from": "190.0.0.0", "to": "190.255.255.254", "m": 8, "label": "South-American ISPs (190.x)"},
{"from": "191.0.0.0", "to": "191.255.255.254", "m": 8, "label": "South-American ISPs (191.x)"},
{"from": "192.124.170.0", "to": "192.124.182.254", "label": "Relcom [CZ]"},
{"from": "195.37.0.0", "to": "195.37.255.254", "m": 16, "label": "DFN [DE]"},
{"from": "200.0.0.0", "to": "200.255.255.254", "m": 8, "label": "South-American ISPs (200.x)"},
{"from": "201.0.0.0", "to": "201.255.255.254", "m": 8, "label": "South-American ISPs (201.x)"},
{"from": "212.95.128.0", "to": "212.95.159.254", "m": 19, "label": "Asiacell [IQ]"},
{"from": "222.252.0.0", "to": "222.252.255.254", "m": 14, "label": "VNPT [VN]"},
{"from": "2001:4860::::::", "to": "2001:4860:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Google LLC [US]"},
{"from": "2001:0ee0::::::", "to": "2001:ee3:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "label": "VNPT [VN]"},
{"from": "2408:8210::::::", "to": "2408:8210:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "label": "China Unicom [CN]"},
{"from": "2600:1f00::::::", "to": "2600:1fff:ffff:ffff:ffff:ffff:ffff:ffff", "m": 24, "label": "Amazon Cloud"},
{"from": "2603:6010::::::", "to": "2603:6010:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Charter [US]"},
{"from": "2603:8000::::::", "to": "2603:80ff:ffff:ffff:ffff:ffff:ffff:ffff", "m": 24, "label": "Charter [US]"},
{"from": "2607:a400::::::", "to": "2607:a400:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Zenlayer Inc. [US]"},
{"from": "2804:::::::", "to": "2804:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "m": 16, "label": "Inspire [BR]"},
{"from": "2a09:bac2::::::", "to": "2a09:bac2:0:ffff:ffff:ffff:ffff:ffff", "m": 48, "label": "Cloudflare"},
{"from": "2a0a:4cc0::::::", "to": "2a0a:4cc0:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "label": "Netcup [DE]"}
] ]
} }

12
config/known-bots.json
View File

@@ -297,5 +297,17 @@
"r": [], "r": [],
"rx": ["Go\\-http\\-client\\/(\\d+)", "quic\\-go\\-HTTP\\/(\\d+)"], "rx": ["Go\\-http\\-client\\/(\\d+)", "quic\\-go\\-HTTP\\/(\\d+)"],
"url": "https://github.com/golang/goen.wi" "url": "https://github.com/golang/goen.wi"
},
{"id": "bnl",
"n": "BnL Harvester",
"r": [],
"rx": ["NLUX_IAHarvester\\/(\\d+)"],
"url": "http://crawl.bnl.lu/"
},
{"id": "turnitin",
"n": "TurnitinBot",
"r": [],
"rx": ["Turnitin\\s"],
"url": "https://www.turnitin.com/robot/crawlerinfo.html"
} }
] ]

149
config/known-ipranges.json
View File

@@ -1,55 +1,96 @@
[ {
{"from": "3.0.0.0", "to": "3.255.255.254", "m": 8, "label": "Amazon Data Services [US]"}, "groups": [
{"from": "8.127.0.0", "to": "8.223.255.254", "label": "Alibaba [CN]"}, {"id": "alibaba", "name": "Alibaba"},
{"from": "14.160.0.0", "to": "14.191.255.255", "m": 11, "label": "VNPT [VN]"}, {"id": "amazon", "name": "Amazon Data Services"},
{"from": "24.240.0.0", "to": "24.247.255.255", "m": 13, "label": "Charter [US]"}, {"id": "amazon-microsoft", "name": "Amazon or Microsoft"},
{"from": "27.106.0.0", "to": "27.106.127.254", "label": "Huawei [US]"}, {"id": "brasilnet", "name": "BrasilNet"},
{"from": "34.0.0.0", "to": "34.191.255.254", "label": "Google LLC"}, {"id": "charter", "name": "Charter Communications Inc."},
{"from": "43.132.0.0", "to": "43.132.255.254", "m": 16, "label": "Tencent [CN]"}, {"id": "cnisp", "name": "China ISP"},
{"from": "43.133.0.0", "to": "43.133.255.254", "m": 16, "label": "Tencent [CN]"}, {"id": "cnmob", "name": "China Mobile Communications Corp."},
{"from": "45.0.0.0", "to": "45.255.255.254", "label": "Various small ISPs, mostly BR"}, {"id": "google", "name": "Google LLC"},
{"from": "46.250.160.0", "to": "46.250.191.254", "label": "Huawei [MX]"}, {"id": "google-amazon", "name": "Google or Amazon"},
{"from": "47.200.0.0", "to": "47.203.255.255", "m": 14, "label": "Frontier Communications [US]"}, {"id": "hetzner", "name": "Hetzner Hosting (US)"},
{"from": "49.0.200.0", "to": "49.0.255.254", "label": "Huawei [SG]"}, {"id": "huawei", "name": "Huawei"},
{"from": "52.220.0.0", "to": "52.222.255.254", "label": "Amazon Data Services"}, {"id": "misc_sa", "name": "Various South- and Central-American ISPs"},
{"from": "66.249.64.0", "to": "66.249.95.255", "m": 19, "label": "Google LLC"}, {"id": "tencent", "name": "Tencent"},
{"from": "84.37.35.0", "to": "84.37.255.254", "label": "GTT.net [US]"}, {"id": "unicom", "name": "China Unicom"},
{"from": "94.74.64.0", "to": "94.74.127.254", "label": "Huawei [HK]"}, {"id": "vnpt", "name": "Vietnam Telecom"},
{"from": "101.0.0.0", "to": "101.255.255.254", "label": "ChinaNet [CN]"}, {"id": "vdsina", "name": "VDSina NL"},
{"from": "104.196.0.0", "to": "104.199.255.255", "m": 14, "label": "Google LLC"}, {"id": "zenlayer", "name": "Zenlayer"}
{"from": "110.238.80.0", "to": "110.238.127.254", "label": "Huawei [SG]"}, ],
{"from": "111.119.192.0", "to": "111.119.255.254", "label": "Huawei [SG]"}, "ranges": [
{"from": "113.160.0.0", "to": "113.191.255.254", "m": 11, "label": "VNPT [VN]"}, {"from": "3.0.0.0", "to": "3.255.255.254", "m": 8, "g": "amazon"},
{"from": "119.0.0.0", "to": "119.207.255.254", "label": "Unicom [CN]"}, {"from": "5.161.0.0", "to": "5.161.255.255", "m": 16, "g": "hetzner"},
{"from": "121.91.168.", "to": "121.91.175.254", "label": "Huawei [HK]"}, {"from": "8.128.0.0", "to": "8.191.255.254", "m": 10, "g": "alibaba"},
{"from": "122.8.0.0", "to": "122.8.255.254", "label": "CN-ISP [CN]"}, {"from": "13.216.0.0", "to": "13.223.255.254", "m": 11, "g": "amazon"},
{"from": "122.9.0.0", "to": "122.9.255.254", "label": "Huawei [CN]"}, {"from": "14.160.0.0", "to": "14.191.255.254", "m": 11, "g": "vnpt"},
{"from": "123.16.0.0", "to": "123.31.255.255", "m": 12, "label": "VNPT [VN]"}, {"from": "14.192.0.0", "to": "14.255.255.254", "m": 10, "g": "vnpt"},
{"from": "124.243.128.0", "to": "124.243.191.254", "label": "Huawei [SG]"}, {"from": "14.224.0.0", "to": "24.255.255.254", "m": 11, "g": "vnpt"},
{"from": "142.147.128.0", "to": "1142.147.255.254", "label": "Web2Objects LLC [US]"}, {"from": "27.106.0.0", "to": "27.106.127.254", "m": 17, "g": "huawei"},
{"from": "150.40.128.0", "to": "150.40.255.254", "label": "Huawei [HK]"}, {"from": "34.0.0.0", "to": "34.255.255.254", "m": 8, "g": "google-amazon"},
{"from": "159.138.0.0", "to": "159.138.225.254", "label": "Huawei [TH]"}, {"from": "39.64.0.0", "to": "39.95.255.254", "g": "cnmob"},
{"from": "162.128.0.0", "to": "162.128.255.254", "label": "Zenlayer [SG]"}, {"from": "43.132.0.0", "to": "43.132.255.254", "m": 16, "g": "tencent"},
{"from": "166.108.192.0", "to": "166.108.255.254", "label": "Huawei [SG]"}, {"from": "43.133.0.0", "to": "43.133.255.254", "m": 16, "g": "tencent"},
{"from": "177.0.0.0", "to": "177.255.255.254", "label": "BrasilNET [BR]"}, {"from": "44.192.0.0", "to": "44.255.255.254", "m": 16, "g": "tencent"},
{"from": "179.0.0.0", "to": "179.255.255.254", "label": "BrasilNET [BR]"}, {"from": "45.0.0.0", "to": "45.255.255.254", "m": 10, "g": "amazon"},
{"from": "183.87.32.0", "to": "183.87.159.254", "label": "Huawei [HK]"}, {"from": "46.250.160.0", "to": "46.250.191.254", "m": 19, "g": "huawei"},
{"from": "186.0.0.0", "to": "186.255.255.254", "m": 8, "label": "South-American ISPs (186.x)"}, {"from": "47.82.0.0", "to": "47.82.255.254", "m": 16, "g": "alibaba"},
{"from": "187.0.0.0", "to": "187.255.255.254", "m": 8, "label": "South-American ISPs (187.x)"}, {"from": "47.200.0.0", "to": "47.203.255.254", "m": 14, "g": "frontier"},
{"from": "188.0.0.0", "to": "188.255.255.254", "m": 8, "label": "South-American ISPs (188.x)"}, {"from": "49.0.192.0", "to": "49.0.255.254", "m": 18, "g": "huawei"},
{"from": "189.0.0.0", "to": "189.255.255.254", "m": 8, "label": "South-American ISPs (189.x)"}, {"from": "52.0.0.0", "to": "52.255.255.254", "m": 8, "g": "amazon-microsoft"},
{"from": "190.0.0.0", "to": "190.255.255.254", "m": 8, "label": "South-American ISPs (190.x)"}, {"from": "54.0.0.0", "to": "54.255.255.254", "m": 8, "g": "amazon"},
{"from": "191.0.0.0", "to": "191.255.255.254", "m": 8, "label": "South-American ISPs (191.x)"}, {"from": "66.249.64.0", "to": "66.249.95.254", "m": 19, "g": "google"},
{"from": "192.124.170.0", "to": "192.124.182.254", "label": "Relcom [CZ]"}, {"from": "84.37.35.0", "to": "84.37.255.254", "g": "gtt"},
{"from": "195.37.0.0", "to": "195.37.255.255", "m": 16, "label": "DFN [DE]"}, {"from": "94.74.64.0", "to": "94.74.127.254", "m": 18, "g": "huawei"},
{"from": "2001:4860::::::", "to": "2001:4860:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Google LLC [US]"}, {"from": "91.84.96.0", "to": "91.84.127.254", "m": 19, "g": "vdsina"},
{"from": "2001:0ee0::::::", "to": "2001:ee3:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "label": "VNPT [VN]"}, {"from": "101.0.0.0", "to": "101.255.255.254", "m": 8,"g": "chinanet"},
{"from": "2408:8210::::::", "to": "2408:8210:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "label": "China Unicom [CN]"}, {"from": "104.196.0.0", "to": "104.199.255.254", "m": 14, "g": "google"},
{"from": "2600:1f00::::::", "to": "2600:1fff:ffff:ffff:ffff:ffff:ffff:ffff", "m": "Amazon Cloud"}, {"from": "110.238.64.0", "to": "110.238.127.254", "m": 18, "g": "huawei"},
{"from": "2603:6010::::::", "to": "2603:6010:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Charter [US]"}, {"from": "111.119.192.0", "to": "111.119.255.254", "m": 18, "g": "huawei"},
{"from": "2603:8000::::::", "to": "2603:80ff:ffff:ffff:ffff:ffff:ffff:ffff", "m": 24, "label": "Charter [US]"}, {"from": "113.160.0.0", "to": "113.191.255.254", "m": 11, "g": "vnpt"},
{"from": "2607:a400::::::", "to": "2607:a400:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Zenlayer Inc. [US]"}, {"from": "114.208.0.0", "to": "114.223.255.254", "m": 12, "g": "unicom"},
{"from": "2804:::::::", "to": "2804:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "m": 16, "label": "Inspire [BR]"}, {"from": "114.224.0.0", "to": "114.255.255.254", "m": 11, "g": "unicom"},
{"from": "2a09:bac2::::::", "to": "2a09:bac2:0:ffff:ffff:ffff:ffff:ffff", "m": 48, "label": "Cloudflare"}, {"from": "119.8.0.0", "to": "119.8.255.254", "m": 16, "g": "huawei"},
{"from": "2a0a:4cc0::::::", "to": "2a0a:4cc0:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "label": "Netcup [DE]"} {"from": "119.13.0.0", "to": "119.13.255.254", "m": 16, "g": "huawei"},
{"from": "121.91.168.0", "to": "121.91.175.254", "m": 21, "g": "huawei"},
{"from": "122.8.0.0", "to": "122.8.255.254", "g": "cnisp"},
{"from": "122.9.0.0", "to": "122.9.255.254", "m": 16, "g": "huawei"},
{"from": "123.16.0.0", "to": "123.31.255.254", "m": 12, "g": "vnpt"},
{"from": "124.243.128.0", "to": "124.243.191.254", "m": 18, "g": "huawei"},
{"from": "138.59.0.0", "to": "138.59.225.254", "m": 16, "g": "misc_sa"},
{"from": "138.121.0.0", "to": "138.121.225.254", "m": 16, "g": "misc_sa"},
{"from": "142.147.128.0", "to": "1142.147.255.254", "m": 17, "g": "w2obj"},
{"from": "146.174.128.0", "to": "146.174.191.254", "m": 18, "g": "huawei"},
{"from": "150.40.128.0", "to": "150.40.255.254", "m": 17, "g": "huawei"},
{"from": "159.138.0.0", "to": "159.138.225.254", "m": 16, "g": "huawei"},
{"from": "162.128.0.0", "to": "162.128.255.254", "m": 16, "g": "zenlayer"},
{"from": "166.108.192.0", "to": "166.108.255.254", "m": 18, "g": "huawei"},
{"from": "168.232.192.0", "to": "168.232.255.254", "m": 16, "g": "misc_sa"},
{"from": "170.82.0.0", "to": "170.82.255.254", "m": 16, "g": "misc_sa"},
{"from": "170.254.0.0", "to": "170.254.255.254", "m": 16, "g": "misc_sa"},
{"from": "171.224.0.0", "to": "171.239.255.254", "m": 12, "g": "viettel"},
{"from": "177.0.0.0", "to": "177.255.255.254", "m": 8, "g": "brasilnet"},
{"from": "179.0.0.0", "to": "179.255.255.254", "m": 8, "g": "brasilnet"},
{"from": "183.87.32.0", "to": "183.87.63.254", "m": 19, "g": "huawei"},
{"from": "186.0.0.0", "to": "186.255.255.254", "m": 8, "g": "misc_sa"},
{"from": "187.0.0.0", "to": "187.255.255.254", "m": 8, "g": "misc_sa"},
{"from": "188.0.0.0", "to": "188.255.255.254", "m": 8, "g": "misc_sa"},
{"from": "189.0.0.0", "to": "189.255.255.254", "m": 8, "g": "misc_sa"},
{"from": "190.0.0.0", "to": "190.255.255.254", "m": 8, "g": "misc_sa"},
{"from": "191.0.0.0", "to": "191.255.255.254", "m": 8, "g": "misc_sa"},
{"from": "192.124.170.0", "to": "192.124.182.254", "g": "relcom"},
{"from": "200.0.0.0", "to": "200.255.255.254", "m": 8, "g": "misc_sa"},
{"from": "201.0.0.0", "to": "201.255.255.254", "m": 8, "g": "misc_sa"},
{"from": "212.95.128.0", "to": "212.95.159.254", "m": 19, "g": "asiacell"},
{"from": "222.252.0.0", "to": "222.252.255.254", "m": 14, "g": "vnpt"},
{"from": "2001:4860::::::", "to": "2001:4860:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "g": "google"},
{"from": "2001:0ee0::::::", "to": "2001:ee3:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "g": "vnpt"},
{"from": "2408:8210::::::", "to": "2408:8210:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "g": "unicom"},
{"from": "2600:1f00::::::", "to": "2600:1fff:ffff:ffff:ffff:ffff:ffff:ffff", "m": 24, "g": "amazon"},
{"from": "2603:6010::::::", "to": "2603:6010:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "g": "charter"},
{"from": "2603:8000::::::", "to": "2603:80ff:ffff:ffff:ffff:ffff:ffff:ffff", "m": 24, "g": "charter"},
{"from": "2607:a400::::::", "to": "2607:a400:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "g": "zenlayer"},
{"from": "2804:::::::", "to": "2804:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "m": 16, "g": "misc_sa"},
{"from": "2a0a:4cc0::::::", "to": "2a0a:4cc0:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "g": "netcup"}
] ]
}

2
config/known-platforms.json
View File

@@ -21,7 +21,7 @@
}, },
{"n": "Old Android", {"n": "Old Android",
"id": "androidold", "id": "androidold",
"rx": [ "Android[\\s;\\/](\\d)\\." ] "rx": [ "Android[\\s;\\/](\\d)[\\.;\\)]" ]
}, },
{"n": "Android", {"n": "Android",
"id": "android", "id": "android",

196
script.js
View File

@@ -19,6 +19,12 @@ const BM_LOGTYPE = Object.freeze({
'TICKER': 'tck' 'TICKER': 'tck'
}); });
// enumeration of IP versions:
const BM_IPVERSION = Object.freeze({
'IPv4': 4,
'IPv6': 6
});
/* BotMon root object */ /* BotMon root object */
const BotMon = { const BotMon = {
@@ -35,13 +41,18 @@ const BotMon = {
// get the time offset: // get the time offset:
this._timeDiff = BotMon.t._getTimeOffset(); this._timeDiff = BotMon.t._getTimeOffset();
// get yesterday's date:
let d = new Date();
d.setDate(d.getDate() - 1);
this._datestr = d.toISOString().slice(0, 10);
// init the sub-objects: // init the sub-objects:
BotMon.t._callInit(this); BotMon.t._callInit(this);
}, },
_baseDir: null, _baseDir: null,
_lang: 'en', _lang: 'en',
_datestr: (new Date()).toISOString().slice(0, 10), _datestr: '',
_timeDiff: '', _timeDiff: '',
/* internal tools */ /* internal tools */
@@ -105,9 +116,9 @@ const BotMon = {
if (!ip) { if (!ip) {
return 'null'; return 'null';
} else if (ip.indexOf(':') > 0) { /* IP6 */ } else if (ip.indexOf(':') > 0) { /* IP6 */
return (ip.split(':').map(d => ('0000'+d).slice(-4) ).join('')); return (ip.split(':').map(d => ('0000'+d).slice(-4) ).join(':'));
} else { /* IP4 */ } else { /* IP4 */
return Number(ip.split('.').map(d => ('000'+d).slice(-3) ).join('')); return ip.split('.').map(d => ('000'+d).slice(-3) ).join('.');
} }
}, },
@@ -147,7 +158,7 @@ const BotMon = {
} }
}; };
/* everything specific to the "Today" tab is self-contained in the "live" object: */ /* everything specific to the "Latest" tab is self-contained in the "live" object: */
BotMon.live = { BotMon.live = {
init: function() { init: function() {
//console.info('BotMon.live.init()'); //console.info('BotMon.live.init()');
@@ -180,6 +191,9 @@ BotMon.live = {
case 'rules': case 'rules':
data._dispatchRulesLoaded = true; data._dispatchRulesLoaded = true;
break; break;
case 'ipranges':
data._dispatchIPRangesLoaded = true;
break;
case 'bots': case 'bots':
data._dispatchBotsLoaded = true; data._dispatchBotsLoaded = true;
break; break;
@@ -194,7 +208,7 @@ BotMon.live = {
} }
// are all the flags set? // are all the flags set?
if (data._dispatchBotsLoaded && data._dispatchClientsLoaded && data._dispatchPlatformsLoaded && data._dispatchRulesLoaded) { if (data._dispatchBotsLoaded && data._dispatchClientsLoaded && data._dispatchPlatformsLoaded && data._dispatchRulesLoaded && data._dispatchIPRangesLoaded) {
// chain the log files loading: // chain the log files loading:
BotMon.live.data.loadLogFile(BM_LOGTYPE.SERVER, BotMon.live.data._onServerLogLoaded); BotMon.live.data.loadLogFile(BM_LOGTYPE.SERVER, BotMon.live.data._onServerLogLoaded);
} }
@@ -203,6 +217,7 @@ BotMon.live = {
_dispatchBotsLoaded: false, _dispatchBotsLoaded: false,
_dispatchClientsLoaded: false, _dispatchClientsLoaded: false,
_dispatchPlatformsLoaded: false, _dispatchPlatformsLoaded: false,
_dispatchIPRangesLoaded: false,
_dispatchRulesLoaded: false, _dispatchRulesLoaded: false,
// event callback, after the server log has been loaded: // event callback, after the server log has been loaded:
@@ -239,6 +254,7 @@ BotMon.live = {
}, },
// the data model:
model: { model: {
// visitors storage: // visitors storage:
_visitors: [], _visitors: [],
@@ -486,8 +502,12 @@ BotMon.live = {
} }
}, },
// functions to analyse the data:
analytics: { analytics: {
/**
* Initializes the analytics data storage object:
*/
init: function() { init: function() {
//console.info('BotMon.live.data.analytics.init()'); //console.info('BotMon.live.data.analytics.init()');
}, },
@@ -580,12 +600,15 @@ BotMon.live = {
// add // add
v._pageViews.forEach( pv => { v._pageViews.forEach( pv => {
me.addToRefererList(pv._ref); me.addToRefererList(pv._ref);
me.addToPagesList(pv.pg);
}); });
} }
}); });
BotMon.live.gui.status.hideBusy('Done.'); BotMon.live.gui.status.hideBusy('Done.');
console.log(me._pagesList);
}, },
// get a list of known bots: // get a list of known bots:
@@ -635,6 +658,49 @@ BotMon.live = {
return rList; return rList;
}, },
// most visited pages list:
_pagesList: [],
/**
* Add a page view to the list of most visited pages.
* @param {string} pageId - The page ID to add to the list.
* @example
* BotMon.live.data.analytics.addToPagesList('1234567890');
*/
addToPagesList: function(pageId) {
//console.log('BotMon.live.data.analytics.addToPagesList', pageId);
const me = BotMon.live.data.analytics;
// already exists?
let pgObj = null;
for (let i = 0; i < me._pagesList.length; i++) {
if (me._pagesList[i].id == pageId) {
pgObj = me._pagesList[i];
break;
}
}
// if not exists, create it:
if (!pgObj) {
pgObj = {
id: pageId,
count: 1
};
me._pagesList.push(pgObj);
} else {
pgObj.count += 1;
}
},
getTopPages: function(max) {
//console.info('BotMon.live.data.analytics.getTopPages('+max+')');
const me = BotMon.live.data.analytics;
return me._pagesList.toSorted( (a, b) => {
return b.count - a.count;
}).slice(0,max);
},
// Referer List: // Referer List:
_refererList: [], _refererList: [],
@@ -696,6 +762,14 @@ BotMon.live = {
}; };
}, },
/**
* Get a sorted list of the top referers.
* The list is sorted in descending order of count.
* If the array has more items than the given maximum, the rest of the items are added to an "other" item.
* Each item in the list has a "pct" property, which is the percentage of the total count.
* @param {number} max - The maximum number of items to return.
* @return {Array} The sorted list of top referers.
*/
getTopReferers: function(max) { getTopReferers: function(max) {
//console.info(('BotMon.live.data.analytics.getTopReferers(' + max + ')')); //console.info(('BotMon.live.data.analytics.getTopReferers(' + max + ')'));
@@ -704,6 +778,15 @@ BotMon.live = {
return me._makeTopList(me._refererList, max); return me._makeTopList(me._refererList, max);
}, },
/**
* Create a sorted list of top items from a given array.
* The list is sorted in descending order of count.
* If the array has more items than the given maximum, the rest of the items are added to an "other" item.
* Each item in the list has a "pct" property, which is the percentage of the total count.
* @param {Array} arr - The array to sort and truncate.
* @param {number} max - The maximum number of items to return.
* @return {Array} The sorted list of top items.
*/
_makeTopList: function(arr, max) { _makeTopList: function(arr, max) {
//console.info(('BotMon.live.data.analytics._makeTopList(arr,' + max + ')')); //console.info(('BotMon.live.data.analytics._makeTopList(arr,' + max + ')'));
@@ -896,6 +979,7 @@ BotMon.live = {
} }
}, },
// information on "known bots":
bots: { bots: {
// loads the list of known bots from a JSON file: // loads the list of known bots from a JSON file:
init: async function() { init: async function() {
@@ -972,6 +1056,7 @@ BotMon.live = {
_list: [] _list: []
}, },
// information on known clients (browsers):
clients: { clients: {
// loads the list of known clients from a JSON file: // loads the list of known clients from a JSON file:
init: async function() { init: async function() {
@@ -1038,6 +1123,7 @@ BotMon.live = {
}, },
// information on known platforms (operating systems):
platforms: { platforms: {
// loads the list of known platforms from a JSON file: // loads the list of known platforms from a JSON file:
init: async function() { init: async function() {
@@ -1104,15 +1190,43 @@ BotMon.live = {
}, },
// storage and functions for the known bot IP-Ranges:
ipRanges: { ipRanges: {
init: function() { init: function() {
//console.log('BotMon.live.data.ipRanges.init()'); //console.log('BotMon.live.data.ipRanges.init()');
// #TODO: Load from separate IP-Ranges file // #TODO: Load from separate IP-Ranges file
// load the rules file:
const me = BotMon.live.data;
try {
BotMon.live.data._loadSettingsFile(['user-ipranges', 'known-ipranges'],
(json) => {
// groups can be just saved in the data structure:
if (json.groups && json.groups.constructor.name == 'Array') {
me.ipRanges._groups = json.groups;
}
// groups can be just saved in the data structure:
if (json.ranges && json.ranges.constructor.name == 'Array') {
json.ranges.forEach(range => {
me.ipRanges.add(range);
})
}
// finished loading
BotMon.live.gui.status.hideBusy("Status: Done.");
BotMon.live.data._dispatch('ipranges')
});
} catch (error) {
BotMon.live.gui.status.setError("Error while loading the config file: " + error.message);
}
}, },
// the actual bot list is stored here: // the actual bot list is stored here:
_list: [], _list: [],
_groups: [],
add: function(data) { add: function(data) {
//console.log('BotMon.live.data.ipRanges.add(',data,')'); //console.log('BotMon.live.data.ipRanges.add(',data,')');
@@ -1123,14 +1237,29 @@ BotMon.live = {
const ip2Num = BotMon.t._ip2Num; const ip2Num = BotMon.t._ip2Num;
let item = { let item = {
'cidr': data.from.replaceAll(/::+/g, '::') + '/' + ( data.m ? data.m : '??' ),
'from': ip2Num(data.from), 'from': ip2Num(data.from),
'to': ip2Num(data.to), 'to': ip2Num(data.to),
'label': data.label 'm': data.m,
'g': data.g
}; };
me._list.push(item); me._list.push(item);
}, },
getOwner: function(rangeInfo) {
const me = BotMon.live.data.ipRanges;
for (let i=0; i < me._groups.length; i++) {
const it = me._groups[i];
if (it.id == rangeInfo.g) {
return it.name;
}
}
return `Unknown (“${rangeInfo.g})`;
},
match: function(ip) { match: function(ip) {
//console.log('BotMon.live.data.ipRanges.match(',ip,')'); //console.log('BotMon.live.data.ipRanges.match(',ip,')');
@@ -1151,6 +1280,7 @@ BotMon.live = {
} }
}, },
// storage for the rules and related functions
rules: { rules: {
/** /**
@@ -1183,25 +1313,17 @@ BotMon.live = {
if (json.threshold) me._threshold = json.threshold; if (json.threshold) me._threshold = json.threshold;
// set the rules list: // set the rules list:
if (json.rules) { if (json.rules && json.rules.constructor.name == 'Array') {
me.rules._rulesList = json.rules; me.rules._rulesList = json.rules;
} }
// load the IP ranges:
if (json.ipRanges) {
json.ipRanges.forEach( it => me.ipRanges.add(it));
};
}
);
} catch (error) {
BotMon.live.gui.status.setError("Error while loading the config file: " + error.message);
} finally {
BotMon.live.gui.status.hideBusy("Status: Done."); BotMon.live.gui.status.hideBusy("Status: Done.");
BotMon.live.data._dispatch('rules') BotMon.live.data._dispatch('rules')
} }
);
} catch (error) {
BotMon.live.gui.status.setError("Error while loading the config file: " + error.message);
}
}, },
_rulesList: [], // list of rules to find out if a visitor is a bot _rulesList: [], // list of rules to find out if a visitor is a bot
@@ -1717,6 +1839,31 @@ BotMon.live = {
} }
} }
// update the top pages;
const wmpages = document.getElementById('botmon__today__wm_pages');
if (wmpages) {
wmpages.appendChild(makeElement('dt', {}, "Top pages"));
const pgList = BotMon.live.data.analytics.getTopPages(maxItemsPerList);
if (pgList) {
pgList.forEach( (pgInfo) => {
const pgDd = makeElement('dd');
pgDd.appendChild(makeElement('a', {
'class': 'page_icon',
'href': DOKU_BASE + 'doku.php?id=' + encodeURIComponent(pgInfo.id),
'target': 'preview',
'title': "PageID: " + pgInfo.id
}, pgInfo.id));
pgDd.appendChild(makeElement('span', {
'class': 'count',
'title': pgInfo.count + " page views"
}, pgInfo.count));
wmpages.appendChild(pgDd);
});
}
}
// update the top referrers; // update the top referrers;
const wmreferers = document.getElementById('botmon__today__wm_referers'); const wmreferers = document.getElementById('botmon__today__wm_referers');
if (wmreferers) { if (wmreferers) {
@@ -2014,6 +2161,10 @@ BotMon.live = {
dl.appendChild(make('dd', {'class': 'has_icon platform pf_' + (data._platform ? data._platform.id : 'unknown')}, dl.appendChild(make('dd', {'class': 'has_icon platform pf_' + (data._platform ? data._platform.id : 'unknown')},
platformName + ( data._platform.v > 0 ? ' (' + data._platform.v + ')' : '' ) )); platformName + ( data._platform.v > 0 ? ' (' + data._platform.v + ')' : '' ) ));
/*dl.appendChild(make('dt', {}, "ID:"));
dl.appendChild(make('dd', {'class': 'has_icon ip' + data.typ}, data.id));*/
}
dl.appendChild(make('dt', {}, "IP-Address:")); dl.appendChild(make('dt', {}, "IP-Address:"));
const ipItem = make('dd', {'class': 'has_icon ipaddr ip' + ipType}); const ipItem = make('dd', {'class': 'has_icon ipaddr ip' + ipType});
ipItem.appendChild(make('span', {'class': 'address'} , data.ip)); ipItem.appendChild(make('span', {'class': 'address'} , data.ip));
@@ -2031,10 +2182,6 @@ BotMon.live = {
} , "DNS Info")); } , "DNS Info"));
dl.appendChild(ipItem); dl.appendChild(ipItem);
/*dl.appendChild(make('dt', {}, "ID:"));
dl.appendChild(make('dd', {'class': 'has_icon ip' + data.typ}, data.id));*/
}
if (Math.abs(data._lastSeen - data._firstSeen) < 100) { if (Math.abs(data._lastSeen - data._firstSeen) < 100) {
dl.appendChild(make('dt', {}, "Seen:")); dl.appendChild(make('dt', {}, "Seen:"));
dl.appendChild(make('dd', {'class': 'seen'}, data._firstSeen.toLocaleString())); dl.appendChild(make('dd', {'class': 'seen'}, data._firstSeen.toLocaleString()));
@@ -2099,7 +2246,8 @@ BotMon.live = {
if (tObj.func == 'fromKnownBotIP') { if (tObj.func == 'fromKnownBotIP') {
const rangeInfo = BotMon.live.data.ipRanges.match(data.ip); const rangeInfo = BotMon.live.data.ipRanges.match(data.ip);
if (rangeInfo) { if (rangeInfo) {
tDesc += ' (' + (rangeInfo.label ? rangeInfo.label : 'Unknown') + ')'; const owner = BotMon.live.data.ipRanges.getOwner(rangeInfo);
tDesc += ' (range: “' + rangeInfo.cidr + '”, ' + owner + ')';
} }
} }

22
style.less
View File

@@ -373,8 +373,23 @@
&.ref_other::before { background-image: url('img/more.svg') } &.ref_other::before { background-image: url('img/more.svg') }
} }
.page_icon {
display: inline-flex;
column-gap: .25em;
align-items: center;
}
.page_icon::before {
content: '';
display: inline-block;
width: 20px; height: 20px;
background: transparent url('img/page.svg') center no-repeat;
background-position: 0 0;
background-size: 20px;
}
/* grid layout for the overview: */ /* grid layout for the overview: */
.botmon_bots_grid, .botmon_webmetrics_grid { .botmon_bots_grid, .botmon_webmetrics_grid, .botmon_traffic_grid {
& { & {
display: grid; display: grid;
grid-gap: 0 .33em; grid-gap: 0 .33em;
@@ -391,7 +406,10 @@
grid-template-columns: 1fr 1fr 1fr; grid-template-columns: 1fr 1fr 1fr;
} }
.botmon_webmetrics_grid { .botmon_webmetrics_grid {
grid-template-columns: 1fr 1fr 1fr 1fr; grid-template-columns: 1fr 1fr 1fr;
}
.botmon_traffic_grid {
grid-template-columns: 2fr 1fr;
} }
/* the "today" tab: */ /* the "today" tab: */