Web traffic monitoring

This commit is contained in:
Sascha Leib
2025-10-03 21:30:29 +02:00
parent d02708c1ae
commit c31fb4d0a5
9 changed files with 321 additions and 167 deletions

View File

@@ -81,8 +81,6 @@ class action_plugin_botmon extends DokuWiki_Action_Plugin {
$username = ( !empty($INFO['userinfo']) && !empty($INFO['userinfo']['name'])
? $INFO['userinfo']['name'] : '');
// clean the page ID
$pageId = preg_replace('/[\x00-\x1F]/', "\u{FFFD}", $INFO['id'] ?? '');

View File

@@ -42,7 +42,7 @@ class admin_plugin_botmon extends AdminPlugin {
<h1>Bot Monitoring Plugin</h1>
<nav id="botmon__tabs">
<ul class="tabs" role="tablist">
<li role="presentation" class="active"><a role="tab" href="#botmon__panel1" aria-controls="botmon__panel1" id="botmon__tab1" aria-selected="true">Today</a></li>
<li role="presentation" class="active"><a role="tab" href="#botmon__panel1" aria-controls="botmon__panel1" id="botmon__tab1" aria-selected="true">Latest</a></li>
</ul>
</nav>';
@@ -51,7 +51,7 @@ class admin_plugin_botmon extends AdminPlugin {
}
echo '<article role="tabpanel" id="botmon__today"">
<h2 class="a11y">Today</h2>
<h2 class="a11y">Latest data</h2>
<header id="botmon__today__title">Loading&nbsp;&hellip;</header>
<div id="botmon__today__content">
<details id="botmon__today__overview" open>
@@ -68,6 +68,12 @@ class admin_plugin_botmon extends AdminPlugin {
<dl id="botmon__today__wm_overview"></dl>
<dl id="botmon__today__wm_clients"></dl>
<dl id="botmon__today__wm_platforms"></dl>
</div>
</details>
<details id="botmon__today__traffic">
<summary>Web traffic (humans only)</summary>
<div class="botmon_traffic_grid">
<dl id="botmon__today__wm_pages"></dl>
<dl id="botmon__today__wm_referers"></dl>
</div>
</details>

View File

@@ -73,7 +73,7 @@
console.error(err);
} finally {
/* send the next heartbeat signal after x seconds: */
// setTimeout(this._onHeartbeat.bind(this, this._src.replace( this._scriptName, '/tick.php')),this._heartbeat * 1000);
setTimeout(this._onHeartbeat.bind(this, this._src.replace( this._scriptName, '/tick.php')),this._heartbeat * 1000);
}
}
}).init();

View File

@@ -61,74 +61,5 @@
"id": "langMatch", "desc": "Clients Accept-Language header does not match the page language",
"bot": 30
}
],
"ipRanges": [
{"from": "3.0.0.0", "to": "3.255.255.254", "label": "Amazon Data Services [US]"},
{"from": "5.161.0.0", "to": "5.161.255.255", "m": 16, "label": "Hetzner [US]"},
{"from": "8.127.0.0", "to": "8.223.255.254", "label": "Alibaba [CN]"},
{"from": "14.160.0.0", "to": "14.191.255.254", "m": 11, "label": "VNPT [VN]"},
{"from": "14.192.0.0", "to": "14.255.255.254", "m": 10, "label": "VNPT [VN]"},
{"from": "14.224.0.0", "to": "24.255.255.254", "m": 11, "label": "Charter [US]"},
{"from": "27.106.0.0", "to": "27.106.127.254", "m": 17, "label": "Huawei [US]"},
{"from": "34.0.0.0", "to": "34.191.255.254", "label": "Google LLC"},
{"from": "43.132.0.0", "to": "43.132.255.254", "m": 16, "label": "Tencent [CN]"},
{"from": "43.133.0.0", "to": "43.133.255.254", "m": 16, "label": "Tencent [CN]"},
{"from": "45.0.0.0", "to": "45.255.255.254", "label": "Various small ISPs, mostly BR"},
{"from": "46.250.160.0", "to": "46.250.191.254", "m": 19, "label": "Huawei [MX]"},
{"from": "47.200.0.0", "to": "47.203.255.254", "m": 14, "label": "Frontier Communications [US]"},
{"from": "49.0.192.0", "to": "49.0.255.254", "m": 18, "label": "Huawei [SG]"},
{"from": "52.220.0.0", "to": "52.222.255.254", "label": "Amazon Data Services"},
{"from": "66.249.64.0", "to": "66.249.95.254", "m": 19, "label": "Google LLC"},
{"from": "84.37.35.0", "to": "84.37.255.254", "label": "GTT.net [US]"},
{"from": "94.74.64.0", "to": "94.74.127.254", "m": 18, "label": "Huawei [HK]"},
{"from": "101.0.0.0", "to": "101.255.255.254", "m": 8,"label": "ChinaNet [CN]"},
{"from": "104.196.0.0", "to": "104.199.255.254", "m": 14, "label": "Google LLC"},
{"from": "110.238.64.0", "to": "110.238.127.254", "m": 18, "label": "Huawei [SG]"},
{"from": "111.119.192.0", "to": "111.119.255.254", "m": 18, "label": "Huawei [SG]"},
{"from": "113.160.0.0", "to": "113.191.255.254", "m": 11, "label": "VNPT [VN]"},
{"from": "119.8.0.0", "to": "119.8.255.254", "m": 16, "label": "Huawei [HK]"},
{"from": "119.13.0.0", "to": "119.13.255.254", "m": 16, "label": "Huawei [HK]"},
{"from": "121.91.168.0", "to": "121.91.175.254", "m": 21, "label": "Huawei [HK]"},
{"from": "122.8.0.0", "to": "122.8.255.254", "label": "CN-ISP [CN]"},
{"from": "122.9.0.0", "to": "122.9.255.254", "m": 16, "label": "Huawei [CN]"},
{"from": "123.16.0.0", "to": "123.31.255.254", "m": 12, "label": "VNPT [VN]"},
{"from": "124.243.128.0", "to": "124.243.191.254", "m": 18, "label": "Huawei [SG]"},
{"from": "138.59.0.0", "to": "138.59.225.254", "m": 16, "label": "South-American ISPs (138.59.x)"},
{"from": "138.121.0.0", "to": "138.121.225.254", "m": 16, "label": "South-American ISPs (138.121.x)"},
{"from": "142.147.128.0", "to": "1142.147.255.254", "m": 17, "label": "Web2Objects LLC [US]"},
{"from": "146.174.128.0", "to": "146.174.191.254", "m": 18, "label": "Huawei [SG]"},
{"from": "150.40.128.0", "to": "150.40.255.254", "m": 17, "label": "Huawei [HK]"},
{"from": "159.138.0.0", "to": "159.138.225.254", "m": 16, "label": "Huawei [TH]"},
{"from": "162.128.0.0", "to": "162.128.255.254", "m": 16, "label": "Zenlayer [SG]"},
{"from": "166.108.192.0", "to": "166.108.255.254", "m": 18, "label": "Huawei [SG]"},
{"from": "168.232.192.0", "to": "168.232.255.254", "m": 16, "label": "South-American ISPs (168.232.x)"},
{"from": "170.82.0.0", "to": "170.82.255.254", "m": 16, "label": "South-American ISPs (170.254.x)"},
{"from": "170.254.0.0", "to": "170.254.255.254", "m": 16, "label": "South-American ISPs (170.82.x)"},
{"from": "171.224.0.0", "to": "171.239.255.254", "m": 12, "label": "Viettel [VN]"},
{"from": "177.0.0.0", "to": "177.255.255.254", "m": 8, "label": "BrasilNET [BR]"},
{"from": "179.0.0.0", "to": "179.255.255.254", "m": 8, "label": "BrasilNET [BR]"},
{"from": "183.87.32.0", "to": "183.87.63.254", "label": "Huawei [HK]"},
{"from": "186.0.0.0", "to": "186.255.255.254", "m": 8, "label": "South-American ISPs (186.x)"},
{"from": "187.0.0.0", "to": "187.255.255.254", "m": 8, "label": "South-American ISPs (187.x)"},
{"from": "188.0.0.0", "to": "188.255.255.254", "m": 8, "label": "South-American ISPs (188.x)"},
{"from": "189.0.0.0", "to": "189.255.255.254", "m": 8, "label": "South-American ISPs (189.x)"},
{"from": "190.0.0.0", "to": "190.255.255.254", "m": 8, "label": "South-American ISPs (190.x)"},
{"from": "191.0.0.0", "to": "191.255.255.254", "m": 8, "label": "South-American ISPs (191.x)"},
{"from": "192.124.170.0", "to": "192.124.182.254", "label": "Relcom [CZ]"},
{"from": "195.37.0.0", "to": "195.37.255.254", "m": 16, "label": "DFN [DE]"},
{"from": "200.0.0.0", "to": "200.255.255.254", "m": 8, "label": "South-American ISPs (200.x)"},
{"from": "201.0.0.0", "to": "201.255.255.254", "m": 8, "label": "South-American ISPs (201.x)"},
{"from": "212.95.128.0", "to": "212.95.159.254", "m": 19, "label": "Asiacell [IQ]"},
{"from": "222.252.0.0", "to": "222.252.255.254", "m": 14, "label": "VNPT [VN]"},
{"from": "2001:4860::::::", "to": "2001:4860:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Google LLC [US]"},
{"from": "2001:0ee0::::::", "to": "2001:ee3:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "label": "VNPT [VN]"},
{"from": "2408:8210::::::", "to": "2408:8210:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "label": "China Unicom [CN]"},
{"from": "2600:1f00::::::", "to": "2600:1fff:ffff:ffff:ffff:ffff:ffff:ffff", "m": 24, "label": "Amazon Cloud"},
{"from": "2603:6010::::::", "to": "2603:6010:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Charter [US]"},
{"from": "2603:8000::::::", "to": "2603:80ff:ffff:ffff:ffff:ffff:ffff:ffff", "m": 24, "label": "Charter [US]"},
{"from": "2607:a400::::::", "to": "2607:a400:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Zenlayer Inc. [US]"},
{"from": "2804:::::::", "to": "2804:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "m": 16, "label": "Inspire [BR]"},
{"from": "2a09:bac2::::::", "to": "2a09:bac2:0:ffff:ffff:ffff:ffff:ffff", "m": 48, "label": "Cloudflare"},
{"from": "2a0a:4cc0::::::", "to": "2a0a:4cc0:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "label": "Netcup [DE]"}
]
}

View File

@@ -297,5 +297,17 @@
"r": [],
"rx": ["Go\\-http\\-client\\/(\\d+)", "quic\\-go\\-HTTP\\/(\\d+)"],
"url": "https://github.com/golang/goen.wi"
},
{"id": "bnl",
"n": "BnL Harvester",
"r": [],
"rx": ["NLUX_IAHarvester\\/(\\d+)"],
"url": "http://crawl.bnl.lu/"
},
{"id": "turnitin",
"n": "TurnitinBot",
"r": [],
"rx": ["Turnitin\\s"],
"url": "https://www.turnitin.com/robot/crawlerinfo.html"
}
]

View File

@@ -1,55 +1,96 @@
[
{"from": "3.0.0.0", "to": "3.255.255.254", "m": 8, "label": "Amazon Data Services [US]"},
{"from": "8.127.0.0", "to": "8.223.255.254", "label": "Alibaba [CN]"},
{"from": "14.160.0.0", "to": "14.191.255.255", "m": 11, "label": "VNPT [VN]"},
{"from": "24.240.0.0", "to": "24.247.255.255", "m": 13, "label": "Charter [US]"},
{"from": "27.106.0.0", "to": "27.106.127.254", "label": "Huawei [US]"},
{"from": "34.0.0.0", "to": "34.191.255.254", "label": "Google LLC"},
{"from": "43.132.0.0", "to": "43.132.255.254", "m": 16, "label": "Tencent [CN]"},
{"from": "43.133.0.0", "to": "43.133.255.254", "m": 16, "label": "Tencent [CN]"},
{"from": "45.0.0.0", "to": "45.255.255.254", "label": "Various small ISPs, mostly BR"},
{"from": "46.250.160.0", "to": "46.250.191.254", "label": "Huawei [MX]"},
{"from": "47.200.0.0", "to": "47.203.255.255", "m": 14, "label": "Frontier Communications [US]"},
{"from": "49.0.200.0", "to": "49.0.255.254", "label": "Huawei [SG]"},
{"from": "52.220.0.0", "to": "52.222.255.254", "label": "Amazon Data Services"},
{"from": "66.249.64.0", "to": "66.249.95.255", "m": 19, "label": "Google LLC"},
{"from": "84.37.35.0", "to": "84.37.255.254", "label": "GTT.net [US]"},
{"from": "94.74.64.0", "to": "94.74.127.254", "label": "Huawei [HK]"},
{"from": "101.0.0.0", "to": "101.255.255.254", "label": "ChinaNet [CN]"},
{"from": "104.196.0.0", "to": "104.199.255.255", "m": 14, "label": "Google LLC"},
{"from": "110.238.80.0", "to": "110.238.127.254", "label": "Huawei [SG]"},
{"from": "111.119.192.0", "to": "111.119.255.254", "label": "Huawei [SG]"},
{"from": "113.160.0.0", "to": "113.191.255.254", "m": 11, "label": "VNPT [VN]"},
{"from": "119.0.0.0", "to": "119.207.255.254", "label": "Unicom [CN]"},
{"from": "121.91.168.", "to": "121.91.175.254", "label": "Huawei [HK]"},
{"from": "122.8.0.0", "to": "122.8.255.254", "label": "CN-ISP [CN]"},
{"from": "122.9.0.0", "to": "122.9.255.254", "label": "Huawei [CN]"},
{"from": "123.16.0.0", "to": "123.31.255.255", "m": 12, "label": "VNPT [VN]"},
{"from": "124.243.128.0", "to": "124.243.191.254", "label": "Huawei [SG]"},
{"from": "142.147.128.0", "to": "1142.147.255.254", "label": "Web2Objects LLC [US]"},
{"from": "150.40.128.0", "to": "150.40.255.254", "label": "Huawei [HK]"},
{"from": "159.138.0.0", "to": "159.138.225.254", "label": "Huawei [TH]"},
{"from": "162.128.0.0", "to": "162.128.255.254", "label": "Zenlayer [SG]"},
{"from": "166.108.192.0", "to": "166.108.255.254", "label": "Huawei [SG]"},
{"from": "177.0.0.0", "to": "177.255.255.254", "label": "BrasilNET [BR]"},
{"from": "179.0.0.0", "to": "179.255.255.254", "label": "BrasilNET [BR]"},
{"from": "183.87.32.0", "to": "183.87.159.254", "label": "Huawei [HK]"},
{"from": "186.0.0.0", "to": "186.255.255.254", "m": 8, "label": "South-American ISPs (186.x)"},
{"from": "187.0.0.0", "to": "187.255.255.254", "m": 8, "label": "South-American ISPs (187.x)"},
{"from": "188.0.0.0", "to": "188.255.255.254", "m": 8, "label": "South-American ISPs (188.x)"},
{"from": "189.0.0.0", "to": "189.255.255.254", "m": 8, "label": "South-American ISPs (189.x)"},
{"from": "190.0.0.0", "to": "190.255.255.254", "m": 8, "label": "South-American ISPs (190.x)"},
{"from": "191.0.0.0", "to": "191.255.255.254", "m": 8, "label": "South-American ISPs (191.x)"},
{"from": "192.124.170.0", "to": "192.124.182.254", "label": "Relcom [CZ]"},
{"from": "195.37.0.0", "to": "195.37.255.255", "m": 16, "label": "DFN [DE]"},
{"from": "2001:4860::::::", "to": "2001:4860:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Google LLC [US]"},
{"from": "2001:0ee0::::::", "to": "2001:ee3:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "label": "VNPT [VN]"},
{"from": "2408:8210::::::", "to": "2408:8210:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "label": "China Unicom [CN]"},
{"from": "2600:1f00::::::", "to": "2600:1fff:ffff:ffff:ffff:ffff:ffff:ffff", "m": "Amazon Cloud"},
{"from": "2603:6010::::::", "to": "2603:6010:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Charter [US]"},
{"from": "2603:8000::::::", "to": "2603:80ff:ffff:ffff:ffff:ffff:ffff:ffff", "m": 24, "label": "Charter [US]"},
{"from": "2607:a400::::::", "to": "2607:a400:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Zenlayer Inc. [US]"},
{"from": "2804:::::::", "to": "2804:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "m": 16, "label": "Inspire [BR]"},
{"from": "2a09:bac2::::::", "to": "2a09:bac2:0:ffff:ffff:ffff:ffff:ffff", "m": 48, "label": "Cloudflare"},
{"from": "2a0a:4cc0::::::", "to": "2a0a:4cc0:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "label": "Netcup [DE]"}
]
{
"groups": [
{"id": "alibaba", "name": "Alibaba"},
{"id": "amazon", "name": "Amazon Data Services"},
{"id": "amazon-microsoft", "name": "Amazon or Microsoft"},
{"id": "brasilnet", "name": "BrasilNet"},
{"id": "charter", "name": "Charter Communications Inc."},
{"id": "cnisp", "name": "China ISP"},
{"id": "cnmob", "name": "China Mobile Communications Corp."},
{"id": "google", "name": "Google LLC"},
{"id": "google-amazon", "name": "Google or Amazon"},
{"id": "hetzner", "name": "Hetzner Hosting (US)"},
{"id": "huawei", "name": "Huawei"},
{"id": "misc_sa", "name": "Various South- and Central-American ISPs"},
{"id": "tencent", "name": "Tencent"},
{"id": "unicom", "name": "China Unicom"},
{"id": "vnpt", "name": "Vietnam Telecom"},
{"id": "vdsina", "name": "VDSina NL"},
{"id": "zenlayer", "name": "Zenlayer"}
],
"ranges": [
{"from": "3.0.0.0", "to": "3.255.255.254", "m": 8, "g": "amazon"},
{"from": "5.161.0.0", "to": "5.161.255.255", "m": 16, "g": "hetzner"},
{"from": "8.128.0.0", "to": "8.191.255.254", "m": 10, "g": "alibaba"},
{"from": "13.216.0.0", "to": "13.223.255.254", "m": 11, "g": "amazon"},
{"from": "14.160.0.0", "to": "14.191.255.254", "m": 11, "g": "vnpt"},
{"from": "14.192.0.0", "to": "14.255.255.254", "m": 10, "g": "vnpt"},
{"from": "14.224.0.0", "to": "24.255.255.254", "m": 11, "g": "vnpt"},
{"from": "27.106.0.0", "to": "27.106.127.254", "m": 17, "g": "huawei"},
{"from": "34.0.0.0", "to": "34.255.255.254", "m": 8, "g": "google-amazon"},
{"from": "39.64.0.0", "to": "39.95.255.254", "g": "cnmob"},
{"from": "43.132.0.0", "to": "43.132.255.254", "m": 16, "g": "tencent"},
{"from": "43.133.0.0", "to": "43.133.255.254", "m": 16, "g": "tencent"},
{"from": "44.192.0.0", "to": "44.255.255.254", "m": 16, "g": "tencent"},
{"from": "45.0.0.0", "to": "45.255.255.254", "m": 10, "g": "amazon"},
{"from": "46.250.160.0", "to": "46.250.191.254", "m": 19, "g": "huawei"},
{"from": "47.82.0.0", "to": "47.82.255.254", "m": 16, "g": "alibaba"},
{"from": "47.200.0.0", "to": "47.203.255.254", "m": 14, "g": "frontier"},
{"from": "49.0.192.0", "to": "49.0.255.254", "m": 18, "g": "huawei"},
{"from": "52.0.0.0", "to": "52.255.255.254", "m": 8, "g": "amazon-microsoft"},
{"from": "54.0.0.0", "to": "54.255.255.254", "m": 8, "g": "amazon"},
{"from": "66.249.64.0", "to": "66.249.95.254", "m": 19, "g": "google"},
{"from": "84.37.35.0", "to": "84.37.255.254", "g": "gtt"},
{"from": "94.74.64.0", "to": "94.74.127.254", "m": 18, "g": "huawei"},
{"from": "91.84.96.0", "to": "91.84.127.254", "m": 19, "g": "vdsina"},
{"from": "101.0.0.0", "to": "101.255.255.254", "m": 8,"g": "chinanet"},
{"from": "104.196.0.0", "to": "104.199.255.254", "m": 14, "g": "google"},
{"from": "110.238.64.0", "to": "110.238.127.254", "m": 18, "g": "huawei"},
{"from": "111.119.192.0", "to": "111.119.255.254", "m": 18, "g": "huawei"},
{"from": "113.160.0.0", "to": "113.191.255.254", "m": 11, "g": "vnpt"},
{"from": "114.208.0.0", "to": "114.223.255.254", "m": 12, "g": "unicom"},
{"from": "114.224.0.0", "to": "114.255.255.254", "m": 11, "g": "unicom"},
{"from": "119.8.0.0", "to": "119.8.255.254", "m": 16, "g": "huawei"},
{"from": "119.13.0.0", "to": "119.13.255.254", "m": 16, "g": "huawei"},
{"from": "121.91.168.0", "to": "121.91.175.254", "m": 21, "g": "huawei"},
{"from": "122.8.0.0", "to": "122.8.255.254", "g": "cnisp"},
{"from": "122.9.0.0", "to": "122.9.255.254", "m": 16, "g": "huawei"},
{"from": "123.16.0.0", "to": "123.31.255.254", "m": 12, "g": "vnpt"},
{"from": "124.243.128.0", "to": "124.243.191.254", "m": 18, "g": "huawei"},
{"from": "138.59.0.0", "to": "138.59.225.254", "m": 16, "g": "misc_sa"},
{"from": "138.121.0.0", "to": "138.121.225.254", "m": 16, "g": "misc_sa"},
{"from": "142.147.128.0", "to": "1142.147.255.254", "m": 17, "g": "w2obj"},
{"from": "146.174.128.0", "to": "146.174.191.254", "m": 18, "g": "huawei"},
{"from": "150.40.128.0", "to": "150.40.255.254", "m": 17, "g": "huawei"},
{"from": "159.138.0.0", "to": "159.138.225.254", "m": 16, "g": "huawei"},
{"from": "162.128.0.0", "to": "162.128.255.254", "m": 16, "g": "zenlayer"},
{"from": "166.108.192.0", "to": "166.108.255.254", "m": 18, "g": "huawei"},
{"from": "168.232.192.0", "to": "168.232.255.254", "m": 16, "g": "misc_sa"},
{"from": "170.82.0.0", "to": "170.82.255.254", "m": 16, "g": "misc_sa"},
{"from": "170.254.0.0", "to": "170.254.255.254", "m": 16, "g": "misc_sa"},
{"from": "171.224.0.0", "to": "171.239.255.254", "m": 12, "g": "viettel"},
{"from": "177.0.0.0", "to": "177.255.255.254", "m": 8, "g": "brasilnet"},
{"from": "179.0.0.0", "to": "179.255.255.254", "m": 8, "g": "brasilnet"},
{"from": "183.87.32.0", "to": "183.87.63.254", "m": 19, "g": "huawei"},
{"from": "186.0.0.0", "to": "186.255.255.254", "m": 8, "g": "misc_sa"},
{"from": "187.0.0.0", "to": "187.255.255.254", "m": 8, "g": "misc_sa"},
{"from": "188.0.0.0", "to": "188.255.255.254", "m": 8, "g": "misc_sa"},
{"from": "189.0.0.0", "to": "189.255.255.254", "m": 8, "g": "misc_sa"},
{"from": "190.0.0.0", "to": "190.255.255.254", "m": 8, "g": "misc_sa"},
{"from": "191.0.0.0", "to": "191.255.255.254", "m": 8, "g": "misc_sa"},
{"from": "192.124.170.0", "to": "192.124.182.254", "g": "relcom"},
{"from": "200.0.0.0", "to": "200.255.255.254", "m": 8, "g": "misc_sa"},
{"from": "201.0.0.0", "to": "201.255.255.254", "m": 8, "g": "misc_sa"},
{"from": "212.95.128.0", "to": "212.95.159.254", "m": 19, "g": "asiacell"},
{"from": "222.252.0.0", "to": "222.252.255.254", "m": 14, "g": "vnpt"},
{"from": "2001:4860::::::", "to": "2001:4860:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "g": "google"},
{"from": "2001:0ee0::::::", "to": "2001:ee3:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "g": "vnpt"},
{"from": "2408:8210::::::", "to": "2408:8210:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "g": "unicom"},
{"from": "2600:1f00::::::", "to": "2600:1fff:ffff:ffff:ffff:ffff:ffff:ffff", "m": 24, "g": "amazon"},
{"from": "2603:6010::::::", "to": "2603:6010:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "g": "charter"},
{"from": "2603:8000::::::", "to": "2603:80ff:ffff:ffff:ffff:ffff:ffff:ffff", "m": 24, "g": "charter"},
{"from": "2607:a400::::::", "to": "2607:a400:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "g": "zenlayer"},
{"from": "2804:::::::", "to": "2804:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "m": 16, "g": "misc_sa"},
{"from": "2a0a:4cc0::::::", "to": "2a0a:4cc0:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "g": "netcup"}
]
}

View File

@@ -21,7 +21,7 @@
},
{"n": "Old Android",
"id": "androidold",
"rx": [ "Android[\\s;\\/](\\d)\\." ]
"rx": [ "Android[\\s;\\/](\\d)[\\.;\\)]" ]
},
{"n": "Android",
"id": "android",

218
script.js
View File

@@ -19,6 +19,12 @@ const BM_LOGTYPE = Object.freeze({
'TICKER': 'tck'
});
// enumeration of IP versions:
const BM_IPVERSION = Object.freeze({
'IPv4': 4,
'IPv6': 6
});
/* BotMon root object */
const BotMon = {
@@ -35,13 +41,18 @@ const BotMon = {
// get the time offset:
this._timeDiff = BotMon.t._getTimeOffset();
// get yesterday's date:
let d = new Date();
d.setDate(d.getDate() - 1);
this._datestr = d.toISOString().slice(0, 10);
// init the sub-objects:
BotMon.t._callInit(this);
},
_baseDir: null,
_lang: 'en',
_datestr: (new Date()).toISOString().slice(0, 10),
_datestr: '',
_timeDiff: '',
/* internal tools */
@@ -105,9 +116,9 @@ const BotMon = {
if (!ip) {
return 'null';
} else if (ip.indexOf(':') > 0) { /* IP6 */
return (ip.split(':').map(d => ('0000'+d).slice(-4) ).join(''));
return (ip.split(':').map(d => ('0000'+d).slice(-4) ).join(':'));
} else { /* IP4 */
return Number(ip.split('.').map(d => ('000'+d).slice(-3) ).join(''));
return ip.split('.').map(d => ('000'+d).slice(-3) ).join('.');
}
},
@@ -147,7 +158,7 @@ const BotMon = {
}
};
/* everything specific to the "Today" tab is self-contained in the "live" object: */
/* everything specific to the "Latest" tab is self-contained in the "live" object: */
BotMon.live = {
init: function() {
//console.info('BotMon.live.init()');
@@ -180,6 +191,9 @@ BotMon.live = {
case 'rules':
data._dispatchRulesLoaded = true;
break;
case 'ipranges':
data._dispatchIPRangesLoaded = true;
break;
case 'bots':
data._dispatchBotsLoaded = true;
break;
@@ -194,7 +208,7 @@ BotMon.live = {
}
// are all the flags set?
if (data._dispatchBotsLoaded && data._dispatchClientsLoaded && data._dispatchPlatformsLoaded && data._dispatchRulesLoaded) {
if (data._dispatchBotsLoaded && data._dispatchClientsLoaded && data._dispatchPlatformsLoaded && data._dispatchRulesLoaded && data._dispatchIPRangesLoaded) {
// chain the log files loading:
BotMon.live.data.loadLogFile(BM_LOGTYPE.SERVER, BotMon.live.data._onServerLogLoaded);
}
@@ -203,6 +217,7 @@ BotMon.live = {
_dispatchBotsLoaded: false,
_dispatchClientsLoaded: false,
_dispatchPlatformsLoaded: false,
_dispatchIPRangesLoaded: false,
_dispatchRulesLoaded: false,
// event callback, after the server log has been loaded:
@@ -239,6 +254,7 @@ BotMon.live = {
},
// the data model:
model: {
// visitors storage:
_visitors: [],
@@ -486,8 +502,12 @@ BotMon.live = {
}
},
// functions to analyse the data:
analytics: {
/**
* Initializes the analytics data storage object:
*/
init: function() {
//console.info('BotMon.live.data.analytics.init()');
},
@@ -580,12 +600,15 @@ BotMon.live = {
// add
v._pageViews.forEach( pv => {
me.addToRefererList(pv._ref);
me.addToPagesList(pv.pg);
});
}
});
BotMon.live.gui.status.hideBusy('Done.');
console.log(me._pagesList);
},
// get a list of known bots:
@@ -635,6 +658,49 @@ BotMon.live = {
return rList;
},
// most visited pages list:
_pagesList: [],
/**
* Add a page view to the list of most visited pages.
* @param {string} pageId - The page ID to add to the list.
* @example
* BotMon.live.data.analytics.addToPagesList('1234567890');
*/
addToPagesList: function(pageId) {
//console.log('BotMon.live.data.analytics.addToPagesList', pageId);
const me = BotMon.live.data.analytics;
// already exists?
let pgObj = null;
for (let i = 0; i < me._pagesList.length; i++) {
if (me._pagesList[i].id == pageId) {
pgObj = me._pagesList[i];
break;
}
}
// if not exists, create it:
if (!pgObj) {
pgObj = {
id: pageId,
count: 1
};
me._pagesList.push(pgObj);
} else {
pgObj.count += 1;
}
},
getTopPages: function(max) {
//console.info('BotMon.live.data.analytics.getTopPages('+max+')');
const me = BotMon.live.data.analytics;
return me._pagesList.toSorted( (a, b) => {
return b.count - a.count;
}).slice(0,max);
},
// Referer List:
_refererList: [],
@@ -696,6 +762,14 @@ BotMon.live = {
};
},
/**
* Get a sorted list of the top referers.
* The list is sorted in descending order of count.
* If the array has more items than the given maximum, the rest of the items are added to an "other" item.
* Each item in the list has a "pct" property, which is the percentage of the total count.
* @param {number} max - The maximum number of items to return.
* @return {Array} The sorted list of top referers.
*/
getTopReferers: function(max) {
//console.info(('BotMon.live.data.analytics.getTopReferers(' + max + ')'));
@@ -704,6 +778,15 @@ BotMon.live = {
return me._makeTopList(me._refererList, max);
},
/**
* Create a sorted list of top items from a given array.
* The list is sorted in descending order of count.
* If the array has more items than the given maximum, the rest of the items are added to an "other" item.
* Each item in the list has a "pct" property, which is the percentage of the total count.
* @param {Array} arr - The array to sort and truncate.
* @param {number} max - The maximum number of items to return.
* @return {Array} The sorted list of top items.
*/
_makeTopList: function(arr, max) {
//console.info(('BotMon.live.data.analytics._makeTopList(arr,' + max + ')'));
@@ -896,6 +979,7 @@ BotMon.live = {
}
},
// information on "known bots":
bots: {
// loads the list of known bots from a JSON file:
init: async function() {
@@ -972,6 +1056,7 @@ BotMon.live = {
_list: []
},
// information on known clients (browsers):
clients: {
// loads the list of known clients from a JSON file:
init: async function() {
@@ -1038,6 +1123,7 @@ BotMon.live = {
},
// information on known platforms (operating systems):
platforms: {
// loads the list of known platforms from a JSON file:
init: async function() {
@@ -1104,15 +1190,43 @@ BotMon.live = {
},
// storage and functions for the known bot IP-Ranges:
ipRanges: {
init: function() {
//console.log('BotMon.live.data.ipRanges.init()');
// #TODO: Load from separate IP-Ranges file
// load the rules file:
const me = BotMon.live.data;
try {
BotMon.live.data._loadSettingsFile(['user-ipranges', 'known-ipranges'],
(json) => {
// groups can be just saved in the data structure:
if (json.groups && json.groups.constructor.name == 'Array') {
me.ipRanges._groups = json.groups;
}
// groups can be just saved in the data structure:
if (json.ranges && json.ranges.constructor.name == 'Array') {
json.ranges.forEach(range => {
me.ipRanges.add(range);
})
}
// finished loading
BotMon.live.gui.status.hideBusy("Status: Done.");
BotMon.live.data._dispatch('ipranges')
});
} catch (error) {
BotMon.live.gui.status.setError("Error while loading the config file: " + error.message);
}
},
// the actual bot list is stored here:
_list: [],
_groups: [],
add: function(data) {
//console.log('BotMon.live.data.ipRanges.add(',data,')');
@@ -1123,14 +1237,29 @@ BotMon.live = {
const ip2Num = BotMon.t._ip2Num;
let item = {
'cidr': data.from.replaceAll(/::+/g, '::') + '/' + ( data.m ? data.m : '??' ),
'from': ip2Num(data.from),
'to': ip2Num(data.to),
'label': data.label
'm': data.m,
'g': data.g
};
me._list.push(item);
},
getOwner: function(rangeInfo) {
const me = BotMon.live.data.ipRanges;
for (let i=0; i < me._groups.length; i++) {
const it = me._groups[i];
if (it.id == rangeInfo.g) {
return it.name;
}
}
return `Unknown (“${rangeInfo.g})`;
},
match: function(ip) {
//console.log('BotMon.live.data.ipRanges.match(',ip,')');
@@ -1151,6 +1280,7 @@ BotMon.live = {
}
},
// storage for the rules and related functions
rules: {
/**
@@ -1183,25 +1313,17 @@ BotMon.live = {
if (json.threshold) me._threshold = json.threshold;
// set the rules list:
if (json.rules) {
if (json.rules && json.rules.constructor.name == 'Array') {
me.rules._rulesList = json.rules;
}
// load the IP ranges:
if (json.ipRanges) {
json.ipRanges.forEach( it => me.ipRanges.add(it));
};
BotMon.live.gui.status.hideBusy("Status: Done.");
BotMon.live.data._dispatch('rules')
}
);
} catch (error) {
BotMon.live.gui.status.setError("Error while loading the config file: " + error.message);
} finally {
BotMon.live.gui.status.hideBusy("Status: Done.");
BotMon.live.data._dispatch('rules')
}
},
_rulesList: [], // list of rules to find out if a visitor is a bot
@@ -1717,6 +1839,31 @@ BotMon.live = {
}
}
// update the top pages;
const wmpages = document.getElementById('botmon__today__wm_pages');
if (wmpages) {
wmpages.appendChild(makeElement('dt', {}, "Top pages"));
const pgList = BotMon.live.data.analytics.getTopPages(maxItemsPerList);
if (pgList) {
pgList.forEach( (pgInfo) => {
const pgDd = makeElement('dd');
pgDd.appendChild(makeElement('a', {
'class': 'page_icon',
'href': DOKU_BASE + 'doku.php?id=' + encodeURIComponent(pgInfo.id),
'target': 'preview',
'title': "PageID: " + pgInfo.id
}, pgInfo.id));
pgDd.appendChild(makeElement('span', {
'class': 'count',
'title': pgInfo.count + " page views"
}, pgInfo.count));
wmpages.appendChild(pgDd);
});
}
}
// update the top referrers;
const wmreferers = document.getElementById('botmon__today__wm_referers');
if (wmreferers) {
@@ -2014,27 +2161,27 @@ BotMon.live = {
dl.appendChild(make('dd', {'class': 'has_icon platform pf_' + (data._platform ? data._platform.id : 'unknown')},
platformName + ( data._platform.v > 0 ? ' (' + data._platform.v + ')' : '' ) ));
dl.appendChild(make('dt', {}, "IP-Address:"));
const ipItem = make('dd', {'class': 'has_icon ipaddr ip' + ipType});
ipItem.appendChild(make('span', {'class': 'address'} , data.ip));
ipItem.appendChild(make('a', {
'class': 'icon_only extlink dnscheck',
'href': `https://dnschecker.org/ip-location.php?ip=${encodeURIComponent(data.ip)}`,
'target': 'dnscheck',
'title': "View this address on DNSChecker.org"
} , "Check Address"));
ipItem.appendChild(make('a', {
'class': 'icon_only extlink ipinfo',
'href': `https://ipinfo.io/${encodeURIComponent(data.ip)}`,
'target': 'ipinfo',
'title': "View this address on IPInfo.io"
} , "DNS Info"));
dl.appendChild(ipItem);
/*dl.appendChild(make('dt', {}, "ID:"));
dl.appendChild(make('dd', {'class': 'has_icon ip' + data.typ}, data.id));*/
}
dl.appendChild(make('dt', {}, "IP-Address:"));
const ipItem = make('dd', {'class': 'has_icon ipaddr ip' + ipType});
ipItem.appendChild(make('span', {'class': 'address'} , data.ip));
ipItem.appendChild(make('a', {
'class': 'icon_only extlink dnscheck',
'href': `https://dnschecker.org/ip-location.php?ip=${encodeURIComponent(data.ip)}`,
'target': 'dnscheck',
'title': "View this address on DNSChecker.org"
} , "Check Address"));
ipItem.appendChild(make('a', {
'class': 'icon_only extlink ipinfo',
'href': `https://ipinfo.io/${encodeURIComponent(data.ip)}`,
'target': 'ipinfo',
'title': "View this address on IPInfo.io"
} , "DNS Info"));
dl.appendChild(ipItem);
if (Math.abs(data._lastSeen - data._firstSeen) < 100) {
dl.appendChild(make('dt', {}, "Seen:"));
dl.appendChild(make('dd', {'class': 'seen'}, data._firstSeen.toLocaleString()));
@@ -2099,7 +2246,8 @@ BotMon.live = {
if (tObj.func == 'fromKnownBotIP') {
const rangeInfo = BotMon.live.data.ipRanges.match(data.ip);
if (rangeInfo) {
tDesc += ' (' + (rangeInfo.label ? rangeInfo.label : 'Unknown') + ')';
const owner = BotMon.live.data.ipRanges.getOwner(rangeInfo);
tDesc += ' (range: “' + rangeInfo.cidr + '”, ' + owner + ')';
}
}

View File

@@ -373,8 +373,23 @@
&.ref_other::before { background-image: url('img/more.svg') }
}
.page_icon {
display: inline-flex;
column-gap: .25em;
align-items: center;
}
.page_icon::before {
content: '';
display: inline-block;
width: 20px; height: 20px;
background: transparent url('img/page.svg') center no-repeat;
background-position: 0 0;
background-size: 20px;
}
/* grid layout for the overview: */
.botmon_bots_grid, .botmon_webmetrics_grid {
.botmon_bots_grid, .botmon_webmetrics_grid, .botmon_traffic_grid {
& {
display: grid;
grid-gap: 0 .33em;
@@ -391,7 +406,10 @@
grid-template-columns: 1fr 1fr 1fr;
}
.botmon_webmetrics_grid {
grid-template-columns: 1fr 1fr 1fr 1fr;
grid-template-columns: 1fr 1fr 1fr;
}
.botmon_traffic_grid {
grid-template-columns: 2fr 1fr;
}
/* the "today" tab: */