Today
+Latest data
@@ -68,6 +68,12 @@ class admin_plugin_botmon extends AdminPlugin {
+
diff --git a/client.js b/client.js
index 925ffe0..432ae1b 100644
--- a/client.js
+++ b/client.js
@@ -73,7 +73,7 @@
console.error(err);
} finally {
/* send the next heartbeat signal after x seconds: */
- // setTimeout(this._onHeartbeat.bind(this, this._src.replace( this._scriptName, '/tick.php')),this._heartbeat * 1000);
+ setTimeout(this._onHeartbeat.bind(this, this._src.replace( this._scriptName, '/tick.php')),this._heartbeat * 1000);
}
}
}).init();
\ No newline at end of file
diff --git a/config/default-config.json b/config/default-config.json
index 59ae26e..479f454 100644
--- a/config/default-config.json
+++ b/config/default-config.json
@@ -61,74 +61,5 @@
"id": "langMatch", "desc": "Client’s ‘Accept-Language’ header does not match the page language",
"bot": 30
}
- ],
- "ipRanges": [
- {"from": "3.0.0.0", "to": "3.255.255.254", "label": "Amazon Data Services [US]"},
- {"from": "5.161.0.0", "to": "5.161.255.255", "m": 16, "label": "Hetzner [US]"},
- {"from": "8.127.0.0", "to": "8.223.255.254", "label": "Alibaba [CN]"},
- {"from": "14.160.0.0", "to": "14.191.255.254", "m": 11, "label": "VNPT [VN]"},
- {"from": "14.192.0.0", "to": "14.255.255.254", "m": 10, "label": "VNPT [VN]"},
- {"from": "14.224.0.0", "to": "24.255.255.254", "m": 11, "label": "Charter [US]"},
- {"from": "27.106.0.0", "to": "27.106.127.254", "m": 17, "label": "Huawei [US]"},
- {"from": "34.0.0.0", "to": "34.191.255.254", "label": "Google LLC"},
- {"from": "43.132.0.0", "to": "43.132.255.254", "m": 16, "label": "Tencent [CN]"},
- {"from": "43.133.0.0", "to": "43.133.255.254", "m": 16, "label": "Tencent [CN]"},
- {"from": "45.0.0.0", "to": "45.255.255.254", "label": "Various small ISPs, mostly BR"},
- {"from": "46.250.160.0", "to": "46.250.191.254", "m": 19, "label": "Huawei [MX]"},
- {"from": "47.200.0.0", "to": "47.203.255.254", "m": 14, "label": "Frontier Communications [US]"},
- {"from": "49.0.192.0", "to": "49.0.255.254", "m": 18, "label": "Huawei [SG]"},
- {"from": "52.220.0.0", "to": "52.222.255.254", "label": "Amazon Data Services"},
- {"from": "66.249.64.0", "to": "66.249.95.254", "m": 19, "label": "Google LLC"},
- {"from": "84.37.35.0", "to": "84.37.255.254", "label": "GTT.net [US]"},
- {"from": "94.74.64.0", "to": "94.74.127.254", "m": 18, "label": "Huawei [HK]"},
- {"from": "101.0.0.0", "to": "101.255.255.254", "m": 8,"label": "ChinaNet [CN]"},
- {"from": "104.196.0.0", "to": "104.199.255.254", "m": 14, "label": "Google LLC"},
- {"from": "110.238.64.0", "to": "110.238.127.254", "m": 18, "label": "Huawei [SG]"},
- {"from": "111.119.192.0", "to": "111.119.255.254", "m": 18, "label": "Huawei [SG]"},
- {"from": "113.160.0.0", "to": "113.191.255.254", "m": 11, "label": "VNPT [VN]"},
- {"from": "119.8.0.0", "to": "119.8.255.254", "m": 16, "label": "Huawei [HK]"},
- {"from": "119.13.0.0", "to": "119.13.255.254", "m": 16, "label": "Huawei [HK]"},
- {"from": "121.91.168.0", "to": "121.91.175.254", "m": 21, "label": "Huawei [HK]"},
- {"from": "122.8.0.0", "to": "122.8.255.254", "label": "CN-ISP [CN]"},
- {"from": "122.9.0.0", "to": "122.9.255.254", "m": 16, "label": "Huawei [CN]"},
- {"from": "123.16.0.0", "to": "123.31.255.254", "m": 12, "label": "VNPT [VN]"},
- {"from": "124.243.128.0", "to": "124.243.191.254", "m": 18, "label": "Huawei [SG]"},
- {"from": "138.59.0.0", "to": "138.59.225.254", "m": 16, "label": "South-American ISPs (138.59.x)"},
- {"from": "138.121.0.0", "to": "138.121.225.254", "m": 16, "label": "South-American ISPs (138.121.x)"},
- {"from": "142.147.128.0", "to": "1142.147.255.254", "m": 17, "label": "Web2Objects LLC [US]"},
- {"from": "146.174.128.0", "to": "146.174.191.254", "m": 18, "label": "Huawei [SG]"},
- {"from": "150.40.128.0", "to": "150.40.255.254", "m": 17, "label": "Huawei [HK]"},
- {"from": "159.138.0.0", "to": "159.138.225.254", "m": 16, "label": "Huawei [TH]"},
- {"from": "162.128.0.0", "to": "162.128.255.254", "m": 16, "label": "Zenlayer [SG]"},
- {"from": "166.108.192.0", "to": "166.108.255.254", "m": 18, "label": "Huawei [SG]"},
- {"from": "168.232.192.0", "to": "168.232.255.254", "m": 16, "label": "South-American ISPs (168.232.x)"},
- {"from": "170.82.0.0", "to": "170.82.255.254", "m": 16, "label": "South-American ISPs (170.254.x)"},
- {"from": "170.254.0.0", "to": "170.254.255.254", "m": 16, "label": "South-American ISPs (170.82.x)"},
- {"from": "171.224.0.0", "to": "171.239.255.254", "m": 12, "label": "Viettel [VN]"},
- {"from": "177.0.0.0", "to": "177.255.255.254", "m": 8, "label": "BrasilNET [BR]"},
- {"from": "179.0.0.0", "to": "179.255.255.254", "m": 8, "label": "BrasilNET [BR]"},
- {"from": "183.87.32.0", "to": "183.87.63.254", "label": "Huawei [HK]"},
- {"from": "186.0.0.0", "to": "186.255.255.254", "m": 8, "label": "South-American ISPs (186.x)"},
- {"from": "187.0.0.0", "to": "187.255.255.254", "m": 8, "label": "South-American ISPs (187.x)"},
- {"from": "188.0.0.0", "to": "188.255.255.254", "m": 8, "label": "South-American ISPs (188.x)"},
- {"from": "189.0.0.0", "to": "189.255.255.254", "m": 8, "label": "South-American ISPs (189.x)"},
- {"from": "190.0.0.0", "to": "190.255.255.254", "m": 8, "label": "South-American ISPs (190.x)"},
- {"from": "191.0.0.0", "to": "191.255.255.254", "m": 8, "label": "South-American ISPs (191.x)"},
- {"from": "192.124.170.0", "to": "192.124.182.254", "label": "Relcom [CZ]"},
- {"from": "195.37.0.0", "to": "195.37.255.254", "m": 16, "label": "DFN [DE]"},
- {"from": "200.0.0.0", "to": "200.255.255.254", "m": 8, "label": "South-American ISPs (200.x)"},
- {"from": "201.0.0.0", "to": "201.255.255.254", "m": 8, "label": "South-American ISPs (201.x)"},
- {"from": "212.95.128.0", "to": "212.95.159.254", "m": 19, "label": "Asiacell [IQ]"},
- {"from": "222.252.0.0", "to": "222.252.255.254", "m": 14, "label": "VNPT [VN]"},
- {"from": "2001:4860::::::", "to": "2001:4860:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Google LLC [US]"},
- {"from": "2001:0ee0::::::", "to": "2001:ee3:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "label": "VNPT [VN]"},
- {"from": "2408:8210::::::", "to": "2408:8210:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "label": "China Unicom [CN]"},
- {"from": "2600:1f00::::::", "to": "2600:1fff:ffff:ffff:ffff:ffff:ffff:ffff", "m": 24, "label": "Amazon Cloud"},
- {"from": "2603:6010::::::", "to": "2603:6010:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Charter [US]"},
- {"from": "2603:8000::::::", "to": "2603:80ff:ffff:ffff:ffff:ffff:ffff:ffff", "m": 24, "label": "Charter [US]"},
- {"from": "2607:a400::::::", "to": "2607:a400:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Zenlayer Inc. [US]"},
- {"from": "2804:::::::", "to": "2804:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "m": 16, "label": "Inspire [BR]"},
- {"from": "2a09:bac2::::::", "to": "2a09:bac2:0:ffff:ffff:ffff:ffff:ffff", "m": 48, "label": "Cloudflare"},
- {"from": "2a0a:4cc0::::::", "to": "2a0a:4cc0:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "label": "Netcup [DE]"}
]
}
\ No newline at end of file
diff --git a/config/known-bots.json b/config/known-bots.json
index 0fb0cd5..4e68292 100644
--- a/config/known-bots.json
+++ b/config/known-bots.json
@@ -297,5 +297,17 @@
"r": [],
"rx": ["Go\\-http\\-client\\/(\\d+)", "quic\\-go\\-HTTP\\/(\\d+)"],
"url": "https://github.com/golang/goen.wi"
+ },
+ {"id": "bnl",
+ "n": "BnL Harvester",
+ "r": [],
+ "rx": ["NLUX_IAHarvester\\/(\\d+)"],
+ "url": "http://crawl.bnl.lu/"
+ },
+ {"id": "turnitin",
+ "n": "TurnitinBot",
+ "r": [],
+ "rx": ["Turnitin\\s"],
+ "url": "https://www.turnitin.com/robot/crawlerinfo.html"
}
]
\ No newline at end of file
diff --git a/config/known-ipranges.json b/config/known-ipranges.json
index ff73d73..415b6cf 100644
--- a/config/known-ipranges.json
+++ b/config/known-ipranges.json
@@ -1,55 +1,96 @@
-[
- {"from": "3.0.0.0", "to": "3.255.255.254", "m": 8, "label": "Amazon Data Services [US]"},
- {"from": "8.127.0.0", "to": "8.223.255.254", "label": "Alibaba [CN]"},
- {"from": "14.160.0.0", "to": "14.191.255.255", "m": 11, "label": "VNPT [VN]"},
- {"from": "24.240.0.0", "to": "24.247.255.255", "m": 13, "label": "Charter [US]"},
- {"from": "27.106.0.0", "to": "27.106.127.254", "label": "Huawei [US]"},
- {"from": "34.0.0.0", "to": "34.191.255.254", "label": "Google LLC"},
- {"from": "43.132.0.0", "to": "43.132.255.254", "m": 16, "label": "Tencent [CN]"},
- {"from": "43.133.0.0", "to": "43.133.255.254", "m": 16, "label": "Tencent [CN]"},
- {"from": "45.0.0.0", "to": "45.255.255.254", "label": "Various small ISPs, mostly BR"},
- {"from": "46.250.160.0", "to": "46.250.191.254", "label": "Huawei [MX]"},
- {"from": "47.200.0.0", "to": "47.203.255.255", "m": 14, "label": "Frontier Communications [US]"},
- {"from": "49.0.200.0", "to": "49.0.255.254", "label": "Huawei [SG]"},
- {"from": "52.220.0.0", "to": "52.222.255.254", "label": "Amazon Data Services"},
- {"from": "66.249.64.0", "to": "66.249.95.255", "m": 19, "label": "Google LLC"},
- {"from": "84.37.35.0", "to": "84.37.255.254", "label": "GTT.net [US]"},
- {"from": "94.74.64.0", "to": "94.74.127.254", "label": "Huawei [HK]"},
- {"from": "101.0.0.0", "to": "101.255.255.254", "label": "ChinaNet [CN]"},
- {"from": "104.196.0.0", "to": "104.199.255.255", "m": 14, "label": "Google LLC"},
- {"from": "110.238.80.0", "to": "110.238.127.254", "label": "Huawei [SG]"},
- {"from": "111.119.192.0", "to": "111.119.255.254", "label": "Huawei [SG]"},
- {"from": "113.160.0.0", "to": "113.191.255.254", "m": 11, "label": "VNPT [VN]"},
- {"from": "119.0.0.0", "to": "119.207.255.254", "label": "Unicom [CN]"},
- {"from": "121.91.168.", "to": "121.91.175.254", "label": "Huawei [HK]"},
- {"from": "122.8.0.0", "to": "122.8.255.254", "label": "CN-ISP [CN]"},
- {"from": "122.9.0.0", "to": "122.9.255.254", "label": "Huawei [CN]"},
- {"from": "123.16.0.0", "to": "123.31.255.255", "m": 12, "label": "VNPT [VN]"},
- {"from": "124.243.128.0", "to": "124.243.191.254", "label": "Huawei [SG]"},
- {"from": "142.147.128.0", "to": "1142.147.255.254", "label": "Web2Objects LLC [US]"},
- {"from": "150.40.128.0", "to": "150.40.255.254", "label": "Huawei [HK]"},
- {"from": "159.138.0.0", "to": "159.138.225.254", "label": "Huawei [TH]"},
- {"from": "162.128.0.0", "to": "162.128.255.254", "label": "Zenlayer [SG]"},
- {"from": "166.108.192.0", "to": "166.108.255.254", "label": "Huawei [SG]"},
- {"from": "177.0.0.0", "to": "177.255.255.254", "label": "BrasilNET [BR]"},
- {"from": "179.0.0.0", "to": "179.255.255.254", "label": "BrasilNET [BR]"},
- {"from": "183.87.32.0", "to": "183.87.159.254", "label": "Huawei [HK]"},
- {"from": "186.0.0.0", "to": "186.255.255.254", "m": 8, "label": "South-American ISPs (186.x)"},
- {"from": "187.0.0.0", "to": "187.255.255.254", "m": 8, "label": "South-American ISPs (187.x)"},
- {"from": "188.0.0.0", "to": "188.255.255.254", "m": 8, "label": "South-American ISPs (188.x)"},
- {"from": "189.0.0.0", "to": "189.255.255.254", "m": 8, "label": "South-American ISPs (189.x)"},
- {"from": "190.0.0.0", "to": "190.255.255.254", "m": 8, "label": "South-American ISPs (190.x)"},
- {"from": "191.0.0.0", "to": "191.255.255.254", "m": 8, "label": "South-American ISPs (191.x)"},
- {"from": "192.124.170.0", "to": "192.124.182.254", "label": "Relcom [CZ]"},
- {"from": "195.37.0.0", "to": "195.37.255.255", "m": 16, "label": "DFN [DE]"},
- {"from": "2001:4860::::::", "to": "2001:4860:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Google LLC [US]"},
- {"from": "2001:0ee0::::::", "to": "2001:ee3:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "label": "VNPT [VN]"},
- {"from": "2408:8210::::::", "to": "2408:8210:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "label": "China Unicom [CN]"},
- {"from": "2600:1f00::::::", "to": "2600:1fff:ffff:ffff:ffff:ffff:ffff:ffff", "m": "Amazon Cloud"},
- {"from": "2603:6010::::::", "to": "2603:6010:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Charter [US]"},
- {"from": "2603:8000::::::", "to": "2603:80ff:ffff:ffff:ffff:ffff:ffff:ffff", "m": 24, "label": "Charter [US]"},
- {"from": "2607:a400::::::", "to": "2607:a400:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Zenlayer Inc. [US]"},
- {"from": "2804:::::::", "to": "2804:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "m": 16, "label": "Inspire [BR]"},
- {"from": "2a09:bac2::::::", "to": "2a09:bac2:0:ffff:ffff:ffff:ffff:ffff", "m": 48, "label": "Cloudflare"},
- {"from": "2a0a:4cc0::::::", "to": "2a0a:4cc0:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "label": "Netcup [DE]"}
-]
\ No newline at end of file
+{
+ "groups": [
+ {"id": "alibaba", "name": "Alibaba"},
+ {"id": "amazon", "name": "Amazon Data Services"},
+ {"id": "amazon-microsoft", "name": "Amazon or Microsoft"},
+ {"id": "brasilnet", "name": "BrasilNet"},
+ {"id": "charter", "name": "Charter Communications Inc."},
+ {"id": "cnisp", "name": "China ISP"},
+ {"id": "cnmob", "name": "China Mobile Communications Corp."},
+ {"id": "google", "name": "Google LLC"},
+ {"id": "google-amazon", "name": "Google or Amazon"},
+ {"id": "hetzner", "name": "Hetzner Hosting (US)"},
+ {"id": "huawei", "name": "Huawei"},
+ {"id": "misc_sa", "name": "Various South- and Central-American ISPs"},
+ {"id": "tencent", "name": "Tencent"},
+ {"id": "unicom", "name": "China Unicom"},
+ {"id": "vnpt", "name": "Vietnam Telecom"},
+ {"id": "vdsina", "name": "VDSina NL"},
+ {"id": "zenlayer", "name": "Zenlayer"}
+ ],
+ "ranges": [
+ {"from": "3.0.0.0", "to": "3.255.255.254", "m": 8, "g": "amazon"},
+ {"from": "5.161.0.0", "to": "5.161.255.255", "m": 16, "g": "hetzner"},
+ {"from": "8.128.0.0", "to": "8.191.255.254", "m": 10, "g": "alibaba"},
+ {"from": "13.216.0.0", "to": "13.223.255.254", "m": 11, "g": "amazon"},
+ {"from": "14.160.0.0", "to": "14.191.255.254", "m": 11, "g": "vnpt"},
+ {"from": "14.192.0.0", "to": "14.255.255.254", "m": 10, "g": "vnpt"},
+ {"from": "14.224.0.0", "to": "24.255.255.254", "m": 11, "g": "vnpt"},
+ {"from": "27.106.0.0", "to": "27.106.127.254", "m": 17, "g": "huawei"},
+ {"from": "34.0.0.0", "to": "34.255.255.254", "m": 8, "g": "google-amazon"},
+ {"from": "39.64.0.0", "to": "39.95.255.254", "g": "cnmob"},
+ {"from": "43.132.0.0", "to": "43.132.255.254", "m": 16, "g": "tencent"},
+ {"from": "43.133.0.0", "to": "43.133.255.254", "m": 16, "g": "tencent"},
+ {"from": "44.192.0.0", "to": "44.255.255.254", "m": 16, "g": "tencent"},
+ {"from": "45.0.0.0", "to": "45.255.255.254", "m": 10, "g": "amazon"},
+ {"from": "46.250.160.0", "to": "46.250.191.254", "m": 19, "g": "huawei"},
+ {"from": "47.82.0.0", "to": "47.82.255.254", "m": 16, "g": "alibaba"},
+ {"from": "47.200.0.0", "to": "47.203.255.254", "m": 14, "g": "frontier"},
+ {"from": "49.0.192.0", "to": "49.0.255.254", "m": 18, "g": "huawei"},
+ {"from": "52.0.0.0", "to": "52.255.255.254", "m": 8, "g": "amazon-microsoft"},
+ {"from": "54.0.0.0", "to": "54.255.255.254", "m": 8, "g": "amazon"},
+ {"from": "66.249.64.0", "to": "66.249.95.254", "m": 19, "g": "google"},
+ {"from": "84.37.35.0", "to": "84.37.255.254", "g": "gtt"},
+ {"from": "94.74.64.0", "to": "94.74.127.254", "m": 18, "g": "huawei"},
+ {"from": "91.84.96.0", "to": "91.84.127.254", "m": 19, "g": "vdsina"},
+ {"from": "101.0.0.0", "to": "101.255.255.254", "m": 8,"g": "chinanet"},
+ {"from": "104.196.0.0", "to": "104.199.255.254", "m": 14, "g": "google"},
+ {"from": "110.238.64.0", "to": "110.238.127.254", "m": 18, "g": "huawei"},
+ {"from": "111.119.192.0", "to": "111.119.255.254", "m": 18, "g": "huawei"},
+ {"from": "113.160.0.0", "to": "113.191.255.254", "m": 11, "g": "vnpt"},
+ {"from": "114.208.0.0", "to": "114.223.255.254", "m": 12, "g": "unicom"},
+ {"from": "114.224.0.0", "to": "114.255.255.254", "m": 11, "g": "unicom"},
+ {"from": "119.8.0.0", "to": "119.8.255.254", "m": 16, "g": "huawei"},
+ {"from": "119.13.0.0", "to": "119.13.255.254", "m": 16, "g": "huawei"},
+ {"from": "121.91.168.0", "to": "121.91.175.254", "m": 21, "g": "huawei"},
+ {"from": "122.8.0.0", "to": "122.8.255.254", "g": "cnisp"},
+ {"from": "122.9.0.0", "to": "122.9.255.254", "m": 16, "g": "huawei"},
+ {"from": "123.16.0.0", "to": "123.31.255.254", "m": 12, "g": "vnpt"},
+ {"from": "124.243.128.0", "to": "124.243.191.254", "m": 18, "g": "huawei"},
+ {"from": "138.59.0.0", "to": "138.59.225.254", "m": 16, "g": "misc_sa"},
+ {"from": "138.121.0.0", "to": "138.121.225.254", "m": 16, "g": "misc_sa"},
+ {"from": "142.147.128.0", "to": "1142.147.255.254", "m": 17, "g": "w2obj"},
+ {"from": "146.174.128.0", "to": "146.174.191.254", "m": 18, "g": "huawei"},
+ {"from": "150.40.128.0", "to": "150.40.255.254", "m": 17, "g": "huawei"},
+ {"from": "159.138.0.0", "to": "159.138.225.254", "m": 16, "g": "huawei"},
+ {"from": "162.128.0.0", "to": "162.128.255.254", "m": 16, "g": "zenlayer"},
+ {"from": "166.108.192.0", "to": "166.108.255.254", "m": 18, "g": "huawei"},
+ {"from": "168.232.192.0", "to": "168.232.255.254", "m": 16, "g": "misc_sa"},
+ {"from": "170.82.0.0", "to": "170.82.255.254", "m": 16, "g": "misc_sa"},
+ {"from": "170.254.0.0", "to": "170.254.255.254", "m": 16, "g": "misc_sa"},
+ {"from": "171.224.0.0", "to": "171.239.255.254", "m": 12, "g": "viettel"},
+ {"from": "177.0.0.0", "to": "177.255.255.254", "m": 8, "g": "brasilnet"},
+ {"from": "179.0.0.0", "to": "179.255.255.254", "m": 8, "g": "brasilnet"},
+ {"from": "183.87.32.0", "to": "183.87.63.254", "m": 19, "g": "huawei"},
+ {"from": "186.0.0.0", "to": "186.255.255.254", "m": 8, "g": "misc_sa"},
+ {"from": "187.0.0.0", "to": "187.255.255.254", "m": 8, "g": "misc_sa"},
+ {"from": "188.0.0.0", "to": "188.255.255.254", "m": 8, "g": "misc_sa"},
+ {"from": "189.0.0.0", "to": "189.255.255.254", "m": 8, "g": "misc_sa"},
+ {"from": "190.0.0.0", "to": "190.255.255.254", "m": 8, "g": "misc_sa"},
+ {"from": "191.0.0.0", "to": "191.255.255.254", "m": 8, "g": "misc_sa"},
+ {"from": "192.124.170.0", "to": "192.124.182.254", "g": "relcom"},
+ {"from": "200.0.0.0", "to": "200.255.255.254", "m": 8, "g": "misc_sa"},
+ {"from": "201.0.0.0", "to": "201.255.255.254", "m": 8, "g": "misc_sa"},
+ {"from": "212.95.128.0", "to": "212.95.159.254", "m": 19, "g": "asiacell"},
+ {"from": "222.252.0.0", "to": "222.252.255.254", "m": 14, "g": "vnpt"},
+ {"from": "2001:4860::::::", "to": "2001:4860:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "g": "google"},
+ {"from": "2001:0ee0::::::", "to": "2001:ee3:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "g": "vnpt"},
+ {"from": "2408:8210::::::", "to": "2408:8210:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "g": "unicom"},
+ {"from": "2600:1f00::::::", "to": "2600:1fff:ffff:ffff:ffff:ffff:ffff:ffff", "m": 24, "g": "amazon"},
+ {"from": "2603:6010::::::", "to": "2603:6010:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "g": "charter"},
+ {"from": "2603:8000::::::", "to": "2603:80ff:ffff:ffff:ffff:ffff:ffff:ffff", "m": 24, "g": "charter"},
+ {"from": "2607:a400::::::", "to": "2607:a400:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "g": "zenlayer"},
+ {"from": "2804:::::::", "to": "2804:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "m": 16, "g": "misc_sa"},
+ {"from": "2a0a:4cc0::::::", "to": "2a0a:4cc0:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "g": "netcup"}
+ ]
+}
\ No newline at end of file
diff --git a/config/known-platforms.json b/config/known-platforms.json
index 2e3da62..46e6c51 100644
--- a/config/known-platforms.json
+++ b/config/known-platforms.json
@@ -21,7 +21,7 @@
},
{"n": "Old Android",
"id": "androidold",
- "rx": [ "Android[\\s;\\/](\\d)\\." ]
+ "rx": [ "Android[\\s;\\/](\\d)[\\.;\\)]" ]
},
{"n": "Android",
"id": "android",
diff --git a/script.js b/script.js
index 02e481e..659c872 100644
--- a/script.js
+++ b/script.js
@@ -19,6 +19,12 @@ const BM_LOGTYPE = Object.freeze({
'TICKER': 'tck'
});
+// enumeration of IP versions:
+const BM_IPVERSION = Object.freeze({
+ 'IPv4': 4,
+ 'IPv6': 6
+});
+
/* BotMon root object */
const BotMon = {
@@ -35,13 +41,18 @@ const BotMon = {
// get the time offset:
this._timeDiff = BotMon.t._getTimeOffset();
+ // get yesterday's date:
+ let d = new Date();
+ d.setDate(d.getDate() - 1);
+ this._datestr = d.toISOString().slice(0, 10);
+
// init the sub-objects:
BotMon.t._callInit(this);
},
_baseDir: null,
_lang: 'en',
- _datestr: (new Date()).toISOString().slice(0, 10),
+ _datestr: '',
_timeDiff: '',
/* internal tools */
@@ -105,9 +116,9 @@ const BotMon = {
if (!ip) {
return 'null';
} else if (ip.indexOf(':') > 0) { /* IP6 */
- return (ip.split(':').map(d => ('0000'+d).slice(-4) ).join(''));
+ return (ip.split(':').map(d => ('0000'+d).slice(-4) ).join(':'));
} else { /* IP4 */
- return Number(ip.split('.').map(d => ('000'+d).slice(-3) ).join(''));
+ return ip.split('.').map(d => ('000'+d).slice(-3) ).join('.');
}
},
@@ -147,7 +158,7 @@ const BotMon = {
}
};
-/* everything specific to the "Today" tab is self-contained in the "live" object: */
+/* everything specific to the "Latest" tab is self-contained in the "live" object: */
BotMon.live = {
init: function() {
//console.info('BotMon.live.init()');
@@ -180,6 +191,9 @@ BotMon.live = {
case 'rules':
data._dispatchRulesLoaded = true;
break;
+ case 'ipranges':
+ data._dispatchIPRangesLoaded = true;
+ break;
case 'bots':
data._dispatchBotsLoaded = true;
break;
@@ -194,7 +208,7 @@ BotMon.live = {
}
// are all the flags set?
- if (data._dispatchBotsLoaded && data._dispatchClientsLoaded && data._dispatchPlatformsLoaded && data._dispatchRulesLoaded) {
+ if (data._dispatchBotsLoaded && data._dispatchClientsLoaded && data._dispatchPlatformsLoaded && data._dispatchRulesLoaded && data._dispatchIPRangesLoaded) {
// chain the log files loading:
BotMon.live.data.loadLogFile(BM_LOGTYPE.SERVER, BotMon.live.data._onServerLogLoaded);
}
@@ -203,6 +217,7 @@ BotMon.live = {
_dispatchBotsLoaded: false,
_dispatchClientsLoaded: false,
_dispatchPlatformsLoaded: false,
+ _dispatchIPRangesLoaded: false,
_dispatchRulesLoaded: false,
// event callback, after the server log has been loaded:
@@ -239,6 +254,7 @@ BotMon.live = {
},
+ // the data model:
model: {
// visitors storage:
_visitors: [],
@@ -486,8 +502,12 @@ BotMon.live = {
}
},
+ // functions to analyse the data:
analytics: {
+ /**
+ * Initializes the analytics data storage object:
+ */
init: function() {
//console.info('BotMon.live.data.analytics.init()');
},
@@ -580,12 +600,15 @@ BotMon.live = {
// add
v._pageViews.forEach( pv => {
me.addToRefererList(pv._ref);
+ me.addToPagesList(pv.pg);
});
}
});
BotMon.live.gui.status.hideBusy('Done.');
+
+ console.log(me._pagesList);
},
// get a list of known bots:
@@ -635,6 +658,49 @@ BotMon.live = {
return rList;
},
+ // most visited pages list:
+ _pagesList: [],
+
+ /**
+ * Add a page view to the list of most visited pages.
+ * @param {string} pageId - The page ID to add to the list.
+ * @example
+ * BotMon.live.data.analytics.addToPagesList('1234567890');
+ */
+ addToPagesList: function(pageId) {
+ //console.log('BotMon.live.data.analytics.addToPagesList', pageId);
+
+ const me = BotMon.live.data.analytics;
+
+ // already exists?
+ let pgObj = null;
+ for (let i = 0; i < me._pagesList.length; i++) {
+ if (me._pagesList[i].id == pageId) {
+ pgObj = me._pagesList[i];
+ break;
+ }
+ }
+
+ // if not exists, create it:
+ if (!pgObj) {
+ pgObj = {
+ id: pageId,
+ count: 1
+ };
+ me._pagesList.push(pgObj);
+ } else {
+ pgObj.count += 1;
+ }
+ },
+
+ getTopPages: function(max) {
+ //console.info('BotMon.live.data.analytics.getTopPages('+max+')');
+ const me = BotMon.live.data.analytics;
+ return me._pagesList.toSorted( (a, b) => {
+ return b.count - a.count;
+ }).slice(0,max);
+ },
+
// Referer List:
_refererList: [],
@@ -696,6 +762,14 @@ BotMon.live = {
};
},
+ /**
+ * Get a sorted list of the top referers.
+ * The list is sorted in descending order of count.
+ * If the array has more items than the given maximum, the rest of the items are added to an "other" item.
+ * Each item in the list has a "pct" property, which is the percentage of the total count.
+ * @param {number} max - The maximum number of items to return.
+ * @return {Array} The sorted list of top referers.
+ */
getTopReferers: function(max) {
//console.info(('BotMon.live.data.analytics.getTopReferers(' + max + ')'));
@@ -704,6 +778,15 @@ BotMon.live = {
return me._makeTopList(me._refererList, max);
},
+ /**
+ * Create a sorted list of top items from a given array.
+ * The list is sorted in descending order of count.
+ * If the array has more items than the given maximum, the rest of the items are added to an "other" item.
+ * Each item in the list has a "pct" property, which is the percentage of the total count.
+ * @param {Array} arr - The array to sort and truncate.
+ * @param {number} max - The maximum number of items to return.
+ * @return {Array} The sorted list of top items.
+ */
_makeTopList: function(arr, max) {
//console.info(('BotMon.live.data.analytics._makeTopList(arr,' + max + ')'));
@@ -896,6 +979,7 @@ BotMon.live = {
}
},
+ // information on "known bots":
bots: {
// loads the list of known bots from a JSON file:
init: async function() {
@@ -972,6 +1056,7 @@ BotMon.live = {
_list: []
},
+ // information on known clients (browsers):
clients: {
// loads the list of known clients from a JSON file:
init: async function() {
@@ -1038,6 +1123,7 @@ BotMon.live = {
},
+ // information on known platforms (operating systems):
platforms: {
// loads the list of known platforms from a JSON file:
init: async function() {
@@ -1104,15 +1190,43 @@ BotMon.live = {
},
+ // storage and functions for the known bot IP-Ranges:
ipRanges: {
init: function() {
//console.log('BotMon.live.data.ipRanges.init()');
// #TODO: Load from separate IP-Ranges file
+ // load the rules file:
+ const me = BotMon.live.data;
+
+ try {
+ BotMon.live.data._loadSettingsFile(['user-ipranges', 'known-ipranges'],
+ (json) => {
+
+ // groups can be just saved in the data structure:
+ if (json.groups && json.groups.constructor.name == 'Array') {
+ me.ipRanges._groups = json.groups;
+ }
+
+ // groups can be just saved in the data structure:
+ if (json.ranges && json.ranges.constructor.name == 'Array') {
+ json.ranges.forEach(range => {
+ me.ipRanges.add(range);
+ })
+ }
+
+ // finished loading
+ BotMon.live.gui.status.hideBusy("Status: Done.");
+ BotMon.live.data._dispatch('ipranges')
+ });
+ } catch (error) {
+ BotMon.live.gui.status.setError("Error while loading the config file: " + error.message);
+ }
},
// the actual bot list is stored here:
_list: [],
+ _groups: [],
add: function(data) {
//console.log('BotMon.live.data.ipRanges.add(',data,')');
@@ -1123,14 +1237,29 @@ BotMon.live = {
const ip2Num = BotMon.t._ip2Num;
let item = {
+ 'cidr': data.from.replaceAll(/::+/g, '::') + '/' + ( data.m ? data.m : '??' ),
'from': ip2Num(data.from),
'to': ip2Num(data.to),
- 'label': data.label
+ 'm': data.m,
+ 'g': data.g
};
me._list.push(item);
},
+ getOwner: function(rangeInfo) {
+
+ const me = BotMon.live.data.ipRanges;
+
+ for (let i=0; i < me._groups.length; i++) {
+ const it = me._groups[i];
+ if (it.id == rangeInfo.g) {
+ return it.name;
+ }
+ }
+ return `Unknown (“${rangeInfo.g})`;
+ },
+
match: function(ip) {
//console.log('BotMon.live.data.ipRanges.match(',ip,')');
@@ -1151,6 +1280,7 @@ BotMon.live = {
}
},
+ // storage for the rules and related functions
rules: {
/**
@@ -1183,25 +1313,17 @@ BotMon.live = {
if (json.threshold) me._threshold = json.threshold;
// set the rules list:
- if (json.rules) {
+ if (json.rules && json.rules.constructor.name == 'Array') {
me.rules._rulesList = json.rules;
}
- // load the IP ranges:
- if (json.ipRanges) {
- json.ipRanges.forEach( it => me.ipRanges.add(it));
- };
-
+ BotMon.live.gui.status.hideBusy("Status: Done.");
+ BotMon.live.data._dispatch('rules')
}
);
-
} catch (error) {
BotMon.live.gui.status.setError("Error while loading the config file: " + error.message);
- } finally {
- BotMon.live.gui.status.hideBusy("Status: Done.");
- BotMon.live.data._dispatch('rules')
}
-
},
_rulesList: [], // list of rules to find out if a visitor is a bot
@@ -1717,6 +1839,31 @@ BotMon.live = {
}
}
+ // update the top pages;
+ const wmpages = document.getElementById('botmon__today__wm_pages');
+ if (wmpages) {
+
+ wmpages.appendChild(makeElement('dt', {}, "Top pages"));
+
+ const pgList = BotMon.live.data.analytics.getTopPages(maxItemsPerList);
+ if (pgList) {
+ pgList.forEach( (pgInfo) => {
+ const pgDd = makeElement('dd');
+ pgDd.appendChild(makeElement('a', {
+ 'class': 'page_icon',
+ 'href': DOKU_BASE + 'doku.php?id=' + encodeURIComponent(pgInfo.id),
+ 'target': 'preview',
+ 'title': "PageID: " + pgInfo.id
+ }, pgInfo.id));
+ pgDd.appendChild(makeElement('span', {
+ 'class': 'count',
+ 'title': pgInfo.count + " page views"
+ }, pgInfo.count));
+ wmpages.appendChild(pgDd);
+ });
+ }
+ }
+
// update the top referrers;
const wmreferers = document.getElementById('botmon__today__wm_referers');
if (wmreferers) {
@@ -2014,27 +2161,27 @@ BotMon.live = {
dl.appendChild(make('dd', {'class': 'has_icon platform pf_' + (data._platform ? data._platform.id : 'unknown')},
platformName + ( data._platform.v > 0 ? ' (' + data._platform.v + ')' : '' ) ));
- dl.appendChild(make('dt', {}, "IP-Address:"));
- const ipItem = make('dd', {'class': 'has_icon ipaddr ip' + ipType});
- ipItem.appendChild(make('span', {'class': 'address'} , data.ip));
- ipItem.appendChild(make('a', {
- 'class': 'icon_only extlink dnscheck',
- 'href': `https://dnschecker.org/ip-location.php?ip=${encodeURIComponent(data.ip)}`,
- 'target': 'dnscheck',
- 'title': "View this address on DNSChecker.org"
- } , "Check Address"));
- ipItem.appendChild(make('a', {
- 'class': 'icon_only extlink ipinfo',
- 'href': `https://ipinfo.io/${encodeURIComponent(data.ip)}`,
- 'target': 'ipinfo',
- 'title': "View this address on IPInfo.io"
- } , "DNS Info"));
- dl.appendChild(ipItem);
-
/*dl.appendChild(make('dt', {}, "ID:"));
dl.appendChild(make('dd', {'class': 'has_icon ip' + data.typ}, data.id));*/
}
+ dl.appendChild(make('dt', {}, "IP-Address:"));
+ const ipItem = make('dd', {'class': 'has_icon ipaddr ip' + ipType});
+ ipItem.appendChild(make('span', {'class': 'address'} , data.ip));
+ ipItem.appendChild(make('a', {
+ 'class': 'icon_only extlink dnscheck',
+ 'href': `https://dnschecker.org/ip-location.php?ip=${encodeURIComponent(data.ip)}`,
+ 'target': 'dnscheck',
+ 'title': "View this address on DNSChecker.org"
+ } , "Check Address"));
+ ipItem.appendChild(make('a', {
+ 'class': 'icon_only extlink ipinfo',
+ 'href': `https://ipinfo.io/${encodeURIComponent(data.ip)}`,
+ 'target': 'ipinfo',
+ 'title': "View this address on IPInfo.io"
+ } , "DNS Info"));
+ dl.appendChild(ipItem);
+
if (Math.abs(data._lastSeen - data._firstSeen) < 100) {
dl.appendChild(make('dt', {}, "Seen:"));
dl.appendChild(make('dd', {'class': 'seen'}, data._firstSeen.toLocaleString()));
@@ -2099,7 +2246,8 @@ BotMon.live = {
if (tObj.func == 'fromKnownBotIP') {
const rangeInfo = BotMon.live.data.ipRanges.match(data.ip);
if (rangeInfo) {
- tDesc += ' (' + (rangeInfo.label ? rangeInfo.label : 'Unknown') + ')';
+ const owner = BotMon.live.data.ipRanges.getOwner(rangeInfo);
+ tDesc += ' (range: “' + rangeInfo.cidr + '”, ' + owner + ')';
}
}
diff --git a/style.less b/style.less
index 6ef52ab..a148c28 100644
--- a/style.less
+++ b/style.less
@@ -373,8 +373,23 @@
&.ref_other::before { background-image: url('img/more.svg') }
}
+ .page_icon {
+ display: inline-flex;
+ column-gap: .25em;
+ align-items: center;
+ }
+ .page_icon::before {
+ content: '';
+ display: inline-block;
+ width: 20px; height: 20px;
+ background: transparent url('img/page.svg') center no-repeat;
+ background-position: 0 0;
+ background-size: 20px;
+ }
+
+
/* grid layout for the overview: */
- .botmon_bots_grid, .botmon_webmetrics_grid {
+ .botmon_bots_grid, .botmon_webmetrics_grid, .botmon_traffic_grid {
& {
display: grid;
grid-gap: 0 .33em;
@@ -391,7 +406,10 @@
grid-template-columns: 1fr 1fr 1fr;
}
.botmon_webmetrics_grid {
- grid-template-columns: 1fr 1fr 1fr 1fr;
+ grid-template-columns: 1fr 1fr 1fr;
+ }
+ .botmon_traffic_grid {
+ grid-template-columns: 2fr 1fr;
}
/* the "today" tab: */
Web traffic (humans only)
+
+