Multiple updates

This commit is contained in:
Sascha Leib
2025-09-14 19:32:48 +02:00
parent 7884ded35b
commit f40be0ac64
11 changed files with 170 additions and 74 deletions

1
.gitignore vendored
View File

@@ -2,4 +2,5 @@
logs/*.log.txt
logs/*.srv.txt
logs/*.tck.txt
config/user-config.json
php_errors.log

View File

@@ -23,11 +23,11 @@
},
{"func": "smallPageCount", "params": [1],
"id": "onePage", "desc": "Visiter viewed only a single page",
"bot": 40
"bot": 30
},
{"func": "noRecord", "params": ["log"],
"id": "noClient", "desc": "No client-side JS log was recorded",
"bot": 50
"bot": 40
},
{"func": "noRecord", "params": ["tck"],
"id": "noTicks", "desc": "No client ticks were recorded",
@@ -45,17 +45,13 @@
"id": "susClient", "desc": "Client identifier that is popular with bot networks",
"bot": 10
},
{"func": "combinationTest", "params": [["macos", "chrome"]],
"id": "unusualPC", "desc": "Unusual combination of platform and client",
"bot": 10
},
{"func": "combinationTest", "params": [["macos", "chromeold"],["macosold", "brave"],["winold", "edge"],["winold", "brave"]],
{"func": "combinationTest", "params": [["macos","chromeold"],["macos","msie"],["winold","edge"],["winold","brave"]],
"id": "suspPC", "desc": "Suspicious combination of platform and client",
"bot": 30
},
{"func": "combinationTest", "params": [["macos", "msie"], ["win10", "safari"]],
{"func": "combinationTest", "params": [["macos","msie"],["win10","safari"],["macosold","brave"]],
"id": "impPC", "desc": "Impossible combination of platform and client",
"bot": 80
"bot": 70
},
{"func": "loadSpeed", "params": [3, 20],
"id": "speedRun", "desc": "Average time between page loads is less than 20 seconds",
@@ -65,31 +61,32 @@
"id": "noAcc", "desc": "No “Accept-Language” header",
"bot": 40
},
{"func": "clientAccepts", "params": ["zh"],
"id": "zhLang", "desc": "Client accepts Chinese language",
"bot": 60
},
{"func": "matchesCountry", "params": ["BR", "CN", "RU", "US", "MX", "SG", "IN", "UY"],
"id": "isFrom", "desc": "Location is in a known bot-spamming country.",
"bot": 50
},
{"func": "matchesCountry", "params": ["ZZ"],
"id": "zzCtry", "desc": "Location could not be determined",
"bot": 20
}
],
"ipRanges": [
{"from": "3.0.0.0", "to": "3.255.255.254", "label": "Amazon Data Services [US]", "g": "US"},
{"from": "3.0.0.0", "to": "3.255.255.254", "label": "Amazon Data Services [US]"},
{"from": "8.127.0.0", "to": "8.223.255.254", "label": "Alibaba [CN]"},
{"from": "24.240.0.0", "to": "24.247.255.255", "m": 13, "label": "Charter [US]"},
{"from": "27.106.0.0", "to": "27.106.127.254", "label": "Huawei [US]"},
{"from": "34.0.0.0", "to": "34.191.255.254", "label": "Google LLC"},
{"from": "45.0.0.0", "to": "45.255.255.254", "label": "Various small ISPs, mostly BR"},
{"from": "46.250.160.0", "to": "46.250.191.254", "label": "Huawei [MX]"},
{"from": "47.200.0.0", "to": "47.203.255.255", "m": 14, "label": "Frontier Communications [US]"},
{"from": "49.0.200.0", "to": "49.0.255.254", "label": "Huawei [SG]"},
{"from": "66.249.64.0", "to": "66.249.95.255", "m": 19, "label": "Google LLC [US]"},
{"from": "84.37.35.0", "to": "84.37.255.254", "label": "GTT.net [US]"},
{"from": "94.74.64.0", "to": "94.74.127.254", "label": "Huawei [HK]"},
{"from": "101.0.0.0", "to": "101.255.255.254", "label": "ChinaNet [CN]"},
{"from": "110.238.96.0", "to": "110.238.127.254", "label": "Huawei [SG]"},
{"from": "110.238.80.0", "to": "110.238.127.254", "label": "Huawei [SG]"},
{"from": "111.119.192.0", "to": "111.119.255.254", "label": "Huawei [SG]"},
{"from": "119.0.0.0", "to": "101.207.255.254", "label": "Unicom [CN]"},
{"from": "119.0.0.0", "to": "119.207.255.254", "label": "Unicom [CN]"},
{"from": "121.91.168.", "to": "121.91.175.254", "label": "Huawei [HK]"},
{"from": "122.8.0.0", "to": "122.8.255.254", "label": "CN-ISP [CN]"},
{"from": "122.9.0.0", "to": "122.9.255.254", "label": "Huawei [CN]"},
@@ -109,8 +106,9 @@
{"from": "190.0.0.0", "to": "190.255.255.254", "label": "South-American ISPs (190.x)"},
{"from": "192.124.170.0", "to": "192.124.182.254", "label": "Relcom [CZ]"},
{"from": "195.37.0.0", "to": "195.37.255.255", "label": "DFN [DE]"},
{"from": "2001:4800::::::", "to": "2001:4fff:ffff:ffff:ffff:ffff:ffff:ffff", "label": "Rackspace/Google [US]"},
{"from": "2001:4860::::::", "to": "2001:4860:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Google LLC [US]"},
{"from": "2001:0ee0::::::", "to": "2001:ee3:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "label": "VNPT [VN]"},
{"from": "2408:8210::::::", "to": "2408:8210:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "label": "China Unicom [CN]"},
{"from": "2600:1f00::::::", "to": "2600:1fff:ffff:ffff:ffff:ffff:ffff:ffff", "m": "Amazon Cloud [US]"},
{"from": "2603:6010::::::", "to": "2603:6010:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Charter [US]"},
{"from": "2603:8000::::::", "to": "2603:80ff:ffff:ffff:ffff:ffff:ffff:ffff", "m": 24, "label": "Charter [US]"},

View File

@@ -22,6 +22,12 @@
"rx": ["APIs-Google"],
"url": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers"
},
{"id": "googleother",
"n": "GoogleOther", "geo": "US",
"r": ["GoogleOther"],
"rx": ["\\sGoogleOther(\\-\\w+)?[\\)\\/]"],
"url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers#googleother"
},
{"id": "applebot",
"n": "Applebot", "geo": "US",
"r": ["Applebot"],

View File

@@ -23,6 +23,10 @@
"id": "huawei",
"rx": [ "\\sHuaweiBrowser\\/(\\d+\\.\\d+)[\\s\\.]", "\\/harmony360Browser\\/(\\d+\\.\\d+)[\\s\\.]"]
},
{"n": "Ecosia Browser",
"id": "ecosia",
"rx": [ "\\(Ecosia ios@(\\d+)\\." ]
},
{"n": "Silk",
"id": "silk",
"rx": [ "\\Silk\\/(\\d+)\\." ]

View File

@@ -25,11 +25,11 @@
},
{"n": "Android",
"id": "android",
"rx": [ " Android[\\s;\\/](\\d\\d)\\.;\\s" ]
"rx": [ "Android[\\s;\\/](\\d\\d)[\\.;\\s]" ]
},
{"n": "Old MacOS",
"id": "macosold",
"rx": [ "\\sMac OS X 10[\\._](\\d|1[0-3])[\\._;\\s\\)]", "\\sMac OS X (1[123])[\\._]" ]
"rx": [ "\\sMac OS X 10[\\._](\\d|1[0-3])[\\._;\\s\\)]", "\\sMac OS X (1[123])[\\.;_\\s]" ]
},
{"n": "MacOS",
"id": "macos",

View File

@@ -9,7 +9,7 @@
"id": "oldClient", "desc": "Obsolete browser version",
"bot": 40
},
{"func": "matchesPlatform", "params": ["winold", "macosold"],
{"func": "matchesPlatform", "params": ["winold", "macosold", "androidold"],
"id": "oldOS", "desc": "Obsolete platform version",
"bot": 40
},
@@ -17,6 +17,10 @@
"id": "serverOS", "desc": "Server OS",
"bot": 40
},
{"func": "matchesPlatform", "params": ["null"],
"id": "noOS", "desc": "Unknown or missing OS information",
"bot": 40
},
{"func": "smallPageCount", "params": [1],
"id": "onePage", "desc": "Visiter viewed only a single page",
"bot": 40
@@ -77,12 +81,13 @@
"ipRanges": [
{"from": "3.0.0.0", "to": "3.255.255.254", "label": "Amazon Data Services [US]"},
{"from": "8.127.0.0", "to": "8.223.255.254", "label": "Alibaba [CN]"},
{"from": "24.240.0.0", "to": "24.243.255.254", "label": "Charter [US]"},
{"from": "24.240.0.0", "to": "24.247.255.255", "m": 13, "label": "Charter [US]"},
{"from": "27.106.0.0", "to": "27.106.127.254", "label": "Huawei [US]"},
{"from": "34.0.0.0", "to": "34.191.255.254", "label": "Google LLC"},
{"from": "45.0.0.0", "to": "45.255.255.254", "label": "Various small ISPs, mostly BR"},
{"from": "46.250.160.0", "to": "46.250.191.254", "label": "Huawei [MX]"},
{"from": "49.0.200.0", "to": "49.0.255.254", "label": "Huawei [SG]"},
{"from": "66.249.64.0", "to": "66.249.95.255", "m": 19, "label": "Google LLC [US]"},
{"from": "84.37.35.0", "to": "84.37.255.254", "label": "GTT.net [US]"},
{"from": "94.74.64.0", "to": "94.74.127.254", "label": "Huawei [HK]"},
{"from": "101.0.0.0", "to": "101.255.255.254", "label": "ChinaNet [CN]"},
@@ -106,12 +111,14 @@
{"from": "188.0.0.0", "to": "188.255.255.254", "label": "South-American ISPs (188.x)"},
{"from": "189.0.0.0", "to": "189.255.255.254", "label": "South-American ISPs (189.x)"},
{"from": "190.0.0.0", "to": "190.255.255.254", "label": "South-American ISPs (190.x)"},
{"from": "192.124.170.0", "to": "192.124.182.254", "label": "Relcom [CZ]"},
{"from": "192.124.170.0", "to": "192.124.182.254", "label": "Relcom [CZ]"},
{"from": "195.37.0.0", "to": "195.37.255.255", "label": "DFN [DE]"},
{"from": "2001:4800::::::", "to": "2001:4fff:ffff:ffff:ffff:ffff:ffff:ffff", "label": "Rackspace/Google [US]"},
{"from": "2001:0ee0::::::", "to": "2001:ee3:ffff:ffff:ffff:ffff:ffff:ffff", "mask": 30, "label": "VNPT [VN]"},
{"from": "2600:1f00::::::", "to": "2600:1fff:ffff:ffff:ffff:ffff:ffff:ffff", "label": "Amazon Cloud [US]"},
{"from": "2804:::::::", "to": "2804:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "label": "Inspire [BR]"},
{"from": "2001:4860::::::", "to": "2001:4860:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Google LLC [US]"},
{"from": "2001:0ee0::::::", "to": "2001:ee3:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "label": "VNPT [VN]"},
{"from": "2600:1f00::::::", "to": "2600:1fff:ffff:ffff:ffff:ffff:ffff:ffff", "m": "Amazon Cloud [US]"},
{"from": "2603:6010::::::", "to": "2603:6010:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Charter [US]"},
{"from": "2603:8000::::::", "to": "2603:80ff:ffff:ffff:ffff:ffff:ffff:ffff", "m": 24, "label": "Charter [US]"},
{"from": "2804:::::::", "to": "2804:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "m": 16, "label": "Inspire [BR]"},
{"from": "2a0a:4cc0::::::", "to": "2a0a:4cc0:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "label": "Netcup [DE]"}
]
}

Binary file not shown.

Before

Width:  |  Height:  |  Size: 19 KiB

After

Width:  |  Height:  |  Size: 20 KiB

BIN
img/links.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.2 KiB

107
script.js
View File

@@ -107,7 +107,7 @@ const BotMon = {
_formatTime: function(date) {
if (date) {
return ('0'+date.getHours()).slice(-2) + ':' + ('0'+date.getMinutes()).slice(-2) + ':' + ('0'+date.getSeconds()).slice(-2);
return date.getHours() + ':' + ('0'+date.getMinutes()).slice(-2) + ':' + ('0'+date.getSeconds()).slice(-2);
} else {
return null;
}
@@ -433,7 +433,7 @@ BotMon.live = {
// get the page view info:
let pv = model._getPageView(visitor, dat);
if (!pv) {
console.warn(`No page view for visit ID “${dat.id}”, page “${dat.pg}”, registering a new one.`);
console.info(`No page view for visit ID “${dat.id}”, page “${dat.pg}”, registering a new one.`);
pv = model._makePageView(dat, type);
visitor._pageViews.push(pv);
}
@@ -448,19 +448,21 @@ BotMon.live = {
// helper function to create a new "page view" item:
_makePageView: function(data, type) {
// console.info('_makePageView', data);
// try to parse the referrer:
let rUrl = null;
try {
rUrl = ( data.ref && data.ref !== '' ? new URL(data.ref) : null );
} catch (e) {
console.info(`Invalid referer: “${data.ref}”.`);
console.warn(`Invalid referer: “${data.ref}”.`);
}
return {
_by: type,
ip: data.ip,
pg: data.pg,
lang: data.lang || '??',
_ref: rUrl,
_firstSeen: data.ts,
_lastSeen: data.ts,
@@ -1293,6 +1295,16 @@ BotMon.live = {
return false;
},
// the "Accept language" header contains certain entries:
clientAccepts: function(visitor, ...languages) {
//console.info('clientAccepts', visitor.accept, languages);
if (visitor.accept && languages) {;
return ( visitor.accept.split(',').filter(lang => languages.includes(lang)).length > 0 );
}
return false;
},
// Is there an accept-language field defined at all?
noAcceptLang: function(visitor) {
@@ -1886,7 +1898,21 @@ BotMon.live = {
platformName + ( data._platform.v > 0 ? ' (' + data._platform.v + ')' : '' ) ));
dl.appendChild(make('dt', {}, "IP-Address:"));
dl.appendChild(make('dd', {'class': 'has_icon ipaddr ip' + ipType}, data.ip));
const ipItem = make('dd', {'class': 'has_icon ipaddr ip' + ipType});
ipItem.appendChild(make('span', {'class': 'address'} , data.ip));
ipItem.appendChild(make('a', {
'class': 'icon_only extlink dnscheck',
'href': `https://dnschecker.org/ip-location.php?ip=${encodeURIComponent(data.ip)}`,
'target': 'dnscheck',
'title': "View this address on DNSChecker.org"
} , "Check Address"));
ipItem.appendChild(make('a', {
'class': 'icon_only extlink ipinfo',
'href': `https://ipinfo.io/${encodeURIComponent(data.ip)}`,
'target': 'ipinfo',
'title': "View this address on IPInfo.io"
} , "DNS Info"));
dl.appendChild(ipItem);
/*dl.appendChild(make('dt', {}, "ID:"));
dl.appendChild(make('dd', {'class': 'has_icon ip' + data.typ}, data.id));*/
@@ -1906,7 +1932,7 @@ BotMon.live = {
dl.appendChild(make('dd', {'class': 'agent'}, data.agent));
dl.appendChild(make('dt', {}, "Languages:"));
dl.appendChild(make('dd', {'class': 'langs'}, "Client accepts: [" + data.accept + "]; Page: [" + data.lang + ']'));
dl.appendChild(make('dd', {'class': 'langs'}, ` [${data.accept}]`));
if (data.geo && data.geo !=='') {
dl.appendChild(make('dt', {}, "Location:"));
@@ -1931,37 +1957,56 @@ BotMon.live = {
const pageList = make('ul');
/* list all page views */
data._pageViews.sort( (a, b) => a._firstSeen - b._firstSeen );
data._pageViews.forEach( (page) => {
//console.log("page:",page);
const pgLi = make('li');
let visitTimeStr = "Bounce";
const visitDuration = page._lastSeen.getTime() - page._firstSeen.getTime();
if (visitDuration > 0) {
visitTimeStr = Math.floor(visitDuration / 1000) + "s";
}
const lGroup = make('span'); // left group:
pgLi.appendChild(make('span', {}, page.pg)); /* DW Page ID */
if (page._ref) {
pgLi.appendChild(make('span', {
'data-ref': page._ref.host,
'title': "Referrer: " + page._ref.full
}, page._ref.site));
} else {
pgLi.appendChild(make('span', {
}, "No referer"));
}
pgLi.appendChild(make('span', {}, ( page._seenBy ? page._seenBy.join(', ') : '—') + '; ' + page._tickCount));
pgLi.appendChild(make('span', {}, BotMon.t._formatTime(page._firstSeen)));
lGroup.appendChild(make('span', {
'data-lang': page.lang,
'title': "PageID: " + page.pg
}, page.pg)); /* DW Page ID */
pgLi.appendChild(lGroup); // end of left group
const rGroup = make('span'); // right group:
let visitTimeStr = "Bounce";
const visitDuration = page._lastSeen.getTime() - page._firstSeen.getTime();
if (visitDuration > 0) {
visitTimeStr = Math.floor(visitDuration / 1000) + "s";
}
/*if (page._ref) {
rGroup.appendChild(make('span', {
'data-ref': page._ref.host,
'title': "Referrer: " + page._ref.full
}, page._ref.site));
} else {
rGroup.appendChild(make('span', {
}, "No referer"));
}*/
//rGroup.appendChild(make('span', {}, ( page._seenBy ? page._seenBy.join(', ') : '—') + '; ' + page._tickCount));
// get the time difference:
const tDiff = BotMon.t._formatTimeDiff(page._firstSeen, page._lastSeen);
if (tDiff) {
rGroup.appendChild(make('span', {'class': 'visit-length', 'title': 'Last seen: ' + page._lastSeen.toLocaleString()}, tDiff));
} else {
rGroup.appendChild(make('span', {
'class': 'bounce',
'title': "Visitor bounced"}, "Bounce"));
}
rGroup.appendChild(make('span', {
'class': 'first-seen',
'title': "First visited: " + page._firstSeen.toLocaleString()
}, BotMon.t._formatTime(page._firstSeen)));
pgLi.appendChild(rGroup); // end of right group
// get the time difference:
const tDiff = BotMon.t._formatTimeDiff(page._firstSeen, page._lastSeen);
if (tDiff) {
pgLi.appendChild(make('span', {'class': 'visit-length', 'title': 'Last seen: ' + page._lastSeen.toLocaleString()}, tDiff));
} else {
pgLi.appendChild(make('span', {
'class': 'bounce',
'title': "Visitor bounced"}, "Bounce"));
}
pageList.appendChild(pgLi);
});

View File

@@ -40,7 +40,7 @@
/* Bot icons */
&.bot::before { background-image: url('img/bots.png') }
&.bot_googlebot::before, &.bot_googleads::before, &.bot_googleapi::before { background-position-y: -20px }
&.bot_googlebot::before, &.bot_googleads::before, &.bot_googleapi::before, &.bot_googleother::before { background-position-y: -20px }
&.bot_bingbot::before { background-position-y: -40px }
&.bot_applebot::before { background-position-y: -60px }
&.bot_openai::before { background-position-y: -80px }
@@ -84,6 +84,7 @@
&.cl_silk::before { background-position-y: -280px }
&.cl_ffold::before { background-position-y: -300px }
&.cl_chromeold::before { background-position-y: -320px }
&.cl_ecosia::before { background-position-y: -340px }
/* Country flags */
/* Note: flag images and CSS adapted from: https://github.com/lafeber/world-flags-sprite/ */
@@ -332,6 +333,11 @@
&.typ_php::before { background-position-y: -40px }
&.typ_ip::before { background-position-y: -60px }
&.typ_usr::before { background-position-y: -80px }
/* External link icons */
&.extlink::before { background-image: url('img/links.png') }
&.extlink.dnscheck::before { background-position-y: -20px }
&.extlink.ipinfo::before { background-position-y: -40px }
}
/* grid layout for the overview: */
@@ -524,26 +530,49 @@
& {
display: flex;
justify-content: space-between;
align-items: center;
align-items: baseline;
white-space: nowrap;
line-height: 1.2rem;
margin: 0;
padding: 0 .25em;
}
&:nth-child(odd) {
background-color: #DFDFDF;
}
span {
&.visit-length {
min-width: min-content;
}
&.bounce {
width: 1.25em; height: 1.25em;
overflow: hidden;
}
&.bounce::before {
display: inline-block;
content: '';
width: 1.25em; height: 1.25em;
background: transparent url('img/bounce.svg') center no-repeat;
background-size: 1.25em;
}
display: inline-block;
}
}
}
span[data-lang] {
overflow: hidden;
text-overflow: ellipsis;
}
span[data-lang]::after {
content: attr(data-lang);
font-size: smaller;
color: #555;
border: #555 solid 1px;
line-height: 1.25;
border-radius: 2pt;
padding: 0 1pt;
margin-left: .2em;
}
span.first-seen {
min-width: 4.2em;
text-align: right;;
}
span.bounce {
width: 1.25em; height: 1.25em;
overflow: hidden;
}
span.bounce::before {
display: inline-block;
content: '';
width: 1.25em; height: 1.25em;
background: transparent url('img/bounce.svg') center no-repeat;
background-size: 1.25em;
}
}
}
@@ -554,7 +583,7 @@
align-items: center;
}
li:nth-child(odd) {
background-color: #EEE;
background-color: #DFDFDF;
}
li.total {
border-top: #333 solid 1px;

View File

@@ -2,6 +2,12 @@
// Note: this script is normally called in HEAD mode, therefore it can not return any payload.
// quit out if it is called without athe right parameters:
if (!isset($_GET['id']) || !isset($_GET['p'])) {
http_response_code(400);
die("Parameter error.");
}
// what is the session identifier?
$sessionId = preg_replace('/[\x00-\x1F{};\"\']/', "\u{FFFD}", $_GET['id']) /* clean json parameter */
?? session_id()