"use strict"; /* DokuWiki BotMon Plugin Script file */ /* 04.09.2025 - 0.1.8 - pre-release */ /* Authors: Sascha Leib */ const BotMon = { init: function() { //console.info('BotMon.init()'); // find the plugin basedir: this._baseDir = document.currentScript.src.substring(0, document.currentScript.src.indexOf('/exe/')) + '/plugins/botmon/'; // read the page language from the DOM: this._lang = document.getRootNode().documentElement.lang || this._lang; // get the time offset: this._timeDiff = BotMon.t._getTimeOffset(); // init the sub-objects: BotMon.t._callInit(this); }, _baseDir: null, _lang: 'en', _today: (new Date()).toISOString().slice(0, 10), _timeDiff: '', /* internal tools */ t: { /* helper function to call inits of sub-objects */ _callInit: function(obj) { //console.info('BotMon.t._callInit(obj=',obj,')'); /* call init / _init on each sub-object: */ Object.keys(obj).forEach( (key,i) => { const sub = obj[key]; let init = null; if (typeof sub === 'object' && sub.init) { init = sub.init; } // bind to object if (typeof init == 'function') { const init2 = init.bind(sub); init2(obj); } }); }, /* helper function to calculate the time difference to UTC: */ _getTimeOffset: function() { const now = new Date(); let offset = now.getTimezoneOffset(); // in minutes const sign = Math.sign(offset); // +1 or -1 offset = Math.abs(offset); // always positive let hours = 0; while (offset >= 60) { hours += 1; offset -= 60; } return ( hours > 0 ? sign * hours + ' h' : '') + (offset > 0 ? ` ${offset} min` : ''); }, /* helper function to create a new element with all attributes and text content */ _makeElement: function(name, atlist = undefined, text = undefined) { var r = null; try { r = document.createElement(name); if (atlist) { for (let attr in atlist) { r.setAttribute(attr, atlist[attr]); } } if (text) { r.textContent = text.toString(); } } catch(e) { console.error(e); } return r; } } }; /* everything specific to the "Today" tab is self-contained in the "live" object: */ BotMon.live = { init: function() { //console.info('BotMon.live.init()'); // set the title: const tDiff = '(UTC' + (BotMon._timeDiff != '' ? `, ${BotMon._timeDiff}` : '' ) + ')'; BotMon.live.gui.status.setTitle(`Data for ${tDiff}`); // init sub-objects: BotMon.t._callInit(this); }, data: { init: function() { //console.info('BotMon.live.data.init()'); // call sub-inits: BotMon.t._callInit(this); }, // this will be called when the known json files are done loading: _dispatch: function(file) { //console.info('BotMon.live.data._dispatch(,',file,')'); // shortcut to make code more readable: const data = BotMon.live.data; // set the flags: switch(file) { case 'bots': data._dispatchBotsLoaded = true; break; case 'clients': data._dispatchClientsLoaded = true; break; case 'platforms': data._dispatchPlatformsLoaded = true; break; default: // ignore } // are all the flags set? if (data._dispatchBotsLoaded && data._dispatchClientsLoaded && data._dispatchPlatformsLoaded) { // chain the log files loading: BotMon.live.data.loadLogFile('srv', BotMon.live.data._onServerLogLoaded); } }, // flags to track which data files have been loaded: _dispatchBotsLoaded: false, _dispatchClientsLoaded: false, _dispatchPlatformsLoaded: false, // event callback, after the server log has been loaded: _onServerLogLoaded: function() { //console.info('BotMon.live.data._onServerLogLoaded()'); // chain the client log file to load: BotMon.live.data.loadLogFile('log', BotMon.live.data._onClientLogLoaded); }, // event callback, after the client log has been loaded: _onClientLogLoaded: function() { //console.info('BotMon.live.data._onClientLogLoaded()'); // chain the ticks file to load: BotMon.live.data.loadLogFile('tck', BotMon.live.data._onTicksLogLoaded); }, // event callback, after the tiker log has been loaded: _onTicksLogLoaded: function() { //console.info('BotMon.live.data._onTicksLogLoaded()'); // analyse the data: BotMon.live.data.analytics.analyseAll(); // sort the data: // #TODO // display the data: BotMon.live.gui.overview.make(); //console.log(BotMon.live.data.model._visitors); }, model: { // visitors storage: _visitors: [], // find an already existing visitor record: findVisitor: function(id) { // shortcut to make code more readable: const model = BotMon.live.data.model; // loop over all visitors already registered: for (let i=0; i { // count visits and page views: this.data.totalVisits += 1; this.data.totalPageViews += v._pageViews.length; // check for typical bot aspects: let botScore = 0; /*if (v._isBot >= 1.0) { // known bots this.data.bots.known += 1; this.groups.knownBots.push(v); } if (v.usr && v.usr != '') { // known users */ this.groups.users.push(v); this.data.bots.users += 1; /*} else { // not a known bot, nor a known user; check other aspects: // no referrer at all: if (!v._hasReferrer) botScore += 0.2; // no js client logging: if (!v._jsClient) botScore += 0.2; // average time between page views less than 30s: if (v._pageViews.length > 1) { botScore -= 0.2; // more than one view: good! let totalDiff = 0; for (let i=1; i= 0.5) { this.data.bots.suspected += 1; this.groups.suspectedBots.push(v); } else { this.data.bots.human += 1; this.groups.humans.push(v); } }*/ }); console.log(this.data); console.log(this.groups); } }, bots: { // loads the list of known bots from a JSON file: init: async function() { //console.info('BotMon.live.data.bots.init()'); // Load the list of known bots: BotMon.live.gui.status.showBusy("Loading known bots …"); const url = BotMon._baseDir + 'data/known-bots.json'; try { const response = await fetch(url); if (!response.ok) { throw new Error(`${response.status} ${response.statusText}`); } this._list = await response.json(); this._ready = true; } catch (error) { BotMon.live.gui.status.setError("Error while loading the ‘known bots’ file: " + error.message); } finally { BotMon.live.gui.status.hideBusy("Status: Done."); BotMon.live.data._dispatch('bots') } }, // returns bot info if the clientId matches a known bot, null otherwise: match: function(agent) { //console.info('BotMon.live.data.bots.match(',agent,')'); const BotList = BotMon.live.data.bots._list; // default is: not found! let botInfo = null; // check for known bots: if (agent) { BotList.find(bot => { let r = false; for (let j=0; j 1 ? rxr[1] : -1) } r = true; break; } } return r; }); } //console.log("botInfo:", botInfo); return botInfo; }, // indicates if the list is loaded and ready to use: _ready: false, // the actual bot list is stored here: _list: [] }, clients: { // loads the list of known clients from a JSON file: init: async function() { //console.info('BotMon.live.data.clients.init()'); // Load the list of known bots: BotMon.live.gui.status.showBusy("Loading known clients"); const url = BotMon._baseDir + 'data/known-clients.json'; try { const response = await fetch(url); if (!response.ok) { throw new Error(`${response.status} ${response.statusText}`); } BotMon.live.data.clients._list = await response.json(); BotMon.live.data.clients._ready = true; } catch (error) { BotMon.live.gui.status.setError("Error while loading the known clients file: " + error.message); } finally { BotMon.live.gui.status.hideBusy("Status: Done."); BotMon.live.data._dispatch('clients') } }, // returns bot info if the user-agent matches a known bot, null otherwise: match: function(agent) { //console.info('BotMon.live.data.clients.match(',agent,')'); let match = {"n": "Unknown", "v": -1, "id": null}; if (agent) { BotMon.live.data.clients._list.find(client => { let r = false; for (let j=0; j 1 ? rxr[1] : -1); match.id = client.id || null; r = true; break; } } return r; }); } //console.log(match) return match; }, // indicates if the list is loaded and ready to use: _ready: false, // the actual bot list is stored here: _list: [] }, platforms: { // loads the list of known platforms from a JSON file: init: async function() { //console.info('BotMon.live.data.platforms.init()'); // Load the list of known bots: BotMon.live.gui.status.showBusy("Loading known platforms"); const url = BotMon._baseDir + 'data/known-platforms.json'; try { const response = await fetch(url); if (!response.ok) { throw new Error(`${response.status} ${response.statusText}`); } BotMon.live.data.platforms._list = await response.json(); BotMon.live.data.platforms._ready = true; } catch (error) { BotMon.live.gui.status.setError("Error while loading the known platforms file: " + error.message); } finally { BotMon.live.gui.status.hideBusy("Status: Done."); BotMon.live.data._dispatch('platforms') } }, // returns bot info if the browser id matches a known platform: match: function(cid) { //console.info('BotMon.live.data.platforms.match(',cid,')'); let match = {"n": "Unknown", "id": null}; if (cid) { BotMon.live.data.platforms._list.find(platform => { let r = false; for (let j=0; j 1 ? rxr[1] : -1); match.id = platform.id || null; r = true; break; } } return r; }); } return match; }, // indicates if the list is loaded and ready to use: _ready: false, // the actual bot list is stored here: _list: [] }, loadLogFile: async function(type, onLoaded = undefined) { console.info('BotMon.live.data.loadLogFile(',type,')'); let typeName = ''; let columns = []; switch (type) { case "srv": typeName = "Server"; columns = ['ts','ip','pg','id','typ','usr','agent','ref']; break; case "log": typeName = "Page load"; columns = ['ts','ip','pg','id','usr','lt','ref','agent']; break; case "tck": typeName = "Ticker"; columns = ['ts','ip','pg','id','agent']; break; default: console.warn(`Unknown log type ${type}.`); return; } // Show the busy indicator and set the visible status: BotMon.live.gui.status.showBusy(`Loading ${typeName} log file …`); // compose the URL from which to load: const url = BotMon._baseDir + `logs/${BotMon._today}.${type}.txt`; //console.log("Loading:",url); // fetch the data: try { const response = await fetch(url); if (!response.ok) { throw new Error(`${response.status} ${response.statusText}`); } const logtxt = await response.text(); logtxt.split('\n').forEach((line) => { if (line.trim() === '') return; // skip empty lines const cols = line.split('\t'); // assign the columns to an object: const data = {}; cols.forEach( (colVal,i) => { colName = columns[i] || `col${i}`; const colValue = (colName == 'ts' ? new Date(colVal) : colVal.trim()); data[colName] = colValue; }); // register the visit in the model: switch(type) { case 'srv': BotMon.live.data.model.registerVisit(data, type); break; case 'log': data.typ = 'js'; BotMon.live.data.model.updateVisit(data); break; case 'tck': data.typ = 'js'; BotMon.live.data.model.updateTicks(data); break; default: console.warn(`Unknown log type ${type}.`); return; } }); if (onLoaded) { onLoaded(); // callback after loading is finished. } } catch (error) { BotMon.live.gui.status.setError(`Error while loading the ${typeName} log file: ${error.message}.`); } finally { BotMon.live.gui.status.hideBusy("Status: Done."); } } }, gui: { init: function() { // init the lists view: this.lists.init(); }, overview: { make: function() { const data = BotMon.live.data.analytics.data; const parent = document.getElementById('botmon__today__content'); if (parent) { const bounceRate = Math.round(data.totalVisits / data.totalPageViews * 1000) / 10; jQuery(parent).prepend(jQuery(`
Overview
Web metrics
Total visits:${data.totalVisits}
Total page views:${data.totalPageViews}
Bounce rate:${bounceRate} %
∅ load time:${data.avgLoadTime} ms
Bots vs. Humans
Known bots:${data.bots.known}
Suspected bots:${data.bots.suspected}
Probably humans:${data.bots.human}
Registered users:${data.bots.users}
Known bots
`)); } } }, status: { setText: function(txt) { const el = document.getElementById('botmon__today__status'); if (el && BotMon.live.gui.status._errorCount <= 0) { el.innerText = txt; } }, setTitle: function(html) { const el = document.getElementById('botmon__today__title'); if (el) { el.innerHTML = html; } }, setError: function(txt) { console.error(txt); BotMon.live.gui.status._errorCount += 1; const el = document.getElementById('botmon__today__status'); if (el) { el.innerText = "An error occured. See the browser log for details!"; el.classList.add('error'); } }, _errorCount: 0, showBusy: function(txt = null) { BotMon.live.gui.status._busyCount += 1; const el = document.getElementById('botmon__today__busy'); if (el) { el.style.display = 'inline-block'; } if (txt) BotMon.live.gui.status.setText(txt); }, _busyCount: 0, hideBusy: function(txt = null) { const el = document.getElementById('botmon__today__busy'); BotMon.live.gui.status._busyCount -= 1; if (BotMon.live.gui.status._busyCount <= 0) { if (el) el.style.display = 'none'; if (txt) BotMon.live.gui.status.setText(txt); } } }, lists: { init: function() { const parent = document.getElementById('botmon__today__visitorlists'); if (parent) { for (let i=0; i < 4; i++) { // change the id and title by number: let listTitle = ''; let listId = ''; switch (i) { case 0: listTitle = "Registered users"; listId = 'users'; break; case 1: listTitle = "Probably humans"; listId = 'humans'; break; case 2: listTitle = "Suspected bots"; listId = 'suspectedBots'; break; case 3: listTitle = "Known bots"; listId = 'knownBots'; break; default: console.warn('Unknwon list number.'); } const details = BotMon.t._makeElement('details', { 'data-group': listId, 'data-loaded': false }); details.appendChild(BotMon.t._makeElement('summary', undefined, listTitle )); details.addEventListener("toggle", this._onDetailsToggle); parent.appendChild(details); } } }, _onDetailsToggle: function(e) { console.info('BotMon.live.gui.lists._onDetailsToggle()'); const target = e.target; if (target.getAttribute('data-loaded') == 'false') { // only if not loaded yet target.setAttribute('data-loaded', 'loading'); const fillType = target.getAttribute('data-group'); const fillList = BotMon.live.data.analytics.groups[fillType]; if (fillList && fillList.length > 0) { const ul = BotMon.t._makeElement('ul'); fillList.forEach( (it) => { ul.appendChild(BotMon.live.gui.lists._makeVisitorItem(it, fillType)); }); target.appendChild(ul); target.setAttribute('data-loaded', 'true'); } else { target.setAttribute('data-loaded', 'false'); } } }, _makeVisitorItem: function(data, type) { // shortcut for neater code: const make = BotMon.t._makeElement; let ipType = ( data.ip.indexOf(':') >= 0 ? '6' : '4' ); const li = make('li'); // root list item const details = make('details'); const summary = make('summary'); details.appendChild(summary); const span1 = make('span'); /* left-hand group */ if (data._type == 'bot') { /* Bot only */ span1.appendChild(make('span', { /* Bot */ 'class': 'bot bot_' + (data._bot ? data._bot.id : 'unknown'), 'title': "Bot: " + (data._bot ? data._bot.n : 'Unknown') }, (data._bot ? data._bot.n : 'Unknown'))); } else if (data._type == 'usr') { /* User only */ span1.appendChild(make('span', { /* User */ 'class': 'user' + (data._user ? data._user.id : 'unknown'), 'title': "User: " + data.usr }, data.usr)); } else { /* others */ if (data.ip == '127.0.0.1' || data.ip == '::1' ) ipType = '0'; span1.appendChild(make('span', { /* IP-Address */ 'class': 'ipaddr ip' + ipType, 'title': "IP-Address: " + data.ip }, data.ip)); } const platformName = (data._platform ? data._platform.n : 'Unknown'); span1.appendChild(make('span', { /* Platform */ 'class': 'icon platform platform_' + (data._platform ? data._platform.id : 'unknown'), 'title': "Platform: " + platformName }, platformName)); const clientName = (data._client ? data._client.n: 'Unknown'); span1.appendChild(make('span', { /* Client */ 'class': 'icon client client_' + (data._client ? data._client.id : 'unknown'), 'title': "Client: " + clientName }, clientName)); summary.appendChild(span1); const span2 = make('span'); /* right-hand group */ span2.appendChild(make('time', { /* Last seen */ 'data-field': 'last-seen', 'datetime': (data._lastSeen ? data._lastSeen : 'unknown') }, (data._lastSeen ? data._lastSeen.getHours() + ':' + data._lastSeen.getMinutes() + ':' + data._lastSeen.getSeconds() : 'Unknown'))); summary.appendChild(span2); // create expanable section: const dl = make('dl', {'class': 'visitor_details'}); if (data._bot) { dl.appendChild(make('dt', {}, "Bot:")); /* bot info */ dl.appendChild(make('dd', {'class': 'has_icon bot bot_' + (data._bot ? data._bot.id : 'unknown')}, (data._bot ? data._bot.n : 'Unknown'))); } dl.appendChild(make('dt', {}, "Client:")); /* client */ dl.appendChild(make('dd', {'class': 'has_icon client_' + (data._client ? data._client.id : 'unknown')}, clientName + ( data._client.v > 0 ? ' (' + data._client.v + ')' : '' ) )); dl.appendChild(make('dt', {}, "Platform:")); /* platform */ dl.appendChild(make('dd', {'class': 'has_icon platform_' + (data._platform ? data._platform.id : 'unknown')}, platformName + ( data._platform.v > 0 ? ' (' + data._platform.v + ')' : '' ) )); dl.appendChild(make('dt', {}, "IP-Address:")); dl.appendChild(make('dd', {'class': 'has_icon ip' + ipType}, data.ip)); if ((data._lastSeen - data._firstSeen) < 1) { dl.appendChild(make('dt', {}, "Seen:")); dl.appendChild(make('dd', {'class': 'seen'}, data._firstSeen.toLocaleString())); } else { dl.appendChild(make('dt', {}, "First seen:")); dl.appendChild(make('dd', {'class': 'firstSeen'}, data._firstSeen.toLocaleString())); dl.appendChild(make('dt', {}, "Last seen:")); dl.appendChild(make('dd', {'class': 'lastSeen'}, data._lastSeen.toLocaleString())); } dl.appendChild(make('dt', {}, "User-Agent:")); dl.appendChild(make('dd', {'class': 'agent' + ipType}, data.agent)); dl.appendChild(make('dt', {}, "Visited pages:")); const pagesDd = make('dd', {'class': 'pages'}); const pageList = make('ul'); data._pageViews.forEach( (page) => { const pgLi = make('li'); let visitTimeStr = "Bounce"; const visitDuration = page._lastSeen.getTime() - page._firstSeen.getTime(); if (visitDuration > 0) { visitTimeStr = Math.floor(visitDuration / 1000) + "s"; } pgLi.appendChild(make('span', {}, page.pg)); pgLi.appendChild(make('span', {}, page.ref)); pgLi.appendChild(make('span', {}, visitTimeStr)); pageList.appendChild(pgLi); }); pagesDd.appendChild(pageList); dl.appendChild(pagesDd); details.appendChild(dl); li.appendChild(details); return li; } } } }; /* launch only if the BotMon admin panel is open: */ if (document.getElementById('botmon__admin')) { BotMon.init(); }