"use strict"; /* DokuWiki BotMon Plugin Script file */ /* 04.09.2025 - 0.1.8 - pre-release */ /* Authors: Sascha Leib */ // enumeration of user types: const BM_USERTYPE = Object.freeze({ 'UNKNOWN': 'unknown', 'KNOWN_USER': 'user', 'HUMAN': 'human', 'LIKELY_BOT': 'likely_bot', 'KNOWN_BOT': 'known_bot' }); /* BotMon root object */ const BotMon = { init: function() { //console.info('BotMon.init()'); // find the plugin basedir: this._baseDir = document.currentScript.src.substring(0, document.currentScript.src.indexOf('/exe/')) + '/plugins/botmon/'; // read the page language from the DOM: this._lang = document.getRootNode().documentElement.lang || this._lang; // get the time offset: this._timeDiff = BotMon.t._getTimeOffset(); // init the sub-objects: BotMon.t._callInit(this); }, _baseDir: null, _lang: 'en', _today: (new Date()).toISOString().slice(0, 10), _timeDiff: '', /* internal tools */ t: { /* helper function to call inits of sub-objects */ _callInit: function(obj) { //console.info('BotMon.t._callInit(obj=',obj,')'); /* call init / _init on each sub-object: */ Object.keys(obj).forEach( (key,i) => { const sub = obj[key]; let init = null; if (typeof sub === 'object' && sub.init) { init = sub.init; } // bind to object if (typeof init == 'function') { const init2 = init.bind(sub); init2(obj); } }); }, /* helper function to calculate the time difference to UTC: */ _getTimeOffset: function() { const now = new Date(); let offset = now.getTimezoneOffset(); // in minutes const sign = Math.sign(offset); // +1 or -1 offset = Math.abs(offset); // always positive let hours = 0; while (offset >= 60) { hours += 1; offset -= 60; } return ( hours > 0 ? sign * hours + ' h' : '') + (offset > 0 ? ` ${offset} min` : ''); }, /* helper function to create a new element with all attributes and text content */ _makeElement: function(name, atlist = undefined, text = undefined) { var r = null; try { r = document.createElement(name); if (atlist) { for (let attr in atlist) { r.setAttribute(attr, atlist[attr]); } } if (text) { r.textContent = text.toString(); } } catch(e) { console.error(e); } return r; } } }; /* everything specific to the "Today" tab is self-contained in the "live" object: */ BotMon.live = { init: function() { //console.info('BotMon.live.init()'); // set the title: const tDiff = '(UTC' + (BotMon._timeDiff != '' ? `, ${BotMon._timeDiff}` : '' ) + ')'; BotMon.live.gui.status.setTitle(`Data for ${tDiff}`); // init sub-objects: BotMon.t._callInit(this); }, data: { init: function() { //console.info('BotMon.live.data.init()'); // call sub-inits: BotMon.t._callInit(this); }, // this will be called when the known json files are done loading: _dispatch: function(file) { //console.info('BotMon.live.data._dispatch(,',file,')'); // shortcut to make code more readable: const data = BotMon.live.data; // set the flags: switch(file) { case 'bots': data._dispatchBotsLoaded = true; break; case 'clients': data._dispatchClientsLoaded = true; break; case 'platforms': data._dispatchPlatformsLoaded = true; break; default: // ignore } // are all the flags set? if (data._dispatchBotsLoaded && data._dispatchClientsLoaded && data._dispatchPlatformsLoaded) { // chain the log files loading: BotMon.live.data.loadLogFile('srv', BotMon.live.data._onServerLogLoaded); } }, // flags to track which data files have been loaded: _dispatchBotsLoaded: false, _dispatchClientsLoaded: false, _dispatchPlatformsLoaded: false, // event callback, after the server log has been loaded: _onServerLogLoaded: function() { //console.info('BotMon.live.data._onServerLogLoaded()'); // chain the client log file to load: BotMon.live.data.loadLogFile('log', BotMon.live.data._onClientLogLoaded); }, // event callback, after the client log has been loaded: _onClientLogLoaded: function() { //console.info('BotMon.live.data._onClientLogLoaded()'); // chain the ticks file to load: BotMon.live.data.loadLogFile('tck', BotMon.live.data._onTicksLogLoaded); }, // event callback, after the tiker log has been loaded: _onTicksLogLoaded: function() { //console.info('BotMon.live.data._onTicksLogLoaded()'); // analyse the data: BotMon.live.data.analytics.analyseAll(); // sort the data: // #TODO // display the data: BotMon.live.gui.overview.make(); //console.log(BotMon.live.data.model._visitors); }, model: { // visitors storage: _visitors: [], // find an already existing visitor record: findVisitor: function(visitor) { //console.info('BotMon.live.data.model.findVisitor()'); //console.log(visitor); // shortcut to make code more readable: const model = BotMon.live.data.model; // loop over all visitors already registered: for (let i=0; i { // count visits and page views: this.data.totalVisits += 1; this.data.totalPageViews += v._pageViews.length; // check for typical bot aspects: let botScore = 0; if (v._type == BM_USERTYPE.KNOWN_BOT) { // known bots this.data.bots.known += 1; this.groups.knownBots.push(v); } else if (v._type == BM_USERTYPE.KNOWN_USER) { // known users */ this.groups.users.push(v); this.data.bots.users += 1; } else { // TODO: find suspected bots this.data.bots.suspected += 1; this.groups.suspectedBots.push(v); } }); //console.log(this.data); //console.log(this.groups); } }, bots: { // loads the list of known bots from a JSON file: init: async function() { //console.info('BotMon.live.data.bots.init()'); // Load the list of known bots: BotMon.live.gui.status.showBusy("Loading known bots …"); const url = BotMon._baseDir + 'data/known-bots.json'; try { const response = await fetch(url); if (!response.ok) { throw new Error(`${response.status} ${response.statusText}`); } this._list = await response.json(); this._ready = true; } catch (error) { BotMon.live.gui.status.setError("Error while loading the ‘known bots’ file: " + error.message); } finally { BotMon.live.gui.status.hideBusy("Status: Done."); BotMon.live.data._dispatch('bots') } }, // returns bot info if the clientId matches a known bot, null otherwise: match: function(agent) { //console.info('BotMon.live.data.bots.match(',agent,')'); const BotList = BotMon.live.data.bots._list; // default is: not found! let botInfo = null; // check for known bots: if (agent) { BotList.find(bot => { let r = false; for (let j=0; j 1 ? rxr[1] : -1) }; r = true; break; } }; return r; }); } //console.log("botInfo:", botInfo); return botInfo; }, // indicates if the list is loaded and ready to use: _ready: false, // the actual bot list is stored here: _list: [] }, clients: { // loads the list of known clients from a JSON file: init: async function() { //console.info('BotMon.live.data.clients.init()'); // Load the list of known bots: BotMon.live.gui.status.showBusy("Loading known clients"); const url = BotMon._baseDir + 'data/known-clients.json'; try { const response = await fetch(url); if (!response.ok) { throw new Error(`${response.status} ${response.statusText}`); } BotMon.live.data.clients._list = await response.json(); BotMon.live.data.clients._ready = true; } catch (error) { BotMon.live.gui.status.setError("Error while loading the known clients file: " + error.message); } finally { BotMon.live.gui.status.hideBusy("Status: Done."); BotMon.live.data._dispatch('clients') } }, // returns bot info if the user-agent matches a known bot, null otherwise: match: function(agent) { //console.info('BotMon.live.data.clients.match(',agent,')'); let match = {"n": "Unknown", "v": -1, "id": null}; if (agent) { BotMon.live.data.clients._list.find(client => { let r = false; for (let j=0; j 1 ? rxr[1] : -1); match.id = client.id || null; r = true; break; } } return r; }); } //console.log(match) return match; }, // indicates if the list is loaded and ready to use: _ready: false, // the actual bot list is stored here: _list: [] }, platforms: { // loads the list of known platforms from a JSON file: init: async function() { //console.info('BotMon.live.data.platforms.init()'); // Load the list of known bots: BotMon.live.gui.status.showBusy("Loading known platforms"); const url = BotMon._baseDir + 'data/known-platforms.json'; try { const response = await fetch(url); if (!response.ok) { throw new Error(`${response.status} ${response.statusText}`); } BotMon.live.data.platforms._list = await response.json(); BotMon.live.data.platforms._ready = true; } catch (error) { BotMon.live.gui.status.setError("Error while loading the known platforms file: " + error.message); } finally { BotMon.live.gui.status.hideBusy("Status: Done."); BotMon.live.data._dispatch('platforms') } }, // returns bot info if the browser id matches a known platform: match: function(cid) { //console.info('BotMon.live.data.platforms.match(',cid,')'); let match = {"n": "Unknown", "id": null}; if (cid) { BotMon.live.data.platforms._list.find(platform => { let r = false; for (let j=0; j 1 ? rxr[1] : -1); match.id = platform.id || null; r = true; break; } } return r; }); } return match; }, // indicates if the list is loaded and ready to use: _ready: false, // the actual bot list is stored here: _list: [] }, loadLogFile: async function(type, onLoaded = undefined) { console.info('BotMon.live.data.loadLogFile(',type,')'); let typeName = ''; let columns = []; switch (type) { case "srv": typeName = "Server"; columns = ['ts','ip','pg','id','typ','usr','agent','ref']; break; case "log": typeName = "Page load"; columns = ['ts','ip','pg','id','usr','lt','ref','agent']; break; case "tck": typeName = "Ticker"; columns = ['ts','ip','pg','id','agent']; break; default: console.warn(`Unknown log type ${type}.`); return; } // Show the busy indicator and set the visible status: BotMon.live.gui.status.showBusy(`Loading ${typeName} log file …`); // compose the URL from which to load: const url = BotMon._baseDir + `logs/${BotMon._today}.${type}.txt`; //console.log("Loading:",url); // fetch the data: try { const response = await fetch(url); if (!response.ok) { throw new Error(`${response.status} ${response.statusText}`); } const logtxt = await response.text(); logtxt.split('\n').forEach((line) => { if (line.trim() === '') return; // skip empty lines const cols = line.split('\t'); // assign the columns to an object: const data = {}; cols.forEach( (colVal,i) => { colName = columns[i] || `col${i}`; const colValue = (colName == 'ts' ? new Date(colVal) : colVal.trim()); data[colName] = colValue; }); // register the visit in the model: switch(type) { case 'srv': BotMon.live.data.model.registerVisit(data, type); break; case 'log': data.typ = 'js'; BotMon.live.data.model.updateVisit(data); break; case 'tck': data.typ = 'js'; BotMon.live.data.model.updateTicks(data); break; default: console.warn(`Unknown log type ${type}.`); return; } }); if (onLoaded) { onLoaded(); // callback after loading is finished. } } catch (error) { BotMon.live.gui.status.setError(`Error while loading the ${typeName} log file: ${error.message}.`); } finally { BotMon.live.gui.status.hideBusy("Status: Done."); } } }, gui: { init: function() { // init the lists view: this.lists.init(); }, overview: { make: function() { const data = BotMon.live.data.analytics.data; const parent = document.getElementById('botmon__today__content'); // shortcut for neater code: const makeElement = BotMon.t._makeElement; if (parent) { const bounceRate = Math.round(data.totalVisits / data.totalPageViews * 1000) / 10; jQuery(parent).prepend(jQuery(`
Overview
Web metrics
Total visits:${data.totalVisits}
Total page views:${data.totalPageViews}
Bounce rate:${bounceRate} %
∅ load time:${data.avgLoadTime} ms
Bots vs. Humans
Known bots:${data.bots.known}
Suspected bots:${data.bots.suspected}
Probably humans:${data.bots.human}
Registered users:${data.bots.users}
`)); // update known bots list: const block = document.getElementById('botmon__botslist'); block.innerHTML = "
Top known bots
"; let bots = BotMon.live.data.analytics.groups.knownBots.toSorted( (a, b) => { return b._pageViews.length - a._pageViews.length; }); for (let i=0; i < Math.min(bots.length, 4); i++) { const dd = makeElement('dd'); dd.appendChild(makeElement('span', {'class': 'bot bot_' + bots[i]._bot.id}, bots[i]._bot.n)); dd.appendChild(makeElement('span', undefined, bots[i]._pageViews.length)); block.appendChild(dd); } } } }, status: { setText: function(txt) { const el = document.getElementById('botmon__today__status'); if (el && BotMon.live.gui.status._errorCount <= 0) { el.innerText = txt; } }, setTitle: function(html) { const el = document.getElementById('botmon__today__title'); if (el) { el.innerHTML = html; } }, setError: function(txt) { console.error(txt); BotMon.live.gui.status._errorCount += 1; const el = document.getElementById('botmon__today__status'); if (el) { el.innerText = "An error occured. See the browser log for details!"; el.classList.add('error'); } }, _errorCount: 0, showBusy: function(txt = null) { BotMon.live.gui.status._busyCount += 1; const el = document.getElementById('botmon__today__busy'); if (el) { el.style.display = 'inline-block'; } if (txt) BotMon.live.gui.status.setText(txt); }, _busyCount: 0, hideBusy: function(txt = null) { const el = document.getElementById('botmon__today__busy'); BotMon.live.gui.status._busyCount -= 1; if (BotMon.live.gui.status._busyCount <= 0) { if (el) el.style.display = 'none'; if (txt) BotMon.live.gui.status.setText(txt); } } }, lists: { init: function() { const parent = document.getElementById('botmon__today__visitorlists'); if (parent) { for (let i=0; i < 4; i++) { // change the id and title by number: let listTitle = ''; let listId = ''; switch (i) { case 0: listTitle = "Registered users"; listId = 'users'; break; case 1: listTitle = "Probably humans"; listId = 'humans'; break; case 2: listTitle = "Suspected bots"; listId = 'suspectedBots'; break; case 3: listTitle = "Known bots"; listId = 'knownBots'; break; default: console.warn('Unknwon list number.'); } const details = BotMon.t._makeElement('details', { 'data-group': listId, 'data-loaded': false }); details.appendChild(BotMon.t._makeElement('summary', undefined, listTitle )); details.addEventListener("toggle", this._onDetailsToggle); parent.appendChild(details); } } }, _onDetailsToggle: function(e) { //console.info('BotMon.live.gui.lists._onDetailsToggle()'); const target = e.target; if (target.getAttribute('data-loaded') == 'false') { // only if not loaded yet target.setAttribute('data-loaded', 'loading'); const fillType = target.getAttribute('data-group'); const fillList = BotMon.live.data.analytics.groups[fillType]; if (fillList && fillList.length > 0) { const ul = BotMon.t._makeElement('ul'); fillList.forEach( (it) => { ul.appendChild(BotMon.live.gui.lists._makeVisitorItem(it, fillType)); }); target.appendChild(ul); target.setAttribute('data-loaded', 'true'); } else { target.setAttribute('data-loaded', 'false'); } } }, _makeVisitorItem: function(data, type) { // shortcut for neater code: const make = BotMon.t._makeElement; let ipType = ( data.ip.indexOf(':') >= 0 ? '6' : '4' ); const li = make('li'); // root list item const details = make('details'); const summary = make('summary'); details.appendChild(summary); const span1 = make('span'); /* left-hand group */ const platformName = (data._platform ? data._platform.n : 'Unknown'); const clientName = (data._client ? data._client.n: 'Unknown'); if (data._type == BM_USERTYPE.KNOWN_BOT) { /* Bot only */ span1.appendChild(make('span', { /* Bot */ 'class': 'bot bot_' + (data._bot ? data._bot.id : 'unknown'), 'title': "Bot: " + (data._bot ? data._bot.n : 'Unknown') }, (data._bot ? data._bot.n : 'Unknown'))); } else if (data._type == BM_USERTYPE.KNOWN_USER) { /* User only */ span1.appendChild(make('span', { /* User */ 'class': 'user_known', 'title': "User: " + data.usr }, data.usr)); } else { /* others */ if (data.ip == '127.0.0.1' || data.ip == '::1' ) ipType = '0'; span1.appendChild(make('span', { /* IP-Address */ 'class': 'ipaddr ip' + ipType, 'title': "IP-Address: " + data.ip }, data.ip)); } if (data._type !== BM_USERTYPE.KNOWN_BOT) { /* Not for bots */ span1.appendChild(make('span', { /* Platform */ 'class': 'icon platform platform_' + (data._platform ? data._platform.id : 'unknown'), 'title': "Platform: " + platformName }, platformName)); span1.appendChild(make('span', { /* Client */ 'class': 'icon client client_' + (data._client ? data._client.id : 'unknown'), 'title': "Client: " + clientName }, clientName)); } summary.appendChild(span1); const span2 = make('span'); /* right-hand group */ span2.appendChild(make('span', { /* page views */ 'class': 'pageviews' }, data._pageViews.length)); summary.appendChild(span2); // create expanable section: const dl = make('dl', {'class': 'visitor_details'}); if (data._type == BM_USERTYPE.KNOWN_BOT) { dl.appendChild(make('dt', {}, "Bot name:")); /* bot info */ dl.appendChild(make('dd', {'class': 'has_icon bot bot_' + (data._bot ? data._bot.id : 'unknown')}, (data._bot ? data._bot.n : 'Unknown'))); if (data._bot && data._bot.url) { dl.appendChild(make('dt', {}, "Bot info:")); /* bot info */ const botInfoDd = dl.appendChild(make('dd')); botInfoDd.appendChild(make('a', { 'href': data._bot.url, 'target': '_blank' }, data._bot.url)); /* bot info link*/ } } else { /* not for bots */ dl.appendChild(make('dt', {}, "Client:")); /* client */ dl.appendChild(make('dd', {'class': 'has_icon client_' + (data._client ? data._client.id : 'unknown')}, clientName + ( data._client.v > 0 ? ' (' + data._client.v + ')' : '' ) )); dl.appendChild(make('dt', {}, "Platform:")); /* platform */ dl.appendChild(make('dd', {'class': 'has_icon platform_' + (data._platform ? data._platform.id : 'unknown')}, platformName + ( data._platform.v > 0 ? ' (' + data._platform.v + ')' : '' ) )); } dl.appendChild(make('dt', {}, "IP-Address:")); dl.appendChild(make('dd', {'class': 'has_icon ip' + ipType}, data.ip)); dl.appendChild(make('dt', {}, "ID:")); dl.appendChild(make('dd', {'class': 'has_icon ip' + data.typ}, data.id)); if ((data._lastSeen - data._firstSeen) < 1) { dl.appendChild(make('dt', {}, "Seen:")); dl.appendChild(make('dd', {'class': 'seen'}, data._firstSeen.toLocaleString())); } else { dl.appendChild(make('dt', {}, "First seen:")); dl.appendChild(make('dd', {'class': 'firstSeen'}, data._firstSeen.toLocaleString())); dl.appendChild(make('dt', {}, "Last seen:")); dl.appendChild(make('dd', {'class': 'lastSeen'}, data._lastSeen.toLocaleString())); } dl.appendChild(make('dt', {}, "User-Agent:")); dl.appendChild(make('dd', {'class': 'agent' + ipType}, data.agent)); dl.appendChild(make('dt', {}, "Visitor Type:")); dl.appendChild(make('dd', undefined, data._type )); dl.appendChild(make('dt', {}, "Seen by:")); dl.appendChild(make('dd', undefined, data._seenBy.join(', ') )); dl.appendChild(make('dt', {}, "Visited pages:")); const pagesDd = make('dd', {'class': 'pages'}); const pageList = make('ul'); data._pageViews.forEach( (page) => { const pgLi = make('li'); let visitTimeStr = "Bounce"; const visitDuration = page._lastSeen.getTime() - page._firstSeen.getTime(); if (visitDuration > 0) { visitTimeStr = Math.floor(visitDuration / 1000) + "s"; } pgLi.appendChild(make('span', {}, page.pg)); pgLi.appendChild(make('span', {}, page.ref)); pgLi.appendChild(make('span', {}, visitTimeStr)); pageList.appendChild(pgLi); }); pagesDd.appendChild(pageList); dl.appendChild(pagesDd); details.appendChild(dl); li.appendChild(details); return li; } } } }; /* launch only if the BotMon admin panel is open: */ if (document.getElementById('botmon__admin')) { BotMon.init(); }