v0.3

Fixes #4

hippolyzer/apps/model.py

@@ -1,43 +1,15 @@
-import collections
-import codecs
-import copy
 import enum
-import fnmatch
-import io
 import logging
-import pickle
-import queue
-import re
 import typing
-import weakref
 
-from defusedxml import minidom
 from PySide2 import QtCore, QtGui
 
-from hippolyzer.lib.base import llsd
-from hippolyzer.lib.base.datatypes import *
-from hippolyzer.lib.proxy.message import ProxiedMessage
-from hippolyzer.lib.proxy.region import ProxiedRegion, CapType
-import hippolyzer.lib.base.serialization as se
-from hippolyzer.lib.proxy.http_flow import HippoHTTPFlow
-from hippolyzer.lib.proxy.sessions import Session, BaseMessageLogger
-
-from .message_filter import compile_filter, BaseFilterNode, MessageFilterNode, MetaFieldSpecifier
+from hippolyzer.lib.proxy.region import ProxiedRegion
+from hippolyzer.lib.proxy.message_logger import FilteringMessageLogger
 
 LOG = logging.getLogger(__name__)
 
 
-def bytes_unescape(val: bytes) -> bytes:
-    # Only in CPython. bytes -> bytes with escape decoding.
-    # https://stackoverflow.com/a/23151714
-    return codecs.escape_decode(val)[0]  # type: ignore
-
-
-def bytes_escape(val: bytes) -> bytes:
-    # Try to keep newlines as-is
-    return re.sub(rb"(?<!\\)\\n", b"\n", codecs.escape_encode(val)[0])  # type: ignore
-
-
 class MessageLogHeader(enum.IntEnum):
     Host = 0
     Type = enum.auto()
@@ -46,582 +18,23 @@ class MessageLogHeader(enum.IntEnum):
     Summary = enum.auto()
 
 
-class AbstractMessageLogEntry:
-    region: typing.Optional[ProxiedRegion]
-    session: typing.Optional[Session]
-    name: str
-    type: str
-
-    __slots__ = ["_region", "_session", "_region_name", "_agent_id", "_summary", "meta"]
-
-    def __init__(self, region, session):
-        if region and not isinstance(region, weakref.ReferenceType):
-            region = weakref.ref(region)
-        if session and not isinstance(session, weakref.ReferenceType):
-            session = weakref.ref(session)
-
-        self._region: typing.Optional[weakref.ReferenceType] = region
-        self._session: typing.Optional[weakref.ReferenceType] = session
-        self._region_name = None
-        self._agent_id = None
-        self._summary = None
-        if self.region:
-            self._region_name = self.region.name
-        if self.session:
-            self._agent_id = self.session.agent_id
-
-        agent_obj = None
-        if self.region is not None:
-            agent_obj = self.region.objects.lookup_fullid(self.agent_id)
-        self.meta = {
-            "RegionName": self.region_name,
-            "AgentID": self.agent_id,
-            "SessionID": self.session.id if self.session else None,
-            "AgentLocal": agent_obj.LocalID if agent_obj is not None else None,
-            "Method": self.method,
-            "Type": self.type,
-            "SelectedLocal": self._current_selected_local(),
-            "SelectedFull": self._current_selected_full(),
-        }
-
-    def freeze(self):
-        pass
-
-    def cache_summary(self):
-        self._summary = self.summary
-
-    def _current_selected_local(self):
-        if self.session:
-            return self.session.selected.object_local
-        return None
-
-    def _current_selected_full(self):
-        selected_local = self._current_selected_local()
-        if selected_local is None or self.region is None:
-            return None
-        obj = self.region.objects.lookup_localid(selected_local)
-        return obj and obj.FullID
-
-    def _get_meta(self, name: str):
-        # Slight difference in semantics. Filters are meant to return the same
-        # thing no matter when they're run, so SelectedLocal and friends resolve
-        # to the selected items _at the time the message was logged_. To handle
-        # the case where we want to match on the selected object at the time the
-        # filter is evaluated, we resolve these here.
-        if name == "CurrentSelectedLocal":
-            return self._current_selected_local()
-        elif name == "CurrentSelectedFull":
-            return self._current_selected_full()
-        return self.meta.get(name)
-
-    @property
-    def region(self) -> typing.Optional[ProxiedRegion]:
-        if self._region:
-            return self._region()
-        return None
-
-    @property
-    def session(self) -> typing.Optional[Session]:
-        if self._session:
-            return self._session()
-        return None
-
-    @property
-    def region_name(self) -> str:
-        region = self.region
-        if region:
-            self._region_name = region.name
-            return self._region_name
-        # Region may die after a message is logged, need to keep this around.
-        if self._region_name:
-            return self._region_name
-
-        return ""
-
-    @property
-    def agent_id(self) -> typing.Optional[UUID]:
-        if self._agent_id:
-            return self._agent_id
-
-        session = self.session
-        if session:
-            self._agent_id = session.agent_id
-            return self._agent_id
-        return None
-
-    @property
-    def host(self) -> str:
-        region_name = self.region_name
-        if not region_name:
-            return ""
-        session_str = ""
-        agent_id = self.agent_id
-        if agent_id:
-            session_str = f" ({agent_id})"
-        return region_name + session_str
-
-    def request(self, beautify=False, replacements=None):
-        return None
-
-    def response(self, beautify=False):
-        return None
-
-    def _packet_root_matches(self, pattern):
-        if fnmatch.fnmatchcase(self.name, pattern):
-            return True
-        if fnmatch.fnmatchcase(self.type, pattern):
-            return True
-        return False
-
-    def _val_matches(self, operator, val, expected):
-        if isinstance(expected, MetaFieldSpecifier):
-            expected = self._get_meta(str(expected))
-            if not isinstance(expected, (int, float, bytes, str, type(None), tuple)):
-                if callable(expected):
-                    expected = expected()
-                else:
-                    expected = str(expected)
-        elif expected is not None:
-            # Unbox the expected value
-            expected = expected.value
-        if not isinstance(val, (int, float, bytes, str, type(None), tuple, TupleCoord)):
-            val = str(val)
-
-        if not operator:
-            return bool(val)
-        elif operator == "==":
-            return val == expected
-        elif operator == "!=":
-            return val != expected
-        elif operator == "^=":
-            if val is None:
-                return False
-            return val.startswith(expected)
-        elif operator == "$=":
-            if val is None:
-                return False
-            return val.endswith(expected)
-        elif operator == "~=":
-            if val is None:
-                return False
-            return expected in val
-        elif operator == "<":
-            return val < expected
-        elif operator == "<=":
-            return val <= expected
-        elif operator == ">":
-            return val > expected
-        elif operator == ">=":
-            return val >= expected
-        else:
-            raise ValueError(f"Unexpected operator {operator!r}")
-
-    def _base_matches(self, matcher: "MessageFilterNode") -> typing.Optional[bool]:
-        if len(matcher.selector) == 1:
-            # Comparison operators would make no sense here
-            if matcher.value or matcher.operator:
-                return False
-            return self._packet_root_matches(matcher.selector[0])
-        if len(matcher.selector) == 2 and matcher.selector[0] == "Meta":
-            return self._val_matches(matcher.operator, self._get_meta(matcher.selector[1]), matcher.value)
-        return None
-
-    def matches(self, matcher: "MessageFilterNode"):
-        return self._base_matches(matcher) or False
-
-    @property
-    def seq(self):
-        return ""
-
-    @property
-    def method(self):
-        return ""
-
-    @property
-    def summary(self):
-        return ""
-
-    @staticmethod
-    def _format_llsd(parsed):
-        xmlified = llsd.format_pretty_xml(parsed)
-        # dedent <key> by 1 for easier visual scanning
-        xmlified = re.sub(rb" <key>", b"<key>", xmlified)
-        return xmlified.decode("utf8", errors="replace")
-
-
-class LLUDPMessageLogEntry(AbstractMessageLogEntry):
-    __slots__ = ["_message", "_name", "_direction", "_frozen_message", "_seq", "_deserializer"]
-
-    def __init__(self, message: ProxiedMessage, region, session):
-        self._message: ProxiedMessage = message
-        self._deserializer = None
-        self._name = message.name
-        self._direction = message.direction
-        self._frozen_message: typing.Optional[bytes] = None
-        self._seq = message.packet_id
-        super().__init__(region, session)
-
-    _MESSAGE_META_ATTRS = {
-        "Injected", "Dropped", "Extra", "Resent", "Zerocoded", "Acks", "Reliable",
-    }
-
-    def _get_meta(self, name: str):
-        # These may change between when the message is logged and when we
-        # actually filter on it, since logging happens before addons.
-        msg = self.message
-        if name in self._MESSAGE_META_ATTRS:
-            return getattr(msg, name.lower(), None)
-        msg_meta = getattr(msg, "meta", None)
-        if msg_meta is not None:
-            if name in msg_meta:
-                return msg_meta[name]
-        return super()._get_meta(name)
-
-    @property
-    def message(self):
-        if self._message:
-            return self._message
-        elif self._frozen_message:
-            message = pickle.loads(self._frozen_message)
-            message.deserializer = self._deserializer
-            return message
-        else:
-            raise ValueError("Didn't have a fresh or frozen message somehow")
-
-    def freeze(self):
-        self.message.invalidate_caches()
-        # These are expensive to keep around. pickle them and un-pickle on
-        # an as-needed basis.
-        self._deserializer = self.message.deserializer
-        self.message.deserializer = None
-        self._frozen_message = pickle.dumps(self._message, protocol=pickle.HIGHEST_PROTOCOL)
-        self._message = None
-
-    @property
-    def type(self):
-        return "LLUDP"
-
-    @property
-    def name(self):
-        if self._message:
-            self._name = self._message.name
-        return self._name
-
-    @property
-    def method(self):
-        if self._message:
-            self._direction = self._message.direction
-        return self._direction.name if self._direction is not None else ""
-
-    def request(self, beautify=False, replacements=None):
-        return self.message.to_human_string(replacements, beautify)
-
-    def matches(self, matcher):
-        base_matched = self._base_matches(matcher)
-        if base_matched is not None:
-            return base_matched
-
-        if not self._packet_root_matches(matcher.selector[0]):
-            return False
-
-        message = self.message
-
-        selector_len = len(matcher.selector)
-        # name, block_name, var_name(, subfield_name)?
-        if selector_len not in (3, 4):
-            return False
-        for block_name in message.blocks:
-            if not fnmatch.fnmatchcase(block_name, matcher.selector[1]):
-                continue
-            for block in message[block_name]:
-                for var_name in block.vars.keys():
-                    if not fnmatch.fnmatchcase(var_name, matcher.selector[2]):
-                        continue
-                    if selector_len == 3:
-                        if matcher.value is None:
-                            return True
-                        if self._val_matches(matcher.operator, block[var_name], matcher.value):
-                            return True
-                    elif selector_len == 4:
-                        try:
-                            deserialized = block.deserialize_var(var_name)
-                        except KeyError:
-                            continue
-                        # Discard the tag if this is a tagged union, we only want the value
-                        if isinstance(deserialized, TaggedUnion):
-                            deserialized = deserialized.value
-                        if not isinstance(deserialized, dict):
-                            return False
-                        for key in deserialized.keys():
-                            if fnmatch.fnmatchcase(str(key), matcher.selector[3]):
-                                if matcher.value is None:
-                                    return True
-                                if self._val_matches(matcher.operator, deserialized[key], matcher.value):
-                                    return True
-
-        return False
-
-    @property
-    def summary(self):
-        if self._summary is None:
-            self._summary = self.message.to_summary()[:500]
-        return self._summary
-
-    @property
-    def seq(self):
-        if self._message:
-            self._seq = self._message.packet_id
-        return self._seq
-
-
-class EQMessageLogEntry(AbstractMessageLogEntry):
-    __slots__ = ["event"]
-
-    def __init__(self, event, region, session):
-        super().__init__(region, session)
-        self.event = event
-
-    @property
-    def type(self):
-        return "EQ"
-
-    def request(self, beautify=False, replacements=None):
-        return self._format_llsd(self.event["body"])
-
-    @property
-    def name(self):
-        return self.event["message"]
-
-    @property
-    def summary(self):
-        if self._summary is not None:
-            return self._summary
-        self._summary = ""
-        self._summary = llsd.format_notation(self.event["body"]).decode("utf8")[:500]
-        return self._summary
-
-
-class HTTPMessageLogEntry(AbstractMessageLogEntry):
-    __slots__ = ["flow"]
-
-    def __init__(self, flow: HippoHTTPFlow):
-        self.flow: HippoHTTPFlow = flow
-        cap_data = self.flow.cap_data
-        region = cap_data and cap_data.region
-        session = cap_data and cap_data.session
-
-        super().__init__(region, session)
-        # This was a request the proxy made through itself
-        self.meta["Injected"] = flow.request_injected
-
-    @property
-    def type(self):
-        return "HTTP"
-
-    @property
-    def name(self):
-        cap_data = self.flow.cap_data
-        name = cap_data and cap_data.cap_name
-        if name:
-            return name
-        return self.flow.request.url
-
-    @property
-    def method(self):
-        return self.flow.request.method
-
-    def _format_http_message(self, want_request, beautify):
-        message = self.flow.request if want_request else self.flow.response
-        method = self.flow.request.method
-        buf = io.StringIO()
-        cap_data = self.flow.cap_data
-        cap_name = cap_data and cap_data.cap_name
-        base_url = cap_name and cap_data.base_url
-        temporary_cap = cap_data and cap_data.type == CapType.TEMPORARY
-        beautify_url = (beautify and base_url and cap_name and
-                        not temporary_cap and self.session and want_request)
-        if want_request:
-            buf.write(message.method)
-            buf.write(" ")
-            if beautify_url:
-                buf.write(f"[[{cap_name}]]{message.url[len(base_url):]}")
-            else:
-                buf.write(message.url)
-            buf.write(" ")
-            buf.write(message.http_version)
-        else:
-            buf.write(message.http_version)
-            buf.write(" ")
-            buf.write(str(message.status_code))
-            buf.write(" ")
-            buf.write(message.reason)
-        buf.write("\r\n")
-        if beautify_url:
-            buf.write("# ")
-            buf.write(message.url)
-            buf.write("\r\n")
-
-        headers = copy.deepcopy(message.headers)
-        for key in tuple(headers.keys()):
-            if key.lower().startswith("x-hippo-"):
-                LOG.warning(f"Internal header {key!r} leaked out?")
-                # If this header actually came from somewhere untrusted, we can't
-                # include it. It may change the meaning of the message when replayed.
-                headers[f"X-Untrusted-{key}"] = headers[key]
-                headers.pop(key)
-        beautified = None
-        if beautify and message.content:
-            try:
-                serializer = se.HTTP_SERIALIZERS.get(cap_name)
-                if serializer:
-                    if want_request:
-                        beautified = serializer.deserialize_req_body(method, message.content)
-                    else:
-                        beautified = serializer.deserialize_resp_body(method, message.content)
-
-                    if beautified is se.UNSERIALIZABLE:
-                        beautified = None
-                    else:
-                        beautified = self._format_llsd(beautified)
-                        headers["X-Hippo-Beautify"] = "1"
-
-                if not beautified:
-                    content_type = self._guess_content_type(message)
-                    if content_type.startswith("application/llsd"):
-                        beautified = self._format_llsd(llsd.parse(message.content))
-                    elif any(content_type.startswith(x) for x in ("application/xml", "text/xml")):
-                        beautified = minidom.parseString(message.content).toprettyxml(indent="  ")
-                        # kill blank lines. will break cdata sections. meh.
-                        beautified = re.sub(r'\n\s*\n', '\n', beautified, flags=re.MULTILINE)
-                        beautified = re.sub(r'<([\w]+)>\s*</\1>', r'<\1></\1>',
-                                            beautified, flags=re.MULTILINE)
-            except:
-                LOG.exception("Failed to beautify message")
-
-        message_body = beautified or message.content
-        if isinstance(message_body, bytes):
-            try:
-                decoded = message.text
-                # Valid in many codecs, but unprintable.
-                if "\x00" in decoded:
-                    raise ValueError("Embedded null")
-                message_body = decoded
-            except (UnicodeError, ValueError):
-                # non-printable characters, return the escaped version.
-                headers["X-Hippo-Escaped-Body"] = "1"
-                message_body = bytes_escape(message_body).decode("utf8")
-
-        buf.write(bytes(headers).decode("utf8", errors="replace"))
-        buf.write("\r\n")
-
-        buf.write(message_body)
-        return buf.getvalue()
-
-    def request(self, beautify=False, replacements=None):
-        return self._format_http_message(want_request=True, beautify=beautify)
-
-    def response(self, beautify=False):
-        return self._format_http_message(want_request=False, beautify=beautify)
-
-    @property
-    def summary(self):
-        if self._summary is not None:
-            return self._summary
-        msg = self.flow.response
-        self._summary = f"{msg.status_code}: "
-        if not msg.content:
-            return self._summary
-        if len(msg.content) > 1000000:
-            self._summary += "[too large...]"
-            return self._summary
-        content_type = self._guess_content_type(msg)
-        if content_type.startswith("application/llsd"):
-            notation = llsd.format_notation(llsd.parse(msg.content))
-            self._summary += notation.decode("utf8")[:500]
-        return self._summary
-
-    def _guess_content_type(self, message):
-        content_type = message.headers.get("Content-Type", "")
-        if not message.content or content_type.startswith("application/llsd"):
-            return content_type
-        # Sometimes gets sent with `text/plain` or `text/html`. Cool.
-        if message.content.startswith(rb'<?xml version="1.0" ?><llsd>'):
-            return "application/llsd+xml"
-        if message.content.startswith(rb'<llsd>'):
-            return "application/llsd+xml"
-        if message.content.startswith(rb'<?xml '):
-            return "application/xml"
-        return content_type
-
-
-class MessageLogModel(QtCore.QAbstractTableModel, BaseMessageLogger):
+class MessageLogModel(QtCore.QAbstractTableModel, FilteringMessageLogger):
     def __init__(self, parent=None):
         QtCore.QAbstractTableModel.__init__(self, parent)
-        BaseMessageLogger.__init__(self)
-        self._raw_entries = collections.deque(maxlen=2000)
-        self._queued_entries = queue.Queue()
-        self._filtered_entries = []
-        self._paused = False
-        self.filter: typing.Optional[BaseFilterNode] = None
+        FilteringMessageLogger.__init__(self)
 
-    def setFilter(self, filter_str: str):
-        self.filter = compile_filter(filter_str)
+    def _begin_insert(self, insert_idx: int):
+        self.beginInsertRows(QtCore.QModelIndex(), insert_idx, insert_idx)
+
+    def _end_insert(self):
+        self.endInsertRows()
+
+    def _begin_reset(self):
         self.beginResetModel()
-        # Keep any entries that've aged out of the raw entries list that
-        # match the new filter
-        self._filtered_entries = [
-            m for m in self._filtered_entries if
-            m not in self._raw_entries and self.filter.match(m)
-        ]
-        self._filtered_entries.extend((m for m in self._raw_entries if self.filter.match(m)))
+
+    def _end_reset(self):
         self.endResetModel()
 
-    def setPaused(self, paused: bool):
-        self._paused = paused
-
-    def log_lludp_message(self, session: Session, region: ProxiedRegion, message: ProxiedMessage):
-        if self._paused:
-            return
-        self.queueLogEntry(LLUDPMessageLogEntry(message, region, session))
-
-    def log_http_response(self, flow: HippoHTTPFlow):
-        if self._paused:
-            return
-        # These are huge, let's not log them for now.
-        if flow.cap_data and flow.cap_data.asset_server_cap:
-            return
-        self.queueLogEntry(HTTPMessageLogEntry(flow))
-
-    def log_eq_event(self, session: Session, region: ProxiedRegion, event: dict):
-        if self._paused:
-            return
-        self.queueLogEntry(EQMessageLogEntry(event, region, session))
-
-    def appendQueuedEntries(self):
-        while not self._queued_entries.empty():
-            entry: AbstractMessageLogEntry = self._queued_entries.get(block=False)
-            # Paused, throw it away.
-            if self._paused:
-                continue
-            self._raw_entries.append(entry)
-            try:
-                if self.filter.match(entry):
-                    next_idx = len(self._filtered_entries)
-                    self.beginInsertRows(QtCore.QModelIndex(), next_idx, next_idx)
-                    self._filtered_entries.append(entry)
-                    self.endInsertRows()
-
-                    entry.cache_summary()
-                # In the common case we don't need to keep around the serialization
-                # caches anymore. If the filter changes, the caches will be repopulated
-                # as necessary.
-                entry.freeze()
-            except Exception:
-                LOG.exception("Failed to filter queued message")
-
-    def queueLogEntry(self, entry: AbstractMessageLogEntry):
-        self._queued_entries.put(entry, block=False)
-
     def rowCount(self, parent=None, *args, **kwargs):
         return len(self._filtered_entries)
 
@@ -656,14 +69,6 @@ class MessageLogModel(QtCore.QAbstractTableModel, BaseMessageLogger):
         if orientation == QtCore.Qt.Horizontal and role == QtCore.Qt.DisplayRole:
             return MessageLogHeader(col).name
 
-    def clear(self):
-        self.beginResetModel()
-        self._filtered_entries.clear()
-        while not self._queued_entries.empty():
-            self._queued_entries.get(block=False)
-        self._raw_entries.clear()
-        self.endResetModel()
-
 
 class RegionListModel(QtCore.QAbstractListModel):
     def __init__(self, parent, session_manager):

hippolyzer/apps/proxy.py

@@ -144,6 +144,7 @@ def start_proxy(extra_addons: Optional[list] = None, extra_addon_paths: Optional
 
     # Everything in memory at this point should stay
     gc.freeze()
+    gc.set_threshold(5000, 50, 10)
 
     # Serve requests until Ctrl+C is pressed
     print(f"SOCKS and HTTP proxies running on {proxy_host}")

hippolyzer/apps/proxy_gui.py

@@ -20,18 +20,11 @@ import multidict
 from qasync import QEventLoop
 from PySide2 import QtCore, QtWidgets, QtGui
 
-from hippolyzer.apps.model import (
-    AbstractMessageLogEntry,
-    LLUDPMessageLogEntry,
-    MessageLogModel,
-    MessageLogHeader,
-    RegionListModel,
-    bytes_unescape,
-    bytes_escape,
-)
+from hippolyzer.apps.model import MessageLogModel, MessageLogHeader, RegionListModel
 from hippolyzer.apps.proxy import start_proxy
 from hippolyzer.lib.base import llsd
 from hippolyzer.lib.base.datatypes import UUID
+from hippolyzer.lib.base.helpers import bytes_unescape, bytes_escape
 from hippolyzer.lib.base.message.llsd_msg_serializer import LLSDMessageSerializer
 from hippolyzer.lib.base.message.message import Block
 from hippolyzer.lib.base.message.msgtypes import MsgType
@@ -44,6 +37,7 @@ from hippolyzer.lib.proxy.caps_client import CapsClient
 from hippolyzer.lib.proxy.http_proxy import create_proxy_master, HTTPFlowContext
 from hippolyzer.lib.proxy.packets import Direction
 from hippolyzer.lib.proxy.message import ProxiedMessage, VerbatimHumanVal, proxy_eval
+from hippolyzer.lib.proxy.message_logger import LLUDPMessageLogEntry, AbstractMessageLogEntry
 from hippolyzer.lib.proxy.region import ProxiedRegion
 from hippolyzer.lib.proxy.sessions import Session, SessionManager
 from hippolyzer.lib.proxy.templates import CAP_TEMPLATES
@@ -242,10 +236,10 @@ class ProxyGUI(QtWidgets.QMainWindow):
             filter_str = self.lineEditFilter.text()
         else:
             self.lineEditFilter.setText(filter_str)
-        self.model.setFilter(filter_str)
+        self.model.set_filter(filter_str)
 
     def _setPaused(self, checked):
-        self.model.setPaused(checked)
+        self.model.set_paused(checked)
 
     def _messageSelected(self, selected, _deselected):
         indexes = selected.indexes()
@@ -796,7 +790,6 @@ def gui_main():
     window = ProxyGUI()
     timer = QtCore.QTimer(app)
     timer.timeout.connect(window.sessionManager.checkRegions)
-    timer.timeout.connect(window.model.appendQueuedEntries)
     timer.start(100)
     signal.signal(signal.SIGINT, lambda *args: QtWidgets.QApplication.quit())
     window.show()

hippolyzer/lib/base/helpers.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import functools
+import re
 import weakref
 from pprint import PrettyPrinter
 from typing import *
@@ -121,3 +122,14 @@ def proxify(obj: Union[Callable[[], _T], weakref.ReferenceType, _T]) -> _T:
     if obj is not None and not isinstance(obj, weakref.ProxyTypes):
         return weakref.proxy(obj)
     return obj
+
+
+def bytes_unescape(val: bytes) -> bytes:
+    # Only in CPython. bytes -> bytes with escape decoding.
+    # https://stackoverflow.com/a/23151714
+    return codecs.escape_decode(val)[0]  # type: ignore
+
+
+def bytes_escape(val: bytes) -> bytes:
+    # Try to keep newlines as-is
+    return re.sub(rb"(?<!\\)\\n", b"\n", codecs.escape_encode(val)[0])  # type: ignore

hippolyzer/lib/base/message/message_handler.py

@@ -79,8 +79,14 @@ class MessageHandler(Generic[_T]):
 
         notifiers = self._subscribe_all(message_names, _handler_wrapper, predicate=predicate)
 
+        async def _get_wrapper():
+            msg = await msg_queue.get()
+            # Consumption is completion
+            msg_queue.task_done()
+            return msg
+
         try:
-            yield msg_queue.get
+            yield _get_wrapper
         finally:
             for n in notifiers:
                 n.unsubscribe(_handler_wrapper)

hippolyzer/lib/base/objects.py

@@ -18,108 +18,113 @@ You should have received a copy of the GNU Lesser General Public License
 along with this program; if not, write to the Free Software Foundation,
 Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 """
+from __future__ import annotations
 
 from typing import *
 
 import lazy_object_proxy
+import recordclass
 
-from hippolyzer.lib.base.datatypes import Vector3, Quaternion, Vector4
+from hippolyzer.lib.base.datatypes import Vector3, Quaternion, Vector4, UUID
 
 
-class Object:
-    """ represents an Object
+class Object(recordclass.datatuple):  # type: ignore
+    __options__ = {
+        "fast_new": False,
+        "use_weakref": True,
+    }
+    __weakref__: Any
 
-    Initialize the Object class instance
-    >>> obj = Object()
-    """
+    LocalID: Optional[int] = None
+    State: Optional[int] = None
+    FullID: Optional[UUID] = None
+    CRC: Optional[int] = None
+    PCode: Optional[int] = None
+    Material: Optional[int] = None
+    ClickAction: Optional[int] = None
+    Scale: Optional[Vector3] = None
+    ParentID: Optional[int] = None
+    # Actually contains a weakref proxy
+    Parent: Optional[Object] = None
+    UpdateFlags: Optional[int] = None
+    PathCurve: Optional[int] = None
+    ProfileCurve: Optional[int] = None
+    PathBegin: Optional[int] = None
+    PathEnd: Optional[int] = None
+    PathScaleX: Optional[int] = None
+    PathScaleY: Optional[int] = None
+    PathShearX: Optional[int] = None
+    PathShearY: Optional[int] = None
+    PathTwist: Optional[int] = None
+    PathTwistBegin: Optional[int] = None
+    PathRadiusOffset: Optional[int] = None
+    PathTaperX: Optional[int] = None
+    PathTaperY: Optional[int] = None
+    PathRevolutions: Optional[int] = None
+    PathSkew: Optional[int] = None
+    ProfileBegin: Optional[int] = None
+    ProfileEnd: Optional[int] = None
+    ProfileHollow: Optional[int] = None
+    TextureEntry: Optional[Any] = None
+    TextureAnim: Optional[Any] = None
+    NameValue: Optional[Any] = None
+    Data: Optional[Any] = None
+    Text: Optional[str] = None
+    TextColor: Optional[bytes] = None
+    MediaURL: Optional[Any] = None
+    PSBlock: Optional[Any] = None
+    ExtraParams: Optional[Any] = None
+    Sound: Optional[UUID] = None
+    OwnerID: Optional[UUID] = None
+    SoundGain: Optional[float] = None
+    SoundFlags: Optional[int] = None
+    SoundRadius: Optional[float] = None
+    JointType: Optional[int] = None
+    JointPivot: Optional[int] = None
+    JointAxisOrAnchor: Optional[int] = None
+    TreeSpecies: Optional[int] = None
+    ScratchPad: Optional[bytes] = None
+    ObjectCosts: Optional[Dict] = None
+    ChildIDs: Optional[List[int]] = None
+    # Same as parent, contains weakref proxies.
+    Children: Optional[List[Object]] = None
 
-    __slots__ = (
-        "LocalID",
-        "State",
-        "FullID",
-        "CRC",
-        "PCode",
-        "Material",
-        "ClickAction",
-        "Scale",
-        "ParentID",
-        "UpdateFlags",
-        "PathCurve",
-        "ProfileCurve",
-        "PathBegin",
-        "PathEnd",
-        "PathScaleX",
-        "PathScaleY",
-        "PathShearX",
-        "PathShearY",
-        "PathTwist",
-        "PathTwistBegin",
-        "PathRadiusOffset",
-        "PathTaperX",
-        "PathTaperY",
-        "PathRevolutions",
-        "PathSkew",
-        "ProfileBegin",
-        "ProfileEnd",
-        "ProfileHollow",
-        "TextureEntry",
-        "TextureAnim",
-        "NameValue",
-        "Data",
-        "Text",
-        "TextColor",
-        "MediaURL",
-        "PSBlock",
-        "ExtraParams",
-        "Sound",
-        "OwnerID",
-        "SoundGain",
-        "SoundFlags",
-        "SoundRadius",
-        "JointType",
-        "JointPivot",
-        "JointAxisOrAnchor",
-        "TreeSpecies",
-        "ObjectCosts",
-        "FootCollisionPlane",
-        "Position",
-        "Velocity",
-        "Acceleration",
-        "Rotation",
-        "AngularVelocity",
-        "CreatorID",
-        "GroupID",
-        "CreationDate",
-        "BaseMask",
-        "OwnerMask",
-        "GroupMask",
-        "EveryoneMask",
-        "NextOwnerMask",
-        "OwnershipCost",
-        "SaleType",
-        "SalePrice",
-        "AggregatePerms",
-        "AggregatePermTextures",
-        "AggregatePermTexturesOwner",
-        "Category",
-        "InventorySerial",
-        "ItemID",
-        "FolderID",
-        "FromTaskID",
-        "LastOwnerID",
-        "Name",
-        "Description",
-        "TouchName",
-        "SitName",
-        "TextureID",
-        "ChildIDs",
-        "Children",
-        "Parent",
-        "ScratchPad",
-        "__weakref__",
-    )
+    FootCollisionPlane: Optional[Vector4] = None
+    Position: Optional[Vector3] = None
+    Velocity: Optional[Vector3] = None
+    Acceleration: Optional[Vector3] = None
+    Rotation: Optional[Quaternion] = None
+    AngularVelocity: Optional[Vector3] = None
 
-    def __init__(self, *, ID=None, LocalID=None, State=None, FullID=None, CRC=None, PCode=None, Material=None,
+    # from ObjectProperties
+    CreatorID: Optional[UUID] = None
+    GroupID: Optional[UUID] = None
+    CreationDate: Optional[int] = None
+    BaseMask: Optional[int] = None
+    OwnerMask: Optional[int] = None
+    GroupMask: Optional[int] = None
+    EveryoneMask: Optional[int] = None
+    NextOwnerMask: Optional[int] = None
+    OwnershipCost: Optional[int] = None
+    # TaxRate
+    SaleType: Optional[int] = None
+    SalePrice: Optional[int] = None
+    AggregatePerms: Optional[int] = None
+    AggregatePermTextures: Optional[int] = None
+    AggregatePermTexturesOwner: Optional[int] = None
+    Category: Optional[int] = None
+    InventorySerial: Optional[int] = None
+    ItemID: Optional[UUID] = None
+    FolderID: Optional[UUID] = None
+    FromTaskID: Optional[UUID] = None
+    LastOwnerID: Optional[UUID] = None
+    Name: Optional[str] = None
+    Description: Optional[str] = None
+    TouchName: Optional[str] = None
+    SitName: Optional[str] = None
+    TextureID: Optional[Any] = None
+
+    def __init__(self, *, LocalID=None, State=None, FullID=None, CRC=None, PCode=None, Material=None,
                  ClickAction=None, Scale=None, ParentID=None, UpdateFlags=None, PathCurve=None, ProfileCurve=None,
                  PathBegin=None, PathEnd=None, PathScaleX=None, PathScaleY=None, PathShearX=None, PathShearY=None,
                  PathTwist=None, PathTwistBegin=None, PathRadiusOffset=None, PathTaperX=None, PathTaperY=None,
@@ -131,7 +136,7 @@ class Object:
                  AngularVelocity=None, TreeSpecies=None, ObjectCosts=None, ScratchPad=None):
         """ set up the object attributes """
 
-        self.LocalID = LocalID or ID  # U32
+        self.LocalID = LocalID  # U32
         self.State = State  # U8
         self.FullID = FullID  # LLUUID
         self.CRC = CRC  # U32 // TEMPORARY HACK FOR JAMES
@@ -258,8 +263,4 @@ class Object:
         return updated_properties
 
     def to_dict(self):
-        return {
-            x: getattr(self, x) for x in dir(self)
-            if not isinstance(getattr(self.__class__, x, None), property) and
-            not callable(getattr(self, x)) and not x.startswith("_")
-        }
+        return recordclass.asdict(self)

hippolyzer/lib/proxy/circuit.py

@@ -14,7 +14,7 @@ from hippolyzer.lib.proxy.message import ProxiedMessage
 
 if TYPE_CHECKING:
     from hippolyzer.lib.proxy.region import ProxiedRegion
-    from hippolyzer.lib.proxy.sessions import BaseMessageLogger
+    from hippolyzer.lib.proxy.message_logger import BaseMessageLogger
 
 
 class ProxiedCircuit:

hippolyzer/lib/proxy/message_filter.py

@@ -3,28 +3,27 @@ import ast
 import typing
 
 from arpeggio import Optional, ZeroOrMore, EOF, \
-    ParserPython, PTNodeVisitor, visit_parse_tree
-from arpeggio import RegExMatch as _
+    ParserPython, PTNodeVisitor, visit_parse_tree, RegExMatch
 
 
 def literal():
     return [
         # Nightmare. str or bytes literal.
         # https://stackoverflow.com/questions/14366401/#comment79795017_14366904
-        _(r'''b?(\"\"\"|\'\'\'|\"|\')((?<!\\)(\\\\)*\\\1|.)*?\1'''),
-        _(r'\d+(\.\d+)?'),
+        RegExMatch(r'''b?(\"\"\"|\'\'\'|\"|\')((?<!\\)(\\\\)*\\\1|.)*?\1'''),
+        RegExMatch(r'\d+(\.\d+)?'),
         "None",
         "True",
         "False",
         # vector3 (tuple)
-        _(r'\(\s*\d+(\.\d+)?\s*,\s*\d+(\.\d+)?\s*,\s*\d+(\.\d+)?\s*\)'),
+        RegExMatch(r'\(\s*\d+(\.\d+)?\s*,\s*\d+(\.\d+)?\s*,\s*\d+(\.\d+)?\s*\)'),
         # vector4 (tuple)
-        _(r'\(\s*\d+(\.\d+)?\s*,\s*\d+(\.\d+)?\s*,\s*\d+(\.\d+)?\s*,\s*\d+(\.\d+)?\s*\)'),
+        RegExMatch(r'\(\s*\d+(\.\d+)?\s*,\s*\d+(\.\d+)?\s*,\s*\d+(\.\d+)?\s*,\s*\d+(\.\d+)?\s*\)'),
     ]
 
 
 def identifier():
-    return _(r'[a-zA-Z*]([a-zA-Z0-9*]+)?')
+    return RegExMatch(r'[a-zA-Z*]([a-zA-Z0-9*]+)?')
 
 
 def field_specifier():
@@ -134,23 +133,23 @@ class LiteralValue:
 
 
 class MessageFilterVisitor(PTNodeVisitor):
-    def visit_identifier(self, node, children):
+    def visit_identifier(self, node, _children):
         return str(node.value)
 
-    def visit_field_specifier(self, node, children):
+    def visit_field_specifier(self, _node, children):
         return children
 
-    def visit_literal(self, node, children):
+    def visit_literal(self, node, _children):
         return LiteralValue(ast.literal_eval(node.value))
 
-    def visit_meta_field_specifier(self, node, children):
+    def visit_meta_field_specifier(self, _node, children):
         return MetaFieldSpecifier(children[0])
 
-    def visit_unary_field_specifier(self, node, children):
+    def visit_unary_field_specifier(self, _node, children):
         # Looks like a bare field specifier with no operator
         return MessageFilterNode(tuple(children), None, None)
 
-    def visit_unary_expression(self, node, children):
+    def visit_unary_expression(self, _node, children):
         if len(children) == 1:
             if isinstance(children[0], BaseFilterNode):
                 return children[0]
@@ -162,10 +161,10 @@ class MessageFilterVisitor(PTNodeVisitor):
         else:
             raise ValueError(f"Unrecognized unary prefix {children[0]}")
 
-    def visit_binary_expression(self, node, children):
+    def visit_binary_expression(self, _node, children):
         return MessageFilterNode(tuple(children[0]), children[1], children[2])
 
-    def visit_expression(self, node, children):
+    def visit_expression(self, _node, children):
         if self.debug:
             print("Expression {}".format(children))
         if len(children) > 1:

hippolyzer/lib/proxy/message_logger.py

@@ -0,0 +1,626 @@
+from __future__ import annotations
+
+import collections
+import copy
+import fnmatch
+import io
+import logging
+import pickle
+import re
+import typing
+import weakref
+
+from defusedxml import minidom
+
+from hippolyzer.lib.base import serialization as se, llsd
+from hippolyzer.lib.base.datatypes import TaggedUnion, UUID, TupleCoord
+from hippolyzer.lib.base.helpers import bytes_escape
+from hippolyzer.lib.proxy.message_filter import MetaFieldSpecifier, compile_filter, BaseFilterNode, MessageFilterNode
+
+if typing.TYPE_CHECKING:
+    from hippolyzer.lib.proxy.http_flow import HippoHTTPFlow
+    from hippolyzer.lib.proxy.message import ProxiedMessage
+    from hippolyzer.lib.proxy.region import ProxiedRegion, CapType
+    from hippolyzer.lib.proxy.sessions import Session
+
+LOG = logging.getLogger(__name__)
+
+
+class BaseMessageLogger:
+    def log_lludp_message(self, session: Session, region: ProxiedRegion, message: ProxiedMessage):
+        pass
+
+    def log_http_response(self, flow: HippoHTTPFlow):
+        pass
+
+    def log_eq_event(self, session: Session, region: ProxiedRegion, event: dict):
+        pass
+
+
+class FilteringMessageLogger(BaseMessageLogger):
+    def __init__(self):
+        BaseMessageLogger.__init__(self)
+        self._raw_entries = collections.deque(maxlen=2000)
+        self._filtered_entries: typing.List[AbstractMessageLogEntry] = []
+        self._paused = False
+        self.filter: BaseFilterNode = compile_filter("")
+
+    def set_filter(self, filter_str: str):
+        self.filter = compile_filter(filter_str)
+        self._begin_reset()
+        # Keep any entries that've aged out of the raw entries list that
+        # match the new filter
+        self._filtered_entries = [
+            m for m in self._filtered_entries if
+            m not in self._raw_entries and self.filter.match(m)
+        ]
+        self._filtered_entries.extend((m for m in self._raw_entries if self.filter.match(m)))
+        self._end_reset()
+
+    def set_paused(self, paused: bool):
+        self._paused = paused
+
+    def log_lludp_message(self, session: Session, region: ProxiedRegion, message: ProxiedMessage):
+        if self._paused:
+            return
+        self._add_log_entry(LLUDPMessageLogEntry(message, region, session))
+
+    def log_http_response(self, flow: HippoHTTPFlow):
+        if self._paused:
+            return
+        # These are huge, let's not log them for now.
+        if flow.cap_data and flow.cap_data.asset_server_cap:
+            return
+        self._add_log_entry(HTTPMessageLogEntry(flow))
+
+    def log_eq_event(self, session: Session, region: ProxiedRegion, event: dict):
+        if self._paused:
+            return
+        self._add_log_entry(EQMessageLogEntry(event, region, session))
+
+    # Hooks that Qt models will want to implement
+    def _begin_insert(self, insert_idx: int):
+        pass
+
+    def _end_insert(self):
+        pass
+
+    def _begin_reset(self):
+        pass
+
+    def _end_reset(self):
+        pass
+
+    def _add_log_entry(self, entry: AbstractMessageLogEntry):
+        try:
+            # Paused, throw it away.
+            if self._paused:
+                return
+            self._raw_entries.append(entry)
+            if self.filter.match(entry):
+                next_idx = len(self._filtered_entries)
+                self._begin_insert(next_idx)
+                self._filtered_entries.append(entry)
+                self._end_insert()
+
+                entry.cache_summary()
+            # In the common case we don't need to keep around the serialization
+            # caches anymore. If the filter changes, the caches will be repopulated
+            # as necessary.
+            entry.freeze()
+        except Exception:
+            LOG.exception("Failed to filter queued message")
+
+    def clear(self):
+        self._begin_reset()
+        self._filtered_entries.clear()
+        self._raw_entries.clear()
+        self._end_reset()
+
+
+class AbstractMessageLogEntry:
+    region: typing.Optional[ProxiedRegion]
+    session: typing.Optional[Session]
+    name: str
+    type: str
+
+    __slots__ = ["_region", "_session", "_region_name", "_agent_id", "_summary", "meta"]
+
+    def __init__(self, region, session):
+        if region and not isinstance(region, weakref.ReferenceType):
+            region = weakref.ref(region)
+        if session and not isinstance(session, weakref.ReferenceType):
+            session = weakref.ref(session)
+
+        self._region: typing.Optional[weakref.ReferenceType] = region
+        self._session: typing.Optional[weakref.ReferenceType] = session
+        self._region_name = None
+        self._agent_id = None
+        self._summary = None
+        if self.region:
+            self._region_name = self.region.name
+        if self.session:
+            self._agent_id = self.session.agent_id
+
+        agent_obj = None
+        if self.region is not None:
+            agent_obj = self.region.objects.lookup_fullid(self.agent_id)
+        self.meta = {
+            "RegionName": self.region_name,
+            "AgentID": self.agent_id,
+            "SessionID": self.session.id if self.session else None,
+            "AgentLocal": agent_obj.LocalID if agent_obj is not None else None,
+            "Method": self.method,
+            "Type": self.type,
+            "SelectedLocal": self._current_selected_local(),
+            "SelectedFull": self._current_selected_full(),
+        }
+
+    def freeze(self):
+        pass
+
+    def cache_summary(self):
+        self._summary = self.summary
+
+    def _current_selected_local(self):
+        if self.session:
+            return self.session.selected.object_local
+        return None
+
+    def _current_selected_full(self):
+        selected_local = self._current_selected_local()
+        if selected_local is None or self.region is None:
+            return None
+        obj = self.region.objects.lookup_localid(selected_local)
+        return obj and obj.FullID
+
+    def _get_meta(self, name: str):
+        # Slight difference in semantics. Filters are meant to return the same
+        # thing no matter when they're run, so SelectedLocal and friends resolve
+        # to the selected items _at the time the message was logged_. To handle
+        # the case where we want to match on the selected object at the time the
+        # filter is evaluated, we resolve these here.
+        if name == "CurrentSelectedLocal":
+            return self._current_selected_local()
+        elif name == "CurrentSelectedFull":
+            return self._current_selected_full()
+        return self.meta.get(name)
+
+    @property
+    def region(self) -> typing.Optional[ProxiedRegion]:
+        if self._region:
+            return self._region()
+        return None
+
+    @property
+    def session(self) -> typing.Optional[Session]:
+        if self._session:
+            return self._session()
+        return None
+
+    @property
+    def region_name(self) -> str:
+        region = self.region
+        if region:
+            self._region_name = region.name
+            return self._region_name
+        # Region may die after a message is logged, need to keep this around.
+        if self._region_name:
+            return self._region_name
+
+        return ""
+
+    @property
+    def agent_id(self) -> typing.Optional[UUID]:
+        if self._agent_id:
+            return self._agent_id
+
+        session = self.session
+        if session:
+            self._agent_id = session.agent_id
+            return self._agent_id
+        return None
+
+    @property
+    def host(self) -> str:
+        region_name = self.region_name
+        if not region_name:
+            return ""
+        session_str = ""
+        agent_id = self.agent_id
+        if agent_id:
+            session_str = f" ({agent_id})"
+        return region_name + session_str
+
+    def request(self, beautify=False, replacements=None):
+        return None
+
+    def response(self, beautify=False):
+        return None
+
+    def _packet_root_matches(self, pattern):
+        if fnmatch.fnmatchcase(self.name, pattern):
+            return True
+        if fnmatch.fnmatchcase(self.type, pattern):
+            return True
+        return False
+
+    def _val_matches(self, operator, val, expected):
+        if isinstance(expected, MetaFieldSpecifier):
+            expected = self._get_meta(str(expected))
+            if not isinstance(expected, (int, float, bytes, str, type(None), tuple)):
+                if callable(expected):
+                    expected = expected()
+                else:
+                    expected = str(expected)
+        elif expected is not None:
+            # Unbox the expected value
+            expected = expected.value
+        if not isinstance(val, (int, float, bytes, str, type(None), tuple, TupleCoord)):
+            val = str(val)
+
+        if not operator:
+            return bool(val)
+        elif operator == "==":
+            return val == expected
+        elif operator == "!=":
+            return val != expected
+        elif operator == "^=":
+            if val is None:
+                return False
+            return val.startswith(expected)
+        elif operator == "$=":
+            if val is None:
+                return False
+            return val.endswith(expected)
+        elif operator == "~=":
+            if val is None:
+                return False
+            return expected in val
+        elif operator == "<":
+            return val < expected
+        elif operator == "<=":
+            return val <= expected
+        elif operator == ">":
+            return val > expected
+        elif operator == ">=":
+            return val >= expected
+        else:
+            raise ValueError(f"Unexpected operator {operator!r}")
+
+    def _base_matches(self, matcher: "MessageFilterNode") -> typing.Optional[bool]:
+        if len(matcher.selector) == 1:
+            # Comparison operators would make no sense here
+            if matcher.value or matcher.operator:
+                return False
+            return self._packet_root_matches(matcher.selector[0])
+        if len(matcher.selector) == 2 and matcher.selector[0] == "Meta":
+            return self._val_matches(matcher.operator, self._get_meta(matcher.selector[1]), matcher.value)
+        return None
+
+    def matches(self, matcher: "MessageFilterNode"):
+        return self._base_matches(matcher) or False
+
+    @property
+    def seq(self):
+        return ""
+
+    @property
+    def method(self):
+        return ""
+
+    @property
+    def summary(self):
+        return ""
+
+    @staticmethod
+    def _format_llsd(parsed):
+        xmlified = llsd.format_pretty_xml(parsed)
+        # dedent <key> by 1 for easier visual scanning
+        xmlified = re.sub(rb" <key>", b"<key>", xmlified)
+        return xmlified.decode("utf8", errors="replace")
+
+
+class HTTPMessageLogEntry(AbstractMessageLogEntry):
+    __slots__ = ["flow"]
+
+    def __init__(self, flow: HippoHTTPFlow):
+        self.flow: HippoHTTPFlow = flow
+        cap_data = self.flow.cap_data
+        region = cap_data and cap_data.region
+        session = cap_data and cap_data.session
+
+        super().__init__(region, session)
+        # This was a request the proxy made through itself
+        self.meta["Injected"] = flow.request_injected
+
+    @property
+    def type(self):
+        return "HTTP"
+
+    @property
+    def name(self):
+        cap_data = self.flow.cap_data
+        name = cap_data and cap_data.cap_name
+        if name:
+            return name
+        return self.flow.request.url
+
+    @property
+    def method(self):
+        return self.flow.request.method
+
+    def _format_http_message(self, want_request, beautify):
+        message = self.flow.request if want_request else self.flow.response
+        method = self.flow.request.method
+        buf = io.StringIO()
+        cap_data = self.flow.cap_data
+        cap_name = cap_data and cap_data.cap_name
+        base_url = cap_name and cap_data.base_url
+        temporary_cap = cap_data and cap_data.type == CapType.TEMPORARY
+        beautify_url = (beautify and base_url and cap_name and
+                        not temporary_cap and self.session and want_request)
+        if want_request:
+            buf.write(message.method)
+            buf.write(" ")
+            if beautify_url:
+                buf.write(f"[[{cap_name}]]{message.url[len(base_url):]}")
+            else:
+                buf.write(message.url)
+            buf.write(" ")
+            buf.write(message.http_version)
+        else:
+            buf.write(message.http_version)
+            buf.write(" ")
+            buf.write(str(message.status_code))
+            buf.write(" ")
+            buf.write(message.reason)
+        buf.write("\r\n")
+        if beautify_url:
+            buf.write("# ")
+            buf.write(message.url)
+            buf.write("\r\n")
+
+        headers = copy.deepcopy(message.headers)
+        for key in tuple(headers.keys()):
+            if key.lower().startswith("x-hippo-"):
+                LOG.warning(f"Internal header {key!r} leaked out?")
+                # If this header actually came from somewhere untrusted, we can't
+                # include it. It may change the meaning of the message when replayed.
+                headers[f"X-Untrusted-{key}"] = headers[key]
+                headers.pop(key)
+        beautified = None
+        if beautify and message.content:
+            try:
+                serializer = se.HTTP_SERIALIZERS.get(cap_name)
+                if serializer:
+                    if want_request:
+                        beautified = serializer.deserialize_req_body(method, message.content)
+                    else:
+                        beautified = serializer.deserialize_resp_body(method, message.content)
+
+                    if beautified is se.UNSERIALIZABLE:
+                        beautified = None
+                    else:
+                        beautified = self._format_llsd(beautified)
+                        headers["X-Hippo-Beautify"] = "1"
+
+                if not beautified:
+                    content_type = self._guess_content_type(message)
+                    if content_type.startswith("application/llsd"):
+                        beautified = self._format_llsd(llsd.parse(message.content))
+                    elif any(content_type.startswith(x) for x in ("application/xml", "text/xml")):
+                        beautified = minidom.parseString(message.content).toprettyxml(indent="  ")
+                        # kill blank lines. will break cdata sections. meh.
+                        beautified = re.sub(r'\n\s*\n', '\n', beautified, flags=re.MULTILINE)
+                        beautified = re.sub(r'<([\w]+)>\s*</\1>', r'<\1></\1>',
+                                            beautified, flags=re.MULTILINE)
+            except:
+                LOG.exception("Failed to beautify message")
+
+        message_body = beautified or message.content
+        if isinstance(message_body, bytes):
+            try:
+                decoded = message.text
+                # Valid in many codecs, but unprintable.
+                if "\x00" in decoded:
+                    raise ValueError("Embedded null")
+                message_body = decoded
+            except (UnicodeError, ValueError):
+                # non-printable characters, return the escaped version.
+                headers["X-Hippo-Escaped-Body"] = "1"
+                message_body = bytes_escape(message_body).decode("utf8")
+
+        buf.write(bytes(headers).decode("utf8", errors="replace"))
+        buf.write("\r\n")
+
+        buf.write(message_body)
+        return buf.getvalue()
+
+    def request(self, beautify=False, replacements=None):
+        return self._format_http_message(want_request=True, beautify=beautify)
+
+    def response(self, beautify=False):
+        return self._format_http_message(want_request=False, beautify=beautify)
+
+    @property
+    def summary(self):
+        if self._summary is not None:
+            return self._summary
+        msg = self.flow.response
+        self._summary = f"{msg.status_code}: "
+        if not msg.content:
+            return self._summary
+        if len(msg.content) > 1000000:
+            self._summary += "[too large...]"
+            return self._summary
+        content_type = self._guess_content_type(msg)
+        if content_type.startswith("application/llsd"):
+            notation = llsd.format_notation(llsd.parse(msg.content))
+            self._summary += notation.decode("utf8")[:500]
+        return self._summary
+
+    def _guess_content_type(self, message):
+        content_type = message.headers.get("Content-Type", "")
+        if not message.content or content_type.startswith("application/llsd"):
+            return content_type
+        # Sometimes gets sent with `text/plain` or `text/html`. Cool.
+        if message.content.startswith(rb'<?xml version="1.0" ?><llsd>'):
+            return "application/llsd+xml"
+        if message.content.startswith(rb'<llsd>'):
+            return "application/llsd+xml"
+        if message.content.startswith(rb'<?xml '):
+            return "application/xml"
+        return content_type
+
+
+class EQMessageLogEntry(AbstractMessageLogEntry):
+    __slots__ = ["event"]
+
+    def __init__(self, event, region, session):
+        super().__init__(region, session)
+        self.event = event
+
+    @property
+    def type(self):
+        return "EQ"
+
+    def request(self, beautify=False, replacements=None):
+        return self._format_llsd(self.event["body"])
+
+    @property
+    def name(self):
+        return self.event["message"]
+
+    @property
+    def summary(self):
+        if self._summary is not None:
+            return self._summary
+        self._summary = ""
+        self._summary = llsd.format_notation(self.event["body"]).decode("utf8")[:500]
+        return self._summary
+
+
+class LLUDPMessageLogEntry(AbstractMessageLogEntry):
+    __slots__ = ["_message", "_name", "_direction", "_frozen_message", "_seq", "_deserializer"]
+
+    def __init__(self, message: ProxiedMessage, region, session):
+        self._message: ProxiedMessage = message
+        self._deserializer = None
+        self._name = message.name
+        self._direction = message.direction
+        self._frozen_message: typing.Optional[bytes] = None
+        self._seq = message.packet_id
+        super().__init__(region, session)
+
+    _MESSAGE_META_ATTRS = {
+        "Injected", "Dropped", "Extra", "Resent", "Zerocoded", "Acks", "Reliable",
+    }
+
+    def _get_meta(self, name: str):
+        # These may change between when the message is logged and when we
+        # actually filter on it, since logging happens before addons.
+        msg = self.message
+        if name in self._MESSAGE_META_ATTRS:
+            return getattr(msg, name.lower(), None)
+        msg_meta = getattr(msg, "meta", None)
+        if msg_meta is not None:
+            if name in msg_meta:
+                return msg_meta[name]
+        return super()._get_meta(name)
+
+    @property
+    def message(self):
+        if self._message:
+            return self._message
+        elif self._frozen_message:
+            message = pickle.loads(self._frozen_message)
+            message.deserializer = self._deserializer
+            return message
+        else:
+            raise ValueError("Didn't have a fresh or frozen message somehow")
+
+    def freeze(self):
+        self.message.invalidate_caches()
+        # These are expensive to keep around. pickle them and un-pickle on
+        # an as-needed basis.
+        self._deserializer = self.message.deserializer
+        self.message.deserializer = None
+        self._frozen_message = pickle.dumps(self._message, protocol=pickle.HIGHEST_PROTOCOL)
+        self._message = None
+
+    @property
+    def type(self):
+        return "LLUDP"
+
+    @property
+    def name(self):
+        if self._message:
+            self._name = self._message.name
+        return self._name
+
+    @property
+    def method(self):
+        if self._message:
+            self._direction = self._message.direction
+        return self._direction.name if self._direction is not None else ""
+
+    def request(self, beautify=False, replacements=None):
+        return self.message.to_human_string(replacements, beautify)
+
+    def matches(self, matcher):
+        base_matched = self._base_matches(matcher)
+        if base_matched is not None:
+            return base_matched
+
+        if not self._packet_root_matches(matcher.selector[0]):
+            return False
+
+        message = self.message
+
+        selector_len = len(matcher.selector)
+        # name, block_name, var_name(, subfield_name)?
+        if selector_len not in (3, 4):
+            return False
+        for block_name in message.blocks:
+            if not fnmatch.fnmatchcase(block_name, matcher.selector[1]):
+                continue
+            for block in message[block_name]:
+                for var_name in block.vars.keys():
+                    if not fnmatch.fnmatchcase(var_name, matcher.selector[2]):
+                        continue
+                    if selector_len == 3:
+                        if matcher.value is None:
+                            return True
+                        if self._val_matches(matcher.operator, block[var_name], matcher.value):
+                            return True
+                    elif selector_len == 4:
+                        try:
+                            deserialized = block.deserialize_var(var_name)
+                        except KeyError:
+                            continue
+                        # Discard the tag if this is a tagged union, we only want the value
+                        if isinstance(deserialized, TaggedUnion):
+                            deserialized = deserialized.value
+                        if not isinstance(deserialized, dict):
+                            return False
+                        for key in deserialized.keys():
+                            if fnmatch.fnmatchcase(str(key), matcher.selector[3]):
+                                if matcher.value is None:
+                                    return True
+                                if self._val_matches(matcher.operator, deserialized[key], matcher.value):
+                                    return True
+
+        return False
+
+    @property
+    def summary(self):
+        if self._summary is None:
+            self._summary = self.message.to_summary()[:500]
+        return self._summary
+
+    @property
+    def seq(self):
+        if self._message:
+            self._seq = self._message.packet_id
+        return self._seq

hippolyzer/lib/proxy/objects.py

@@ -192,6 +192,7 @@ class ObjectManager:
             "State": block.deserialize_var("State", make_copy=False),
             **block.deserialize_var("ObjectData", make_copy=False).value,
         }
+        object_data["LocalID"] = object_data.pop("ID")
         # Empty == not updated
         if not object_data["TextureEntry"]:
             object_data.pop("TextureEntry")
@@ -211,7 +212,7 @@ class ObjectManager:
         for block in packet['ObjectData']:
             object_data = self._normalize_object_update(block)
 
-            seen_locals.append(object_data["ID"])
+            seen_locals.append(object_data["LocalID"])
             obj = self.lookup_fullid(object_data["FullID"])
             if obj:
                 self._update_existing_object(obj, object_data)
@@ -226,6 +227,7 @@ class ObjectManager:
             **dict(block.items()),
             "TextureEntry": block.deserialize_var("TextureEntry", make_copy=False),
         }
+        object_data["LocalID"] = object_data.pop("ID")
         object_data.pop("Data")
         # Empty == not updated
         if object_data["TextureEntry"] is None:
@@ -236,19 +238,19 @@ class ObjectManager:
         seen_locals = []
         for block in packet['ObjectData']:
             object_data = self._normalize_terse_object_update(block)
-            obj = self.lookup_localid(object_data["ID"])
+            obj = self.lookup_localid(object_data["LocalID"])
             # Can only update existing object with this message
             if obj:
                 # Need the Object as context because decoding state requires PCode.
                 state_deserializer = ObjectStateSerializer.deserialize
                 object_data["State"] = state_deserializer(ctx_obj=obj, val=object_data["State"])
 
-            seen_locals.append(object_data["ID"])
+            seen_locals.append(object_data["LocalID"])
             if obj:
                 self._update_existing_object(obj, object_data)
             else:
-                self.missing_locals.add(object_data["ID"])
-                LOG.debug(f"Received terse update for unknown object {object_data['ID']}")
+                self.missing_locals.add(object_data["LocalID"])
+                LOG.debug(f"Received terse update for unknown object {object_data['LocalID']}")
 
         packet.meta["ObjectUpdateIDs"] = tuple(seen_locals)
 
@@ -288,6 +290,7 @@ class ObjectManager:
             "PSBlock": ps_block.value,
             # Parent flag not set means explicitly un-parented
             "ParentID": compressed.pop("ParentID", None) or 0,
+            "LocalID": compressed.pop("ID"),
             **compressed,
             **dict(block.items()),
             "UpdateFlags": block.deserialize_var("UpdateFlags", make_copy=False),
@@ -304,8 +307,8 @@ class ObjectManager:
         seen_locals = []
         for block in packet['ObjectData']:
             object_data = self._normalize_object_update_compressed(block)
-            obj = self.lookup_localid(object_data["ID"])
-            seen_locals.append(object_data["ID"])
+            obj = self.lookup_localid(object_data["LocalID"])
+            seen_locals.append(object_data["LocalID"])
             if obj:
                 self._update_existing_object(obj, object_data)
             else:

hippolyzer/lib/proxy/sessions.py

@@ -12,12 +12,9 @@ from hippolyzer.lib.base.datatypes import UUID
 from hippolyzer.lib.proxy.circuit import ProxiedCircuit
 from hippolyzer.lib.proxy.http_asset_repo import HTTPAssetRepo
 from hippolyzer.lib.proxy.http_proxy import HTTPFlowContext, is_asset_server_cap_name, SerializedCapData
+from hippolyzer.lib.proxy.message_logger import BaseMessageLogger
 from hippolyzer.lib.proxy.region import ProxiedRegion, CapType
 
-if TYPE_CHECKING:
-    from hippolyzer.lib.proxy.http_flow import HippoHTTPFlow
-    from hippolyzer.lib.proxy.message import ProxiedMessage
-
 
 class Session:
     def __init__(self, session_id, secure_session_id, agent_id, circuit_code,
@@ -144,17 +141,6 @@ class Session:
         return "<%s %s>" % (self.__class__.__name__, self.id)
 
 
-class BaseMessageLogger:
-    def log_lludp_message(self, session: Session, region: ProxiedRegion, message: ProxiedMessage):
-        pass
-
-    def log_http_response(self, flow: HippoHTTPFlow):
-        pass
-
-    def log_eq_event(self, session: Session, region: ProxiedRegion, event: dict):
-        pass
-
-
 class SessionManager:
     def __init__(self):
         self.sessions: List[Session] = []

setup.py

@@ -25,7 +25,7 @@ from setuptools import setup, find_packages
 
 here = path.abspath(path.dirname(__file__))
 
-version = '0.2.1'
+version = '0.3'
 
 with open(path.join(here, 'README.md')) as readme_fh:
     readme = readme_fh.read()

tests/proxy/integration/test_lludp.py

@@ -13,6 +13,7 @@ from hippolyzer.lib.base.objects import Object
 from hippolyzer.lib.proxy.addon_utils import BaseAddon
 from hippolyzer.lib.proxy.addons import AddonManager
 from hippolyzer.lib.proxy.message import ProxiedMessage
+from hippolyzer.lib.proxy.message_logger import FilteringMessageLogger
 from hippolyzer.lib.proxy.packets import ProxiedUDPPacket, Direction
 from hippolyzer.lib.proxy.region import ProxiedRegion
 from hippolyzer.lib.proxy.sessions import Session
@@ -35,6 +36,12 @@ class MockAddon(BaseAddon):
         self.events.append(("object_update", session.id, region.circuit_addr, obj.LocalID))
 
 
+class SimpleMessageLogger(FilteringMessageLogger):
+    @property
+    def entries(self):
+        return self._filtered_entries
+
+
 class LLUDPIntegrationTests(BaseIntegrationTest):
     def setUp(self) -> None:
         super().setUp()
@@ -169,3 +176,14 @@ class LLUDPIntegrationTests(BaseIntegrationTest):
         obj = self.session.regions[0].objects.lookup_localid(1234)
         self.assertIsInstance(obj.TextureEntry, lazy_object_proxy.Proxy)
         self.assertEqual(obj.TextureEntry.Textures[None], UUID("89556747-24cb-43ed-920b-47caed15465f"))
+
+    async def test_message_logger(self):
+        message_logger = SimpleMessageLogger()
+        self.session_manager.message_logger = message_logger
+        self._setup_circuit()
+        obj_update = self._make_objectupdate_compressed(1234)
+        self.protocol.datagram_received(obj_update, self.region_addr)
+        await self._wait_drained()
+        entries = message_logger.entries
+        self.assertEqual(len(entries), 1)
+        self.assertEqual(entries[0].name, "ObjectUpdateCompressed")

tests/proxy/test_message_filter.py

@@ -5,8 +5,8 @@ from hippolyzer.lib.base.message.message import Block
 from hippolyzer.lib.base.message.udpdeserializer import UDPMessageDeserializer
 from hippolyzer.lib.base.settings import Settings
 from hippolyzer.lib.proxy.message import ProxiedMessage as Message
-from hippolyzer.apps.model import LLUDPMessageLogEntry
-from hippolyzer.apps.message_filter import compile_filter
+from hippolyzer.lib.proxy.message_logger import LLUDPMessageLogEntry
+from hippolyzer.lib.proxy.message_filter import compile_filter
 
 
 OBJECT_UPDATE = b'\xc0\x00\x00\x00Q\x00\x0c\x00\x01\xea\x03\x00\x02\xe6\x03\x00\x01\xbe\xff\x01\x06\xbc\x8e\x0b\x00' \
@@ -17,7 +17,7 @@ OBJECT_UPDATE = b'\xc0\x00\x00\x00Q\x00\x0c\x00\x01\xea\x03\x00\x02\xe6\x03\x00\
                 b'\x00\x02d&\x00\x03\x0e\x00\x01\x0e\x00\x01\x19\x00\x01\x80\x00\x01\x80\x00\x01\x80\x00\x01\x80\x00' \
                 b'\x01\x80\x00\x01\x80\x91\x11\xd2^/\x12\x8f\x81U\xa7@:x\xb3\x0e-\x00\x10\x03\x01\x00\x03\x1e%n\xa2' \
                 b'\xff\xc5\xe0\x83\x00\x01\x06\x00\x01\r\r\x01\x00\x11\x0e\xdc\x9b\x83\x98\x9aJv\xac\xc3\xdb\xbf7Ta' \
-                b'\x88\x00" '
+                b'\x88\x00"'
 
 
 class MessageFilterTests(unittest.TestCase):