v0.9.0

Previously only the first match was being highlighted.

.github/workflows/bundle_windows.yml

@@ -29,6 +29,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
+        pip install -r requirements.txt
         pip install -e .
         pip install cx_freeze
 

.github/workflows/pytest.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8, 3.9]
+        python-version: ["3.8", "3.10"]
 
     steps:
     - uses: actions/checkout@v2

README.md

@@ -375,6 +375,12 @@ To have your client's traffic proxied through Hippolyzer the general flow is:
 * The proxy needs to use content sniffing to figure out which requests are login requests,
   so make sure your request would pass `MITMProxyEventManager._is_login_request()`
 
+#### Do I have to do all that?
+
+You might be able to automate some of it on Linux by using
+[LinHippoAutoProxy](https://github.com/SaladDais/LinHippoAutoProxy). If you're on Windows or MacOS the
+above is your only option.
+
 ### Should I use this library to make an SL client in Python?
 
 No. If you just want to write a client in Python, you should instead look at using 

addon_examples/blueish_object_list.py

@@ -11,7 +11,7 @@ import enum
 import os.path
 from typing import *
 
-from PySide2 import QtCore, QtGui, QtWidgets
+from PySide6 import QtCore, QtGui, QtWidgets
 
 from hippolyzer.lib.base.datatypes import Vector3
 from hippolyzer.lib.base.message.message import Block, Message
@@ -80,7 +80,7 @@ class BlueishObjectListGUIAddon(BaseAddon):
             raise
 
     def _highlight_object(self, session: Session, obj: Object):
-        session.main_region.circuit.send_message(Message(
+        session.main_region.circuit.send(Message(
             "ForceObjectSelect",
             Block("Header", ResetList=False),
             Block("Data", LocalID=obj.LocalID),
@@ -88,7 +88,7 @@ class BlueishObjectListGUIAddon(BaseAddon):
         ))
 
     def _teleport_to_object(self, session: Session, obj: Object):
-        session.main_region.circuit.send_message(Message(
+        session.main_region.circuit.send(Message(
             "TeleportLocationRequest",
             Block("AgentData", AgentID=session.agent_id, SessionID=session.id),
             Block(

addon_examples/local_anim.py

@@ -20,13 +20,13 @@ bulk upload, like changing priority or removing a joint.
 """
 
 import asyncio
-import os
 import pathlib
 from abc import abstractmethod
 from typing import *
 
 from hippolyzer.lib.base import serialization as se
 from hippolyzer.lib.base.datatypes import UUID
+from hippolyzer.lib.base.helpers import get_mtime
 from hippolyzer.lib.base.llanim import Animation
 from hippolyzer.lib.base.message.message import Block, Message
 from hippolyzer.lib.proxy import addon_ctx
@@ -39,13 +39,6 @@ from hippolyzer.lib.proxy.region import ProxiedRegion
 from hippolyzer.lib.proxy.sessions import Session, SessionManager
 
 
-def _get_mtime(path: str):
-    try:
-        return os.stat(path).st_mtime
-    except:
-        return None
-
-
 class LocalAnimAddon(BaseAddon):
     # name -> path, only for anims actually from files
     local_anim_paths: Dict[str, str] = SessionProperty(dict)
@@ -166,7 +159,7 @@ class LocalAnimAddon(BaseAddon):
             cls.local_anim_playing_ids.pop(anim_name, None)
             cls.local_anim_bytes.pop(anim_name, None)
 
-        region.circuit.send_message(new_msg)
+        region.circuit.send(new_msg)
         print(f"Changing {anim_name} to {next_id}")
 
     @classmethod
@@ -176,7 +169,7 @@ class LocalAnimAddon(BaseAddon):
         anim_data = None
         if anim_path:
             old_mtime = cls.local_anim_mtimes.get(anim_name)
-            mtime = _get_mtime(anim_path)
+            mtime = get_mtime(anim_path)
             if only_if_changed and old_mtime == mtime:
                 return
 

addon_examples/local_mesh.py

@@ -81,17 +81,16 @@ class MeshUploadInterceptingAddon(BaseAddon):
 
     @handle_command()
     async def set_local_mesh_target(self, session: Session, region: ProxiedRegion):
-        """Set the currently selected object as the target for local mesh"""
-        parent_object = region.objects.lookup_localid(session.selected.object_local)
-        if not parent_object:
+        """Set the currently selected objects as the target for local mesh"""
+        selected_links = [region.objects.lookup_localid(l_id) for l_id in session.selected.object_locals]
+        selected_links = [o for o in selected_links if o is not None]
+        if not selected_links:
             show_message("Nothing selected")
             return
-        linkset_objects = [parent_object] + parent_object.Children
-
         old_locals = self.local_mesh_target_locals
         self.local_mesh_target_locals = [
             x.LocalID
-            for x in linkset_objects
+            for x in selected_links
             if ExtraParamType.MESH in x.ExtraParams
         ]
 

addon_examples/mesh_mangler.py

@@ -16,18 +16,23 @@ import local_mesh
 AddonManager.hot_reload(local_mesh, require_addons_loaded=True)
 
 
-def _reorient_coord(coord, orientation):
+def _reorient_coord(coord, orientation, normals=False):
     coords = []
     for axis in orientation:
         axis_idx = abs(axis) - 1
-        coords.append(coord[axis_idx] if axis >= 0 else 1.0 - coord[axis_idx])
+        if normals:
+            # Normals have a static domain from -1.0 to 1.0, just negate.
+            new_coord = coord[axis_idx] if axis >= 0 else -coord[axis_idx]
+        else:
+            new_coord = coord[axis_idx] if axis >= 0 else 1.0 - coord[axis_idx]
+        coords.append(new_coord)
     if coord.__class__ in (list, tuple):
         return coord.__class__(coords)
     return coord.__class__(*coords)
 
 
-def _reorient_coord_list(coord_list, orientation):
-    return [_reorient_coord(x, orientation) for x in coord_list]
+def _reorient_coord_list(coord_list, orientation, normals=False):
+    return [_reorient_coord(x, orientation, normals) for x in coord_list]
 
 
 def reorient_mesh(orientation):
@@ -42,7 +47,7 @@ def reorient_mesh(orientation):
             # flipping the axes around.
             material["Position"] = _reorient_coord_list(material["Position"], orientation)
             # Are you even supposed to do this to the normals?
-            material["Normal"] = _reorient_coord_list(material["Normal"], orientation)
+            material["Normal"] = _reorient_coord_list(material["Normal"], orientation, normals=True)
         return mesh
     return _reorienter
 

addon_examples/message_mirror.py

@@ -126,14 +126,14 @@ class MessageMirrorAddon(BaseAddon):
 
         # Send the message normally first if we're mirroring
         if message.name in MIRROR:
-            region.circuit.send_message(message)
+            region.circuit.send(message)
 
         # We're going to send the message on a new circuit, we need to take
         # it so we get a new packet ID and clean ACKs
         message = message.take()
 
         self._lludp_fixups(target_session, message)
-        target_region.circuit.send_message(message)
+        target_region.circuit.send(message)
         return True
 
     def _lludp_fixups(self, target_session: Session, message: Message):
@@ -206,7 +206,7 @@ class MessageMirrorAddon(BaseAddon):
                 return
             caps_source = target_region
 
-        new_base_url = caps_source.caps.get(cap_data.cap_name)
+        new_base_url = caps_source.cap_urls.get(cap_data.cap_name)
         if not new_base_url:
             print("No equiv cap?")
             return

addon_examples/mock_proxy_cap.py

@@ -0,0 +1,49 @@
+"""
+Example of proxy-provided caps
+
+Useful for mocking out a cap that isn't actually implemented by the server
+while developing the viewer-side pieces of it.
+
+Implements a cap that accepts an `obj_id` UUID query parameter and returns
+the name of the object.
+"""
+import asyncio
+import asgiref.wsgi
+
+from flask import Flask, Response, request
+
+from hippolyzer.lib.base.datatypes import UUID
+from hippolyzer.lib.proxy import addon_ctx
+from hippolyzer.lib.proxy.webapp_cap_addon import WebAppCapAddon
+
+app = Flask("GetObjectNameCapApp")
+
+
+@app.route('/')
+async def get_object_name():
+    # Should always have the current region, the cap handler is bound to one.
+    # Just need to pull it from the `addon_ctx` module's global.
+    obj_mgr = addon_ctx.region.get().objects
+    obj_id = UUID(request.args['obj_id'])
+    obj = obj_mgr.lookup_fullid(obj_id)
+    if not obj:
+        return Response(f"Couldn't find {obj_id!r}", status=404, mimetype="text/plain")
+
+    try:
+        await asyncio.wait_for(obj_mgr.request_object_properties(obj)[0], 1.0)
+    except asyncio.TimeoutError:
+        return Response(f"Timed out requesting {obj_id!r}'s properties", status=500, mimetype="text/plain")
+
+    return Response(obj.Name, mimetype="text/plain")
+
+
+class MockProxyCapExampleAddon(WebAppCapAddon):
+    # A cap URL with this name will be tied to each region when
+    # the sim is first connected to. The URL will be returned to the
+    # viewer in the Seed if the viewer requests it by name.
+    CAP_NAME = "GetObjectNameExample"
+    # Any asgi app should be fine.
+    APP = asgiref.wsgi.WsgiToAsgi(app)
+
+
+addons = [MockProxyCapExampleAddon()]

addon_examples/payday.py

@@ -37,7 +37,7 @@ class PaydayAddon(BaseAddon):
             chat_type=ChatType.SHOUT,
         )
         # Do the traditional money dance.
-        session.main_region.circuit.send_message(Message(
+        session.main_region.circuit.send(Message(
             "AgentAnimation",
             Block("AgentData", AgentID=session.agent_id, SessionID=session.id),
             Block("AnimationList", AnimID=UUID("928cae18-e31d-76fd-9cc9-2f55160ff818"), StartAnim=True),

addon_examples/pixel_artist.py

@@ -9,7 +9,7 @@ import asyncio
 import struct
 from typing import *
 
-from PySide2.QtGui import QImage
+from PySide6.QtGui import QImage
 
 from hippolyzer.lib.base.datatypes import UUID, Vector3, Quaternion
 from hippolyzer.lib.base.helpers import to_chunks
@@ -42,7 +42,7 @@ class PixelArtistAddon(BaseAddon):
             return
         img = QImage()
         with open(filename, "rb") as f:
-            img.loadFromData(f.read(), aformat=None)
+            img.loadFromData(f.read(), format=None)
         img = img.convertToFormat(QImage.Format_RGBA8888)
         height = img.height()
         width = img.width()
@@ -80,7 +80,7 @@ class PixelArtistAddon(BaseAddon):
                 # TODO: We don't track the land group or user's active group, so
                 #   "anyone can build" must be on for rezzing to work.
                 group_id = UUID()
-                region.circuit.send_message(Message(
+                region.circuit.send(Message(
                     'ObjectAdd',
                     Block('AgentData', AgentID=session.agent_id, SessionID=session.id, GroupID=group_id),
                     Block(
@@ -129,7 +129,7 @@ class PixelArtistAddon(BaseAddon):
             # Set the prim color to the color from the pixel
             te.Color[None] = pixel_color
             # Set the prim texture and color
-            region.circuit.send_message(Message(
+            region.circuit.send(Message(
                 'ObjectImage',
                 Block('AgentData', AgentID=session.agent_id, SessionID=session.id),
                 Block('ObjectData', ObjectLocalID=obj.LocalID, MediaURL=b'', TextureEntry_=te),
@@ -149,7 +149,7 @@ class PixelArtistAddon(BaseAddon):
 
         # Move the "pixels" to their correct position in chunks
         for chunk in to_chunks(positioning_blocks, 25):
-            region.circuit.send_message(Message(
+            region.circuit.send(Message(
                 'MultipleObjectUpdate',
                 Block('AgentData', AgentID=session.agent_id, SessionID=session.id),
                 *chunk,

addon_examples/recapitator.py

@@ -116,7 +116,7 @@ class RecapitatorAddon(BaseAddon):
         except:
             logging.exception("Exception while recapitating")
         # Tell the viewer about the status of its original upload
-        region.circuit.send_message(Message(
+        region.circuit.send(Message(
             "AssetUploadComplete",
             Block("AssetBlock", UUID=asset_id, Type=asset_block["Type"], Success=success),
             direction=Direction.IN,

addon_examples/simulate_packet_loss.py

@@ -0,0 +1,22 @@
+import random
+
+from hippolyzer.lib.proxy.addon_utils import BaseAddon
+from hippolyzer.lib.base.message.message import Message
+from hippolyzer.lib.proxy.region import ProxiedRegion
+from hippolyzer.lib.proxy.sessions import Session
+
+
+class SimulatePacketLossAddon(BaseAddon):
+    def handle_lludp_message(self, session: Session, region: ProxiedRegion, message: Message):
+        # Messing with these may kill your circuit
+        if message.name in {"PacketAck", "StartPingCheck", "CompletePingCheck", "UseCircuitCode",
+                            "CompleteAgentMovement", "AgentMovementComplete"}:
+            return
+        # Simulate 30% packet loss
+        if random.random() > 0.7:
+            # Do nothing, drop this packet on the floor
+            return True
+        return
+
+
+addons = [SimulatePacketLossAddon()]

addon_examples/transfer_example.py

@@ -3,7 +3,7 @@ Example of how to request a Transfer
 """
 from typing import *
 
-from hippolyzer.lib.base.legacy_inv import InventoryModel, InventoryItem
+from hippolyzer.lib.base.inventory import InventoryModel, InventoryItem
 from hippolyzer.lib.base.message.message import Block, Message
 from hippolyzer.lib.base.templates import (
     AssetType,
@@ -35,7 +35,7 @@ class TransferExampleAddon(BaseAddon):
     async def get_first_script(self, session: Session, region: ProxiedRegion):
         """Get the contents of the first script in the selected object"""
         # Ask for the object inventory so we can find a script
-        region.circuit.send_message(Message(
+        region.circuit.send(Message(
             'RequestTaskInventory',
             Block('AgentData', AgentID=session.agent_id, SessionID=session.id),
             Block('InventoryData', LocalID=session.selected.object_local),

addon_examples/turbo_object_inventory.py

@@ -64,12 +64,12 @@ class TurboObjectInventoryAddon(BaseAddon):
             # Any previous requests will have triggered a delete of the inventory file
             # by marking it complete on the server-side. Re-send our RequestTaskInventory
             # To make sure there's a fresh copy.
-            region.circuit.send_message(request_msg.take())
+            region.circuit.send(request_msg.take())
             inv_message = await region.message_handler.wait_for(('ReplyTaskInventory',), timeout=5.0)
             # No task inventory, send the reply as-is
             file_name = inv_message["InventoryData"]["Filename"]
             if not file_name:
-                region.circuit.send_message(inv_message)
+                region.circuit.send(inv_message)
                 return
 
             xfer = region.xfer_manager.request(
@@ -87,7 +87,7 @@ class TurboObjectInventoryAddon(BaseAddon):
                 continue
 
             # Send the original ReplyTaskInventory to the viewer so it knows the file is ready
-            region.circuit.send_message(inv_message)
+            region.circuit.send(inv_message)
             proxied_xfer = Xfer(data=xfer.reassemble_chunks())
 
             # Wait for the viewer to request the inventory file

addon_examples/uploader.py

@@ -102,7 +102,7 @@ class UploaderAddon(BaseAddon):
             ais_item_to_inventory_data(ais_item),
             direction=Direction.IN
         )
-        region.circuit.send_message(message)
+        region.circuit.send(message)
 
 
 addons = [UploaderAddon()]

addon_examples/xfer_example.py

@@ -2,7 +2,7 @@
 Example of how to request an Xfer
 """
 from hippolyzer.lib.base.datatypes import UUID
-from hippolyzer.lib.base.legacy_inv import InventoryModel
+from hippolyzer.lib.base.inventory import InventoryModel
 from hippolyzer.lib.base.templates import XferFilePath, AssetType, InventoryType, WearableType
 from hippolyzer.lib.base.message.message import Block, Message
 from hippolyzer.lib.proxy.addon_utils import BaseAddon, show_message
@@ -15,7 +15,7 @@ class XferExampleAddon(BaseAddon):
     @handle_command()
     async def get_mute_list(self, session: Session, region: ProxiedRegion):
         """Fetch the current user's mute list"""
-        region.circuit.send_message(Message(
+        region.circuit.send(Message(
             'MuteListRequest',
             Block('AgentData', AgentID=session.agent_id, SessionID=session.id),
             Block("MuteData", MuteCRC=0),
@@ -35,7 +35,7 @@ class XferExampleAddon(BaseAddon):
     @handle_command()
     async def get_task_inventory(self, session: Session, region: ProxiedRegion):
         """Get the inventory of the currently selected object"""
-        region.circuit.send_message(Message(
+        region.circuit.send(Message(
             'RequestTaskInventory',
             # If no session is passed in we'll use the active session when the coro was created
             Block('AgentData', AgentID=session.agent_id, SessionID=session.id),
@@ -98,7 +98,7 @@ textures 1
             data=asset_data,
             transaction_id=transaction_id
         )
-        region.circuit.send_message(Message(
+        region.circuit.send(Message(
             'CreateInventoryItem',
             Block('AgentData', AgentID=session.agent_id, SessionID=session.id),
             Block(

hippolyzer/apps/model.py

@@ -2,7 +2,7 @@ import enum
 import logging
 import typing
 
-from PySide2 import QtCore, QtGui
+from PySide6 import QtCore, QtGui
 
 from hippolyzer.lib.proxy.region import ProxiedRegion
 from hippolyzer.lib.proxy.message_logger import FilteringMessageLogger

hippolyzer/apps/proxy.py

@@ -7,6 +7,7 @@ import sys
 import time
 from typing import Optional
 
+import mitmproxy.ctx
 import mitmproxy.exceptions
 
 from hippolyzer.lib.base import llsd
@@ -131,6 +132,9 @@ def start_proxy(session_manager: SessionManager, extra_addons: Optional[list] =
         daemon=True,
     )
     http_proc.start()
+    # These need to be set for mitmproxy's ASGIApp serving code to work.
+    mitmproxy.ctx.master = None
+    mitmproxy.ctx.log = logging.getLogger("mitmproxy log")
 
     server = SLSOCKS5Server(session_manager)
     coro = asyncio.start_server(server.handle_connection, proxy_host, udp_proxy_port)

hippolyzer/apps/proxy_gui.py

@@ -18,7 +18,7 @@ from typing import *
 
 import multidict
 from qasync import QEventLoop, asyncSlot
-from PySide2 import QtCore, QtWidgets, QtGui
+from PySide6 import QtCore, QtWidgets, QtGui
 
 from hippolyzer.apps.model import MessageLogModel, MessageLogHeader, RegionListModel
 from hippolyzer.apps.proxy import start_proxy
@@ -35,6 +35,7 @@ from hippolyzer.lib.base.message.message_formatting import (
 )
 from hippolyzer.lib.base.message.msgtypes import MsgType
 from hippolyzer.lib.base.message.template_dict import DEFAULT_TEMPLATE_DICT
+from hippolyzer.lib.base.settings import SettingDescriptor
 from hippolyzer.lib.base.ui_helpers import loadUi
 import hippolyzer.lib.base.serialization as se
 from hippolyzer.lib.base.network.transport import Direction, SocketUDPTransport
@@ -61,7 +62,7 @@ def show_error_message(error_msg, parent=None):
     error_dialog = QtWidgets.QErrorMessage(parent=parent)
     # No obvious way to set this to plaintext, yuck...
     error_dialog.showMessage(html.escape(error_msg))
-    error_dialog.exec_()
+    error_dialog.exec()
     error_dialog.raise_()
 
 
@@ -88,13 +89,13 @@ class GUISessionManager(SessionManager, QtCore.QObject):
         self.all_regions = new_regions
 
 
-class GUIInteractionManager(BaseInteractionManager, QtCore.QObject):
-    def __init__(self, parent):
+class GUIInteractionManager(BaseInteractionManager):
+    def __init__(self, parent: QtWidgets.QWidget):
         BaseInteractionManager.__init__(self)
-        QtCore.QObject.__init__(self, parent=parent)
+        self._parent = parent
 
     def main_window_handle(self) -> Any:
-        return self.parent()
+        return self._parent
 
     def _dialog_async_exec(self, dialog: QtWidgets.QDialog):
         future = asyncio.Future()
@@ -106,7 +107,7 @@ class GUIInteractionManager(BaseInteractionManager, QtCore.QObject):
             self, caption: str, directory: str, filter_str: str, mode: QtWidgets.QFileDialog.FileMode,
             default_suffix: str = '',
     ) -> Tuple[bool, QtWidgets.QFileDialog]:
-        dialog = QtWidgets.QFileDialog(self.parent(), caption=caption, directory=directory, filter=filter_str)
+        dialog = QtWidgets.QFileDialog(self._parent, caption=caption, directory=directory, filter=filter_str)
         dialog.setFileMode(mode)
         if mode == QtWidgets.QFileDialog.FileMode.AnyFile:
             dialog.setAcceptMode(QtWidgets.QFileDialog.AcceptMode.AcceptSave)
@@ -154,7 +155,7 @@ class GUIInteractionManager(BaseInteractionManager, QtCore.QObject):
             title,
             caption,
             QtWidgets.QMessageBox.Ok | QtWidgets.QMessageBox.Cancel,
-            self.parent(),
+            self._parent,
         )
         fut = asyncio.Future()
         msg.finished.connect(lambda r: fut.set_result(r))
@@ -163,6 +164,8 @@ class GUIInteractionManager(BaseInteractionManager, QtCore.QObject):
 
 
 class GUIProxySettings(ProxySettings):
+    FIRST_RUN: bool = SettingDescriptor(True)
+
     """Persistent settings backed by QSettings"""
     def __init__(self, settings: QtCore.QSettings):
         super().__init__()
@@ -265,7 +268,7 @@ class MessageLogWindow(QtWidgets.QMainWindow):
         self.lineEditFilter.editingFinished.connect(self.setFilter)
         self.btnMessageBuilder.clicked.connect(self._sendToMessageBuilder)
         self.btnCopyRepr.clicked.connect(self._copyRepr)
-        self.actionInstallHTTPSCerts.triggered.connect(self._installHTTPSCerts)
+        self.actionInstallHTTPSCerts.triggered.connect(self.installHTTPSCerts)
         self.actionManageAddons.triggered.connect(self._manageAddons)
         self.actionManageFilters.triggered.connect(self._manageFilters)
         self.actionOpenMessageBuilder.triggered.connect(self._openMessageBuilder)
@@ -300,7 +303,7 @@ class MessageLogWindow(QtWidgets.QMainWindow):
 
     def _populateFilterMenu(self):
         def _addFilterAction(text, filter_str):
-            filter_action = QtWidgets.QAction(text, self)
+            filter_action = QtGui.QAction(text, self)
             filter_action.triggered.connect(lambda: self.setFilter(filter_str))
             self._filterMenu.addAction(filter_action)
 
@@ -311,13 +314,16 @@ class MessageLogWindow(QtWidgets.QMainWindow):
         for preset_name, preset_filter in filters.items():
             _addFilterAction(preset_name, preset_filter)
 
+    def getFilterDict(self):
+        return self.settings.FILTERS
+
     def setFilterDict(self, val: dict):
         self.settings.FILTERS = val
         self._populateFilterMenu()
 
     def _manageFilters(self):
         dialog = FilterDialog(self)
-        dialog.exec_()
+        dialog.exec()
 
     @nonFatalExceptions
     def setFilter(self, filter_str=None):
@@ -354,21 +360,20 @@ class MessageLogWindow(QtWidgets.QMainWindow):
             beautify=self.checkBeautify.isChecked(),
             replacements=buildReplacements(entry.session, entry.region),
         )
-        highlight_range = None
-        if isinstance(req, SpannedString):
-            match_result = self.model.filter.match(entry)
-            # Match result was a tuple indicating what matched
-            if isinstance(match_result, tuple):
-                highlight_range = req.spans.get(match_result)
-
         self.textRequest.setPlainText(req)
-        if highlight_range:
-            cursor = self.textRequest.textCursor()
-            cursor.setPosition(highlight_range[0], QtGui.QTextCursor.MoveAnchor)
-            cursor.setPosition(highlight_range[1], QtGui.QTextCursor.KeepAnchor)
-            highlight_format = QtGui.QTextBlockFormat()
-            highlight_format.setBackground(QtCore.Qt.yellow)
-            cursor.setBlockFormat(highlight_format)
+        # The string has a map of fields and their associated positions within the string,
+        # use that to highlight any individual fields the filter matched on.
+        if isinstance(req, SpannedString):
+            for field in self.model.filter.match(entry, short_circuit=False).fields:
+                field_span = req.spans.get(field)
+                if not field_span:
+                    continue
+                cursor = self.textRequest.textCursor()
+                cursor.setPosition(field_span[0], QtGui.QTextCursor.MoveAnchor)
+                cursor.setPosition(field_span[1], QtGui.QTextCursor.KeepAnchor)
+                highlight_format = QtGui.QTextBlockFormat()
+                highlight_format.setBackground(QtCore.Qt.yellow)
+                cursor.setBlockFormat(highlight_format)
 
         resp = entry.response(beautify=self.checkBeautify.isChecked())
         if resp:
@@ -441,10 +446,10 @@ class MessageLogWindow(QtWidgets.QMainWindow):
         with open(log_file, "wb") as f:
             f.write(export_log_entries(self.model))
 
-    def _installHTTPSCerts(self):
+    def installHTTPSCerts(self):
         msg = QtWidgets.QMessageBox()
-        msg.setText("This will install the proxy's HTTPS certificate in the config dir"
-                    " of any installed viewers, continue?")
+        msg.setText("Would you like to install the proxy's HTTPS certificate in the config dir"
+                    " of any installed viewers so that HTTPS connections will work?")
         yes_btn = msg.addButton("Yes", QtWidgets.QMessageBox.NoRole)
         msg.addButton("No", QtWidgets.QMessageBox.NoRole)
         msg.exec()
@@ -476,7 +481,7 @@ class MessageLogWindow(QtWidgets.QMainWindow):
 
     def _manageAddons(self):
         dialog = AddonDialog(self)
-        dialog.exec_()
+        dialog.exec()
 
     def getAddonList(self) -> List[str]:
         return self.sessionManager.settings.ADDON_SCRIPTS
@@ -565,7 +570,7 @@ class MessageBuilderWindow(QtWidgets.QMainWindow):
             else:
                 self.comboUntrusted.addItem(message_name)
 
-        cap_names = sorted(set(itertools.chain(*[r.caps.keys() for r in self.regionModel.regions])))
+        cap_names = sorted(set(itertools.chain(*[r.cap_urls.keys() for r in self.regionModel.regions])))
         for cap_name in cap_names:
             if cap_name.endswith("ProxyWrapper"):
                 continue
@@ -596,7 +601,7 @@ class MessageBuilderWindow(QtWidgets.QMainWindow):
                 break
         self.textRequest.setPlainText(
             f"""{method} [[{cap_name}]]{path}{params} HTTP/1.1
-# {region.caps.get(cap_name, "<unknown URI>")}
+# {region.cap_urls.get(cap_name, "<unknown URI>")}
 {headers}
 {body}"""
         )
@@ -697,7 +702,7 @@ class MessageBuilderWindow(QtWidgets.QMainWindow):
             else:
                 self._sendHTTPRequest(
                     "POST",
-                    region.caps["UntrustedSimulatorMessage"],
+                    region.cap_urls["UntrustedSimulatorMessage"],
                     {"Content-Type": "application/llsd+xml", "Accept": "application/llsd+xml"},
                     self.llsdSerializer.serialize(msg),
                 )
@@ -706,7 +711,7 @@ class MessageBuilderWindow(QtWidgets.QMainWindow):
             off_circuit = self.checkOffCircuit.isChecked()
             if off_circuit:
                 transport = SocketUDPTransport(socket.socket(socket.AF_INET, socket.SOCK_DGRAM))
-            region.circuit.send_message(msg, transport=transport)
+            region.circuit.send(msg, transport=transport)
             if off_circuit:
                 transport.close()
 
@@ -741,7 +746,7 @@ class MessageBuilderWindow(QtWidgets.QMainWindow):
             cap_name = match.group(1)
             cap_url = session.global_caps.get(cap_name)
             if not cap_url:
-                cap_url = region.caps.get(cap_name)
+                cap_url = region.cap_urls.get(cap_name)
             if not cap_url:
                 raise ValueError("Don't have a Cap for %s" % cap_name)
             uri = cap_url + match.group(2)
@@ -915,6 +920,10 @@ def gui_main():
     http_host = None
     if window.sessionManager.settings.REMOTELY_ACCESSIBLE:
         http_host = "0.0.0.0"
+    if settings.FIRST_RUN:
+        settings.FIRST_RUN = False
+        # Automatically offer to install the HTTPS certs on first run.
+        window.installHTTPSCerts()
     start_proxy(
         session_manager=window.sessionManager,
         extra_addon_paths=window.getAddonList(),

hippolyzer/lib/base/datatypes.py

@@ -273,7 +273,8 @@ class JankStringyBytes(bytes):
     Treat bytes as UTF8 if used in string context
 
     Sinful, but necessary evil for now since templates don't specify what's
-    binary and what's a string.
+    binary and what's a string. There are also certain fields where the value
+    may be either binary _or_ a string, depending on the context.
     """
     __slots__ = ()
 
@@ -288,6 +289,11 @@ class JankStringyBytes(bytes):
     def __ne__(self, other):
         return not self.__eq__(other)
 
+    def __contains__(self, item):
+        if isinstance(item, str):
+            return item in str(self)
+        return item in bytes(self)
+
 
 class RawBytes(bytes):
     __slots__ = ()

hippolyzer/lib/base/helpers.py

@@ -2,6 +2,8 @@ from __future__ import annotations
 
 import codecs
 import functools
+import os
+
 import pkg_resources
 import re
 import weakref
@@ -145,3 +147,10 @@ def to_chunks(chunkable: Sequence[_T], chunk_size: int) -> Generator[_T, None, N
     while chunkable:
         yield chunkable[:chunk_size]
         chunkable = chunkable[chunk_size:]
+
+
+def get_mtime(path):
+    try:
+        return os.stat(path).st_mtime
+    except:
+        return None

hippolyzer/lib/base/inventory.py

@@ -9,6 +9,7 @@ import dataclasses
 import datetime as dt
 import itertools
 import logging
+import struct
 import weakref
 from io import StringIO
 from typing import *
@@ -33,6 +34,17 @@ LOG = logging.getLogger(__name__)
 _T = TypeVar("_T")
 
 
+class SchemaFlagField(SchemaHexInt):
+    """Like a hex int, but must be serialized as bytes in LLSD due to being a U32"""
+    @classmethod
+    def from_llsd(cls, val: Any) -> int:
+        return struct.unpack("!I", val)[0]
+
+    @classmethod
+    def to_llsd(cls, val: int) -> Any:
+        return struct.pack("!I", val)
+
+
 def _yield_schema_tokens(reader: StringIO):
     in_bracket = False
     # empty str == EOF in Python
@@ -76,7 +88,7 @@ class InventoryBase(SchemaBase):
             if schema_name != cls.SCHEMA_NAME:
                 raise ValueError(f"Expected schema name {schema_name!r} to be {cls.SCHEMA_NAME!r}")
 
-        fields = cls._fields_dict()
+        fields = cls._get_fields_dict()
         obj_dict = {}
         for key, val in tok_iter:
             if key in fields:
@@ -100,7 +112,7 @@ class InventoryBase(SchemaBase):
     def to_writer(self, writer: StringIO):
         writer.write(f"\t{self.SCHEMA_NAME}\t0\n")
         writer.write("\t{\n")
-        for field_name, field in self._fields_dict().items():
+        for field_name, field in self._get_fields_dict().items():
             spec = field.metadata.get("spec")
             # Not meant to be serialized
             if not spec:
@@ -147,12 +159,38 @@ class InventoryModel(InventoryBase):
         model.reparent_nodes()
         return model
 
+    @classmethod
+    def from_llsd(cls, llsd_val: List[Dict]) -> InventoryModel:
+        model = cls()
+        for obj_dict in llsd_val:
+            if InventoryCategory.ID_ATTR in obj_dict:
+                if (obj := InventoryCategory.from_llsd(obj_dict)) is not None:
+                    model.add_container(obj)
+            elif InventoryObject.ID_ATTR in obj_dict:
+                if (obj := InventoryObject.from_llsd(obj_dict)) is not None:
+                    model.add_container(obj)
+            elif InventoryItem.ID_ATTR in obj_dict:
+                if (obj := InventoryItem.from_llsd(obj_dict)) is not None:
+                    model.add_item(obj)
+            else:
+                LOG.warning(f"Unknown object type {obj_dict!r}")
+        model.reparent_nodes()
+        return model
+
     def to_writer(self, writer: StringIO):
         for container in self.containers.values():
             container.to_writer(writer)
         for item in self.items.values():
             item.to_writer(writer)
 
+    def to_llsd(self):
+        vals = []
+        for container in self.containers.values():
+            vals.append(container.to_llsd())
+        for item in self.items.values():
+            vals.append(item.to_llsd())
+        return vals
+
     def add_container(self, container: InventoryContainerBase):
         self.containers[container.node_id] = container
         container.model = weakref.proxy(self)
@@ -246,7 +284,7 @@ class InventoryCategory(InventoryContainerBase):
     SCHEMA_NAME: ClassVar[str] = "inv_object"
 
     cat_id: UUID = schema_field(SchemaUUID)
-    pref_type: str = schema_field(SchemaStr)
+    pref_type: str = schema_field(SchemaStr, llsd_name="preferred_type")
     owner_id: UUID = schema_field(SchemaUUID)
     version: int = schema_field(SchemaInt)
 
@@ -259,10 +297,10 @@ class InventoryItem(InventoryNodeBase):
     item_id: UUID = schema_field(SchemaUUID)
     type: str = schema_field(SchemaStr)
     inv_type: str = schema_field(SchemaStr)
-    flags: int = schema_field(SchemaHexInt)
+    flags: int = schema_field(SchemaFlagField)
     name: str = schema_field(SchemaMultilineStr)
     desc: str = schema_field(SchemaMultilineStr)
-    creation_date: dt.datetime = schema_field(SchemaDate)
+    creation_date: dt.datetime = schema_field(SchemaDate, llsd_name="created_at")
     permissions: InventoryPermissions = schema_field(InventoryPermissions)
     sale_info: InventorySaleInfo = schema_field(InventorySaleInfo)
     asset_id: Optional[UUID] = schema_field(SchemaUUID, default=None)

hippolyzer/lib/base/jp2_utils.py

@@ -1,7 +1,6 @@
 import os
 import tempfile
 from io import BytesIO
-from typing import *
 
 import defusedxml.ElementTree
 from glymur import jp2box, Jp2k
@@ -10,12 +9,6 @@ from glymur import jp2box, Jp2k
 jp2box.ET = defusedxml.ElementTree
 
 
-SL_DEFAULT_ENCODE = {
-    "cratios": (1920.0, 480.0, 120.0, 30.0, 10.0),
-    "irreversible": True,
-}
-
-
 class BufferedJp2k(Jp2k):
     """
     For manipulating JP2K from within a binary buffer.
@@ -24,12 +17,7 @@ class BufferedJp2k(Jp2k):
     based on filename, so this is the least brittle approach.
     """
 
-    def __init__(self, contents: bytes, encode_kwargs: Optional[Dict] = None):
-        if encode_kwargs is None:
-            self.encode_kwargs = SL_DEFAULT_ENCODE.copy()
-        else:
-            self.encode_kwargs = encode_kwargs
-
+    def __init__(self, contents: bytes):
         stream = BytesIO(contents)
         self.temp_file = tempfile.NamedTemporaryFile(delete=False)
         stream.seek(0)
@@ -44,11 +32,12 @@ class BufferedJp2k(Jp2k):
             os.remove(self.temp_file.name)
         self.temp_file = None
 
-    def _write(self, img_array, verbose=False, **kwargs):
-        # Glymur normally only lets you control encode params when a write happens within
-        # the constructor. Keep around the encode params from the constructor and pass
-        # them to successive write calls.
-        return super()._write(img_array, verbose=False, **self.encode_kwargs, **kwargs)
+    def _populate_cparams(self, img_array):
+        if self._cratios is None:
+            self._cratios = (1920.0, 480.0, 120.0, 30.0, 10.0)
+        if self._irreversible is None:
+            self.irreversible = True
+        return super()._populate_cparams(img_array)
 
     def __bytes__(self):
         with open(self.temp_file.name, "rb") as f:

hippolyzer/lib/base/legacy_schema.py

@@ -31,6 +31,14 @@ class SchemaFieldSerializer(abc.ABC, Generic[_T]):
     def serialize(cls, val: _T) -> str:
         pass
 
+    @classmethod
+    def from_llsd(cls, val: Any) -> _T:
+        return val
+
+    @classmethod
+    def to_llsd(cls, val: _T) -> Any:
+        return val
+
 
 class SchemaDate(SchemaFieldSerializer[dt.datetime]):
     @classmethod
@@ -41,6 +49,14 @@ class SchemaDate(SchemaFieldSerializer[dt.datetime]):
     def serialize(cls, val: dt.datetime) -> str:
         return str(calendar.timegm(val.utctimetuple()))
 
+    @classmethod
+    def from_llsd(cls, val: Any) -> dt.datetime:
+        return dt.datetime.utcfromtimestamp(val)
+
+    @classmethod
+    def to_llsd(cls, val: dt.datetime):
+        return calendar.timegm(val.utctimetuple())
+
 
 class SchemaHexInt(SchemaFieldSerializer[int]):
     @classmethod
@@ -95,10 +111,11 @@ class SchemaUUID(SchemaFieldSerializer[UUID]):
 
 
 def schema_field(spec: Type[Union[SchemaBase, SchemaFieldSerializer]], *, default=dataclasses.MISSING, init=True,
-                 repr=True, hash=None, compare=True) -> dataclasses.Field:  # noqa
+                 repr=True, hash=None, compare=True, llsd_name=None) -> dataclasses.Field:  # noqa
     """Describe a field in the inventory schema and the shape of its value"""
     return dataclasses.field(
-        metadata={"spec": spec}, default=default, init=init, repr=repr, hash=hash, compare=compare
+        metadata={"spec": spec, "llsd_name": llsd_name}, default=default,
+        init=init, repr=repr, hash=hash, compare=compare,
     )
 
 
@@ -121,8 +138,14 @@ def parse_schema_line(line: str):
 @dataclasses.dataclass
 class SchemaBase(abc.ABC):
     @classmethod
-    def _fields_dict(cls):
-        return {f.name: f for f in dataclasses.fields(cls)}
+    def _get_fields_dict(cls, llsd=False):
+        fields_dict = {}
+        for field in dataclasses.fields(cls):
+            field_name = field.name
+            if llsd:
+                field_name = field.metadata.get("llsd_name") or field_name
+            fields_dict[field_name] = field
+        return fields_dict
 
     @classmethod
     def from_str(cls, text: str):
@@ -137,6 +160,30 @@ class SchemaBase(abc.ABC):
     def from_bytes(cls, data: bytes):
         return cls.from_str(data.decode("utf8"))
 
+    @classmethod
+    def from_llsd(cls, inv_dict: Dict):
+        fields = cls._get_fields_dict(llsd=True)
+        obj_dict = {}
+        for key, val in inv_dict.items():
+            if key in fields:
+                field: dataclasses.Field = fields[key]
+                key = field.name
+                spec = field.metadata.get("spec")
+                # Not a real key, an internal var on our dataclass
+                if not spec:
+                    LOG.warning(f"Internal key {key!r}")
+                    continue
+                # some kind of nested structure like sale_info
+                if issubclass(spec, SchemaBase):
+                    obj_dict[key] = spec.from_llsd(val)
+                elif issubclass(spec, SchemaFieldSerializer):
+                    obj_dict[key] = spec.from_llsd(val)
+                else:
+                    raise ValueError(f"Unsupported spec for {key!r}, {spec!r}")
+            else:
+                LOG.warning(f"Unknown key {key!r}")
+        return cls._obj_from_dict(obj_dict)
+
     def to_bytes(self) -> bytes:
         return self.to_str().encode("utf8")
 
@@ -146,6 +193,28 @@ class SchemaBase(abc.ABC):
         writer.seek(0)
         return writer.read()
 
+    def to_llsd(self):
+        obj_dict = {}
+        for field_name, field in self._get_fields_dict(llsd=True).items():
+            spec = field.metadata.get("spec")
+            # Not meant to be serialized
+            if not spec:
+                continue
+
+            val = getattr(self, field.name)
+            if val is None:
+                continue
+
+            # Some kind of nested structure like sale_info
+            if isinstance(val, SchemaBase):
+                val = val.to_llsd()
+            elif issubclass(spec, SchemaFieldSerializer):
+                val = spec.to_llsd(val)
+            else:
+                raise ValueError(f"Bad inventory spec {spec!r}")
+            obj_dict[field_name] = val
+        return obj_dict
+
     @abc.abstractmethod
     def to_writer(self, writer: StringIO):
         pass

hippolyzer/lib/base/mesh.py

@@ -270,8 +270,8 @@ LOD_SEGMENT_SERIALIZER = SegmentSerializer({
     # Each position represents a single vert.
     "Position": se.Collection(None, se.Vector3U16(0.0, 1.0)),
     "TexCoord0": se.Collection(None, se.Vector2U16(0.0, 1.0)),
-    # Normals have a static domain between -1 and 1
-    "Normal": se.Collection(None, se.Vector3U16(0.0, 1.0)),
+    # Normals have a static domain between -1 and 1, so just use that.
+    "Normal": se.Collection(None, se.Vector3U16(-1.0, 1.0)),
     "Weights": se.Collection(None, VertexWeights)
 })
 

hippolyzer/lib/base/message/circuit.py

@@ -1,6 +1,9 @@
 from __future__ import annotations
 
 import abc
+import asyncio
+import copy
+import dataclasses
 import datetime as dt
 import logging
 from typing import *
@@ -13,6 +16,14 @@ from .msgtypes import PacketFlags
 from .udpserializer import UDPMessageSerializer
 
 
+@dataclasses.dataclass
+class ReliableResendInfo:
+    last_resent: dt.datetime
+    message: Message
+    completed: asyncio.Future = dataclasses.field(default_factory=asyncio.Future)
+    tries_left: int = 10
+
+
 class Circuit:
     def __init__(self, near_host: Optional[ADDR_TUPLE], far_host: ADDR_TUPLE, transport):
         self.near_host: Optional[ADDR_TUPLE] = near_host
@@ -22,6 +33,8 @@ class Circuit:
         self.serializer = UDPMessageSerializer()
         self.last_packet_at = dt.datetime.now()
         self.packet_id_base = 0
+        self.unacked_reliable: Dict[Tuple[Direction, int], ReliableResendInfo] = {}
+        self.resend_every: float = 3.0
 
     def _send_prepared_message(self, message: Message, transport=None):
         try:
@@ -46,24 +59,69 @@ class Circuit:
             raise RuntimeError(f"Trying to re-send finalized {message!r}")
         message.packet_id = self.packet_id_base
         self.packet_id_base += 1
-        if not message.acks:
-            message.send_flags &= PacketFlags.ACK
+        if message.acks:
+            message.send_flags |= PacketFlags.ACK
+        else:
+            message.send_flags &= ~PacketFlags.ACK
         # If it was queued, it's not anymore
         message.queued = False
         message.finalized = True
 
-    def send_message(self, message: Message, transport=None):
+    def send(self, message: Message, transport=None) -> UDPPacket:
         if self.prepare_message(message):
+            # If the message originates from us then we're responsible for resends.
+            if message.reliable and message.synthetic:
+                self.unacked_reliable[(message.direction, message.packet_id)] = ReliableResendInfo(
+                    last_resent=dt.datetime.now(),
+                    message=message,
+                )
             return self._send_prepared_message(message, transport)
 
+    # Temporary alias
+    send_message = send
+
+    def send_reliable(self, message: Message, transport=None) -> asyncio.Future:
+        """send() wrapper that always sends reliably and allows `await`ing ACK receipt"""
+        if not message.synthetic:
+            raise ValueError("Not able to send non-synthetic message reliably!")
+        message.send_flags |= PacketFlags.RELIABLE
+        self.send(message, transport)
+        return self.unacked_reliable[(message.direction, message.packet_id)].completed
+
+    def collect_acks(self, message: Message):
+        effective_acks = list(message.acks)
+        if message.name == "PacketAck":
+            effective_acks.extend(x["ID"] for x in message["Packets"])
+        for ack in effective_acks:
+            resend_info = self.unacked_reliable.pop((~message.direction, ack), None)
+            if resend_info:
+                resend_info.completed.set_result(None)
+
+    def resend_unacked(self):
+        for resend_info in list(self.unacked_reliable.values()):
+            # Not time to attempt a resend yet
+            if dt.datetime.now() - resend_info.last_resent < dt.timedelta(seconds=self.resend_every):
+                continue
+
+            msg = copy.copy(resend_info.message)
+            resend_info.tries_left -= 1
+            # We were on our last try and we never received an ack
+            if not resend_info.tries_left:
+                logging.warning(f"Giving up on unacked {msg.packet_id}")
+                del self.unacked_reliable[(msg.direction, msg.packet_id)]
+                resend_info.completed.set_exception(TimeoutError("Exceeded resend limit"))
+                continue
+            resend_info.last_resent = dt.datetime.now()
+            msg.send_flags |= PacketFlags.RESENT
+            self._send_prepared_message(msg)
+
     def send_acks(self, to_ack: Sequence[int], direction=Direction.OUT, packet_id=None):
         logging.debug("%r acking %r" % (direction, to_ack))
         # TODO: maybe tack this onto `.acks` for next message?
         message = Message('PacketAck', *[Block('Packets', ID=x) for x in to_ack])
         message.packet_id = packet_id
         message.direction = direction
-        message.injected = True
-        self.send_message(message)
+        self.send(message)
 
     def __repr__(self):
         return "<%s %r : %r>" % (self.__class__.__name__, self.near_host, self.host)

hippolyzer/lib/base/message/message.py

@@ -188,7 +188,7 @@ class MsgBlockList(List["Block"]):
 class Message:
     __slots__ = ("name", "send_flags", "packet_id", "acks", "body_boundaries", "queued",
                  "offset", "raw_extra", "raw_body", "deserializer", "_blocks", "finalized",
-                 "direction", "meta", "injected", "dropped", "sender")
+                 "direction", "meta", "synthetic", "dropped", "sender")
 
     def __init__(self, name, *args, packet_id=None, flags=0, acks=None, direction=None):
         # TODO: Do this on a timer or something.
@@ -213,7 +213,7 @@ class Message:
         self.queued: bool = False
         self._blocks: BLOCK_DICT = {}
         self.meta = {}
-        self.injected = False
+        self.synthetic = packet_id is None
         self.dropped = False
         self.sender: Optional[ADDR_TUPLE] = None
 
@@ -312,7 +312,7 @@ class Message:
                 "packet_id": self.packet_id,
                 "meta": self.meta.copy(),
                 "dropped": self.dropped,
-                "injected": self.injected,
+                "synthetic": self.synthetic,
                 "direction": self.direction.name,
                 "send_flags": int(self.send_flags),
                 "extra": self.extra,
@@ -334,7 +334,7 @@ class Message:
             msg.packet_id = dict_val['packet_id']
             msg.meta = dict_val['meta']
             msg.dropped = dict_val['dropped']
-            msg.injected = dict_val['injected']
+            msg.synthetic = dict_val['synthetic']
             msg.direction = Direction[dict_val['direction']]
             msg.send_flags = dict_val['send_flags']
             msg.extra = dict_val['extra']
@@ -386,6 +386,7 @@ class Message:
         message_copy.packet_id = None
         message_copy.dropped = False
         message_copy.finalized = False
+        message_copy.queued = False
         return message_copy
 
     def to_summary(self):

hippolyzer/lib/base/message/message_formatting.py

@@ -62,9 +62,16 @@ class HumanMessageSerializer:
                 continue
 
             if first_line:
-                direction, message_name = line.split(" ", 1)
+                first_split = [x for x in line.split(" ") if x]
+                direction, message_name = first_split[:2]
+                options = [x.strip("[]") for x in first_split[2:]]
                 msg = Message(message_name)
                 msg.direction = Direction[direction.upper()]
+                for option in options:
+                    if option in PacketFlags.__members__:
+                        msg.send_flags |= PacketFlags[option]
+                    elif re.match(r"^\d+$", option):
+                        msg.send_flags |= int(option)
                 first_line = False
                 continue
 
@@ -137,9 +144,17 @@ class HumanMessageSerializer:
         if msg.direction is not None:
             string += f'{msg.direction.name} '
         string += msg.name
+        flags = msg.send_flags
+        for poss_flag in iter(PacketFlags):
+            if flags & poss_flag:
+                flags &= ~poss_flag
+                string += f" [{poss_flag.name}]"
+        # Make sure flags with unknown meanings don't get lost
+        if flags:
+            string += f" [{int(flags)}]"
         if msg.packet_id is not None:
-            string += f'\n# {msg.packet_id}: {PacketFlags(msg.send_flags)!r}'
-            string += f'{", DROPPED" if msg.dropped else ""}{", INJECTED" if msg.injected else ""}'
+            string += f'\n# ID: {msg.packet_id}'
+            string += f'{", DROPPED" if msg.dropped else ""}{", SYNTHETIC" if msg.synthetic else ""}'
         if msg.extra:
             string += f'\n# EXTRA: {msg.extra!r}'
         string += '\n\n'

hippolyzer/lib/base/message/udpdeserializer.py

@@ -68,7 +68,7 @@ class UDPMessageDeserializer:
         self.settings = settings or Settings()
         self.template_dict = self.DEFAULT_TEMPLATE
 
-    def deserialize(self, msg_buff: bytes):
+    def deserialize(self, msg_buff: bytes) -> Message:
         msg = self._parse_message_header(msg_buff)
         if not self.settings.ENABLE_DEFERRED_PACKET_PARSING:
             try:
@@ -85,6 +85,7 @@ class UDPMessageDeserializer:
         reader = se.BufferReader("!", data)
 
         msg: Message = Message("Placeholder")
+        msg.synthetic = False
         msg.send_flags = reader.read(se.U8)
         msg.packet_id = reader.read(se.U32)
 

hippolyzer/lib/base/templates.py

@@ -1600,6 +1600,7 @@ class RegionHandshakeReplyFlags(IntFlag):
 @se.flag_field_serializer("TeleportStart", "Info", "TeleportFlags")
 @se.flag_field_serializer("TeleportProgress", "Info", "TeleportFlags")
 @se.flag_field_serializer("TeleportFinish", "Info", "TeleportFlags")
+@se.flag_field_serializer("TeleportLocal", "Info", "TeleportFlags")
 @se.flag_field_serializer("TeleportLureRequest", "Info", "TeleportFlags")
 class TeleportFlags(IntFlag):
     SET_HOME_TO_TARGET = 1 << 0  # newbie leaving prelude (starter area)
@@ -1618,6 +1619,8 @@ class TeleportFlags(IntFlag):
     IS_FLYING = 1 << 13
     SHOW_RESET_HOME = 1 << 14
     FORCE_REDIRECT = 1 << 15
+    VIA_GLOBAL_COORDS = 1 << 16
+    WITHIN_REGION = 1 << 17
 
 
 @se.http_serializer("RenderMaterials")

hippolyzer/lib/base/transfer_manager.py

@@ -94,7 +94,7 @@ class TransferManager:
         if params_dict.get("SessionID", dataclasses.MISSING) is None:
             params.SessionID = self._session_id
 
-        self._connection_holder.circuit.send_message(Message(
+        self._connection_holder.circuit.send(Message(
             'TransferRequest',
             Block(
                 'TransferInfo',

hippolyzer/lib/base/ui_helpers.py

@@ -1,5 +1,5 @@
-from PySide2.QtCore import QMetaObject
-from PySide2.QtUiTools import QUiLoader
+from PySide6.QtCore import QMetaObject
+from PySide6.QtUiTools import QUiLoader
 
 
 class UiLoader(QUiLoader):

hippolyzer/lib/base/wearables.py

@@ -13,7 +13,7 @@ from xml.etree.ElementTree import parse as parse_etree
 
 from hippolyzer.lib.base.datatypes import UUID
 from hippolyzer.lib.base.helpers import get_resource_filename
-from hippolyzer.lib.base.legacy_inv import InventorySaleInfo, InventoryPermissions
+from hippolyzer.lib.base.inventory import InventorySaleInfo, InventoryPermissions
 from hippolyzer.lib.base.legacy_schema import SchemaBase, parse_schema_line, SchemaParsingError
 from hippolyzer.lib.base.templates import WearableType
 

hippolyzer/lib/base/xfer_manager.py

@@ -110,7 +110,7 @@ class XferManager:
             direction: Direction = Direction.OUT,
     ) -> Xfer:
         xfer_id = xfer_id if xfer_id is not None else random.getrandbits(64)
-        self._connection_holder.circuit.send_message(Message(
+        self._connection_holder.circuit.send(Message(
             'RequestXfer',
             Block(
                 'XferID',
@@ -174,7 +174,7 @@ class XferManager:
             to_ack = range(xfer.next_ackable, ack_max)
             xfer.next_ackable = ack_max
         for ack_id in to_ack:
-            self._connection_holder.circuit.send_message(Message(
+            self._connection_holder.circuit.send(Message(
                 "ConfirmXferPacket",
                 Block("XferID", ID=xfer.xfer_id, Packet=ack_id),
                 direction=xfer.direction,
@@ -216,7 +216,7 @@ class XferManager:
         else:
             inline_data = data
 
-        self._connection_holder.circuit.send_message(Message(
+        self._connection_holder.circuit.send(Message(
             "AssetUploadRequest",
             Block(
                 "AssetBlock",
@@ -272,7 +272,7 @@ class XferManager:
             chunk = xfer.chunks.pop(packet_id)
             # EOF if there are no chunks left
             packet_val = XferPacket(PacketID=packet_id, IsEOF=not bool(xfer.chunks))
-            self._connection_holder.circuit.send_message(Message(
+            self._connection_holder.circuit.send(Message(
                 "SendXferPacket",
                 Block("XferID", ID=xfer.xfer_id, Packet_=packet_val),
                 Block("DataPacket", Data=chunk),

hippolyzer/lib/client/object_manager.py

@@ -116,8 +116,8 @@ class ClientObjectManager:
                 *[Block("ObjectData", ObjectLocalID=x) for x in ids_to_req[:255]],
             ]
             # Selecting causes ObjectProperties to be sent
-            self._region.circuit.send_message(Message("ObjectSelect", blocks))
-            self._region.circuit.send_message(Message("ObjectDeselect", blocks))
+            self._region.circuit.send(Message("ObjectSelect", blocks))
+            self._region.circuit.send(Message("ObjectDeselect", blocks))
             ids_to_req = ids_to_req[255:]
 
         futures = []
@@ -150,7 +150,7 @@ class ClientObjectManager:
 
         ids_to_req = local_ids
         while ids_to_req:
-            self._region.circuit.send_message(Message(
+            self._region.circuit.send(Message(
                 "RequestMultipleObjects",
                 Block("AgentData", AgentID=session.agent_id, SessionID=session.id),
                 *[Block("ObjectData", CacheMissType=0, ID=x) for x in ids_to_req[:255]],

hippolyzer/lib/proxy/addon_utils.py

@@ -73,17 +73,17 @@ def show_message(text, session=None) -> None:
         direction=Direction.IN,
     )
     if session:
-        session.main_region.circuit.send_message(message)
+        session.main_region.circuit.send(message)
     else:
         for session in AddonManager.SESSION_MANAGER.sessions:
-            session.main_region.circuit.send_message(copy.copy(message))
+            session.main_region.circuit.send(copy.copy(message))
 
 
 def send_chat(message: Union[bytes, str], channel=0, chat_type=ChatType.NORMAL, session=None):
     session = session or addon_ctx.session.get(None) or None
     if not session:
         raise RuntimeError("Tried to send chat without session")
-    session.main_region.circuit.send_message(Message(
+    session.main_region.circuit.send(Message(
         "ChatFromViewer",
         Block(
             "AgentData",
@@ -181,6 +181,9 @@ class BaseAddon(abc.ABC):
     def handle_region_changed(self, session: Session, region: ProxiedRegion):
         pass
 
+    def handle_region_registered(self, session: Session, region: ProxiedRegion):
+        pass
+
     def handle_circuit_created(self, session: Session, region: ProxiedRegion):
         pass
 

hippolyzer/lib/proxy/addons.py

@@ -16,6 +16,7 @@ from types import ModuleType
 from typing import *
 
 from hippolyzer.lib.base.datatypes import UUID
+from hippolyzer.lib.base.helpers import get_mtime
 from hippolyzer.lib.base.message.message import Message
 from hippolyzer.lib.base.network.transport import UDPPacket
 from hippolyzer.lib.proxy import addon_ctx
@@ -31,13 +32,6 @@ if TYPE_CHECKING:
 LOG = logging.getLogger(__name__)
 
 
-def _get_mtime(path):
-    try:
-        return os.stat(path).st_mtime
-    except:
-        return None
-
-
 class BaseInteractionManager:
     @abc.abstractmethod
     async def open_dir(self, caption: str = '', directory: str = '', filter_str: str = '') -> Optional[str]:
@@ -187,7 +181,7 @@ class AddonManager:
     def _check_hotreloads(cls):
         """Mark addons that rely on changed files for reloading"""
         for filename, importers in cls.HOTRELOAD_IMPORTERS.items():
-            mtime = _get_mtime(filename)
+            mtime = get_mtime(filename)
             if not mtime or mtime == cls.FILE_MTIMES.get(filename, None):
                 continue
 
@@ -216,7 +210,7 @@ class AddonManager:
         # Mark the caller as having imported (and being dependent on) `module`
         stack = inspect.stack()[1]
         cls.HOTRELOAD_IMPORTERS[imported_file].add(stack.filename)
-        cls.FILE_MTIMES[imported_file] = _get_mtime(imported_file)
+        cls.FILE_MTIMES[imported_file] = get_mtime(imported_file)
 
         importing_spec = next((s for s in cls.BASE_ADDON_SPECS if s.origin == stack.filename), None)
         imported_spec = next((s for s in cls.BASE_ADDON_SPECS if s.origin == imported_file), None)
@@ -264,7 +258,7 @@ class AddonManager:
         for spec in cls.BASE_ADDON_SPECS[:]:
             had_mod = spec.name in cls.FRESH_ADDON_MODULES
             try:
-                mtime = _get_mtime(spec.origin)
+                mtime = get_mtime(spec.origin)
                 mtime_changed = mtime != cls.FILE_MTIMES.get(spec.origin, None)
                 if not mtime_changed and had_mod:
                     continue
@@ -527,6 +521,11 @@ class AddonManager:
         with addon_ctx.push(session, region):
             return cls._call_all_addon_hooks("handle_region_changed", session, region)
 
+    @classmethod
+    def handle_region_registered(cls, session: Session, region: ProxiedRegion):
+        with addon_ctx.push(session, region):
+            return cls._call_all_addon_hooks("handle_region_registered", session, region)
+
     @classmethod
     def handle_circuit_created(cls, session: Session, region: ProxiedRegion):
         with addon_ctx.push(session, region):

hippolyzer/lib/proxy/caps.py

@@ -24,6 +24,10 @@ class CapType(enum.Enum):
     WRAPPER = enum.auto()
     PROXY_ONLY = enum.auto()
 
+    @property
+    def fake(self) -> bool:
+        return self == CapType.PROXY_ONLY or self == CapType.WRAPPER
+
 
 class SerializedCapData(typing.NamedTuple):
     cap_name: typing.Optional[str] = None

hippolyzer/lib/proxy/caps_client.py

@@ -20,7 +20,7 @@ class ProxyCapsClient(CapsClient):
     def _get_caps(self) -> Optional[CAPS_DICT]:
         if not self._region:
             return None
-        return self._region.caps
+        return self._region.cap_urls
 
     def _request_fixups(self, cap_or_url: str, headers: Dict, proxy: Optional[bool], ssl: Any):
         # We want to proxy this through Hippolyzer

hippolyzer/lib/proxy/circuit.py

@@ -25,7 +25,7 @@ class ProxiedCircuit(Circuit):
         except:
             logging.exception(f"Failed to serialize: {message.to_dict()!r}")
             raise
-        if self.logging_hook and message.injected:
+        if self.logging_hook and message.synthetic:
             self.logging_hook(message)
         return self.send_datagram(serialized, message.direction, transport=transport)
 
@@ -34,47 +34,46 @@ class ProxiedCircuit(Circuit):
             return self.out_injections, self.in_injections
         return self.in_injections, self.out_injections
 
-    def prepare_message(self, message: Message, direction=None):
+    def prepare_message(self, message: Message):
         if message.finalized:
             raise RuntimeError(f"Trying to re-send finalized {message!r}")
         if message.queued:
             # This is due to be dropped, nothing should be sending the original
             raise RuntimeError(f"Trying to send original of queued {message!r}")
-        direction = direction or getattr(message, 'direction')
-        fwd_injections, reverse_injections = self._get_injections(direction)
+        fwd_injections, reverse_injections = self._get_injections(message.direction)
+
+        message.finalized = True
 
         # Injected, let's gen an ID
         if message.packet_id is None:
             message.packet_id = fwd_injections.gen_injectable_id()
-            message.injected = True
-        else:
+            message.synthetic = True
+        # This message wasn't injected by the proxy so we need to rewrite packet IDs
+        # to account for IDs the real creator of the packet couldn't have known about.
+        elif not message.synthetic:
             # was_dropped needs the unmodified packet ID
             if fwd_injections.was_dropped(message.packet_id) and message.name != "PacketAck":
                 logging.warning("Attempting to re-send previously dropped %s:%s, did we ack?" %
                                 (message.packet_id, message.name))
             message.packet_id = fwd_injections.get_effective_id(message.packet_id)
             fwd_injections.track_seen(message.packet_id)
-
-        message.finalized = True
-
-        if not message.injected:
-            # This message wasn't injected by the proxy so we need to rewrite packet IDs
-            # to account for IDs the other parties couldn't have known about.
             message.acks = tuple(
                 reverse_injections.get_original_id(x) for x in message.acks
                 if not reverse_injections.was_injected(x)
             )
 
             if message.name == "PacketAck":
-                if not self._rewrite_packet_ack(message, reverse_injections):
-                    logging.debug(f"Dropping {direction} ack for injected packets!")
+                if not self._rewrite_packet_ack(message, reverse_injections) and not message.acks:
+                    logging.debug(f"Dropping {message.direction} ack for injected packets!")
                     # Let caller know this shouldn't be sent at all, it's strictly ACKs for
                     # injected packets.
                     return False
             elif message.name == "StartPingCheck":
                 self._rewrite_start_ping_check(message, fwd_injections)
 
-        if not message.acks:
+        if message.acks:
+            message.send_flags |= PacketFlags.ACK
+        else:
             message.send_flags &= ~PacketFlags.ACK
         return True
 
@@ -100,15 +99,18 @@ class ProxiedCircuit(Circuit):
         new_id = fwd_injections.get_effective_id(orig_id)
         if orig_id != new_id:
             logging.debug("Rewrote oldest unacked %s -> %s" % (orig_id, new_id))
+        # Get a list of unacked IDs for the direction this StartPingCheck is heading
+        fwd_unacked = (a for (d, a) in self.unacked_reliable.keys() if d == message.direction)
+        # Use the proxy's oldest unacked ID if it's older than the client's
+        new_id = min((new_id, *fwd_unacked))
         message["PingID"]["OldestUnacked"] = new_id
 
-    def drop_message(self, message: Message, orig_direction=None):
+    def drop_message(self, message: Message):
         if message.finalized:
             raise RuntimeError(f"Trying to drop finalized {message!r}")
         if message.packet_id is None:
             return
-        orig_direction = orig_direction or message.direction
-        fwd_injections, reverse_injections = self._get_injections(orig_direction)
+        fwd_injections, reverse_injections = self._get_injections(message.direction)
 
         fwd_injections.mark_dropped(message.packet_id)
         message.dropped = True
@@ -116,7 +118,7 @@ class ProxiedCircuit(Circuit):
 
         # Was sent reliably, tell the other end that we saw it and to shut up.
         if message.reliable:
-            self.send_acks([message.packet_id], ~orig_direction)
+            self.send_acks([message.packet_id], ~message.direction)
 
         # This packet had acks for the other end, send them in a separate PacketAck
         effective_acks = tuple(
@@ -124,7 +126,7 @@ class ProxiedCircuit(Circuit):
             if not reverse_injections.was_injected(x)
         )
         if effective_acks:
-            self.send_acks(effective_acks, orig_direction, packet_id=message.packet_id)
+            self.send_acks(effective_acks, message.direction, packet_id=message.packet_id)
 
 
 class InjectionTracker:

hippolyzer/lib/proxy/http_event_manager.py

@@ -83,16 +83,19 @@ class MITMProxyEventManager:
         finally:
             # If someone has taken this request out of the regular callback flow,
             # they'll manually send a callback at some later time.
-            if not flow.taken:
-                self.to_proxy_queue.put(("callback", flow.id, flow.get_state()))
+            if not flow.taken and not flow.resumed:
+                # Addon hasn't taken ownership of this flow, send it back to mitmproxy
+                # ourselves.
+                flow.resume()
 
     def _handle_request(self, flow: HippoHTTPFlow):
         url = flow.request.url
         cap_data = self.session_manager.resolve_cap(url)
         flow.cap_data = cap_data
-        # Don't do anything special with the proxy's own requests,
-        # we only pass it through for logging purposes.
-        if flow.request_injected:
+        # Don't do anything special with the proxy's own requests unless the requested
+        # URL can only be handled by the proxy. Ideally we only pass the request through
+        # for logging purposes.
+        if flow.request_injected and (not cap_data or not cap_data.type.fake):
             return
 
         # The local asset repo gets first bite at the apple
@@ -104,7 +107,7 @@ class MITMProxyEventManager:
         AddonManager.handle_http_request(flow)
         if cap_data and cap_data.cap_name.endswith("ProxyWrapper"):
             orig_cap_name = cap_data.cap_name.rsplit("ProxyWrapper", 1)[0]
-            orig_cap_url = cap_data.region().caps[orig_cap_name]
+            orig_cap_url = cap_data.region().cap_urls[orig_cap_name]
             split_orig_url = urllib.parse.urlsplit(orig_cap_url)
             orig_cap_host = split_orig_url[1]
 
@@ -135,7 +138,7 @@ class MITMProxyEventManager:
                 )
         elif cap_data and cap_data.asset_server_cap:
             # Both the wrapper request and the actual asset server request went through
-            # the proxy
+            # the proxy. Don't bother trying the redirect strategy anymore.
             self._asset_server_proxied = True
             logging.warning("noproxy not used, switching to URI rewrite strategy")
         elif cap_data and cap_data.cap_name == "EventQueueGet":
@@ -159,6 +162,17 @@ class MITMProxyEventManager:
                         "Connection": "close",
                     },
                 )
+        elif cap_data and cap_data.cap_name == "Seed":
+            # Drop any proxy-only caps from the seed request we send to the server,
+            # add those cap names as metadata so we know to send their urls in the response
+            parsed_seed: List[str] = llsd.parse_xml(flow.request.content)
+            flow.metadata['needed_proxy_caps'] = []
+            for known_cap_name, (known_cap_type, known_cap_url) in cap_data.region().caps.items():
+                if known_cap_type == CapType.PROXY_ONLY and known_cap_name in parsed_seed:
+                    parsed_seed.remove(known_cap_name)
+                    flow.metadata['needed_proxy_caps'].append(known_cap_name)
+            if flow.metadata['needed_proxy_caps']:
+                flow.request.content = llsd.format_xml(parsed_seed)
         elif not cap_data:
             if self._is_login_request(flow):
                 # Not strictly a Cap, but makes it easier to filter on.
@@ -198,10 +212,14 @@ class MITMProxyEventManager:
     def _handle_response(self, flow: HippoHTTPFlow):
         message_logger = self.session_manager.message_logger
         if message_logger:
-            message_logger.log_http_response(flow)
+            try:
+                message_logger.log_http_response(flow)
+            except:
+                logging.exception("Failed while logging HTTP flow")
 
-        # Don't handle responses for requests injected by the proxy
-        if flow.request_injected:
+        # Don't process responses for requests or responses injected by the proxy.
+        # We already processed it, it came from us!
+        if flow.request_injected or flow.response_injected:
             return
 
         status = flow.response.status_code
@@ -262,7 +280,10 @@ class MITMProxyEventManager:
                 for cap_name in wrappable_caps:
                     if cap_name in parsed:
                         parsed[cap_name] = region.register_wrapper_cap(cap_name)
-                flow.response.content = llsd.format_pretty_xml(parsed)
+                # Send the client the URLs for any proxy-only caps it requested
+                for cap_name in flow.metadata['needed_proxy_caps']:
+                    parsed[cap_name] = region.cap_urls[cap_name]
+                flow.response.content = llsd.format_xml(parsed)
             elif cap_data.cap_name == "EventQueueGet":
                 parsed_eq_resp = llsd.parse_xml(flow.response.content)
                 if parsed_eq_resp:
@@ -281,13 +302,13 @@ class MITMProxyEventManager:
                     # HACK: see note in above request handler for EventQueueGet
                     req_ack_id = llsd.parse_xml(flow.request.content)["ack"]
                     eq_manager.cache_last_poll_response(req_ack_id, parsed_eq_resp)
-                flow.response.content = llsd.format_pretty_xml(parsed_eq_resp)
+                flow.response.content = llsd.format_xml(parsed_eq_resp)
             elif cap_data.cap_name in self.UPLOAD_CREATING_CAPS:
                 if not region:
                     return
                 parsed = llsd.parse_xml(flow.response.content)
                 if "uploader" in parsed:
-                    region.register_temporary_cap(cap_data.cap_name + "Uploader", parsed["uploader"])
+                    region.register_cap(cap_data.cap_name + "Uploader", parsed["uploader"], CapType.TEMPORARY)
         except:
             logging.exception("OOPS, blew up in HTTP proxy!")
 

hippolyzer/lib/proxy/http_flow.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
 import copy
+import multiprocessing
+import weakref
 from typing import *
 from typing import Optional
 
@@ -20,16 +22,18 @@ class HippoHTTPFlow:
     Hides the nastiness of writing to flow.metadata so we can pass
     state back and forth between the two proxies
     """
-    __slots__ = ("flow",)
+    __slots__ = ("flow", "callback_queue", "resumed", "taken")
 
-    def __init__(self, flow: HTTPFlow):
+    def __init__(self, flow: HTTPFlow, callback_queue: Optional[multiprocessing.Queue] = None):
         self.flow: HTTPFlow = flow
+        self.resumed = False
+        self.taken = False
+        self.callback_queue = weakref.ref(callback_queue) if callback_queue else None
         meta = self.flow.metadata
-        meta.setdefault("taken", False)
         meta.setdefault("can_stream", True)
         meta.setdefault("response_injected", False)
         meta.setdefault("request_injected", False)
-        meta.setdefault("cap_data", None)
+        meta.setdefault("cap_data", CapData())
         meta.setdefault("from_browser", False)
 
     @property
@@ -91,12 +95,21 @@ class HippoHTTPFlow:
 
     def take(self) -> HippoHTTPFlow:
         """Don't automatically pass this flow back to mitmproxy"""
-        self.metadata["taken"] = True
+        # TODO: Having to explicitly take / release Flows to use them in an async
+        #  context is kind of janky. The HTTP callback handling code should probably
+        #  be made totally async, including the addon hooks. Would coroutine per-callback
+        #  be expensive?
+        assert not self.taken and not self.resumed
+        self.taken = True
         return self
 
-    @property
-    def taken(self) -> bool:
-        return self.metadata["taken"]
+    def resume(self):
+        """Release the HTTP flow back to the normal processing flow"""
+        assert self.callback_queue
+        assert not self.resumed
+        self.taken = False
+        self.resumed = True
+        self.callback_queue().put(("callback", self.flow.id, self.get_state()))
 
     @property
     def is_replay(self) -> bool:
@@ -120,11 +133,14 @@ class HippoHTTPFlow:
         flow: Optional[HTTPFlow] = HTTPFlow.from_state(flow_state)
         assert flow is not None
         cap_data_ser = flow.metadata.get("cap_data_ser")
+        callback_queue = None
+        if session_manager:
+            callback_queue = session_manager.flow_context.to_proxy_queue
         if cap_data_ser is not None:
             flow.metadata["cap_data"] = CapData.deserialize(cap_data_ser, session_manager)
         else:
             flow.metadata["cap_data"] = None
-        return cls(flow)
+        return cls(flow, callback_queue)
 
     def copy(self) -> HippoHTTPFlow:
         # HACK: flow.copy() expects the flow to be fully JSON serializable, but

hippolyzer/lib/proxy/http_proxy.py

@@ -136,9 +136,6 @@ class IPCInterceptionAddon:
                 if event_type == "callback":
                     orig_flow = self.intercepted_flows.pop(flow_id)
                     orig_flow.set_state(flow_state)
-                    # Remove the taken flag from the flow if present, the flow by definition
-                    # isn't take()n anymore once it's been passed back to the proxy.
-                    orig_flow.metadata.pop("taken", None)
                 elif event_type == "replay":
                     flow: HTTPFlow = HTTPFlow.from_state(flow_state)
                     # mitmproxy won't replay intercepted flows, this is an old flow so
@@ -178,7 +175,7 @@ class IPCInterceptionAddon:
     def responseheaders(self, flow: HTTPFlow):
         # The response was injected earlier in an earlier handler,
         # we don't want to touch this anymore.
-        if flow.metadata["response_injected"]:
+        if flow.metadata.get("response_injected"):
             return
 
         # Someone fucked up and put a mimetype in Content-Encoding.
@@ -189,7 +186,10 @@ class IPCInterceptionAddon:
             flow.response.headers["Content-Encoding"] = "identity"
 
     def response(self, flow: HTTPFlow):
-        if flow.metadata["response_injected"]:
+        cap_data: typing.Optional[SerializedCapData] = flow.metadata.get("cap_data")
+        if flow.metadata.get("response_injected") and cap_data and cap_data.asset_server_cap:
+            # Don't bother intercepting asset server requests where we injected a response.
+            # We don't want to log them and they don't need any more processing by user hooks.
             return
         self._queue_flow_interception("response", flow)
 
@@ -197,10 +197,10 @@ class IPCInterceptionAddon:
 class SLMITMAddon(IPCInterceptionAddon):
     def responseheaders(self, flow: HTTPFlow):
         super().responseheaders(flow)
-        cap_data: typing.Optional[SerializedCapData] = flow.metadata["cap_data_ser"]
+        cap_data: typing.Optional[SerializedCapData] = flow.metadata.get("cap_data_ser")
 
         # Request came from the proxy itself, don't touch it.
-        if flow.metadata["request_injected"]:
+        if flow.metadata.get("request_injected"):
             return
 
         # This is an asset server response that we're not interested in intercepting.
@@ -209,7 +209,7 @@ class SLMITMAddon(IPCInterceptionAddon):
             # Can't stream if we injected our own response or we were asked not to stream
             if not flow.metadata["response_injected"] and flow.metadata["can_stream"]:
                 flow.response.stream = True
-        elif not cap_data and not flow.metadata["from_browser"]:
+        elif not cap_data and not flow.metadata.get("from_browser"):
             object_name = flow.response.headers.get("X-SecondLife-Object-Name", "")
             # Meh. Add some fake Cap data for this so it can be matched on.
             if object_name.startswith("#Firestorm LSL Bridge"):

hippolyzer/lib/proxy/lludp_proxy.py

@@ -1,3 +1,4 @@
+import asyncio
 import logging
 import weakref
 from typing import Optional, Tuple
@@ -35,6 +36,17 @@ class InterceptingLLUDPProxyProtocol(UDPProxyProtocol):
         )
         self.message_xml = MessageDotXML()
         self.session: Optional[Session] = None
+        self.resend_task = asyncio.get_event_loop().create_task(self.attempt_resends())
+
+    async def attempt_resends(self):
+        while True:
+            await asyncio.sleep(0.1)
+            if self.session is None:
+                continue
+            for region in self.session.regions:
+                if not region.circuit or not region.circuit.is_alive:
+                    continue
+                region.circuit.resend_unacked()
 
     def _ensure_message_allowed(self, msg: Message):
         if not self.message_xml.validate_udp_msg(msg.name):
@@ -99,6 +111,9 @@ class InterceptingLLUDPProxyProtocol(UDPProxyProtocol):
             LOG.error("No circuit for %r, dropping packet!" % (packet.far_addr,))
             return
 
+        # Process any ACKs for messages we injected first
+        region.circuit.collect_acks(message)
+
         if message.name == "AgentMovementComplete":
             self.session.main_region = region
             if region.handle is None:
@@ -148,7 +163,7 @@ class InterceptingLLUDPProxyProtocol(UDPProxyProtocol):
 
         # Send the message if it wasn't explicitly dropped or sent before
         if not message.finalized:
-            region.circuit.send_message(message)
+            region.circuit.send(message)
 
     def close(self):
         super().close()
@@ -156,3 +171,4 @@ class InterceptingLLUDPProxyProtocol(UDPProxyProtocol):
             AddonManager.handle_session_closed(self.session)
             self.session_manager.close_session(self.session)
         self.session = None
+        self.resend_task.cancel()

hippolyzer/lib/proxy/message_filter.py

@@ -3,7 +3,7 @@ import ast
 import typing
 
 from arpeggio import Optional, ZeroOrMore, EOF, \
-    ParserPython, PTNodeVisitor, visit_parse_tree, RegExMatch
+    ParserPython, PTNodeVisitor, visit_parse_tree, RegExMatch, OneOrMore
 
 
 def literal():
@@ -26,7 +26,9 @@ def literal():
 
 
 def identifier():
-    return RegExMatch(r'[a-zA-Z*]([a-zA-Z0-9_*]+)?')
+    # Identifiers are allowed to have "-". It's not a special character
+    # in our grammar, and we expect them to show up some places, like header names.
+    return RegExMatch(r'[a-zA-Z*]([a-zA-Z0-9_*-]+)?')
 
 
 def field_specifier():
@@ -42,7 +44,7 @@ def unary_expression():
 
 
 def meta_field_specifier():
-    return "Meta", ".", identifier
+    return "Meta", OneOrMore(".", identifier)
 
 
 def enum_field_specifier():
@@ -69,12 +71,17 @@ def message_filter():
     return expression, EOF
 
 
-MATCH_RESULT = typing.Union[bool, typing.Tuple]
+class MatchResult(typing.NamedTuple):
+    result: bool
+    fields: typing.List[typing.Tuple]
+
+    def __bool__(self):
+        return self.result
 
 
 class BaseFilterNode(abc.ABC):
     @abc.abstractmethod
-    def match(self, msg) -> MATCH_RESULT:
+    def match(self, msg, short_circuit=True) -> MatchResult:
         raise NotImplementedError()
 
     @property
@@ -104,18 +111,36 @@ class BinaryFilterNode(BaseFilterNode, abc.ABC):
 
 
 class UnaryNotFilterNode(UnaryFilterNode):
-    def match(self, msg) -> MATCH_RESULT:
-        return not self.node.match(msg)
+    def match(self, msg, short_circuit=True) -> MatchResult:
+        # Should we pass fields up here? Maybe not.
+        return MatchResult(not self.node.match(msg, short_circuit), [])
 
 
 class OrFilterNode(BinaryFilterNode):
-    def match(self, msg) -> MATCH_RESULT:
-        return self.left_node.match(msg) or self.right_node.match(msg)
+    def match(self, msg, short_circuit=True) -> MatchResult:
+        left_match = self.left_node.match(msg, short_circuit)
+        if left_match and short_circuit:
+            return MatchResult(True, left_match.fields)
+
+        right_match = self.right_node.match(msg, short_circuit)
+        if right_match and short_circuit:
+            return MatchResult(True, right_match.fields)
+
+        if left_match or right_match:
+            # Fine since fields should be empty when result=False
+            return MatchResult(True, left_match.fields + right_match.fields)
+        return MatchResult(False, [])
 
 
 class AndFilterNode(BinaryFilterNode):
-    def match(self, msg) -> MATCH_RESULT:
-        return self.left_node.match(msg) and self.right_node.match(msg)
+    def match(self, msg, short_circuit=True) -> MatchResult:
+        left_match = self.left_node.match(msg, short_circuit)
+        if not left_match:
+            return MatchResult(False, [])
+        right_match = self.right_node.match(msg, short_circuit)
+        if not right_match:
+            return MatchResult(False, [])
+        return MatchResult(True, left_match.fields + right_match.fields)
 
 
 class MessageFilterNode(BaseFilterNode):
@@ -124,15 +149,15 @@ class MessageFilterNode(BaseFilterNode):
         self.operator = operator
         self.value = value
 
-    def match(self, msg) -> MATCH_RESULT:
-        return msg.matches(self)
+    def match(self, msg, short_circuit=True) -> MatchResult:
+        return msg.matches(self, short_circuit)
 
     @property
     def children(self):
         return self.selector, self.operator, self.value
 
 
-class MetaFieldSpecifier(str):
+class MetaFieldSpecifier(tuple):
     pass
 
 
@@ -158,7 +183,7 @@ class MessageFilterVisitor(PTNodeVisitor):
         return LiteralValue(ast.literal_eval(node.value))
 
     def visit_meta_field_specifier(self, _node, children):
-        return MetaFieldSpecifier(children[0])
+        return MetaFieldSpecifier(children)
 
     def visit_enum_field_specifier(self, _node, children):
         return EnumFieldSpecifier(*children)

hippolyzer/lib/proxy/message_logger.py

@@ -21,7 +21,7 @@ from hippolyzer.lib.base.datatypes import TaggedUnion, UUID, TupleCoord
 from hippolyzer.lib.base.helpers import bytes_escape
 from hippolyzer.lib.base.message.message_formatting import HumanMessageSerializer
 from hippolyzer.lib.proxy.message_filter import MetaFieldSpecifier, compile_filter, BaseFilterNode, MessageFilterNode, \
-    EnumFieldSpecifier
+    EnumFieldSpecifier, MatchResult
 from hippolyzer.lib.proxy.http_flow import HippoHTTPFlow
 from hippolyzer.lib.proxy.caps import CapType, SerializedCapData
 
@@ -235,7 +235,7 @@ class AbstractMessageLogEntry(abc.ABC):
         obj = self.region.objects.lookup_localid(selected_local)
         return obj and obj.FullID
 
-    def _get_meta(self, name: str):
+    def _get_meta(self, name: str) -> typing.Any:
         # Slight difference in semantics. Filters are meant to return the same
         # thing no matter when they're run, so SelectedLocal and friends resolve
         # to the selected items _at the time the message was logged_. To handle
@@ -308,7 +308,9 @@ class AbstractMessageLogEntry(abc.ABC):
 
     def _val_matches(self, operator, val, expected):
         if isinstance(expected, MetaFieldSpecifier):
-            expected = self._get_meta(str(expected))
+            if len(expected) != 1:
+                raise ValueError(f"Can only support single-level Meta specifiers, not {expected!r}")
+            expected = self._get_meta(str(expected[0]))
             if not isinstance(expected, (int, float, bytes, str, type(None), tuple)):
                 if callable(expected):
                     expected = expected()
@@ -362,12 +364,18 @@ class AbstractMessageLogEntry(abc.ABC):
             if matcher.value or matcher.operator:
                 return False
             return self._packet_root_matches(matcher.selector[0])
-        if len(matcher.selector) == 2 and matcher.selector[0] == "Meta":
-            return self._val_matches(matcher.operator, self._get_meta(matcher.selector[1]), matcher.value)
+        if matcher.selector[0] == "Meta":
+            if len(matcher.selector) == 2:
+                return self._val_matches(matcher.operator, self._get_meta(matcher.selector[1]), matcher.value)
+            elif len(matcher.selector) == 3:
+                meta_dict = self._get_meta(matcher.selector[1])
+                if not meta_dict or not hasattr(meta_dict, 'get'):
+                    return False
+                return self._val_matches(matcher.operator, meta_dict.get(matcher.selector[2]), matcher.value)
         return None
 
-    def matches(self, matcher: "MessageFilterNode"):
-        return self._base_matches(matcher) or False
+    def matches(self, matcher: "MessageFilterNode", short_circuit=True) -> "MatchResult":
+        return MatchResult(self._base_matches(matcher) or False, [])
 
     @property
     def seq(self):
@@ -388,6 +396,14 @@ class AbstractMessageLogEntry(abc.ABC):
         xmlified = re.sub(rb" <key>", b"<key>", xmlified)
         return xmlified.decode("utf8", errors="replace")
 
+    @staticmethod
+    def _format_xml(content):
+        beautified = minidom.parseString(content).toprettyxml(indent="  ")
+        # kill blank lines. will break cdata sections. meh.
+        beautified = re.sub(r'\n\s*\n', '\n', beautified, flags=re.MULTILINE)
+        return re.sub(r'<([\w]+)>\s*</\1>', r'<\1></\1>',
+                      beautified, flags=re.MULTILINE)
+
 
 class HTTPMessageLogEntry(AbstractMessageLogEntry):
     __slots__ = ["flow"]
@@ -400,7 +416,7 @@ class HTTPMessageLogEntry(AbstractMessageLogEntry):
 
         super().__init__(region, session)
         # This was a request the proxy made through itself
-        self.meta["Injected"] = flow.request_injected
+        self.meta["Synthetic"] = flow.request_injected
 
     @property
     def type(self):
@@ -476,13 +492,17 @@ class HTTPMessageLogEntry(AbstractMessageLogEntry):
                 if not beautified:
                     content_type = self._guess_content_type(message)
                     if content_type.startswith("application/llsd"):
-                        beautified = self._format_llsd(llsd.parse(message.content))
+                        try:
+                            beautified = self._format_llsd(llsd.parse(message.content))
+                        except llsd.LLSDParseError:
+                            # Sometimes LL sends plain XML with a Content-Type of application/llsd+xml.
+                            # Try to detect that case and work around it
+                            if content_type == "application/llsd+xml" and message.content.startswith(b'<'):
+                                beautified = self._format_xml(message.content)
+                            else:
+                                raise
                     elif any(content_type.startswith(x) for x in ("application/xml", "text/xml")):
-                        beautified = minidom.parseString(message.content).toprettyxml(indent="  ")
-                        # kill blank lines. will break cdata sections. meh.
-                        beautified = re.sub(r'\n\s*\n', '\n', beautified, flags=re.MULTILINE)
-                        beautified = re.sub(r'<([\w]+)>\s*</\1>', r'<\1></\1>',
-                                            beautified, flags=re.MULTILINE)
+                        beautified = self._format_xml(message.content)
             except:
                 LOG.exception("Failed to beautify message")
 
@@ -541,6 +561,20 @@ class HTTPMessageLogEntry(AbstractMessageLogEntry):
             return "application/xml"
         return content_type
 
+    def _get_meta(self, name: str) -> typing.Any:
+        lower_name = name.lower()
+        if lower_name == "url":
+            return self.flow.request.url
+        elif lower_name == "reqheaders":
+            return self.flow.request.headers
+        elif lower_name == "respheaders":
+            return self.flow.response.headers
+        elif lower_name == "host":
+            return self.flow.request.host.lower()
+        elif lower_name == "status":
+            return self.flow.response.status_code
+        return super()._get_meta(name)
+
     def to_dict(self):
         val = super().to_dict()
         val['flow'] = self.flow.get_state()
@@ -613,7 +647,7 @@ class LLUDPMessageLogEntry(AbstractMessageLogEntry):
         super().__init__(region, session)
 
     _MESSAGE_META_ATTRS = {
-        "Injected", "Dropped", "Extra", "Resent", "Zerocoded", "Acks", "Reliable",
+        "Synthetic", "Dropped", "Extra", "Resent", "Zerocoded", "Acks", "Reliable",
     }
 
     def _get_meta(self, name: str):
@@ -671,20 +705,21 @@ class LLUDPMessageLogEntry(AbstractMessageLogEntry):
     def request(self, beautify=False, replacements=None):
         return HumanMessageSerializer.to_human_string(self.message, replacements, beautify)
 
-    def matches(self, matcher):
+    def matches(self, matcher, short_circuit=True) -> "MatchResult":
         base_matched = self._base_matches(matcher)
         if base_matched is not None:
-            return base_matched
+            return MatchResult(base_matched, [])
 
         if not self._packet_root_matches(matcher.selector[0]):
-            return False
+            return MatchResult(False, [])
 
         message = self.message
 
         selector_len = len(matcher.selector)
         # name, block_name, var_name(, subfield_name)?
         if selector_len not in (3, 4):
-            return False
+            return MatchResult(False, [])
+        found_field_keys = []
         for block_name in message.blocks:
             if not fnmatch.fnmatchcase(block_name, matcher.selector[1]):
                 continue
@@ -693,13 +728,13 @@ class LLUDPMessageLogEntry(AbstractMessageLogEntry):
                     if not fnmatch.fnmatchcase(var_name, matcher.selector[2]):
                         continue
                     # So we know where the match happened
-                    span_key = (message.name, block_name, block_num, var_name)
+                    field_key = (message.name, block_name, block_num, var_name)
                     if selector_len == 3:
                         # We're just matching on the var existing, not having any particular value
                         if matcher.value is None:
-                            return span_key
-                        if self._val_matches(matcher.operator, block[var_name], matcher.value):
-                            return span_key
+                            found_field_keys.append(field_key)
+                        elif self._val_matches(matcher.operator, block[var_name], matcher.value):
+                            found_field_keys.append(field_key)
                     # Need to invoke a special unpacker
                     elif selector_len == 4:
                         try:
@@ -710,15 +745,21 @@ class LLUDPMessageLogEntry(AbstractMessageLogEntry):
                         if isinstance(deserialized, TaggedUnion):
                             deserialized = deserialized.value
                         if not isinstance(deserialized, dict):
-                            return False
+                            continue
                         for key in deserialized.keys():
                             if fnmatch.fnmatchcase(str(key), matcher.selector[3]):
                                 if matcher.value is None:
-                                    return span_key
-                                if self._val_matches(matcher.operator, deserialized[key], matcher.value):
-                                    return span_key
+                                    # Short-circuiting checking individual subfields is fine since
+                                    # we only highlight fields anyway.
+                                    found_field_keys.append(field_key)
+                                    break
+                                elif self._val_matches(matcher.operator, deserialized[key], matcher.value):
+                                    found_field_keys.append(field_key)
+                                    break
 
-        return False
+                    if short_circuit and found_field_keys:
+                        return MatchResult(True, found_field_keys)
+        return MatchResult(bool(found_field_keys), found_field_keys)
 
     @property
     def summary(self):

hippolyzer/lib/proxy/object_manager.py

@@ -130,9 +130,9 @@ class ProxyWorldObjectManager(ClientWorldObjectManager):
             if obj.PCode == PCode.AVATAR and "ParentID" in updated_props:
                 if obj.ParentID and not region.objects.lookup_localid(obj.ParentID):
                     # If an avatar just sat on an object we don't know about, add it to the queued
-                    # cache misses and request if if the viewer doesn't. This should happen
-                    # regardless of the auto-request object setting because otherwise we have no way
-                    # to get a sitting agent's true region location, even if it's ourself.
+                    # cache misses and request it if the viewer doesn't. This should happen
+                    # regardless of the auto-request missing objects setting because otherwise we
+                    # have no way to get a sitting agent's true region location, even if it's ourselves.
                     region.objects.queued_cache_misses.add(obj.ParentID)
                     region.objects.request_missed_cached_objects_soon()
         AddonManager.handle_object_updated(self._session, region, obj, updated_props)

hippolyzer/lib/proxy/region.py

@@ -51,10 +51,11 @@ class ProxiedRegion(BaseClientRegion):
         self.cache_id: Optional[UUID] = None
         self.circuit: Optional[ProxiedCircuit] = None
         self.circuit_addr = circuit_addr
-        self._caps = CapsMultiDict()
+        self.caps = CapsMultiDict()
+        # Reverse lookup for URL -> cap data
         self._caps_url_lookup: Dict[str, Tuple[CapType, str]] = {}
         if seed_cap:
-            self._caps["Seed"] = (CapType.NORMAL, seed_cap)
+            self.caps["Seed"] = (CapType.NORMAL, seed_cap)
         self.session: Callable[[], Session] = weakref.ref(session)
         self.message_handler: MessageHandler[Message, str] = MessageHandler()
         self.http_message_handler: MessageHandler[HippoHTTPFlow, str] = MessageHandler()
@@ -77,8 +78,8 @@ class ProxiedRegion(BaseClientRegion):
         self._name = val
 
     @property
-    def caps(self):
-        return multidict.MultiDict((x, y[1]) for x, y in self._caps.items())
+    def cap_urls(self) -> multidict.MultiDict[str, str]:
+        return multidict.MultiDict((x, y[1]) for x, y in self.caps.items())
 
     @property
     def global_pos(self) -> Vector3:
@@ -95,12 +96,12 @@ class ProxiedRegion(BaseClientRegion):
     def update_caps(self, caps: Mapping[str, str]):
         for cap_name, cap_url in caps.items():
             if isinstance(cap_url, str) and cap_url.startswith('http'):
-                self._caps.add(cap_name, (CapType.NORMAL, cap_url))
+                self.caps.add(cap_name, (CapType.NORMAL, cap_url))
                 self._recalc_caps()
 
     def _recalc_caps(self):
         self._caps_url_lookup.clear()
-        for name, cap_info in self._caps.items():
+        for name, cap_info in self.caps.items():
             cap_type, cap_url = cap_info
             self._caps_url_lookup[cap_url] = (cap_type, name)
 
@@ -109,32 +110,31 @@ class ProxiedRegion(BaseClientRegion):
         Wrap an existing, non-unique cap with a unique URL
 
         caps like ViewerAsset may be the same globally and wouldn't let us infer
-        which session / region the request was related to without a wrapper
+        which session / region the request was related to without a wrapper URL
+        that we inject into the seed response sent to the viewer.
         """
-        parsed = list(urllib.parse.urlsplit(self._caps[name][1]))
-        seed_id = self._caps["Seed"][1].split("/")[-1].encode("utf8")
+        parsed = list(urllib.parse.urlsplit(self.caps[name][1]))
+        seed_id = self.caps["Seed"][1].split("/")[-1].encode("utf8")
         # Give it a unique domain tied to the current Seed URI
         parsed[1] = f"{name.lower()}-{hashlib.sha256(seed_id).hexdigest()[:16]}.hippo-proxy.localhost"
         # Force the URL to HTTP, we're going to handle the request ourselves so it doesn't need
         # to be secure. This should save on expensive TLS context setup for each req.
         parsed[0] = "http"
         wrapper_url = urllib.parse.urlunsplit(parsed)
-        self._caps.add(name + "ProxyWrapper", (CapType.WRAPPER, wrapper_url))
-        self._recalc_caps()
+        # Register it with "ProxyWrapper" appended so we don't shadow the real cap URL
+        # in our own view of the caps
+        self.register_cap(name + "ProxyWrapper", wrapper_url, CapType.WRAPPER)
         return wrapper_url
 
     def register_proxy_cap(self, name: str):
-        """
-        Register a cap to be completely handled by the proxy
-        """
-        cap_url = f"https://caps.hippo-proxy.localhost/cap/{uuid.uuid4()!s}"
-        self._caps.add(name, (CapType.PROXY_ONLY, cap_url))
-        self._recalc_caps()
+        """Register a cap to be completely handled by the proxy"""
+        cap_url = f"http://{uuid.uuid4()!s}.caps.hippo-proxy.localhost"
+        self.register_cap(name, cap_url, CapType.PROXY_ONLY)
         return cap_url
 
-    def register_temporary_cap(self, name: str, cap_url: str):
+    def register_cap(self, name: str, cap_url: str, cap_type: CapType = CapType.NORMAL):
         """Register a Cap that only has meaning the first time it's used"""
-        self._caps.add(name, (CapType.TEMPORARY, cap_url))
+        self.caps.add(name, (cap_type, cap_url))
         self._recalc_caps()
 
     def resolve_cap(self, url: str, consume=True) -> Optional[Tuple[str, str, CapType]]:
@@ -143,9 +143,9 @@ class ProxiedRegion(BaseClientRegion):
                 cap_type, name = self._caps_url_lookup[cap_url]
                 if cap_type == CapType.TEMPORARY and consume:
                     # Resolving a temporary cap pops it out of the dict
-                    temporary_caps = self._caps.popall(name)
+                    temporary_caps = self.caps.popall(name)
                     temporary_caps.remove((cap_type, cap_url))
-                    self._caps.extend((name, x) for x in temporary_caps)
+                    self.caps.extend((name, x) for x in temporary_caps)
                     self._recalc_caps()
                 return name, cap_url, cap_type
         return None

hippolyzer/lib/proxy/sessions.py

@@ -99,12 +99,12 @@ class Session(BaseClientSession):
 
         for region in self.regions:
             if region.circuit_addr == circuit_addr:
-                if seed_url and region.caps.get("Seed") != seed_url:
+                if seed_url and region.cap_urls.get("Seed") != seed_url:
                     region.update_caps({"Seed": seed_url})
                 if handle:
                     region.handle = handle
                 return region
-            if seed_url and region.caps.get("Seed") == seed_url:
+            if seed_url and region.cap_urls.get("Seed") == seed_url:
                 return region
 
         if not circuit_addr:
@@ -113,6 +113,7 @@ class Session(BaseClientSession):
         logging.info("Registering region for %r" % (circuit_addr,))
         region = ProxiedRegion(circuit_addr, seed_url, self, handle=handle)
         self.regions.append(region)
+        AddonManager.handle_region_registered(self, region)
         return region
 
     def region_by_circuit_addr(self, circuit_addr) -> Optional[ProxiedRegion]:

hippolyzer/lib/proxy/test_utils.py

@@ -37,6 +37,9 @@ class BaseProxyTest(unittest.IsolatedAsyncioTestCase):
         self.serializer = UDPMessageSerializer()
         self.session.objects.track_region_objects(123)
 
+    def tearDown(self) -> None:
+        self.protocol.close()
+
     async def _wait_drained(self):
         await asyncio.sleep(0.001)
 

hippolyzer/lib/proxy/webapp_cap_addon.py

@@ -0,0 +1,46 @@
+import abc
+
+from mitmproxy.addons import asgiapp
+from mitmproxy.controller import DummyReply
+
+from hippolyzer.lib.proxy.addon_utils import BaseAddon
+from hippolyzer.lib.proxy.http_flow import HippoHTTPFlow
+from hippolyzer.lib.proxy.region import ProxiedRegion
+from hippolyzer.lib.proxy.sessions import Session, SessionManager
+
+
+async def serve(app, flow: HippoHTTPFlow):
+    """Serve a request based on a Hippolyzer HTTP flow using a provided app"""
+    # Shove this on mitmproxy's flow object so asgiapp doesn't explode when it tries
+    # to commit the flow reply. Our take / commit semantics are different than mitmproxy
+    # proper, so we ignore what mitmproxy sets here anyhow.
+    flow.flow.reply = DummyReply()
+    flow.flow.reply.take()
+    await asgiapp.serve(app, flow.flow)
+    flow.flow.reply = None
+    # Send the modified flow object back to mitmproxy
+    flow.resume()
+
+
+class WebAppCapAddon(BaseAddon, abc.ABC):
+    """
+    Addon that provides a cap via an ASGI webapp
+
+    Handles all registration of the cap URL and routing of the request.
+    """
+    CAP_NAME: str
+    APP: any
+
+    def handle_region_registered(self, session: Session, region: ProxiedRegion):
+        # Register a fake URL for our cap. This will add the cap URL to the Seed
+        # response that gets sent back to the client if that cap name was requested.
+        if self.CAP_NAME not in region.cap_urls:
+            region.register_proxy_cap(self.CAP_NAME)
+
+    def handle_http_request(self, session_manager: SessionManager, flow: HippoHTTPFlow):
+        if flow.cap_data.cap_name != self.CAP_NAME:
+            return
+        # This request may take a while to generate a response for, take it out of the normal
+        # HTTP handling flow and handle it in a async task.
+        # TODO: Make all HTTP handling hooks async so this isn't necessary
+        self._schedule_task(serve(self.APP, flow.take()))

requirements.txt

@@ -1,65 +1,66 @@
-aiohttp==3.7.4.post0
+aiohttp==3.8.1
+aiosignal==1.2.0
 appdirs==1.4.4
 Arpeggio==1.10.2
 asgiref==3.4.1
-async-timeout==3.0.1
+async-timeout==4.0.1
 attrs==21.2.0
 blinker==1.4
 Brotli==1.0.9
-certifi==2021.5.30
-cffi==1.14.6
-chardet==4.0.0
-charset-normalizer==2.0.3
-click==8.0.1
-cryptography==3.4.7
+certifi==2021.10.8
+cffi==1.15.0
+charset-normalizer==2.0.9
+click==8.0.3
+cryptography==3.4.8
 defusedxml==0.7.1
-Flask==2.0.1
-Glymur==0.9.3
+Flask==2.0.2
+frozenlist==1.2.0
+Glymur==0.9.6
 h11==0.12.0
-h2==4.0.0
+h2==4.1.0
 hpack==4.0.0
 hyperframe==6.0.1
 idna==2.10
 itsdangerous==2.0.1
-jedi==0.18.0
-Jinja2==3.0.1
+jedi==0.18.1
+Jinja2==3.0.3
 kaitaistruct==0.9
 lazy-object-proxy==1.6.0
-ldap3==2.9
+ldap3==2.9.1
 llbase==1.2.11
-lxml==4.6.3
+lxml==4.6.4
 MarkupSafe==2.0.1
-mitmproxy==7.0.2
-msgpack==1.0.2
-multidict==5.1.0
-numpy==1.21.0
-parso==0.8.2
+mitmproxy==7.0.4
+msgpack==1.0.3
+multidict==5.2.0
+numpy==1.21.4
+parso==0.8.3
 passlib==1.7.4
-prompt-toolkit==3.0.19
-protobuf==3.17.3
-ptpython==3.0.19
+prompt-toolkit==3.0.23
+protobuf==3.18.1
+ptpython==3.0.20
 publicsuffix2==2.20191221
 pyasn1==0.4.8
-pycparser==2.20
-Pygments==2.9.0
+pycparser==2.21
+Pygments==2.10.0
 pyOpenSSL==20.0.1
 pyparsing==2.4.7
 pyperclip==1.8.2
-PySide2==5.15.2
-qasync==0.17.0
+PySide6==6.2.2
+qasync==0.22.0
 recordclass==0.14.3
 requests==2.26.0
-ruamel.yaml==0.17.10
+ruamel.yaml==0.17.16
 ruamel.yaml.clib==0.2.6
-shiboken2==5.15.2
+shiboken6==6.2.2
 six==1.16.0
 sortedcontainers==2.4.0
 tornado==6.1
-typing-extensions==3.10.0.0
-urllib3==1.26.6
+typing-extensions==4.0.1
+urllib3==1.26.7
 urwid==2.1.2
 wcwidth==0.2.5
-Werkzeug==2.0.1
+Werkzeug==2.0.2
 wsproto==1.0.0
-yarl==1.6.3
+yarl==1.7.2
 zstandard==0.15.2

setup.py

@@ -25,7 +25,7 @@ from setuptools import setup, find_packages
 
 here = path.abspath(path.dirname(__file__))
 
-version = '0.7.0'
+version = '0.9.0'
 
 with open(path.join(here, 'README.md')) as readme_fh:
     readme = readme_fh.read()
@@ -44,6 +44,7 @@ setup(
         "Programming Language :: Python :: 3 :: Only",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: Implementation :: CPython",
         "Topic :: System :: Networking :: Monitoring",
         "Topic :: Software Development :: Libraries :: Python Modules",
@@ -82,7 +83,7 @@ setup(
         'llbase>=1.2.5',
         'defusedxml',
         'aiohttp<4.0.0',
-        'recordclass',
+        'recordclass<0.15',
         'lazy-object-proxy',
         'arpeggio',
         # requests breaks with newer idna
@@ -92,10 +93,10 @@ setup(
         # For REPLs
         'ptpython<4.0',
         # JP2 codec
-        'Glymur<1.0',
+        'Glymur<0.9.7',
         'numpy<2.0',
         # These could be in extras_require if you don't want a GUI.
-        'pyside2<6.0',
+        'pyside6',
         'qasync',
     ],
     tests_require=[

setup_cxfreeze.py

@@ -9,20 +9,21 @@ from cx_Freeze import setup, Executable
 
 # We don't need any of these and they make the archive huge.
 TO_DELETE = [
-    "lib/PySide2/Qt3DRender.pyd",
-    "lib/PySide2/Qt53DRender.dll",
-    "lib/PySide2/Qt5Charts.dll",
-    "lib/PySide2/Qt5Location.dll",
-    "lib/PySide2/Qt5Pdf.dll",
-    "lib/PySide2/Qt5Quick.dll",
-    "lib/PySide2/Qt5WebEngineCore.dll",
-    "lib/PySide2/QtCharts.pyd",
-    "lib/PySide2/QtMultimedia.pyd",
-    "lib/PySide2/QtOpenGLFunctions.pyd",
-    "lib/PySide2/QtOpenGLFunctions.pyi",
-    "lib/PySide2/d3dcompiler_47.dll",
-    "lib/PySide2/opengl32sw.dll",
-    "lib/PySide2/translations",
+    "lib/PySide6/Qt6DRender.pyd",
+    "lib/PySide6/Qt63DRender.dll",
+    "lib/PySide6/Qt6Charts.dll",
+    "lib/PySide6/Qt6Location.dll",
+    "lib/PySide6/Qt6Pdf.dll",
+    "lib/PySide6/Qt6Quick.dll",
+    "lib/PySide6/Qt6WebEngineCore.dll",
+    "lib/PySide6/QtCharts.pyd",
+    "lib/PySide6/QtMultimedia.pyd",
+    "lib/PySide6/QtOpenGLFunctions.pyd",
+    "lib/PySide6/QtOpenGLFunctions.pyi",
+    "lib/PySide6/d3dcompiler_47.dll",
+    "lib/PySide6/opengl32sw.dll",
+    "lib/PySide6/lupdate.exe",
+    "lib/PySide6/translations",
     "lib/aiohttp/_find_header.c",
     "lib/aiohttp/_frozenlist.c",
     "lib/aiohttp/_helpers.c",
@@ -112,7 +113,7 @@ executables = [
 
 setup(
     name="hippolyzer_gui",
-    version="0.7.0",
+    version="0.9.0",
     description="Hippolyzer GUI",
     options=options,
     executables=executables,

tests/base/test_datatypes.py

@@ -134,3 +134,15 @@ class TestDatatypes(unittest.TestCase):
         val = llsd.parse_binary(llsd.format_binary(orig))
         self.assertIsInstance(val, UUID)
         self.assertEqual(orig, val)
+
+    def test_jank_stringy_bytes(self):
+        val = JankStringyBytes(b"foo\x00")
+        self.assertTrue("o" in val)
+        self.assertTrue(b"o" in val)
+        self.assertFalse(b"z" in val)
+        self.assertFalse("z" in val)
+        self.assertEqual("foo", val)
+        self.assertEqual(b"foo\x00", val)
+        self.assertNotEqual(b"foo", val)
+        self.assertEqual(b"foo", JankStringyBytes(b"foo"))
+        self.assertEqual("foo", JankStringyBytes(b"foo"))

tests/base/test_legacy_schema.py

@@ -1,7 +1,7 @@
 import unittest
 
 from hippolyzer.lib.base.datatypes import *
-from hippolyzer.lib.base.legacy_inv import InventoryModel
+from hippolyzer.lib.base.inventory import InventoryModel
 from hippolyzer.lib.base.wearables import Wearable, VISUAL_PARAMS
 
 SIMPLE_INV = """\tinv_object\t0
@@ -61,6 +61,51 @@ class TestLegacyInv(unittest.TestCase):
         self.assertEqual(item.sale_info.sale_type, "not")
         self.assertEqual(item.model, model)
 
+    def test_llsd_serialization(self):
+        model = InventoryModel.from_str(SIMPLE_INV)
+        self.assertEqual(
+            model.to_llsd(),
+            [
+                {
+                    'name': 'Contents',
+                    'obj_id': UUID('f4d91477-def1-487a-b4f3-6fa201c17376'),
+                    'parent_id': UUID('00000000-0000-0000-0000-000000000000'),
+                    'type': 'category'
+                },
+                {
+                    'asset_id': UUID('00000000-0000-0000-0000-000000000000'),
+                    'created_at': 1587367239,
+                    'desc': '2020-04-20 04:20:39 lsl2 script',
+                    'flags': b'\x00\x00\x00\x00',
+                    'inv_type': 'script',
+                    'item_id': UUID('dd163122-946b-44df-99f6-a6030e2b9597'),
+                    'name': 'New Script',
+                    'parent_id': UUID('f4d91477-def1-487a-b4f3-6fa201c17376'),
+                    'permissions': {
+                        'base_mask': 2147483647,
+                        'creator_id': UUID('a2e76fcd-9360-4f6d-a924-000000000003'),
+                        'everyone_mask': 0,
+                        'group_id': UUID('00000000-0000-0000-0000-000000000000'),
+                        'group_mask': 0,
+                        'last_owner_id': UUID('a2e76fcd-9360-4f6d-a924-000000000003'),
+                        'next_owner_mask': 581632,
+                        'owner_id': UUID('a2e76fcd-9360-4f6d-a924-000000000003'),
+                        'owner_mask': 2147483647
+                    },
+                    'sale_info': {
+                        'sale_price': 10,
+                        'sale_type': 'not'
+                    },
+                    'type': 'lsltext'
+                }
+            ]
+        )
+
+    def test_llsd_legacy_equality(self):
+        model = InventoryModel.from_str(SIMPLE_INV)
+        new_model = InventoryModel.from_llsd(model.to_llsd())
+        self.assertEqual(model, new_model)
+
 
 GIRL_NEXT_DOOR_SHAPE = """LLWearable version 22
 Girl Next Door - C2 - med - Adam n Eve

tests/base/test_message_wrapper.py

@@ -300,3 +300,14 @@ class HumanReadableMessageTests(unittest.TestCase):
 
         with self.assertRaises(ValueError):
             HumanMessageSerializer.from_human_string(val)
+
+    def test_flags(self):
+        val = """
+OUT FooMessage [ZEROCODED] [RELIABLE] [1]
+
+[SomeBlock]
+  foo = 1
+        """
+
+        msg = HumanMessageSerializer.from_human_string(val)
+        self.assertEqual(HumanMessageSerializer.to_human_string(msg).strip(), val.strip())

tests/base/test_xfer_transfer.py

@@ -70,7 +70,7 @@ class XferManagerTests(BaseTransferTests):
             manager = XferManager(self.server_connection)
             xfer = await manager.request(vfile_id=asset_id, vfile_type=AssetType.BODYPART)
             self.received_bytes = xfer.reassemble_chunks()
-        self.server_circuit.send_message(Message(
+        self.server_circuit.send(Message(
             "AssetUploadComplete",
             Block("AssetBlock", UUID=asset_id, Type=asset_block["Type"], Success=True),
             direction=Direction.IN,
@@ -109,7 +109,7 @@ class TestTransferManager(BaseTransferTests):
         self.assertEqual(EstateAssetType.COVENANT, params.EstateAssetType)
         data = self.LARGE_PAYLOAD
 
-        self.server_circuit.send_message(Message(
+        self.server_circuit.send(Message(
             'TransferInfo',
             Block(
                 'TransferInfo',
@@ -125,7 +125,7 @@ class TestTransferManager(BaseTransferTests):
         while True:
             chunk = data[:1000]
             data = data[1000:]
-            self.server_circuit.send_message(Message(
+            self.server_circuit.send(Message(
                 'TransferPacket',
                 Block(
                     'TransferData',

tests/proxy/integration/test_http.py

@@ -145,25 +145,26 @@ class TestMITMProxy(BaseProxyTest):
         super().setUp()
         self._setup_default_circuit()
         self.caps_client = self.session.main_region.caps_client
-
-    def test_mitmproxy_works(self):
         proxy_port = 9905
         self.session_manager.settings.HTTP_PROXY_PORT = proxy_port
 
-        http_proc = multiprocessing.Process(
+        self.http_proc = multiprocessing.Process(
             target=run_http_proxy_process,
             args=("127.0.0.1", proxy_port, self.session_manager.flow_context),
             daemon=True,
         )
-        http_proc.start()
-
+        self.http_proc.start()
         self.session_manager.flow_context.mitmproxy_ready.wait(1.0)
 
-        http_event_manager = MITMProxyEventManager(self.session_manager, self.session_manager.flow_context)
+        self.http_event_manager = MITMProxyEventManager(
+            self.session_manager,
+            self.session_manager.flow_context
+        )
 
+    def test_mitmproxy_works(self):
         async def _request_example_com():
             # Pump callbacks from mitmproxy
-            asyncio.create_task(http_event_manager.run())
+            asyncio.create_task(self.http_event_manager.run())
             try:
                 async with self.caps_client.get("http://example.com/", timeout=0.5) as resp:
                     self.assertIn(b"Example Domain", await resp.read())
@@ -173,4 +174,4 @@ class TestMITMProxy(BaseProxyTest):
                 # Tell the event pump and mitmproxy they need to shut down
                 self.session_manager.flow_context.shutdown_signal.set()
         asyncio.run(_request_example_com())
-        http_proc.join()
+        self.http_proc.join()

tests/proxy/integration/test_lludp.py

@@ -12,7 +12,6 @@ from hippolyzer.lib.base.datatypes import UUID
 from hippolyzer.lib.base.message.message import Block, Message
 from hippolyzer.lib.base.message.udpdeserializer import UDPMessageDeserializer
 from hippolyzer.lib.base.objects import Object
-import hippolyzer.lib.base.serialization as se
 from hippolyzer.lib.proxy.addon_utils import BaseAddon
 from hippolyzer.lib.proxy.addons import AddonManager
 from hippolyzer.lib.proxy.message_logger import FilteringMessageLogger, LLUDPMessageLogEntry
@@ -205,8 +204,8 @@ class LLUDPIntegrationTests(BaseProxyTest):
         self.protocol.datagram_received(obj_update, self.region_addr)
         await self._wait_drained()
         entries = message_logger.entries
-        self.assertEqual(len(entries), 1)
-        self.assertEqual(entries[0].name, "ObjectUpdateCompressed")
+        self.assertEqual(1, len(entries))
+        self.assertEqual("ObjectUpdateCompressed", entries[0].name)
 
     async def test_filtering_logged_messages(self):
         message_logger = SimpleMessageLogger()
@@ -223,8 +222,8 @@ class LLUDPIntegrationTests(BaseProxyTest):
         await self._wait_drained()
         message_logger.set_filter("ObjectUpdateCompressed")
         entries = message_logger.entries
-        self.assertEqual(len(entries), 1)
-        self.assertEqual(entries[0].name, "ObjectUpdateCompressed")
+        self.assertEqual(1, len(entries))
+        self.assertEqual("ObjectUpdateCompressed", entries[0].name)
 
     async def test_logging_taken_message(self):
         message_logger = SimpleMessageLogger()
@@ -262,11 +261,6 @@ class LLUDPIntegrationTests(BaseProxyTest):
                     # Don't have a serializer, onto the next field
                     continue
                 deser = serializer.deserialize(block, orig_val)
-                # For now we consider returning UNSERIALIZABLE to be acceptable.
-                # We should probably consider raising instead of returning that.
-                if deser is se.UNSERIALIZABLE:
-                    continue
-
                 new_val = serializer.serialize(block, deser)
                 if orig_val != new_val:
                     raise AssertionError(f"{block.name}.{var_name} didn't reserialize correctly,"

tests/proxy/test_httpflows.py

@@ -1,5 +1,6 @@
 from mitmproxy.test import tflow, tutils
 
+from hippolyzer.lib.proxy.caps import CapType
 from hippolyzer.lib.proxy.http_flow import HippoHTTPFlow
 from hippolyzer.lib.proxy.message_logger import HTTPMessageLogEntry
 from hippolyzer.lib.proxy.test_utils import BaseProxyTest
@@ -84,8 +85,8 @@ content-length: 33\r
         self.assertEqual(b"foobar", flow.response.content)
 
     def test_temporary_cap_resolution(self):
-        self.region.register_temporary_cap("TempExample", "http://not.example.com")
-        self.region.register_temporary_cap("TempExample", "http://not2.example.com")
+        self.region.register_cap("TempExample", "http://not.example.com", CapType.TEMPORARY)
+        self.region.register_cap("TempExample", "http://not2.example.com", CapType.TEMPORARY)
         # Resolving the cap should consume it
         cap_data = self.session_manager.resolve_cap("http://not.example.com")
         self.assertEqual(cap_data.cap_name, "TempExample")

tests/proxy/test_message_filter.py

@@ -141,6 +141,16 @@ class MessageFilterTests(unittest.IsolatedAsyncioTestCase):
         self.assertTrue(self._filter_matches("FakeCap", entry))
         self.assertFalse(self._filter_matches("NotFakeCap", entry))
 
+    def test_http_header_filter(self):
+        session_manager = SessionManager(ProxySettings())
+        fake_flow = tflow.tflow(req=tutils.treq(), resp=tutils.tresp())
+        fake_flow.request.headers["Cookie"] = 'foo="bar"'
+        flow = HippoHTTPFlow.from_state(fake_flow.get_state(), session_manager)
+        entry = HTTPMessageLogEntry(flow)
+        # The header map is case-insensitive!
+        self.assertTrue(self._filter_matches('Meta.ReqHeaders.cookie ~= "foo"', entry))
+        self.assertFalse(self._filter_matches('Meta.ReqHeaders.foobar ~= "foo"', entry))
+
     def test_export_import_http_flow(self):
         fake_flow = tflow.tflow(req=tutils.treq(), resp=tutils.tresp())
         fake_flow.metadata["cap_data_ser"] = SerializedCapData(

tests/proxy/test_messages.py

@@ -17,17 +17,17 @@ class MockedProxyCircuit(ProxiedCircuit):
         self.in_injections = InjectionTracker(0, maxlen=10)
 
     def _send_prepared_message(self, msg: Message, transport=None):
-        self.sent_simple.append((msg.packet_id, msg.name, msg.direction, msg.injected, msg.acks))
+        self.sent_simple.append((msg.packet_id, msg.name, msg.direction, msg.synthetic, msg.acks))
         self.sent_msgs.append(msg)
 
 
-class PacketIDTests(unittest.TestCase):
+class PacketIDTests(unittest.IsolatedAsyncioTestCase):
     def setUp(self) -> None:
         self.circuit = MockedProxyCircuit()
 
     def _send_message(self, msg, outgoing=True):
         msg.direction = Direction.OUT if outgoing else Direction.IN
-        return self.circuit.send_message(msg)
+        return self.circuit.send(msg)
 
     def test_basic(self):
         self._send_message(Message('ChatFromViewer', packet_id=1))
@@ -178,10 +178,7 @@ class PacketIDTests(unittest.TestCase):
 
     def test_drop_proxied_message(self):
         self._send_message(Message('ChatFromViewer', packet_id=1))
-        self.circuit.drop_message(
-            Message('ChatFromViewer', packet_id=2, flags=PacketFlags.RELIABLE),
-            Direction.OUT,
-        )
+        self.circuit.drop_message(Message('ChatFromViewer', packet_id=2, flags=PacketFlags.RELIABLE))
         self._send_message(Message('ChatFromViewer', packet_id=3))
 
         self.assertSequenceEqual(self.circuit.sent_simple, [
@@ -193,10 +190,7 @@ class PacketIDTests(unittest.TestCase):
 
     def test_unreliable_proxied_message(self):
         self._send_message(Message('ChatFromViewer', packet_id=1))
-        self.circuit.drop_message(
-            Message('ChatFromViewer', packet_id=2),
-            Direction.OUT,
-        )
+        self.circuit.drop_message(Message('ChatFromViewer', packet_id=2))
         self._send_message(Message('ChatFromViewer', packet_id=3))
 
         self.assertSequenceEqual(self.circuit.sent_simple, [
@@ -209,10 +203,7 @@ class PacketIDTests(unittest.TestCase):
         self._send_message(Message('ChatFromViewer', packet_id=2))
         self._send_message(Message('ChatFromViewer', packet_id=3))
         self._send_message(Message('ChatFromSimulator'), outgoing=False)
-        self.circuit.drop_message(
-            Message('ChatFromViewer', packet_id=4, acks=(4,)),
-            Direction.OUT,
-        )
+        self.circuit.drop_message(Message('ChatFromViewer', packet_id=4, acks=(4,)))
         self._send_message(Message('ChatFromViewer', packet_id=5))
 
         self.assertSequenceEqual(self.circuit.sent_simple, [
@@ -230,7 +221,7 @@ class PacketIDTests(unittest.TestCase):
         self.assertEqual(self.circuit.sent_msgs[4]["Packets"][0]["ID"], 3)
 
     def test_resending_or_dropping(self):
-        self.circuit.send_message(Message('ChatFromViewer', packet_id=1))
+        self.circuit.send(Message('ChatFromViewer', packet_id=1))
         to_drop = Message('ChatFromViewer', packet_id=2, flags=PacketFlags.RELIABLE)
         self.circuit.drop_message(to_drop)
         with self.assertRaises(RuntimeError):
@@ -238,12 +229,72 @@ class PacketIDTests(unittest.TestCase):
             self.circuit.drop_message(to_drop)
         # Returns a new message without finalized flag
         new_msg = to_drop.take()
-        self.circuit.send_message(new_msg)
+        self.circuit.send(new_msg)
         with self.assertRaises(RuntimeError):
-            self.circuit.send_message(new_msg)
+            self.circuit.send(new_msg)
         self.assertSequenceEqual(self.circuit.sent_simple, [
             (1, "ChatFromViewer", Direction.OUT, False, ()),
             (1, "PacketAck", Direction.IN, True, ()),
             # ended up getting the same packet ID when injected
             (2, "ChatFromViewer", Direction.OUT, True, ()),
         ])
+
+    def test_reliable_unacked_queueing(self):
+        self._send_message(Message('ChatFromViewer', flags=PacketFlags.RELIABLE))
+        self._send_message(Message('ChatFromViewer', flags=PacketFlags.RELIABLE, packet_id=2))
+        # Only the first, injected message should be queued for resends
+        self.assertEqual({(Direction.OUT, 1)}, set(self.circuit.unacked_reliable))
+
+    def test_reliable_resend_cadence(self):
+        self._send_message(Message('ChatFromViewer', flags=PacketFlags.RELIABLE))
+        resend_info = self.circuit.unacked_reliable[(Direction.OUT, 1)]
+        self.circuit.resend_unacked()
+        # Should have been too soon to retry
+        self.assertEqual(10, resend_info.tries_left)
+        # Switch to allowing resends every 0s
+        self.circuit.resend_every = 0.0
+        self.circuit.resend_unacked()
+        self.assertSequenceEqual(self.circuit.sent_simple, [
+            (1, "ChatFromViewer", Direction.OUT, True, ()),
+            # Should have resent
+            (1, "ChatFromViewer", Direction.OUT, True, ()),
+        ])
+        self.assertEqual(9, resend_info.tries_left)
+        for _ in range(resend_info.tries_left):
+            self.circuit.resend_unacked()
+        # Should have used up all the retry attempts and been kicked out of the retry queue
+        self.assertEqual(set(), set(self.circuit.unacked_reliable))
+
+    def test_reliable_ack_collection(self):
+        msg = Message('ChatFromViewer', flags=PacketFlags.RELIABLE)
+        fut = self.circuit.send_reliable(msg)
+        self.assertEqual(1, len(self.circuit.unacked_reliable))
+        # Shouldn't count, this is an ACK going in the wrong direction!
+        ack_msg = Message("PacketAck", Block("Packets", ID=msg.packet_id))
+        self.circuit.collect_acks(ack_msg)
+        self.assertEqual(1, len(self.circuit.unacked_reliable))
+        self.assertFalse(fut.done())
+        # But it should count if the ACK message is heading in
+        ack_msg.direction = Direction.IN
+        self.circuit.collect_acks(ack_msg)
+        self.assertEqual(0, len(self.circuit.unacked_reliable))
+        self.assertTrue(fut.done())
+
+    def test_start_ping_check(self):
+        # Should not break if no unacked
+        self._send_message(Message(
+            "StartPingCheck",
+            Block("PingID", PingID=0, OldestUnacked=20),
+            packet_id=5,
+        ))
+
+        injected_msg = Message('ChatFromViewer', flags=PacketFlags.RELIABLE)
+        self._send_message(injected_msg)
+
+        self._send_message(Message(
+            "StartPingCheck",
+            Block("PingID", PingID=0, OldestUnacked=20),
+            packet_id=8,
+        ))
+        # Oldest unacked should have been replaced with the injected packet's ID, it's older!
+        self.assertEqual(self.circuit.sent_msgs[2]["PingID"]["OldestUnacked"], injected_msg.packet_id)