v0.6.3

Cutting a release before working on mitmproxy upgrade

README.md

@@ -2,7 +2,7 @@
 
 ![Python Test Status](https://github.com/SaladDais/Hippolyzer/workflows/Run%20Python%20Tests/badge.svg) [![codecov](https://codecov.io/gh/SaladDais/Hippolyzer/branch/master/graph/badge.svg?token=HCTFA4RAXX)](https://codecov.io/gh/SaladDais/Hippolyzer)
 
-[Hippolyzer](http://wiki.secondlife.com/wiki/Hippo) is a fork of Linden Lab's abandoned
+[Hippolyzer](http://wiki.secondlife.com/wiki/Hippo) is a revival of Linden Lab's
 [PyOGP library](http://wiki.secondlife.com/wiki/PyOGP)
 targeting modern Python 3, with a focus on debugging issues in Second Life-compatible
 servers and clients. There is a secondary focus on mocking up new features without requiring a
@@ -224,7 +224,7 @@ OUT ObjectAdd
 ```
 
 The repeat spinner at the bottom of the window lets you send a message multiple times.
-an `i` variable is put into the eval context and can be used to vary messages accros repeats.
+an `i` variable is put into the eval context and can be used to vary messages across repeats.
 With repeat set to two:
 
 ```

addon_examples/local_mesh.py

@@ -280,4 +280,23 @@ class MeshUploadInterceptingAddon(BaseAddon):
                 cls._replace_local_mesh(session.main_region, asset_repo, mesh_list)
 
 
+class BaseMeshManglerAddon(BaseAddon):
+    """Base class for addons that mangle uploaded or local mesh"""
+    MESH_MANGLERS: List[Callable[[MeshAsset], MeshAsset]]
+
+    def handle_init(self, session_manager: SessionManager):
+        # Add our manglers into the list
+        MeshUploadInterceptingAddon.mesh_manglers.extend(self.MESH_MANGLERS)
+        # Tell the local mesh plugin that the mangler list changed, and to re-apply
+        MeshUploadInterceptingAddon.remangle_local_mesh(session_manager)
+
+    def handle_unload(self, session_manager: SessionManager):
+        # Clean up our manglers before we go away
+        mangler_list = MeshUploadInterceptingAddon.mesh_manglers
+        for mangler in self.MESH_MANGLERS:
+            if mangler in mangler_list:
+                mangler_list.remove(mangler)
+        MeshUploadInterceptingAddon.remangle_local_mesh(session_manager)
+
+
 addons = [MeshUploadInterceptingAddon()]

addon_examples/mesh_mangler.py

@@ -11,8 +11,6 @@ to add to give a mesh an arbitrary center of rotation / scaling.
 
 from hippolyzer.lib.base.mesh import MeshAsset
 from hippolyzer.lib.proxy.addons import AddonManager
-from hippolyzer.lib.proxy.addon_utils import BaseAddon
-from hippolyzer.lib.proxy.sessions import SessionManager
 
 import local_mesh
 AddonManager.hot_reload(local_mesh, require_addons_loaded=True)
@@ -37,6 +35,9 @@ def reorient_mesh(orientation):
     # X=1, Y=2, Z=3
     def _reorienter(mesh: MeshAsset):
         for material in mesh.iter_lod_materials():
+            if "Position" not in material:
+                # Must be a NoGeometry LOD
+                continue
             # We don't need to use positions_(to/from)_domain here since we're just naively
             # flipping the axes around.
             material["Position"] = _reorient_coord_list(material["Position"], orientation)
@@ -46,28 +47,11 @@ def reorient_mesh(orientation):
     return _reorienter
 
 
-OUR_MANGLERS = [
-    # Negate the X and Y axes on any mesh we upload or create temp
-    reorient_mesh((-1, -2, 3)),
-]
+class ExampleMeshManglerAddon(local_mesh.BaseMeshManglerAddon):
+    MESH_MANGLERS = [
+        # Negate the X and Y axes on any mesh we upload or create temp
+        reorient_mesh((-1, -2, 3)),
+    ]
 
 
-class MeshManglerExampleAddon(BaseAddon):
-    def handle_init(self, session_manager: SessionManager):
-        # Add our manglers into the list
-        local_mesh_addon = local_mesh.MeshUploadInterceptingAddon
-        local_mesh_addon.mesh_manglers.extend(OUR_MANGLERS)
-        # Tell the local mesh plugin that the mangler list changed, and to re-apply
-        local_mesh_addon.remangle_local_mesh(session_manager)
-
-    def handle_unload(self, session_manager: SessionManager):
-        # Clean up our manglers before we go away
-        local_mesh_addon = local_mesh.MeshUploadInterceptingAddon
-        mangler_list = local_mesh_addon.mesh_manglers
-        for mangler in OUR_MANGLERS:
-            if mangler in mangler_list:
-                mangler_list.remove(mangler)
-        local_mesh_addon.remangle_local_mesh(session_manager)
-
-
-addons = [MeshManglerExampleAddon()]
+addons = [ExampleMeshManglerAddon()]

addon_examples/shield.py

@@ -6,7 +6,13 @@ from hippolyzer.lib.base.network.transport import Direction
 from hippolyzer.lib.proxy.region import ProxiedRegion
 from hippolyzer.lib.proxy.sessions import Session
 
-SUSPICIOUS_PACKETS = {"TransferRequest", "UUIDNameRequest", "UUIDGroupNameRequest", "OpenCircuit"}
+SUSPICIOUS_PACKETS = {
+    "TransferRequest",
+    "UUIDNameRequest",
+    "UUIDGroupNameRequest",
+    "OpenCircuit",
+    "AddCircuitCode",
+}
 REGULAR_IM_DIALOGS = (IMDialogType.TYPING_STOP, IMDialogType.TYPING_STOP, IMDialogType.NOTHING_SPECIAL)
 
 

hippolyzer/apps/proxy_gui.py

@@ -34,7 +34,7 @@ from hippolyzer.lib.base.message.message_formatting import (
     SpannedString,
 )
 from hippolyzer.lib.base.message.msgtypes import MsgType
-from hippolyzer.lib.base.message.template_dict import TemplateDictionary
+from hippolyzer.lib.base.message.template_dict import DEFAULT_TEMPLATE_DICT
 from hippolyzer.lib.base.ui_helpers import loadUi
 import hippolyzer.lib.base.serialization as se
 from hippolyzer.lib.base.network.transport import Direction, SocketUDPTransport
@@ -446,7 +446,7 @@ class MessageBuilderWindow(QtWidgets.QMainWindow):
     def __init__(self, parent, session_manager):
         super().__init__(parent=parent)
         loadUi(MESSAGE_BUILDER_UI_PATH, self)
-        self.templateDict = TemplateDictionary()
+        self.templateDict = DEFAULT_TEMPLATE_DICT
         self.llsdSerializer = LLSDMessageSerializer()
         self.sessionManager: SessionManager = session_manager
         self.regionModel = RegionListModel(self, self.sessionManager)
@@ -575,24 +575,9 @@ class MessageBuilderWindow(QtWidgets.QMainWindow):
         if var.name in ("TaskID", "ObjectID"):
             return VerbatimHumanVal("[[SELECTED_FULL]]")
 
-        if var.type.is_int:
-            return 0
-        elif var.type.is_float:
-            return 0.0
-        elif var.type == MsgType.MVT_LLUUID:
-            return UUID()
-        elif var.type == MsgType.MVT_BOOL:
-            return False
-        elif var.type == MsgType.MVT_VARIABLE:
-            return ""
-        elif var.type in (MsgType.MVT_LLVector3, MsgType.MVT_LLVector3d, MsgType.MVT_LLQuaternion):
-            return VerbatimHumanVal("(0.0, 0.0, 0.0)")
-        elif var.type == MsgType.MVT_LLVector4:
-            return VerbatimHumanVal("(0.0, 0.0, 0.0, 0.0)")
-        elif var.type == MsgType.MVT_FIXED:
-            return b"\x00" * var.size
-        elif var.type == MsgType.MVT_IP_ADDR:
-            return "0.0.0.0"
+        default_val = var.default_value
+        if default_val is not None:
+            return default_val
         return VerbatimHumanVal("")
 
     @nonFatalExceptions
@@ -632,7 +617,7 @@ class MessageBuilderWindow(QtWidgets.QMainWindow):
         msg = HumanMessageSerializer.from_human_string(msg_text, replacements, env, safe=False)
         if self.checkLLUDPViaCaps.isChecked():
             if msg.direction == Direction.IN:
-                region.eq_manager.queue_event(
+                region.eq_manager.inject_event(
                     self.llsdSerializer.serialize(msg, as_dict=True)
                 )
             else:
@@ -656,7 +641,7 @@ class MessageBuilderWindow(QtWidgets.QMainWindow):
             raise RuntimeError("Need a valid session and region to send EQ event")
         message_line, _, body = (x.strip() for x in msg_text.partition("\n"))
         message_name = message_line.rsplit(" ", 1)[-1]
-        region.eq_manager.queue_event({
+        region.eq_manager.inject_event({
             "message": message_name,
             "body": llsd.parse_xml(body.encode("utf8")),
         })

hippolyzer/lib/base/datatypes.py

@@ -294,6 +294,17 @@ class RawBytes(bytes):
     pass
 
 
+_T = TypeVar("_T")
+
+
+class Pretty(Generic[_T]):
+    """Wrapper for var values so Messages will know to serialize"""
+    __slots__ = ("value",)
+
+    def __init__(self, value: _T):
+        self.value: _T = value
+
+
 class StringEnum(str, enum.Enum):
     def __str__(self):
         return self.value
@@ -333,5 +344,5 @@ class TaggedUnion(recordclass.datatuple):  # type: ignore
 __all__ = [
     "Vector3", "Vector4", "Vector2", "Quaternion", "TupleCoord",
     "UUID", "RawBytes", "StringEnum", "JankStringyBytes", "TaggedUnion",
-    "IntEnum", "IntFlag", "flags_to_pod"
+    "IntEnum", "IntFlag", "flags_to_pod", "Pretty"
 ]

hippolyzer/lib/base/message/llsd_msg_serializer.py

@@ -5,14 +5,13 @@ from hippolyzer.lib.base import llsd
 from hippolyzer.lib.base.message.data_packer import LLSDDataPacker
 from hippolyzer.lib.base.message.message import Message
 from hippolyzer.lib.base.message.template import MessageTemplateVariable
-from hippolyzer.lib.base.message.template_dict import TemplateDictionary
-
+from hippolyzer.lib.base.message.template_dict import TemplateDictionary, DEFAULT_TEMPLATE_DICT
 
 VAR_PAIR = Tuple[dict, MessageTemplateVariable]
 
 
 class LLSDMessageSerializer:
-    DEFAULT_TEMPLATE = TemplateDictionary()
+    DEFAULT_TEMPLATE = DEFAULT_TEMPLATE_DICT
 
     def __init__(self, message_template=None, message_cls: Type[Message] = Message):
         if message_template is not None:

hippolyzer/lib/base/message/message.py

@@ -32,6 +32,7 @@ from typing import *
 from hippolyzer.lib.base.datatypes import *
 import hippolyzer.lib.base.serialization as se
 import hippolyzer.lib.base.templates as templates
+from hippolyzer.lib.base.datatypes import Pretty
 from hippolyzer.lib.base.message.msgtypes import PacketFlags
 from hippolyzer.lib.base.network.transport import Direction, ADDR_TUPLE
 
@@ -62,11 +63,12 @@ class Block:
     Block expects a name, and kwargs for variables (var_name = value)
     """
     __slots__ = ('name', 'size', 'vars', 'message_name', '_ser_cache', 'fill_missing',)
+    PARENT_MESSAGE_NAME: ClassVar[Optional[str]] = None
 
     def __init__(self, name, /, *, fill_missing=False, **kwargs):
         self.name = name
         self.size = 0
-        self.message_name: Optional[str] = None
+        self.message_name: Optional[str] = self.PARENT_MESSAGE_NAME
         self.vars: Dict[str, VAR_TYPE] = {}
         self._ser_cache: Dict[str, Any] = {}
         self.fill_missing = fill_missing
@@ -83,6 +85,9 @@ class Block:
         return self.vars[name]
 
     def __setitem__(self, key, value):
+        if isinstance(value, Pretty):
+            return self.serialize_var(key, value.value)
+
         # These don't pickle well since they're likely to get hot-reloaded
         if isinstance(value, (enum.IntEnum, enum.IntFlag)):
             value = int(value)

hippolyzer/lib/base/message/template.py

@@ -22,6 +22,7 @@ Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 import typing
 
 from .msgtypes import MsgType, MsgBlockType
+from ..datatypes import UUID
 
 
 class MessageTemplateVariable:
@@ -61,6 +62,32 @@ class MessageTemplateVariable:
             self._probably_text = self._probably_text and self.name != "NameValue"
         return self._probably_text
 
+    @property
+    def default_value(self):
+        if self.type.is_int:
+            return 0
+        elif self.type.is_float:
+            return 0.0
+        elif self.type == MsgType.MVT_LLUUID:
+            return UUID()
+        elif self.type == MsgType.MVT_BOOL:
+            return False
+        elif self.type == MsgType.MVT_VARIABLE:
+            if self.probably_binary:
+                return b""
+            if self.probably_text:
+                return ""
+            return b""
+        elif self.type in (MsgType.MVT_LLVector3, MsgType.MVT_LLVector3d, MsgType.MVT_LLQuaternion):
+            return 0.0, 0.0, 0.0
+        elif self.type == MsgType.MVT_LLVector4:
+            return 0.0, 0.0, 0.0, 0.0
+        elif self.type == MsgType.MVT_FIXED:
+            return b"\x00" * self.size
+        elif self.type == MsgType.MVT_IP_ADDR:
+            return "0.0.0.0"
+        return None
+
 
 class MessageTemplateBlock:
     def __init__(self, name):

hippolyzer/lib/base/message/template_dict.py

@@ -27,25 +27,35 @@ from .template import MessageTemplate
 from .template_parser import MessageTemplateParser
 
 
+DEFAULT_PARSER = MessageTemplateParser(msg_tmpl)
+
+
 class TemplateDictionary:
     """the dictionary with all known templates"""
 
     def __init__(self, template_list=None, message_template=None):
         if template_list is None:
             if message_template is None:
-                parser = MessageTemplateParser(msg_tmpl)
+                parser = DEFAULT_PARSER
             else:
                 parser = MessageTemplateParser(message_template)
             template_list = parser.message_templates
 
-        self.template_list: typing.List[MessageTemplate] = template_list
-
+        self.template_list: typing.List[MessageTemplate] = []
         # maps name to template
         self.message_templates = {}
 
         # maps (freq,num) to template
         self.message_dict = {}
 
+        self.load_templates(template_list)
+
+    def load_templates(self, template_list):
+        self.template_list.clear()
+        self.template_list.extend(template_list)
+        self.message_templates.clear()
+        self.message_dict.clear()
+
         self.build_dictionaries(template_list)
         self.build_message_ids()
 
@@ -99,3 +109,6 @@ class TemplateDictionary:
 
     def __iter__(self):
         return iter(self.template_list)
+
+
+DEFAULT_TEMPLATE_DICT = TemplateDictionary()

hippolyzer/lib/base/message/udpdeserializer.py

@@ -26,7 +26,7 @@ from logging import getLogger
 from hippolyzer.lib.base.datatypes import JankStringyBytes
 from hippolyzer.lib.base.settings import Settings
 from .template import MessageTemplateVariable
-from .template_dict import TemplateDictionary
+from .template_dict import DEFAULT_TEMPLATE_DICT
 from .msgtypes import MsgType, MsgBlockType, PacketLayout
 from .data_packer import TemplateDataPacker
 from .message import Message, Block
@@ -62,7 +62,7 @@ def _parse_msg_num(reader: se.BufferReader):
 
 
 class UDPMessageDeserializer:
-    DEFAULT_TEMPLATE = TemplateDictionary()
+    DEFAULT_TEMPLATE = DEFAULT_TEMPLATE_DICT
 
     def __init__(self, settings=None):
         self.settings = settings or Settings()

hippolyzer/lib/base/message/udpserializer.py

@@ -26,7 +26,7 @@ from .data_packer import TemplateDataPacker
 from .message import Message, MsgBlockList
 from .msgtypes import MsgType, MsgBlockType
 from .template import MessageTemplateVariable, MessageTemplateBlock
-from .template_dict import TemplateDictionary
+from .template_dict import TemplateDictionary, DEFAULT_TEMPLATE_DICT
 from hippolyzer.lib.base import exc
 from hippolyzer.lib.base import serialization as se
 from hippolyzer.lib.base.datatypes import RawBytes
@@ -35,7 +35,7 @@ logger = getLogger('message.udpserializer')
 
 
 class UDPMessageSerializer:
-    DEFAULT_TEMPLATE = TemplateDictionary(None)
+    DEFAULT_TEMPLATE = DEFAULT_TEMPLATE_DICT
 
     def __init__(self, message_template=None):
         if message_template is not None:

hippolyzer/lib/base/network/transport.py

@@ -30,6 +30,7 @@ class UDPPacket:
         self.dst_addr = dst_addr
         self.data = data
         self.direction = direction
+        self.meta = {}
 
     @property
     def outgoing(self):

hippolyzer/lib/proxy/http_event_manager.py

@@ -137,6 +137,27 @@ class MITMProxyEventManager:
             # the proxy
             self._asset_server_proxied = True
             logging.warning("noproxy not used, switching to URI rewrite strategy")
+        elif cap_data and cap_data.cap_name == "EventQueueGet":
+            # HACK: The sim's EQ acking mechanism doesn't seem to actually work.
+            # if the client drops the connection due to timeout before we can
+            # proxy back the response then it will be lost forever. Keep around
+            # the last EQ response we got so we can re-send it if the client repeats
+            # its previous request.
+            req_ack_id = llsd.parse_xml(flow.request.content)["ack"]
+            eq_manager = cap_data.region().eq_manager
+            cached_resp = eq_manager.get_cached_poll_response(req_ack_id)
+            if cached_resp:
+                logging.warning("Had to serve a cached EventQueueGet due to client desync")
+                flow.response = mitmproxy.http.HTTPResponse.make(
+                    200,
+                    llsd.format_xml(cached_resp),
+                    {
+                        "Content-Type": "application/llsd+xml",
+                        # So we can differentiate these in the log
+                        "X-Hippo-Fake-EQ": "1",
+                        "Connection": "close",
+                    },
+                )
         elif not cap_data:
             if self._is_login_request(flow):
                 # Not strictly a Cap, but makes it easier to filter on.
@@ -251,11 +272,14 @@ class MITMProxyEventManager:
                             new_events.append(event)
                     # Add on any fake events that've been queued by addons
                     eq_manager = cap_data.region().eq_manager
-                    new_events.extend(eq_manager.take_events())
+                    new_events.extend(eq_manager.take_injected_events())
                     parsed_eq_resp["events"] = new_events
+                    # Empty event list is an error, need to return undef instead.
                     if old_events and not new_events:
-                        # Need at least one event or the viewer will refuse to ack!
-                        new_events.append({"message": "NOP", "body": {}})
+                        parsed_eq_resp = None
+                    # HACK: see note in above request handler for EventQueueGet
+                    req_ack_id = llsd.parse_xml(flow.request.content)["ack"]
+                    eq_manager.cache_last_poll_response(req_ack_id, parsed_eq_resp)
                 flow.response.content = llsd.format_pretty_xml(parsed_eq_resp)
             elif cap_data.cap_name in self.UPLOAD_CREATING_CAPS:
                 if not region:

hippolyzer/lib/proxy/lludp_proxy.py

@@ -58,6 +58,7 @@ class InterceptingLLUDPProxyProtocol(UDPProxyProtocol):
         message = self.deserializer.deserialize(packet.data)
         message.direction = packet.direction
         message.sender = packet.src_addr
+        message.meta.update(packet.meta)
 
         assert message is not None
         # Check for UDP bans on inbound messages

hippolyzer/lib/proxy/namecache.py

@@ -32,6 +32,9 @@ class ProxyNameCache(NameCache):
                     with open(namecache_file, "rb") as f:
                         namecache_bytes = f.read()
                     agents = llsd.parse_xml(namecache_bytes)["agents"]
+                    # Can be `None` if the file was just created
+                    if not agents:
+                        continue
                     for agent_id, agent_data in agents.items():
                         # Don't set display name if they just have the default
                         display_name = None

hippolyzer/lib/proxy/region.py

@@ -162,6 +162,7 @@ class ProxiedRegion(BaseClientRegion):
         if self.circuit:
             self.circuit.is_alive = False
         self.objects.clear()
+        self.eq_manager.clear()
 
     def __repr__(self):
         return "<%s %s>" % (self.__class__.__name__, self.name)
@@ -172,11 +173,27 @@ class EventQueueManager:
         # TODO: Per-EQ InjectionTracker so we can inject fake responses on 499
         self._queued_events = []
         self._region = weakref.proxy(region)
+        self._last_ack: Optional[int] = None
+        self._last_payload: Optional[Any] = None
 
-    def queue_event(self, event: dict):
+    def inject_event(self, event: dict):
         self._queued_events.append(event)
 
-    def take_events(self):
+    def take_injected_events(self):
         events = self._queued_events
         self._queued_events = []
         return events
+
+    def cache_last_poll_response(self, req_ack: int, payload: Any):
+        self._last_ack = req_ack
+        self._last_payload = payload
+
+    def get_cached_poll_response(self, req_ack: Optional[int]) -> Optional[Any]:
+        if self._last_ack == req_ack:
+            return self._last_payload
+        return None
+
+    def clear(self):
+        self._queued_events.clear()
+        self._last_ack = None
+        self._last_payload = None

setup.py

@@ -25,7 +25,7 @@ from setuptools import setup, find_packages
 
 here = path.abspath(path.dirname(__file__))
 
-version = '0.6.1'
+version = '0.6.3'
 
 with open(path.join(here, 'README.md')) as readme_fh:
     readme = readme_fh.read()

setup_cxfreeze.py

@@ -112,7 +112,7 @@ executables = [
 
 setup(
     name="hippolyzer_gui",
-    version="0.6.1",
+    version="0.6.3",
     description="Hippolyzer GUI",
     options=options,
     executables=executables,

tests/proxy/test_templates.py

@@ -5,10 +5,10 @@ from hippolyzer.lib.base.datatypes import UUID
 from hippolyzer.lib.base.message.message_formatting import HumanMessageSerializer
 from hippolyzer.lib.base.templates import TextureEntrySubfieldSerializer, TEFaceBitfield, TextureEntry
 
-EXAMPLE_TE = b"\x89UgG$\xcbC\xed\x92\x0bG\xca\xed\x15F_\x08\xe7\xb2\x98\x04\xca\x10;\x85\x94\x05Lj\x8d\xd4" \
-             b"\x0b\x1f\x01B\xcb\xe6|\x1d,\xa7sc\xa6\x1a\xa2L\xb1u\x01\x00\x00\x00\x00\x00\x00\x00\x00\x80?" \
-             b"\x00\x00\x00\x80?\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" \
-             b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+EXAMPLE_TE = b'\x89UgG$\xcbC\xed\x92\x0bG\xca\xed\x15F_\x08\xca*\x98:\x18\x02,\r\xf4\x1e\xc6\xf5\x91\x01]\x83\x014' \
+             b'\x00\x90i+\x10\x80\xa1\xaa\xa2g\x11o\xa8]\xc6\x00\x00\x00\x00\x00\x00\x00\x00\x80?\x00\x00\x00\x80?' \
+             b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' \
+             b'\x00\x00\x00\x00\x00\x00\x00'
 
 
 class TemplateTests(unittest.TestCase):
@@ -58,7 +58,10 @@ class TemplateTests(unittest.TestCase):
         str_msg = HumanMessageSerializer.to_human_string(msg, beautify=True)
         msg = HumanMessageSerializer.from_human_string(str_msg)
         spec = msg["ObjectData"][0].get_serializer("TextureEntry")
-        deser = spec.deserialize(None, msg["ObjectData"]["TextureEntry"], pod=True)
+        data_field = msg["ObjectData"]["TextureEntry"]
+        # Serialization order and format should match indra's exactly
+        self.assertEqual(EXAMPLE_TE, data_field)
+        deser = spec.deserialize(None, data_field, pod=True)
         self.assertEqual(deser, pod_te)
 
     def test_textureentry_defaults(self):