Handle unknown messages better

hippolyzer/lib/base/message/message.py

@@ -188,7 +188,7 @@ class MsgBlockList(List["Block"]):
 class Message:
     __slots__ = ("name", "send_flags", "packet_id", "acks", "body_boundaries", "queued",
                  "offset", "raw_extra", "raw_body", "deserializer", "_blocks", "finalized",
-                 "direction", "meta", "synthetic", "dropped", "sender")
+                 "direction", "meta", "synthetic", "dropped", "sender", "unknown_message")
 
     def __init__(self, name, *args, packet_id=None, flags=0, acks=None, direction=None):
         # TODO: Do this on a timer or something.
@@ -200,6 +200,7 @@ class Message:
 
         self.acks = acks if acks is not None else tuple()
         self.body_boundaries = (-1, -1)
+        self.unknown_message = False
         self.offset = 0
         self.raw_extra = b""
         self.direction: Direction = direction if direction is not None else Direction.OUT
@@ -288,7 +289,7 @@ class Message:
 
     def ensure_parsed(self):
         # This is a little magic, think about whether we want this.
-        if self.raw_body and self.deserializer():
+        if self.raw_body and self.deserializer and self.deserializer():
             self.deserializer().parse_message_body(self)
 
     def to_dict(self, extended=False):

hippolyzer/lib/base/message/udpdeserializer.py

@@ -126,8 +126,14 @@ class UDPMessageDeserializer:
         frequency, num = _parse_msg_num(reader)
         current_template = self.template_dict.get_template_by_pair(frequency, num)
         if current_template is None:
-            raise exc.MessageTemplateNotFound("deserializing data", f"{frequency}:{num}")
-        msg.name = current_template.name
+            if self.settings.ALLOW_UNKNOWN_MESSAGES:
+                LOG.warning(f"Unknown message type {frequency}:{num}")
+                msg.unknown_message = True
+                msg.name = "UnknownMessage:%d" % num
+            else:
+                raise exc.MessageTemplateNotFound("deserializing data", f"{frequency}:{num}")
+        else:
+            msg.name = current_template.name
 
         # extra field, see note regarding msg.offset
         msg.raw_extra = reader.read_bytes(msg.offset)
@@ -143,6 +149,12 @@ class UDPMessageDeserializer:
         # Already parsed if we don't have a raw body
         if not raw_body:
             return
+
+        if msg.unknown_message:
+            # We can't parse this, we don't know anything about it
+            msg.deserializer = None
+            return
+
         msg.raw_body = None
         msg.deserializer = None
 

hippolyzer/lib/base/message/udpserializer.py

@@ -45,7 +45,7 @@ class UDPMessageSerializer:
 
     def serialize(self, msg: Message):
         current_template = self.template_dict.get_template_by_name(msg.name)
-        if current_template is None:
+        if current_template is None and msg.raw_body is None:
             raise exc.MessageSerializationError("message name", "invalid message name")
 
         # Header and trailers are all big-endian

hippolyzer/lib/base/settings.py

@@ -55,6 +55,7 @@ class SettingDescriptor(Generic[_T]):
 
 class Settings:
     ENABLE_DEFERRED_PACKET_PARSING: bool = SettingDescriptor(True)
+    ALLOW_UNKNOWN_MESSAGES: bool = SettingDescriptor(True)
 
     def __init__(self):
         self._settings: Dict[str, Any] = {}

tests/base/test_packetdata.py

@@ -50,6 +50,8 @@ OBJECT_UPDATE = binascii.unhexlify(''.join(OBJECT_UPDATE.split()))
 
 COARSE_LOCATION_UPDATE = b'\x00\x00\x00\x00E\x00\xff\x06\x00\xff\xff\xff\xff\x00'
 
+UNKNOWN_PACKET = b'\x00\x00\x00\x00E\x00\xff\xf0\x00\xff\xff\xff\xff\x00'
+
 
 class TestPacketDecode(unittest.TestCase):
 
@@ -110,3 +112,12 @@ class TestPacketDecode(unittest.TestCase):
         parsed = deserializer.deserialize(message)
         logging.debug("Parsed blocks: %r " % (list(parsed.blocks.keys()),))
         self.assertEqual(message, serializer.serialize(parsed))
+
+    def test_unknown_packet_roundtrips(self):
+        message = UNKNOWN_PACKET
+        deserializer = UDPMessageDeserializer(settings=self.settings)
+        serializer = UDPMessageSerializer()
+        parsed = deserializer.deserialize(message)
+        logging.debug("Parsed blocks: %r " % (list(parsed.blocks.keys()),))
+        self.assertEqual("UnknownMessage:240", parsed.name)
+        self.assertEqual(message, serializer.serialize(parsed))

tests/proxy/integration/test_lludp.py

@@ -21,6 +21,9 @@ from hippolyzer.lib.proxy.sessions import Session
 from hippolyzer.lib.proxy.test_utils import BaseProxyTest
 
 
+UNKNOWN_PACKET = b'\x00\x00\x00\x00E\x00\xff\xf0\x00\xff\xff\xff\xff\x00'
+
+
 class MockAddon(BaseAddon):
     def __init__(self):
         self.events = []
@@ -242,6 +245,21 @@ class LLUDPIntegrationTests(BaseProxyTest):
         self.assertEqual(entry.name, "UndoLand")
         self.assertEqual(entry.message.dropped, True)
 
+    async def test_logging_unknown_message(self):
+        message_logger = SimpleMessageLogger()
+        self.session_manager.message_logger = message_logger
+        self._setup_default_circuit()
+        self.protocol.datagram_received(UNKNOWN_PACKET, self.region_addr)
+        await self._wait_drained()
+        entries = message_logger.entries
+        self.assertEqual(len(entries), 1)
+        entry: LLUDPMessageLogEntry = entries[0]  # type: ignore
+        # Freezing shouldn't affect this
+        entry.freeze()
+        self.assertEqual(entry.name, "UnknownMessage:240")
+        self.assertEqual(entry.message.dropped, False)
+        self.assertEqual(entry.message.unknown_message, True)
+
     async def test_session_message_handler(self):
         self._setup_default_circuit()
         obj_update = self._make_objectupdate_compressed(1234)