diff --git a/hippolyzer/lib/base/message/message.py b/hippolyzer/lib/base/message/message.py index ab5777b..b1fc5d8 100644 --- a/hippolyzer/lib/base/message/message.py +++ b/hippolyzer/lib/base/message/message.py @@ -188,7 +188,7 @@ class MsgBlockList(List["Block"]): class Message: __slots__ = ("name", "send_flags", "packet_id", "acks", "body_boundaries", "queued", "offset", "raw_extra", "raw_body", "deserializer", "_blocks", "finalized", - "direction", "meta", "synthetic", "dropped", "sender") + "direction", "meta", "synthetic", "dropped", "sender", "unknown_message") def __init__(self, name, *args, packet_id=None, flags=0, acks=None, direction=None): # TODO: Do this on a timer or something. @@ -200,6 +200,7 @@ class Message: self.acks = acks if acks is not None else tuple() self.body_boundaries = (-1, -1) + self.unknown_message = False self.offset = 0 self.raw_extra = b"" self.direction: Direction = direction if direction is not None else Direction.OUT @@ -288,7 +289,7 @@ class Message: def ensure_parsed(self): # This is a little magic, think about whether we want this. - if self.raw_body and self.deserializer(): + if self.raw_body and self.deserializer and self.deserializer(): self.deserializer().parse_message_body(self) def to_dict(self, extended=False): diff --git a/hippolyzer/lib/base/message/udpdeserializer.py b/hippolyzer/lib/base/message/udpdeserializer.py index f33fb4c..805b152 100644 --- a/hippolyzer/lib/base/message/udpdeserializer.py +++ b/hippolyzer/lib/base/message/udpdeserializer.py @@ -126,8 +126,14 @@ class UDPMessageDeserializer: frequency, num = _parse_msg_num(reader) current_template = self.template_dict.get_template_by_pair(frequency, num) if current_template is None: - raise exc.MessageTemplateNotFound("deserializing data", f"{frequency}:{num}") - msg.name = current_template.name + if self.settings.ALLOW_UNKNOWN_MESSAGES: + LOG.warning(f"Unknown message type {frequency}:{num}") + msg.unknown_message = True + msg.name = "UnknownMessage:%d" % num + else: + raise exc.MessageTemplateNotFound("deserializing data", f"{frequency}:{num}") + else: + msg.name = current_template.name # extra field, see note regarding msg.offset msg.raw_extra = reader.read_bytes(msg.offset) @@ -143,6 +149,12 @@ class UDPMessageDeserializer: # Already parsed if we don't have a raw body if not raw_body: return + + if msg.unknown_message: + # We can't parse this, we don't know anything about it + msg.deserializer = None + return + msg.raw_body = None msg.deserializer = None diff --git a/hippolyzer/lib/base/message/udpserializer.py b/hippolyzer/lib/base/message/udpserializer.py index 3a67679..dc5ba55 100644 --- a/hippolyzer/lib/base/message/udpserializer.py +++ b/hippolyzer/lib/base/message/udpserializer.py @@ -45,7 +45,7 @@ class UDPMessageSerializer: def serialize(self, msg: Message): current_template = self.template_dict.get_template_by_name(msg.name) - if current_template is None: + if current_template is None and msg.raw_body is None: raise exc.MessageSerializationError("message name", "invalid message name") # Header and trailers are all big-endian diff --git a/hippolyzer/lib/base/settings.py b/hippolyzer/lib/base/settings.py index 9d751b1..2738c36 100644 --- a/hippolyzer/lib/base/settings.py +++ b/hippolyzer/lib/base/settings.py @@ -55,6 +55,7 @@ class SettingDescriptor(Generic[_T]): class Settings: ENABLE_DEFERRED_PACKET_PARSING: bool = SettingDescriptor(True) + ALLOW_UNKNOWN_MESSAGES: bool = SettingDescriptor(True) def __init__(self): self._settings: Dict[str, Any] = {} diff --git a/tests/base/test_packetdata.py b/tests/base/test_packetdata.py index b33b460..bb38894 100644 --- a/tests/base/test_packetdata.py +++ b/tests/base/test_packetdata.py @@ -50,6 +50,8 @@ OBJECT_UPDATE = binascii.unhexlify(''.join(OBJECT_UPDATE.split())) COARSE_LOCATION_UPDATE = b'\x00\x00\x00\x00E\x00\xff\x06\x00\xff\xff\xff\xff\x00' +UNKNOWN_PACKET = b'\x00\x00\x00\x00E\x00\xff\xf0\x00\xff\xff\xff\xff\x00' + class TestPacketDecode(unittest.TestCase): @@ -110,3 +112,12 @@ class TestPacketDecode(unittest.TestCase): parsed = deserializer.deserialize(message) logging.debug("Parsed blocks: %r " % (list(parsed.blocks.keys()),)) self.assertEqual(message, serializer.serialize(parsed)) + + def test_unknown_packet_roundtrips(self): + message = UNKNOWN_PACKET + deserializer = UDPMessageDeserializer(settings=self.settings) + serializer = UDPMessageSerializer() + parsed = deserializer.deserialize(message) + logging.debug("Parsed blocks: %r " % (list(parsed.blocks.keys()),)) + self.assertEqual("UnknownMessage:240", parsed.name) + self.assertEqual(message, serializer.serialize(parsed)) diff --git a/tests/proxy/integration/test_lludp.py b/tests/proxy/integration/test_lludp.py index 1b8a600..e8f76d0 100644 --- a/tests/proxy/integration/test_lludp.py +++ b/tests/proxy/integration/test_lludp.py @@ -21,6 +21,9 @@ from hippolyzer.lib.proxy.sessions import Session from hippolyzer.lib.proxy.test_utils import BaseProxyTest +UNKNOWN_PACKET = b'\x00\x00\x00\x00E\x00\xff\xf0\x00\xff\xff\xff\xff\x00' + + class MockAddon(BaseAddon): def __init__(self): self.events = [] @@ -242,6 +245,21 @@ class LLUDPIntegrationTests(BaseProxyTest): self.assertEqual(entry.name, "UndoLand") self.assertEqual(entry.message.dropped, True) + async def test_logging_unknown_message(self): + message_logger = SimpleMessageLogger() + self.session_manager.message_logger = message_logger + self._setup_default_circuit() + self.protocol.datagram_received(UNKNOWN_PACKET, self.region_addr) + await self._wait_drained() + entries = message_logger.entries + self.assertEqual(len(entries), 1) + entry: LLUDPMessageLogEntry = entries[0] # type: ignore + # Freezing shouldn't affect this + entry.freeze() + self.assertEqual(entry.name, "UnknownMessage:240") + self.assertEqual(entry.message.dropped, False) + self.assertEqual(entry.message.unknown_message, True) + async def test_session_message_handler(self): self._setup_default_circuit() obj_update = self._make_objectupdate_compressed(1234)