diff --git a/hippolyzer/lib/base/objects.py b/hippolyzer/lib/base/objects.py index 5082832..dc164a1 100644 --- a/hippolyzer/lib/base/objects.py +++ b/hippolyzer/lib/base/objects.py @@ -21,6 +21,8 @@ Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. from __future__ import annotations import dataclasses +import logging +import struct from typing import * import lazy_object_proxy @@ -29,7 +31,20 @@ import recordclass from hippolyzer.lib.base.datatypes import Vector3, Quaternion, Vector4, UUID, TaggedUnion from hippolyzer.lib.base.message.message import Block from hippolyzer.lib.base.namevalue import NameValueCollection -from hippolyzer.lib.base.templates import ObjectUpdateCompressedDataSerializer +import hippolyzer.lib.base.serialization as se +from hippolyzer.lib.base.templates import ( + PCode, + AgentState, + CompressedFlags, + AttachmentStateAdapter, + PSBLOCK_TEMPLATE, + Color4, + EXTRA_PARAM_COLLECTION, + SoundFlags, + NAMEVALUES_TERMINATED_TEMPLATE, + DATA_PACKER_TE_TEMPLATE, + TA_TEMPLATE, +) class Object(recordclass.datatuple): # type: ignore @@ -340,7 +355,7 @@ def normalize_terse_object_update(block: Block): def normalize_object_update_compressed_data(data: bytes): # Shared by ObjectUpdateCompressed and VOCache case - compressed = ObjectUpdateCompressedDataSerializer.deserialize(None, data) + compressed = FastObjectUpdateCompressedDataDeserializer.read(data) # TODO: ObjectUpdateCompressed doesn't provide a default value for unused # fields, whereas ObjectUpdate and friends do (TextColor, etc.) # need some way to normalize ObjectUpdates so they won't appear to have @@ -376,3 +391,146 @@ def normalize_object_update_compressed(block: Block): compressed = normalize_object_update_compressed_data(block["Data"]) compressed["UpdateFlags"] = block.deserialize_var("UpdateFlags", make_copy=False) return compressed + + +class SimpleStructReader(se.BufferReader): + def read_struct(self, spec: struct.Struct, peek=False) -> Tuple[Any, ...]: + val = spec.unpack_from(self._buffer, self._pos) + if not peek: + self._pos += spec.size + return val + + def read_bytes_null_term(self) -> bytes: + old_offset = self._pos + while self._buffer[self._pos] != 0: + self._pos += 1 + val = self._buffer[old_offset:self._pos] + self._pos += 1 + return val + + +class FastObjectUpdateCompressedDataDeserializer: + HEADER_STRUCT = struct.Struct("<16sIBBIBB3f3f3fI16s") + ANGULAR_VELOCITY_STRUCT = struct.Struct("<3f") + PARENT_ID_STRUCT = struct.Struct(" Dict: + reader = SimpleStructReader("<", data) + foo = reader.read_struct(cls.HEADER_STRUCT) + full_id, local_id, pcode, state, crc, material, click_action, \ + scalex, scaley, scalez, posx, posy, posz, rotx, roty, rotz, \ + flags, owner_id = foo + scale = Vector3(scalex, scaley, scalez) + full_id = UUID(bytes=full_id) + pcode = PCode(pcode) + if pcode == PCode.AVATAR: + state = AgentState(state) + elif pcode == PCode.PRIMITIVE: + state = cls.ATTACHMENT_STATE_ADAPTER.decode(state, None) + pos = Vector3(posx, posy, posz) + rot = Quaternion(rotx, roty, rotz) + owner_id = UUID(bytes=owner_id) + ang_vel = None + if flags & CompressedFlags.ANGULAR_VELOCITY.value: + ang_vel = Vector3(*reader.read_struct(cls.ANGULAR_VELOCITY_STRUCT)) + parent_id = None + if flags & CompressedFlags.PARENT_ID.value: + parent_id = reader.read_struct(cls.PARENT_ID_STRUCT)[0] + tree_species = None + if flags & CompressedFlags.TREE.value: + tree_species = reader.read_struct(cls.TREE_SPECIES_STRUCT)[0] + scratchpad = None + if flags & CompressedFlags.SCRATCHPAD.value: + scratchpad = reader.read_bytes(reader.read_struct(cls.DATAPACKER_LEN)[0]) + text = None + text_color = None + if flags & CompressedFlags.TEXT.value: + text = reader.read_bytes_null_term().decode("utf8") + text_color = cls.COLOR_ADAPTER.decode(reader.read_bytes(4), ctx=None) + media_url = None + if flags & CompressedFlags.MEDIA_URL.value: + media_url = reader.read_bytes_null_term().decode("utf8") + psblock = None + if flags & CompressedFlags.PARTICLES.value: + psblock = reader.read(cls.PARTICLES_OLD) + extra_params = reader.read(EXTRA_PARAM_COLLECTION) + sound, sound_gain, sound_flags, sound_radius = None, None, None, None + if flags & CompressedFlags.SOUND.value: + sound, sound_gain, sound_flags, sound_radius = reader.read_struct(cls.SOUND_STRUCT) + sound = UUID(bytes=sound) + sound_flags = SoundFlags(sound_flags) + name_value = None + if flags & CompressedFlags.NAME_VALUES.value: + name_value = reader.read(NAMEVALUES_TERMINATED_TEMPLATE) + path_curve, profile_curve, path_begin, path_end, path_scale_x, path_scale_y, \ + path_shear_x, path_shear_y, path_twist, path_twist_begin, path_radius_offset, \ + path_taper_x, path_taper_y, path_revolutions, path_skew, profile_begin, \ + profile_end, profile_hollow = reader.read_struct(cls.PRIM_PARAMS_STRUCT) + texture_entry = reader.read(DATA_PACKER_TE_TEMPLATE) + texture_anim = None + if flags & CompressedFlags.TEXTURE_ANIM.value: + texture_anim = reader.read(se.TypedByteArray(se.U32, TA_TEMPLATE)) + psblock_new = None + if flags & CompressedFlags.PARTICLES_NEW.value: + psblock_new = reader.read(PSBLOCK_TEMPLATE) + + if len(reader): + logging.warning(f"{len(reader)} bytes left at end of buffer for compressed {data!r}") + + return { + "FullID": full_id, + "ID": local_id, + "PCode": pcode, + "State": state, + "CRC": crc, + "Material": material, + "ClickAction": click_action, + "Scale": scale, + "Position": pos, + "Rotation": rot, + "Flags": flags, + "OwnerID": owner_id, + "AngularVelocity": ang_vel, + "ParentID": parent_id, + "TreeSpecies": tree_species, + "ScratchPad": scratchpad, + "Text": text, + "TextColor": text_color, + "MediaURL": media_url, + "PSBlock": psblock, + "ExtraParams": extra_params, + "Sound": sound, + "SoundGain": sound_gain, + "SoundFlags": sound_flags, + "SoundRadius": sound_radius, + "NameValue": name_value, + "PathCurve": path_curve, + "ProfileCurve": profile_curve, + "PathBegin": path_begin, # 0 to 1, quanta = 0.01 + "PathEnd": path_end, # 0 to 1, quanta = 0.01 + "PathScaleX": path_scale_x, # 0 to 1, quanta = 0.01 + "PathScaleY": path_scale_y, # 0 to 1, quanta = 0.01 + "PathShearX": path_shear_x, # -.5 to .5, quanta = 0.01 + "PathShearY": path_shear_y, # -.5 to .5, quanta = 0.01 + "PathTwist": path_twist, # -1 to 1, quanta = 0.01 + "PathTwistBegin": path_twist_begin, # -1 to 1, quanta = 0.01 + "PathRadiusOffset": path_radius_offset, # -1 to 1, quanta = 0.01 + "PathTaperX": path_taper_x, # -1 to 1, quanta = 0.01 + "PathTaperY": path_taper_y, # -1 to 1, quanta = 0.01 + "PathRevolutions": path_revolutions, # 0 to 3, quanta = 0.015 + "PathSkew": path_skew, # -1 to 1, quanta = 0.01 + "ProfileBegin": profile_begin, # 0 to 1, quanta = 0.01 + "ProfileEnd": profile_end, # 0 to 1, quanta = 0.01 + "ProfileHollow": profile_hollow, # 0 to 1, quanta = 0.01 + "TextureEntry": texture_entry, + "TextureAnim": texture_anim, + "PSBlockNew": psblock_new, + } diff --git a/hippolyzer/lib/base/templates.py b/hippolyzer/lib/base/templates.py index a4a979c..1fc08af 100644 --- a/hippolyzer/lib/base/templates.py +++ b/hippolyzer/lib/base/templates.py @@ -878,13 +878,13 @@ class MediaFlags: MEDIA_FLAGS = se.BitfieldDataclass(MediaFlags, se.U8, shift=False) -class Color4(se.SerializableBase): +class Color4(se.Adapter): def __init__(self, invert_bytes=False, invert_alpha=False): # There's several different ways of representing colors, presumably # to allow for more efficient zerocoding in common cases. self.invert_bytes = invert_bytes self.invert_alpha = invert_alpha - self._bytes_templ = se.BytesFixed(4) + super().__init__(se.BytesFixed(4)) def _invert(self, val: bytes) -> bytes: if self.invert_bytes: @@ -893,11 +893,11 @@ class Color4(se.SerializableBase): val = val[:3] + bytes((~val[4] & 0xFF,)) return val - def serialize(self, val, writer: se.BufferWriter, ctx=None): - self._bytes_templ.serialize(self._invert(val), writer, ctx) + def encode(self, val: bytes, ctx: Optional[se.ParseContext]) -> bytes: + return self._invert(val) - def deserialize(self, reader: se.BufferReader, ctx=None): - return self._invert(self._bytes_templ.deserialize(reader, ctx)) + def decode(self, val: bytes, ctx: Optional[se.ParseContext], pod: bool = False) -> bytes: + return self._invert(val) class TEFaceBitfield(se.SerializableBase): @@ -1311,12 +1311,9 @@ class CompressedFlags(IntFlag): PARTICLES_NEW = 1 << 10 -UPDATE_COMPRESSED_FLAGS = se.IntFlag(CompressedFlags, se.U32) - - class CompressedOption(se.OptionalFlagged): def __init__(self, flag_val, spec): - super().__init__("Flags", UPDATE_COMPRESSED_FLAGS, flag_val, spec) + super().__init__("Flags", se.IntFlag(CompressedFlags, se.U32), flag_val, spec) NAMEVALUES_TERMINATED_TEMPLATE = se.TypedBytesTerminated( @@ -1338,7 +1335,7 @@ class ObjectUpdateCompressedDataSerializer(se.SimpleSubfieldSerializer): "Scale": se.Vector3, "Position": se.Vector3, "Rotation": se.PackedQuat(se.Vector3), - "Flags": UPDATE_COMPRESSED_FLAGS, + "Flags": se.IntFlag(CompressedFlags, se.U32), # Only non-null if there's an attached sound "OwnerID": se.UUID, "AngularVelocity": CompressedOption(CompressedFlags.ANGULAR_VELOCITY, se.Vector3), @@ -1356,7 +1353,7 @@ class ObjectUpdateCompressedDataSerializer(se.SimpleSubfieldSerializer): "ExtraParams": EXTRA_PARAM_COLLECTION, "Sound": CompressedOption(CompressedFlags.SOUND, se.UUID), "SoundGain": CompressedOption(CompressedFlags.SOUND, se.F32), - "SoundFlags": CompressedOption(CompressedFlags.SOUND, se.U8), + "SoundFlags": CompressedOption(CompressedFlags.SOUND, se.IntFlag(SoundFlags, se.U8)), "SoundRadius": CompressedOption(CompressedFlags.SOUND, se.F32), "NameValue": CompressedOption(CompressedFlags.NAME_VALUES, NAMEVALUES_TERMINATED_TEMPLATE), # Intentionally not de-quantizing to preserve their real ranges. diff --git a/tests/proxy/integration/test_lludp.py b/tests/proxy/integration/test_lludp.py index 97998d4..118f0f9 100644 --- a/tests/proxy/integration/test_lludp.py +++ b/tests/proxy/integration/test_lludp.py @@ -2,8 +2,8 @@ from __future__ import annotations import random import struct -from typing import * import unittest +from typing import * import lazy_object_proxy diff --git a/tests/proxy/test_object_manager.py b/tests/proxy/test_object_manager.py index b8bd716..fd88539 100644 --- a/tests/proxy/test_object_manager.py +++ b/tests/proxy/test_object_manager.py @@ -9,7 +9,8 @@ from hippolyzer.lib.base.message.message import Block from hippolyzer.lib.base.message.message_handler import MessageHandler from hippolyzer.lib.base.message.udpdeserializer import UDPMessageDeserializer from hippolyzer.lib.base.message.udpserializer import UDPMessageSerializer -from hippolyzer.lib.base.objects import Object +from hippolyzer.lib.base.objects import Object, normalize_object_update_compressed_data +from hippolyzer.lib.base.templates import ExtraParamType, SculptTypeData, SculptType from hippolyzer.lib.proxy.addons import AddonManager from hippolyzer.lib.proxy.addon_utils import BaseAddon from hippolyzer.lib.proxy.objects import ObjectManager @@ -18,6 +19,20 @@ from hippolyzer.lib.proxy.templates import PCode from hippolyzer.lib.proxy.vocache import RegionViewerObjectCacheChain, RegionViewerObjectCache, ViewerObjectCacheEntry +OBJECT_UPDATE_COMPRESSED_DATA = ( + b"\x12\x12\x10\xbf\x16XB~\x8f\xb4\xfb\x00\x1a\xcd\x9b\xe5\xd2\x04\x00\x00\t\x00\xcdG\x00\x00" + b"\x03\x00\x00\x00\x1cB\x00\x00\x1cB\xcd\xcc\xcc=\xedG," + b"B\x9e\xb1\x9eBff\xa0A\x00\x00\x00\x00\x00\x00\x00\x00[" + b"\x8b\xf8\xbe\xc0\x00\x00\x00k\x9b\xc4\xfe3\nOa\xbb\xe2\xe4\xb2C\xac7\xbd\x00\x00\x00\x00" + b"\x00\x00\x00\x00\x00\x00\xa2=\x010\x00\x11\x00\x00\x00\x89UgG$\xcbC\xed\x92\x0bG\xca\xed" + b"\x15F_@ \x00\x00\x00\x00d\x96\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + b"\x00?\x00\x00\x00\x1c\x9fJoI\x8dH\xa0\x9d\xc4&''\x19=g\x00\x00\x00\x003\x00ff\x86\xbf" + b"\x00ff\x86?\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x89UgG$\xcbC" + b"\xed\x92\x0bG\xca\xed\x15F_\x10\x00\x00\x003\x00\x01\x01\x00\x00\x00\x00\xdb\x0f\xc9@\xa6" + b"\x9b\xc4=" +) + + class MockRegion: def __init__(self, message_handler: MessageHandler): self.session = lambda: None @@ -330,27 +345,74 @@ class ObjectManagerTests(unittest.TestCase): av = self.object_manager.lookup_avatar(obj.FullID) self.assertEqual(av.Name, "firstname Resident") + def test_normalize_cache_data(self): + normalized = normalize_object_update_compressed_data(OBJECT_UPDATE_COMPRESSED_DATA) + expected = { + 'PSBlock': None, + 'ParentID': 0, + 'LocalID': 1234, + 'FullID': UUID('121210bf-1658-427e-8fb4-fb001acd9be5'), + 'PCode': PCode.PRIMITIVE, + 'State': 0, + 'CRC': 18381, + 'Material': 3, + 'ClickAction': 0, + 'Scale': Vector3(39.0, 39.0, 0.10000000149011612), + 'Position': Vector3(43.07024002075195, 79.34690856933594, 20.049999237060547), + 'Rotation': Quaternion(0.0, 0.0, -0.48543819785118103, 0.8742709854884798), + 'OwnerID': UUID('6b9bc4fe-330a-4f61-bbe2-e4b243ac37bd'), + 'AngularVelocity': Vector3(0.0, 0.0, 0.0791015625), + 'TreeSpecies': None, + 'ScratchPad': None, + 'Text': None, + 'TextColor': None, + 'MediaURL': None, + 'ExtraParams': { + ExtraParamType.SCULPT: { + 'Texture': UUID('89556747-24cb-43ed-920b-47caed15465f'), + 'TypeData': SculptTypeData(Type=SculptType.NONE, Invert=True, Mirror=False) + } + }, + 'Sound': None, + 'SoundGain': None, + 'SoundFlags': None, + 'SoundRadius': None, + 'NameValue': [], + 'PathCurve': 32, + 'ProfileCurve': 0, + 'PathBegin': 0, + 'PathEnd': 25600, + 'PathScaleX': 150, + 'PathScaleY': 0, + 'PathShearX': 0, + 'PathShearY': 0, + 'PathTwist': 0, + 'PathTwistBegin': 0, + 'PathRadiusOffset': 0, + 'PathTaperX': 0, + 'PathTaperY': 0, + 'PathRevolutions': 0, + 'PathSkew': 0, + 'ProfileBegin': 0, + 'ProfileEnd': 0, + 'ProfileHollow': 0 + } + filtered_normalized = {k: v for k, v in normalized.items() if k in expected} + self.assertEqual(filtered_normalized, expected) + self.assertIsNotNone(normalized['TextureAnim']) + self.assertIsNotNone(normalized['TextureEntry']) + def test_object_cache(self): self.mock_get_region_object_cache_chain.return_value = RegionViewerObjectCacheChain([ RegionViewerObjectCache(self.region.cache_id, [ ViewerObjectCacheEntry( local_id=1234, crc=22, - data=b"\x12\x12\x10\xbf\x16XB~\x8f\xb4\xfb\x00\x1a\xcd\x9b\xe5\xd2\x04\x00\x00\t\x00\xcdG\x00\x00" - b"\x03\x00\x00\x00\x1cB\x00\x00\x1cB\xcd\xcc\xcc=\xedG," - b"B\x9e\xb1\x9eBff\xa0A\x00\x00\x00\x00\x00\x00\x00\x00[" - b"\x8b\xf8\xbe\xc0\x00\x00\x00k\x9b\xc4\xfe3\nOa\xbb\xe2\xe4\xb2C\xac7\xbd\x00\x00\x00\x00" - b"\x00\x00\x00\x00\x00\x00\xa2=\x010\x00\x11\x00\x00\x00\x89UgG$\xcbC\xed\x92\x0bG\xca\xed" - b"\x15F_@ \x00\x00\x00\x00d\x96\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" - b"\x00?\x00\x00\x00\x1c\x9fJoI\x8dH\xa0\x9d\xc4&''\x19=g\x00\x00\x00\x003\x00ff\x86\xbf" - b"\x00ff\x86?\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x89UgG$\xcbC" - b"\xed\x92\x0bG\xca\xed\x15F_\x10\x00\x00\x003\x00\x01\x01\x00\x00\x00\x00\xdb\x0f\xc9@\xa6" - b"\x9b\xc4=" + data=OBJECT_UPDATE_COMPRESSED_DATA, ) ]) ]) - self.object_manager.load_cache() - self.message_handler.handle(Message( + cache_msg = Message( 'ObjectUpdateCached', Block( "ObjectData", @@ -358,7 +420,11 @@ class ObjectManagerTests(unittest.TestCase): CRC=22, UpdateFlags=4321, ) - )) + ) + obj = self.object_manager.lookup_localid(1234) + self.assertIsNone(obj) + self.object_manager.load_cache() + self.message_handler.handle(cache_msg) obj = self.object_manager.lookup_localid(1234) self.assertEqual(obj.FullID, UUID('121210bf-1658-427e-8fb4-fb001acd9be5')) # Flags from the ObjectUpdateCached should have been merged in