Miko/Hippolyzer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add fast path for ObjectUpdateCompressed decoding

Browse Source 
Speeds up ObjectUpdateCompressed handling by 25%

Resolves #9
This commit is contained in:
Salad Dais
2021-05-28 02:07:49 +00:00
parent 2608a02d5c
commit 9dbb719d52
4 changed files with 250 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

162
hippolyzer/lib/base/objects.py
View File

@@ -21,6 +21,8 @@ Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
from __future__ import annotations
import dataclasses
import logging
import struct
from typing import *
import lazy_object_proxy
@@ -29,7 +31,20 @@ import recordclass
from hippolyzer.lib.base.datatypes import Vector3, Quaternion, Vector4, UUID, TaggedUnion
from hippolyzer.lib.base.message.message import Block
from hippolyzer.lib.base.namevalue import NameValueCollection
from hippolyzer.lib.base.templates import ObjectUpdateCompressedDataSerializer
import hippolyzer.lib.base.serialization as se
from hippolyzer.lib.base.templates import (
PCode,
AgentState,
CompressedFlags,
AttachmentStateAdapter,
PSBLOCK_TEMPLATE,
Color4,
EXTRA_PARAM_COLLECTION,
SoundFlags,
NAMEVALUES_TERMINATED_TEMPLATE,
DATA_PACKER_TE_TEMPLATE,
TA_TEMPLATE,
)
class Object(recordclass.datatuple): # type: ignore
@@ -340,7 +355,7 @@ def normalize_terse_object_update(block: Block):
def normalize_object_update_compressed_data(data: bytes):
# Shared by ObjectUpdateCompressed and VOCache case
compressed = ObjectUpdateCompressedDataSerializer.deserialize(None, data)
compressed = FastObjectUpdateCompressedDataDeserializer.read(data)
# TODO: ObjectUpdateCompressed doesn't provide a default value for unused
# fields, whereas ObjectUpdate and friends do (TextColor, etc.)
# need some way to normalize ObjectUpdates so they won't appear to have
@@ -376,3 +391,146 @@ def normalize_object_update_compressed(block: Block):
compressed = normalize_object_update_compressed_data(block["Data"])
compressed["UpdateFlags"] = block.deserialize_var("UpdateFlags", make_copy=False)
return compressed
class SimpleStructReader(se.BufferReader):
def read_struct(self, spec: struct.Struct, peek=False) -> Tuple[Any, ...]:
val = spec.unpack_from(self._buffer, self._pos)
if not peek:
self._pos += spec.size
return val
def read_bytes_null_term(self) -> bytes:
old_offset = self._pos
while self._buffer[self._pos] != 0:
self._pos += 1
val = self._buffer[old_offset:self._pos]
self._pos += 1
return val
class FastObjectUpdateCompressedDataDeserializer:
HEADER_STRUCT = struct.Struct("<16sIBBIBB3f3f3fI16s")
ANGULAR_VELOCITY_STRUCT = struct.Struct("<3f")
PARENT_ID_STRUCT = struct.Struct("<I")
TREE_SPECIES_STRUCT = struct.Struct("<B")
DATAPACKER_LEN = struct.Struct("<I")
COLOR_ADAPTER = Color4()
PARTICLES_OLD = se.TypedBytesFixed(86, PSBLOCK_TEMPLATE)
SOUND_STRUCT = struct.Struct("<16sfBf")
PRIM_PARAMS_STRUCT = struct.Struct("<BBHHBBBBbbbbbBbHHH")
ATTACHMENT_STATE_ADAPTER = AttachmentStateAdapter(None)
@classmethod
def read(cls, data: bytes) -> Dict:
reader = SimpleStructReader("<", data)
foo = reader.read_struct(cls.HEADER_STRUCT)
full_id, local_id, pcode, state, crc, material, click_action, \
scalex, scaley, scalez, posx, posy, posz, rotx, roty, rotz, \
flags, owner_id = foo
scale = Vector3(scalex, scaley, scalez)
full_id = UUID(bytes=full_id)
pcode = PCode(pcode)
if pcode == PCode.AVATAR:
state = AgentState(state)
elif pcode == PCode.PRIMITIVE:
state = cls.ATTACHMENT_STATE_ADAPTER.decode(state, None)
pos = Vector3(posx, posy, posz)
rot = Quaternion(rotx, roty, rotz)
owner_id = UUID(bytes=owner_id)
ang_vel = None
if flags & CompressedFlags.ANGULAR_VELOCITY.value:
ang_vel = Vector3(*reader.read_struct(cls.ANGULAR_VELOCITY_STRUCT))
parent_id = None
if flags & CompressedFlags.PARENT_ID.value:
parent_id = reader.read_struct(cls.PARENT_ID_STRUCT)[0]
tree_species = None
if flags & CompressedFlags.TREE.value:
tree_species = reader.read_struct(cls.TREE_SPECIES_STRUCT)[0]
scratchpad = None
if flags & CompressedFlags.SCRATCHPAD.value:
scratchpad = reader.read_bytes(reader.read_struct(cls.DATAPACKER_LEN)[0])
text = None
text_color = None
if flags & CompressedFlags.TEXT.value:
text = reader.read_bytes_null_term().decode("utf8")
text_color = cls.COLOR_ADAPTER.decode(reader.read_bytes(4), ctx=None)
media_url = None
if flags & CompressedFlags.MEDIA_URL.value:
media_url = reader.read_bytes_null_term().decode("utf8")
psblock = None
if flags & CompressedFlags.PARTICLES.value:
psblock = reader.read(cls.PARTICLES_OLD)
extra_params = reader.read(EXTRA_PARAM_COLLECTION)
sound, sound_gain, sound_flags, sound_radius = None, None, None, None
if flags & CompressedFlags.SOUND.value:
sound, sound_gain, sound_flags, sound_radius = reader.read_struct(cls.SOUND_STRUCT)
sound = UUID(bytes=sound)
sound_flags = SoundFlags(sound_flags)
name_value = None
if flags & CompressedFlags.NAME_VALUES.value:
name_value = reader.read(NAMEVALUES_TERMINATED_TEMPLATE)
path_curve, profile_curve, path_begin, path_end, path_scale_x, path_scale_y, \
path_shear_x, path_shear_y, path_twist, path_twist_begin, path_radius_offset, \
path_taper_x, path_taper_y, path_revolutions, path_skew, profile_begin, \
profile_end, profile_hollow = reader.read_struct(cls.PRIM_PARAMS_STRUCT)
texture_entry = reader.read(DATA_PACKER_TE_TEMPLATE)
texture_anim = None
if flags & CompressedFlags.TEXTURE_ANIM.value:
texture_anim = reader.read(se.TypedByteArray(se.U32, TA_TEMPLATE))
psblock_new = None
if flags & CompressedFlags.PARTICLES_NEW.value:
psblock_new = reader.read(PSBLOCK_TEMPLATE)
if len(reader):
logging.warning(f"{len(reader)} bytes left at end of buffer for compressed {data!r}")
return {
"FullID": full_id,
"ID": local_id,
"PCode": pcode,
"State": state,
"CRC": crc,
"Material": material,
"ClickAction": click_action,
"Scale": scale,
"Position": pos,
"Rotation": rot,
"Flags": flags,
"OwnerID": owner_id,
"AngularVelocity": ang_vel,
"ParentID": parent_id,
"TreeSpecies": tree_species,
"ScratchPad": scratchpad,
"Text": text,
"TextColor": text_color,
"MediaURL": media_url,
"PSBlock": psblock,
"ExtraParams": extra_params,
"Sound": sound,
"SoundGain": sound_gain,
"SoundFlags": sound_flags,
"SoundRadius": sound_radius,
"NameValue": name_value,
"PathCurve": path_curve,
"ProfileCurve": profile_curve,
"PathBegin": path_begin, # 0 to 1, quanta = 0.01
"PathEnd": path_end, # 0 to 1, quanta = 0.01
"PathScaleX": path_scale_x, # 0 to 1, quanta = 0.01
"PathScaleY": path_scale_y, # 0 to 1, quanta = 0.01
"PathShearX": path_shear_x, # -.5 to .5, quanta = 0.01
"PathShearY": path_shear_y, # -.5 to .5, quanta = 0.01
"PathTwist": path_twist, # -1 to 1, quanta = 0.01
"PathTwistBegin": path_twist_begin, # -1 to 1, quanta = 0.01
"PathRadiusOffset": path_radius_offset, # -1 to 1, quanta = 0.01
"PathTaperX": path_taper_x, # -1 to 1, quanta = 0.01
"PathTaperY": path_taper_y, # -1 to 1, quanta = 0.01
"PathRevolutions": path_revolutions, # 0 to 3, quanta = 0.015
"PathSkew": path_skew, # -1 to 1, quanta = 0.01
"ProfileBegin": profile_begin, # 0 to 1, quanta = 0.01
"ProfileEnd": profile_end, # 0 to 1, quanta = 0.01
"ProfileHollow": profile_hollow, # 0 to 1, quanta = 0.01
"TextureEntry": texture_entry,
"TextureAnim": texture_anim,
"PSBlockNew": psblock_new,
}

21
hippolyzer/lib/base/templates.py
View File

@@ -878,13 +878,13 @@ class MediaFlags:
MEDIA_FLAGS = se.BitfieldDataclass(MediaFlags, se.U8, shift=False)
class Color4(se.SerializableBase):
class Color4(se.Adapter):
def __init__(self, invert_bytes=False, invert_alpha=False):
# There's several different ways of representing colors, presumably
# to allow for more efficient zerocoding in common cases.
self.invert_bytes = invert_bytes
self.invert_alpha = invert_alpha
self._bytes_templ = se.BytesFixed(4)
super().__init__(se.BytesFixed(4))
def _invert(self, val: bytes) -> bytes:
if self.invert_bytes:
@@ -893,11 +893,11 @@ class Color4(se.SerializableBase):
val = val[:3] + bytes((~val[4] & 0xFF,))
return val
def serialize(self, val, writer: se.BufferWriter, ctx=None):
self._bytes_templ.serialize(self._invert(val), writer, ctx)
def encode(self, val: bytes, ctx: Optional[se.ParseContext]) -> bytes:
return self._invert(val)
def deserialize(self, reader: se.BufferReader, ctx=None):
return self._invert(self._bytes_templ.deserialize(reader, ctx))
def decode(self, val: bytes, ctx: Optional[se.ParseContext], pod: bool = False) -> bytes:
return self._invert(val)
class TEFaceBitfield(se.SerializableBase):
@@ -1311,12 +1311,9 @@ class CompressedFlags(IntFlag):
PARTICLES_NEW = 1 << 10
UPDATE_COMPRESSED_FLAGS = se.IntFlag(CompressedFlags, se.U32)
class CompressedOption(se.OptionalFlagged):
def __init__(self, flag_val, spec):
super().__init__("Flags", UPDATE_COMPRESSED_FLAGS, flag_val, spec)
super().__init__("Flags", se.IntFlag(CompressedFlags, se.U32), flag_val, spec)
NAMEVALUES_TERMINATED_TEMPLATE = se.TypedBytesTerminated(
@@ -1338,7 +1335,7 @@ class ObjectUpdateCompressedDataSerializer(se.SimpleSubfieldSerializer):
"Scale": se.Vector3,
"Position": se.Vector3,
"Rotation": se.PackedQuat(se.Vector3),
"Flags": UPDATE_COMPRESSED_FLAGS,
"Flags": se.IntFlag(CompressedFlags, se.U32),
# Only non-null if there's an attached sound
"OwnerID": se.UUID,
"AngularVelocity": CompressedOption(CompressedFlags.ANGULAR_VELOCITY, se.Vector3),
@@ -1356,7 +1353,7 @@ class ObjectUpdateCompressedDataSerializer(se.SimpleSubfieldSerializer):
"ExtraParams": EXTRA_PARAM_COLLECTION,
"Sound": CompressedOption(CompressedFlags.SOUND, se.UUID),
"SoundGain": CompressedOption(CompressedFlags.SOUND, se.F32),
"SoundFlags": CompressedOption(CompressedFlags.SOUND, se.U8),
"SoundFlags": CompressedOption(CompressedFlags.SOUND, se.IntFlag(SoundFlags, se.U8)),
"SoundRadius": CompressedOption(CompressedFlags.SOUND, se.F32),
"NameValue": CompressedOption(CompressedFlags.NAME_VALUES, NAMEVALUES_TERMINATED_TEMPLATE),
# Intentionally not de-quantizing to preserve their real ranges.

2
tests/proxy/integration/test_lludp.py
View File

@@ -2,8 +2,8 @@ from __future__ import annotations
import random
import struct
from typing import *
import unittest
from typing import *
import lazy_object_proxy

94
tests/proxy/test_object_manager.py
View File

@@ -9,7 +9,8 @@ from hippolyzer.lib.base.message.message import Block
from hippolyzer.lib.base.message.message_handler import MessageHandler
from hippolyzer.lib.base.message.udpdeserializer import UDPMessageDeserializer
from hippolyzer.lib.base.message.udpserializer import UDPMessageSerializer
from hippolyzer.lib.base.objects import Object
from hippolyzer.lib.base.objects import Object, normalize_object_update_compressed_data
from hippolyzer.lib.base.templates import ExtraParamType, SculptTypeData, SculptType
from hippolyzer.lib.proxy.addons import AddonManager
from hippolyzer.lib.proxy.addon_utils import BaseAddon
from hippolyzer.lib.proxy.objects import ObjectManager
@@ -18,6 +19,20 @@ from hippolyzer.lib.proxy.templates import PCode
from hippolyzer.lib.proxy.vocache import RegionViewerObjectCacheChain, RegionViewerObjectCache, ViewerObjectCacheEntry
OBJECT_UPDATE_COMPRESSED_DATA = (
b"\x12\x12\x10\xbf\x16XB~\x8f\xb4\xfb\x00\x1a\xcd\x9b\xe5\xd2\x04\x00\x00\t\x00\xcdG\x00\x00"
b"\x03\x00\x00\x00\x1cB\x00\x00\x1cB\xcd\xcc\xcc=\xedG,"
b"B\x9e\xb1\x9eBff\xa0A\x00\x00\x00\x00\x00\x00\x00\x00["
b"\x8b\xf8\xbe\xc0\x00\x00\x00k\x9b\xc4\xfe3\nOa\xbb\xe2\xe4\xb2C\xac7\xbd\x00\x00\x00\x00"
b"\x00\x00\x00\x00\x00\x00\xa2=\x010\x00\x11\x00\x00\x00\x89UgG$\xcbC\xed\x92\x0bG\xca\xed"
b"\x15F_@ \x00\x00\x00\x00d\x96\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
b"\x00?\x00\x00\x00\x1c\x9fJoI\x8dH\xa0\x9d\xc4&''\x19=g\x00\x00\x00\x003\x00ff\x86\xbf"
b"\x00ff\x86?\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x89UgG$\xcbC"
b"\xed\x92\x0bG\xca\xed\x15F_\x10\x00\x00\x003\x00\x01\x01\x00\x00\x00\x00\xdb\x0f\xc9@\xa6"
b"\x9b\xc4="
)
class MockRegion:
def __init__(self, message_handler: MessageHandler):
self.session = lambda: None
@@ -330,27 +345,74 @@ class ObjectManagerTests(unittest.TestCase):
av = self.object_manager.lookup_avatar(obj.FullID)
self.assertEqual(av.Name, "firstname Resident")
def test_normalize_cache_data(self):
normalized = normalize_object_update_compressed_data(OBJECT_UPDATE_COMPRESSED_DATA)
expected = {
'PSBlock': None,
'ParentID': 0,
'LocalID': 1234,
'FullID': UUID('121210bf-1658-427e-8fb4-fb001acd9be5'),
'PCode': PCode.PRIMITIVE,
'State': 0,
'CRC': 18381,
'Material': 3,
'ClickAction': 0,
'Scale': Vector3(39.0, 39.0, 0.10000000149011612),
'Position': Vector3(43.07024002075195, 79.34690856933594, 20.049999237060547),
'Rotation': Quaternion(0.0, 0.0, -0.48543819785118103, 0.8742709854884798),
'OwnerID': UUID('6b9bc4fe-330a-4f61-bbe2-e4b243ac37bd'),
'AngularVelocity': Vector3(0.0, 0.0, 0.0791015625),
'TreeSpecies': None,
'ScratchPad': None,
'Text': None,
'TextColor': None,
'MediaURL': None,
'ExtraParams': {
ExtraParamType.SCULPT: {
'Texture': UUID('89556747-24cb-43ed-920b-47caed15465f'),
'TypeData': SculptTypeData(Type=SculptType.NONE, Invert=True, Mirror=False)
}
},
'Sound': None,
'SoundGain': None,
'SoundFlags': None,
'SoundRadius': None,
'NameValue': [],
'PathCurve': 32,
'ProfileCurve': 0,
'PathBegin': 0,
'PathEnd': 25600,
'PathScaleX': 150,
'PathScaleY': 0,
'PathShearX': 0,
'PathShearY': 0,
'PathTwist': 0,
'PathTwistBegin': 0,
'PathRadiusOffset': 0,
'PathTaperX': 0,
'PathTaperY': 0,
'PathRevolutions': 0,
'PathSkew': 0,
'ProfileBegin': 0,
'ProfileEnd': 0,
'ProfileHollow': 0
}
filtered_normalized = {k: v for k, v in normalized.items() if k in expected}
self.assertEqual(filtered_normalized, expected)
self.assertIsNotNone(normalized['TextureAnim'])
self.assertIsNotNone(normalized['TextureEntry'])
def test_object_cache(self):
self.mock_get_region_object_cache_chain.return_value = RegionViewerObjectCacheChain([
RegionViewerObjectCache(self.region.cache_id, [
ViewerObjectCacheEntry(
local_id=1234,
crc=22,
data=b"\x12\x12\x10\xbf\x16XB~\x8f\xb4\xfb\x00\x1a\xcd\x9b\xe5\xd2\x04\x00\x00\t\x00\xcdG\x00\x00"
b"\x03\x00\x00\x00\x1cB\x00\x00\x1cB\xcd\xcc\xcc=\xedG,"
b"B\x9e\xb1\x9eBff\xa0A\x00\x00\x00\x00\x00\x00\x00\x00["
b"\x8b\xf8\xbe\xc0\x00\x00\x00k\x9b\xc4\xfe3\nOa\xbb\xe2\xe4\xb2C\xac7\xbd\x00\x00\x00\x00"
b"\x00\x00\x00\x00\x00\x00\xa2=\x010\x00\x11\x00\x00\x00\x89UgG$\xcbC\xed\x92\x0bG\xca\xed"
b"\x15F_@ \x00\x00\x00\x00d\x96\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
b"\x00?\x00\x00\x00\x1c\x9fJoI\x8dH\xa0\x9d\xc4&''\x19=g\x00\x00\x00\x003\x00ff\x86\xbf"
b"\x00ff\x86?\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x89UgG$\xcbC"
b"\xed\x92\x0bG\xca\xed\x15F_\x10\x00\x00\x003\x00\x01\x01\x00\x00\x00\x00\xdb\x0f\xc9@\xa6"
b"\x9b\xc4="
data=OBJECT_UPDATE_COMPRESSED_DATA,
)
])
])
self.object_manager.load_cache()
self.message_handler.handle(Message(
cache_msg = Message(
'ObjectUpdateCached',
Block(
"ObjectData",
@@ -358,7 +420,11 @@ class ObjectManagerTests(unittest.TestCase):
CRC=22,
UpdateFlags=4321,
)
))
)
obj = self.object_manager.lookup_localid(1234)
self.assertIsNone(obj)
self.object_manager.load_cache()
self.message_handler.handle(cache_msg)
obj = self.object_manager.lookup_localid(1234)
self.assertEqual(obj.FullID, UUID('121210bf-1658-427e-8fb4-fb001acd9be5'))
# Flags from the ObjectUpdateCached should have been merged in