v0.14.2

llsd.parse_notation() is slow as hell, no way around it.

.github/workflows/pytest.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.11"]
+        python-version: ["3.9", "3.11"]
 
     steps:
     - uses: actions/checkout@v2

addon_examples/get_task_inventory_cap.py

@@ -17,7 +17,7 @@ from hippolyzer.lib.base import llsd
 from hippolyzer.lib.base.datatypes import UUID
 from hippolyzer.lib.base.inventory import InventoryModel, InventoryObject
 from hippolyzer.lib.base.message.message import Message, Block
-from hippolyzer.lib.base.templates import XferFilePath
+from hippolyzer.lib.base.templates import XferFilePath, AssetType
 from hippolyzer.lib.proxy import addon_ctx
 from hippolyzer.lib.proxy.webapp_cap_addon import WebAppCapAddon
 
@@ -64,7 +64,7 @@ async def get_task_inventory():
                         InventoryObject(
                             name="Contents",
                             parent_id=UUID.ZERO,
-                            type="category",
+                            type=AssetType.CATEGORY,
                             obj_id=obj_id
                         ).to_llsd()
                     ],

hippolyzer/lib/base/helpers.py

@@ -132,6 +132,13 @@ def proxify(obj: Union[Callable[[], _T], weakref.ReferenceType, _T]) -> _T:
     return obj
 
 
+class BiDiDict(Generic[_T]):
+    """Dictionary for bidirectional lookups"""
+    def __init__(self, values: Dict[_T, _T]):
+        self.forward = {**values}
+        self.backward = {value: key for (key, value) in values.items()}
+
+
 def bytes_unescape(val: bytes) -> bytes:
     # Only in CPython. bytes -> bytes with escape decoding.
     # https://stackoverflow.com/a/23151714

hippolyzer/lib/base/inventory.py

@@ -3,13 +3,20 @@ Parse the horrible legacy inventory-related format.
 
 It's typically only used for object contents now.
 """
+
+# TODO: Maybe handle CRC calculation? Does anything care about that?
+#  I don't think anything in the viewer actually looks at the result
+#  of the CRC check for UDP stuff.
+
 from __future__ import annotations
 
+import abc
 import dataclasses
 import datetime as dt
+import inspect
 import logging
+import secrets
 import struct
-import typing
 import weakref
 from io import StringIO
 from typing import *
@@ -29,6 +36,8 @@ from hippolyzer.lib.base.legacy_schema import (
     SchemaUUID,
     schema_field,
 )
+from hippolyzer.lib.base.message.message import Block
+from hippolyzer.lib.base.templates import SaleType, InventoryType, LookupIntEnum, AssetType, FolderType
 
 MAGIC_ID = UUID("3c115e51-04f4-523c-9fa6-98aff1034730")
 LOG = logging.getLogger(__name__)
@@ -38,12 +47,42 @@ _T = TypeVar("_T")
 class SchemaFlagField(SchemaHexInt):
     """Like a hex int, but must be serialized as bytes in LLSD due to being a U32"""
     @classmethod
-    def from_llsd(cls, val: Any) -> int:
-        return struct.unpack("!I", val)[0]
+    def from_llsd(cls, val: Any, flavor: str) -> int:
+        # Sometimes values in S32 range will just come through normally
+        if isinstance(val, int):
+            return val
+
+        if flavor == "legacy":
+            return struct.unpack("!I", val)[0]
+        return val
 
     @classmethod
-    def to_llsd(cls, val: int) -> Any:
-        return struct.pack("!I", val)
+    def to_llsd(cls, val: int, flavor: str) -> Any:
+        if flavor == "legacy":
+            return struct.pack("!I", val)
+        return val
+
+
+class SchemaEnumField(SchemaStr, Generic[_T]):
+    def __init__(self, enum_cls: Type[LookupIntEnum]):
+        super().__init__()
+        self._enum_cls = enum_cls
+
+    def deserialize(self, val: str) -> _T:
+        return self._enum_cls.from_lookup_name(val)
+
+    def serialize(self, val: _T) -> str:
+        return self._enum_cls(val).to_lookup_name()
+
+    def from_llsd(self, val: Union[str, int], flavor: str) -> _T:
+        if flavor == "legacy":
+            return self.deserialize(val)
+        return self._enum_cls(val)
+
+    def to_llsd(self, val: _T, flavor: str) -> Union[int, str]:
+        if flavor == "legacy":
+            return self.serialize(val)
+        return int(val)
 
 
 def _yield_schema_tokens(reader: StringIO):
@@ -99,10 +138,14 @@ class InventoryBase(SchemaBase):
                 if not spec:
                     LOG.warning(f"Internal key {key!r}")
                     continue
+
+                spec_cls = spec
+                if not inspect.isclass(spec_cls):
+                    spec_cls = spec_cls.__class__
                 # some kind of nested structure like sale_info
-                if issubclass(spec, SchemaBase):
+                if issubclass(spec_cls, SchemaBase):
                     obj_dict[key] = spec.from_reader(reader)
-                elif issubclass(spec, SchemaFieldSerializer):
+                elif issubclass(spec_cls, SchemaFieldSerializer):
                     obj_dict[key] = spec.deserialize(val)
                 else:
                     raise ValueError(f"Unsupported spec for {key!r}, {spec!r}")
@@ -111,9 +154,21 @@ class InventoryBase(SchemaBase):
         return cls._obj_from_dict(obj_dict)
 
     def to_writer(self, writer: StringIO):
-        writer.write(f"\t{self.SCHEMA_NAME}\t0\n")
+        writer.write(f"\t{self.SCHEMA_NAME}")
+        if self.SCHEMA_NAME == "permissions":
+            writer.write(" 0\n")
+        else:
+            writer.write("\t0\n")
         writer.write("\t{\n")
-        for field_name, field in self._get_fields_dict().items():
+
+        # Make sure the ID field always comes first, if there is one.
+        fields_dict = {}
+        if hasattr(self, "ID_ATTR"):
+            fields_dict = {getattr(self, "ID_ATTR"): None}
+        # update()ing will put all fields that aren't yet in the dict after the ID attr.
+        fields_dict.update(self._get_fields_dict())
+
+        for field_name, field in fields_dict.items():
             spec = field.metadata.get("spec")
             # Not meant to be serialized
             if not spec:
@@ -122,20 +177,23 @@ class InventoryBase(SchemaBase):
                 continue
 
             val = getattr(self, field_name)
-            if val is None:
+            if val is None and not field.metadata.get("include_none"):
                 continue
 
+            spec_cls = spec
+            if not inspect.isclass(spec_cls):
+                spec_cls = spec_cls.__class__
             # Some kind of nested structure like sale_info
             if isinstance(val, SchemaBase):
                 val.to_writer(writer)
-            elif issubclass(spec, SchemaFieldSerializer):
+            elif issubclass(spec_cls, SchemaFieldSerializer):
                 writer.write(f"\t\t{field_name}\t{spec.serialize(val)}\n")
             else:
                 raise ValueError(f"Bad inventory spec {spec!r}")
         writer.write("\t}\n")
 
 
-class InventoryDifferences(typing.NamedTuple):
+class InventoryDifferences(NamedTuple):
     changed: List[InventoryNodeBase]
     removed: List[InventoryNodeBase]
 
@@ -166,12 +224,12 @@ class InventoryModel(InventoryBase):
         return model
 
     @classmethod
-    def from_llsd(cls, llsd_val: List[Dict]) -> InventoryModel:
+    def from_llsd(cls, llsd_val: List[Dict], flavor: str = "legacy") -> InventoryModel:
         model = cls()
         for obj_dict in llsd_val:
             for inv_type in INVENTORY_TYPES:
                 if inv_type.ID_ATTR in obj_dict:
-                    if (obj := inv_type.from_llsd(obj_dict)) is not None:
+                    if (obj := inv_type.from_llsd(obj_dict, flavor)) is not None:
                         model.add(obj)
                     break
                 LOG.warning(f"Unknown object type {obj_dict!r}")
@@ -203,8 +261,8 @@ class InventoryModel(InventoryBase):
         for node in self.ordered_nodes:
             node.to_writer(writer)
 
-    def to_llsd(self):
-        return list(node.to_llsd() for node in self.ordered_nodes)
+    def to_llsd(self, flavor: str = "legacy"):
+        return list(node.to_llsd(flavor) for node in self.ordered_nodes)
 
     def add(self, node: InventoryNodeBase):
         if node.node_id in self.nodes:
@@ -280,24 +338,31 @@ class InventoryPermissions(InventoryBase):
     group_id: UUID = schema_field(SchemaUUID)
     # Nothing actually cares about this, but it could be there.
     # It's kind of redundant since it just means owner_id == NULL_KEY && group_id != NULL_KEY.
-    is_owner_group: int = schema_field(SchemaInt, default=0, llsd_only=True)
+    is_owner_group: Optional[int] = schema_field(SchemaInt, default=None, llsd_only=True)
 
 
 @dataclasses.dataclass
 class InventorySaleInfo(InventoryBase):
     SCHEMA_NAME: ClassVar[str] = "sale_info"
 
-    sale_type: str = schema_field(SchemaStr)
+    sale_type: SaleType = schema_field(SchemaEnumField(SaleType))
     sale_price: int = schema_field(SchemaInt)
 
 
-@dataclasses.dataclass
-class InventoryNodeBase(InventoryBase):
-    ID_ATTR: ClassVar[str]
-
+class _HasName(abc.ABC):
+    """
+    Only exists so that we can assert that all subclasses should have this without forcing
+    a particular serialization order, as would happen if this was present on InventoryNodeBase.
+    """
     name: str
 
+
+@dataclasses.dataclass
+class InventoryNodeBase(InventoryBase, _HasName):
+    ID_ATTR: ClassVar[str]
+
     parent_id: Optional[UUID] = schema_field(SchemaUUID)
+
     model: Optional[InventoryModel] = dataclasses.field(
         default=None, init=False, hash=False, compare=False, repr=False
     )
@@ -338,8 +403,7 @@ class InventoryNodeBase(InventoryBase):
 
 @dataclasses.dataclass
 class InventoryContainerBase(InventoryNodeBase):
-    type: str = schema_field(SchemaStr)
-    name: str = schema_field(SchemaMultilineStr)
+    type: AssetType = schema_field(SchemaEnumField(AssetType))
 
     @property
     def children(self) -> Sequence[InventoryNodeBase]:
@@ -368,8 +432,8 @@ class InventoryContainerBase(InventoryNodeBase):
             name=name,
             cat_id=UUID.random(),
             parent_id=self.node_id,
-            type="category",
-            pref_type="-1",
+            type=AssetType.CATEGORY,
+            pref_type=FolderType.NONE,
             owner_id=getattr(self, 'owner_id', UUID.ZERO),
             version=1,
         )
@@ -386,7 +450,8 @@ class InventoryObject(InventoryContainerBase):
     ID_ATTR: ClassVar[str] = "obj_id"
 
     obj_id: UUID = schema_field(SchemaUUID)
-    metadata: Optional[Dict[str, Any]] = schema_field(SchemaLLSD, default=None)
+    name: str = schema_field(SchemaMultilineStr)
+    metadata: Optional[Dict[str, Any]] = schema_field(SchemaLLSD, default=None, include_none=True)
 
     __hash__ = InventoryNodeBase.__hash__
 
@@ -398,10 +463,43 @@ class InventoryCategory(InventoryContainerBase):
     VERSION_NONE: ClassVar[int] = -1
 
     cat_id: UUID = schema_field(SchemaUUID)
-    pref_type: str = schema_field(SchemaStr, llsd_name="preferred_type")
-    owner_id: UUID = schema_field(SchemaUUID)
-    version: int = schema_field(SchemaInt)
-    metadata: Optional[Dict[str, Any]] = schema_field(SchemaLLSD, default=None)
+    pref_type: FolderType = schema_field(SchemaEnumField(FolderType), llsd_name="preferred_type")
+    name: str = schema_field(SchemaMultilineStr)
+    owner_id: Optional[UUID] = schema_field(SchemaUUID, default=None)
+    version: int = schema_field(SchemaInt, default=VERSION_NONE, llsd_only=True)
+    metadata: Optional[Dict[str, Any]] = schema_field(SchemaLLSD, default=None, include_none=False)
+
+    def to_folder_data(self) -> Block:
+        return Block(
+            "FolderData",
+            FolderID=self.cat_id,
+            ParentID=self.parent_id,
+            CallbackID=0,
+            Type=self.pref_type,
+            Name=self.name,
+        )
+
+    @classmethod
+    def from_folder_data(cls, block: Block):
+        return cls(
+            cat_id=block["FolderID"],
+            parent_id=block["ParentID"],
+            pref_type=block["Type"],
+            name=block["Name"],
+            type=AssetType.CATEGORY,
+        )
+
+    @classmethod
+    def _get_fields_dict(cls, llsd_flavor: Optional[str] = None):
+        fields = super()._get_fields_dict(llsd_flavor)
+        if llsd_flavor == "ais":
+            # AIS is smart enough to know that all categories are asset type category...
+            fields.pop("type")
+            # These have different names though
+            fields["type_default"] = fields.pop("preferred_type")
+            fields["agent_id"] = fields.pop("owner_id")
+            fields["category_id"] = fields.pop("cat_id")
+        return fields
 
     __hash__ = InventoryNodeBase.__hash__
 
@@ -412,17 +510,17 @@ class InventoryItem(InventoryNodeBase):
     ID_ATTR: ClassVar[str] = "item_id"
 
     item_id: UUID = schema_field(SchemaUUID)
-    type: str = schema_field(SchemaStr)
-    inv_type: str = schema_field(SchemaStr)
-    flags: int = schema_field(SchemaFlagField)
-    name: str = schema_field(SchemaMultilineStr)
-    desc: str = schema_field(SchemaMultilineStr)
-    creation_date: dt.datetime = schema_field(SchemaDate, llsd_name="created_at")
     permissions: InventoryPermissions = schema_field(InventoryPermissions)
-    sale_info: InventorySaleInfo = schema_field(InventorySaleInfo)
     asset_id: Optional[UUID] = schema_field(SchemaUUID, default=None)
     shadow_id: Optional[UUID] = schema_field(SchemaUUID, default=None)
-    metadata: Optional[Dict[str, Any]] = schema_field(SchemaLLSD, default=None)
+    type: Optional[AssetType] = schema_field(SchemaEnumField(AssetType), default=None)
+    inv_type: Optional[InventoryType] = schema_field(SchemaEnumField(InventoryType), default=None)
+    flags: Optional[int] = schema_field(SchemaFlagField, default=None)
+    sale_info: Optional[InventorySaleInfo] = schema_field(InventorySaleInfo, default=None)
+    name: Optional[str] = schema_field(SchemaMultilineStr, default=None)
+    desc: Optional[str] = schema_field(SchemaMultilineStr, default=None)
+    metadata: Optional[Dict[str, Any]] = schema_field(SchemaLLSD, default=None, include_none=True)
+    creation_date: Optional[dt.datetime] = schema_field(SchemaDate, llsd_name="created_at", default=None)
 
     __hash__ = InventoryNodeBase.__hash__
 
@@ -432,5 +530,68 @@ class InventoryItem(InventoryNodeBase):
             return self.asset_id
         return self.shadow_id ^ MAGIC_ID
 
+    def to_inventory_data(self) -> Block:
+        return Block(
+            "InventoryData",
+            ItemID=self.item_id,
+            FolderID=self.parent_id,
+            CallbackID=0,
+            CreatorID=self.permissions.creator_id,
+            OwnerID=self.permissions.owner_id,
+            GroupID=self.permissions.group_id,
+            BaseMask=self.permissions.base_mask,
+            OwnerMask=self.permissions.owner_mask,
+            GroupMask=self.permissions.group_mask,
+            EveryoneMask=self.permissions.everyone_mask,
+            NextOwnerMask=self.permissions.next_owner_mask,
+            GroupOwned=self.permissions.owner_id == UUID.ZERO and self.permissions.group_id != UUID.ZERO,
+            AssetID=self.true_asset_id,
+            Type=self.type,
+            InvType=self.inv_type,
+            Flags=self.flags,
+            SaleType=self.sale_info.sale_type,
+            SalePrice=self.sale_info.sale_price,
+            Name=self.name,
+            Description=self.desc,
+            CreationDate=SchemaDate.to_llsd(self.creation_date, "legacy"),
+            # Meaningless here
+            CRC=secrets.randbits(32),
+        )
+
+    @classmethod
+    def from_inventory_data(cls, block: Block):
+        return cls(
+            item_id=block["ItemID"],
+            parent_id=block["ParentID"],
+            permissions=InventoryPermissions(
+                creator_id=block["CreatorID"],
+                owner_id=block["OwnerID"],
+                group_id=block["GroupID"],
+                base_mask=block["BaseMask"],
+                owner_mask=block["OwnerMask"],
+                group_mask=block["GroupMask"],
+                everyone_mask=block["EveryoneMask"],
+                next_owner_mask=block["NextOwnerMask"],
+            ),
+            asset_id=block["AssetID"],
+            type=AssetType(block["Type"]),
+            inv_type=InventoryType(block["InvType"]),
+            flags=block["Flags"],
+            sale_info=InventorySaleInfo(
+                sale_type=SaleType(block["SaleType"]),
+                sale_price=block["SalePrice"],
+            ),
+            name=block["Name"],
+            desc=block["Description"],
+            creation_date=block["CreationDate"],
+        )
+
+    def to_llsd(self, flavor: str = "legacy"):
+        val = super().to_llsd(flavor=flavor)
+        if flavor == "ais":
+            # There's little chance this differs from owner ID, just place it.
+            val["agent_id"] = val["permissions"]["owner_id"]
+        return val
+
 
 INVENTORY_TYPES: Tuple[Type[InventoryNodeBase], ...] = (InventoryCategory, InventoryObject, InventoryItem)

hippolyzer/lib/base/legacy_schema.py

@@ -9,6 +9,7 @@ import abc
 import calendar
 import dataclasses
 import datetime as dt
+import inspect
 import logging
 import re
 from io import StringIO
@@ -34,11 +35,11 @@ class SchemaFieldSerializer(abc.ABC, Generic[_T]):
         pass
 
     @classmethod
-    def from_llsd(cls, val: Any) -> _T:
+    def from_llsd(cls, val: Any, flavor: str) -> _T:
         return val
 
     @classmethod
-    def to_llsd(cls, val: _T) -> Any:
+    def to_llsd(cls, val: _T, flavor: str) -> Any:
         return val
 
 
@@ -52,11 +53,11 @@ class SchemaDate(SchemaFieldSerializer[dt.datetime]):
         return str(calendar.timegm(val.utctimetuple()))
 
     @classmethod
-    def from_llsd(cls, val: Any) -> dt.datetime:
+    def from_llsd(cls, val: Any, flavor: str) -> dt.datetime:
         return dt.datetime.utcfromtimestamp(val)
 
     @classmethod
-    def to_llsd(cls, val: dt.datetime):
+    def to_llsd(cls, val: dt.datetime, flavor: str):
         return calendar.timegm(val.utctimetuple())
 
 
@@ -103,6 +104,13 @@ class SchemaStr(SchemaFieldSerializer[str]):
 
 
 class SchemaUUID(SchemaFieldSerializer[UUID]):
+    @classmethod
+    def from_llsd(cls, val: Any, flavor: str) -> UUID:
+        # FetchInventory2 will return a string, but we want a UUID. It's not an issue
+        # for us to return a UUID later there because it'll just cast to string if
+        # that's what it wants
+        return UUID(val)
+
     @classmethod
     def deserialize(cls, val: str) -> UUID:
         return UUID(val)
@@ -116,19 +124,24 @@ class SchemaLLSD(SchemaFieldSerializer[_T]):
     """Arbitrary LLSD embedded in a field"""
     @classmethod
     def deserialize(cls, val: str) -> _T:
-        return llsd.parse_notation(val.encode("utf8"))
+        return llsd.parse_xml(val.partition("|")[0].encode("utf8"))
 
     @classmethod
     def serialize(cls, val: _T) -> str:
-        return llsd.format_notation(val).decode("utf8")
+        # Don't include the XML header
+        return llsd.format_xml(val).split(b">", 1)[1].decode("utf8") + "\n|"
 
 
-def schema_field(spec: Type[Union[SchemaBase, SchemaFieldSerializer]], *, default=dataclasses.MISSING, init=True,
-                 repr=True, hash=None, compare=True, llsd_name=None, llsd_only=False) -> dataclasses.Field:  # noqa
+_SCHEMA_SPEC = Union[Type[Union["SchemaBase", SchemaFieldSerializer]], SchemaFieldSerializer]
+
+
+def schema_field(spec: _SCHEMA_SPEC, *, default=dataclasses.MISSING, init=True,
+                 repr=True, hash=None, compare=True, llsd_name=None, llsd_only=False,
+                 include_none=False) -> dataclasses.Field:  # noqa
     """Describe a field in the inventory schema and the shape of its value"""
     return dataclasses.field(  # noqa
-        metadata={"spec": spec, "llsd_name": llsd_name, "llsd_only": llsd_only}, default=default,
-        init=init, repr=repr, hash=hash, compare=compare,
+        metadata={"spec": spec, "llsd_name": llsd_name, "llsd_only": llsd_only, "include_none": include_none},
+        default=default, init=init, repr=repr, hash=hash, compare=compare,
     )
 
 
@@ -151,11 +164,11 @@ def parse_schema_line(line: str):
 @dataclasses.dataclass
 class SchemaBase(abc.ABC):
     @classmethod
-    def _get_fields_dict(cls, llsd=False):
+    def _get_fields_dict(cls, llsd_flavor: Optional[str] = None):
         fields_dict = {}
         for field in dataclasses.fields(cls):
             field_name = field.name
-            if llsd:
+            if llsd_flavor:
                 field_name = field.metadata.get("llsd_name") or field_name
             fields_dict[field_name] = field
         return fields_dict
@@ -174,8 +187,8 @@ class SchemaBase(abc.ABC):
         return cls.from_str(data.decode("utf8"))
 
     @classmethod
-    def from_llsd(cls, inv_dict: Dict):
-        fields = cls._get_fields_dict(llsd=True)
+    def from_llsd(cls, inv_dict: Dict, flavor: str = "legacy"):
+        fields = cls._get_fields_dict(llsd_flavor=flavor)
         obj_dict = {}
         for key, val in inv_dict.items():
             if key in fields:
@@ -186,15 +199,23 @@ class SchemaBase(abc.ABC):
                 if not spec:
                     LOG.warning(f"Internal key {key!r}")
                     continue
+
+                spec_cls = spec
+                if not inspect.isclass(spec_cls):
+                    spec_cls = spec_cls.__class__
+
                 # some kind of nested structure like sale_info
-                if issubclass(spec, SchemaBase):
-                    obj_dict[key] = spec.from_llsd(val)
-                elif issubclass(spec, SchemaFieldSerializer):
-                    obj_dict[key] = spec.from_llsd(val)
+                if issubclass(spec_cls, SchemaBase):
+                    obj_dict[key] = spec.from_llsd(val, flavor)
+                elif issubclass(spec_cls, SchemaFieldSerializer):
+                    obj_dict[key] = spec.from_llsd(val, flavor)
                 else:
                     raise ValueError(f"Unsupported spec for {key!r}, {spec!r}")
             else:
-                LOG.warning(f"Unknown key {key!r}")
+                if flavor != "ais":
+                    # AIS has a number of different fields that are irrelevant depending on
+                    # what exactly sent the payload
+                    LOG.warning(f"Unknown key {key!r}")
         return cls._obj_from_dict(obj_dict)
 
     def to_bytes(self) -> bytes:
@@ -206,9 +227,9 @@ class SchemaBase(abc.ABC):
         writer.seek(0)
         return writer.read()
 
-    def to_llsd(self):
+    def to_llsd(self, flavor: str = "legacy"):
         obj_dict = {}
-        for field_name, field in self._get_fields_dict(llsd=True).items():
+        for field_name, field in self._get_fields_dict(llsd_flavor=flavor).items():
             spec = field.metadata.get("spec")
             # Not meant to be serialized
             if not spec:
@@ -218,11 +239,15 @@ class SchemaBase(abc.ABC):
             if val is None:
                 continue
 
+            spec_cls = spec
+            if not inspect.isclass(spec_cls):
+                spec_cls = spec_cls.__class__
+
             # Some kind of nested structure like sale_info
             if isinstance(val, SchemaBase):
-                val = val.to_llsd()
-            elif issubclass(spec, SchemaFieldSerializer):
-                val = spec.to_llsd(val)
+                val = val.to_llsd(flavor)
+            elif issubclass(spec_cls, SchemaFieldSerializer):
+                val = spec.to_llsd(val, flavor)
             else:
                 raise ValueError(f"Bad inventory spec {spec!r}")
             obj_dict[field_name] = val

hippolyzer/lib/base/serialization.py

@@ -1580,8 +1580,15 @@ def bitfield_field(bits: int, *, adapter: Optional[Adapter] = None, default=0, i
 
 
 class BitfieldDataclass(DataclassAdapter):
-    def __init__(self, data_cls: Type,
+    PRIM_SPEC: ClassVar[Optional[SerializablePrimitive]] = None
+
+    def __init__(self, data_cls: Optional[Type] = None,
                  prim_spec: Optional[SerializablePrimitive] = None, shift: bool = True):
+        if not dataclasses.is_dataclass(data_cls):
+            raise ValueError(f"{data_cls!r} is not a dataclass")
+        if prim_spec is None:
+            prim_spec = getattr(data_cls, 'PRIM_SPEC', None)
+
         super().__init__(data_cls, prim_spec)
         self._shift = shift
         self._bitfield_spec = self._build_bitfield(data_cls)

hippolyzer/lib/base/templates.py

@@ -15,9 +15,40 @@ from typing import *
 import hippolyzer.lib.base.serialization as se
 from hippolyzer.lib.base import llsd
 from hippolyzer.lib.base.datatypes import UUID, IntEnum, IntFlag, Vector3, Quaternion
+from hippolyzer.lib.base.helpers import BiDiDict
 from hippolyzer.lib.base.namevalue import NameValuesSerializer
 
 
+class LookupIntEnum(IntEnum):
+    """
+    Used for enums that have legacy string names, may be used in the legacy schema
+
+    Generally this is the string returned by `LLWhateverType::lookup()` in indra
+    """
+    @abc.abstractmethod
+    def to_lookup_name(self) -> str:
+        raise NotImplementedError()
+
+    @classmethod
+    def from_lookup_name(cls, legacy_name: str):
+        raise NotImplementedError()
+
+
+_ASSET_TYPE_BIDI: BiDiDict[str] = BiDiDict({
+    "animation": "animatn",
+    "callingcard": "callcard",
+    "lsl_text": "lsltext",
+    "lsl_bytecode": "lslbyte",
+    "texture_tga": "txtr_tga",
+    "image_tga": "img_tga",
+    "image_jpeg": "jpg",
+    "sound_wav": "snd_wav",
+    "folder_link": "link_f",
+    "unknown": "invalid",
+    "none": "-1",
+})
+
+
 @se.enum_field_serializer("RequestXfer", "XferID", "VFileType")
 @se.enum_field_serializer("AssetUploadRequest", "AssetBlock", "Type")
 @se.enum_field_serializer("AssetUploadComplete", "AssetBlock", "Type")
@@ -26,7 +57,7 @@ from hippolyzer.lib.base.namevalue import NameValuesSerializer
 @se.enum_field_serializer("RezObject", "InventoryData", "Type")
 @se.enum_field_serializer("RezScript", "InventoryBlock", "Type")
 @se.enum_field_serializer("UpdateTaskInventory", "InventoryData", "Type")
-class AssetType(IntEnum):
+class AssetType(LookupIntEnum):
     TEXTURE = 0
     SOUND = 1
     CALLINGCARD = 2
@@ -47,7 +78,7 @@ class AssetType(IntEnum):
     GESTURE = 21
     SIMSTATE = 22
     LINK = 24
-    LINK_FOLDER = 25
+    FOLDER_LINK = 25
     MARKETPLACE_FOLDER = 26
     WIDGET = 40
     PERSON = 45
@@ -62,16 +93,14 @@ class AssetType(IntEnum):
     UNKNOWN = 255
     NONE = -1
 
-    @property
-    def human_name(self):
+    def to_lookup_name(self) -> str:
         lower = self.name.lower()
-        return {
-            "animation": "animatn",
-            "callingcard": "callcard",
-            "texture_tga": "txtr_tga",
-            "image_tga": "img_tga",
-            "sound_wav": "snd_wav",
-        }.get(lower, lower)
+        return _ASSET_TYPE_BIDI.forward.get(lower, lower)
+
+    @classmethod
+    def from_lookup_name(cls, legacy_name: str):
+        reg_name = _ASSET_TYPE_BIDI.backward.get(legacy_name, legacy_name).upper()
+        return cls[reg_name]
 
     @property
     def inventory_type(self):
@@ -99,12 +128,19 @@ class AssetType(IntEnum):
         }.get(self, AssetType.NONE)
 
 
+_INV_TYPE_BIDI: BiDiDict[str] = BiDiDict({
+    "callingcard": "callcard",
+    "attachment": "attach",
+    "none": "-1",
+})
+
+
 @se.enum_field_serializer("UpdateCreateInventoryItem", "InventoryData", "InvType")
 @se.enum_field_serializer("CreateInventoryItem", "InventoryBlock", "InvType")
 @se.enum_field_serializer("RezObject", "InventoryData", "InvType")
 @se.enum_field_serializer("RezScript", "InventoryBlock", "InvType")
 @se.enum_field_serializer("UpdateTaskInventory", "InventoryData", "InvType")
-class InventoryType(IntEnum):
+class InventoryType(LookupIntEnum):
     TEXTURE = 0
     SOUND = 1
     CALLINGCARD = 2
@@ -133,16 +169,37 @@ class InventoryType(IntEnum):
     UNKNOWN = 255
     NONE = -1
 
-    @property
-    def human_name(self):
+    def to_lookup_name(self) -> str:
         lower = self.name.lower()
-        return {
-            "callingcard": "callcard",
-            "none": "-1",
-        }.get(lower, lower)
+        return _INV_TYPE_BIDI.forward.get(lower, lower)
+
+    @classmethod
+    def from_lookup_name(cls, legacy_name: str):
+        reg_name = _INV_TYPE_BIDI.backward.get(legacy_name, legacy_name).upper()
+        return cls[reg_name]
 
 
-class FolderType(IntEnum):
+_FOLDER_TYPE_BIDI: BiDiDict[str] = BiDiDict({
+    "callingcard": "callcard",
+    "lsl_text": "lsltext",
+    "animation": "animatn",
+    "snapshot_category": "snapshot",
+    "lost_and_found": "lstndfnd",
+    "ensemble_start": "ensemble",
+    "ensemble_end": "ensemble",
+    "current_outfit": "current",
+    "my_outfits": "my_otfts",
+    "basic_root": "basic_rt",
+    "marketplace_listings": "merchant",
+    "marketplace_stock": "stock",
+    "marketplace_version": "version",
+    "my_suitcase": "suitcase",
+    "root_inventory": "root_inv",
+    "none": "-1",
+})
+
+
+class FolderType(LookupIntEnum):
     TEXTURE = 0
     SOUND = 1
     CALLINGCARD = 2
@@ -161,6 +218,7 @@ class FolderType(IntEnum):
     ANIMATION = 20
     GESTURE = 21
     FAVORITE = 23
+    # The "ensemble" values aren't used, no idea what they were for.
     ENSEMBLE_START = 26
     ENSEMBLE_END = 45
     # This range is reserved for special clothing folder types.
@@ -177,7 +235,7 @@ class FolderType(IntEnum):
     # Note: We actually *never* create folders with that type. This is used for icon override only.
     MARKETPLACE_VERSION = 55
     SETTINGS = 56
-    # Firestorm folders, may not actually exist
+    # Firestorm folders, may not actually exist in legacy schema
     FIRESTORM = 57
     PHOENIX = 58
     RLV = 59
@@ -185,6 +243,15 @@ class FolderType(IntEnum):
     MY_SUITCASE = 100
     NONE = -1
 
+    def to_lookup_name(self) -> str:
+        lower = self.name.lower()
+        return _FOLDER_TYPE_BIDI.forward.get(lower, lower)
+
+    @classmethod
+    def from_lookup_name(cls, legacy_name: str):
+        reg_name = _FOLDER_TYPE_BIDI.backward.get(legacy_name, legacy_name).upper()
+        return cls[reg_name]
+
 
 @se.enum_field_serializer("AgentIsNowWearing", "WearableData", "WearableType")
 @se.enum_field_serializer("AgentWearablesUpdate", "WearableData", "WearableType")
@@ -244,6 +311,9 @@ class Permissions(IntFlag):
     RESERVED = 1 << 31
 
 
+_SALE_TYPE_LEGACY_NAMES = ("not", "orig", "copy", "cntn")
+
+
 @se.enum_field_serializer("ObjectSaleInfo", "ObjectData", "SaleType")
 @se.enum_field_serializer("ObjectProperties", "ObjectData", "SaleType")
 @se.enum_field_serializer("ObjectPropertiesFamily", "ObjectData", "SaleType")
@@ -252,12 +322,19 @@ class Permissions(IntFlag):
 @se.enum_field_serializer("RezObject", "InventoryData", "SaleType")
 @se.enum_field_serializer("UpdateTaskInventory", "InventoryData", "SaleType")
 @se.enum_field_serializer("UpdateCreateInventoryItem", "InventoryData", "SaleType")
-class SaleInfo(IntEnum):
+class SaleType(LookupIntEnum):
     NOT = 0
     ORIGINAL = 1
     COPY = 2
     CONTENTS = 3
 
+    @classmethod
+    def from_lookup_name(cls, legacy_name: str):
+        return cls(_SALE_TYPE_LEGACY_NAMES.index(legacy_name))
+
+    def to_lookup_name(self) -> str:
+        return _SALE_TYPE_LEGACY_NAMES[int(self.value)]
+
 
 @se.flag_field_serializer("ParcelInfoReply", "Data", "Flags")
 class ParcelInfoFlags(IntFlag):

hippolyzer/lib/client/asset_uploader.py

@@ -30,12 +30,12 @@ class AssetUploader:
     async def initiate_asset_upload(self, name: str, asset_type: AssetType,
                                     body: bytes, flags: Optional[int] = None) -> UploadToken:
         payload = {
-            "asset_type": asset_type.human_name,
+            "asset_type": asset_type.to_lookup_name(),
             "description": "(No Description)",
             "everyone_mask": 0,
             "group_mask": 0,
             "folder_id": UUID.ZERO,  # Puts it in the default folder, I guess. Undocumented.
-            "inventory_type": asset_type.inventory_type.human_name,
+            "inventory_type": asset_type.inventory_type.to_lookup_name(),
             "name": name,
             "next_owner_mask": 581632,
         }

hippolyzer/lib/client/inventory_manager.py

@@ -2,14 +2,13 @@ from __future__ import annotations
 
 import gzip
 import logging
-import secrets
 from pathlib import Path
 from typing import Union, List, Tuple, Set
 
 from hippolyzer.lib.base import llsd
 from hippolyzer.lib.base.datatypes import UUID
 from hippolyzer.lib.base.inventory import InventoryModel, InventoryCategory, InventoryItem
-from hippolyzer.lib.base.message.message import Block
+from hippolyzer.lib.base.templates import AssetType, FolderType
 from hippolyzer.lib.client.state import BaseClientSession
 
 
@@ -33,8 +32,8 @@ class InventoryManager:
                 # Don't use the version from the skeleton, this flags the inventory as needing
                 # completion from the inventory cache. This matches indra's behavior.
                 version=InventoryCategory.VERSION_NONE,
-                type="category",
-                pref_type=skel_cat.get("type_default", "-1"),
+                type=AssetType.CATEGORY,
+                pref_type=FolderType(skel_cat.get("type_default", FolderType.NONE)),
                 owner_id=self._session.agent_id,
             ))
 
@@ -87,6 +86,7 @@ class InventoryManager:
             self.model.add(cached_item)
 
     def _parse_cache(self, path: Union[str, Path]) -> Tuple[List[InventoryCategory], List[InventoryItem]]:
+        """Warning, may be incredibly slow due to llsd.parse_notation() behavior"""
         categories: List[InventoryCategory] = []
         items: List[InventoryItem] = []
         # Parse our cached items and categories out of the compressed inventory cache
@@ -112,81 +112,3 @@ class InventoryManager:
                 else:
                     LOG.warning(f"Unknown node type in inv cache: {node_llsd!r}")
         return categories, items
-
-
-# Thankfully we have 9 billion different ways to represent inventory data.
-def ais_item_to_inventory_data(ais_item: dict) -> Block:
-    return Block(
-        "InventoryData",
-        ItemID=ais_item["item_id"],
-        FolderID=ais_item["parent_id"],
-        CallbackID=0,
-        CreatorID=ais_item["permissions"]["creator_id"],
-        OwnerID=ais_item["permissions"]["owner_id"],
-        GroupID=ais_item["permissions"]["group_id"],
-        BaseMask=ais_item["permissions"]["base_mask"],
-        OwnerMask=ais_item["permissions"]["owner_mask"],
-        GroupMask=ais_item["permissions"]["group_mask"],
-        EveryoneMask=ais_item["permissions"]["everyone_mask"],
-        NextOwnerMask=ais_item["permissions"]["next_owner_mask"],
-        GroupOwned=0,
-        AssetID=ais_item["asset_id"],
-        Type=ais_item["type"],
-        InvType=ais_item["inv_type"],
-        Flags=ais_item["flags"],
-        SaleType=ais_item["sale_info"]["sale_type"],
-        SalePrice=ais_item["sale_info"]["sale_price"],
-        Name=ais_item["name"],
-        Description=ais_item["desc"],
-        CreationDate=ais_item["created_at"],
-        # Meaningless here
-        CRC=secrets.randbits(32),
-    )
-
-
-def inventory_data_to_ais_item(inventory_data: Block) -> dict:
-    return dict(
-        item_id=inventory_data["ItemID"],
-        parent_id=inventory_data["ParentID"],
-        permissions=dict(
-            creator_id=inventory_data["CreatorID"],
-            owner_id=inventory_data["OwnerID"],
-            group_id=inventory_data["GroupID"],
-            base_mask=inventory_data["BaseMask"],
-            owner_mask=inventory_data["OwnerMask"],
-            group_mask=inventory_data["GroupMask"],
-            everyone_mask=inventory_data["EveryoneMask"],
-            next_owner_mask=inventory_data["NextOwnerMask"],
-        ),
-        asset_id=inventory_data["AssetID"],
-        type=inventory_data["Type"],
-        inv_type=inventory_data["InvType"],
-        flags=inventory_data["Flags"],
-        sale_info=dict(
-            sale_type=inventory_data["SaleType"],
-            sale_price=inventory_data["SalePrice"],
-        ),
-        name=inventory_data["Name"],
-        description=inventory_data["Description"],
-        creation_at=inventory_data["CreationDate"],
-    )
-
-
-def ais_folder_to_inventory_data(ais_folder: dict) -> Block:
-    return Block(
-        "FolderData",
-        FolderID=ais_folder["cat_id"],
-        ParentID=ais_folder["parent_id"],
-        CallbackID=0,
-        Type=ais_folder["preferred_type"],
-        Name=ais_folder["name"],
-    )
-
-
-def inventory_data_to_ais_folder(inventory_data: Block) -> dict:
-    return dict(
-        cat_id=inventory_data["FolderID"],
-        parent_id=inventory_data["ParentID"],
-        preferred_type=inventory_data["Type"],
-        name=inventory_data["Name"],
-    )

hippolyzer/lib/proxy/asset_uploader.py

@@ -1,8 +1,8 @@
 from hippolyzer.lib.base.datatypes import UUID
+from hippolyzer.lib.base.inventory import InventoryItem
 from hippolyzer.lib.base.message.message import Message, Block
 from hippolyzer.lib.base.network.transport import Direction
 from hippolyzer.lib.client.asset_uploader import AssetUploader
-from hippolyzer.lib.client.inventory_manager import ais_item_to_inventory_data
 
 
 class ProxyAssetUploader(AssetUploader):
@@ -22,7 +22,7 @@ class ProxyAssetUploader(AssetUploader):
             ]
         }
         async with self._region.caps_client.post('FetchInventory2', llsd=ais_req_data) as resp:
-            ais_item = (await resp.read_llsd())["items"][0]
+            ais_item = InventoryItem.from_llsd((await resp.read_llsd())["items"][0], flavor="ais")
 
         # Got it, ship it off to the viewer
         message = Message(
@@ -33,7 +33,7 @@ class ProxyAssetUploader(AssetUploader):
                 SimApproved=1,
                 TransactionID=UUID.random(),
             ),
-            ais_item_to_inventory_data(ais_item),
+            ais_item.to_inventory_data(),
             direction=Direction.IN
         )
         self._region.circuit.send(message)

hippolyzer/lib/proxy/inventory_manager.py

@@ -1,5 +1,5 @@
+import asyncio
 import datetime as dt
-import logging
 
 from hippolyzer.lib.base.helpers import get_mtime
 from hippolyzer.lib.client.inventory_manager import InventoryManager
@@ -12,6 +12,8 @@ class ProxyInventoryManager(InventoryManager):
         super().__init__(session)
         newest_cache = None
         newest_timestamp = dt.datetime(year=1970, month=1, day=1, tzinfo=dt.timezone.utc)
+        # So consumers know when the inventory should be complete
+        self.cache_loaded: asyncio.Event = asyncio.Event()
         # Look for the newest version of the cached inventory and use that.
         # Not foolproof, but close enough if we're not sure what viewer is being used.
         for cache_dir in iter_viewer_cache_dirs():
@@ -26,7 +28,8 @@ class ProxyInventoryManager(InventoryManager):
                 newest_cache = inv_cache_path
 
         if newest_cache:
-            try:
-                self.load_cache(newest_cache)
-            except:
-                logging.exception("Failed to load invcache")
+            cache_load_fut = asyncio.ensure_future(asyncio.to_thread(self.load_cache, newest_cache))
+            # Meh. Don't care if it fails.
+            cache_load_fut.add_done_callback(lambda *args: self.cache_loaded.set())
+        else:
+            self.cache_loaded.set()

setup.py

@@ -25,7 +25,7 @@ from setuptools import setup, find_packages
 
 here = path.abspath(path.dirname(__file__))
 
-version = '0.14.0'
+version = '0.14.2'
 
 with open(path.join(here, 'README.md')) as readme_fh:
     readme = readme_fh.read()
@@ -42,7 +42,6 @@ setup(
         "Operating System :: POSIX",
         "Operating System :: Microsoft :: Windows",
         "Programming Language :: Python :: 3 :: Only",
-        "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
@@ -80,7 +79,7 @@ setup(
         }
     },
     zip_safe=False,
-    python_requires='>=3.8',
+    python_requires='>=3.9',
     install_requires=[
         'llsd<1.1.0',
         'defusedxml',

tests/base/test_legacy_schema.py

@@ -2,7 +2,7 @@ import copy
 import unittest
 
 from hippolyzer.lib.base.datatypes import *
-from hippolyzer.lib.base.inventory import InventoryModel
+from hippolyzer.lib.base.inventory import InventoryModel, SaleType
 from hippolyzer.lib.base.wearables import Wearable, VISUAL_PARAMS
 
 SIMPLE_INV = """\tinv_object\t0
@@ -11,6 +11,8 @@ SIMPLE_INV = """\tinv_object\t0
 \t\tparent_id\t00000000-0000-0000-0000-000000000000
 \t\ttype\tcategory
 \t\tname\tContents|
+\t\tmetadata\t<llsd><undef /></llsd>
+|
 \t}
 \tinv_item\t0
 \t{
@@ -39,10 +41,23 @@ SIMPLE_INV = """\tinv_object\t0
 \t}
 \t\tname\tNew Script|
 \t\tdesc\t2020-04-20 04:20:39 lsl2 script|
+\t\tmetadata\t<llsd><map><key>experience</key><uuid>a2e76fcd-9360-4f6d-a924-000000000003</uuid></map></llsd>
+|
 \t\tcreation_date\t1587367239
 \t}
 """
 
+INV_CATEGORY = """\tinv_category\t0
+\t{
+\t\tcat_id\tf4d91477-def1-487a-b4f3-6fa201c17376
+\t\tparent_id\t00000000-0000-0000-0000-000000000000
+\t\ttype\tlsltext
+\t\tpref_type\tlsltext
+\t\tname\tScripts|
+\t\towner_id\ta2e76fcd-9360-4f6d-a924-000000000003
+\t}
+"""
+
 
 class TestLegacyInv(unittest.TestCase):
     def setUp(self) -> None:
@@ -52,15 +67,27 @@ class TestLegacyInv(unittest.TestCase):
         self.assertTrue(UUID('f4d91477-def1-487a-b4f3-6fa201c17376') in self.model.nodes)
         self.assertIsNotNone(self.model.root)
 
+    def test_parse_category(self):
+        model = InventoryModel.from_str(INV_CATEGORY)
+        self.assertEqual(UUID('f4d91477-def1-487a-b4f3-6fa201c17376'), model.root.node_id)
+
     def test_serialize(self):
-        self.model = InventoryModel.from_str(SIMPLE_INV)
         new_model = InventoryModel.from_str(self.model.to_str())
         self.assertEqual(self.model, new_model)
 
+    def test_serialize_category(self):
+        model = InventoryModel.from_str(INV_CATEGORY)
+        new_model = InventoryModel.from_str(model.to_str())
+        self.assertEqual(model, new_model)
+
+    def test_category_legacy_serialization(self):
+        self.assertEqual(INV_CATEGORY, InventoryModel.from_str(INV_CATEGORY).to_str())
+
     def test_item_access(self):
         item = self.model.nodes[UUID('dd163122-946b-44df-99f6-a6030e2b9597')]
         self.assertEqual(item.name, "New Script")
-        self.assertEqual(item.sale_info.sale_type, "not")
+        self.assertEqual(item.sale_info.sale_type, SaleType.NOT)
+        self.assertDictEqual(item.metadata, {"experience": UUID("a2e76fcd-9360-4f6d-a924-000000000003")})
         self.assertEqual(item.model, self.model)
 
     def test_access_children(self):
@@ -112,6 +139,7 @@ class TestLegacyInv(unittest.TestCase):
                     'inv_type': 'script',
                     'item_id': UUID('dd163122-946b-44df-99f6-a6030e2b9597'),
                     'name': 'New Script',
+                    'metadata': {"experience": UUID("a2e76fcd-9360-4f6d-a924-000000000003")},
                     'parent_id': UUID('f4d91477-def1-487a-b4f3-6fa201c17376'),
                     'permissions': {
                         'base_mask': 2147483647,
@@ -123,7 +151,6 @@ class TestLegacyInv(unittest.TestCase):
                         'next_owner_mask': 581632,
                         'owner_id': UUID('a2e76fcd-9360-4f6d-a924-000000000003'),
                         'owner_mask': 2147483647,
-                        'is_owner_group': 0,
                     },
                     'sale_info': {
                         'sale_price': 10,
@@ -134,12 +161,31 @@ class TestLegacyInv(unittest.TestCase):
             ]
         )
 
+    def test_llsd_serialization_ais(self):
+        model = InventoryModel.from_str(INV_CATEGORY)
+        self.assertEqual(
+            [
+                {
+                    'agent_id': UUID('a2e76fcd-9360-4f6d-a924-000000000003'),
+                    'category_id': UUID('f4d91477-def1-487a-b4f3-6fa201c17376'),
+                    'name': 'Scripts',
+                    'parent_id': UUID('00000000-0000-0000-0000-000000000000'),
+                    'type_default': 10,
+                    'version': -1
+                }
+            ],
+            model.to_llsd("ais")
+        )
+
     def test_llsd_legacy_equality(self):
         new_model = InventoryModel.from_llsd(self.model.to_llsd())
         self.assertEqual(self.model, new_model)
         new_model.root.name = "foo"
         self.assertNotEqual(self.model, new_model)
 
+    def test_legacy_serialization(self):
+        self.assertEqual(SIMPLE_INV, self.model.to_str())
+
     def test_difference_added(self):
         new_model = InventoryModel.from_llsd(self.model.to_llsd())
         diff = self.model.get_differences(new_model)