Add start of proxy inventory manager

hippolyzer/lib/base/inventory.py

@@ -117,6 +117,8 @@ class InventoryBase(SchemaBase):
             # Not meant to be serialized
             if not spec:
                 continue
+            if field.metadata.get("llsd_only"):
+                continue
 
             val = getattr(self, field_name)
             if val is None:
@@ -166,16 +168,11 @@ class InventoryModel(InventoryBase):
     def from_llsd(cls, llsd_val: List[Dict]) -> InventoryModel:
         model = cls()
         for obj_dict in llsd_val:
-            if InventoryCategory.ID_ATTR in obj_dict:
-                if (obj := InventoryCategory.from_llsd(obj_dict)) is not None:
-                    model.add(obj)
-            elif InventoryObject.ID_ATTR in obj_dict:
-                if (obj := InventoryObject.from_llsd(obj_dict)) is not None:
-                    model.add(obj)
-            elif InventoryItem.ID_ATTR in obj_dict:
-                if (obj := InventoryItem.from_llsd(obj_dict)) is not None:
-                    model.add(obj)
-            else:
+            for inv_type in INVENTORY_TYPES:
+                if inv_type.ID_ATTR in obj_dict:
+                    if (obj := inv_type.from_llsd(obj_dict)) is not None:
+                        model.add(obj)
+                    break
                 LOG.warning(f"Unknown object type {obj_dict!r}")
         return model
 
@@ -218,13 +215,13 @@ class InventoryModel(InventoryBase):
                 self.root = node
         node.model = weakref.proxy(self)
 
-    def unlink(self, node: InventoryNodeBase) -> Sequence[InventoryNodeBase]:
+    def unlink(self, node: InventoryNodeBase, single_only: bool = False) -> Sequence[InventoryNodeBase]:
         """Unlink a node and its descendants from the tree, returning the removed nodes"""
         assert node.model == self
         if node == self.root:
             self.root = None
         unlinked = [node]
-        if isinstance(node, InventoryContainerBase):
+        if isinstance(node, InventoryContainerBase) and not single_only:
             for child in node.children:
                 unlinked.extend(self.unlink(child))
         self.nodes.pop(node.node_id, None)
@@ -257,6 +254,15 @@ class InventoryModel(InventoryBase):
             removed=removed_in_other,
         )
 
+    def __getitem__(self, item: UUID) -> InventoryNodeBase:
+        return self.nodes[item]
+
+    def __contains__(self, item: UUID):
+        return item in self.nodes
+
+    def get(self, item: UUID) -> Optional[InventoryNodeBase]:
+        return self.nodes.get(item)
+
 
 @dataclasses.dataclass
 class InventoryPermissions(InventoryBase):
@@ -271,6 +277,9 @@ class InventoryPermissions(InventoryBase):
     owner_id: UUID = schema_field(SchemaUUID)
     last_owner_id: UUID = schema_field(SchemaUUID)
     group_id: UUID = schema_field(SchemaUUID)
+    # Nothing actually cares about this, but it could be there.
+    # It's kind of redundant since it just means owner_id == NULL_KEY && group_id != NULL_KEY.
+    is_owner_group: int = schema_field(SchemaInt, default=0, llsd_only=True)
 
 
 @dataclasses.dataclass
@@ -384,6 +393,7 @@ class InventoryObject(InventoryContainerBase):
 class InventoryCategory(InventoryContainerBase):
     ID_ATTR: ClassVar[str] = "cat_id"
     SCHEMA_NAME: ClassVar[str] = "inv_category"
+    VERSION_NONE: ClassVar[int] = -1
 
     cat_id: UUID = schema_field(SchemaUUID)
     pref_type: str = schema_field(SchemaStr, llsd_name="preferred_type")
@@ -417,3 +427,6 @@ class InventoryItem(InventoryNodeBase):
         if self.asset_id is not None:
             return self.asset_id
         return self.shadow_id ^ MAGIC_ID
+
+
+INVENTORY_TYPES: Tuple[Type[InventoryNodeBase], ...] = (InventoryCategory, InventoryObject, InventoryItem)

hippolyzer/lib/base/legacy_schema.py

@@ -111,10 +111,10 @@ class SchemaUUID(SchemaFieldSerializer[UUID]):
 
 
 def schema_field(spec: Type[Union[SchemaBase, SchemaFieldSerializer]], *, default=dataclasses.MISSING, init=True,
-                 repr=True, hash=None, compare=True, llsd_name=None) -> dataclasses.Field:  # noqa
+                 repr=True, hash=None, compare=True, llsd_name=None, llsd_only=False) -> dataclasses.Field:  # noqa
     """Describe a field in the inventory schema and the shape of its value"""
     return dataclasses.field(
-        metadata={"spec": spec, "llsd_name": llsd_name}, default=default,
+        metadata={"spec": spec, "llsd_name": llsd_name, "llsd_only": llsd_only}, default=default,
         init=init, repr=repr, hash=hash, compare=compare,
     )
 

hippolyzer/lib/client/inventory_manager.py

@@ -0,0 +1,90 @@
+import gzip
+import logging
+from pathlib import Path
+from typing import Union, List, Tuple, Set
+
+from hippolyzer.lib.base import llsd
+from hippolyzer.lib.base.datatypes import UUID
+from hippolyzer.lib.base.inventory import InventoryModel, InventoryCategory, InventoryItem
+from hippolyzer.lib.client.state import BaseClientSession
+
+
+LOG = logging.getLogger(__name__)
+
+
+class InventoryManager:
+    def __init__(self, session: BaseClientSession):
+        self._session = session
+        self.model: InventoryModel = InventoryModel()
+
+    def load_cache(self, path: Union[str, Path]):
+        # Per indra, rough flow for loading inv on login is:
+        # 1. Look at inventory skeleton from login response
+        # 2. Pre-populate model with categories from the skeleton, including their versions
+        # 3. Read the inventory cache, tracking categories and items separately
+        # 4. Walk the list of categories in our cache. If the cat exists in the skeleton and the versions
+        #    match, then we may load the category and its descendants from cache.
+        # 5. Any categories in the skeleton but not in the cache, or those with mismatched versions must be fetched.
+        #    The viewer does this by setting the local version of the cats to -1 and forcing a descendent fetch
+        #    over AIS.
+        #
+        # By the time you call this function call, you should have already loaded the inventory skeleton
+        # into the model set its inventory category versions to VERSION_NONE.
+
+        skel_cats: List[dict] = self._session.login_data['inventory-skeleton']
+        # UUID -> version map for inventory skeleton
+        skel_versions = {UUID(cat["folder_id"]): cat["version"] for cat in skel_cats}
+        cached_categories, cached_items = self._parse_cache(path)
+        loaded_cat_ids: Set[UUID] = set()
+
+        for cached_cat in cached_categories:
+            existing_cat: InventoryCategory = self.model.get(cached_cat.cat_id)  # noqa
+            # Don't clobber an existing cat unless it just has a placeholder version,
+            # maybe from loading the skeleton?
+            if existing_cat and existing_cat.version != InventoryCategory.VERSION_NONE:
+                continue
+            # Cached cat isn't the same as what the inv server says it should be, can't use it.
+            if cached_cat.version != skel_versions.get(cached_cat.cat_id):
+                continue
+            if existing_cat:
+                # Remove the category so that we can replace it, but leave any children in place
+                self.model.unlink(existing_cat, single_only=True)
+            self.model.add(cached_cat)
+            # Any items in this category in our cache file are usable and should be added
+            loaded_cat_ids.add(cached_cat.cat_id)
+
+        for cached_item in cached_items:
+            # The skeleton doesn't have any items, so if we run into any items they should be exactly the
+            # same as what we're trying to add. No point clobbering.
+            if cached_item.item_id in self.model:
+                continue
+            # The parent category didn't have a cache hit against the inventory skeleton, can't add!
+            if cached_item.parent_id not in loaded_cat_ids:
+                continue
+            self.model.add(cached_item)
+
+    def _parse_cache(self, path: Union[str, Path]) -> Tuple[List[InventoryCategory], List[InventoryItem]]:
+        categories: List[InventoryCategory] = []
+        items: List[InventoryItem] = []
+        # Parse our cached items and categories out of the compressed inventory cache
+        first_line = True
+        with gzip.open(path, "rb") as f:
+            # Line-delimited LLSD notation!
+            for line in f.readlines():
+                node_llsd = llsd.parse_notation(line)
+                if first_line:
+                    # First line is the file header
+                    first_line = False
+                    if node_llsd['inv_cache_version'] != 2:
+                        raise ValueError(f"Unknown cache version: {node_llsd!r}")
+                    continue
+
+                if InventoryCategory.ID_ATTR in node_llsd:
+                    if (cat_node := InventoryCategory.from_llsd(node_llsd)) is not None:
+                        categories.append(cat_node)
+                elif InventoryItem.ID_ATTR in node_llsd:
+                    if (item_node := InventoryItem.from_llsd(node_llsd)) is not None:
+                        items.append(item_node)
+                else:
+                    LOG.warning(f"Unknown node type in inv cache: {node_llsd!r}")
+        return categories, items

hippolyzer/lib/client/state.py

@@ -36,3 +36,4 @@ class BaseClientSession(abc.ABC):
     region_by_handle: Callable[[int], Optional[BaseClientRegion]]
     region_by_circuit_addr: Callable[[ADDR_TUPLE], Optional[BaseClientRegion]]
     objects: ClientWorldObjectManager
+    login_data: Dict[str, Any]

tests/base/test_legacy_schema.py

@@ -122,7 +122,8 @@ class TestLegacyInv(unittest.TestCase):
                         'last_owner_id': UUID('a2e76fcd-9360-4f6d-a924-000000000003'),
                         'next_owner_mask': 581632,
                         'owner_id': UUID('a2e76fcd-9360-4f6d-a924-000000000003'),
-                        'owner_mask': 2147483647
+                        'owner_mask': 2147483647,
+                        'is_owner_group': 0,
                     },
                     'sale_info': {
                         'sale_price': 10,