v0.3.2

This used to be the behaviour, but switching from queueing to
immediately adding messages to the log removed the implicit delay

.github/workflows/bundle_windows.yml

@@ -0,0 +1,46 @@
+# Have to manually unzip this (it gets double zipped) and add it
+# onto the release after it gets created. Don't want actions with repo write.
+name: Bundle Windows EXE
+
+on:
+  # Only trigger on release creation
+  release:
+    types:
+      - created
+  workflow_dispatch:
+
+
+jobs:
+  build:
+
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        python-version: [3.9]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e .
+        pip install cx_freeze
+
+    - name: Bundle with cx_Freeze
+      run: |
+        python setup_cxfreeze.py build_exe
+        pip install pip-licenses
+        pip-licenses --format=plain-vertical --with-license-file --no-license-path --output-file=lib_licenses.txt
+        python setup_cxfreeze.py finalize_cxfreeze
+
+    - name: Upload the artifact
+      uses: actions/upload-artifact@v2
+      with:
+        name: hippolyzer-gui-windows-${{ github.sha }}
+        path: ./dist/**

.github/workflows/pypi_publish.yml

@@ -6,6 +6,8 @@ on:
   release:
     types:
       - created
+  workflow_dispatch:
+
 
 # based on https://github.com/pypa/gh-action-pypi-publish
 

.github/workflows/pytest.yml

@@ -12,16 +12,33 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
+
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install flake8 pytest
+        pip install flake8 pytest pytest-cov
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
     - name: Test with pytest
       run: |
-        pytest
+        pytest --cov=./hippolyzer --cov=./tests --cov-report=xml
 
+    # Keep this in a workflow without any other secrets in it.
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v1
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        files: ./coverage.xml
+        directory: ./coverage/reports/
+        flags: unittests
+        env_vars: OS,PYTHON
+        name: codecov-umbrella
+        fail_ci_if_error: false
+        # We don't care if coverage drops
+        continue-on-error: true
+        path_to_write_report: ./coverage/codecov_report.txt
+        verbose: false

.gitignore

@@ -1,6 +1,7 @@
 #use glob syntax
 syntax: glob
 
+__pycache__
 *.pyc
 build/*
 *.egg-info

README.md

@@ -1,5 +1,7 @@
 # Hippolyzer
 
+![Python Test Status](https://github.com/SaladDais/Hippolyzer/workflows/Run%20Python%20Tests/badge.svg) [![codecov](https://codecov.io/gh/SaladDais/Hippolyzer/branch/master/graph/badge.svg?token=HCTFA4RAXX)](https://codecov.io/gh/SaladDais/Hippolyzer)
+
 [Hippolyzer](http://wiki.secondlife.com/wiki/Hippo) is a fork of Linden Lab's abandoned
 [PyOGP library](http://wiki.secondlife.com/wiki/PyOGP)
 targeting modern Python 3, with a focus on debugging issues in Second Life-compatible
@@ -22,6 +24,9 @@ with low-level SL details. See the [Local Animation addon example](https://githu
 ![Screenshot of proxy GUI](https://github.com/SaladDais/Hippolyzer/blob/master/static/screenshot.png?raw=true)
 
 ## Setup
+
+### From Source
+
 * Python 3.8 or above is **required**. If you're unable to upgrade your system Python package due to
   being on a stable distro, you can use [pyenv](https://github.com/pyenv/pyenv) to create
   a self-contained Python install with the appropriate version.
@@ -32,6 +37,11 @@ with low-level SL details. See the [Local Animation addon example](https://githu
 * * Under Windows it's `<virtualenv_dir>\Scripts\activate.bat`
 * Run `pip install hippolyzer`, or run `pip install -e .` in a cloned repo to install an editable version
 
+### Binary Windows Builds
+
+Binary Windows builds are available on the [Releases page](https://github.com/SaladDais/Hippolyzer/releases/).
+I don't extensively test these, building from source is recommended.
+
 ## Proxy
 
 A proxy is provided with both a CLI and Qt-based interface. The proxy application wraps a
@@ -289,8 +299,6 @@ If you are a viewer developer, please put them in a viewer.
 
 ## Potential Changes
 
-* Make package-able for PyPI
-* GitHub action to build binary packages and pull together licenses bundle
 * AISv3 wrapper?
 * Higher level wrappers for common things? I don't really need these, so only if people want to write them.
 * Highlight matched portion of message in log view, if applicable

codecov.yml

@@ -0,0 +1,11 @@
+coverage:
+  precision: 1
+  round: down
+  range: "50...80"
+  status:
+    project:
+      default:
+        # Do not fail PRs if the code coverage drops.
+        target: 0%
+        threshold: 100%
+        base: auto

hippolyzer/apps/proxy.py

@@ -186,3 +186,8 @@ def _windows_timeout_killer(pid: int):
 def main():
     multiprocessing.set_start_method("spawn")
     start_proxy()
+
+
+if __name__ == "__main__":
+    multiprocessing.freeze_support()
+    main()

hippolyzer/apps/proxy_gui.py

@@ -8,7 +8,6 @@ import json
 import logging
 import pathlib
 import multiprocessing
-import os
 import re
 import signal
 import socket
@@ -24,7 +23,7 @@ from hippolyzer.apps.model import MessageLogModel, MessageLogHeader, RegionListM
 from hippolyzer.apps.proxy import start_proxy
 from hippolyzer.lib.base import llsd
 from hippolyzer.lib.base.datatypes import UUID
-from hippolyzer.lib.base.helpers import bytes_unescape, bytes_escape
+from hippolyzer.lib.base.helpers import bytes_unescape, bytes_escape, get_resource_filename
 from hippolyzer.lib.base.message.llsd_msg_serializer import LLSDMessageSerializer
 from hippolyzer.lib.base.message.message import Block
 from hippolyzer.lib.base.message.msgtypes import MsgType
@@ -44,11 +43,10 @@ from hippolyzer.lib.proxy.templates import CAP_TEMPLATES
 
 LOG = logging.getLogger(__name__)
 
-BASE_PATH = os.path.dirname(os.path.abspath(__file__))
-MAIN_WINDOW_UI_PATH = os.path.join(BASE_PATH, "proxy_mainwindow.ui")
-MESSAGE_BUILDER_UI_PATH = os.path.join(BASE_PATH, "message_builder.ui")
-ADDON_DIALOG_UI_PATH = os.path.join(BASE_PATH, "addon_dialog.ui")
-FILTER_DIALOG_UI_PATH = os.path.join(BASE_PATH, "filter_dialog.ui")
+MAIN_WINDOW_UI_PATH = get_resource_filename("apps/proxy_mainwindow.ui")
+MESSAGE_BUILDER_UI_PATH = get_resource_filename("apps/message_builder.ui")
+ADDON_DIALOG_UI_PATH = get_resource_filename("apps/addon_dialog.ui")
+FILTER_DIALOG_UI_PATH = get_resource_filename("apps/filter_dialog.ui")
 
 
 def show_error_message(error_msg, parent=None):
@@ -802,3 +800,8 @@ def gui_main():
         extra_addon_paths=window.getAddonList(),
         proxy_host=http_host,
     )
+
+
+if __name__ == "__main__":
+    multiprocessing.freeze_support()
+    gui_main()

hippolyzer/lib/base/helpers.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import functools
+import pkg_resources
 import re
 import weakref
 from pprint import PrettyPrinter
@@ -133,3 +134,7 @@ def bytes_unescape(val: bytes) -> bytes:
 def bytes_escape(val: bytes) -> bytes:
     # Try to keep newlines as-is
     return re.sub(rb"(?<!\\)\\n", b"\n", codecs.escape_encode(val)[0])  # type: ignore
+
+
+def get_resource_filename(resource_filename: str):
+    return pkg_resources.resource_filename("hippolyzer", resource_filename)

hippolyzer/lib/base/message/data/__init__.py

@@ -22,6 +22,8 @@ Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 import os
 
-msg_tmpl = open(os.path.join(os.path.dirname(__file__), 'message_template.msg'))
-with open(os.path.join(os.path.dirname(__file__), 'message.xml'), "rb") as _f:
+from hippolyzer.lib.base.helpers import get_resource_filename
+
+msg_tmpl = open(get_resource_filename("lib/base/message/data/message_template.msg"))
+with open(get_resource_filename("lib/base/message/data/message.xml"), "rb") as _f:
     msg_details = _f.read()

hippolyzer/lib/base/serialization.py

@@ -1703,7 +1703,7 @@ class BaseSubfieldSerializer(abc.ABC):
         """Guess at which template a val might correspond to"""
         if dataclasses.is_dataclass(val):
             val = dataclasses.asdict(val)  # noqa
-        if isinstance(val, bytes):
+        if isinstance(val, (bytes, bytearray)):
             template_checker = cls._template_sizes_match
         elif isinstance(val, dict):
             template_checker = cls._template_keys_match

hippolyzer/lib/proxy/http_proxy.py

@@ -5,7 +5,6 @@ import multiprocessing
 import os
 import re
 import sys
-import pkg_resources
 import queue
 import typing
 import uuid
@@ -20,6 +19,7 @@ from mitmproxy.addons import core, clientplayback
 from mitmproxy.http import HTTPFlow
 import OpenSSL
 
+from hippolyzer.lib.base.helpers import get_resource_filename
 from hippolyzer.lib.base.multiprocessing_utils import ParentProcessWatcher
 
 orig_sethostflags = OpenSSL.SSL._lib.X509_VERIFY_PARAM_set_hostflags  # noqa
@@ -230,7 +230,7 @@ def create_proxy_master(host, port, flow_context: HTTPFlowContext):  # pragma: n
         os.path.join(opts.confdir, "config.yml"),
     )
     # Use SL's CA bundle so LL's CA certs won't cause verification errors
-    ca_bundle = pkg_resources.resource_filename("hippolyzer.lib.base", "network/data/ca-bundle.crt")
+    ca_bundle = get_resource_filename("lib/base/network/data/ca-bundle.crt")
     opts.update(
         ssl_verify_upstream_trusted_ca=ca_bundle,
         listen_host=host,

hippolyzer/lib/proxy/lludp_proxy.py

@@ -129,13 +129,14 @@ class InterceptingLLUDPProxyProtocol(BaseLLUDPProxyProtocol):
             LOG.exception("Failed in region message handler")
 
         message_logger = self.session_manager.message_logger
-        if message_logger:
-            message_logger.log_lludp_message(self.session, region, message)
 
         handled = AddonManager.handle_lludp_message(
             self.session, region, message
         )
 
+        if message_logger:
+            message_logger.log_lludp_message(self.session, region, message)
+
         if handled:
             return
 

hippolyzer/lib/proxy/message_logger.py

@@ -16,11 +16,12 @@ from hippolyzer.lib.base import serialization as se, llsd
 from hippolyzer.lib.base.datatypes import TaggedUnion, UUID, TupleCoord
 from hippolyzer.lib.base.helpers import bytes_escape
 from hippolyzer.lib.proxy.message_filter import MetaFieldSpecifier, compile_filter, BaseFilterNode, MessageFilterNode
+from hippolyzer.lib.proxy.region import CapType
 
 if typing.TYPE_CHECKING:
     from hippolyzer.lib.proxy.http_flow import HippoHTTPFlow
     from hippolyzer.lib.proxy.message import ProxiedMessage
-    from hippolyzer.lib.proxy.region import ProxiedRegion, CapType
+    from hippolyzer.lib.proxy.region import ProxiedRegion
     from hippolyzer.lib.proxy.sessions import Session
 
 LOG = logging.getLogger(__name__)

hippolyzer/lib/proxy/sessions.py

@@ -12,9 +12,11 @@ from hippolyzer.lib.base.datatypes import UUID
 from hippolyzer.lib.proxy.circuit import ProxiedCircuit
 from hippolyzer.lib.proxy.http_asset_repo import HTTPAssetRepo
 from hippolyzer.lib.proxy.http_proxy import HTTPFlowContext, is_asset_server_cap_name, SerializedCapData
-from hippolyzer.lib.proxy.message_logger import BaseMessageLogger
 from hippolyzer.lib.proxy.region import ProxiedRegion, CapType
 
+if TYPE_CHECKING:
+    from hippolyzer.lib.proxy.message_logger import BaseMessageLogger
+
 
 class Session:
     def __init__(self, session_id, secure_session_id, agent_id, circuit_code,

hippolyzer/lib/proxy/vfs.py

@@ -1,10 +1,9 @@
 import dataclasses
 from typing import *
 
-import pkg_resources
-
 import hippolyzer.lib.base.serialization as se
 from hippolyzer.lib.base.datatypes import UUID
+from hippolyzer.lib.base.helpers import get_resource_filename
 from hippolyzer.lib.proxy.templates import AssetType
 
 
@@ -64,5 +63,5 @@ class VFS:
         return self._data_fh.read(block.size)
 
 
-_static_path = pkg_resources.resource_filename("hippolyzer.lib.proxy", "data/static_index.db2")
+_static_path = get_resource_filename("lib/proxy/data/static_index.db2")
 STATIC_VFS = VFS(_static_path)

setup.py

@@ -25,7 +25,7 @@ from setuptools import setup, find_packages
 
 here = path.abspath(path.dirname(__file__))
 
-version = '0.3'
+version = '0.3.2'
 
 with open(path.join(here, 'README.md')) as readme_fh:
     readme = readme_fh.read()

setup_cxfreeze.py

@@ -0,0 +1,121 @@
+import setuptools  # noqa
+
+import os
+import shutil
+from distutils.core import Command
+from pathlib import Path
+
+from cx_Freeze import setup, Executable
+
+# We don't need any of these and they make the archive huge.
+TO_DELETE = [
+    "lib/PySide2/Qt3DRender.pyd",
+    "lib/PySide2/Qt53DRender.dll",
+    "lib/PySide2/Qt5Charts.dll",
+    "lib/PySide2/Qt5Location.dll",
+    "lib/PySide2/Qt5Pdf.dll",
+    "lib/PySide2/Qt5Quick.dll",
+    "lib/PySide2/Qt5WebEngineCore.dll",
+    "lib/PySide2/QtCharts.pyd",
+    "lib/PySide2/QtMultimedia.pyd",
+    "lib/PySide2/QtOpenGLFunctions.pyd",
+    "lib/PySide2/QtOpenGLFunctions.pyi",
+    "lib/PySide2/d3dcompiler_47.dll",
+    "lib/PySide2/opengl32sw.dll",
+    "lib/PySide2/translations",
+    "lib/aiohttp/_find_header.c",
+    "lib/aiohttp/_frozenlist.c",
+    "lib/aiohttp/_helpers.c",
+    "lib/aiohttp/_http_parser.c",
+    "lib/aiohttp/_http_writer.c",
+    "lib/aiohttp/_websocket.c",
+    # Improve this to work with different versions.
+    "lib/aiohttp/python39.dll",
+    "lib/lazy_object_proxy/python39.dll",
+    "lib/lxml/python39.dll",
+    "lib/markupsafe/python39.dll",
+    "lib/multidict/python39.dll",
+    "lib/numpy/core/python39.dll",
+    "lib/numpy/fft/python39.dll",
+    "lib/numpy/linalg/python39.dll",
+    "lib/numpy/random/python39.dll",
+    "lib/python39.dll",
+    "lib/recordclass/python39.dll",
+    "lib/regex/python39.dll",
+    "lib/test",
+    "lib/yarl/python39.dll",
+]
+
+COPY_TO_ZIP = [
+    "LICENSE.txt",
+    "README.md",
+    "NOTICE.md",
+    # Must have been generated with pip-licenses before. Many dependencies
+    # require their license to be distributed with their binaries.
+    "lib_licenses.txt",
+]
+
+
+BASE_DIR = Path(__file__).parent.absolute()
+
+
+class FinalizeCXFreezeCommand(Command):
+    description = "Prepare cx_Freeze build dirs and create a zip"
+    user_options = []
+
+    def initialize_options(self) -> None:
+        pass
+
+    def finalize_options(self) -> None:
+        pass
+
+    def run(self):
+        (BASE_DIR / "dist").mkdir(exist_ok=True)
+        for path in (BASE_DIR / "build").iterdir():
+            if path.name.startswith("exe.") and path.is_dir():
+                for cleanse_suffix in TO_DELETE:
+                    cleanse_path = path / cleanse_suffix
+                    shutil.rmtree(cleanse_path, ignore_errors=True)
+                    try:
+                        os.unlink(cleanse_path)
+                    except:
+                        pass
+                for to_copy in COPY_TO_ZIP:
+                    shutil.copy(BASE_DIR / to_copy, path / to_copy)
+                zip_path = BASE_DIR / "dist" / path.name
+                shutil.make_archive(zip_path, "zip", path)
+
+
+options = {
+    "build_exe": {
+        "packages": [
+            "passlib",
+            "_cffi_backend",
+            "hippolyzer",
+        ],
+        # exclude packages that are not really needed
+        "excludes": [
+            "tkinter",
+        ],
+        "include_msvcr": True,
+    }
+}
+
+executables = [
+    Executable(
+        "hippolyzer/apps/proxy_gui.py",
+        base=None,
+        target_name="hippolyzer_gui"
+    ),
+]
+
+setup(
+    name="hippolyzer_gui",
+    version="0.3.2",
+    description="Hippolyzer GUI",
+    options=options,
+    executables=executables,
+    cmdclass={
+        "finalize_cxfreeze": FinalizeCXFreezeCommand,
+    }
+)

tests/base/test_mesh.py

@@ -1,16 +1,18 @@
-import pkg_resources
+import os
 import unittest
 
 from hippolyzer.lib.base.mesh import LLMeshSerializer, MeshAsset
 import hippolyzer.lib.base.serialization as se
 
+BASE_PATH = os.path.dirname(os.path.abspath(__file__))
+
 
 class TestMesh(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls) -> None:
         # Use a rigged cube SLM from the upload process as a test file
-        slm_file = pkg_resources.resource_filename(__name__, "test_resources/testslm.slm")
+        slm_file = os.path.join(BASE_PATH, "test_resources", "testslm.slm")
         with open(slm_file, "rb") as f:
             cls.slm_bytes = f.read()