From c42e0d72917296dbfc54002599ea996985e24b27 Mon Sep 17 00:00:00 2001 From: Salad Dais Date: Mon, 11 Dec 2023 18:35:56 +0000 Subject: [PATCH] Make client login testable --- .github/workflows/pytest.yml | 1 + hippolyzer/lib/base/test_utils.py | 41 +++++++++ hippolyzer/lib/client/hippo_client.py | 98 +++++++++++--------- hippolyzer/lib/proxy/test_utils.py | 22 +---- setup.py | 20 +++-- tests/base/test_xfer_transfer.py | 19 +--- tests/client/__init__.py | 0 tests/client/test_hippo_client.py | 125 ++++++++++++++++++++++++++ 8 files changed, 236 insertions(+), 90 deletions(-) create mode 100644 hippolyzer/lib/base/test_utils.py create mode 100644 tests/client/__init__.py create mode 100644 tests/client/test_hippo_client.py diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index fab0ac8..9d0d99a 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -24,6 +24,7 @@ jobs: pip install -r requirements.txt pip install -r requirements-test.txt sudo apt-get install libopenjp2-7 + pip install -e . - name: Run Flake8 run: | flake8 . diff --git a/hippolyzer/lib/base/test_utils.py b/hippolyzer/lib/base/test_utils.py new file mode 100644 index 0000000..c8f5c63 --- /dev/null +++ b/hippolyzer/lib/base/test_utils.py @@ -0,0 +1,41 @@ +import asyncio +from typing import Any, Optional, List, Tuple + +from hippolyzer.lib.base.message.circuit import Circuit, ConnectionHolder +from hippolyzer.lib.base.message.message import Message +from hippolyzer.lib.base.message.message_handler import MessageHandler +from hippolyzer.lib.base.network.transport import AbstractUDPTransport, ADDR_TUPLE, UDPPacket + + +class MockTransport(AbstractUDPTransport): + def sendto(self, data: Any, addr: Optional[ADDR_TUPLE] = ...) -> None: + pass + + def abort(self) -> None: + pass + + def close(self) -> None: + pass + + def __init__(self): + super().__init__() + self.packets: List[Tuple[bytes, Tuple[str, int]]] = [] + + def send_packet(self, packet: UDPPacket) -> None: + self.packets.append((packet.data, packet.dst_addr)) + + +class MockHandlingCircuit(Circuit): + def __init__(self, handler: MessageHandler[Message, str]): + super().__init__(("127.0.0.1", 1), ("127.0.0.1", 2), None) + self.handler = handler + + def _send_prepared_message(self, message: Message, transport=None): + loop = asyncio.get_event_loop_policy().get_event_loop() + loop.call_soon(self.handler.handle, message) + + +class MockConnectionHolder(ConnectionHolder): + def __init__(self, circuit, message_handler): + self.circuit = circuit + self.message_handler = message_handler diff --git a/hippolyzer/lib/client/hippo_client.py b/hippolyzer/lib/client/hippo_client.py index 50ebcc1..1b7bfbc 100644 --- a/hippolyzer/lib/client/hippo_client.py +++ b/hippolyzer/lib/client/hippo_client.py @@ -19,7 +19,7 @@ from hippolyzer.lib.base.message.message_dot_xml import MessageDotXML from hippolyzer.lib.base.message.message_handler import MessageHandler from hippolyzer.lib.base.message.udpdeserializer import UDPMessageDeserializer from hippolyzer.lib.base.network.caps_client import CapsClient -from hippolyzer.lib.base.network.transport import ADDR_TUPLE, Direction, SocketUDPTransport +from hippolyzer.lib.base.network.transport import ADDR_TUPLE, Direction, SocketUDPTransport, AbstractUDPTransport from hippolyzer.lib.base.settings import Settings from hippolyzer.lib.base.templates import RegionHandshakeReplyFlags from hippolyzer.lib.base.transfer_manager import TransferManager @@ -39,17 +39,11 @@ class HippoCapsClient(CapsClient): class HippoClientProtocol(asyncio.DatagramProtocol): def __init__(self, session: HippoClientSession): - self.session = session - self.transport: Optional[SocketUDPTransport] = None + self.session = proxify(session) self.message_xml = MessageDotXML() self.deserializer = UDPMessageDeserializer( settings=self.session.session_manager.settings, ) - loop = asyncio.get_event_loop_policy().get_event_loop() - self.resend_task = loop.create_task(self.attempt_resends()) - - def connection_made(self, transport: asyncio.DatagramTransport): - self.transport = SocketUDPTransport(transport) def datagram_received(self, data, source_addr: ADDR_TUPLE): region = self.session.region_by_circuit_addr(source_addr) @@ -69,28 +63,18 @@ class HippoClientProtocol(asyncio.DatagramProtocol): region.circuit.collect_acks(message) + # TODO: Ignore resends we already have, our ACK may have been missed + if message.reliable: + # This is a bit crap. We send an ACK immediately through a PacketAck. + # This is pretty wasteful, we should batch them up and send them on a timer. + region.circuit.send_acks((message.packet_id,)) + try: self.session.message_handler.handle(message) except: LOG.exception("Failed in region message handler") region.message_handler.handle(message) - async def attempt_resends(self): - while True: - await asyncio.sleep(0.1) - if self.session is None: - continue - for region in self.session.regions: - if not region.circuit or not region.circuit.is_alive: - continue - region.circuit.resend_unacked() - - def close(self): - logging.info("Closing UDP transport") - self.transport.close() - self.session = None - self.resend_task.cancel() - class HippoClientRegion(BaseClientRegion): def __init__(self, circuit_addr, seed_cap: str, session: HippoClientSession, handle=None): @@ -129,6 +113,7 @@ class HippoClientSession(BaseClientSession): super().__init__(id, secure_session_id, agent_id, circuit_code, session_manager, login_data=login_data) self.http_session = session_manager.http_session self.objects = ClientWorldObjectManager(proxify(self), session_manager.settings, None) + self.transport: Optional[SocketUDPTransport] = None self.protocol: Optional[HippoClientProtocol] = None def register_region(self, circuit_addr: Optional[ADDR_TUPLE] = None, seed_url: Optional[str] = None, @@ -140,7 +125,7 @@ class HippoClientSession(BaseClientSession): if region.circuit_addr == circuit_addr: valid_circuit = False if not region.circuit or not region.circuit.is_alive: - region.circuit = Circuit(("127.0.0.1", 0), circuit_addr, self.protocol.transport) + region.circuit = Circuit(("127.0.0.1", 0), circuit_addr, self.transport) region.circuit.is_alive = False valid_circuit = True if region.circuit and region.circuit.is_alive: @@ -310,15 +295,32 @@ class HippoClient(BaseClientSessionManager): self._password: Optional[str] = None self._mac = uuid.getnode() self._options = options if options is not None else self.DEFAULT_OPTIONS - self.http_session = aiohttp.ClientSession() + self.http_session: Optional[aiohttp.ClientSession] = aiohttp.ClientSession() self.session: Optional[HippoClientSession] = None self.settings = Settings() + self._resend_task: Optional[asyncio.Task] = None async def aclose(self): try: - await self.logout() + if self.session: + await self.logout() finally: - await self.http_session.close() + if self.http_session: + await self.http_session.close() + self.http_session = None + + def __del__(self): + # Make sure we don't leak resources if someone was lazy. + if self.http_session: + asyncio.get_event_loop_policy().get_event_loop().create_task(self.aclose()) + + async def _create_transport(self) -> Tuple[AbstractUDPTransport, HippoClientProtocol]: + loop = asyncio.get_event_loop_policy().get_event_loop() + transport, protocol = await loop.create_datagram_endpoint( + lambda: HippoClientProtocol(self.session), + local_addr=('0.0.0.0', 0)) + transport = SocketUDPTransport(transport) + return transport, protocol async def login( self, @@ -370,12 +372,9 @@ class HippoClient(BaseClientSessionManager): login_data = xmlrpc.client.loads((await resp.read()).decode("utf8"))[0][0] self.session = HippoClientSession.from_login_data(login_data, self) - loop = asyncio.get_event_loop_policy().get_event_loop() - transport, protocol = await loop.create_datagram_endpoint( - lambda: HippoClientProtocol(self.session), - local_addr=('0.0.0.0', 0)) - self.session.protocol = protocol - protocol.transport = SocketUDPTransport(transport) + self.session.transport, self.session.protocol = await self._create_transport() + self._resend_task = asyncio.create_task(self._attempt_resends()) + assert self.session.open_circuit(self.session.regions[-1].circuit_addr) region = self.session.regions[-1] self.session.main_region = region @@ -428,15 +427,28 @@ class HippoClient(BaseClientSessionManager): async def logout(self): if not self.session: return + if self._resend_task: + self._resend_task.cancel() + self._resend_task = None + session = self.session self.session = None - if not session.main_region or not session.main_region.is_alive: - # Nothing to do - return - # Don't need to send reliably, there's a good chance the server won't ACK anyway. - session.main_region.circuit.send( - Message( - "LogoutRequest", - Block("AgentData", AgentID=session.agent_id, SessionID=session.id), + if session.main_region and session.main_region.is_alive: + # Don't need to send reliably, there's a good chance the server won't ACK anyway. + session.main_region.circuit.send( + Message( + "LogoutRequest", + Block("AgentData", AgentID=session.agent_id, SessionID=session.id), + ) ) - ) + session.transport.close() + + async def _attempt_resends(self): + while True: + await asyncio.sleep(0.1) + if self.session is None: + continue + for region in self.session.regions: + if not region.circuit or not region.circuit.is_alive: + continue + region.circuit.resend_unacked() diff --git a/hippolyzer/lib/proxy/test_utils.py b/hippolyzer/lib/proxy/test_utils.py index 295a78d..87e2a5f 100644 --- a/hippolyzer/lib/proxy/test_utils.py +++ b/hippolyzer/lib/proxy/test_utils.py @@ -1,11 +1,11 @@ import asyncio import unittest -from typing import Any, Optional, List, Tuple from hippolyzer.lib.base.datatypes import UUID from hippolyzer.lib.base.message.message import Message from hippolyzer.lib.base.message.udpserializer import UDPMessageSerializer -from hippolyzer.lib.base.network.transport import UDPPacket, AbstractUDPTransport, ADDR_TUPLE +from hippolyzer.lib.base.network.transport import UDPPacket +from hippolyzer.lib.base.test_utils import MockTransport from hippolyzer.lib.proxy.lludp_proxy import InterceptingLLUDPProxyProtocol from hippolyzer.lib.proxy.region import ProxiedRegion from hippolyzer.lib.proxy.sessions import SessionManager @@ -63,21 +63,3 @@ class BaseProxyTest(unittest.IsolatedAsyncioTestCase): def _msg_to_datagram(self, msg: Message, src, dst, socks_header=True): packet = self._msg_to_packet(msg, src, dst) return SOCKS5UDPTransport.serialize(packet, force_socks_header=socks_header) - - -class MockTransport(AbstractUDPTransport): - def sendto(self, data: Any, addr: Optional[ADDR_TUPLE] = ...) -> None: - pass - - def abort(self) -> None: - pass - - def close(self) -> None: - pass - - def __init__(self): - super().__init__() - self.packets: List[Tuple[bytes, Tuple[str, int]]] = [] - - def send_packet(self, packet: UDPPacket) -> None: - self.packets.append((packet.data, packet.dst_addr)) diff --git a/setup.py b/setup.py index 48fe170..2af17d7 100644 --- a/setup.py +++ b/setup.py @@ -83,30 +83,32 @@ setup( python_requires='>=3.8', install_requires=[ 'llsd<1.1.0', - 'outleap<1.0', 'defusedxml', 'aiohttp<4.0.0', # Newer recordclasses break! 'recordclass>0.15,<0.18.3', 'lazy-object-proxy', - 'arpeggio', # requests breaks with newer idna 'idna<3,>=2.5', + # Needed for mesh format conversion tooling + 'pycollada', + 'transformations', + 'gltflib', + # JP2 codec + 'Glymur<0.9.7', + 'numpy<2.0', + + # Proxy-specific stuff + 'outleap<1.0', + 'arpeggio', # 7.x will be a major change. 'mitmproxy>=8.0.0,<8.1', 'Werkzeug<3.0', # For REPLs 'ptpython<4.0', - # JP2 codec - 'Glymur<0.9.7', - 'numpy<2.0', # These could be in extras_require if you don't want a GUI. 'pyside6-essentials', 'qasync', - # Needed for mesh format conversion tooling - 'pycollada', - 'transformations', - 'gltflib', ], tests_require=[ "pytest", diff --git a/tests/base/test_xfer_transfer.py b/tests/base/test_xfer_transfer.py index cebc2a2..cd9a14a 100644 --- a/tests/base/test_xfer_transfer.py +++ b/tests/base/test_xfer_transfer.py @@ -6,7 +6,6 @@ from typing import * from hippolyzer.lib.base.datatypes import UUID from hippolyzer.lib.base.message.message import Block, Message from hippolyzer.lib.base.message.message_handler import MessageHandler -from hippolyzer.lib.base.message.circuit import ConnectionHolder from hippolyzer.lib.base.templates import ( AssetType, EstateAssetType, @@ -16,26 +15,10 @@ from hippolyzer.lib.base.templates import ( TransferTargetType, TransferStatus, ) -from hippolyzer.lib.proxy.circuit import ProxiedCircuit from hippolyzer.lib.base.network.transport import Direction from hippolyzer.lib.base.transfer_manager import TransferManager, Transfer from hippolyzer.lib.base.xfer_manager import XferManager - - -class MockHandlingCircuit(ProxiedCircuit): - def __init__(self, handler: MessageHandler[Message, str]): - super().__init__(("127.0.0.1", 1), ("127.0.0.1", 2), None) - self.handler = handler - - def _send_prepared_message(self, message: Message, transport=None): - loop = asyncio.get_event_loop_policy().get_event_loop() - loop.call_soon(self.handler.handle, message) - - -class MockConnectionHolder(ConnectionHolder): - def __init__(self, circuit, message_handler): - self.circuit = circuit - self.message_handler = message_handler +from hippolyzer.lib.base.test_utils import MockHandlingCircuit, MockConnectionHolder class BaseTransferTests(unittest.IsolatedAsyncioTestCase): diff --git a/tests/client/__init__.py b/tests/client/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/client/test_hippo_client.py b/tests/client/test_hippo_client.py new file mode 100644 index 0000000..f281549 --- /dev/null +++ b/tests/client/test_hippo_client.py @@ -0,0 +1,125 @@ +import asyncio +import copy +import unittest +import xmlrpc.client +from typing import Tuple, Optional + +import aioresponses + +from hippolyzer.lib.base.datatypes import UUID +from hippolyzer.lib.base.message.circuit import Circuit +from hippolyzer.lib.base.message.message import Message, Block +from hippolyzer.lib.base.message.message_handler import MessageHandler +from hippolyzer.lib.base.message.udpdeserializer import UDPMessageDeserializer +from hippolyzer.lib.base.network.transport import AbstractUDPTransport, UDPPacket, Direction +from hippolyzer.lib.base.test_utils import MockTransport, MockConnectionHolder +from hippolyzer.lib.client.hippo_client import HippoClient, HippoClientProtocol + + +class MockServer(MockConnectionHolder): + def __init__(self, circuit, message_handler): + super().__init__(circuit, message_handler) + self.deserializer = UDPMessageDeserializer() + self.protocol: Optional[HippoClientProtocol] = None + + def process_inbound(self, packet: UDPPacket): + """Process a packet that the client sent to us""" + message = self.deserializer.deserialize(packet.data) + message.direction = Direction.IN + if message.reliable: + self.circuit.send_acks((message.packet_id,)) + self.circuit.collect_acks(message) + if message.name != "PacketAck": + self.message_handler.handle(message) + + +class PacketForwardingTransport(MockTransport): + def __init__(self): + super().__init__() + self.protocol: Optional[HippoClientProtocol] = None + + def send_packet(self, packet: UDPPacket): + super().send_packet(packet) + self.protocol.datagram_received(packet.data, packet.src_addr) + + +class MockServerTransport(MockTransport): + """Used for the client to send packets out""" + def __init__(self, server: MockServer): + super().__init__() + self._server = server + + def send_packet(self, packet: UDPPacket) -> None: + super().send_packet(packet) + # Directly pass the packet to the server + packet = copy.copy(packet) + packet.direction = Direction.IN + # Delay calling so the client can do its ACK bookkeeping first + asyncio.get_event_loop_policy().get_event_loop().call_soon(lambda: self._server.process_inbound(packet)) + + +class MockHippoClient(HippoClient): + def __init__(self, server: MockServer): + super().__init__() + self.server = server + + async def _create_transport(self) -> Tuple[AbstractUDPTransport, HippoClientProtocol]: + protocol = HippoClientProtocol(self.session) + # TODO: This isn't great, but whatever. + self.server.circuit.transport.protocol = protocol + return MockServerTransport(self.server), protocol + + +class TestHippoClient(unittest.IsolatedAsyncioTestCase): + FAKE_LOGIN_URI = "http://127.0.0.1:1/login.cgi" + FAKE_LOGIN_RESP = { + "session_id": str(UUID(int=1)), + "secure_session_id": str(UUID(int=2)), + "agent_id": str(UUID(int=3)), + "circuit_code": 123, + "sim_ip": "127.0.0.1", + "sim_port": 2, + "region_x": 0, + "region_y": 123, + "seed_capability": "https://127.0.0.1:4/foo", + } + + def setUp(self): + self.server_handler = MessageHandler() + self.server_transport = PacketForwardingTransport() + self.server_circuit = Circuit(("127.0.0.1", 2), ("127.0.0.1", 99), self.server_transport) + self.server = MockServer(self.server_circuit, self.server_handler) + + def _make_fake_login_body(self): + return xmlrpc.client.dumps((self.FAKE_LOGIN_RESP,), None, True) + + async def test_login(self): + client = MockHippoClient(self.server) + + async def _do_login(): + with aioresponses.aioresponses() as m: + m.post(self.FAKE_LOGIN_URI, body=self._make_fake_login_body()) + await client.login("foo", "bar", login_uri=self.FAKE_LOGIN_URI) + await client.logout() + + login_task = asyncio.create_task(_do_login()) + with self.server_handler.subscribe_async( + ("*",), + ) as get_msg: + async def _get_msg_soon(): + return await asyncio.wait_for(get_msg(), timeout=1.0) + + assert (await _get_msg_soon()).name == "UseCircuitCode" + assert (await _get_msg_soon()).name == "CompleteAgentMovement" + self.server.circuit.send(Message( + 'RegionHandshake', + Block('RegionInfo', fill_missing=True), + Block('RegionInfo2', fill_missing=True), + Block('RegionInfo3', fill_missing=True), + Block('RegionInfo4', fill_missing=True), + )) + assert (await _get_msg_soon()).name == "RegionHandshakeReply" + assert (await _get_msg_soon()).name == "AgentThrottle" + assert (await _get_msg_soon()).name == "LogoutRequest" + + await login_task