From cf69c42f67e91de9eb1922830814142df9cf66a6 Mon Sep 17 00:00:00 2001 From: Salad Dais Date: Sat, 17 Jul 2021 15:09:17 +0000 Subject: [PATCH] Rework HTTP proxying code to work with mitmproxy 7.0.0 --- addon_examples/horror_animator.py | 2 +- addon_examples/local_mesh.py | 2 +- hippolyzer/apps/proxy.py | 3 +- hippolyzer/lib/proxy/http_asset_repo.py | 2 +- hippolyzer/lib/proxy/http_event_manager.py | 4 +- hippolyzer/lib/proxy/http_flow.py | 6 +-- hippolyzer/lib/proxy/http_proxy.py | 49 +++++++++++------ requirements.txt | 62 ++++++++++------------ setup.py | 2 +- tests/proxy/integration/test_http.py | 5 +- tests/proxy/test_message_filter.py | 2 +- 11 files changed, 75 insertions(+), 64 deletions(-) diff --git a/addon_examples/horror_animator.py b/addon_examples/horror_animator.py index 2938966..e0cff7f 100644 --- a/addon_examples/horror_animator.py +++ b/addon_examples/horror_animator.py @@ -105,7 +105,7 @@ class HorrorAnimatorAddon(BaseAddon): # send the response back immediately block = STATIC_VFS[orig_anim_id] anim_data = STATIC_VFS.read_block(block) - flow.response = mitmproxy.http.HTTPResponse.make( + flow.response = mitmproxy.http.Response.make( 200, _mutate_anim_bytes(anim_data), { diff --git a/addon_examples/local_mesh.py b/addon_examples/local_mesh.py index fba2480..a9e8852 100644 --- a/addon_examples/local_mesh.py +++ b/addon_examples/local_mesh.py @@ -201,7 +201,7 @@ class MeshUploadInterceptingAddon(BaseAddon): self.local_mesh_mapping = {x["mesh_name"]: x["mesh"] for x in instances} # Fake a response, we don't want to actually send off the request. - flow.response = mitmproxy.http.HTTPResponse.make( + flow.response = mitmproxy.http.Response.make( 200, b"", { diff --git a/hippolyzer/apps/proxy.py b/hippolyzer/apps/proxy.py index 8048cdc..daa9dac 100644 --- a/hippolyzer/apps/proxy.py +++ b/hippolyzer/apps/proxy.py @@ -89,7 +89,6 @@ def run_http_proxy_process(proxy_host, http_proxy_port, flow_context: HTTPFlowCo mitmproxy_master = create_http_proxy(proxy_host, http_proxy_port, flow_context) mitmproxy_master.start_server() gc.freeze() - flow_context.mitmproxy_ready.set() mitm_loop.run_forever() @@ -120,7 +119,7 @@ def start_proxy(session_manager: SessionManager, extra_addons: Optional[list] = if sys.argv[1] == "--setup-ca": try: mitmproxy_master = create_http_proxy(proxy_host, http_proxy_port, flow_context) - except mitmproxy.exceptions.ServerException: + except mitmproxy.exceptions.MitmproxyException: # Proxy already running, create the master so we don't try to bind to a port mitmproxy_master = create_proxy_master(proxy_host, http_proxy_port, flow_context) setup_ca(sys.argv[2], mitmproxy_master) diff --git a/hippolyzer/lib/proxy/http_asset_repo.py b/hippolyzer/lib/proxy/http_asset_repo.py index a6d0721..c6047b3 100644 --- a/hippolyzer/lib/proxy/http_asset_repo.py +++ b/hippolyzer/lib/proxy/http_asset_repo.py @@ -58,7 +58,7 @@ class HTTPAssetRepo(collections.UserDict): return False asset = self[asset_id] - flow.response = http.HTTPResponse.make( + flow.response = http.Response.make( content=asset.data, headers={ "Content-Type": "application/octet-stream", diff --git a/hippolyzer/lib/proxy/http_event_manager.py b/hippolyzer/lib/proxy/http_event_manager.py index 409ae87..1b6e6f8 100644 --- a/hippolyzer/lib/proxy/http_event_manager.py +++ b/hippolyzer/lib/proxy/http_event_manager.py @@ -120,7 +120,7 @@ class MITMProxyEventManager: if not flow.can_stream or self._asset_server_proxied: flow.request.url = redir_url else: - flow.response = mitmproxy.http.HTTPResponse.make( + flow.response = mitmproxy.http.Response.make( 307, # Can't provide explanation in the body because this results in failing Range requests under # mitmproxy that return garbage data. Chances are there's weird interactions @@ -166,7 +166,7 @@ class MITMProxyEventManager: if cap_data and cap_data.type == CapType.PROXY_ONLY: # A proxy addon was supposed to respond itself, but it didn't. if not flow.taken and not flow.response_injected: - flow.response = mitmproxy.http.HTTPResponse.make( + flow.response = mitmproxy.http.Response.make( 500, b"Proxy didn't handle proxy-only Cap correctly", { diff --git a/hippolyzer/lib/proxy/http_flow.py b/hippolyzer/lib/proxy/http_flow.py index 739bb88..7679a4f 100644 --- a/hippolyzer/lib/proxy/http_flow.py +++ b/hippolyzer/lib/proxy/http_flow.py @@ -30,11 +30,11 @@ class HippoHTTPFlow: meta.setdefault("from_browser", False) @property - def request(self) -> mitmproxy.http.HTTPRequest: + def request(self) -> mitmproxy.http.Request: return self.flow.request @property - def response(self) -> Optional[mitmproxy.http.HTTPResponse]: + def response(self) -> Optional[mitmproxy.http.Response]: return self.flow.response @property @@ -42,7 +42,7 @@ class HippoHTTPFlow: return self.flow.id @response.setter - def response(self, val: Optional[mitmproxy.http.HTTPResponse]): + def response(self, val: Optional[mitmproxy.http.Response]): self.flow.metadata["response_injected"] = True self.flow.response = val diff --git a/hippolyzer/lib/proxy/http_proxy.py b/hippolyzer/lib/proxy/http_proxy.py index 78aadd0..61a1334 100644 --- a/hippolyzer/lib/proxy/http_proxy.py +++ b/hippolyzer/lib/proxy/http_proxy.py @@ -15,7 +15,8 @@ import mitmproxy.log import mitmproxy.master import mitmproxy.options import mitmproxy.proxy -from mitmproxy.addons import core, clientplayback +from mitmproxy.addons import core, clientplayback, proxyserver, next_layer, disable_h2c +from mitmproxy.connection import ConnectionState from mitmproxy.http import HTTPFlow import OpenSSL @@ -41,14 +42,15 @@ OpenSSL.SSL._lib.X509_VERIFY_PARAM_set_hostflags = _sethostflags_wrapper # noqa class SLCertStore(mitmproxy.certs.CertStore): - def get_cert(self, commonname: typing.Optional[bytes], sans: typing.List[bytes], *args): - cert, privkey, chain = super().get_cert(commonname, sans, *args) - x509: OpenSSL.crypto.X509 = cert.x509 + def get_cert(self, commonname: typing.Optional[str], sans: typing.List[str], *args, **kwargs): + entry = super().get_cert(commonname, sans, *args, **kwargs) + cert, privkey, chain = entry.cert, entry.privatekey, entry.chain_file + x509 = cert.to_pyopenssl() for i in range(0, x509.get_extension_count()): ext = x509.get_extension(i) # This cert already has a subject key id, pass through. if ext.get_short_name() == b"subjectKeyIdentifier": - return cert, privkey, chain + return entry # Need to add a subject key ID onto this cert or the viewer will reject it. x509.add_extensions([ @@ -58,17 +60,23 @@ class SLCertStore(mitmproxy.certs.CertStore): uuid.uuid4().hex.encode("utf8"), ), ]) - x509.sign(privkey, "sha256") # type: ignore - return cert, privkey, chain + x509.sign(OpenSSL.crypto.PKey.from_cryptography_key(privkey), "sha256") # type: ignore + new_entry = mitmproxy.certs.CertStoreEntry( + mitmproxy.certs.Cert.from_pyopenssl(x509), privkey, chain + ) + self.certs[(commonname, tuple(sans))] = new_entry + self.expire_queue.pop(-1) + self.expire(new_entry) + return new_entry -class SLProxyConfig(mitmproxy.proxy.ProxyConfig): - def configure(self, options, updated) -> None: - super().configure(options, updated) +class SLTlsConfig(mitmproxy.addons.tlsconfig.TlsConfig): + def running(self): + super().running() old_cert_store = self.certstore # Replace the cert store with one that knows how to add # a subject key ID extension. - self.certstore = SLCertStore( # noqa + self.certstore = SLCertStore( default_privatekey=old_cert_store.default_privatekey, default_ca=old_cert_store.default_ca, default_chain_file=old_cert_store.default_chain_file, @@ -92,12 +100,13 @@ class IPCInterceptionAddon: flow which is merged in and resumed. """ def __init__(self, flow_context: HTTPFlowContext): + self.mitmproxy_ready = flow_context.mitmproxy_ready self.intercepted_flows: typing.Dict[str, HTTPFlow] = {} self.from_proxy_queue: multiprocessing.Queue = flow_context.from_proxy_queue self.to_proxy_queue: multiprocessing.Queue = flow_context.to_proxy_queue self.shutdown_signal: multiprocessing.Event = flow_context.shutdown_signal - def log(self, entry: mitmproxy.log.LogEntry): + def add_log(self, entry: mitmproxy.log.LogEntry): if entry.level == "debug": logging.debug(entry.msg) elif entry.level in ("alert", "info"): @@ -112,6 +121,8 @@ class IPCInterceptionAddon: def running(self): # register to pump the events or something here asyncio.create_task(self._pump_callbacks()) + # Tell the main process mitmproxy is ready to handle requests + self.mitmproxy_ready.set() async def _pump_callbacks(self): watcher = ParentProcessWatcher(self.shutdown_signal) @@ -126,6 +137,11 @@ class IPCInterceptionAddon: continue if event_type == "callback": orig_flow = self.intercepted_flows.pop(flow_id) + # HACK: Have to temporarily lie and say that the connection is closed so + # we can do a no-op assignment on server connection address inside `from_state()`. + # Will need an upstream fix. + if orig_flow.server_conn: + orig_flow.server_conn.state = ConnectionState.CLOSED orig_flow.set_state(flow_state) # Remove the taken flag from the flow if present, the flow by definition # isn't take()n anymore once it's been passed back to the proxy. @@ -213,7 +229,11 @@ class SLMITMMaster(mitmproxy.master.Master): self.addons.add( core.Core(), clientplayback.ClientPlayback(), - SLMITMAddon(flow_context) + disable_h2c.DisableH2C(), + proxyserver.Proxyserver(), + next_layer.NextLayer(), + SLTlsConfig(), + SLMITMAddon(flow_context), ) def start_server(self): @@ -242,9 +262,6 @@ def create_proxy_master(host, port, flow_context: HTTPFlowContext): # pragma: n def create_http_proxy(bind_host, port, flow_context: HTTPFlowContext): # pragma: no cover master = create_proxy_master(bind_host, port, flow_context) - pconf = SLProxyConfig(master.options) - server = mitmproxy.proxy.server.ProxyServer(pconf) - master.server = server return master diff --git a/requirements.txt b/requirements.txt index 75f9c48..826ba3a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,69 +1,65 @@ aiohttp==3.7.4.post0 appdirs==1.4.4 Arpeggio==1.10.2 -asgiref==3.3.4 +asgiref==3.4.1 async-timeout==3.0.1 -attrs==20.3.0 -black==21.4b2 +attrs==21.2.0 blinker==1.4 Brotli==1.0.9 -certifi==2020.12.5 -cffi==1.14.5 +certifi==2021.5.30 +cffi==1.14.6 chardet==4.0.0 -click==7.1.2 -cryptography==3.3.2 +charset-normalizer==2.0.3 +click==8.0.1 +cryptography==3.4.7 defusedxml==0.7.1 -Flask==1.1.2 +Flask==2.0.1 Glymur==0.9.3 h11==0.12.0 h2==4.0.0 hpack==4.0.0 hyperframe==6.0.1 idna==2.10 -itsdangerous==1.1.0 +itsdangerous==2.0.1 jedi==0.18.0 -Jinja2==2.11.3 +Jinja2==3.0.1 kaitaistruct==0.9 lazy-object-proxy==1.6.0 -ldap3==2.8.1 -llbase==1.2.10 +ldap3==2.9 +llbase==1.2.11 lxml==4.6.3 -MarkupSafe==1.1.1 -mitmproxy==6.0.2 +MarkupSafe==2.0.1 +mitmproxy==7.0.0 msgpack==1.0.2 multidict==5.1.0 -mypy-extensions==0.4.3 -numpy==1.20.2 +numpy==1.21.0 parso==0.8.2 passlib==1.7.4 -pathspec==0.8.1 -prompt-toolkit==3.0.18 -protobuf==3.14.0 -ptpython==3.0.17 +prompt-toolkit==3.0.19 +protobuf==3.17.3 +ptpython==3.0.19 publicsuffix2==2.20191221 pyasn1==0.4.8 pycparser==2.20 -Pygments==2.8.1 +Pygments==2.9.0 pyOpenSSL==20.0.1 pyparsing==2.4.7 pyperclip==1.8.2 PySide2==5.15.2 -qasync==0.15.0 +qasync==0.17.0 recordclass==0.14.3 -regex==2021.4.4 -requests==2.25.1 -ruamel.yaml==0.16.13 -ruamel.yaml.clib==0.2.2 +requests==2.26.0 +ruamel.yaml==0.17.10 +ruamel.yaml.clib==0.2.6 shiboken2==5.15.2 -six==1.15.0 -sortedcontainers==2.3.0 -toml==0.10.2 +six==1.16.0 +sortedcontainers==2.4.0 tornado==6.1 -typing-extensions==3.7.4.3 -urllib3==1.26.5 +typing-extensions==3.10.0.0 +urllib3==1.26.6 urwid==2.1.2 wcwidth==0.2.5 -Werkzeug==1.0.1 +Werkzeug==2.0.1 wsproto==1.0.0 yarl==1.6.3 -zstandard==0.14.1 +zstandard==0.15.2 diff --git a/setup.py b/setup.py index 0464e54..30f643d 100644 --- a/setup.py +++ b/setup.py @@ -88,7 +88,7 @@ setup( # requests breaks with newer idna 'idna<3,>=2.5', # 7.x will be a major change. - 'mitmproxy<7.0.0', + 'mitmproxy>=7.0,<8.0', # For REPLs 'ptpython<4.0', # JP2 codec diff --git a/tests/proxy/integration/test_http.py b/tests/proxy/integration/test_http.py index 06e4412..ee401b1 100644 --- a/tests/proxy/integration/test_http.py +++ b/tests/proxy/integration/test_http.py @@ -6,9 +6,8 @@ import multiprocessing from urllib.parse import urlparse import aioresponses -from mitmproxy.net import http from mitmproxy.test import tflow, tutils -from mitmproxy.http import HTTPFlow +from mitmproxy.http import HTTPFlow, Headers from yarl import URL from hippolyzer.apps.proxy import run_http_proxy_process @@ -88,7 +87,7 @@ class HTTPIntegrationTests(BaseProxyTest): fake_flow = tflow.tflow( req=tutils.treq(host="example.com", content=b'getZOffsets|'), resp=tutils.tresp( - headers=http.Headers(( + headers=Headers(( (b"X-SecondLife-Object-Name", b"#Firestorm LSL Bridge v99999"), (b"X-SecondLife-Owner-Key", str(self.session.agent_id).encode("utf8")), )), diff --git a/tests/proxy/test_message_filter.py b/tests/proxy/test_message_filter.py index 4ec2887..2509cff 100644 --- a/tests/proxy/test_message_filter.py +++ b/tests/proxy/test_message_filter.py @@ -24,7 +24,7 @@ OBJECT_UPDATE = b'\xc0\x00\x00\x00Q\x00\x0c\x00\x01\xea\x03\x00\x02\xe6\x03\x00\ b'\x88\x00"' -class MessageFilterTests(unittest.TestCase): +class MessageFilterTests(unittest.IsolatedAsyncioTestCase): def _filter_matches(self, filter_str, message): compiled = compile_filter(filter_str) return compiled.match(message)