Miko/Hippolyzer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rework HTTP proxying code to work with mitmproxy 7.0.0

Browse Source
This commit is contained in:
Salad Dais
2021-07-17 15:09:17 +00:00
parent be658b9026
commit cf69c42f67
11 changed files with 75 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
addon_examples/horror_animator.py
View File

@@ -105,7 +105,7 @@ class HorrorAnimatorAddon(BaseAddon):
# send the response back immediately
block = STATIC_VFS[orig_anim_id]
anim_data = STATIC_VFS.read_block(block)
flow.response = mitmproxy.http.HTTPResponse.make(
flow.response = mitmproxy.http.Response.make(
200,
_mutate_anim_bytes(anim_data),
{

2
addon_examples/local_mesh.py
View File

@@ -201,7 +201,7 @@ class MeshUploadInterceptingAddon(BaseAddon):
self.local_mesh_mapping = {x["mesh_name"]: x["mesh"] for x in instances}
# Fake a response, we don't want to actually send off the request.
flow.response = mitmproxy.http.HTTPResponse.make(
flow.response = mitmproxy.http.Response.make(
200,
b"",
{

3
hippolyzer/apps/proxy.py
View File

@@ -89,7 +89,6 @@ def run_http_proxy_process(proxy_host, http_proxy_port, flow_context: HTTPFlowCo
mitmproxy_master = create_http_proxy(proxy_host, http_proxy_port, flow_context)
mitmproxy_master.start_server()
gc.freeze()
flow_context.mitmproxy_ready.set()
mitm_loop.run_forever()
@@ -120,7 +119,7 @@ def start_proxy(session_manager: SessionManager, extra_addons: Optional[list] =
if sys.argv[1] == "--setup-ca":
try:
mitmproxy_master = create_http_proxy(proxy_host, http_proxy_port, flow_context)
except mitmproxy.exceptions.ServerException:
except mitmproxy.exceptions.MitmproxyException:
# Proxy already running, create the master so we don't try to bind to a port
mitmproxy_master = create_proxy_master(proxy_host, http_proxy_port, flow_context)
setup_ca(sys.argv[2], mitmproxy_master)

2
hippolyzer/lib/proxy/http_asset_repo.py
View File

@@ -58,7 +58,7 @@ class HTTPAssetRepo(collections.UserDict):
return False
asset = self[asset_id]
flow.response = http.HTTPResponse.make(
flow.response = http.Response.make(
content=asset.data,
headers={
"Content-Type": "application/octet-stream",

4
hippolyzer/lib/proxy/http_event_manager.py
View File

@@ -120,7 +120,7 @@ class MITMProxyEventManager:
if not flow.can_stream or self._asset_server_proxied:
flow.request.url = redir_url
else:
flow.response = mitmproxy.http.HTTPResponse.make(
flow.response = mitmproxy.http.Response.make(
307,
# Can't provide explanation in the body because this results in failing Range requests under
# mitmproxy that return garbage data. Chances are there's weird interactions
@@ -166,7 +166,7 @@ class MITMProxyEventManager:
if cap_data and cap_data.type == CapType.PROXY_ONLY:
# A proxy addon was supposed to respond itself, but it didn't.
if not flow.taken and not flow.response_injected:
flow.response = mitmproxy.http.HTTPResponse.make(
flow.response = mitmproxy.http.Response.make(
500,
b"Proxy didn't handle proxy-only Cap correctly",
{

6
hippolyzer/lib/proxy/http_flow.py
View File

@@ -30,11 +30,11 @@ class HippoHTTPFlow:
meta.setdefault("from_browser", False)
@property
def request(self) -> mitmproxy.http.HTTPRequest:
def request(self) -> mitmproxy.http.Request:
return self.flow.request
@property
def response(self) -> Optional[mitmproxy.http.HTTPResponse]:
def response(self) -> Optional[mitmproxy.http.Response]:
return self.flow.response
@property
@@ -42,7 +42,7 @@ class HippoHTTPFlow:
return self.flow.id
@response.setter
def response(self, val: Optional[mitmproxy.http.HTTPResponse]):
def response(self, val: Optional[mitmproxy.http.Response]):
self.flow.metadata["response_injected"] = True
self.flow.response = val

49
hippolyzer/lib/proxy/http_proxy.py
View File

@@ -15,7 +15,8 @@ import mitmproxy.log
import mitmproxy.master
import mitmproxy.options
import mitmproxy.proxy
from mitmproxy.addons import core, clientplayback
from mitmproxy.addons import core, clientplayback, proxyserver, next_layer, disable_h2c
from mitmproxy.connection import ConnectionState
from mitmproxy.http import HTTPFlow
import OpenSSL
@@ -41,14 +42,15 @@ OpenSSL.SSL._lib.X509_VERIFY_PARAM_set_hostflags = _sethostflags_wrapper # noqa
class SLCertStore(mitmproxy.certs.CertStore):
def get_cert(self, commonname: typing.Optional[bytes], sans: typing.List[bytes], *args):
cert, privkey, chain = super().get_cert(commonname, sans, *args)
x509: OpenSSL.crypto.X509 = cert.x509
def get_cert(self, commonname: typing.Optional[str], sans: typing.List[str], *args, **kwargs):
entry = super().get_cert(commonname, sans, *args, **kwargs)
cert, privkey, chain = entry.cert, entry.privatekey, entry.chain_file
x509 = cert.to_pyopenssl()
for i in range(0, x509.get_extension_count()):
ext = x509.get_extension(i)
# This cert already has a subject key id, pass through.
if ext.get_short_name() == b"subjectKeyIdentifier":
return cert, privkey, chain
return entry
# Need to add a subject key ID onto this cert or the viewer will reject it.
x509.add_extensions([
@@ -58,17 +60,23 @@ class SLCertStore(mitmproxy.certs.CertStore):
uuid.uuid4().hex.encode("utf8"),
),
])
x509.sign(privkey, "sha256") # type: ignore
return cert, privkey, chain
x509.sign(OpenSSL.crypto.PKey.from_cryptography_key(privkey), "sha256") # type: ignore
new_entry = mitmproxy.certs.CertStoreEntry(
mitmproxy.certs.Cert.from_pyopenssl(x509), privkey, chain
)
self.certs[(commonname, tuple(sans))] = new_entry
self.expire_queue.pop(-1)
self.expire(new_entry)
return new_entry
class SLProxyConfig(mitmproxy.proxy.ProxyConfig):
def configure(self, options, updated) -> None:
super().configure(options, updated)
class SLTlsConfig(mitmproxy.addons.tlsconfig.TlsConfig):
def running(self):
super().running()
old_cert_store = self.certstore
# Replace the cert store with one that knows how to add
# a subject key ID extension.
self.certstore = SLCertStore( # noqa
self.certstore = SLCertStore(
default_privatekey=old_cert_store.default_privatekey,
default_ca=old_cert_store.default_ca,
default_chain_file=old_cert_store.default_chain_file,
@@ -92,12 +100,13 @@ class IPCInterceptionAddon:
flow which is merged in and resumed.
"""
def __init__(self, flow_context: HTTPFlowContext):
self.mitmproxy_ready = flow_context.mitmproxy_ready
self.intercepted_flows: typing.Dict[str, HTTPFlow] = {}
self.from_proxy_queue: multiprocessing.Queue = flow_context.from_proxy_queue
self.to_proxy_queue: multiprocessing.Queue = flow_context.to_proxy_queue
self.shutdown_signal: multiprocessing.Event = flow_context.shutdown_signal
def log(self, entry: mitmproxy.log.LogEntry):
def add_log(self, entry: mitmproxy.log.LogEntry):
if entry.level == "debug":
logging.debug(entry.msg)
elif entry.level in ("alert", "info"):
@@ -112,6 +121,8 @@ class IPCInterceptionAddon:
def running(self):
# register to pump the events or something here
asyncio.create_task(self._pump_callbacks())
# Tell the main process mitmproxy is ready to handle requests
self.mitmproxy_ready.set()
async def _pump_callbacks(self):
watcher = ParentProcessWatcher(self.shutdown_signal)
@@ -126,6 +137,11 @@ class IPCInterceptionAddon:
continue
if event_type == "callback":
orig_flow = self.intercepted_flows.pop(flow_id)
# HACK: Have to temporarily lie and say that the connection is closed so
# we can do a no-op assignment on server connection address inside `from_state()`.
# Will need an upstream fix.
if orig_flow.server_conn:
orig_flow.server_conn.state = ConnectionState.CLOSED
orig_flow.set_state(flow_state)
# Remove the taken flag from the flow if present, the flow by definition
# isn't take()n anymore once it's been passed back to the proxy.
@@ -213,7 +229,11 @@ class SLMITMMaster(mitmproxy.master.Master):
self.addons.add(
core.Core(),
clientplayback.ClientPlayback(),
SLMITMAddon(flow_context)
disable_h2c.DisableH2C(),
proxyserver.Proxyserver(),
next_layer.NextLayer(),
SLTlsConfig(),
SLMITMAddon(flow_context),
)
def start_server(self):
@@ -242,9 +262,6 @@ def create_proxy_master(host, port, flow_context: HTTPFlowContext): # pragma: n
def create_http_proxy(bind_host, port, flow_context: HTTPFlowContext): # pragma: no cover
master = create_proxy_master(bind_host, port, flow_context)
pconf = SLProxyConfig(master.options)
server = mitmproxy.proxy.server.ProxyServer(pconf)
master.server = server
return master

62
requirements.txt
View File

@@ -1,69 +1,65 @@
aiohttp==3.7.4.post0
appdirs==1.4.4
Arpeggio==1.10.2
asgiref==3.3.4
asgiref==3.4.1
async-timeout==3.0.1
attrs==20.3.0
black==21.4b2
attrs==21.2.0
blinker==1.4
Brotli==1.0.9
certifi==2020.12.5
cffi==1.14.5
certifi==2021.5.30
cffi==1.14.6
chardet==4.0.0
click==7.1.2
cryptography==3.3.2
charset-normalizer==2.0.3
click==8.0.1
cryptography==3.4.7
defusedxml==0.7.1
Flask==1.1.2
Flask==2.0.1
Glymur==0.9.3
h11==0.12.0
h2==4.0.0
hpack==4.0.0
hyperframe==6.0.1
idna==2.10
itsdangerous==1.1.0
itsdangerous==2.0.1
jedi==0.18.0
Jinja2==2.11.3
Jinja2==3.0.1
kaitaistruct==0.9
lazy-object-proxy==1.6.0
ldap3==2.8.1
llbase==1.2.10
ldap3==2.9
llbase==1.2.11
lxml==4.6.3
MarkupSafe==1.1.1
mitmproxy==6.0.2
MarkupSafe==2.0.1
mitmproxy==7.0.0
msgpack==1.0.2
multidict==5.1.0
mypy-extensions==0.4.3
numpy==1.20.2
numpy==1.21.0
parso==0.8.2
passlib==1.7.4
pathspec==0.8.1
prompt-toolkit==3.0.18
protobuf==3.14.0
ptpython==3.0.17
prompt-toolkit==3.0.19
protobuf==3.17.3
ptpython==3.0.19
publicsuffix2==2.20191221
pyasn1==0.4.8
pycparser==2.20
Pygments==2.8.1
Pygments==2.9.0
pyOpenSSL==20.0.1
pyparsing==2.4.7
pyperclip==1.8.2
PySide2==5.15.2
qasync==0.15.0
qasync==0.17.0
recordclass==0.14.3
regex==2021.4.4
requests==2.25.1
ruamel.yaml==0.16.13
ruamel.yaml.clib==0.2.2
requests==2.26.0
ruamel.yaml==0.17.10
ruamel.yaml.clib==0.2.6
shiboken2==5.15.2
six==1.15.0
sortedcontainers==2.3.0
toml==0.10.2
six==1.16.0
sortedcontainers==2.4.0
tornado==6.1
typing-extensions==3.7.4.3
urllib3==1.26.5
typing-extensions==3.10.0.0
urllib3==1.26.6
urwid==2.1.2
wcwidth==0.2.5
Werkzeug==1.0.1
Werkzeug==2.0.1
wsproto==1.0.0
yarl==1.6.3
zstandard==0.14.1
zstandard==0.15.2

2
setup.py
View File

@@ -88,7 +88,7 @@ setup(
# requests breaks with newer idna
'idna<3,>=2.5',
# 7.x will be a major change.
'mitmproxy<7.0.0',
'mitmproxy>=7.0,<8.0',
# For REPLs
'ptpython<4.0',
# JP2 codec

5
tests/proxy/integration/test_http.py
View File

@@ -6,9 +6,8 @@ import multiprocessing
from urllib.parse import urlparse
import aioresponses
from mitmproxy.net import http
from mitmproxy.test import tflow, tutils
from mitmproxy.http import HTTPFlow
from mitmproxy.http import HTTPFlow, Headers
from yarl import URL
from hippolyzer.apps.proxy import run_http_proxy_process
@@ -88,7 +87,7 @@ class HTTPIntegrationTests(BaseProxyTest):
fake_flow = tflow.tflow(
req=tutils.treq(host="example.com", content=b'<llsd><string>getZOffsets|'),
resp=tutils.tresp(
headers=http.Headers((
headers=Headers((
(b"X-SecondLife-Object-Name", b"#Firestorm LSL Bridge v99999"),
(b"X-SecondLife-Owner-Key", str(self.session.agent_id).encode("utf8")),
)),

2
tests/proxy/test_message_filter.py
View File

@@ -24,7 +24,7 @@ OBJECT_UPDATE = b'\xc0\x00\x00\x00Q\x00\x0c\x00\x01\xea\x03\x00\x02\xe6\x03\x00\
b'\x88\x00"'
class MessageFilterTests(unittest.TestCase):
class MessageFilterTests(unittest.IsolatedAsyncioTestCase):
def _filter_matches(self, filter_str, message):
compiled = compile_filter(filter_str)
return compiled.match(message)