We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Handle `bytes` in request parameters
When POST data is encoded as UTF8, `har_dump.py` would bail with

```
TypeError: b'xxxxx' is not JSON serializable
```

(please excuse my poor python, feel free to reject and solve in some other canonical way)


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `examples/har_dump.py`
Content:
```
1 """
2 This inline script can be used to dump flows as HAR files.
3 """
4 
5 
6 import pprint
7 import json
8 import sys
9 import base64
10 import zlib
11 
12 from datetime import datetime
13 import pytz
14 
15 import mitmproxy
16 
17 from mitmproxy import version
18 from mitmproxy.utils import strutils
19 from mitmproxy.net.http import cookies
20 
21 HAR = {}
22 
23 # A list of server seen till now is maintained so we can avoid
24 # using 'connect' time for entries that use an existing connection.
25 SERVERS_SEEN = set()
26 
27 
28 def start():
29     """
30         Called once on script startup before any other events.
31     """
32     if len(sys.argv) != 2:
33         raise ValueError(
34             'Usage: -s "har_dump.py filename" '
35             '(- will output to stdout, filenames ending with .zhar '
36             'will result in compressed har)'
37         )
38 
39     HAR.update({
40         "log": {
41             "version": "1.2",
42             "creator": {
43                 "name": "mitmproxy har_dump",
44                 "version": "0.1",
45                 "comment": "mitmproxy version %s" % version.MITMPROXY
46             },
47             "entries": []
48         }
49     })
50 
51 
52 def response(flow):
53     """
54        Called when a server response has been received.
55     """
56 
57     # -1 indicates that these values do not apply to current request
58     ssl_time = -1
59     connect_time = -1
60 
61     if flow.server_conn and flow.server_conn not in SERVERS_SEEN:
62         connect_time = (flow.server_conn.timestamp_tcp_setup -
63                         flow.server_conn.timestamp_start)
64 
65         if flow.server_conn.timestamp_ssl_setup is not None:
66             ssl_time = (flow.server_conn.timestamp_ssl_setup -
67                         flow.server_conn.timestamp_tcp_setup)
68 
69         SERVERS_SEEN.add(flow.server_conn)
70 
71     # Calculate raw timings from timestamps. DNS timings can not be calculated
72     # for lack of a way to measure it. The same goes for HAR blocked.
73     # mitmproxy will open a server connection as soon as it receives the host
74     # and port from the client connection. So, the time spent waiting is actually
75     # spent waiting between request.timestamp_end and response.timestamp_start
76     # thus it correlates to HAR wait instead.
77     timings_raw = {
78         'send': flow.request.timestamp_end - flow.request.timestamp_start,
79         'receive': flow.response.timestamp_end - flow.response.timestamp_start,
80         'wait': flow.response.timestamp_start - flow.request.timestamp_end,
81         'connect': connect_time,
82         'ssl': ssl_time,
83     }
84 
85     # HAR timings are integers in ms, so we re-encode the raw timings to that format.
86     timings = dict([(k, int(1000 * v)) for k, v in timings_raw.items()])
87 
88     # full_time is the sum of all timings.
89     # Timings set to -1 will be ignored as per spec.
90     full_time = sum(v for v in timings.values() if v > -1)
91 
92     started_date_time = format_datetime(datetime.utcfromtimestamp(flow.request.timestamp_start))
93 
94     # Response body size and encoding
95     response_body_size = len(flow.response.raw_content)
96     response_body_decoded_size = len(flow.response.content)
97     response_body_compression = response_body_decoded_size - response_body_size
98 
99     entry = {
100         "startedDateTime": started_date_time,
101         "time": full_time,
102         "request": {
103             "method": flow.request.method,
104             "url": flow.request.url,
105             "httpVersion": flow.request.http_version,
106             "cookies": format_request_cookies(flow.request.cookies.fields),
107             "headers": name_value(flow.request.headers),
108             "queryString": name_value(flow.request.query or {}),
109             "headersSize": len(str(flow.request.headers)),
110             "bodySize": len(flow.request.content),
111         },
112         "response": {
113             "status": flow.response.status_code,
114             "statusText": flow.response.reason,
115             "httpVersion": flow.response.http_version,
116             "cookies": format_response_cookies(flow.response.cookies.fields),
117             "headers": name_value(flow.response.headers),
118             "content": {
119                 "size": response_body_size,
120                 "compression": response_body_compression,
121                 "mimeType": flow.response.headers.get('Content-Type', '')
122             },
123             "redirectURL": flow.response.headers.get('Location', ''),
124             "headersSize": len(str(flow.response.headers)),
125             "bodySize": response_body_size,
126         },
127         "cache": {},
128         "timings": timings,
129     }
130 
131     # Store binay data as base64
132     if strutils.is_mostly_bin(flow.response.content):
133         b64 = base64.b64encode(flow.response.content)
134         entry["response"]["content"]["text"] = b64.decode('ascii')
135         entry["response"]["content"]["encoding"] = "base64"
136     else:
137         entry["response"]["content"]["text"] = flow.response.text
138 
139     if flow.request.method in ["POST", "PUT", "PATCH"]:
140         entry["request"]["postData"] = {
141             "mimeType": flow.request.headers.get("Content-Type", "").split(";")[0],
142             "text": flow.request.content,
143             "params": name_value(flow.request.urlencoded_form)
144         }
145 
146     if flow.server_conn:
147         entry["serverIPAddress"] = str(flow.server_conn.ip_address.address[0])
148 
149     HAR["log"]["entries"].append(entry)
150 
151 
152 def done():
153     """
154         Called once on script shutdown, after any other events.
155     """
156     dump_file = sys.argv[1]
157 
158     if dump_file == '-':
159         mitmproxy.ctx.log(pprint.pformat(HAR))
160     else:
161         json_dump = json.dumps(HAR, indent=2)
162 
163         if dump_file.endswith('.zhar'):
164             json_dump = zlib.compress(json_dump, 9)
165 
166         with open(dump_file, "w") as f:
167             f.write(json_dump)
168 
169         mitmproxy.ctx.log("HAR dump finished (wrote %s bytes to file)" % len(json_dump))
170 
171 
172 def format_datetime(dt):
173     return dt.replace(tzinfo=pytz.timezone("UTC")).isoformat()
174 
175 
176 def format_cookies(cookie_list):
177     rv = []
178 
179     for name, value, attrs in cookie_list:
180         cookie_har = {
181             "name": name,
182             "value": value,
183         }
184 
185         # HAR only needs some attributes
186         for key in ["path", "domain", "comment"]:
187             if key in attrs:
188                 cookie_har[key] = attrs[key]
189 
190         # These keys need to be boolean!
191         for key in ["httpOnly", "secure"]:
192             cookie_har[key] = bool(key in attrs)
193 
194         # Expiration time needs to be formatted
195         expire_ts = cookies.get_expiration_ts(attrs)
196         if expire_ts is not None:
197             cookie_har["expires"] = format_datetime(datetime.fromtimestamp(expire_ts))
198 
199         rv.append(cookie_har)
200 
201     return rv
202 
203 
204 def format_request_cookies(fields):
205     return format_cookies(cookies.group_cookies(fields))
206 
207 
208 def format_response_cookies(fields):
209     return format_cookies((c[0], c[1].value, c[1].attrs) for c in fields)
210 
211 
212 def name_value(obj):
213     """
214         Convert (key, value) pairs to HAR format.
215     """
216     return [{"name": k, "value": v} for k, v in obj.items()]
217
```

Path: `pathod/language/generators.py`
Content:
```
1 import string
2 import random
3 import mmap
4 
5 import sys
6 
7 DATATYPES = dict(
8     ascii_letters=string.ascii_letters.encode(),
9     ascii_lowercase=string.ascii_lowercase.encode(),
10     ascii_uppercase=string.ascii_uppercase.encode(),
11     digits=string.digits.encode(),
12     hexdigits=string.hexdigits.encode(),
13     octdigits=string.octdigits.encode(),
14     punctuation=string.punctuation.encode(),
15     whitespace=string.whitespace.encode(),
16     ascii=string.printable.encode(),
17     bytes=bytes(bytearray(range(256)))
18 )
19 
20 
21 class TransformGenerator:
22 
23     """
24         Perform a byte-by-byte transform another generator - that is, for each
25         input byte, the transformation must produce one output byte.
26 
27         gen: A generator to wrap
28         transform: A function (offset, data) -> transformed
29     """
30 
31     def __init__(self, gen, transform):
32         self.gen = gen
33         self.transform = transform
34 
35     def __len__(self):
36         return len(self.gen)
37 
38     def __getitem__(self, x):
39         d = self.gen.__getitem__(x)
40         if isinstance(x, slice):
41             return self.transform(x.start, d)
42         return self.transform(x, d)
43 
44     def __repr__(self):
45         return "'transform(%s)'" % self.gen
46 
47 
48 def rand_byte(chars):
49     """
50         Return a random character as byte from a charset.
51     """
52     # bytearray has consistent behaviour on both Python 2 and 3
53     # while bytes does not
54     return bytes([random.choice(chars)])
55 
56 
57 class RandomGenerator:
58 
59     def __init__(self, dtype, length):
60         self.dtype = dtype
61         self.length = length
62 
63     def __len__(self):
64         return self.length
65 
66     def __getitem__(self, x):
67         chars = DATATYPES[self.dtype]
68         if isinstance(x, slice):
69             return b"".join(rand_byte(chars) for _ in range(*x.indices(min(self.length, sys.maxsize))))
70         return rand_byte(chars)
71 
72     def __repr__(self):
73         return "%s random from %s" % (self.length, self.dtype)
74 
75 
76 class FileGenerator:
77 
78     def __init__(self, path):
79         self.path = path
80         self.fp = open(path, "rb")
81         self.map = mmap.mmap(self.fp.fileno(), 0, access=mmap.ACCESS_READ)
82 
83     def __len__(self):
84         return len(self.map)
85 
86     def __getitem__(self, x):
87         if isinstance(x, slice):
88             return self.map.__getitem__(x)
89         # A slice of length 1 returns a byte object (not an integer)
90         return self.map.__getitem__(slice(x, x + 1 or self.map.size()))
91 
92     def __repr__(self):
93         return "<%s" % self.path
94
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/examples/har_dump.py b/examples/har_dump.py
--- a/examples/har_dump.py
+++ b/examples/har_dump.py
@@ -3,7 +3,6 @@
 """
 
 
-import pprint
 import json
 import sys
 import base64
@@ -128,19 +127,22 @@
         "timings": timings,
     }
 
-    # Store binay data as base64
+    # Store binary data as base64
     if strutils.is_mostly_bin(flow.response.content):
-        b64 = base64.b64encode(flow.response.content)
-        entry["response"]["content"]["text"] = b64.decode('ascii')
+        entry["response"]["content"]["text"] = base64.b64encode(flow.response.content).decode()
         entry["response"]["content"]["encoding"] = "base64"
     else:
-        entry["response"]["content"]["text"] = flow.response.text
+        entry["response"]["content"]["text"] = flow.response.get_text(strict=False)
 
     if flow.request.method in ["POST", "PUT", "PATCH"]:
+        params = [
+            {"name": a.decode("utf8", "surrogateescape"), "value": b.decode("utf8", "surrogateescape")}
+            for a, b in flow.request.urlencoded_form.items(multi=True)
+        ]
         entry["request"]["postData"] = {
             "mimeType": flow.request.headers.get("Content-Type", "").split(";")[0],
-            "text": flow.request.content,
-            "params": name_value(flow.request.urlencoded_form)
+            "text": flow.request.get_text(strict=False),
+            "params": params
         }
 
     if flow.server_conn:
@@ -155,16 +157,17 @@
     """
     dump_file = sys.argv[1]
 
+    json_dump = json.dumps(HAR, indent=2)  # type: str
+
     if dump_file == '-':
-        mitmproxy.ctx.log(pprint.pformat(HAR))
+        mitmproxy.ctx.log(json_dump)
     else:
-        json_dump = json.dumps(HAR, indent=2)
-
+        raw = json_dump.encode()  # type: bytes
         if dump_file.endswith('.zhar'):
-            json_dump = zlib.compress(json_dump, 9)
+            raw = zlib.compress(raw, 9)
 
-        with open(dump_file, "w") as f:
-            f.write(json_dump)
+        with open(dump_file, "wb") as f:
+            f.write(raw)
 
         mitmproxy.ctx.log("HAR dump finished (wrote %s bytes to file)" % len(json_dump))
 
diff --git a/pathod/language/generators.py b/pathod/language/generators.py
--- a/pathod/language/generators.py
+++ b/pathod/language/generators.py
@@ -14,7 +14,7 @@
     punctuation=string.punctuation.encode(),
     whitespace=string.whitespace.encode(),
     ascii=string.printable.encode(),
-    bytes=bytes(bytearray(range(256)))
+    bytes=bytes(range(256))
 )

{"golden_diff": "diff --git a/examples/har_dump.py b/examples/har_dump.py\n--- a/examples/har_dump.py\n+++ b/examples/har_dump.py\n@@ -3,7 +3,6 @@\n \"\"\"\n \n \n-import pprint\n import json\n import sys\n import base64\n@@ -128,19 +127,22 @@\n         \"timings\": timings,\n     }\n \n-    # Store binay data as base64\n+    # Store binary data as base64\n     if strutils.is_mostly_bin(flow.response.content):\n-        b64 = base64.b64encode(flow.response.content)\n-        entry[\"response\"][\"content\"][\"text\"] = b64.decode('ascii')\n+        entry[\"response\"][\"content\"][\"text\"] = base64.b64encode(flow.response.content).decode()\n         entry[\"response\"][\"content\"][\"encoding\"] = \"base64\"\n     else:\n-        entry[\"response\"][\"content\"][\"text\"] = flow.response.text\n+        entry[\"response\"][\"content\"][\"text\"] = flow.response.get_text(strict=False)\n \n     if flow.request.method in [\"POST\", \"PUT\", \"PATCH\"]:\n+        params = [\n+            {\"name\": a.decode(\"utf8\", \"surrogateescape\"), \"value\": b.decode(\"utf8\", \"surrogateescape\")}\n+            for a, b in flow.request.urlencoded_form.items(multi=True)\n+        ]\n         entry[\"request\"][\"postData\"] = {\n             \"mimeType\": flow.request.headers.get(\"Content-Type\", \"\").split(\";\")[0],\n-            \"text\": flow.request.content,\n-            \"params\": name_value(flow.request.urlencoded_form)\n+            \"text\": flow.request.get_text(strict=False),\n+            \"params\": params\n         }\n \n     if flow.server_conn:\n@@ -155,16 +157,17 @@\n     \"\"\"\n     dump_file = sys.argv[1]\n \n+    json_dump = json.dumps(HAR, indent=2)  # type: str\n+\n     if dump_file == '-':\n-        mitmproxy.ctx.log(pprint.pformat(HAR))\n+        mitmproxy.ctx.log(json_dump)\n     else:\n-        json_dump = json.dumps(HAR, indent=2)\n-\n+        raw = json_dump.encode()  # type: bytes\n         if dump_file.endswith('.zhar'):\n-            json_dump = zlib.compress(json_dump, 9)\n+            raw = zlib.compress(raw, 9)\n \n-        with open(dump_file, \"w\") as f:\n-            f.write(json_dump)\n+        with open(dump_file, \"wb\") as f:\n+            f.write(raw)\n \n         mitmproxy.ctx.log(\"HAR dump finished (wrote %s bytes to file)\" % len(json_dump))\n \ndiff --git a/pathod/language/generators.py b/pathod/language/generators.py\n--- a/pathod/language/generators.py\n+++ b/pathod/language/generators.py\n@@ -14,7 +14,7 @@\n     punctuation=string.punctuation.encode(),\n     whitespace=string.whitespace.encode(),\n     ascii=string.printable.encode(),\n-    bytes=bytes(bytearray(range(256)))\n+    bytes=bytes(range(256))\n )\n", "issue": "Handle `bytes` in request parameters\nWhen POST data is encoded as UTF8, `har_dump.py` would bail with\n\n```\nTypeError: b'xxxxx' is not JSON serializable\n```\n\n(please excuse my poor python, feel free to reject and solve in some other canonical way)\n\n", "before_files": [{"content": "\"\"\"\nThis inline script can be used to dump flows as HAR files.\n\"\"\"\n\n\nimport pprint\nimport json\nimport sys\nimport base64\nimport zlib\n\nfrom datetime import datetime\nimport pytz\n\nimport mitmproxy\n\nfrom mitmproxy import version\nfrom mitmproxy.utils import strutils\nfrom mitmproxy.net.http import cookies\n\nHAR = {}\n\n# A list of server seen till now is maintained so we can avoid\n# using 'connect' time for entries that use an existing connection.\nSERVERS_SEEN = set()\n\n\ndef start():\n    \"\"\"\n        Called once on script startup before any other events.\n    \"\"\"\n    if len(sys.argv) != 2:\n        raise ValueError(\n            'Usage: -s \"har_dump.py filename\" '\n            '(- will output to stdout, filenames ending with .zhar '\n            'will result in compressed har)'\n        )\n\n    HAR.update({\n        \"log\": {\n            \"version\": \"1.2\",\n            \"creator\": {\n                \"name\": \"mitmproxy har_dump\",\n                \"version\": \"0.1\",\n                \"comment\": \"mitmproxy version %s\" % version.MITMPROXY\n            },\n            \"entries\": []\n        }\n    })\n\n\ndef response(flow):\n    \"\"\"\n       Called when a server response has been received.\n    \"\"\"\n\n    # -1 indicates that these values do not apply to current request\n    ssl_time = -1\n    connect_time = -1\n\n    if flow.server_conn and flow.server_conn not in SERVERS_SEEN:\n        connect_time = (flow.server_conn.timestamp_tcp_setup -\n                        flow.server_conn.timestamp_start)\n\n        if flow.server_conn.timestamp_ssl_setup is not None:\n            ssl_time = (flow.server_conn.timestamp_ssl_setup -\n                        flow.server_conn.timestamp_tcp_setup)\n\n        SERVERS_SEEN.add(flow.server_conn)\n\n    # Calculate raw timings from timestamps. DNS timings can not be calculated\n    # for lack of a way to measure it. The same goes for HAR blocked.\n    # mitmproxy will open a server connection as soon as it receives the host\n    # and port from the client connection. So, the time spent waiting is actually\n    # spent waiting between request.timestamp_end and response.timestamp_start\n    # thus it correlates to HAR wait instead.\n    timings_raw = {\n        'send': flow.request.timestamp_end - flow.request.timestamp_start,\n        'receive': flow.response.timestamp_end - flow.response.timestamp_start,\n        'wait': flow.response.timestamp_start - flow.request.timestamp_end,\n        'connect': connect_time,\n        'ssl': ssl_time,\n    }\n\n    # HAR timings are integers in ms, so we re-encode the raw timings to that format.\n    timings = dict([(k, int(1000 * v)) for k, v in timings_raw.items()])\n\n    # full_time is the sum of all timings.\n    # Timings set to -1 will be ignored as per spec.\n    full_time = sum(v for v in timings.values() if v > -1)\n\n    started_date_time = format_datetime(datetime.utcfromtimestamp(flow.request.timestamp_start))\n\n    # Response body size and encoding\n    response_body_size = len(flow.response.raw_content)\n    response_body_decoded_size = len(flow.response.content)\n    response_body_compression = response_body_decoded_size - response_body_size\n\n    entry = {\n        \"startedDateTime\": started_date_time,\n        \"time\": full_time,\n        \"request\": {\n            \"method\": flow.request.method,\n            \"url\": flow.request.url,\n            \"httpVersion\": flow.request.http_version,\n            \"cookies\": format_request_cookies(flow.request.cookies.fields),\n            \"headers\": name_value(flow.request.headers),\n            \"queryString\": name_value(flow.request.query or {}),\n            \"headersSize\": len(str(flow.request.headers)),\n            \"bodySize\": len(flow.request.content),\n        },\n        \"response\": {\n            \"status\": flow.response.status_code,\n            \"statusText\": flow.response.reason,\n            \"httpVersion\": flow.response.http_version,\n            \"cookies\": format_response_cookies(flow.response.cookies.fields),\n            \"headers\": name_value(flow.response.headers),\n            \"content\": {\n                \"size\": response_body_size,\n                \"compression\": response_body_compression,\n                \"mimeType\": flow.response.headers.get('Content-Type', '')\n            },\n            \"redirectURL\": flow.response.headers.get('Location', ''),\n            \"headersSize\": len(str(flow.response.headers)),\n            \"bodySize\": response_body_size,\n        },\n        \"cache\": {},\n        \"timings\": timings,\n    }\n\n    # Store binay data as base64\n    if strutils.is_mostly_bin(flow.response.content):\n        b64 = base64.b64encode(flow.response.content)\n        entry[\"response\"][\"content\"][\"text\"] = b64.decode('ascii')\n        entry[\"response\"][\"content\"][\"encoding\"] = \"base64\"\n    else:\n        entry[\"response\"][\"content\"][\"text\"] = flow.response.text\n\n    if flow.request.method in [\"POST\", \"PUT\", \"PATCH\"]:\n        entry[\"request\"][\"postData\"] = {\n            \"mimeType\": flow.request.headers.get(\"Content-Type\", \"\").split(\";\")[0],\n            \"text\": flow.request.content,\n            \"params\": name_value(flow.request.urlencoded_form)\n        }\n\n    if flow.server_conn:\n        entry[\"serverIPAddress\"] = str(flow.server_conn.ip_address.address[0])\n\n    HAR[\"log\"][\"entries\"].append(entry)\n\n\ndef done():\n    \"\"\"\n        Called once on script shutdown, after any other events.\n    \"\"\"\n    dump_file = sys.argv[1]\n\n    if dump_file == '-':\n        mitmproxy.ctx.log(pprint.pformat(HAR))\n    else:\n        json_dump = json.dumps(HAR, indent=2)\n\n        if dump_file.endswith('.zhar'):\n            json_dump = zlib.compress(json_dump, 9)\n\n        with open(dump_file, \"w\") as f:\n            f.write(json_dump)\n\n        mitmproxy.ctx.log(\"HAR dump finished (wrote %s bytes to file)\" % len(json_dump))\n\n\ndef format_datetime(dt):\n    return dt.replace(tzinfo=pytz.timezone(\"UTC\")).isoformat()\n\n\ndef format_cookies(cookie_list):\n    rv = []\n\n    for name, value, attrs in cookie_list:\n        cookie_har = {\n            \"name\": name,\n            \"value\": value,\n        }\n\n        # HAR only needs some attributes\n        for key in [\"path\", \"domain\", \"comment\"]:\n            if key in attrs:\n                cookie_har[key] = attrs[key]\n\n        # These keys need to be boolean!\n        for key in [\"httpOnly\", \"secure\"]:\n            cookie_har[key] = bool(key in attrs)\n\n        # Expiration time needs to be formatted\n        expire_ts = cookies.get_expiration_ts(attrs)\n        if expire_ts is not None:\n            cookie_har[\"expires\"] = format_datetime(datetime.fromtimestamp(expire_ts))\n\n        rv.append(cookie_har)\n\n    return rv\n\n\ndef format_request_cookies(fields):\n    return format_cookies(cookies.group_cookies(fields))\n\n\ndef format_response_cookies(fields):\n    return format_cookies((c[0], c[1].value, c[1].attrs) for c in fields)\n\n\ndef name_value(obj):\n    \"\"\"\n        Convert (key, value) pairs to HAR format.\n    \"\"\"\n    return [{\"name\": k, \"value\": v} for k, v in obj.items()]\n", "path": "examples/har_dump.py"}, {"content": "import string\nimport random\nimport mmap\n\nimport sys\n\nDATATYPES = dict(\n    ascii_letters=string.ascii_letters.encode(),\n    ascii_lowercase=string.ascii_lowercase.encode(),\n    ascii_uppercase=string.ascii_uppercase.encode(),\n    digits=string.digits.encode(),\n    hexdigits=string.hexdigits.encode(),\n    octdigits=string.octdigits.encode(),\n    punctuation=string.punctuation.encode(),\n    whitespace=string.whitespace.encode(),\n    ascii=string.printable.encode(),\n    bytes=bytes(bytearray(range(256)))\n)\n\n\nclass TransformGenerator:\n\n    \"\"\"\n        Perform a byte-by-byte transform another generator - that is, for each\n        input byte, the transformation must produce one output byte.\n\n        gen: A generator to wrap\n        transform: A function (offset, data) -> transformed\n    \"\"\"\n\n    def __init__(self, gen, transform):\n        self.gen = gen\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.gen)\n\n    def __getitem__(self, x):\n        d = self.gen.__getitem__(x)\n        if isinstance(x, slice):\n            return self.transform(x.start, d)\n        return self.transform(x, d)\n\n    def __repr__(self):\n        return \"'transform(%s)'\" % self.gen\n\n\ndef rand_byte(chars):\n    \"\"\"\n        Return a random character as byte from a charset.\n    \"\"\"\n    # bytearray has consistent behaviour on both Python 2 and 3\n    # while bytes does not\n    return bytes([random.choice(chars)])\n\n\nclass RandomGenerator:\n\n    def __init__(self, dtype, length):\n        self.dtype = dtype\n        self.length = length\n\n    def __len__(self):\n        return self.length\n\n    def __getitem__(self, x):\n        chars = DATATYPES[self.dtype]\n        if isinstance(x, slice):\n            return b\"\".join(rand_byte(chars) for _ in range(*x.indices(min(self.length, sys.maxsize))))\n        return rand_byte(chars)\n\n    def __repr__(self):\n        return \"%s random from %s\" % (self.length, self.dtype)\n\n\nclass FileGenerator:\n\n    def __init__(self, path):\n        self.path = path\n        self.fp = open(path, \"rb\")\n        self.map = mmap.mmap(self.fp.fileno(), 0, access=mmap.ACCESS_READ)\n\n    def __len__(self):\n        return len(self.map)\n\n    def __getitem__(self, x):\n        if isinstance(x, slice):\n            return self.map.__getitem__(x)\n        # A slice of length 1 returns a byte object (not an integer)\n        return self.map.__getitem__(slice(x, x + 1 or self.map.size()))\n\n    def __repr__(self):\n        return \"<%s\" % self.path\n", "path": "pathod/language/generators.py"}], "after_files": [{"content": "\"\"\"\nThis inline script can be used to dump flows as HAR files.\n\"\"\"\n\n\nimport json\nimport sys\nimport base64\nimport zlib\n\nfrom datetime import datetime\nimport pytz\n\nimport mitmproxy\n\nfrom mitmproxy import version\nfrom mitmproxy.utils import strutils\nfrom mitmproxy.net.http import cookies\n\nHAR = {}\n\n# A list of server seen till now is maintained so we can avoid\n# using 'connect' time for entries that use an existing connection.\nSERVERS_SEEN = set()\n\n\ndef start():\n    \"\"\"\n        Called once on script startup before any other events.\n    \"\"\"\n    if len(sys.argv) != 2:\n        raise ValueError(\n            'Usage: -s \"har_dump.py filename\" '\n            '(- will output to stdout, filenames ending with .zhar '\n            'will result in compressed har)'\n        )\n\n    HAR.update({\n        \"log\": {\n            \"version\": \"1.2\",\n            \"creator\": {\n                \"name\": \"mitmproxy har_dump\",\n                \"version\": \"0.1\",\n                \"comment\": \"mitmproxy version %s\" % version.MITMPROXY\n            },\n            \"entries\": []\n        }\n    })\n\n\ndef response(flow):\n    \"\"\"\n       Called when a server response has been received.\n    \"\"\"\n\n    # -1 indicates that these values do not apply to current request\n    ssl_time = -1\n    connect_time = -1\n\n    if flow.server_conn and flow.server_conn not in SERVERS_SEEN:\n        connect_time = (flow.server_conn.timestamp_tcp_setup -\n                        flow.server_conn.timestamp_start)\n\n        if flow.server_conn.timestamp_ssl_setup is not None:\n            ssl_time = (flow.server_conn.timestamp_ssl_setup -\n                        flow.server_conn.timestamp_tcp_setup)\n\n        SERVERS_SEEN.add(flow.server_conn)\n\n    # Calculate raw timings from timestamps. DNS timings can not be calculated\n    # for lack of a way to measure it. The same goes for HAR blocked.\n    # mitmproxy will open a server connection as soon as it receives the host\n    # and port from the client connection. So, the time spent waiting is actually\n    # spent waiting between request.timestamp_end and response.timestamp_start\n    # thus it correlates to HAR wait instead.\n    timings_raw = {\n        'send': flow.request.timestamp_end - flow.request.timestamp_start,\n        'receive': flow.response.timestamp_end - flow.response.timestamp_start,\n        'wait': flow.response.timestamp_start - flow.request.timestamp_end,\n        'connect': connect_time,\n        'ssl': ssl_time,\n    }\n\n    # HAR timings are integers in ms, so we re-encode the raw timings to that format.\n    timings = dict([(k, int(1000 * v)) for k, v in timings_raw.items()])\n\n    # full_time is the sum of all timings.\n    # Timings set to -1 will be ignored as per spec.\n    full_time = sum(v for v in timings.values() if v > -1)\n\n    started_date_time = format_datetime(datetime.utcfromtimestamp(flow.request.timestamp_start))\n\n    # Response body size and encoding\n    response_body_size = len(flow.response.raw_content)\n    response_body_decoded_size = len(flow.response.content)\n    response_body_compression = response_body_decoded_size - response_body_size\n\n    entry = {\n        \"startedDateTime\": started_date_time,\n        \"time\": full_time,\n        \"request\": {\n            \"method\": flow.request.method,\n            \"url\": flow.request.url,\n            \"httpVersion\": flow.request.http_version,\n            \"cookies\": format_request_cookies(flow.request.cookies.fields),\n            \"headers\": name_value(flow.request.headers),\n            \"queryString\": name_value(flow.request.query or {}),\n            \"headersSize\": len(str(flow.request.headers)),\n            \"bodySize\": len(flow.request.content),\n        },\n        \"response\": {\n            \"status\": flow.response.status_code,\n            \"statusText\": flow.response.reason,\n            \"httpVersion\": flow.response.http_version,\n            \"cookies\": format_response_cookies(flow.response.cookies.fields),\n            \"headers\": name_value(flow.response.headers),\n            \"content\": {\n                \"size\": response_body_size,\n                \"compression\": response_body_compression,\n                \"mimeType\": flow.response.headers.get('Content-Type', '')\n            },\n            \"redirectURL\": flow.response.headers.get('Location', ''),\n            \"headersSize\": len(str(flow.response.headers)),\n            \"bodySize\": response_body_size,\n        },\n        \"cache\": {},\n        \"timings\": timings,\n    }\n\n    # Store binary data as base64\n    if strutils.is_mostly_bin(flow.response.content):\n        entry[\"response\"][\"content\"][\"text\"] = base64.b64encode(flow.response.content).decode()\n        entry[\"response\"][\"content\"][\"encoding\"] = \"base64\"\n    else:\n        entry[\"response\"][\"content\"][\"text\"] = flow.response.get_text(strict=False)\n\n    if flow.request.method in [\"POST\", \"PUT\", \"PATCH\"]:\n        params = [\n            {\"name\": a.decode(\"utf8\", \"surrogateescape\"), \"value\": b.decode(\"utf8\", \"surrogateescape\")}\n            for a, b in flow.request.urlencoded_form.items(multi=True)\n        ]\n        entry[\"request\"][\"postData\"] = {\n            \"mimeType\": flow.request.headers.get(\"Content-Type\", \"\").split(\";\")[0],\n            \"text\": flow.request.get_text(strict=False),\n            \"params\": params\n        }\n\n    if flow.server_conn:\n        entry[\"serverIPAddress\"] = str(flow.server_conn.ip_address.address[0])\n\n    HAR[\"log\"][\"entries\"].append(entry)\n\n\ndef done():\n    \"\"\"\n        Called once on script shutdown, after any other events.\n    \"\"\"\n    dump_file = sys.argv[1]\n\n    json_dump = json.dumps(HAR, indent=2)  # type: str\n\n    if dump_file == '-':\n        mitmproxy.ctx.log(json_dump)\n    else:\n        raw = json_dump.encode()  # type: bytes\n        if dump_file.endswith('.zhar'):\n            raw = zlib.compress(raw, 9)\n\n        with open(dump_file, \"wb\") as f:\n            f.write(raw)\n\n        mitmproxy.ctx.log(\"HAR dump finished (wrote %s bytes to file)\" % len(json_dump))\n\n\ndef format_datetime(dt):\n    return dt.replace(tzinfo=pytz.timezone(\"UTC\")).isoformat()\n\n\ndef format_cookies(cookie_list):\n    rv = []\n\n    for name, value, attrs in cookie_list:\n        cookie_har = {\n            \"name\": name,\n            \"value\": value,\n        }\n\n        # HAR only needs some attributes\n        for key in [\"path\", \"domain\", \"comment\"]:\n            if key in attrs:\n                cookie_har[key] = attrs[key]\n\n        # These keys need to be boolean!\n        for key in [\"httpOnly\", \"secure\"]:\n            cookie_har[key] = bool(key in attrs)\n\n        # Expiration time needs to be formatted\n        expire_ts = cookies.get_expiration_ts(attrs)\n        if expire_ts is not None:\n            cookie_har[\"expires\"] = format_datetime(datetime.fromtimestamp(expire_ts))\n\n        rv.append(cookie_har)\n\n    return rv\n\n\ndef format_request_cookies(fields):\n    return format_cookies(cookies.group_cookies(fields))\n\n\ndef format_response_cookies(fields):\n    return format_cookies((c[0], c[1].value, c[1].attrs) for c in fields)\n\n\ndef name_value(obj):\n    \"\"\"\n        Convert (key, value) pairs to HAR format.\n    \"\"\"\n    return [{\"name\": k, \"value\": v} for k, v in obj.items()]\n", "path": "examples/har_dump.py"}, {"content": "import string\nimport random\nimport mmap\n\nimport sys\n\nDATATYPES = dict(\n    ascii_letters=string.ascii_letters.encode(),\n    ascii_lowercase=string.ascii_lowercase.encode(),\n    ascii_uppercase=string.ascii_uppercase.encode(),\n    digits=string.digits.encode(),\n    hexdigits=string.hexdigits.encode(),\n    octdigits=string.octdigits.encode(),\n    punctuation=string.punctuation.encode(),\n    whitespace=string.whitespace.encode(),\n    ascii=string.printable.encode(),\n    bytes=bytes(range(256))\n)\n\n\nclass TransformGenerator:\n\n    \"\"\"\n        Perform a byte-by-byte transform another generator - that is, for each\n        input byte, the transformation must produce one output byte.\n\n        gen: A generator to wrap\n        transform: A function (offset, data) -> transformed\n    \"\"\"\n\n    def __init__(self, gen, transform):\n        self.gen = gen\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.gen)\n\n    def __getitem__(self, x):\n        d = self.gen.__getitem__(x)\n        if isinstance(x, slice):\n            return self.transform(x.start, d)\n        return self.transform(x, d)\n\n    def __repr__(self):\n        return \"'transform(%s)'\" % self.gen\n\n\ndef rand_byte(chars):\n    \"\"\"\n        Return a random character as byte from a charset.\n    \"\"\"\n    # bytearray has consistent behaviour on both Python 2 and 3\n    # while bytes does not\n    return bytes([random.choice(chars)])\n\n\nclass RandomGenerator:\n\n    def __init__(self, dtype, length):\n        self.dtype = dtype\n        self.length = length\n\n    def __len__(self):\n        return self.length\n\n    def __getitem__(self, x):\n        chars = DATATYPES[self.dtype]\n        if isinstance(x, slice):\n            return b\"\".join(rand_byte(chars) for _ in range(*x.indices(min(self.length, sys.maxsize))))\n        return rand_byte(chars)\n\n    def __repr__(self):\n        return \"%s random from %s\" % (self.length, self.dtype)\n\n\nclass FileGenerator:\n\n    def __init__(self, path):\n        self.path = path\n        self.fp = open(path, \"rb\")\n        self.map = mmap.mmap(self.fp.fileno(), 0, access=mmap.ACCESS_READ)\n\n    def __len__(self):\n        return len(self.map)\n\n    def __getitem__(self, x):\n        if isinstance(x, slice):\n            return self.map.__getitem__(x)\n        # A slice of length 1 returns a byte object (not an integer)\n        return self.map.__getitem__(slice(x, x + 1 or self.map.size()))\n\n    def __repr__(self):\n        return \"<%s\" % self.path\n", "path": "pathod/language/generators.py"}]}