Prevent reading past EOF in multipart parser (Fixes #307) (#309)

src/microdot/multipart.py

@@ -31,7 +31,10 @@ class FormDataIter:
     the next iteration, as the internal stream stored in ``FileUpload``
     instances is invalidated at the end of the iteration.
     """
-    #: The size of the buffer used to read chunks of the request body.
+    #: The size of the buffer used to read chunks of the request body. This
+    #: size must be large enough to hold at least one complete header or
+    #: boundary line, so it is not recommended to lower it, but it can be made
+    #: higher to improve performance at the expense of RAM.
     buffer_size = 256
 
     def __init__(self, request):
@@ -59,6 +62,7 @@ class FormDataIter:
             pass
 
         # make sure we are at a boundary
+        await self._fill_buffer()
         s = self.buffer.split(self.boundary, 1)
         if len(s) != 2 or s[0] != b'':
             abort(400)  # pragma: no cover
@@ -111,6 +115,9 @@ class FormDataIter:
         return name, FileUpload(filename, content_type, self._read_buffer)
 
     async def _fill_buffer(self):
+        if self.buffer[-len(self.boundary) - 4:] == self.boundary + b'--\r\n':
+            # we have reached the end of the body
+            return
         self.buffer += await self.request.stream.read(
             self.buffer_size + self.extra_size - len(self.buffer))
 

tests/test_multipart.py

@@ -99,6 +99,37 @@ class TestMultipart(unittest.TestCase):
                                     'g': 'g|text/html|<p>hello</p>'})
         FileUpload.max_memory_size = saved_max_memory_size
 
+    def test_large_file_upload(self):
+        saved_buffer_size = FormDataIter.buffer_size
+        FormDataIter.buffer_size = 100
+        saved_max_memory_size = FileUpload.max_memory_size
+        FileUpload.max_memory_size = 200
+
+        app = Microdot()
+
+        @app.post('/')
+        @with_form_data
+        async def index(req):
+            return {"len": len(await req.files['f'].read())}
+
+        client = TestClient(app)
+
+        res = self._run(client.post(
+            '/', headers={
+                'Content-Type': 'multipart/form-data; boundary=boundary',
+            },
+            body=(
+                b'--boundary\r\n'
+                b'Content-Disposition: form-data; name="f"; filename="f"\r\n'
+                b'Content-Type: text/plain\r\n\r\n' + b'*' * 398 + b'\r\n'
+                b'--boundary--\r\n')
+        ))
+        self.assertEqual(res.status_code, 200)
+        self.assertEqual(res.json, {'len': 398})
+
+        FormDataIter.buffer_size = saved_buffer_size
+        FileUpload.max_memory_size = saved_max_memory_size
+
     def test_file_save(self):
         app = Microdot()