Support streamed request payloads (Fixes #26)

bin/mkrelease

@@ -1,33 +0,0 @@
-#!/bin/bash
-VERSION=$1
-if [[ "$VERSION" == "" ]]; then
-    echo Usage: $0 "<version>"
-    exit 1
-fi
-
-git diff --cached --exit-code >/dev/null
-if [[ "$?" != "0" ]]; then
-    echo Commit your changes before using this script.
-    exit 1
-fi
-
-set -e
-for PKG in microdot*; do
-    echo Building $PKG...
-    cd $PKG
-    sed -i "" "s/version.*$/version=\"$VERSION\",/" setup.py
-    git add setup.py
-    rm -rf dist
-    python setup.py sdist bdist_wheel --universal
-    cd ..
-done
-git commit -m "Release v$VERSION"
-git tag v$VERSION
-git push --tags origin master
-
-for PKG in microdot*; do
-    echo Releasing $PKG...
-    cd $PKG
-    twine upload dist/*
-    cd ..
-done

src/microdot.py

@@ -18,6 +18,11 @@ try:
 except ImportError:
     import errno
 
+try:
+    import uio as io
+except ImportError:
+    import io
+
 concurrency_mode = 'threaded'
 
 try:  # pragma: no cover
@@ -181,7 +186,8 @@ class Request():
     :var cookies: A dictionary with the cookies included in the request.
     :var content_length: The parsed ``Content-Length`` header.
     :var content_type: The parsed ``Content-Type`` header.
-    :var body: A stream from where the body can be read.
+    :var stream: The input stream, containing the request body.
+    :var body: The body of the request, as bytes.
     :var json: The parsed JSON body, as a dictionary or list, or ``None`` if
                the request does not have a JSON body.
     :var form: The parsed form submission body, as a :class:`MultiDict` object,
@@ -198,6 +204,17 @@ class Request():
     #:    Request.max_content_length = 1 * 1024 * 1024  # 1MB requests allowed
     max_content_length = 16 * 1024
 
+    #: Specify the maximum payload size that can be stored in ``body``.
+    #: Requests with payloads that are larger than this size and up to
+    #: ``max_content_length`` bytes will be accepted, but the application will
+    #: only be able to access the body of the request by reading from
+    #: ``stream``. Set to 0 if you always access the body as a stream.
+    #:
+    #: Example::
+    #:
+    #:    Request.max_body_length = 4 * 1024  # up to 4KB bodies read
+    max_body_length = 16 * 1024
+
     #: Specify the maximum length allowed for a line in the request. Requests
     #: with longer lines will not be correctly interpreted. Applications can
     #: change this maximum as necessary.
@@ -211,7 +228,7 @@ class Request():
         pass
 
     def __init__(self, app, client_addr, method, url, http_version, headers,
-                 body):
+                 body, stream):
         self.app = app
         self.client_addr = client_addr
         self.method = method
@@ -238,6 +255,7 @@ class Request():
                     name, value = cookie.strip().split('=', 1)
                     self.cookies[name] = value
         self.body = body
+        self.stream = stream
         self._json = None
         self._form = None
         self.g = Request.G()
@@ -275,15 +293,18 @@ class Request():
 
         # body
         body = b''
-        if content_length and content_length <= Request.max_content_length:
+        if content_length and content_length <= Request.max_body_length:
             while len(body) < content_length:
                 data = client_stream.read(content_length - len(body))
                 if len(data) == 0:  # pragma: no cover
                     raise EOFError()
                 body += data
+            stream = io.BytesIO(body)
+        else:
+            stream = client_stream
 
         return Request(app, client_addr, method, url, http_version, headers,
-                       body)
+                       body, stream)
 
     def _parse_urlencoded(self, urlencoded):
         data = MultiDict()

src/microdot_asyncio.py

@@ -10,6 +10,12 @@ try:
     import uasyncio as asyncio
 except ImportError:
     import asyncio
+
+try:
+    import uio as io
+except ImportError:
+    import io
+
 from microdot import Microdot as BaseMicrodot
 from microdot import print_exception
 from microdot import Request as BaseRequest
@@ -20,6 +26,23 @@ def _iscoroutine(coro):
     return hasattr(coro, 'send') and hasattr(coro, 'throw')
 
 
+class _AsyncBytesIO:
+    def __init__(self, data):
+        self.stream = io.BytesIO(data)
+
+    async def read(self, n=-1):
+        return self.stream.read(n)
+
+    async def readline(self):  # pragma: no cover
+        return self.stream.readline()
+
+    async def readexactly(self, n):  # pragma: no cover
+        return self.stream.read(n)
+
+    async def readuntil(self, separator=b'\n'):  # pragma: no cover
+        return self.stream.readuntil(separator=separator)
+
+
 class Request(BaseRequest):
     @staticmethod
     async def create(app, client_stream, client_addr):
@@ -55,12 +78,15 @@ class Request(BaseRequest):
                 content_length = int(value)
 
         # body
-        body = await client_stream.readexactly(content_length) \
+        body = b''
-            if content_length and \
+        if content_length and content_length <= Request.max_body_length:
-            content_length <= Request.max_content_length else b''
+            body = await client_stream.readexactly(content_length)
+            stream = _AsyncBytesIO(body)
+        else:
+            stream = client_stream
 
         return Request(app, client_addr, method, url, http_version, headers,
-                       body)
+                       body, stream)
 
     @staticmethod
     async def _safe_readline(stream):

tests/microdot/test_request.py

@@ -91,14 +91,26 @@ class TestRequest(unittest.TestCase):
 
         Request.max_readline = saved_max_readline
 
+    def test_stream(self):
+        fd = get_request_fd('GET', '/foo', headers={
+            'Content-Type': 'application/x-www-form-urlencoded'},
+            body='foo=bar&abc=def&x=y')
+        req = Request.create('app', fd, 'addr')
+        self.assertEqual(req.body, b'foo=bar&abc=def&x=y')
+        self.assertEqual(req.stream.read(), b'foo=bar&abc=def&x=y')
+
     def test_large_payload(self):
         saved_max_content_length = Request.max_content_length
-        Request.max_content_length = 16
+        saved_max_body_length = Request.max_body_length
+        Request.max_content_length = 32
+        Request.max_body_length = 16
 
         fd = get_request_fd('GET', '/foo', headers={
             'Content-Type': 'application/x-www-form-urlencoded'},
             body='foo=bar&abc=def&x=y')
         req = Request.create('app', fd, 'addr')
         self.assertEqual(req.body, b'')
+        self.assertEqual(req.stream.read(), b'foo=bar&abc=def&x=y')
 
         Request.max_content_length = saved_max_content_length
+        Request.max_body_length = saved_max_body_length

tests/microdot_asyncio/test_request_asyncio.py

@@ -101,14 +101,30 @@ class TestRequestAsync(unittest.TestCase):
 
         Request.max_readline = saved_max_readline
 
+    def test_stream(self):
+        fd = get_async_request_fd('GET', '/foo', headers={
+            'Content-Type': 'application/x-www-form-urlencoded',
+            'Content-Length': '19'},
+            body='foo=bar&abc=def&x=y')
+        req = _run(Request.create('app', fd, 'addr'))
+        self.assertEqual(req.body, b'foo=bar&abc=def&x=y')
+        data = _run(req.stream.read())
+        self.assertEqual(data, b'foo=bar&abc=def&x=y')
+
     def test_large_payload(self):
         saved_max_content_length = Request.max_content_length
-        Request.max_content_length = 16
+        saved_max_body_length = Request.max_body_length
+        Request.max_content_length = 32
+        Request.max_body_length = 16
 
         fd = get_async_request_fd('GET', '/foo', headers={
-            'Content-Type': 'application/x-www-form-urlencoded'},
+            'Content-Type': 'application/x-www-form-urlencoded',
+            'Content-Length': '19'},
             body='foo=bar&abc=def&x=y')
         req = _run(Request.create('app', fd, 'addr'))
         self.assertEqual(req.body, b'')
+        data = _run(req.stream.read())
+        self.assertEqual(data, b'foo=bar&abc=def&x=y')
 
         Request.max_content_length = saved_max_content_length
+        Request.max_body_length = saved_max_body_length

tests/mock_socket.py

@@ -56,7 +56,7 @@ class FakeStreamAsync:
     async def readline(self):
         return self.stream.readline()
 
-    async def read(self, n):
+    async def read(self, n=-1):
         return self.stream.read(n)
 
     async def readexactly(self, n):