diff --git a/bin/mkrelease b/bin/mkrelease deleted file mode 100755 index b72b98e..0000000 --- a/bin/mkrelease +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -VERSION=$1 -if [[ "$VERSION" == "" ]]; then - echo Usage: $0 "" - exit 1 -fi - -git diff --cached --exit-code >/dev/null -if [[ "$?" != "0" ]]; then - echo Commit your changes before using this script. - exit 1 -fi - -set -e -for PKG in microdot*; do - echo Building $PKG... - cd $PKG - sed -i "" "s/version.*$/version=\"$VERSION\",/" setup.py - git add setup.py - rm -rf dist - python setup.py sdist bdist_wheel --universal - cd .. -done -git commit -m "Release v$VERSION" -git tag v$VERSION -git push --tags origin master - -for PKG in microdot*; do - echo Releasing $PKG... - cd $PKG - twine upload dist/* - cd .. -done diff --git a/src/microdot.py b/src/microdot.py index e6d5449..5733de6 100644 --- a/src/microdot.py +++ b/src/microdot.py @@ -18,6 +18,11 @@ try: except ImportError: import errno +try: + import uio as io +except ImportError: + import io + concurrency_mode = 'threaded' try: # pragma: no cover @@ -181,7 +186,8 @@ class Request(): :var cookies: A dictionary with the cookies included in the request. :var content_length: The parsed ``Content-Length`` header. :var content_type: The parsed ``Content-Type`` header. - :var body: A stream from where the body can be read. + :var stream: The input stream, containing the request body. + :var body: The body of the request, as bytes. :var json: The parsed JSON body, as a dictionary or list, or ``None`` if the request does not have a JSON body. :var form: The parsed form submission body, as a :class:`MultiDict` object, @@ -198,6 +204,17 @@ class Request(): #: Request.max_content_length = 1 * 1024 * 1024 # 1MB requests allowed max_content_length = 16 * 1024 + #: Specify the maximum payload size that can be stored in ``body``. + #: Requests with payloads that are larger than this size and up to + #: ``max_content_length`` bytes will be accepted, but the application will + #: only be able to access the body of the request by reading from + #: ``stream``. Set to 0 if you always access the body as a stream. + #: + #: Example:: + #: + #: Request.max_body_length = 4 * 1024 # up to 4KB bodies read + max_body_length = 16 * 1024 + #: Specify the maximum length allowed for a line in the request. Requests #: with longer lines will not be correctly interpreted. Applications can #: change this maximum as necessary. @@ -211,7 +228,7 @@ class Request(): pass def __init__(self, app, client_addr, method, url, http_version, headers, - body): + body, stream): self.app = app self.client_addr = client_addr self.method = method @@ -238,6 +255,7 @@ class Request(): name, value = cookie.strip().split('=', 1) self.cookies[name] = value self.body = body + self.stream = stream self._json = None self._form = None self.g = Request.G() @@ -275,15 +293,18 @@ class Request(): # body body = b'' - if content_length and content_length <= Request.max_content_length: + if content_length and content_length <= Request.max_body_length: while len(body) < content_length: data = client_stream.read(content_length - len(body)) if len(data) == 0: # pragma: no cover raise EOFError() body += data + stream = io.BytesIO(body) + else: + stream = client_stream return Request(app, client_addr, method, url, http_version, headers, - body) + body, stream) def _parse_urlencoded(self, urlencoded): data = MultiDict() diff --git a/src/microdot_asyncio.py b/src/microdot_asyncio.py index 6eb28ee..ea51e19 100644 --- a/src/microdot_asyncio.py +++ b/src/microdot_asyncio.py @@ -10,6 +10,12 @@ try: import uasyncio as asyncio except ImportError: import asyncio + +try: + import uio as io +except ImportError: + import io + from microdot import Microdot as BaseMicrodot from microdot import print_exception from microdot import Request as BaseRequest @@ -20,6 +26,23 @@ def _iscoroutine(coro): return hasattr(coro, 'send') and hasattr(coro, 'throw') +class _AsyncBytesIO: + def __init__(self, data): + self.stream = io.BytesIO(data) + + async def read(self, n=-1): + return self.stream.read(n) + + async def readline(self): # pragma: no cover + return self.stream.readline() + + async def readexactly(self, n): # pragma: no cover + return self.stream.read(n) + + async def readuntil(self, separator=b'\n'): # pragma: no cover + return self.stream.readuntil(separator=separator) + + class Request(BaseRequest): @staticmethod async def create(app, client_stream, client_addr): @@ -55,12 +78,15 @@ class Request(BaseRequest): content_length = int(value) # body - body = await client_stream.readexactly(content_length) \ - if content_length and \ - content_length <= Request.max_content_length else b'' + body = b'' + if content_length and content_length <= Request.max_body_length: + body = await client_stream.readexactly(content_length) + stream = _AsyncBytesIO(body) + else: + stream = client_stream return Request(app, client_addr, method, url, http_version, headers, - body) + body, stream) @staticmethod async def _safe_readline(stream): diff --git a/tests/microdot/test_request.py b/tests/microdot/test_request.py index 2aef941..ef1e7b6 100644 --- a/tests/microdot/test_request.py +++ b/tests/microdot/test_request.py @@ -91,14 +91,26 @@ class TestRequest(unittest.TestCase): Request.max_readline = saved_max_readline + def test_stream(self): + fd = get_request_fd('GET', '/foo', headers={ + 'Content-Type': 'application/x-www-form-urlencoded'}, + body='foo=bar&abc=def&x=y') + req = Request.create('app', fd, 'addr') + self.assertEqual(req.body, b'foo=bar&abc=def&x=y') + self.assertEqual(req.stream.read(), b'foo=bar&abc=def&x=y') + def test_large_payload(self): saved_max_content_length = Request.max_content_length - Request.max_content_length = 16 + saved_max_body_length = Request.max_body_length + Request.max_content_length = 32 + Request.max_body_length = 16 fd = get_request_fd('GET', '/foo', headers={ 'Content-Type': 'application/x-www-form-urlencoded'}, body='foo=bar&abc=def&x=y') req = Request.create('app', fd, 'addr') self.assertEqual(req.body, b'') + self.assertEqual(req.stream.read(), b'foo=bar&abc=def&x=y') Request.max_content_length = saved_max_content_length + Request.max_body_length = saved_max_body_length diff --git a/tests/microdot_asyncio/test_request_asyncio.py b/tests/microdot_asyncio/test_request_asyncio.py index b4a63b6..f1466a2 100644 --- a/tests/microdot_asyncio/test_request_asyncio.py +++ b/tests/microdot_asyncio/test_request_asyncio.py @@ -101,14 +101,30 @@ class TestRequestAsync(unittest.TestCase): Request.max_readline = saved_max_readline + def test_stream(self): + fd = get_async_request_fd('GET', '/foo', headers={ + 'Content-Type': 'application/x-www-form-urlencoded', + 'Content-Length': '19'}, + body='foo=bar&abc=def&x=y') + req = _run(Request.create('app', fd, 'addr')) + self.assertEqual(req.body, b'foo=bar&abc=def&x=y') + data = _run(req.stream.read()) + self.assertEqual(data, b'foo=bar&abc=def&x=y') + def test_large_payload(self): saved_max_content_length = Request.max_content_length - Request.max_content_length = 16 + saved_max_body_length = Request.max_body_length + Request.max_content_length = 32 + Request.max_body_length = 16 fd = get_async_request_fd('GET', '/foo', headers={ - 'Content-Type': 'application/x-www-form-urlencoded'}, + 'Content-Type': 'application/x-www-form-urlencoded', + 'Content-Length': '19'}, body='foo=bar&abc=def&x=y') req = _run(Request.create('app', fd, 'addr')) self.assertEqual(req.body, b'') + data = _run(req.stream.read()) + self.assertEqual(data, b'foo=bar&abc=def&x=y') Request.max_content_length = saved_max_content_length + Request.max_body_length = saved_max_body_length diff --git a/tests/mock_socket.py b/tests/mock_socket.py index ccb66f2..edb7fd5 100644 --- a/tests/mock_socket.py +++ b/tests/mock_socket.py @@ -56,7 +56,7 @@ class FakeStreamAsync: async def readline(self): return self.stream.readline() - async def read(self, n): + async def read(self, n=-1): return self.stream.read(n) async def readexactly(self, n):