TonBERRY/microdot
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support streamed request payloads (Fixes #26)

Browse Source
This commit is contained in:
Miguel Grinberg
2022-02-18 17:30:18 +00:00
parent e16fb94b2d
commit 992fa722c1
6 changed files with 87 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
bin/mkrelease
View File

@@ -1,33 +0,0 @@
#!/bin/bash
VERSION=$1
if [[ "$VERSION" == "" ]]; then
echo Usage: $0 "<version>"
exit 1
fi
git diff --cached --exit-code >/dev/null
if [[ "$?" != "0" ]]; then
echo Commit your changes before using this script.
exit 1
fi
set -e
for PKG in microdot*; do
echo Building $PKG...
cd $PKG
sed -i "" "s/version.*$/version=\"$VERSION\",/" setup.py
git add setup.py
rm -rf dist
python setup.py sdist bdist_wheel --universal
cd ..
done
git commit -m "Release v$VERSION"
git tag v$VERSION
git push --tags origin master
for PKG in microdot*; do
echo Releasing $PKG...
cd $PKG
twine upload dist/*
cd ..
done

29
src/microdot.py
View File

@@ -18,6 +18,11 @@ try:
except ImportError:
import errno
try:
import uio as io
except ImportError:
import io
concurrency_mode = 'threaded'
try: # pragma: no cover
@@ -181,7 +186,8 @@ class Request():
:var cookies: A dictionary with the cookies included in the request.
:var content_length: The parsed ``Content-Length`` header.
:var content_type: The parsed ``Content-Type`` header.
:var body: A stream from where the body can be read.
:var stream: The input stream, containing the request body.
:var body: The body of the request, as bytes.
:var json: The parsed JSON body, as a dictionary or list, or ``None`` if
the request does not have a JSON body.
:var form: The parsed form submission body, as a :class:`MultiDict` object,
@@ -198,6 +204,17 @@ class Request():
#: Request.max_content_length = 1 * 1024 * 1024 # 1MB requests allowed
max_content_length = 16 * 1024
#: Specify the maximum payload size that can be stored in ``body``.
#: Requests with payloads that are larger than this size and up to
#: ``max_content_length`` bytes will be accepted, but the application will
#: only be able to access the body of the request by reading from
#: ``stream``. Set to 0 if you always access the body as a stream.
#:
#: Example::
#:
#: Request.max_body_length = 4 * 1024 # up to 4KB bodies read
max_body_length = 16 * 1024
#: Specify the maximum length allowed for a line in the request. Requests
#: with longer lines will not be correctly interpreted. Applications can
#: change this maximum as necessary.
@@ -211,7 +228,7 @@ class Request():
pass
def __init__(self, app, client_addr, method, url, http_version, headers,
body):
body, stream):
self.app = app
self.client_addr = client_addr
self.method = method
@@ -238,6 +255,7 @@ class Request():
name, value = cookie.strip().split('=', 1)
self.cookies[name] = value
self.body = body
self.stream = stream
self._json = None
self._form = None
self.g = Request.G()
@@ -275,15 +293,18 @@ class Request():
# body
body = b''
if content_length and content_length <= Request.max_content_length:
if content_length and content_length <= Request.max_body_length:
while len(body) < content_length:
data = client_stream.read(content_length - len(body))
if len(data) == 0: # pragma: no cover
raise EOFError()
body += data
stream = io.BytesIO(body)
else:
stream = client_stream
return Request(app, client_addr, method, url, http_version, headers,
body)
body, stream)
def _parse_urlencoded(self, urlencoded):
data = MultiDict()

34
src/microdot_asyncio.py
View File

@@ -10,6 +10,12 @@ try:
import uasyncio as asyncio
except ImportError:
import asyncio
try:
import uio as io
except ImportError:
import io
from microdot import Microdot as BaseMicrodot
from microdot import print_exception
from microdot import Request as BaseRequest
@@ -20,6 +26,23 @@ def _iscoroutine(coro):
return hasattr(coro, 'send') and hasattr(coro, 'throw')
class _AsyncBytesIO:
def __init__(self, data):
self.stream = io.BytesIO(data)
async def read(self, n=-1):
return self.stream.read(n)
async def readline(self): # pragma: no cover
return self.stream.readline()
async def readexactly(self, n): # pragma: no cover
return self.stream.read(n)
async def readuntil(self, separator=b'\n'): # pragma: no cover
return self.stream.readuntil(separator=separator)
class Request(BaseRequest):
@staticmethod
async def create(app, client_stream, client_addr):
@@ -55,12 +78,15 @@ class Request(BaseRequest):
content_length = int(value)
# body
body = await client_stream.readexactly(content_length) \
if content_length and \
content_length <= Request.max_content_length else b''
body = b''
if content_length and content_length <= Request.max_body_length:
body = await client_stream.readexactly(content_length)
stream = _AsyncBytesIO(body)
else:
stream = client_stream
return Request(app, client_addr, method, url, http_version, headers,
body)
body, stream)
@staticmethod
async def _safe_readline(stream):

14
tests/microdot/test_request.py
View File

@@ -91,14 +91,26 @@ class TestRequest(unittest.TestCase):
Request.max_readline = saved_max_readline
def test_stream(self):
fd = get_request_fd('GET', '/foo', headers={
'Content-Type': 'application/x-www-form-urlencoded'},
body='foo=bar&abc=def&x=y')
req = Request.create('app', fd, 'addr')
self.assertEqual(req.body, b'foo=bar&abc=def&x=y')
self.assertEqual(req.stream.read(), b'foo=bar&abc=def&x=y')
def test_large_payload(self):
saved_max_content_length = Request.max_content_length
Request.max_content_length = 16
saved_max_body_length = Request.max_body_length
Request.max_content_length = 32
Request.max_body_length = 16
fd = get_request_fd('GET', '/foo', headers={
'Content-Type': 'application/x-www-form-urlencoded'},
body='foo=bar&abc=def&x=y')
req = Request.create('app', fd, 'addr')
self.assertEqual(req.body, b'')
self.assertEqual(req.stream.read(), b'foo=bar&abc=def&x=y')
Request.max_content_length = saved_max_content_length
Request.max_body_length = saved_max_body_length

20
tests/microdot_asyncio/test_request_asyncio.py
View File

@@ -101,14 +101,30 @@ class TestRequestAsync(unittest.TestCase):
Request.max_readline = saved_max_readline
def test_stream(self):
fd = get_async_request_fd('GET', '/foo', headers={
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': '19'},
body='foo=bar&abc=def&x=y')
req = _run(Request.create('app', fd, 'addr'))
self.assertEqual(req.body, b'foo=bar&abc=def&x=y')
data = _run(req.stream.read())
self.assertEqual(data, b'foo=bar&abc=def&x=y')
def test_large_payload(self):
saved_max_content_length = Request.max_content_length
Request.max_content_length = 16
saved_max_body_length = Request.max_body_length
Request.max_content_length = 32
Request.max_body_length = 16
fd = get_async_request_fd('GET', '/foo', headers={
'Content-Type': 'application/x-www-form-urlencoded'},
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': '19'},
body='foo=bar&abc=def&x=y')
req = _run(Request.create('app', fd, 'addr'))
self.assertEqual(req.body, b'')
data = _run(req.stream.read())
self.assertEqual(data, b'foo=bar&abc=def&x=y')
Request.max_content_length = saved_max_content_length
Request.max_body_length = saved_max_body_length

2
tests/mock_socket.py
View File

@@ -56,7 +56,7 @@ class FakeStreamAsync:
async def readline(self):
return self.stream.readline()
async def read(self, n):
async def read(self, n=-1):
return self.stream.read(n)
async def readexactly(self, n):