Release 2.0.3

.github/workflows/tests.yml

@@ -16,6 +16,7 @@ jobs:
       - run: python -m pip install --upgrade pip wheel
       - run: pip install tox tox-gh-actions
       - run: tox -eflake8
+      - run: tox -edocs
   tests:
     name: tests
     strategy:

CHANGES.md

@@ -1,5 +1,12 @@
 # Microdot change log
 
+**Release 2.0.3** - 2024-01-07
+
+- Add a limit to WebSocket message size [#193](https://github.com/miguelgrinberg/microdot/issues/193) ([commit](https://github.com/miguelgrinberg/microdot/commit/5d188e8c0ddef6ce633ca702dbdd4a90f2799597))
+- Pass keyword arguments to thread executor in the correct way [#195](https://github.com/miguelgrinberg/microdot/issues/195) ([commit](https://github.com/miguelgrinberg/microdot/commit/6712c47400d7c426c88032f65ab74466524eccab))
+- Update uasyncio library used in tests to include new TLS support ([commit](https://github.com/miguelgrinberg/microdot/commit/c8c91e83457d24320f22c9a74e80b15e06b072ca))
+- Documentation improvements ([commit](https://github.com/miguelgrinberg/microdot/commit/b80b6b64d02d21400ca8a5077f5ed1127cc202ae))
+
 **Release 2.0.2** - 2023-12-28
 
 - Support binary data in the SSE extension ([commit](https://github.com/miguelgrinberg/microdot/commit/1fc11193da0d298f5539e2ad218836910a13efb2))

docs/api.rst

@@ -1,8 +1,8 @@
 API Reference
 =============
 
-``microdot`` module
--------------------
+Core API
+--------
 
 .. autoclass:: microdot.Microdot
    :members:
@@ -14,51 +14,57 @@ API Reference
    :members:
 
 
-``websocket`` extension
------------------------
+WebSocket
+---------
 
 .. automodule:: microdot.websocket
    :members:
 
-``utemplate`` templating extension
-----------------------------------
+Server-Sent Events (SSE)
+------------------------
+
+.. automodule:: microdot.sse
+   :members:
+
+Templates (uTemplate)
+---------------------
 
 .. automodule:: microdot.utemplate
    :members:
 
-``jinja`` templating extension
-------------------------------
+Templates (Jinja)
+-----------------
 
 .. automodule:: microdot.jinja
    :members:
 
-``session`` extension
----------------------
+User Sessions
+-------------
 
 .. automodule:: microdot.session
    :members:
 
-``cors`` extension
-------------------
+Cross-Origin Resource Sharing (CORS)
+------------------------------------
 
 .. automodule:: microdot.cors
    :members:
 
-``test_client`` extension
--------------------------
+Test Client
+-----------
 
 .. automodule:: microdot.test_client
    :members:
 
-``asgi`` extension
-------------------
+ASGI
+----
 
 .. autoclass:: microdot.asgi.Microdot
    :members:
    :exclude-members: shutdown, run
 
-``wsgi`` extension
--------------------
+WSGI
+----
 
 .. autoclass:: microdot.wsgi.Microdot
    :members:

docs/intro.rst

@@ -25,14 +25,15 @@ and incorporated into a custom MicroPython firmware.
 
 Use the following guidelines to know what files to copy:
 
-- For a minimal setup with only the base web server functionality, copy
+* For a minimal setup with only the base web server functionality, copy
   `microdot.py <https://github.com/miguelgrinberg/microdot/blob/main/src/microdot/microdot.py>`_
   into your project.
-- For a configuration that includes one or more optional extensions, create a
+* For a configuration that includes one or more optional extensions, create a
   *microdot* directory in your device and copy the following files:
-   - `__init__.py <https://github.com/miguelgrinberg/microdot/blob/main/src/microdot/__init__.py>`_
-   - `microdot.py <https://github.com/miguelgrinberg/microdot/blob/main/src/microdot/microdot.py>`_
-   - any needed `extensions <https://github.com/miguelgrinberg/microdot/tree/main/src/microdot>`_.
+
+  * `__init__.py <https://github.com/miguelgrinberg/microdot/blob/main/src/microdot/__init__.py>`_
+  * `microdot.py <https://github.com/miguelgrinberg/microdot/blob/main/src/microdot/microdot.py>`_
+  * any needed `extensions <https://github.com/miguelgrinberg/microdot/tree/main/src/microdot>`_.
 
 
 Getting Started
@@ -171,10 +172,8 @@ configure the web server.
     app.run(port=4443, debug=True, ssl=sslctx)
 
 .. note::
-   The ``ssl`` argument can only be used with CPython at this time, because
-   MicroPython's asyncio module does not currently support SSL certificates or
-   TLS encryption. Work on this is
-   `in progress <https://github.com/micropython/micropython/pull/11897>`_.
+   When using CPython, the certificate and key files must be given in PEM
+   format. When using MicroPython, these files must be given in DER format.
 
 Defining Routes
 ~~~~~~~~~~~~~~~

examples/tls/hello.py

@@ -1,4 +1,5 @@
 import ssl
+import sys
 from microdot import Microdot
 
 app = Microdot()
@@ -31,6 +32,7 @@ async def shutdown(request):
     return 'The server is shutting down...'
 
 
+ext = 'der' if sys.implementation.name == 'micropython' else 'pem'
 sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
-sslctx.load_cert_chain('cert.pem', 'key.pem')
+sslctx.load_cert_chain('cert.' + ext, 'key.' + ext)
 app.run(port=4443, debug=True, ssl=sslctx)

libs/micropython/asyncio/stream.py

@@ -13,12 +13,6 @@ class Stream:
     def get_extra_info(self, v):
         return self.e[v]
 
-    async def __aenter__(self):
-        return self
-
-    async def __aexit__(self, exc_type, exc, tb):
-        await self.close()
-
     def close(self):
         pass
 
@@ -63,6 +57,8 @@ class Stream:
         while True:
             yield core._io_queue.queue_read(self.s)
             l2 = self.s.readline()  # may do multiple reads but won't block
+            if l2 is None:
+                continue
             l += l2
             if not l2 or l[-1] == 10:  # \n (check l in case l2 is str)
                 return l
@@ -100,19 +96,29 @@ StreamWriter = Stream
 # Create a TCP stream connection to a remote host
 #
 # async
-def open_connection(host, port):
+def open_connection(host, port, ssl=None, server_hostname=None):
     from errno import EINPROGRESS
     import socket
 
     ai = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)[0]  # TODO this is blocking!
     s = socket.socket(ai[0], ai[1], ai[2])
     s.setblocking(False)
-    ss = Stream(s)
     try:
         s.connect(ai[-1])
     except OSError as er:
         if er.errno != EINPROGRESS:
             raise er
+    # wrap with SSL, if requested
+    if ssl:
+        if ssl is True:
+            import ssl as _ssl
+
+            ssl = _ssl.SSLContext(_ssl.PROTOCOL_TLS_CLIENT)
+        if not server_hostname:
+            server_hostname = host
+        s = ssl.wrap_socket(s, server_hostname=server_hostname, do_handshake_on_connect=False)
+        s.setblocking(False)
+    ss = Stream(s)
     yield core._io_queue.queue_write(s)
     return ss, ss
 
@@ -135,7 +141,7 @@ class Server:
     async def wait_closed(self):
         await self.task
 
-    async def _serve(self, s, cb):
+    async def _serve(self, s, cb, ssl):
         self.state = False
         # Accept incoming connections
         while True:
@@ -156,6 +162,13 @@ class Server:
             except:
                 # Ignore a failed accept
                 continue
+            if ssl:
+                try:
+                    s2 = ssl.wrap_socket(s2, server_side=True, do_handshake_on_connect=False)
+                except OSError as e:
+                    core.sys.print_exception(e)
+                    s2.close()
+                    continue
             s2.setblocking(False)
             s2s = Stream(s2, {"peername": addr})
             core.create_task(cb(s2s, s2s))
@@ -163,7 +176,7 @@ class Server:
 
 # Helper function to start a TCP stream server, running as a new task
 # TODO could use an accept-callback on socket read activity instead of creating a task
-async def start_server(cb, host, port, backlog=5):
+async def start_server(cb, host, port, backlog=5, ssl=None):
     import socket
 
     # Create and bind server socket.
@@ -176,7 +189,7 @@ async def start_server(cb, host, port, backlog=5):
 
     # Create and return server object and task.
     srv = Server()
-    srv.task = core.create_task(srv._serve(s, cb))
+    srv.task = core.create_task(srv._serve(s, cb, ssl))
     try:
         # Ensure that the _serve task has been scheduled so that it gets to
         # handle cancellation.

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "microdot"
-version = "2.0.2"
+version = "2.0.3"
 authors = [
     { name = "Miguel Grinberg", email = "miguel.grinberg@gmail.com" },
 ]

src/microdot/asgi.py

@@ -45,6 +45,12 @@ class _BodyStream:  # pragma: no cover
 
 
 class Microdot(BaseMicrodot):
+    """A subclass of the core :class:`Microdot <microdot.Microdot>` class that
+    implements the ASGI protocol.
+
+    This class must be used as the application instance when running under an
+    ASGI web server.
+    """
     def __init__(self):
         super().__init__()
         self.embedded_server = False

src/microdot/microdot.py

@@ -13,6 +13,7 @@ import time
 
 try:
     from inspect import iscoroutinefunction, iscoroutine
+    from functools import partial
 
     async def invoke_handler(handler, *args, **kwargs):
         """Invoke a handler and return the result.
@@ -23,7 +24,7 @@ try:
             ret = await handler(*args, **kwargs)
         else:
             ret = await asyncio.get_running_loop().run_in_executor(
-                None, handler, *args, **kwargs)
+                None, partial(handler, *args, **kwargs))
         return ret
 except ImportError:  # pragma: no cover
     def iscoroutine(coro):

src/microdot/session.py

@@ -134,7 +134,7 @@ def with_session(f):
             return 'Hello, World!'
 
     Note that the decorator does not save the session. To update the session,
-    call the :func:`update_session <microdot.session.update_session>` function.
+    call the :func:`session.save() <microdot.session.SessionDict.save>` method.
     """
     async def wrapper(request, *args, **kwargs):
         return await invoke_handler(

src/microdot/sse.py

@@ -3,6 +3,11 @@ import json
 
 
 class SSE:
+    """Server-Sent Events object.
+
+    An object of this class is sent to handler functions to manage the SSE
+    connection.
+    """
     def __init__(self):
         self.event = asyncio.Event()
         self.queue = []
@@ -40,19 +45,9 @@ def sse_response(request, event_function, *args, **kwargs):
     :param args: additional positional arguments to be passed to the response.
     :param kwargs: additional keyword arguments to be passed to the response.
 
-    Example::
-
-        @app.route('/events')
-        async def events_route(request):
-            async def events(request, sse):
-                # send an unnamed event with string data
-                await sse.send('hello')
-                # send an unnamed event with JSON data
-                await sse.send({'foo': 'bar'})
-                # send a named event
-                await sse.send('hello', event='greeting')
-
-            return sse_response(request, events)
+    This is a low-level function that can be used to implement a custom SSE
+    endpoint. In general the :func:`microdot.sse.with_sse` decorator should be
+    used instead.
     """
     sse = SSE()
 
@@ -95,9 +90,14 @@ def with_sse(f):
         @app.route('/events')
         @with_sse
         async def events(request, sse):
-            for i in range(10):
-                await asyncio.sleep(1)
-                await sse.send(f'{i}')
+            # send an unnamed event with string data
+            await sse.send('hello')
+
+            # send an unnamed event with JSON data
+            await sse.send({'foo': 'bar'})
+
+            # send a named event
+            await sse.send('hello', event='greeting')
     """
     async def sse_handler(request, *args, **kwargs):
         return sse_response(request, f, *args, **kwargs)

src/microdot/test_client.py

@@ -292,6 +292,8 @@ class TestClient:
             async def awrite(self, data):
                 if self.started:
                     h = WebSocket._parse_frame_header(data[0:2])
+                    if h[1] not in [WebSocket.TEXT, WebSocket.BINARY]:
+                        return
                     if h[3] < 0:
                         data = data[2 - h[3]:]
                     else:

src/microdot/websocket.py

@@ -1,10 +1,20 @@
 import binascii
 import hashlib
-from microdot import Response
-from microdot.microdot import MUTED_SOCKET_ERRORS
+from microdot import Request, Response
+from microdot.microdot import MUTED_SOCKET_ERRORS, print_exception
+
+
+class WebSocketError(Exception):
+    """Exception raised when an error occurs in a WebSocket connection."""
+    pass
 
 
 class WebSocket:
+    """A WebSocket connection object.
+
+    An instance of this class is sent to handler functions to manage the
+    WebSocket connection.
+    """
     CONT = 0
     TEXT = 1
     BINARY = 2
@@ -12,6 +22,18 @@ class WebSocket:
     PING = 9
     PONG = 10
 
+    #: Specify the maximum message size that can be received when calling the
+    #: ``receive()`` method. Messages with payloads that are larger than this
+    #: size will be rejected and the connection closed. Set to 0 to disable
+    #: the size check (be aware of potential security issues if you do this),
+    #: or to -1 to use the value set in
+    #: ``Request.max_body_length``. The default is -1.
+    #:
+    #: Example::
+    #:
+    #:    WebSocket.max_message_length = 4 * 1024  # up to 4KB messages
+    max_message_length = -1
+
     def __init__(self, request):
         self.request = request
         self.closed = False
@@ -26,6 +48,7 @@ class WebSocket:
             b'Sec-WebSocket-Accept: ' + response + b'\r\n\r\n')
 
     async def receive(self):
+        """Receive a message from the client."""
         while True:
             opcode, payload = await self._read_frame()
             send_opcode, data = self._process_websocket_frame(opcode, payload)
@@ -35,12 +58,20 @@ class WebSocket:
                 return data
 
     async def send(self, data, opcode=None):
+        """Send a message to the client.
+
+        :param data: the data to send, given as a string or bytes.
+        :param opcode: a custom frame opcode to use. If not given, the opcode
+                       is ``TEXT`` or ``BINARY`` depending on the type of the
+                       data.
+        """
         frame = self._encode_websocket_frame(
             opcode or (self.TEXT if isinstance(data, str) else self.BINARY),
             data)
         await self.request.sock[1].awrite(frame)
 
     async def close(self):
+        """Close the websocket connection."""
         if not self.closed:  # pragma: no cover
             self.closed = True
             await self.send(b'', self.CLOSE)
@@ -72,7 +103,7 @@ class WebSocket:
         fin = header[0] & 0x80
         opcode = header[0] & 0x0f
         if fin == 0 or opcode == cls.CONT:  # pragma: no cover
-            raise OSError(32, 'Continuation frames not supported')
+            raise WebSocketError('Continuation frames not supported')
         has_mask = header[1] & 0x80
         length = header[1] & 0x7f
         if length == 126:
@@ -87,7 +118,7 @@ class WebSocket:
         elif opcode == self.BINARY:
             pass
         elif opcode == self.CLOSE:
-            raise OSError(32, 'Websocket connection closed')
+            raise WebSocketError('Websocket connection closed')
         elif opcode == self.PING:
             return self.PONG, payload
         elif opcode == self.PONG:  # pragma: no branch
@@ -114,7 +145,7 @@ class WebSocket:
     async def _read_frame(self):
         header = await self.request.sock[0].read(2)
         if len(header) != 2:  # pragma: no cover
-            raise OSError(32, 'Websocket connection closed')
+            raise WebSocketError('Websocket connection closed')
         fin, opcode, has_mask, length = self._parse_frame_header(header)
         if length == -2:
             length = await self.request.sock[0].read(2)
@@ -122,6 +153,10 @@ class WebSocket:
         elif length == -8:
             length = await self.request.sock[0].read(8)
             length = int.from_bytes(length, 'big')
+        max_allowed_length = Request.max_body_length \
+            if self.max_message_length == -1 else self.max_message_length
+        if length > max_allowed_length:
+            raise WebSocketError('Message too large')
         if has_mask:  # pragma: no cover
             mask = await self.request.sock[0].read(4)
         payload = await self.request.sock[0].read(length)
@@ -161,11 +196,19 @@ def websocket_wrapper(f, upgrade_function):
         ws = await upgrade_function(request)
         try:
             await f(request, ws, *args, **kwargs)
-            await ws.close()  # pragma: no cover
         except OSError as exc:
             if exc.errno not in MUTED_SOCKET_ERRORS:  # pragma: no cover
                 raise
-        return ''
+        except WebSocketError:
+            pass
+        except Exception as exc:
+            print_exception(exc)
+        finally:  # pragma: no cover
+            try:
+                await ws.close()
+            except Exception:
+                pass
+        return Response.already_handled
     return wrapper
 
 

src/microdot/wsgi.py

@@ -9,6 +9,12 @@ from microdot.websocket import WebSocket, websocket_upgrade, \
 
 
 class Microdot(BaseMicrodot):
+    """A subclass of the core :class:`Microdot <microdot.Microdot>` class that
+    implements the WSGI protocol.
+
+    This class must be used as the application instance when running under a
+    WSGI web server.
+    """
     def __init__(self):
         super().__init__()
         self.loop = asyncio.new_event_loop()

tests/test_microdot.py

@@ -25,6 +25,14 @@ class TestMicrodot(unittest.TestCase):
         async def index2(req):
             return 'foo-async'
 
+        @app.route('/arg/<id>')
+        def index3(req, id):
+            return id
+
+        @app.route('/arg/async/<id>')
+        async def index4(req, id):
+            return f'async-{id}'
+
         client = TestClient(app)
 
         res = self._run(client.get('/'))
@@ -45,6 +53,24 @@ class TestMicrodot(unittest.TestCase):
         self.assertEqual(res.body, b'foo-async')
         self.assertEqual(res.json, None)
 
+        res = self._run(client.get('/arg/123'))
+        self.assertEqual(res.status_code, 200)
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
+        self.assertEqual(res.headers['Content-Length'], '3')
+        self.assertEqual(res.text, '123')
+        self.assertEqual(res.body, b'123')
+        self.assertEqual(res.json, None)
+
+        res = self._run(client.get('/arg/async/123'))
+        self.assertEqual(res.status_code, 200)
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
+        self.assertEqual(res.headers['Content-Length'], '9')
+        self.assertEqual(res.text, 'async-123')
+        self.assertEqual(res.body, b'async-123')
+        self.assertEqual(res.json, None)
+
     def test_post_request(self):
         app = Microdot()
 

tests/test_websocket.py

@@ -1,8 +1,8 @@
 import asyncio
 import sys
 import unittest
-from microdot import Microdot
-from microdot.websocket import with_websocket, WebSocket
+from microdot import Microdot, Request
+from microdot.websocket import with_websocket, WebSocket, WebSocketError
 from microdot.test_client import TestClient
 
 
@@ -17,6 +17,7 @@ class TestWebSocket(unittest.TestCase):
         return self.loop.run_until_complete(coro)
 
     def test_websocket_echo(self):
+        WebSocket.max_message_length = 65537
         app = Microdot()
 
         @app.route('/echo')
@@ -26,34 +27,10 @@ class TestWebSocket(unittest.TestCase):
                 data = await ws.receive()
                 await ws.send(data)
 
-        results = []
-
-        def ws():
-            data = yield 'hello'
-            results.append(data)
-            data = yield b'bye'
-            results.append(data)
-            data = yield b'*' * 300
-            results.append(data)
-            data = yield b'+' * 65537
-            results.append(data)
-
-        client = TestClient(app)
-        res = self._run(client.websocket('/echo', ws))
-        self.assertIsNone(res)
-        self.assertEqual(results, ['hello', b'bye', b'*' * 300, b'+' * 65537])
-
-    @unittest.skipIf(sys.implementation.name == 'micropython',
-                     'no support for async generators in MicroPython')
-    def test_websocket_echo_async_client(self):
-        app = Microdot()
-
-        @app.route('/echo')
+        @app.route('/divzero')
         @with_websocket
-        async def index(req, ws):
-            while True:
-                data = await ws.receive()
-                await ws.send(data)
+        async def divzero(req, ws):
+            1 / 0
 
         results = []
 
@@ -72,6 +49,35 @@ class TestWebSocket(unittest.TestCase):
         self.assertIsNone(res)
         self.assertEqual(results, ['hello', b'bye', b'*' * 300, b'+' * 65537])
 
+        res = self._run(client.websocket('/divzero', ws))
+        self.assertIsNone(res)
+        WebSocket.max_message_length = -1
+
+    @unittest.skipIf(sys.implementation.name == 'micropython',
+                     'no support for async generators in MicroPython')
+    def test_websocket_large_message(self):
+        saved_max_body_length = Request.max_body_length
+        Request.max_body_length = 10
+        app = Microdot()
+
+        @app.route('/echo')
+        @with_websocket
+        async def index(req, ws):
+            data = await ws.receive()
+            await ws.send(data)
+
+        results = []
+
+        async def ws():
+            data = yield '0123456789abcdef'
+            results.append(data)
+
+        client = TestClient(app)
+        res = self._run(client.websocket('/echo', ws))
+        self.assertIsNone(res)
+        self.assertEqual(results, [])
+        Request.max_body_length = saved_max_body_length
+
     def test_bad_websocket_request(self):
         app = Microdot()
 
@@ -106,7 +112,7 @@ class TestWebSocket(unittest.TestCase):
                          (None, 'foo'))
         self.assertEqual(ws._process_websocket_frame(WebSocket.BINARY, b'foo'),
                          (None, b'foo'))
-        self.assertRaises(OSError, ws._process_websocket_frame,
+        self.assertRaises(WebSocketError, ws._process_websocket_frame,
                           WebSocket.CLOSE, b'')
         self.assertEqual(ws._process_websocket_frame(WebSocket.PING, b'foo'),
                          (WebSocket.PONG, b'foo'))

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist=flake8,py38,py39,py310,py311,py312,upy,cpy,benchmark
+envlist=flake8,py38,py39,py310,py311,py312,upy,cpy,benchmark,docs
 skipsdist=True
 skip_missing_interpreters=True
 
@@ -36,7 +36,6 @@ commands=sh -c "bin/circuitpython run_tests.py"
 [testenv:upy-mac]
 allowlist_externals=micropython
 commands=micropython run_tests.py
-deps=
 
 [testenv:benchmark]
 deps=
@@ -59,3 +58,13 @@ deps=
     flake8
 commands=
     flake8 --ignore=W503 --exclude examples/templates/utemplate/templates src tests examples
+
+[testenv:docs]
+changedir=docs
+deps=
+    sphinx
+    pyjwt
+allowlist_externals=
+    make
+commands=
+    make html