memory comparison benchmark

.github/workflows/tests.yml

@@ -51,3 +51,12 @@ jobs:
       - run: pip install tox tox-gh-actions codecov
       - run: tox
       - run: codecov
+  benchmark:
+    name: benchmark
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - run: python -m pip install --upgrade pip wheel
+      - run: pip install tox tox-gh-actions
+      - run: tox -ebenchmark

examples/benchmark/README.md

@@ -0,0 +1,5 @@
+This directory contains a few example applications for different
+configurations of Microdot, plus similar implementations for other web
+frameworks.
+
+The *run.py* script runs these applications and reports memory usage for each.

examples/benchmark/mem.py

@@ -0,0 +1,11 @@
+from microdot import Microdot
+
+app = Microdot()
+
+
+@app.get('/')
+def index(req):
+    return {'hello': 'world'}
+
+
+app.run()

examples/benchmark/mem_asgi.py

@@ -0,0 +1,8 @@
+from microdot_asgi import Microdot
+
+app = Microdot()
+
+
+@app.get('/')
+async def index(req):
+    return {'hello': 'world'}

examples/benchmark/mem_async.py

@@ -0,0 +1,11 @@
+from microdot_asyncio import Microdot
+
+app = Microdot()
+
+
+@app.get('/')
+async def index(req):
+    return {'hello': 'world'}
+
+
+app.run()

examples/benchmark/mem_fastapi.py

@@ -0,0 +1,8 @@
+from fastapi import FastAPI
+
+app = FastAPI()
+
+
+@app.get('/')
+def index():
+    return {'hello': 'world'}

examples/benchmark/mem_flask.py

@@ -0,0 +1,8 @@
+from flask import Flask
+
+app = Flask(__name__)
+
+
+@app.get('/')
+def index():
+    return {'hello': 'world'}

examples/benchmark/mem_quart.py

@@ -0,0 +1,8 @@
+from quart import Quart
+
+app = Quart(__name__)
+
+
+@app.get('/')
+def index():
+    return {'hello': 'world'}

examples/benchmark/mem_wsgi.py

@@ -0,0 +1,8 @@
+from microdot_wsgi import Microdot
+
+app = Microdot()
+
+
+@app.get('/')
+def index(req):
+    return {'hello': 'world'}

examples/benchmark/requirements.txt

@@ -0,0 +1,33 @@
+aiofiles==0.8.0
+anyio==3.6.1
+blinker==1.5
+certifi==2022.6.15
+charset-normalizer==2.1.0
+click==8.1.3
+fastapi==0.79.0
+Flask==2.2.1
+gunicorn==20.1.0
+h11==0.13.0
+h2==4.1.0
+hpack==4.0.0
+humanize==4.3.0
+hypercorn==0.13.2
+hyperframe==6.0.1
+idna==3.3
+itsdangerous==2.1.2
+Jinja2==3.1.2
+MarkupSafe==2.1.1
+microdot
+priority==2.0.0
+psutil==5.9.1
+pydantic==1.9.1
+quart==0.18.0
+requests==2.28.1
+sniffio==1.2.0
+starlette==0.19.1
+toml==0.10.2
+typing_extensions==4.3.0
+urllib3==1.26.11
+uvicorn==0.18.2
+Werkzeug==2.2.1
+wsproto==1.1.0

examples/benchmark/results.txt

@@ -0,0 +1,14 @@
+❯ curl -X GET http://localhost:5000/        <-- microdot
+{"ram": 8429568}%
+❯ curl -X GET http://localhost:5000/        <-- microdot_asyncio
+{"ram": 12410880}%
+❯ curl -X GET http://localhost:8000/        <-- microdot_wsgi
+{"ram": 9101312}%
+❯ curl -X GET http://localhost:8000/        <-- microdot_asgi
+{"ram": 18620416}%
+❯ curl -X GET http://localhost:5000/        <-- flask app.run
+{"ram":25460736}
+❯ curl -X GET http://localhost:5000/        <-- flask run
+{"ram":26210304}
+❯ curl -X GET http://localhost:5000/        <-- quart run
+{"ram":31748096}%

examples/benchmark/run.py

@@ -0,0 +1,94 @@
+import os
+import subprocess
+import time
+import requests
+import psutil
+import humanize
+
+apps = [
+    (
+        ['micropython', '-c', 'import time; time.sleep(10)'],
+        {},
+        'baseline-micropython'
+    ),
+    (
+        'micropython mem.py',
+        {'MICROPYPATH': '../../src'},
+        'microdot-micropython-sync'
+    ),
+    (
+        'micropython mem_async.py',
+        {'MICROPYPATH': '../../src:../../libs/micropython'},
+        'microdot-micropython-async'
+    ),
+    (
+        ['python', '-c', 'import time; time.sleep(10)'],
+        {},
+        'baseline-python'
+    ),
+    (
+        'python mem.py',
+        {'PYTHONPATH': '../../src'},
+        'microdot-cpython-sync'
+    ),
+    (
+        'python mem_async.py',
+        {'PYTHONPATH': '../../src'},
+        'microdot-cpython-async'
+    ),
+    (
+        'gunicorn --workers 1 --bind :5000 mem_wsgi:app',
+        {'PYTHONPATH': '../../src'},
+        'microdot-gunicorn-sync'
+    ),
+    (
+        'uvicorn --workers 1 --port 5000 mem_asgi:app',
+        {'PYTHONPATH': '../../src'},
+        'microdot-uvicorn-async'
+    ),
+    (
+        'flask run',
+        {'FLASK_APP': 'mem_flask.py'},
+        'flask-run-sync'
+    ),
+    (
+        'quart run',
+        {'QUART_APP': 'mem_quart.py'},
+        'quart-run-async'
+    ),
+    (
+        'gunicorn --workers 1 --bind :5000 mem_flask:app',
+        {},
+        'flask-gunicorn-sync'
+    ),
+    (
+        'uvicorn --workers 1 --port 5000 mem_quart:app',
+        {},
+        'quart-uvicorn-async'
+    ),
+    (
+        'uvicorn --workers 1 --port 5000 mem_fastapi:app',
+        {},
+        'fastapi-uvicorn-async'
+    ),
+]
+
+for app, env, name in apps:
+    p = subprocess.Popen(
+        app.split() if isinstance(app, str) else app,
+        env={'PATH': os.environ['PATH'], **env},
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL
+    )
+    time.sleep(1)
+    if not name.startswith('baseline'):
+        r = requests.get('http://localhost:5000')
+        r.raise_for_status()
+    proc = psutil.Process(p.pid)
+    mem = proc.memory_info().rss
+    for child in proc.children(recursive=True):
+        mem += child.memory_info().rss
+    bar = '*' * (mem // (1024 * 1024))
+    print(f'{name:<28}{humanize.naturalsize(mem):>10} {bar}')
+    p.terminate()
+    time.sleep(1)

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist=flake8,py36,py37,py38,py39,py310,upy
+envlist=flake8,py36,py37,py38,py39,py310,upy,benchmark
 skipsdist=True
 skip_missing_interpreters=True
 
@@ -38,3 +38,20 @@ commands=sh -c "bin/micropython run_tests.py"
 whitelist_externals=micropython
 commands=micropython run_tests.py
 deps=
+
+[testenv:benchmark]
+deps=
+    flask
+    quart
+    fastapi
+    gunicorn
+    uvicorn
+    requests
+    psutil
+    humanize
+changedir=examples/benchmark
+commands=
+    python run.py
+setenv=
+  PATH={env:PATH}{:}../../bin
+