Charset handling in Content-Type headers (Fixes #60)

src/microdot.py

@@ -59,11 +59,11 @@ MUTED_SOCKET_ERRORS = [
 ]
 
 
-def urldecode(string):
-    string = string.replace('+', ' ')
-    parts = string.split('%')
+def urldecode_str(s):
+    s = s.replace('+', ' ')
+    parts = s.split('%')
     if len(parts) == 1:
-        return string
+        return s
     result = [parts[0]]
     for item in parts[1:]:
         if item == '':
@@ -75,6 +75,22 @@ def urldecode(string):
     return ''.join(result)
 
 
+def urldecode_bytes(s):
+    s = s.replace(b'+', b' ')
+    parts = s.split(b'%')
+    if len(parts) == 1:
+        return s.decode()
+    result = [parts[0]]
+    for item in parts[1:]:
+        if item == b'':
+            result.append(b'%')
+        else:
+            code = item[:2]
+            result.append(bytes([int(code, 16)]))
+            result.append(item[2:])
+    return b''.join(result).decode()
+
+
 class MultiDict(dict):
     """A subclass of dictionary that can hold multiple values for the same
     key. It is used to hold key/value pairs decoded from query strings and
@@ -287,9 +303,15 @@ class Request():
 
     def _parse_urlencoded(self, urlencoded):
         data = MultiDict()
-        if urlencoded:
-            for k, v in [pair.split('=', 1) for pair in urlencoded.split('&')]:
-                data[urldecode(k)] = urldecode(v)
+        if len(urlencoded) > 0:
+            if isinstance(urlencoded, str):
+                for k, v in [pair.split('=', 1)
+                             for pair in urlencoded.split('&')]:
+                    data[urldecode_str(k)] = urldecode_str(v)
+            elif isinstance(urlencoded, bytes):  # pragma: no branch
+                for k, v in [pair.split(b'=', 1)
+                             for pair in urlencoded.split(b'&')]:
+                    data[urldecode_bytes(k)] = urldecode_bytes(v)
         return data
 
     @property
@@ -342,7 +364,7 @@ class Request():
             mime_type = self.content_type.split(';')[0]
             if mime_type != 'application/x-www-form-urlencoded':
                 return None
-            self._form = self._parse_urlencoded(self.body.decode())
+            self._form = self._parse_urlencoded(self.body)
         return self._form
 
     def after_request(self, f):
@@ -419,7 +441,7 @@ class Response():
         self.reason = reason
         if isinstance(body, (dict, list)):
             self.body = json.dumps(body).encode()
-            self.headers['Content-Type'] = 'application/json'
+            self.headers['Content-Type'] = 'application/json; charset=UTF-8'
         elif isinstance(body, str):
             self.body = body.encode()
         else:
@@ -468,6 +490,8 @@ class Response():
             self.headers['Content-Length'] = str(len(self.body))
         if 'Content-Type' not in self.headers:
             self.headers['Content-Type'] = self.default_content_type
+            if 'charset=' not in self.headers['Content-Type']:
+                self.headers['Content-Type'] += '; charset=UTF-8'
 
     def write(self, stream):
         self.complete()

src/microdot_test_client.py

@@ -48,7 +48,7 @@ class TestResponse:
     def _process_json_body(self):
         for name, value in self.headers.items():  # pragma: no branch
             if name.lower() == 'content-type':
-                if value.lower() == 'application/json':
+                if value.lower().split(';')[0] == 'application/json':
                     self.json = json.loads(self.text)
                 break
 

tests/test_microdot.py

@@ -37,7 +37,8 @@ class TestMicrodot(unittest.TestCase):
         client = TestClient(app)
         res = client.get('/')
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.headers['Content-Length'], '3')
         self.assertEqual(res.text, 'foo')
         self.assertEqual(res.body, b'foo')
@@ -57,7 +58,8 @@ class TestMicrodot(unittest.TestCase):
         client = TestClient(app)
         res = client.post('/')
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.headers['Content-Length'], '3')
         self.assertEqual(res.text, 'bar')
 
@@ -73,7 +75,8 @@ class TestMicrodot(unittest.TestCase):
         app.run()
         self.assertTrue(fd.response.startswith(b'HTTP/1.0 400 N/A\r\n'))
         self.assertIn(b'Content-Length: 11\r\n', fd.response)
-        self.assertIn(b'Content-Type: text/plain\r\n', fd.response)
+        self.assertIn(b'Content-Type: text/plain; charset=UTF-8\r\n',
+                      fd.response)
         self.assertTrue(fd.response.endswith(b'\r\n\r\nBad request'))
 
         self._unmock()
@@ -106,7 +109,8 @@ class TestMicrodot(unittest.TestCase):
         for method in methods:
             res = getattr(client, method.lower())('/' + method.lower())
             self.assertEqual(res.status_code, 200)
-            self.assertEqual(res.headers['Content-Type'], 'text/plain')
+            self.assertEqual(res.headers['Content-Type'],
+                             'text/plain; charset=UTF-8')
             self.assertEqual(res.text, method)
 
     def test_headers(self):
@@ -119,7 +123,8 @@ class TestMicrodot(unittest.TestCase):
         client = TestClient(app)
         res = client.get('/', headers={'X-Foo': 'bar'})
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, 'bar')
 
     def test_cookies(self):
@@ -133,7 +138,8 @@ class TestMicrodot(unittest.TestCase):
         client = TestClient(app, cookies={'one': '1', 'two': '2'})
         res = client.get('/', headers={'Cookie': 'three=3'})
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, '123')
 
     def test_binary_payload(self):
@@ -146,7 +152,8 @@ class TestMicrodot(unittest.TestCase):
         client = TestClient(app)
         res = client.post('/', body=b'foo')
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, 'foo')
 
     def test_json_payload(self):
@@ -165,12 +172,14 @@ class TestMicrodot(unittest.TestCase):
 
         res = client.post('/dict', body={'foo': 'bar'})
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, 'bar')
 
         res = client.post('/list', body=['foo', 'bar'])
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, 'foo')
 
     def test_tuple_responses(self):
@@ -190,18 +199,21 @@ class TestMicrodot(unittest.TestCase):
 
         @app.route('/body-status-headers')
         def four(req):
-            return '<p>four</p>', 202, {'Content-Type': 'text/html'}
+            return '<p>four</p>', 202, \
+                {'Content-Type': 'text/html; charset=UTF-8'}
 
         client = TestClient(app)
 
         res = client.get('/body')
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, 'one')
 
         res = client.get('/body-status')
         self.assertEqual(res.status_code, 202)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, 'two')
 
         res = client.get('/body-headers')
@@ -211,7 +223,8 @@ class TestMicrodot(unittest.TestCase):
 
         res = client.get('/body-status-headers')
         self.assertEqual(res.status_code, 202)
-        self.assertEqual(res.headers['Content-Type'], 'text/html')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/html; charset=UTF-8')
         self.assertEqual(res.text, '<p>four</p>')
 
     def test_before_after_request(self):
@@ -248,7 +261,8 @@ class TestMicrodot(unittest.TestCase):
 
         res = client.get('/bar')
         self.assertEqual(res.status_code, 202)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.headers['Set-Cookie'], ['foo=bar'])
         self.assertEqual(res.headers['X-One'], '1')
         self.assertEqual(res.headers['X-Two'], '2')
@@ -257,7 +271,8 @@ class TestMicrodot(unittest.TestCase):
 
         res = client.get('/baz')
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.headers['Set-Cookie'], ['foo=bar'])
         self.assertEqual(res.headers['X-One'], '1')
         self.assertFalse('X-Two' in res.headers)
@@ -276,7 +291,8 @@ class TestMicrodot(unittest.TestCase):
         app.run()
         self.assertTrue(fd.response.startswith(b'HTTP/1.0 400 N/A\r\n'))
         self.assertIn(b'Content-Length: 11\r\n', fd.response)
-        self.assertIn(b'Content-Type: text/plain\r\n', fd.response)
+        self.assertIn(b'Content-Type: text/plain; charset=UTF-8\r\n',
+                      fd.response)
         self.assertTrue(fd.response.endswith(b'\r\n\r\nBad request'))
 
         self._unmock()
@@ -297,7 +313,8 @@ class TestMicrodot(unittest.TestCase):
         app.run()
         self.assertTrue(fd.response.startswith(b'HTTP/1.0 200 OK\r\n'))
         self.assertIn(b'Content-Length: 3\r\n', fd.response)
-        self.assertIn(b'Content-Type: text/plain\r\n', fd.response)
+        self.assertIn(b'Content-Type: text/plain; charset=UTF-8\r\n',
+                      fd.response)
         self.assertTrue(fd.response.endswith(b'\r\n\r\n400'))
 
         self._unmock()
@@ -312,7 +329,8 @@ class TestMicrodot(unittest.TestCase):
         client = TestClient(app)
         res = client.post('/foo')
         self.assertEqual(res.status_code, 404)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, 'Not found')
 
     def test_404_handler(self):
@@ -329,7 +347,8 @@ class TestMicrodot(unittest.TestCase):
         client = TestClient(app)
         res = client.post('/foo')
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, '404')
 
     def test_405(self):
@@ -342,7 +361,8 @@ class TestMicrodot(unittest.TestCase):
         client = TestClient(app)
         res = client.post('/foo')
         self.assertEqual(res.status_code, 405)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, 'Not found')
 
     def test_405_handler(self):
@@ -359,7 +379,8 @@ class TestMicrodot(unittest.TestCase):
         client = TestClient(app)
         res = client.patch('/foo')
         self.assertEqual(res.status_code, 405)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, '405')
 
     def test_413(self):
@@ -372,7 +393,8 @@ class TestMicrodot(unittest.TestCase):
         client = TestClient(app)
         res = client.post('/foo', body='x' * 17000)
         self.assertEqual(res.status_code, 413)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, 'Payload too large')
 
     def test_413_handler(self):
@@ -389,7 +411,8 @@ class TestMicrodot(unittest.TestCase):
         client = TestClient(app)
         res = client.post('/foo', body='x' * 17000)
         self.assertEqual(res.status_code, 400)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, '413')
 
     def test_500(self):
@@ -402,7 +425,8 @@ class TestMicrodot(unittest.TestCase):
         client = TestClient(app)
         res = client.get('/')
         self.assertEqual(res.status_code, 500)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, 'Internal server error')
 
     def test_500_handler(self):
@@ -419,7 +443,8 @@ class TestMicrodot(unittest.TestCase):
         client = TestClient(app)
         res = client.get('/')
         self.assertEqual(res.status_code, 501)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, '501')
 
     def test_exception_handler(self):
@@ -436,7 +461,8 @@ class TestMicrodot(unittest.TestCase):
         client = TestClient(app)
         res = client.get('/')
         self.assertEqual(res.status_code, 501)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, '501')
 
     def test_abort(self):
@@ -450,7 +476,8 @@ class TestMicrodot(unittest.TestCase):
         client = TestClient(app)
         res = client.get('/')
         self.assertEqual(res.status_code, 406)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, 'Not acceptable')
 
     def test_abort_handler(self):
@@ -468,7 +495,8 @@ class TestMicrodot(unittest.TestCase):
         client = TestClient(app)
         res = client.get('/')
         self.assertEqual(res.status_code, 406)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, '406')
 
     def test_json_response(self):
@@ -486,12 +514,14 @@ class TestMicrodot(unittest.TestCase):
 
         res = client.get('/dict')
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'application/json')
+        self.assertEqual(res.headers['Content-Type'],
+                         'application/json; charset=UTF-8')
         self.assertEqual(res.json, {'foo': 'bar'})
 
         res = client.get('/list')
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'application/json')
+        self.assertEqual(res.headers['Content-Type'],
+                         'application/json; charset=UTF-8')
         self.assertEqual(res.json, ['foo', 'bar'])
 
     def test_binary_response(self):
@@ -523,7 +553,8 @@ class TestMicrodot(unittest.TestCase):
         client = TestClient(app)
         res = client.get('/')
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, 'foobar')
 
     def test_already_handled_response(self):
@@ -563,12 +594,14 @@ class TestMicrodot(unittest.TestCase):
 
         res = client.get('/app')
         self.assertEqual(res.status_code, 404)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, '404')
 
         res = client.get('/sub/app')
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, 'before:foo:after')
 
     def test_ssl(self):
@@ -590,7 +623,8 @@ class TestMicrodot(unittest.TestCase):
         app.run(ssl=FakeSSL())
         self.assertTrue(fd.response.startswith(b'HTTP/1.0 200 OK\r\n'))
         self.assertIn(b'Content-Length: 3\r\n', fd.response)
-        self.assertIn(b'Content-Type: text/plain\r\n', fd.response)
+        self.assertIn(b'Content-Type: text/plain; charset=UTF-8\r\n',
+                      fd.response)
         self.assertTrue(fd.response.endswith(b'\r\n\r\nbar'))
 
         self._unmock()

tests/test_microdot_asgi.py

@@ -82,10 +82,12 @@ class TestMicrodotASGI(unittest.TestCase):
         async def send(packet):
             if packet['type'] == 'http.response.start':
                 self.assertEqual(packet['status'], 200)
-                expected_headers = [('Content-Length', '8'),
-                                    ('Content-Type', 'text/plain'),
-                                    ('Set-Cookie', 'foo=foo'),
-                                    ('Set-Cookie', 'bar=bar; HttpOnly')]
+                expected_headers = [
+                    ('Content-Length', '8'),
+                    ('Content-Type', 'text/plain; charset=UTF-8'),
+                    ('Set-Cookie', 'foo=foo'),
+                    ('Set-Cookie', 'bar=bar; HttpOnly')
+                ]
                 self.assertEqual(len(packet['headers']), len(expected_headers))
                 for header in expected_headers:
                     self.assertIn(header, packet['headers'])

tests/test_microdot_asyncio.py

@@ -50,7 +50,8 @@ class TestMicrodotAsync(unittest.TestCase):
 
         res = self._run(client.get('/'))
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.headers['Content-Length'], '3')
         self.assertEqual(res.text, 'foo')
         self.assertEqual(res.body, b'foo')
@@ -58,7 +59,8 @@ class TestMicrodotAsync(unittest.TestCase):
 
         res = self._run(client.get('/async'))
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.headers['Content-Length'], '9')
         self.assertEqual(res.text, 'foo-async')
         self.assertEqual(res.body, b'foo-async')
@@ -83,7 +85,8 @@ class TestMicrodotAsync(unittest.TestCase):
 
         res = self._run(client.post('/'))
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.headers['Content-Length'], '3')
         self.assertEqual(res.text, 'bar')
         self.assertEqual(res.body, b'bar')
@@ -91,7 +94,8 @@ class TestMicrodotAsync(unittest.TestCase):
 
         res = self._run(client.post('/async'))
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.headers['Content-Length'], '9')
         self.assertEqual(res.text, 'bar-async')
         self.assertEqual(res.body, b'bar-async')
@@ -107,7 +111,8 @@ class TestMicrodotAsync(unittest.TestCase):
         app.run()
         self.assertTrue(fd.response.startswith(b'HTTP/1.0 400 N/A\r\n'))
         self.assertIn(b'Content-Length: 11\r\n', fd.response)
-        self.assertIn(b'Content-Type: text/plain\r\n', fd.response)
+        self.assertIn(b'Content-Type: text/plain; charset=UTF-8\r\n',
+                      fd.response)
         self.assertTrue(fd.response.endswith(b'\r\n\r\nBad request'))
 
     def test_method_decorators(self):
@@ -139,7 +144,8 @@ class TestMicrodotAsync(unittest.TestCase):
             res = self._run(getattr(
                 client, method.lower())('/' + method.lower()))
             self.assertEqual(res.status_code, 200)
-            self.assertEqual(res.headers['Content-Type'], 'text/plain')
+            self.assertEqual(res.headers['Content-Type'],
+                             'text/plain; charset=UTF-8')
             self.assertEqual(res.text, method)
 
     def test_headers(self):
@@ -152,7 +158,8 @@ class TestMicrodotAsync(unittest.TestCase):
         client = TestClient(app)
         res = self._run(client.get('/', headers={'X-Foo': 'bar'}))
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, 'bar')
 
     def test_cookies(self):
@@ -166,7 +173,8 @@ class TestMicrodotAsync(unittest.TestCase):
         client = TestClient(app, cookies={'one': '1', 'two': '2'})
         res = self._run(client.get('/', headers={'Cookie': 'three=3'}))
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, '123')
 
     def test_binary_payload(self):
@@ -179,7 +187,8 @@ class TestMicrodotAsync(unittest.TestCase):
         client = TestClient(app)
         res = self._run(client.post('/', body=b'foo'))
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, 'foo')
 
     def test_json_payload(self):
@@ -198,12 +207,14 @@ class TestMicrodotAsync(unittest.TestCase):
 
         res = self._run(client.post('/dict', body={'foo': 'bar'}))
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, 'bar')
 
         res = self._run(client.post('/list', body=['foo', 'bar']))
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, 'foo')
 
     def test_tuple_responses(self):
@@ -223,18 +234,21 @@ class TestMicrodotAsync(unittest.TestCase):
 
         @app.route('/body-status-headers')
         def four(req):
-            return '<p>four</p>', 202, {'Content-Type': 'text/html'}
+            return '<p>four</p>', 202, \
+                {'Content-Type': 'text/html; charset=UTF-8'}
 
         client = TestClient(app)
 
         res = self._run(client.get('/body'))
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, 'one')
 
         res = self._run(client.get('/body-status'))
         self.assertEqual(res.status_code, 202)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, 'two')
 
         res = self._run(client.get('/body-headers'))
@@ -244,7 +258,8 @@ class TestMicrodotAsync(unittest.TestCase):
 
         res = self._run(client.get('/body-status-headers'))
         self.assertEqual(res.status_code, 202)
-        self.assertEqual(res.headers['Content-Type'], 'text/html')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/html; charset=UTF-8')
         self.assertEqual(res.text, '<p>four</p>')
 
     def test_before_after_request(self):
@@ -281,7 +296,8 @@ class TestMicrodotAsync(unittest.TestCase):
 
         res = self._run(client.get('/bar'))
         self.assertEqual(res.status_code, 202)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.headers['Set-Cookie'], ['foo=bar'])
         self.assertEqual(res.headers['X-One'], '1')
         self.assertEqual(res.headers['X-Two'], '2')
@@ -290,7 +306,8 @@ class TestMicrodotAsync(unittest.TestCase):
 
         res = self._run(client.get('/baz'))
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.headers['Set-Cookie'], ['foo=bar'])
         self.assertEqual(res.headers['X-One'], '1')
         self.assertFalse('X-Two' in res.headers)
@@ -309,7 +326,8 @@ class TestMicrodotAsync(unittest.TestCase):
         app.run()
         self.assertTrue(fd.response.startswith(b'HTTP/1.0 400 N/A\r\n'))
         self.assertIn(b'Content-Length: 11\r\n', fd.response)
-        self.assertIn(b'Content-Type: text/plain\r\n', fd.response)
+        self.assertIn(b'Content-Type: text/plain; charset=UTF-8\r\n',
+                      fd.response)
         self.assertTrue(fd.response.endswith(b'\r\n\r\nBad request'))
 
         self._unmock()
@@ -330,7 +348,8 @@ class TestMicrodotAsync(unittest.TestCase):
         app.run()
         self.assertTrue(fd.response.startswith(b'HTTP/1.0 200 OK\r\n'))
         self.assertIn(b'Content-Length: 3\r\n', fd.response)
-        self.assertIn(b'Content-Type: text/plain\r\n', fd.response)
+        self.assertIn(b'Content-Type: text/plain; charset=UTF-8\r\n',
+                      fd.response)
         self.assertTrue(fd.response.endswith(b'\r\n\r\n400'))
 
         self._unmock()
@@ -345,7 +364,8 @@ class TestMicrodotAsync(unittest.TestCase):
         client = TestClient(app)
         res = self._run(client.post('/foo'))
         self.assertEqual(res.status_code, 404)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, 'Not found')
 
     def test_404_handler(self):
@@ -362,7 +382,8 @@ class TestMicrodotAsync(unittest.TestCase):
         client = TestClient(app)
         res = self._run(client.post('/foo'))
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, '404')
 
     def test_405(self):
@@ -375,7 +396,8 @@ class TestMicrodotAsync(unittest.TestCase):
         client = TestClient(app)
         res = self._run(client.post('/foo'))
         self.assertEqual(res.status_code, 405)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, 'Not found')
 
     def test_405_handler(self):
@@ -392,7 +414,8 @@ class TestMicrodotAsync(unittest.TestCase):
         client = TestClient(app)
         res = self._run(client.patch('/foo'))
         self.assertEqual(res.status_code, 405)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, '405')
 
     def test_413(self):
@@ -405,7 +428,8 @@ class TestMicrodotAsync(unittest.TestCase):
         client = TestClient(app)
         res = self._run(client.post('/foo', body='x' * 17000))
         self.assertEqual(res.status_code, 413)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, 'Payload too large')
 
     def test_413_handler(self):
@@ -422,7 +446,8 @@ class TestMicrodotAsync(unittest.TestCase):
         client = TestClient(app)
         res = self._run(client.post('/foo', body='x' * 17000))
         self.assertEqual(res.status_code, 400)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, '413')
 
     def test_500(self):
@@ -435,7 +460,8 @@ class TestMicrodotAsync(unittest.TestCase):
         client = TestClient(app)
         res = self._run(client.get('/'))
         self.assertEqual(res.status_code, 500)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, 'Internal server error')
 
     def test_500_handler(self):
@@ -452,7 +478,8 @@ class TestMicrodotAsync(unittest.TestCase):
         client = TestClient(app)
         res = self._run(client.get('/'))
         self.assertEqual(res.status_code, 501)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, '501')
 
     def test_exception_handler(self):
@@ -469,7 +496,8 @@ class TestMicrodotAsync(unittest.TestCase):
         client = TestClient(app)
         res = self._run(client.get('/'))
         self.assertEqual(res.status_code, 501)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, '501')
 
     def test_abort(self):
@@ -483,7 +511,8 @@ class TestMicrodotAsync(unittest.TestCase):
         client = TestClient(app)
         res = self._run(client.get('/'))
         self.assertEqual(res.status_code, 406)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, 'Not acceptable')
 
     def test_abort_handler(self):
@@ -501,7 +530,8 @@ class TestMicrodotAsync(unittest.TestCase):
         client = TestClient(app)
         res = self._run(client.get('/'))
         self.assertEqual(res.status_code, 406)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, '406')
 
     def test_json_response(self):
@@ -519,12 +549,14 @@ class TestMicrodotAsync(unittest.TestCase):
 
         res = self._run(client.get('/dict'))
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'application/json')
+        self.assertEqual(res.headers['Content-Type'],
+                         'application/json; charset=UTF-8')
         self.assertEqual(res.json, {'foo': 'bar'})
 
         res = self._run(client.get('/list'))
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'application/json')
+        self.assertEqual(res.headers['Content-Type'],
+                         'application/json; charset=UTF-8')
         self.assertEqual(res.json, ['foo', 'bar'])
 
     def test_binary_response(self):
@@ -568,7 +600,8 @@ class TestMicrodotAsync(unittest.TestCase):
         client = TestClient(app)
         res = self._run(client.get('/'))
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers['Content-Type'], 'text/plain')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
         self.assertEqual(res.text, 'foobar')
 
     def test_already_handled_response(self):

tests/test_microdot_wsgi.py

@@ -54,7 +54,7 @@ class TestMicrodotWSGI(unittest.TestCase):
         def start_response(status, headers):
             self.assertEqual(status, '200 OK')
             expected_headers = [('Content-Length', '8'),
-                                ('Content-Type', 'text/plain'),
+                                ('Content-Type', 'text/plain; charset=UTF-8'),
                                 ('Set-Cookie', 'foo=foo'),
                                 ('Set-Cookie', 'bar=bar; HttpOnly')]
             self.assertEqual(len(headers), len(expected_headers))

tests/test_response.py

@@ -20,7 +20,7 @@ class TestResponse(unittest.TestCase):
         response = fd.getvalue()
         self.assertIn(b'HTTP/1.0 200 OK\r\n', response)
         self.assertIn(b'Content-Length: 3\r\n', response)
-        self.assertIn(b'Content-Type: text/plain\r\n', response)
+        self.assertIn(b'Content-Type: text/plain; charset=UTF-8\r\n', response)
         self.assertTrue(response.endswith(b'\r\n\r\nfoo'))
 
     def test_create_from_string_with_content_length(self):
@@ -33,7 +33,7 @@ class TestResponse(unittest.TestCase):
         response = fd.getvalue()
         self.assertIn(b'HTTP/1.0 200 OK\r\n', response)
         self.assertIn(b'Content-Length: 2\r\n', response)
-        self.assertIn(b'Content-Type: text/plain\r\n', response)
+        self.assertIn(b'Content-Type: text/plain; charset=UTF-8\r\n', response)
         self.assertTrue(response.endswith(b'\r\n\r\nfoo'))
 
     def test_create_from_bytes(self):
@@ -46,7 +46,7 @@ class TestResponse(unittest.TestCase):
         response = fd.getvalue()
         self.assertIn(b'HTTP/1.0 200 OK\r\n', response)
         self.assertIn(b'Content-Length: 3\r\n', response)
-        self.assertIn(b'Content-Type: text/plain\r\n', response)
+        self.assertIn(b'Content-Type: text/plain; charset=UTF-8\r\n', response)
         self.assertTrue(response.endswith(b'\r\n\r\nfoo'))
 
     def test_create_empty(self):
@@ -60,32 +60,36 @@ class TestResponse(unittest.TestCase):
         self.assertIn(b'HTTP/1.0 200 OK\r\n', response)
         self.assertIn(b'X-Foo: Bar\r\n', response)
         self.assertIn(b'Content-Length: 0\r\n', response)
-        self.assertIn(b'Content-Type: text/plain\r\n', response)
+        self.assertIn(b'Content-Type: text/plain; charset=UTF-8\r\n', response)
         self.assertTrue(response.endswith(b'\r\n\r\n'))
 
     def test_create_json(self):
         res = Response({'foo': 'bar'})
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers, {'Content-Type': 'application/json'})
+        self.assertEqual(res.headers,
+                         {'Content-Type': 'application/json; charset=UTF-8'})
         self.assertEqual(res.body, b'{"foo": "bar"}')
         fd = io.BytesIO()
         res.write(fd)
         response = fd.getvalue()
         self.assertIn(b'HTTP/1.0 200 OK\r\n', response)
         self.assertIn(b'Content-Length: 14\r\n', response)
-        self.assertIn(b'Content-Type: application/json\r\n', response)
+        self.assertIn(b'Content-Type: application/json; charset=UTF-8\r\n',
+                      response)
         self.assertTrue(response.endswith(b'\r\n\r\n{"foo": "bar"}'))
 
         res = Response([1, '2'])
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers, {'Content-Type': 'application/json'})
+        self.assertEqual(res.headers,
+                         {'Content-Type': 'application/json; charset=UTF-8'})
         self.assertEqual(res.body, b'[1, "2"]')
         fd = io.BytesIO()
         res.write(fd)
         response = fd.getvalue()
         self.assertIn(b'HTTP/1.0 200 OK\r\n', response)
         self.assertIn(b'Content-Length: 8\r\n', response)
-        self.assertIn(b'Content-Type: application/json\r\n', response)
+        self.assertIn(b'Content-Type: application/json; charset=UTF-8\r\n',
+                      response)
         self.assertTrue(response.endswith(b'\r\n\r\n[1, "2"]'))
 
     def test_create_from_none(self):
@@ -97,7 +101,7 @@ class TestResponse(unittest.TestCase):
         response = fd.getvalue()
         self.assertIn(b'HTTP/1.0 204 N/A\r\n', response)
         self.assertIn(b'Content-Length: 0\r\n', response)
-        self.assertIn(b'Content-Type: text/plain\r\n', response)
+        self.assertIn(b'Content-Type: text/plain; charset=UTF-8\r\n', response)
         self.assertTrue(response.endswith(b'\r\n\r\n'))
 
     def test_create_from_other(self):
@@ -230,3 +234,18 @@ class TestResponse(unittest.TestCase):
             response,
             b'HTTP/1.0 200 OK\r\nContent-Type: text/html\r\n\r\nfoo\n')
         Response.send_file_buffer_size = original_buffer_size
+
+    def test_default_content_type(self):
+        original_content_type = Response.default_content_type
+        res = Response('foo')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/plain; charset=UTF-8')
+        Response.default_content_type = 'text/html'
+        res = Response('foo')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/html; charset=UTF-8')
+        Response.default_content_type = 'text/html; charset=ISO-8859-1'
+        res = Response('foo')
+        self.assertEqual(res.headers['Content-Type'],
+                         'text/html; charset=ISO-8859-1')
+        Response.default_content_type = original_content_type

tests/test_response_asyncio.py

@@ -22,7 +22,8 @@ class TestResponseAsync(unittest.TestCase):
         _run(res.write(fd))
         self.assertIn(b'HTTP/1.0 200 OK\r\n', fd.response)
         self.assertIn(b'Content-Length: 3\r\n', fd.response)
-        self.assertIn(b'Content-Type: text/plain\r\n', fd.response)
+        self.assertIn(b'Content-Type: text/plain; charset=UTF-8\r\n',
+                      fd.response)
         self.assertTrue(fd.response.endswith(b'\r\n\r\nfoo'))
 
     def test_create_from_string_with_content_length(self):
@@ -34,7 +35,8 @@ class TestResponseAsync(unittest.TestCase):
         _run(res.write(fd))
         self.assertIn(b'HTTP/1.0 200 OK\r\n', fd.response)
         self.assertIn(b'Content-Length: 2\r\n', fd.response)
-        self.assertIn(b'Content-Type: text/plain\r\n', fd.response)
+        self.assertIn(b'Content-Type: text/plain; charset=UTF-8\r\n',
+                      fd.response)
         self.assertTrue(fd.response.endswith(b'\r\n\r\nfoo'))
 
     def test_create_from_bytes(self):
@@ -46,7 +48,8 @@ class TestResponseAsync(unittest.TestCase):
         _run(res.write(fd))
         self.assertIn(b'HTTP/1.0 200 OK\r\n', fd.response)
         self.assertIn(b'Content-Length: 3\r\n', fd.response)
-        self.assertIn(b'Content-Type: text/plain\r\n', fd.response)
+        self.assertIn(b'Content-Type: text/plain; charset=UTF-8\r\n',
+                      fd.response)
         self.assertTrue(fd.response.endswith(b'\r\n\r\nfoo'))
 
     def test_create_empty(self):
@@ -59,30 +62,35 @@ class TestResponseAsync(unittest.TestCase):
         self.assertIn(b'HTTP/1.0 200 OK\r\n', fd.response)
         self.assertIn(b'X-Foo: Bar\r\n', fd.response)
         self.assertIn(b'Content-Length: 0\r\n', fd.response)
-        self.assertIn(b'Content-Type: text/plain\r\n', fd.response)
+        self.assertIn(b'Content-Type: text/plain; charset=UTF-8\r\n',
+                      fd.response)
         self.assertTrue(fd.response.endswith(b'\r\n\r\n'))
 
     def test_create_json(self):
         res = Response({'foo': 'bar'})
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers, {'Content-Type': 'application/json'})
+        self.assertEqual(res.headers,
+                         {'Content-Type': 'application/json; charset=UTF-8'})
         self.assertEqual(res.body, b'{"foo": "bar"}')
         fd = FakeStreamAsync()
         _run(res.write(fd))
         self.assertIn(b'HTTP/1.0 200 OK\r\n', fd.response)
         self.assertIn(b'Content-Length: 14\r\n', fd.response)
-        self.assertIn(b'Content-Type: application/json\r\n', fd.response)
+        self.assertIn(b'Content-Type: application/json; charset=UTF-8\r\n',
+                      fd.response)
         self.assertTrue(fd.response.endswith(b'\r\n\r\n{"foo": "bar"}'))
 
         res = Response([1, '2'])
         self.assertEqual(res.status_code, 200)
-        self.assertEqual(res.headers, {'Content-Type': 'application/json'})
+        self.assertEqual(res.headers,
+                         {'Content-Type': 'application/json; charset=UTF-8'})
         self.assertEqual(res.body, b'[1, "2"]')
         fd = FakeStreamAsync()
         _run(res.write(fd))
         self.assertIn(b'HTTP/1.0 200 OK\r\n', fd.response)
         self.assertIn(b'Content-Length: 8\r\n', fd.response)
-        self.assertIn(b'Content-Type: application/json\r\n', fd.response)
+        self.assertIn(b'Content-Type: application/json; charset=UTF-8\r\n',
+                      fd.response)
         self.assertTrue(fd.response.endswith(b'\r\n\r\n[1, "2"]'))
 
     def test_create_with_reason(self):