feat: support alternate request charset (@W-14001583) (#131)

* add test for utf-16 request body encoding

* handle other charsets for json request body

* add logging test for mimer errors

* log invalid mime translations

* changelog + version bump

* add request body to mimer error logs

Author: dshafer

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.29.0
+current_version = 0.30.0
 
 [bumpversion:file:pyproject.toml]
 

CHANGELOG.md

@@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+# [0.30.0] - 2023-08-23
+- [PR 131](https://github.com/salesforce/django-declarative-apis/pull/131) Support non-utf8 request body
+
 # [0.29.0] - 2023-08-16
 - [PR 129](https://github.com/salesforce/django-declarative-apis/pull/129) Remove outdated HttpResponse wrapper 
 

django_declarative_apis/resources/utils.py

@@ -5,14 +5,19 @@
 # For full license text, see the LICENSE file in the repo root or https://opensource.org/licenses/BSD-3-Clause
 #
 
+import logging
 import warnings
 from pydoc import locate
 
 from decorator import decorator
 from django import get_version as django_version
+from django.conf import settings
 from django.http import HttpResponse
 
 
+logger = logging.getLogger(__name__)
+
+
 def format_error(error):
     return "Django Declarative APIs (Django %s) crash report:\n\n%s" % (
         django_version(),
@@ -189,15 +194,30 @@ def translate(self):
             if loadee:
                 try:
                     data = self.request.body
+                    charset = self.request.encoding or getattr(
+                        settings, "DEFAULT_CHARSET", "utf-8"
+                    )
                     # PY3: Loaders usually don't work with bytes:
-                    data = data.decode("utf-8")
+                    data = data.decode(charset)
                     self.request.data = loadee(data)
 
                     # Reset both POST and PUT from request, as its
                     # misleading having their presence around.
                     self.request.POST = self.request.PUT = dict()
                 except (TypeError, ValueError):
                     # This also catches if loadee is None.
+                    log_mimer_data_exception = getattr(
+                        settings, "DDA_LOG_MIMER_DATA_EXCEPTION", False
+                    )
+                    if log_mimer_data_exception:
+                        # using the exception logger should give a better hint of what exactly went wrong
+                        logger.exception(
+                            'ev=dda_mime_data_exception, content_type="%s", body="%s"',
+                            self.request.headers.get(
+                                "content-type", "missing content type header!"
+                            ),
+                            self.request.body,
+                        )
                     raise MimerDataException
             else:
                 self.request.data = None

docs/source/conf.py

@@ -74,7 +74,7 @@
 # built documents.
 
 # The full version, including alpha/beta/rc tags.
-release = '0.29.0'  # set by bumpversion
+release = '0.30.0'  # set by bumpversion
 
 # The short X.Y version.
 version = release.rsplit('.', 1)[0]

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "django-declarative-apis"
-version = "0.29.0"  # set by bumpversion
+version = "0.30.0"  # set by bumpversion
 description = "Simple, readable, declarative APIs for Django"
 readme = "README.md"
 dependencies = [

tests/resources/test_resource.py

@@ -11,6 +11,7 @@
 import django.conf
 import django.core.exceptions
 import django.test
+from django.test.utils import override_settings
 from unittest import mock
 
 from django_declarative_apis.authentication.oauthlib import oauth_errors
@@ -53,6 +54,49 @@ class Handler:
             resource_instance = res(req)
             self.assertEqual(resource_instance.content, b"Bad Request")
 
+    def test_call_alternate_charset(self):
+        class Handler:
+            allowed_methods = ("POST",)
+            method_handlers = {
+                "POST": lambda req, *args, **kwargs: (http.HTTPStatus.OK, "")
+            }
+
+        body = {"foo": "bar"}
+        req = self.create_request(
+            method="POST",
+            body=body,
+            content_type="application/json; charset=utf-16",
+            use_auth_header_signature=True,
+        )
+
+        res = resource.Resource(lambda: Handler())
+        resource_instance = res(req)
+        self.assertEqual(200, resource_instance.status_code)
+
+    def test_call_invalid_charset(self):
+        class Handler:
+            allowed_methods = ("POST",)
+            method_handlers = {
+                "POST": lambda req, *args, **kwargs: (http.HTTPStatus.OK, "")
+            }
+
+        body = {"foo": "bar"}
+        req = self.create_request(
+            method="POST",
+            body=body,
+            content_type="application/json; charset=utf-16",
+            use_auth_header_signature=True,
+        )
+        req.encoding = "utf-8"
+        res = resource.Resource(lambda: Handler())
+        with override_settings(DDA_LOG_MIMER_DATA_EXCEPTION=True):
+            with self.assertLogs("django_declarative_apis.resources.utils") as logs:
+                resource_instance = res(req)
+                self.assertTrue(
+                    any(["ev=dda_mime_data_exception" in o for o in logs.output])
+                )
+        self.assertEqual(400, resource_instance.status_code)
+
     def test_call_put(self):
         class Handler:
             allowed_methods = ("PUT",)

tests/testutils.py

@@ -28,6 +28,7 @@
 _ENCODERS = {
     DEFAULT_CONTENT_TYPE: lambda data: urllib.parse.urlencode(data),
     "application/json": lambda data: json.dumps(data),
+    "application/json; charset=utf-16": lambda data: json.dumps(data),
 }