From: anish-yen (@anish-yen) <noreply+anish-yen@github.com>
To: psycopg/psycopg <noreply+psycopg-psycopg@github.com>
Subject: [psycopg/psycopg] PR #1505: Decode JSON data using the connection encoding
Date: Thu, 19 Nov 2026 17:37:38 +0000
Message-ID: <gh-psycopg-psycopg-1505@github.com> (raw)
Fix the `json` and `jsonb` loaders to decode server data using the connection encoding instead of assuming UTF-8.
When `client_encoding` is not UTF-8 (e.g. a `LATIN1` database), the server sends JSON text in that encoding, but `_JsonLoader.load()` passed the raw bytes to `json.loads()`, which only auto-detects UTF-8/16/32. Any non-ASCII character in a JSON value then raised `UnicodeDecodeError`, while casting the same value to `text` loaded fine.
`_JsonLoader` now records the connection encoding the same way `TextLoader` does and decodes explicitly when it is not UTF-8/ASCII, keeping the bytes fast path for UTF-8 connections. The same handling applies to `JsonbBinaryLoader`, whose payload the server converts to the client encoding as well.
Fixes #1504. Thanks @fppfurtado for the precise report and the suggested approach.
Regression test: `test_load_non_utf8_encoding` covers `json`/`jsonb` in text and binary formats under `client_encoding=latin1`. Verified locally against the loaders with a LATIN1 connection encoding (the new behavior fails with `UnicodeDecodeError` on the unfixed code and passes with the fix); flake8/black/mypy clean.
This contribution was developed with AI assistance and verified by the author.
inline patch 1: psycopg/psycopg/types/json.py, tests/types/test_json.py (+25/-4 lines) download inline-1.patchdiff --git a/psycopg/psycopg/types/json.py b/psycopg/psycopg/types/json.pyindex 7ec732a6a..f69c923ad 100644--- a/psycopg/psycopg/types/json.py+++ b/psycopg/psycopg/types/json.py@@ -18,6 +18,7 @@
from ..pq import Format
from ..adapt import AdaptersMap, Buffer, Dumper, Loader, PyFormat
from ..errors import DataError
+from .._encodings import conn_encoding
JsonDumpsFunction: TypeAlias = Callable[[Any], str | bytes]
JsonLoadsFunction: TypeAlias = Callable[[str | bytes], Any]
@@ -250,12 +251,22 @@ class _JsonLoader(Loader):
def __init__(self, oid: int, context: abc.AdaptContext | None = None):
super().__init__(oid, context)
self.loads = self.__class__._loads
+ enc = conn_encoding(self.connection)+ # json.loads() can auto-detect the UTF encodings from the input+ # bytes; for other connection encodings the data must be decoded+ # explicitly (#1504).+ self._encoding = "" if enc in ("utf-8", "ascii") else enc
def load(self, data: Buffer) -> Any:
+ return self.loads(self._decode(data))++ def _decode(self, data: Buffer) -> str | bytes:
# json.loads() cannot work on memoryview.
if not isinstance(data, bytes):
data = bytes(data)
- return self.loads(data)+ if self._encoding:+ return data.decode(self._encoding)+ return data
class JsonLoader(_JsonLoader):
@@ -276,9 +287,7 @@ class JsonbBinaryLoader(_JsonLoader):
def load(self, data: Buffer) -> Any:
if data and data[0] != 1:
raise DataError(f"unknown jsonb binary format: {data[0]}")
- if not isinstance((data := data[1:]), bytes):- data = bytes(data)- return self.loads(data)+ return self.loads(self._decode(data[1:]))
def _get_current_dumper(
diff --git a/tests/types/test_json.py b/tests/types/test_json.pyindex f649ebdf3..b49c78541 100644--- a/tests/types/test_json.py+++ b/tests/types/test_json.py@@ -127,6 +127,18 @@ def test_load_copy(conn, val, jtype, fmt_out):
assert got == json.loads(val)
+@pytest.mark.crdb_skip("encoding")+@pytest.mark.parametrize("jtype", ["json", "jsonb"])+@pytest.mark.parametrize("fmt_out", pq.Format)+def test_load_non_utf8_encoding(conn, jtype, fmt_out):+ # The loaders must decode the server data with the connection encoding,+ # not always as UTF-8 (#1504).+ cur = conn.cursor(binary=fmt_out)+ conn.execute("set client_encoding to latin1")+ cur.execute(f"select %s::{jtype}", ('{"a": "\xe9"}',))+ assert cur.fetchone()[0] == {"a": "\xe9"}++
@pytest.mark.parametrize("fmt_in", PyFormat)
@pytest.mark.parametrize("wrapper", ["Json", "Jsonb"])
def test_dump_customise(conn, wrapper, fmt_in):