Skip to content

Introduce jsonpickle serializers #12875

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,15 @@
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
# Ruff version.
rev: v0.12.3
rev: v0.12.4
hooks:
- id: ruff
args: [--fix, --exit-non-zero-on-fix]
# Run the formatter.
- id: ruff-format

- repo: https://github.com/pre-commit/mirrors-mypy
rev: v1.16.1
rev: v1.17.0
hooks:
- id: mypy
entry: bash -c 'cd localstack-core && mypy --install-types --non-interactive'
Expand Down
4 changes: 4 additions & 0 deletions localstack-core/localstack/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -1254,6 +1254,9 @@ def use_custom_dns():
# This flag enables all responses from LocalStack to contain a `x-localstack` HTTP header.
LOCALSTACK_RESPONSE_HEADER_ENABLED = is_env_not_false("LOCALSTACK_RESPONSE_HEADER_ENABLED")

# Serialization backend for LocalStack state (stores and moto's backend dicts). Can be either `dill` or `json` (experimental)
STATE_SERIALIZATION_BACKEND = os.environ.get("STATE_SERIALIZATION_BACKEND", "").strip() or "dill"

# List of environment variable names used for configuration that are passed from the host into the LocalStack container.
# => Synchronize this list with the above and the configuration docs:
# https://docs.localstack.cloud/references/configuration/
Expand Down Expand Up @@ -1390,6 +1393,7 @@ def use_custom_dns():
"SQS_ENDPOINT_STRATEGY",
"SQS_DISABLE_CLOUDWATCH_METRICS",
"SQS_CLOUDWATCH_METRICS_REPORT_INTERVAL",
"STATE_SERIALIZATION_BACKEND",
"STRICT_SERVICE_LOADING",
"TF_COMPAT_MODE",
"USE_SSL",
Expand Down
68 changes: 68 additions & 0 deletions localstack-core/localstack/state/json.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
from typing import IO, Any, Callable, Type, TypeVar

import jsonpickle
from jsonpickle.handlers import BaseHandler

from localstack.state import Decoder, Encoder

T = TypeVar("T", bound=BaseHandler)


def jsonpickle_register(cls: Type = None) -> Callable[[Type[T]], Type[T]]:
"""
Decorator to register a custom handler for jsonpickle serialization.
It provides a clean way to register custom jsonpickle handlers.
:param cls: the type to register the handler for
:raise ValueError: if the handler class does not extend handlers.BaseHandler
:return:
Example::
@jsonpickle_register(MyObject)
class MyObjectHandler(handlers.BaseHandler):
def flatten(self, obj: MyObject, data: dict) -> dict:
# ...
return data
def restore(self, data: dict) -> MyObject:
# ...
return MyObject(...)
"""

def _wrapper(handler_class):
if not issubclass(handler_class, BaseHandler):
raise ValueError(f"Cannot register {handler_class}")

jsonpickle.handlers.register(cls, handler_class)
return handler_class

return _wrapper


class JsonEncoder(Encoder):
"""
An Encoder that uses ``jsonpickle`` under the hood.
"""

def __init__(self, pickler_class: Type[jsonpickle.Pickler] = None):
self.pickler_class = pickler_class or jsonpickle.Pickler()

def encode(self, obj: Any, file: IO[bytes]):
json_str = jsonpickle.encode(obj, context=self.pickler_class)
file.write(json_str.encode("utf-8"))


class JsonDecoder(Decoder):
"""
A Decoder that uses ``jsonpickle`` under the hood.
"""

unpickler_class: Type[jsonpickle.Unpickler]

def __init__(self, unpickler_class: Type[jsonpickle.Unpickler] = None):
self.unpickler_class = unpickler_class or jsonpickle.Unpickler()

def decode(self, file: IO[bytes]) -> Any:
json_str = file.read().decode("utf-8")
return jsonpickle.decode(json_str, context=self.unpickler_class)
43 changes: 31 additions & 12 deletions localstack-core/localstack/state/pickle.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,10 @@ def _recreate(obj_type, obj_queue):
import dill
from dill._dill import MetaCatchingDict

from localstack import config

from .core import Decoder, Encoder
from .json import JsonDecoder, JsonEncoder

_T = TypeVar("_T")

Expand Down Expand Up @@ -134,44 +137,52 @@ def remove_dispatch_entry(cls: Type):
pass


def dumps(obj: Any) -> bytes:
def dumps(obj: Any, encoder: Encoder | None = None) -> bytes:
"""
Pickle an object into bytes using a ``PickleEncoder``.
Pickle an object into bytes using an ``Encoder``.
:param obj: the object to pickle
:param encoder: the encoder
:return: the pickled object
"""
return PickleEncoder().encodes(obj)
encoder = encoder or _default_encoder
return encoder.encodes(obj)


def dump(obj: Any, file: BinaryIO):
def dump(obj: Any, file: BinaryIO, encoder: Encoder | None = None):
"""
Pickle an object into a buffer using a ``PickleEncoder``.
Pickle an object into a buffer using an ``Encoder``.
:param obj: the object to pickle
:param encoder: the encoder
:param file: the IO buffer
"""
return PickleEncoder().encode(obj, file)
encoder = encoder or _default_encoder
return encoder.encode(obj, file)


def loads(data: bytes) -> Any:
def loads(data: bytes, decoder: Decoder | None = None) -> Any:
"""
Unpickle am object from bytes using a ``PickleDecoder``.
Unpickle am object from bytes using a ``Decoder``.
:param data: the pickled object
:param decoder: the decoder
:return: the unpickled object
"""
return PickleDecoder().decodes(data)
decoder = decoder or _default_decoder
return decoder.decodes(data)


def load(file: BinaryIO) -> Any:
def load(file: BinaryIO, decoder: Decoder | None = None) -> Any:
"""
Unpickle am object from a buffer using a ``PickleDecoder``.
Unpickle am object from a buffer using a ``Decoder``.
:param file: the buffer containing the pickled object
:param decoder: the decoder
:return: the unpickled object
"""
return PickleDecoder().decode(file)
decoder = decoder or _default_decoder
return decoder.decode(file)


class _SuperclassMatchingTypeDict(MetaCatchingDict):
Expand Down Expand Up @@ -255,6 +266,14 @@ def decode(self, file: BinaryIO) -> Any:
return self.unpickler_class(file).load()


_default_encoder = (
PickleEncoder() if config.STATE_SERIALIZATION_BACKEND == "dill" else JsonEncoder()
)
_default_decoder = (
PickleDecoder() if config.STATE_SERIALIZATION_BACKEND == "dill" else JsonDecoder()
)


class ObjectStateReducer(Generic[_T]):
"""
A generalization of the following pattern::
Expand Down
14 changes: 14 additions & 0 deletions localstack-core/localstack/testing/pytest/fixtures.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@
CrossRegionAttribute,
LocalAttribute,
)
from localstack.state import pickle
from localstack.state.json import JsonDecoder, JsonEncoder
from localstack.state.pickle import PickleDecoder, PickleEncoder
from localstack.testing.aws.cloudformation_utils import load_template_file, render_template
from localstack.testing.aws.util import get_lambda_logs, is_aws_cloud
from localstack.testing.config import (
Expand Down Expand Up @@ -2616,3 +2619,14 @@ def _delete_log_group():
call_safe(_delete_log_group)

yield _clean_up


@pytest.fixture(
params=[(PickleEncoder(), PickleDecoder()), (JsonEncoder(), JsonDecoder())],
ids=["dill", "jsonpickle"],
)
def patch_default_encoder(request, monkeypatch):
encoder, decoder = request.param
monkeypatch.setattr(pickle, "_default_encoder", encoder)
monkeypatch.setattr(pickle, "_default_decoder", decoder)
return encoder, decoder
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ base-runtime = [
"dnspython>=1.16.0",
"docker>=6.1.1",
"jsonpatch>=1.24",
"jsonpickle==4.1.1",
"hypercorn>=0.14.4",
"localstack-twisted>=23.0",
"openapi-core>=0.19.2",
Expand Down
4 changes: 3 additions & 1 deletion requirements-base-runtime.txt
Original file line number Diff line number Diff line change
Expand Up @@ -77,9 +77,11 @@ jmespath==1.0.1
# botocore
jsonpatch==1.33
# via localstack-core (pyproject.toml)
jsonpickle==4.1.1
# via localstack-core (pyproject.toml)
jsonpointer==3.0.0
# via jsonpatch
jsonschema==4.24.0
jsonschema==4.24.1
# via
# openapi-core
# openapi-schema-validator
Expand Down
12 changes: 8 additions & 4 deletions requirements-dev.txt
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ aws-cdk-asset-node-proxy-agent-v6==2.1.0
# via aws-cdk-lib
aws-cdk-cloud-assembly-schema==45.2.0
# via aws-cdk-lib
aws-cdk-lib==2.204.0
aws-cdk-lib==2.206.0
# via localstack-core
aws-sam-translator==1.99.0
# via
Expand Down Expand Up @@ -121,7 +121,7 @@ dill==0.3.6
# via
# localstack-core
# localstack-core (pyproject.toml)
distlib==0.3.9
distlib==0.4.0
# via virtualenv
dnslib==0.9.26
# via
Expand Down Expand Up @@ -215,9 +215,13 @@ jsonpath-ng==1.7.0
# moto-ext
jsonpath-rw==1.4.0
# via localstack-core
jsonpickle==4.1.1
# via
# aws-xray-sdk
# localstack-core
jsonpointer==3.0.0
# via jsonpatch
jsonschema==4.24.0
jsonschema==4.24.1
# via
# aws-sam-translator
# moto-ext
Expand Down Expand Up @@ -433,7 +437,7 @@ rsa==4.7.2
# via awscli
rstr==3.2.2
# via localstack-core (pyproject.toml)
ruff==0.12.3
ruff==0.12.4
# via localstack-core (pyproject.toml)
s3transfer==0.13.0
# via
Expand Down
4 changes: 3 additions & 1 deletion requirements-runtime.txt
Original file line number Diff line number Diff line change
Expand Up @@ -155,9 +155,11 @@ jsonpath-ng==1.7.0
# moto-ext
jsonpath-rw==1.4.0
# via localstack-core (pyproject.toml)
jsonpickle==4.1.1
# via localstack-core
jsonpointer==3.0.0
# via jsonpatch
jsonschema==4.24.0
jsonschema==4.24.1
# via
# aws-sam-translator
# moto-ext
Expand Down
8 changes: 6 additions & 2 deletions requirements-test.txt
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ aws-cdk-asset-node-proxy-agent-v6==2.1.0
# via aws-cdk-lib
aws-cdk-cloud-assembly-schema==45.2.0
# via aws-cdk-lib
aws-cdk-lib==2.204.0
aws-cdk-lib==2.206.0
# via localstack-core (pyproject.toml)
aws-sam-translator==1.99.0
# via
Expand Down Expand Up @@ -199,9 +199,13 @@ jsonpath-ng==1.7.0
# moto-ext
jsonpath-rw==1.4.0
# via localstack-core
jsonpickle==4.1.1
# via
# aws-xray-sdk
# localstack-core
jsonpointer==3.0.0
# via jsonpatch
jsonschema==4.24.0
jsonschema==4.24.1
# via
# aws-sam-translator
# moto-ext
Expand Down
Loading
Loading