Skip to content

DNM: External backends for stores #11657

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 78 additions & 0 deletions localstack-core/localstack/services/stores.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,14 +34,92 @@ class SqsStore(BaseStore):
from threading import RLock
from typing import Any, Generic, Iterator, Type, TypeVar, Union

import redis

from localstack import config
from localstack.utils.aws.arns import parse_arn
from localstack.utils.aws.aws_stack import get_valid_regions_for_service

LOCAL_ATTR_PREFIX = "attr_"

BaseStoreType = TypeVar("BaseStoreType")


#
# Hackathon
#


class SuperRedisDict(dict):
def __init__(
self,
account_id: str,
region_name: str,
hashmap_name: str,
arn_builder: Callable,
serializer: Callable,
deserializer: Callable,
):
self.redis = redis.StrictRedis(host="localhost", port=6379, db=0)

self.account_id = account_id
self.region_name = region_name
self.hashmap_name = hashmap_name

self.arn_builder = arn_builder
self.serializer = serializer
self.deserializer = deserializer

def __getitem__(self, key) -> Any:
arn = self.arn_builder(key, self.account_id, self.region_name)
obj: bytes | None = self.redis.hget(self.hashmap_name, arn)
if obj is not None:
return self.deserializer(obj)

def __setitem__(self, key, value) -> Any:
ser: bytes = self.serializer(value)
arn = self.arn_builder(key, self.account_id, self.region_name)
self.redis.hset(self.hashmap_name, arn, ser)

def keys(self):
keylist = []
for arn in self.redis.hkeys(self.hashmap_name):
arn_data = parse_arn(arn)
name = arn_data["resource"].removeprefix(f"{self.hashmap_name}/")
keylist.append(name)

return keylist

def values(self):
valuelist = []
for ser in self.redis.hvals(self.hashmap_name):
obj = self.deserializer(ser)
valuelist.append(obj)

return valuelist


class LocalRedisResource:
def __init__(
self, name: str, arn_builder: Callable, serializer: Callable, deserializer: Callable
):
self.name = name
self.arn_builder = arn_builder

self.serializer = serializer
self.deserializer = deserializer

def __get__(self, obj: BaseStoreType, objtype=None) -> Any:
return SuperRedisDict(
obj._account_id,
obj._region_name,
self.name,
self.arn_builder,
self.serializer,
self.deserializer,
)


#
# Descriptor protocol classes
#
Expand Down
14 changes: 11 additions & 3 deletions localstack-core/localstack/services/transcribe/models.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,17 @@
from localstack.aws.api.transcribe import TranscriptionJob, TranscriptionJobName
from localstack.services.stores import AccountRegionBundle, BaseStore, LocalAttribute
import json

from localstack.services.stores import AccountRegionBundle, BaseStore, LocalRedisResource
from localstack.utils.aws.arns import transcribe_transcription_job_arn
from localstack.utils.json import CustomEncoder


class TranscribeStore(BaseStore):
transcription_jobs: dict[TranscriptionJobName, TranscriptionJob] = LocalAttribute(default=dict)
transcription_jobs = LocalRedisResource(
name="transcription-job",
arn_builder=transcribe_transcription_job_arn,
serializer=lambda obj: json.dumps(obj, cls=CustomEncoder),
deserializer=json.loads,
)


transcribe_stores = AccountRegionBundle("transcribe", TranscribeStore)
10 changes: 10 additions & 0 deletions localstack-core/localstack/utils/aws/arns.py
Original file line number Diff line number Diff line change
Expand Up @@ -503,6 +503,16 @@ def route53_resolver_query_log_config_arn(id: str, account_id: str, region_name:
return _resource_arn(id, pattern, account_id=account_id, region_name=region_name)


#
# Transcribe
#


def transcribe_transcription_job_arn(id: str, account_id: str, region_name: str) -> str:
pattern = "arn:%s:transcribe:%s:%s:transcription-job/%s"
return _resource_arn(id, pattern, account_id=account_id, region_name=region_name)


#
# Other ARN related helpers
#
Expand Down
Loading