Skip to content

Commit

Permalink
fix: use correct anon key for s3 and azure
Browse files Browse the repository at this point in the history
Signed-off-by: machichima <nary12321@gmail.com>
  • Loading branch information
machichima committed Jan 8, 2025
1 parent 0ca6dbc commit 42cc75f
Showing 1 changed file with 6 additions and 8 deletions.
14 changes: 6 additions & 8 deletions flytekit/core/data_persistence.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,11 @@
from flytekit.loggers import logger
from flytekit.utils.asyn import loop_manager

# Refer to https://github.com/fsspec/s3fs/blob/50bafe4d8766c3b2a4e1fc09669cf02fb2d71454/s3fs/core.py#L198
# Refer to https://github.com/developmentseed/obstore/blob/33654fc37f19a657689eb93327b621e9f9e01494/obstore/python/obstore/store/_aws.pyi#L11
# for key and secret
_FSSPEC_S3_KEY_ID = "access_key_id"
_FSSPEC_S3_SECRET = "secret_access_key"
_ANON = "anon"
_ANON = "skip_signature"

Uploadable = typing.Union[str, os.PathLike, pathlib.Path, bytes, io.BufferedReader, io.BytesIO, io.StringIO]

Expand All @@ -68,7 +68,8 @@ def s3_setup_args(s3_cfg: configuration.S3Config, bucket: str = "", anonymous: b
# S3fs takes this as a special arg
if s3_cfg.endpoint is not None:
store_kwargs["endpoint_url"] = s3_cfg.endpoint
# kwargs["client_kwargs"] = {"endpoint_url": s3_cfg.endpoint}
if anonymous:
store_kwargs[_ANON] = "true"

store = S3Store.from_env(
bucket,
Expand All @@ -81,9 +82,6 @@ def s3_setup_args(s3_cfg: configuration.S3Config, bucket: str = "", anonymous: b

kwargs["retries"] = s3_cfg.retries

if anonymous:
kwargs[_ANON] = True

kwargs["store"] = store

return kwargs
Expand Down Expand Up @@ -157,6 +155,8 @@ def azure_setup_args(
store_kwargs["client_secret"] = azure_cfg.client_secret
if azure_cfg.tenant_id:
store_kwargs["tenant_id"] = azure_cfg.tenant_id
if anonymous:
kwargs[_ANON] = "true"

store = AzureStore.from_env(
container,
Expand All @@ -167,8 +167,6 @@ def azure_setup_args(

kwargs["store"] = store

if anonymous:
kwargs[_ANON] = True

return kwargs

Expand Down

0 comments on commit 42cc75f

Please sign in to comment.