Initial Commit
This commit is contained in:
@@ -0,0 +1,74 @@
|
||||
from .exceptions import IncompleteFieldDefinitionWarning, SettingsError
|
||||
from .main import BaseSettings, CliApp, SettingsConfigDict
|
||||
from .sources import (
|
||||
CLI_SUPPRESS,
|
||||
AWSSecretsManagerSettingsSource,
|
||||
AWSSystemsManagerSettingsSource,
|
||||
AzureKeyVaultSettingsSource,
|
||||
CliDualFlag,
|
||||
CliExplicitFlag,
|
||||
CliImplicitFlag,
|
||||
CliMutuallyExclusiveGroup,
|
||||
CliPositionalArg,
|
||||
CliSettingsSource,
|
||||
CliSubCommand,
|
||||
CliSuppress,
|
||||
CliToggleFlag,
|
||||
CliUnknownArgs,
|
||||
CliVariadicArg,
|
||||
DotEnvSettingsSource,
|
||||
EnvSettingsSource,
|
||||
ForceDecode,
|
||||
GoogleSecretManagerSettingsSource,
|
||||
InitSettingsSource,
|
||||
JsonConfigSettingsSource,
|
||||
NestedSecretsSettingsSource,
|
||||
NoDecode,
|
||||
PydanticBaseSettingsSource,
|
||||
PyprojectTomlConfigSettingsSource,
|
||||
SecretsSettingsSource,
|
||||
TomlConfigSettingsSource,
|
||||
YamlConfigSettingsSource,
|
||||
get_subcommand,
|
||||
)
|
||||
from .version import VERSION
|
||||
|
||||
__all__ = (
|
||||
'CLI_SUPPRESS',
|
||||
'AWSSecretsManagerSettingsSource',
|
||||
'AWSSystemsManagerSettingsSource',
|
||||
'AzureKeyVaultSettingsSource',
|
||||
'BaseSettings',
|
||||
'CliApp',
|
||||
'CliDualFlag',
|
||||
'CliExplicitFlag',
|
||||
'CliImplicitFlag',
|
||||
'CliMutuallyExclusiveGroup',
|
||||
'CliPositionalArg',
|
||||
'CliSettingsSource',
|
||||
'CliSubCommand',
|
||||
'CliSuppress',
|
||||
'CliToggleFlag',
|
||||
'CliUnknownArgs',
|
||||
'CliVariadicArg',
|
||||
'DotEnvSettingsSource',
|
||||
'EnvSettingsSource',
|
||||
'ForceDecode',
|
||||
'GoogleSecretManagerSettingsSource',
|
||||
'IncompleteFieldDefinitionWarning',
|
||||
'InitSettingsSource',
|
||||
'JsonConfigSettingsSource',
|
||||
'NestedSecretsSettingsSource',
|
||||
'NoDecode',
|
||||
'PydanticBaseSettingsSource',
|
||||
'PyprojectTomlConfigSettingsSource',
|
||||
'SecretsSettingsSource',
|
||||
'SettingsConfigDict',
|
||||
'SettingsError',
|
||||
'TomlConfigSettingsSource',
|
||||
'YamlConfigSettingsSource',
|
||||
'__version__',
|
||||
'get_subcommand',
|
||||
)
|
||||
|
||||
__version__ = VERSION
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,10 @@
|
||||
class SettingsError(ValueError):
|
||||
"""Base exception for settings-related errors."""
|
||||
|
||||
|
||||
class IncompleteFieldDefinitionWarning(UserWarning):
|
||||
"""Warning emitted when a field with an incomplete definition is used during settings resolution.
|
||||
|
||||
A field definition is incomplete when its annotation contains unresolved forward references,
|
||||
in which case settings sources may fail to correctly resolve its value.
|
||||
"""
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,91 @@
|
||||
"""Package for handling configuration sources in pydantic-settings."""
|
||||
|
||||
from .base import (
|
||||
ConfigFileSourceMixin,
|
||||
DefaultSettingsSource,
|
||||
InitSettingsSource,
|
||||
PydanticBaseEnvSettingsSource,
|
||||
PydanticBaseSettingsSource,
|
||||
get_subcommand,
|
||||
)
|
||||
from .providers.aws import AWSSecretsManagerSettingsSource, AWSSystemsManagerSettingsSource
|
||||
from .providers.azure import AzureKeyVaultSettingsSource
|
||||
from .providers.cli import (
|
||||
CLI_SUPPRESS,
|
||||
CliDualFlag,
|
||||
CliExplicitFlag,
|
||||
CliImplicitFlag,
|
||||
CliMutuallyExclusiveGroup,
|
||||
CliPositionalArg,
|
||||
CliSettingsSource,
|
||||
CliSubCommand,
|
||||
CliSuppress,
|
||||
CliToggleFlag,
|
||||
CliUnknownArgs,
|
||||
CliVariadicArg,
|
||||
)
|
||||
from .providers.dotenv import DotEnvSettingsSource, read_env_file
|
||||
from .providers.env import EnvSettingsSource
|
||||
from .providers.gcp import GoogleSecretManagerSettingsSource
|
||||
from .providers.json import JsonConfigSettingsSource
|
||||
from .providers.nested_secrets import NestedSecretsSettingsSource
|
||||
from .providers.pyproject import PyprojectTomlConfigSettingsSource
|
||||
from .providers.secrets import SecretsSettingsSource
|
||||
from .providers.toml import TomlConfigSettingsSource
|
||||
from .providers.yaml import YamlConfigSettingsSource
|
||||
from .types import (
|
||||
DEFAULT_PATH,
|
||||
ENV_FILE_SENTINEL,
|
||||
ConfigFileSourceType,
|
||||
DotenvFiltering,
|
||||
DotenvType,
|
||||
EnvPrefixTarget,
|
||||
ForceDecode,
|
||||
NoDecode,
|
||||
PathType,
|
||||
PydanticModel,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
'CLI_SUPPRESS',
|
||||
'DEFAULT_PATH',
|
||||
'ENV_FILE_SENTINEL',
|
||||
'AWSSecretsManagerSettingsSource',
|
||||
'AWSSystemsManagerSettingsSource',
|
||||
'AzureKeyVaultSettingsSource',
|
||||
'CliDualFlag',
|
||||
'CliExplicitFlag',
|
||||
'CliImplicitFlag',
|
||||
'CliMutuallyExclusiveGroup',
|
||||
'CliPositionalArg',
|
||||
'CliSettingsSource',
|
||||
'CliSubCommand',
|
||||
'CliSuppress',
|
||||
'CliToggleFlag',
|
||||
'CliUnknownArgs',
|
||||
'CliVariadicArg',
|
||||
'ConfigFileSourceMixin',
|
||||
'ConfigFileSourceType',
|
||||
'DefaultSettingsSource',
|
||||
'DotEnvSettingsSource',
|
||||
'DotenvFiltering',
|
||||
'DotenvType',
|
||||
'EnvPrefixTarget',
|
||||
'EnvSettingsSource',
|
||||
'ForceDecode',
|
||||
'GoogleSecretManagerSettingsSource',
|
||||
'InitSettingsSource',
|
||||
'JsonConfigSettingsSource',
|
||||
'NestedSecretsSettingsSource',
|
||||
'NoDecode',
|
||||
'PathType',
|
||||
'PydanticBaseEnvSettingsSource',
|
||||
'PydanticBaseSettingsSource',
|
||||
'PydanticModel',
|
||||
'PyprojectTomlConfigSettingsSource',
|
||||
'SecretsSettingsSource',
|
||||
'TomlConfigSettingsSource',
|
||||
'YamlConfigSettingsSource',
|
||||
'get_subcommand',
|
||||
'read_env_file',
|
||||
]
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,746 @@
|
||||
"""Base classes and core functionality for pydantic-settings sources."""
|
||||
|
||||
from __future__ import annotations as _annotations
|
||||
|
||||
import json
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable, Hashable, Mapping, Sequence
|
||||
from dataclasses import asdict, is_dataclass
|
||||
from dataclasses import fields as dataclass_fields
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, cast, get_args, get_type_hints
|
||||
|
||||
from pydantic import AliasChoices, AliasPath, TypeAdapter
|
||||
from pydantic._internal._typing_extra import ( # type: ignore[attr-defined]
|
||||
get_origin,
|
||||
)
|
||||
from pydantic._internal._utils import deep_update, is_model_class
|
||||
from pydantic.dataclasses import is_pydantic_dataclass
|
||||
from pydantic.fields import FieldInfo
|
||||
from typing_inspection.introspection import is_union_origin
|
||||
|
||||
from ..exceptions import SettingsError
|
||||
from .types import (
|
||||
ConfigFileSourceType,
|
||||
EnvNoneType,
|
||||
EnvPrefixTarget,
|
||||
ForceDecode,
|
||||
NoDecode,
|
||||
PydanticModel,
|
||||
_CliSubCommand,
|
||||
)
|
||||
from .utils import (
|
||||
InitState,
|
||||
_annotation_is_complex,
|
||||
_get_alias_names,
|
||||
_get_field_annotation_and_metadata,
|
||||
_get_field_metadata,
|
||||
_get_model_config,
|
||||
_get_model_fields,
|
||||
_strip_annotated,
|
||||
_union_is_complex,
|
||||
_validate_by_alias_enabled,
|
||||
_validate_by_name_enabled,
|
||||
_warn_if_field_info_incomplete,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pydantic_settings.main import BaseSettings
|
||||
|
||||
from .types import Traversable
|
||||
|
||||
|
||||
def get_subcommand(
|
||||
model: PydanticModel,
|
||||
is_required: bool = True,
|
||||
cli_exit_on_error: bool | None = None,
|
||||
_suppress_errors: list[SettingsError | SystemExit] | None = None,
|
||||
) -> PydanticModel | None:
|
||||
"""
|
||||
Get the subcommand from a model.
|
||||
|
||||
Args:
|
||||
model: The model to get the subcommand from.
|
||||
is_required: Determines whether a model must have subcommand set and raises error if not
|
||||
found. Defaults to `True`.
|
||||
cli_exit_on_error: Determines whether this function exits with error if no subcommand is found.
|
||||
Defaults to model_config `cli_exit_on_error` value if set. Otherwise, defaults to `True`.
|
||||
|
||||
Returns:
|
||||
The subcommand model if found, otherwise `None`.
|
||||
|
||||
Raises:
|
||||
SystemExit: When no subcommand is found and is_required=`True` and cli_exit_on_error=`True`
|
||||
(the default).
|
||||
SettingsError: When no subcommand is found and is_required=`True` and
|
||||
cli_exit_on_error=`False`.
|
||||
"""
|
||||
|
||||
model_cls = type(model)
|
||||
if cli_exit_on_error is None and is_model_class(model_cls):
|
||||
model_default = model_cls.model_config.get('cli_exit_on_error')
|
||||
if isinstance(model_default, bool):
|
||||
cli_exit_on_error = model_default
|
||||
if cli_exit_on_error is None:
|
||||
cli_exit_on_error = True
|
||||
|
||||
subcommands: list[str] = []
|
||||
for field_name, field_info in _get_model_fields(model_cls).items():
|
||||
if _CliSubCommand in field_info.metadata:
|
||||
if getattr(model, field_name) is not None:
|
||||
return getattr(model, field_name)
|
||||
subcommands.append(field_name)
|
||||
# TODO: we could warn if no sub command was found and one of `field_info`s
|
||||
# isn't complete.
|
||||
|
||||
if is_required:
|
||||
error_message = (
|
||||
f'Error: CLI subcommand is required {{{", ".join(subcommands)}}}'
|
||||
if subcommands
|
||||
else 'Error: CLI subcommand is required but no subcommands were found.'
|
||||
)
|
||||
err = SystemExit(error_message) if cli_exit_on_error else SettingsError(error_message)
|
||||
if _suppress_errors is None:
|
||||
raise err
|
||||
_suppress_errors.append(err)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
class PydanticBaseSettingsSource(ABC):
|
||||
"""
|
||||
Abstract base class for settings sources, every settings source classes should inherit from it.
|
||||
"""
|
||||
|
||||
def __init__(self, settings_cls: type[BaseSettings], _init_state: InitState | None = None):
|
||||
self._init_state: InitState = {} if _init_state is None else _init_state
|
||||
self.settings_cls = settings_cls
|
||||
self.config = settings_cls.model_config
|
||||
self._current_state: dict[str, Any] = {}
|
||||
self._settings_sources_data: dict[str, dict[str, Any]] = {}
|
||||
|
||||
def _set_current_state(self, state: dict[str, Any]) -> None:
|
||||
"""
|
||||
Record the state of settings from the previous settings sources. This should
|
||||
be called right before __call__.
|
||||
"""
|
||||
self._current_state = state
|
||||
|
||||
def _set_settings_sources_data(self, states: dict[str, dict[str, Any]]) -> None:
|
||||
"""
|
||||
Record the state of settings from all previous settings sources. This should
|
||||
be called right before __call__.
|
||||
"""
|
||||
self._settings_sources_data = states
|
||||
|
||||
@property
|
||||
def current_state(self) -> dict[str, Any]:
|
||||
"""
|
||||
The current state of the settings, populated by the previous settings sources.
|
||||
"""
|
||||
return self._current_state
|
||||
|
||||
@property
|
||||
def settings_sources_data(self) -> dict[str, dict[str, Any]]:
|
||||
"""
|
||||
The state of all previous settings sources.
|
||||
"""
|
||||
return self._settings_sources_data
|
||||
|
||||
@abstractmethod
|
||||
def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
|
||||
"""
|
||||
Gets the value, the key for model creation, and a flag to determine whether value is complex.
|
||||
|
||||
This is an abstract method that should be overridden in every settings source classes.
|
||||
|
||||
Args:
|
||||
field: The field.
|
||||
field_name: The field name.
|
||||
|
||||
Returns:
|
||||
A tuple that contains the value, key and a flag to determine whether value is complex.
|
||||
"""
|
||||
|
||||
def field_is_complex(self, field: FieldInfo) -> bool:
|
||||
"""
|
||||
Checks whether a field is complex, in which case it will attempt to be parsed as JSON.
|
||||
|
||||
Args:
|
||||
field: The field.
|
||||
|
||||
Returns:
|
||||
Whether the field is complex.
|
||||
"""
|
||||
return _annotation_is_complex(field.annotation, field.metadata, self._init_state)
|
||||
|
||||
def prepare_field_value(self, field_name: str, field: FieldInfo, value: Any, value_is_complex: bool) -> Any:
|
||||
"""
|
||||
Prepares the value of a field.
|
||||
|
||||
Args:
|
||||
field_name: The field name.
|
||||
field: The field.
|
||||
value: The value of the field that has to be prepared.
|
||||
value_is_complex: A flag to determine whether value is complex.
|
||||
|
||||
Returns:
|
||||
The prepared value.
|
||||
"""
|
||||
if value is not None and (self.field_is_complex(field) or value_is_complex):
|
||||
return self.decode_complex_value(field_name, field, value)
|
||||
return value
|
||||
|
||||
def decode_complex_value(self, field_name: str, field: FieldInfo, value: Any) -> Any:
|
||||
"""
|
||||
Decode the value for a complex field
|
||||
|
||||
Args:
|
||||
field_name: The field name.
|
||||
field: The field.
|
||||
value: The value of the field that has to be prepared.
|
||||
|
||||
Returns:
|
||||
The decoded value for further preparation
|
||||
"""
|
||||
metadata = _get_field_metadata(field) if field else []
|
||||
if field and (
|
||||
NoDecode in metadata or (self.config.get('enable_decoding') is False and ForceDecode not in metadata)
|
||||
):
|
||||
return value
|
||||
|
||||
return json.loads(value)
|
||||
|
||||
@abstractmethod
|
||||
def __call__(self) -> dict[str, Any]:
|
||||
pass
|
||||
|
||||
|
||||
class ConfigFileSourceMixin(ABC):
|
||||
@staticmethod
|
||||
def _existing_files(files: ConfigFileSourceType | None) -> list[Path | Traversable]:
|
||||
"""Return the given files that exist, with `~` expanded in native paths."""
|
||||
if files is None:
|
||||
return []
|
||||
if not isinstance(files, Sequence) or isinstance(files, str):
|
||||
files = [files]
|
||||
existing: list[Path | Traversable] = []
|
||||
for file in files:
|
||||
file_path: Path | Traversable = Path(file) if isinstance(file, str) else file
|
||||
if isinstance(file_path, Path):
|
||||
file_path = file_path.expanduser()
|
||||
if file_path.is_file():
|
||||
existing.append(file_path)
|
||||
return existing
|
||||
|
||||
def _read_files(self, files: ConfigFileSourceType | None, deep_merge: bool = False) -> dict[str, Any]:
|
||||
vars: dict[str, Any] = {}
|
||||
for file_path in self._existing_files(files):
|
||||
updating_vars = self._read_file(file_path)
|
||||
if deep_merge:
|
||||
vars = deep_update(vars, updating_vars)
|
||||
else:
|
||||
vars.update(updating_vars)
|
||||
return vars
|
||||
|
||||
@abstractmethod
|
||||
def _read_file(self, path: Path | Traversable) -> dict[str, Any]:
|
||||
pass
|
||||
|
||||
|
||||
def _unwrap_optional_annotation(annotation: Any) -> Any:
|
||||
"""Return the inner type of an ``Optional[T]`` annotation, otherwise the annotation unchanged."""
|
||||
if is_union_origin(get_origin(annotation)):
|
||||
args = get_args(annotation)
|
||||
if len(args) == 2 and type(None) in args:
|
||||
for arg in args: # pragma: no branch
|
||||
if arg is not type(None):
|
||||
return arg
|
||||
return annotation
|
||||
|
||||
|
||||
def _is_model_or_dataclass(annotation: Any) -> bool:
|
||||
"""Whether the annotation is a pydantic model or a pydantic dataclass, i.e. a type with nested fields."""
|
||||
return is_model_class(annotation) or is_pydantic_dataclass(annotation)
|
||||
|
||||
|
||||
def _get_discriminator(field_info: FieldInfo) -> str | Callable[[Any], Hashable] | None:
|
||||
"""Get the discriminator of a field using a discriminated union (via Annotated or Field(discriminator=...))."""
|
||||
from pydantic import Discriminator
|
||||
|
||||
discriminator = field_info.discriminator
|
||||
if discriminator is None:
|
||||
discriminator = next((m for m in field_info.metadata if isinstance(m, Discriminator)), None)
|
||||
return discriminator.discriminator if isinstance(discriminator, Discriminator) else discriminator
|
||||
|
||||
|
||||
def _get_member_fields(model_cls: type[Any]) -> dict[str, FieldInfo]:
|
||||
"""Get the fields of a discriminated union member: a model, a pydantic dataclass or a stdlib dataclass."""
|
||||
if _is_model_or_dataclass(model_cls):
|
||||
return _get_model_fields(model_cls)
|
||||
# Pydantic doesn't attach `__pydantic_fields__` to a stdlib dataclass, so collect the `FieldInfo`s itself:
|
||||
# `Field()` is kept as the dataclass field's default, or in its `Annotated` metadata.
|
||||
fields: dict[str, FieldInfo] = {}
|
||||
hints = get_type_hints(model_cls, include_extras=True)
|
||||
for dc_field in dataclass_fields(model_cls):
|
||||
if isinstance(dc_field.default, FieldInfo):
|
||||
fields[dc_field.name] = dc_field.default
|
||||
else:
|
||||
metadata = get_args(hints.get(dc_field.name))[1:]
|
||||
field_info = next((m for m in metadata if isinstance(m, FieldInfo)), None)
|
||||
if field_info is not None:
|
||||
fields[dc_field.name] = field_info
|
||||
return fields
|
||||
|
||||
|
||||
def _get_discriminator_keys(model_cls: type[Any], discriminator: str) -> tuple[str, ...]:
|
||||
"""Get the keys that can hold the discriminator value of a discriminated union member: its name and alias."""
|
||||
alias = getattr(_get_member_fields(model_cls).get(discriminator), 'validation_alias', None)
|
||||
return (discriminator, alias) if isinstance(alias, str) else (discriminator,)
|
||||
|
||||
|
||||
def _get_nested_discriminators(annotation: Any) -> list[str] | None:
|
||||
"""Get the discriminators nested inside a discriminated union's annotation.
|
||||
|
||||
A member can be guarded by more than one discriminator, e.g. in
|
||||
``Annotated[Annotated[BlackCat | WhiteCat, Discriminator('color')] | Dog, Discriminator('pet_type')]``
|
||||
reaching ``BlackCat`` requires both ``pet_type='cat'`` and ``color='black'``. Each one has to keep the
|
||||
default's value for its default to still apply.
|
||||
|
||||
Returns ``None`` if a nested discriminator can't be resolved to a field name (i.e. it is callable), as
|
||||
the member it selects can't be looked up in the incoming value.
|
||||
"""
|
||||
from pydantic import Discriminator
|
||||
|
||||
discriminators: list[str] = []
|
||||
for arg in get_args(annotation):
|
||||
if arg is type(None):
|
||||
continue
|
||||
for metadata in get_args(arg)[1:]:
|
||||
if isinstance(metadata, Discriminator):
|
||||
if not isinstance(metadata.discriminator, str):
|
||||
return None
|
||||
discriminators.append(metadata.discriminator)
|
||||
nested = _get_nested_discriminators(arg)
|
||||
if nested is None:
|
||||
return None
|
||||
discriminators.extend(nested)
|
||||
return discriminators
|
||||
|
||||
|
||||
class DefaultSettingsSource(PydanticBaseSettingsSource):
|
||||
"""
|
||||
Source class for loading default object values.
|
||||
|
||||
Args:
|
||||
settings_cls: The Settings class.
|
||||
nested_model_default_partial_update: Whether to allow partial updates on nested model default object fields.
|
||||
Defaults to `False`.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
settings_cls: type[BaseSettings],
|
||||
nested_model_default_partial_update: bool | None = None,
|
||||
_init_state: InitState | None = None,
|
||||
):
|
||||
super().__init__(settings_cls, _init_state)
|
||||
self.defaults: dict[str, Any] = {}
|
||||
# Discriminated union fields: per discriminator guarding the default's member, the keys that can hold
|
||||
# its value, and the default's value
|
||||
self._discriminator_tags: dict[str, list[tuple[tuple[str, ...], Any]]] = {}
|
||||
self.nested_model_default_partial_update = (
|
||||
nested_model_default_partial_update
|
||||
if nested_model_default_partial_update is not None
|
||||
else self.config.get('nested_model_default_partial_update', False)
|
||||
)
|
||||
if self.nested_model_default_partial_update:
|
||||
validate_by_alias = _validate_by_alias_enabled(self.config)
|
||||
for field_name, field_info in settings_cls.model_fields.items():
|
||||
_warn_if_field_info_incomplete(field_info, field_name, self._init_state)
|
||||
discriminator = _get_discriminator(field_info)
|
||||
if callable(discriminator):
|
||||
# The member selected by a callable discriminator can't be looked up in the incoming value
|
||||
continue
|
||||
nested = _get_nested_discriminators(field_info.annotation) if discriminator is not None else []
|
||||
if nested is None:
|
||||
# Same for a nested callable discriminator
|
||||
continue
|
||||
alias_names, *_ = _get_alias_names(field_name, field_info, validate_by_alias=validate_by_alias)
|
||||
preferred_alias = alias_names[0]
|
||||
if is_dataclass(type(field_info.default)):
|
||||
self.defaults[preferred_alias] = asdict(field_info.default)
|
||||
elif is_model_class(type(field_info.default)):
|
||||
self.defaults[preferred_alias] = field_info.default.model_dump()
|
||||
if discriminator is not None and preferred_alias in self.defaults:
|
||||
default_cls = type(field_info.default)
|
||||
self._discriminator_tags[preferred_alias] = [
|
||||
(_get_discriminator_keys(default_cls, name), getattr(field_info.default, name, None))
|
||||
for name in (discriminator, *nested)
|
||||
]
|
||||
|
||||
def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
|
||||
# Nothing to do here. Only implement the return statement to make mypy happy
|
||||
return None, '', False
|
||||
|
||||
def __call__(self) -> dict[str, Any]:
|
||||
defaults = self.defaults.copy()
|
||||
for alias, discriminators in self._discriminator_tags.items():
|
||||
value = self.current_state.get(alias)
|
||||
if not isinstance(value, dict):
|
||||
continue
|
||||
# Pydantic reads each tag from the discriminator field's name or its alias, so a value under any of
|
||||
# those keys selects a member. A key that is absent leaves the default's member in place.
|
||||
# The default's fields don't apply when any discriminator selects another member.
|
||||
if any(value[key] != default_tag for keys, default_tag in discriminators for key in keys if key in value):
|
||||
del defaults[alias]
|
||||
return defaults
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f'{self.__class__.__name__}(nested_model_default_partial_update={self.nested_model_default_partial_update})'
|
||||
)
|
||||
|
||||
|
||||
class InitSettingsSource(PydanticBaseSettingsSource):
|
||||
"""
|
||||
Source class for loading values provided during settings class initialization.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
settings_cls: type[BaseSettings],
|
||||
init_kwargs: dict[str, Any],
|
||||
nested_model_default_partial_update: bool | None = None,
|
||||
_init_state: InitState | None = None,
|
||||
):
|
||||
super().__init__(settings_cls, _init_state)
|
||||
case_sensitive = self.config.get('case_sensitive', False)
|
||||
include_name = _validate_by_name_enabled(self.config)
|
||||
validate_by_alias = _validate_by_alias_enabled(self.config)
|
||||
|
||||
def normalize(name: str) -> str:
|
||||
# When case_sensitive is False, matching is done on lower-cased names.
|
||||
return name if case_sensitive else name.lower()
|
||||
|
||||
self.init_kwargs = {}
|
||||
init_kwarg_names = set(init_kwargs.keys())
|
||||
# Map each normalized name to all provided keys sharing it, so field aliases can
|
||||
# be matched without losing the original key used to look up the value. A 1-to-many
|
||||
# mapping is required because case_sensitive=False can collapse distinct keys
|
||||
# (e.g. ``TeSt`` and ``TEST``) onto the same normalized name.
|
||||
normalized_provided: dict[str, list[str]] = {}
|
||||
for key in init_kwargs:
|
||||
normalized_provided.setdefault(normalize(key), []).append(key)
|
||||
for field_name, field_info in settings_cls.model_fields.items():
|
||||
_warn_if_field_info_incomplete(field_info, field_name, self._init_state)
|
||||
# Canonical (case-preserving) alias names determine the output key, so the value
|
||||
# still matches pydantic's validation aliases, which are always case-sensitive.
|
||||
canonical_alias_names, *_ = _get_alias_names(field_name, field_info, validate_by_alias=validate_by_alias)
|
||||
# Match alias names respect case_sensitive when matching against provided keys.
|
||||
match_alias_names, *_ = _get_alias_names(
|
||||
field_name, field_info, case_sensitive=case_sensitive, validate_by_alias=validate_by_alias
|
||||
)
|
||||
# When populate_by_name is True, allow using the field name as an input key,
|
||||
# but normalize to the preferred alias to keep keys consistent across sources.
|
||||
matchable_names = list(match_alias_names)
|
||||
if include_name:
|
||||
name = normalize(field_name)
|
||||
if name not in matchable_names:
|
||||
matchable_names.append(name)
|
||||
# All provided keys matching this field, in matchable_names preference order
|
||||
# (aliases first, then the field name). Only keys not yet consumed by an
|
||||
# earlier field are considered.
|
||||
matched_keys = [
|
||||
original
|
||||
for name in matchable_names
|
||||
if name in normalized_provided
|
||||
for original in normalized_provided[name]
|
||||
if original in init_kwarg_names
|
||||
]
|
||||
if matched_keys:
|
||||
preferred_alias = canonical_alias_names[0] if canonical_alias_names else field_name
|
||||
# Prefer the first match in preference order; drop the remaining matches
|
||||
# so they are not re-added as extras.
|
||||
init_kwarg_names.difference_update(matched_keys)
|
||||
self.init_kwargs[preferred_alias] = init_kwargs[matched_keys[0]]
|
||||
# Include any remaining init kwargs (e.g., extras) unchanged
|
||||
# Note: If populate_by_name is True and the provided key is the field name, but
|
||||
# no alias exists, we keep it as-is so it can be processed as extra if allowed.
|
||||
self.init_kwargs.update({key: val for key, val in init_kwargs.items() if key in init_kwarg_names})
|
||||
|
||||
self.nested_model_default_partial_update = (
|
||||
nested_model_default_partial_update
|
||||
if nested_model_default_partial_update is not None
|
||||
else self.config.get('nested_model_default_partial_update', False)
|
||||
)
|
||||
|
||||
def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
|
||||
# Nothing to do here. Only implement the return statement to make mypy happy
|
||||
return None, '', False
|
||||
|
||||
def __call__(self) -> dict[str, Any]:
|
||||
return (
|
||||
TypeAdapter(dict[str, Any]).dump_python(self.init_kwargs)
|
||||
if self.nested_model_default_partial_update
|
||||
else self.init_kwargs
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f'{self.__class__.__name__}(init_kwargs={self.init_kwargs!r})'
|
||||
|
||||
|
||||
class PydanticBaseEnvSettingsSource(PydanticBaseSettingsSource):
|
||||
def __init__(
|
||||
self,
|
||||
settings_cls: type[BaseSettings],
|
||||
case_sensitive: bool | None = None,
|
||||
env_prefix: str | None = None,
|
||||
env_prefix_target: EnvPrefixTarget | None = None,
|
||||
env_ignore_empty: bool | None = None,
|
||||
env_parse_none_str: str | None = None,
|
||||
env_parse_enums: bool | None = None,
|
||||
_init_state: InitState | None = None,
|
||||
) -> None:
|
||||
super().__init__(settings_cls, _init_state)
|
||||
self.case_sensitive = case_sensitive if case_sensitive is not None else self.config.get('case_sensitive', False)
|
||||
self.env_prefix = env_prefix if env_prefix is not None else self.config.get('env_prefix', '')
|
||||
self.env_prefix_target = (
|
||||
env_prefix_target if env_prefix_target is not None else self.config.get('env_prefix_target', 'variable')
|
||||
)
|
||||
self.env_ignore_empty = (
|
||||
env_ignore_empty if env_ignore_empty is not None else self.config.get('env_ignore_empty', False)
|
||||
)
|
||||
self.env_parse_none_str = (
|
||||
env_parse_none_str if env_parse_none_str is not None else self.config.get('env_parse_none_str')
|
||||
)
|
||||
self.env_parse_enums = env_parse_enums if env_parse_enums is not None else self.config.get('env_parse_enums')
|
||||
|
||||
def _apply_case_sensitive(self, value: str) -> str:
|
||||
return value.lower() if not self.case_sensitive else value
|
||||
|
||||
def _extract_field_info(
|
||||
self, field: FieldInfo, field_name: str, model_config: Mapping[str, Any] | None = None
|
||||
) -> list[tuple[str, str, bool]]:
|
||||
"""
|
||||
Extracts field info. This info is used to get the value of field from environment variables.
|
||||
|
||||
It returns a list of tuples, each tuple contains:
|
||||
* field_key: The key of field that has to be used in model creation.
|
||||
* env_name: The environment variable name of the field.
|
||||
* value_is_complex: A flag to determine whether the value from environment variable
|
||||
is complex and has to be parsed.
|
||||
|
||||
Args:
|
||||
field (FieldInfo): The field.
|
||||
field_name (str): The field name.
|
||||
model_config (Mapping[str, Any] | None): The config of the model owning the field, which decides
|
||||
whether its alias and name are accepted. Defaults to the settings config, pass it for a field
|
||||
of a nested model.
|
||||
|
||||
Returns:
|
||||
list[tuple[str, str, bool]]: List of tuples, each tuple contains field_key, env_name, and value_is_complex.
|
||||
"""
|
||||
_warn_if_field_info_incomplete(field, field_name, self._init_state)
|
||||
field_info: list[tuple[str, str, bool]] = []
|
||||
v_alias: str | list[str | int] | list[list[str | int]] | None
|
||||
if model_config is None:
|
||||
model_config = self.config
|
||||
if not _validate_by_alias_enabled(model_config):
|
||||
# pydantic ignores the aliases, so only the field name can populate the field.
|
||||
v_alias = None
|
||||
elif isinstance(field.validation_alias, (AliasChoices, AliasPath)):
|
||||
v_alias = field.validation_alias.convert_to_aliases()
|
||||
else:
|
||||
v_alias = field.validation_alias
|
||||
|
||||
if v_alias:
|
||||
env_prefix = self.env_prefix if self.env_prefix_target in ('alias', 'all') else ''
|
||||
if isinstance(v_alias, list): # AliasChoices, AliasPath
|
||||
if isinstance(v_alias[0], list): # AliasChoices: one entry per choice
|
||||
for alias in v_alias:
|
||||
first_arg = cast(str, alias[0]) # first item of an AliasChoices must be a str
|
||||
field_info.append(
|
||||
(
|
||||
first_arg,
|
||||
self._apply_case_sensitive(env_prefix + first_arg),
|
||||
len(alias) > 1,
|
||||
)
|
||||
)
|
||||
else: # bare AliasPath: a single flat path, e.g. ['a', 0, 'b']
|
||||
# Only the head names the env var to read; the remaining segments are
|
||||
# navigated in-memory once the value has been JSON-decoded, so they must
|
||||
# never be looked up as env vars of their own (previously each string
|
||||
# segment produced its own candidate). Complexity must reflect the number
|
||||
# of segments in the whole path, not the length of the head's name.
|
||||
head = cast(str, v_alias[0]) # first item of an AliasPath must be a str
|
||||
field_info.append((head, self._apply_case_sensitive(env_prefix + head), len(v_alias) > 1))
|
||||
else: # string validation alias
|
||||
field_info.append((v_alias, self._apply_case_sensitive(env_prefix + v_alias), False))
|
||||
|
||||
if not v_alias or _validate_by_name_enabled(model_config):
|
||||
annotation, metadata = _get_field_annotation_and_metadata(field)
|
||||
env_prefix = self.env_prefix if self.env_prefix_target in ('variable', 'all') else ''
|
||||
if is_union_origin(get_origin(annotation)) and _union_is_complex(annotation, metadata, self._init_state):
|
||||
field_info.append((field_name, self._apply_case_sensitive(env_prefix + field_name), True))
|
||||
else:
|
||||
field_info.append((field_name, self._apply_case_sensitive(env_prefix + field_name), False))
|
||||
|
||||
return field_info
|
||||
|
||||
def _replace_field_names_case_insensitively(self, field: FieldInfo, field_values: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
Replace field names in values dict by looking in models fields insensitively.
|
||||
|
||||
By having the following models:
|
||||
|
||||
```py
|
||||
class SubSubSub(BaseModel):
|
||||
VaL3: str
|
||||
|
||||
class SubSub(BaseModel):
|
||||
Val2: str
|
||||
SUB_sub_SuB: SubSubSub
|
||||
|
||||
class Sub(BaseModel):
|
||||
VAL1: str
|
||||
SUB_sub: SubSub
|
||||
|
||||
class Settings(BaseSettings):
|
||||
nested: Sub
|
||||
|
||||
model_config = SettingsConfigDict(env_nested_delimiter='__')
|
||||
```
|
||||
|
||||
Then:
|
||||
_replace_field_names_case_insensitively(
|
||||
field,
|
||||
{"val1": "v1", "sub_SUB": {"VAL2": "v2", "sub_SUB_sUb": {"vAl3": "v3"}}}
|
||||
)
|
||||
Returns {'VAL1': 'v1', 'SUB_sub': {'Val2': 'v2', 'SUB_sub_SuB': {'VaL3': 'v3'}}}
|
||||
"""
|
||||
values: dict[str, Any] = {}
|
||||
|
||||
for name, value in field_values.items():
|
||||
sub_model_field: FieldInfo | None = None
|
||||
|
||||
annotation = _strip_annotated(_unwrap_optional_annotation(field.annotation))
|
||||
|
||||
if not _is_model_or_dataclass(annotation):
|
||||
values[name] = value
|
||||
continue
|
||||
model_fields = _get_model_fields(annotation)
|
||||
validate_by_alias = _validate_by_alias_enabled(_get_model_config(annotation))
|
||||
|
||||
# Find field in sub model by looking in fields case insensitively
|
||||
field_key: str | None = None
|
||||
for sub_model_field_name, sub_model_field in model_fields.items():
|
||||
_warn_if_field_info_incomplete(sub_model_field, sub_model_field_name, self._init_state)
|
||||
aliases, _ = _get_alias_names(
|
||||
sub_model_field_name, sub_model_field, validate_by_alias=validate_by_alias
|
||||
)
|
||||
_search = (alias for alias in aliases if alias.lower() == name.lower())
|
||||
if field_key := next(_search, None):
|
||||
break
|
||||
|
||||
if not field_key:
|
||||
values[name] = value
|
||||
continue
|
||||
|
||||
if (
|
||||
sub_model_field is not None
|
||||
and _is_model_or_dataclass(_strip_annotated(_unwrap_optional_annotation(sub_model_field.annotation)))
|
||||
and isinstance(value, dict)
|
||||
):
|
||||
values[field_key] = self._replace_field_names_case_insensitively(sub_model_field, value)
|
||||
else:
|
||||
values[field_key] = value
|
||||
|
||||
return values
|
||||
|
||||
def _replace_env_none_type_values(self, field_value: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
Recursively parse values that are of "None" type(EnvNoneType) to `None` type(None).
|
||||
"""
|
||||
values: dict[str, Any] = {}
|
||||
|
||||
for key, value in field_value.items():
|
||||
if not isinstance(value, EnvNoneType):
|
||||
values[key] = value if not isinstance(value, dict) else self._replace_env_none_type_values(value)
|
||||
else:
|
||||
values[key] = None
|
||||
|
||||
return values
|
||||
|
||||
def _get_resolved_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
|
||||
"""
|
||||
Gets the value, the preferred alias key for model creation, and a flag to determine whether value
|
||||
is complex.
|
||||
|
||||
Note:
|
||||
In V3, this method should either be made public, or, this method should be removed and the
|
||||
abstract method get_field_value should be updated to include a "use_preferred_alias" flag.
|
||||
|
||||
Args:
|
||||
field: The field.
|
||||
field_name: The field name.
|
||||
|
||||
Returns:
|
||||
A tuple that contains the value, preferred key and a flag to determine whether value is complex.
|
||||
"""
|
||||
field_value, field_key, value_is_complex = self.get_field_value(field, field_name)
|
||||
if not (value_is_complex or (_validate_by_name_enabled(self.config) and (field_key == field_name))):
|
||||
field_infos = self._extract_field_info(field, field_name)
|
||||
preferred_key, _, preferred_is_complex = field_infos[0]
|
||||
# Only normalize to preferred_key when it's a simple string alias.
|
||||
# When the preferred key comes from an AliasPath (complex entry), skip normalization
|
||||
# to avoid using the AliasPath's first element as the key (see #766).
|
||||
if not preferred_is_complex:
|
||||
return field_value, preferred_key, value_is_complex
|
||||
return field_value, field_key, value_is_complex
|
||||
|
||||
def __call__(self) -> dict[str, Any]:
|
||||
data: dict[str, Any] = {}
|
||||
|
||||
for field_name, field in self.settings_cls.model_fields.items():
|
||||
_warn_if_field_info_incomplete(field, field_name, self._init_state)
|
||||
try:
|
||||
field_value, field_key, value_is_complex = self._get_resolved_field_value(field, field_name)
|
||||
except Exception as e:
|
||||
raise SettingsError(
|
||||
f'error getting value for field "{field_name}" from source "{self.__class__.__name__}"'
|
||||
) from e
|
||||
|
||||
try:
|
||||
field_value = self.prepare_field_value(field_name, field, field_value, value_is_complex)
|
||||
except ValueError as e:
|
||||
raise SettingsError(
|
||||
f'error parsing value for field "{field_name}" from source "{self.__class__.__name__}"'
|
||||
) from e
|
||||
|
||||
if field_value is not None:
|
||||
if self.env_parse_none_str is not None:
|
||||
if isinstance(field_value, dict):
|
||||
field_value = self._replace_env_none_type_values(field_value)
|
||||
elif isinstance(field_value, EnvNoneType):
|
||||
field_value = None
|
||||
if (
|
||||
not self.case_sensitive
|
||||
# and _lenient_issubclass(field.annotation, BaseModel)
|
||||
and isinstance(field_value, dict)
|
||||
):
|
||||
data[field_key] = self._replace_field_names_case_insensitively(field, field_value)
|
||||
else:
|
||||
data[field_key] = field_value
|
||||
|
||||
return data
|
||||
|
||||
|
||||
__all__ = [
|
||||
'ConfigFileSourceMixin',
|
||||
'DefaultSettingsSource',
|
||||
'InitSettingsSource',
|
||||
'PydanticBaseEnvSettingsSource',
|
||||
'PydanticBaseSettingsSource',
|
||||
'SettingsError',
|
||||
]
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Package containing individual source implementations."""
|
||||
|
||||
from .aws import AWSSecretsManagerSettingsSource, AWSSystemsManagerSettingsSource
|
||||
from .azure import AzureKeyVaultSettingsSource
|
||||
from .cli import (
|
||||
CliDualFlag,
|
||||
CliExplicitFlag,
|
||||
CliImplicitFlag,
|
||||
CliMutuallyExclusiveGroup,
|
||||
CliPositionalArg,
|
||||
CliSettingsSource,
|
||||
CliSubCommand,
|
||||
CliSuppress,
|
||||
CliToggleFlag,
|
||||
CliVariadicArg,
|
||||
)
|
||||
from .dotenv import DotEnvSettingsSource
|
||||
from .env import EnvSettingsSource
|
||||
from .gcp import GoogleSecretManagerSettingsSource
|
||||
from .json import JsonConfigSettingsSource
|
||||
from .pyproject import PyprojectTomlConfigSettingsSource
|
||||
from .secrets import SecretsSettingsSource
|
||||
from .toml import TomlConfigSettingsSource
|
||||
from .yaml import YamlConfigSettingsSource
|
||||
|
||||
__all__ = [
|
||||
'AWSSecretsManagerSettingsSource',
|
||||
'AWSSystemsManagerSettingsSource',
|
||||
'AzureKeyVaultSettingsSource',
|
||||
'CliDualFlag',
|
||||
'CliExplicitFlag',
|
||||
'CliImplicitFlag',
|
||||
'CliMutuallyExclusiveGroup',
|
||||
'CliPositionalArg',
|
||||
'CliSettingsSource',
|
||||
'CliSubCommand',
|
||||
'CliSuppress',
|
||||
'CliToggleFlag',
|
||||
'CliVariadicArg',
|
||||
'DotEnvSettingsSource',
|
||||
'EnvSettingsSource',
|
||||
'GoogleSecretManagerSettingsSource',
|
||||
'JsonConfigSettingsSource',
|
||||
'PyprojectTomlConfigSettingsSource',
|
||||
'SecretsSettingsSource',
|
||||
'TomlConfigSettingsSource',
|
||||
'YamlConfigSettingsSource',
|
||||
]
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,158 @@
|
||||
from __future__ import annotations as _annotations # important for BaseSettings import to work
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from ...exceptions import SettingsError
|
||||
from ..utils import InitState, parse_env_vars
|
||||
from .env import EnvSettingsSource
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from types_boto3_secretsmanager.client import SecretsManagerClient
|
||||
from types_boto3_ssm.client import SSMClient
|
||||
|
||||
from pydantic_settings.main import BaseSettings
|
||||
|
||||
|
||||
boto3_client = None
|
||||
|
||||
|
||||
def import_aws_secrets_manager() -> None:
|
||||
global boto3_client
|
||||
|
||||
try:
|
||||
from boto3 import client as boto3_client
|
||||
except ImportError as e: # pragma: no cover
|
||||
raise ImportError(
|
||||
'AWS Secrets Manager dependencies are not installed, run `pip install pydantic-settings[aws-secrets-manager]`'
|
||||
) from e
|
||||
|
||||
|
||||
def import_aws_systems_manager() -> None:
|
||||
global boto3_client
|
||||
|
||||
try:
|
||||
from boto3 import client as boto3_client
|
||||
except ImportError as e: # pragma: no cover
|
||||
raise ImportError(
|
||||
'AWS Systems Manager dependencies are not installed, run `pip install pydantic-settings[aws-systems-manager]`'
|
||||
) from e
|
||||
|
||||
|
||||
class AWSSecretsManagerSettingsSource(EnvSettingsSource):
|
||||
_secret_id: str
|
||||
_secretsmanager_client: SecretsManagerClient
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
settings_cls: type[BaseSettings],
|
||||
secret_id: str,
|
||||
region_name: str | None = None,
|
||||
endpoint_url: str | None = None,
|
||||
case_sensitive: bool | None = True,
|
||||
env_prefix: str | None = None,
|
||||
env_nested_delimiter: str | None = '--',
|
||||
env_parse_none_str: str | None = None,
|
||||
env_parse_enums: bool | None = None,
|
||||
version_id: str | None = None,
|
||||
_init_state: InitState | None = None,
|
||||
) -> None:
|
||||
import_aws_secrets_manager()
|
||||
self._secretsmanager_client = boto3_client('secretsmanager', region_name=region_name, endpoint_url=endpoint_url) # type: ignore
|
||||
self._secret_id = secret_id
|
||||
self._version_id = version_id
|
||||
super().__init__(
|
||||
settings_cls,
|
||||
case_sensitive=case_sensitive,
|
||||
env_prefix=env_prefix,
|
||||
env_nested_delimiter=env_nested_delimiter,
|
||||
env_ignore_empty=False,
|
||||
env_parse_none_str=env_parse_none_str,
|
||||
env_parse_enums=env_parse_enums,
|
||||
_init_state=_init_state,
|
||||
)
|
||||
|
||||
def _load_env_vars(self) -> Mapping[str, str | None]:
|
||||
request = {'SecretId': self._secret_id}
|
||||
|
||||
if self._version_id:
|
||||
request['VersionId'] = self._version_id
|
||||
|
||||
response = self._secretsmanager_client.get_secret_value(**request)
|
||||
|
||||
return parse_env_vars(
|
||||
json.loads(response['SecretString']),
|
||||
self.case_sensitive,
|
||||
self.env_ignore_empty,
|
||||
self.env_parse_none_str,
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f'{self.__class__.__name__}(secret_id={self._secret_id!r}, '
|
||||
f'env_nested_delimiter={self.env_nested_delimiter!r})'
|
||||
)
|
||||
|
||||
|
||||
class AWSSystemsManagerSettingsSource(EnvSettingsSource):
|
||||
_ssm_path: str
|
||||
_ssm_client: SSMClient
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
settings_cls: type[BaseSettings],
|
||||
ssm_path: str = '/',
|
||||
region_name: str | None = None,
|
||||
endpoint_url: str | None = None,
|
||||
case_sensitive: bool | None = True,
|
||||
env_prefix: str | None = None,
|
||||
env_nested_delimiter: str | None = '/',
|
||||
env_parse_none_str: str | None = None,
|
||||
env_parse_enums: bool | None = None,
|
||||
_init_state: InitState | None = None,
|
||||
) -> None:
|
||||
if not ssm_path.startswith('/'):
|
||||
raise SettingsError(f'AWS Systems Manager path must start with "/": {ssm_path!r}')
|
||||
|
||||
import_aws_systems_manager()
|
||||
self._ssm_client = boto3_client('ssm', region_name=region_name, endpoint_url=endpoint_url) # type: ignore
|
||||
self._ssm_path = ssm_path
|
||||
super().__init__(
|
||||
settings_cls,
|
||||
case_sensitive=case_sensitive,
|
||||
env_prefix=env_prefix,
|
||||
env_nested_delimiter=env_nested_delimiter,
|
||||
env_ignore_empty=False,
|
||||
env_parse_none_str=env_parse_none_str,
|
||||
env_parse_enums=env_parse_enums,
|
||||
_init_state=_init_state,
|
||||
)
|
||||
|
||||
def _load_env_vars(self) -> Mapping[str, str | None]:
|
||||
prefix = self._ssm_path.rstrip('/') + '/'
|
||||
|
||||
paginator = self._ssm_client.get_paginator('get_parameters_by_path')
|
||||
page_iterator = paginator.paginate(Path=self._ssm_path, Recursive=True, WithDecryption=True)
|
||||
|
||||
parameters: dict[str, str | None] = {}
|
||||
for page in page_iterator:
|
||||
for parameter in page['Parameters']:
|
||||
key = parameter['Name'].removeprefix(prefix)
|
||||
parameters[key] = parameter.get('Value')
|
||||
|
||||
return parse_env_vars(
|
||||
parameters,
|
||||
self.case_sensitive,
|
||||
self.env_ignore_empty,
|
||||
self.env_parse_none_str,
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f'{self.__class__.__name__}(ssm_path={self._ssm_path!r}, env_nested_delimiter={self.env_nested_delimiter!r})'
|
||||
|
||||
|
||||
__all__ = [
|
||||
'AWSSecretsManagerSettingsSource',
|
||||
'AWSSystemsManagerSettingsSource',
|
||||
]
|
||||
@@ -0,0 +1,246 @@
|
||||
"""Azure Key Vault settings source."""
|
||||
|
||||
from __future__ import annotations as _annotations
|
||||
|
||||
from collections.abc import Iterable, Iterator, Mapping
|
||||
from typing import TYPE_CHECKING, Any, get_args, get_origin
|
||||
|
||||
from pydantic._internal._utils import is_model_class
|
||||
from pydantic.alias_generators import to_snake
|
||||
from pydantic.dataclasses import is_pydantic_dataclass
|
||||
from pydantic.fields import FieldInfo
|
||||
from typing_inspection.introspection import is_union_origin
|
||||
|
||||
from ...utils import _lenient_issubclass
|
||||
from ..utils import (
|
||||
InitState,
|
||||
_get_model_config,
|
||||
_get_model_fields,
|
||||
_parse_env_none_str,
|
||||
_resolve_type_alias,
|
||||
_strip_annotated,
|
||||
)
|
||||
from .env import EnvSettingsSource
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from azure.core.credentials import TokenCredential
|
||||
from azure.core.exceptions import ResourceNotFoundError
|
||||
from azure.keyvault.secrets import SecretClient
|
||||
|
||||
from pydantic_settings.main import BaseSettings
|
||||
else:
|
||||
TokenCredential = None
|
||||
ResourceNotFoundError = None
|
||||
SecretClient = None
|
||||
|
||||
|
||||
def import_azure_key_vault() -> None:
|
||||
global TokenCredential
|
||||
global SecretClient
|
||||
global ResourceNotFoundError
|
||||
|
||||
try:
|
||||
from azure.core.credentials import TokenCredential
|
||||
from azure.core.exceptions import ResourceNotFoundError
|
||||
from azure.keyvault.secrets import SecretClient
|
||||
except ImportError as e: # pragma: no cover
|
||||
raise ImportError(
|
||||
'Azure Key Vault dependencies are not installed, run `pip install pydantic-settings[azure-key-vault]`'
|
||||
) from e
|
||||
|
||||
|
||||
def _collect_nested_types(annotation: Any, models: list[Any], dict_value_types: list[Any]) -> None:
|
||||
"""Collect the models and dict value types that nested keys of `annotation` can belong to."""
|
||||
annotation = _strip_annotated(_resolve_type_alias(annotation))
|
||||
origin = get_origin(annotation)
|
||||
if is_union_origin(origin):
|
||||
for arg in get_args(annotation):
|
||||
_collect_nested_types(arg, models, dict_value_types)
|
||||
elif _lenient_issubclass(origin, dict):
|
||||
# A bare `typing.Dict` has no value type, so its keys are left as-is.
|
||||
if args := get_args(annotation):
|
||||
dict_value_types.append(args[-1])
|
||||
elif is_model_class(annotation) or is_pydantic_dataclass(annotation):
|
||||
models.append(annotation)
|
||||
|
||||
|
||||
class AzureKeyVaultMapping(Mapping[str, str | None]):
|
||||
_loaded_secrets: dict[str, str | None]
|
||||
_secret_client: SecretClient
|
||||
_secret_names: list[str]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
secret_client: SecretClient,
|
||||
case_sensitive: bool,
|
||||
snake_case_conversion: bool,
|
||||
env_prefix: str | None,
|
||||
env_parse_none_str: str | None = None,
|
||||
) -> None:
|
||||
self._loaded_secrets = {}
|
||||
self._secret_client = secret_client
|
||||
self._case_sensitive = case_sensitive
|
||||
self._snake_case_conversion = snake_case_conversion
|
||||
self._env_prefix = env_prefix or ''
|
||||
self._env_parse_none_str = env_parse_none_str
|
||||
self._secret_map: dict[str, str] = self._load_remote()
|
||||
|
||||
def _load_remote(self) -> dict[str, str]:
|
||||
secret_names: Iterator[str] = (
|
||||
secret.name for secret in self._secret_client.list_properties_of_secrets() if secret.name and secret.enabled
|
||||
)
|
||||
|
||||
if self._snake_case_conversion:
|
||||
name_map: dict[str, str] = {}
|
||||
for name in secret_names:
|
||||
if name.startswith(self._env_prefix):
|
||||
name_map[f'{self._env_prefix}{to_snake(name[len(self._env_prefix) :])}'] = name
|
||||
else:
|
||||
name_map[to_snake(name)] = name
|
||||
return name_map
|
||||
|
||||
if self._case_sensitive:
|
||||
return {name: name for name in secret_names}
|
||||
|
||||
return {name.lower(): name for name in secret_names}
|
||||
|
||||
def __getitem__(self, key: str) -> str | None:
|
||||
new_key = key
|
||||
|
||||
if self._snake_case_conversion:
|
||||
if key.startswith(self._env_prefix):
|
||||
new_key = f'{self._env_prefix}{to_snake(key[len(self._env_prefix) :])}'
|
||||
else:
|
||||
new_key = to_snake(key)
|
||||
|
||||
elif not self._case_sensitive:
|
||||
new_key = key.lower()
|
||||
|
||||
if new_key not in self._loaded_secrets:
|
||||
if new_key in self._secret_map:
|
||||
self._loaded_secrets[new_key] = self._secret_client.get_secret(self._secret_map[new_key]).value
|
||||
else:
|
||||
raise KeyError(key)
|
||||
|
||||
return _parse_env_none_str(self._loaded_secrets[new_key], self._env_parse_none_str)
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._secret_map)
|
||||
|
||||
def __iter__(self) -> Iterator[str]:
|
||||
return iter(self._secret_map.keys())
|
||||
|
||||
|
||||
class AzureKeyVaultSettingsSource(EnvSettingsSource):
|
||||
_url: str
|
||||
_credential: TokenCredential
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
settings_cls: type[BaseSettings],
|
||||
url: str,
|
||||
credential: TokenCredential,
|
||||
dash_to_underscore: bool = False,
|
||||
case_sensitive: bool | None = None,
|
||||
snake_case_conversion: bool = False,
|
||||
env_prefix: str | None = None,
|
||||
env_parse_none_str: str | None = None,
|
||||
env_parse_enums: bool | None = None,
|
||||
_init_state: InitState | None = None,
|
||||
) -> None:
|
||||
import_azure_key_vault()
|
||||
self._url = url
|
||||
self._credential = credential
|
||||
self._dash_to_underscore = dash_to_underscore
|
||||
self._snake_case_conversion = snake_case_conversion
|
||||
super().__init__(
|
||||
settings_cls,
|
||||
case_sensitive=True if snake_case_conversion else case_sensitive,
|
||||
env_prefix=env_prefix,
|
||||
env_nested_delimiter='__' if snake_case_conversion else '--',
|
||||
env_ignore_empty=False,
|
||||
env_parse_none_str=env_parse_none_str,
|
||||
env_parse_enums=env_parse_enums,
|
||||
_init_state=_init_state,
|
||||
)
|
||||
|
||||
def _load_env_vars(self) -> Mapping[str, str | None]:
|
||||
secret_client = SecretClient(vault_url=self._url, credential=self._credential)
|
||||
return AzureKeyVaultMapping(
|
||||
secret_client=secret_client,
|
||||
case_sensitive=self.case_sensitive,
|
||||
snake_case_conversion=self._snake_case_conversion,
|
||||
env_prefix=self.env_prefix,
|
||||
env_parse_none_str=self.env_parse_none_str,
|
||||
)
|
||||
|
||||
def _extract_field_info(
|
||||
self, field: FieldInfo, field_name: str, model_config: Mapping[str, Any] | None = None
|
||||
) -> list[tuple[str, str, bool]]:
|
||||
field_info = super()._extract_field_info(field, field_name, model_config)
|
||||
if self._dash_to_underscore and not self._snake_case_conversion:
|
||||
return [
|
||||
(field_key, env_name.replace('_', '-'), is_complex) for field_key, env_name, is_complex in field_info
|
||||
]
|
||||
return field_info
|
||||
|
||||
def explode_env_vars(self, field_name: str, field: FieldInfo, env_vars: Mapping[str, str | None]) -> dict[str, Any]:
|
||||
values = super().explode_env_vars(field_name, field, env_vars)
|
||||
if self._dash_to_underscore and not self._snake_case_conversion:
|
||||
self._translate_nested_keys(values, [field.annotation])
|
||||
return values
|
||||
|
||||
def _translate_nested_keys(self, values: dict[str, Any], annotations: Iterable[Any]) -> None:
|
||||
"""Rename the kebab-case keys of `values` to the keys of the nested model fields they belong to.
|
||||
|
||||
The parent prefix has already been removed from the keys in `values`.
|
||||
"""
|
||||
models: list[Any] = []
|
||||
dict_value_types: list[Any] = []
|
||||
for annotation in annotations:
|
||||
_collect_nested_types(annotation, models, dict_value_types)
|
||||
|
||||
targets = self._nested_field_targets(models)
|
||||
field_keys = {field_key for field_targets in targets.values() for field_key, _ in field_targets}
|
||||
|
||||
# Translate each nested dict once, against every field it may belong to.
|
||||
nested: dict[int, tuple[dict[str, Any], list[Any]]] = {}
|
||||
for env_name, field_targets in targets.items():
|
||||
if env_name not in values:
|
||||
continue
|
||||
value = values[env_name]
|
||||
for field_key, annotation in field_targets:
|
||||
values.setdefault(field_key, value)
|
||||
if isinstance(value, dict):
|
||||
nested.setdefault(id(value), (value, []))[1].append(annotation)
|
||||
# Keep the dashed key when it is itself a field key, e.g. an alias of another union member.
|
||||
if env_name not in field_keys:
|
||||
del values[env_name]
|
||||
|
||||
for value, field_annotations in nested.values():
|
||||
self._translate_nested_keys(value, field_annotations)
|
||||
|
||||
if dict_value_types:
|
||||
for value in values.values():
|
||||
if isinstance(value, dict):
|
||||
self._translate_nested_keys(value, dict_value_types)
|
||||
|
||||
def _nested_field_targets(self, models: list[Any]) -> dict[str, list[tuple[str, Any]]]:
|
||||
"""Map the dashed secret name segment of each field of `models` to its field keys and annotations.
|
||||
|
||||
A dashed name may belong to several fields, e.g. the same field in two union members.
|
||||
"""
|
||||
targets: dict[str, list[tuple[str, Any]]] = {}
|
||||
for model in models:
|
||||
config = _get_model_config(model)
|
||||
for field_name, field in _get_model_fields(model).items():
|
||||
for field_key, _, _ in self._extract_field_info(field, field_name, config):
|
||||
env_name = self._apply_case_sensitive(field_key).replace('_', '-')
|
||||
targets.setdefault(env_name, []).append((field_key, field.annotation))
|
||||
return targets
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f'{self.__class__.__name__}(url={self._url!r}, env_nested_delimiter={self.env_nested_delimiter!r})'
|
||||
|
||||
|
||||
__all__ = ['AzureKeyVaultMapping', 'AzureKeyVaultSettingsSource']
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,226 @@
|
||||
"""Dotenv file settings source."""
|
||||
|
||||
from __future__ import annotations as _annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import warnings
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from dotenv import dotenv_values
|
||||
from pydantic._internal._typing_extra import ( # type: ignore[attr-defined]
|
||||
get_origin,
|
||||
)
|
||||
from typing_inspection.introspection import is_union_origin
|
||||
|
||||
from ...utils import _settings_debug_enabled, logger
|
||||
from ..types import ENV_FILE_SENTINEL, DotenvFiltering, DotenvType, EnvPrefixTarget
|
||||
from ..utils import (
|
||||
InitState,
|
||||
_annotation_is_complex,
|
||||
_get_field_annotation_and_metadata,
|
||||
_resolve_config_file,
|
||||
_union_is_complex,
|
||||
parse_env_vars,
|
||||
)
|
||||
from .env import EnvSettingsSource
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pydantic_settings.main import BaseSettings
|
||||
|
||||
|
||||
class DotEnvSettingsSource(EnvSettingsSource):
|
||||
"""
|
||||
Source class for loading settings values from env files.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
settings_cls: type[BaseSettings],
|
||||
env_file: DotenvType | None = ENV_FILE_SENTINEL,
|
||||
env_file_encoding: str | None = None,
|
||||
dotenv_filtering: DotenvFiltering | None = None,
|
||||
case_sensitive: bool | None = None,
|
||||
env_prefix: str | None = None,
|
||||
env_prefix_target: EnvPrefixTarget | None = None,
|
||||
env_nested_delimiter: str | None = None,
|
||||
env_nested_max_split: int | None = None,
|
||||
env_ignore_empty: bool | None = None,
|
||||
env_parse_none_str: str | None = None,
|
||||
env_parse_enums: bool | None = None,
|
||||
env_file_depth: int | None = None,
|
||||
_init_state: InitState | None = None,
|
||||
) -> None:
|
||||
self.env_file = env_file if env_file != ENV_FILE_SENTINEL else settings_cls.model_config.get('env_file')
|
||||
self.env_file_encoding = (
|
||||
env_file_encoding if env_file_encoding is not None else settings_cls.model_config.get('env_file_encoding')
|
||||
)
|
||||
self.env_file_depth = (
|
||||
env_file_depth if env_file_depth is not None else settings_cls.model_config.get('env_file_depth', 0)
|
||||
)
|
||||
self.dotenv_filtering = (
|
||||
dotenv_filtering if dotenv_filtering is not None else settings_cls.model_config.get('dotenv_filtering')
|
||||
)
|
||||
super().__init__(
|
||||
settings_cls,
|
||||
case_sensitive,
|
||||
env_prefix,
|
||||
env_prefix_target,
|
||||
env_nested_delimiter,
|
||||
env_nested_max_split,
|
||||
env_ignore_empty,
|
||||
env_parse_none_str,
|
||||
env_parse_enums,
|
||||
_init_state,
|
||||
)
|
||||
|
||||
def _load_env_vars(self) -> Mapping[str, str | None]:
|
||||
return self._read_env_files()
|
||||
|
||||
@staticmethod
|
||||
def _static_read_env_file(
|
||||
file_path: Path,
|
||||
*,
|
||||
encoding: str | None = None,
|
||||
case_sensitive: bool = False,
|
||||
ignore_empty: bool = False,
|
||||
parse_none_str: str | None = None,
|
||||
) -> Mapping[str, str | None]:
|
||||
encoding = encoding if encoding is not None else 'utf-8'
|
||||
file_vars: dict[str, str | None] = dotenv_values(file_path, encoding=encoding)
|
||||
return parse_env_vars(file_vars, case_sensitive, ignore_empty, parse_none_str)
|
||||
|
||||
def _read_env_file(
|
||||
self,
|
||||
file_path: Path,
|
||||
) -> Mapping[str, str | None]:
|
||||
return self._static_read_env_file(
|
||||
file_path,
|
||||
encoding=self.env_file_encoding,
|
||||
case_sensitive=self.case_sensitive,
|
||||
ignore_empty=self.env_ignore_empty,
|
||||
parse_none_str=self.env_parse_none_str,
|
||||
)
|
||||
|
||||
def _read_env_files(self) -> Mapping[str, str | None]:
|
||||
env_files = self.env_file
|
||||
if env_files is None:
|
||||
return {}
|
||||
|
||||
if isinstance(env_files, (str, os.PathLike)):
|
||||
env_files = [env_files]
|
||||
|
||||
debug = _settings_debug_enabled() and logger.isEnabledFor(logging.DEBUG)
|
||||
|
||||
dotenv_vars: dict[str, str | None] = {}
|
||||
for env_file in env_files:
|
||||
env_path = _resolve_config_file(env_file, self.env_file_depth, allow_fifo=True)
|
||||
if env_path is not None:
|
||||
if debug:
|
||||
logger.debug('Loading env file: %s', env_path.resolve())
|
||||
dotenv_vars.update(self._read_env_file(env_path))
|
||||
elif debug:
|
||||
logger.debug('Env file not found, skipping: %s', Path(env_file).expanduser().resolve())
|
||||
|
||||
return dotenv_vars
|
||||
|
||||
def __call__(self) -> dict[str, Any]: # noqa: C901
|
||||
data: dict[str, Any] = super().__call__()
|
||||
if self.dotenv_filtering == 'only_existing':
|
||||
# This case behaves like the EnvSettingsSource, only return existing fields
|
||||
return data
|
||||
if self.dotenv_filtering == 'match_prefix':
|
||||
# In this case add all env vars that match the prefix, stripping the prefix.
|
||||
prefix = self._apply_case_sensitive(self.env_prefix)
|
||||
for env_name, env_value in self.env_vars.items():
|
||||
if env_name.startswith(prefix):
|
||||
# Slice with the case-normalized prefix: `str.lower()` can change
|
||||
# length (e.g. `'İ'` lowers to two code points), so the raw
|
||||
# `env_prefix` length would under-strip and leave a partial prefix.
|
||||
normalized_env_name = env_name[len(prefix) :]
|
||||
if (
|
||||
self.env_nested_delimiter
|
||||
and self.env_nested_delimiter in normalized_env_name
|
||||
and normalized_env_name.partition(self.env_nested_delimiter)[0] in data
|
||||
):
|
||||
continue
|
||||
if normalized_env_name not in data:
|
||||
data[normalized_env_name] = env_value
|
||||
return data
|
||||
|
||||
is_extra_allowed = self.config.get('extra') != 'forbid'
|
||||
# `env_vars` keys are case-normalised, so the prefix must be too before it
|
||||
# is matched against them.
|
||||
env_prefix = self._apply_case_sensitive(self.env_prefix)
|
||||
|
||||
# As `extra` config is allowed in dotenv settings source, We have to
|
||||
# update data with extra env variables from dotenv file.
|
||||
for env_name, env_value in self.env_vars.items():
|
||||
if not env_value or env_name in data or (self.env_prefix and env_name in self.settings_cls.model_fields):
|
||||
continue
|
||||
env_used = False
|
||||
for field_name, field in self.settings_cls.model_fields.items():
|
||||
annotation, metadata = _get_field_annotation_and_metadata(field)
|
||||
for _, field_env_name, _ in self._extract_field_info(field, field_name):
|
||||
if env_name == field_env_name or (
|
||||
(
|
||||
_annotation_is_complex(annotation, metadata, self._init_state)
|
||||
or (
|
||||
is_union_origin(get_origin(annotation))
|
||||
and _union_is_complex(annotation, metadata, self._init_state)
|
||||
)
|
||||
)
|
||||
# A var only belongs to a complex field when it sits at the nested
|
||||
# delimiter boundary (`db<delim>...`, matching what explode_env_vars
|
||||
# consumes). A bare name-prefix match (e.g. `dbx_token` vs field `db`)
|
||||
# must not claim the var, otherwise it is silently dropped instead of
|
||||
# being kept as an extra.
|
||||
and self.env_nested_delimiter
|
||||
and env_name.startswith(f'{field_env_name}{self.env_nested_delimiter}')
|
||||
):
|
||||
env_used = True
|
||||
break
|
||||
if env_used:
|
||||
break
|
||||
if not env_used:
|
||||
if is_extra_allowed and env_name.startswith(env_prefix):
|
||||
# env_prefix should be respected and removed from the env_name
|
||||
normalized_env_name = env_name[len(env_prefix) :]
|
||||
data[normalized_env_name] = env_value
|
||||
else:
|
||||
data[env_name] = env_value
|
||||
return data
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f'{self.__class__.__name__}(env_file={self.env_file!r}, env_file_encoding={self.env_file_encoding!r}, '
|
||||
f'env_file_depth={self.env_file_depth!r}, env_nested_delimiter={self.env_nested_delimiter!r}, '
|
||||
f'env_prefix_len={self.env_prefix_len!r})'
|
||||
)
|
||||
|
||||
|
||||
def read_env_file(
|
||||
file_path: Path,
|
||||
*,
|
||||
encoding: str | None = None,
|
||||
case_sensitive: bool = False,
|
||||
ignore_empty: bool = False,
|
||||
parse_none_str: str | None = None,
|
||||
) -> Mapping[str, str | None]:
|
||||
warnings.warn(
|
||||
'read_env_file will be removed in the next version, use DotEnvSettingsSource._static_read_env_file if you must',
|
||||
DeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return DotEnvSettingsSource._static_read_env_file(
|
||||
file_path,
|
||||
encoding=encoding,
|
||||
case_sensitive=case_sensitive,
|
||||
ignore_empty=ignore_empty,
|
||||
parse_none_str=parse_none_str,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ['DotEnvSettingsSource', 'read_env_file']
|
||||
@@ -0,0 +1,393 @@
|
||||
from __future__ import annotations as _annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
cast,
|
||||
get_args,
|
||||
get_origin,
|
||||
)
|
||||
|
||||
from pydantic import AliasChoices, AliasPath, Json, TypeAdapter, ValidationError
|
||||
from pydantic._internal._utils import deep_update, is_model_class
|
||||
from pydantic.dataclasses import is_pydantic_dataclass
|
||||
from pydantic.fields import FieldInfo
|
||||
from typing_inspection.introspection import is_union_origin
|
||||
|
||||
from ...utils import _lenient_issubclass
|
||||
from ..base import PydanticBaseEnvSettingsSource
|
||||
from ..types import EnvNoneType, EnvPrefixTarget
|
||||
from ..utils import (
|
||||
InitState,
|
||||
_annotation_contains_types,
|
||||
_annotation_enum_name_to_val,
|
||||
_annotation_is_complex,
|
||||
_get_field_annotation_and_metadata,
|
||||
_get_model_config,
|
||||
_get_model_fields,
|
||||
_literal_has_numeric_enum,
|
||||
_union_has_strict_types,
|
||||
_union_is_complex,
|
||||
parse_env_vars,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pydantic_settings.main import BaseSettings
|
||||
|
||||
|
||||
def _environ_is_case_insensitive() -> bool:
|
||||
"""Whether ``os.environ`` is the OS-backed, case-insensitive mapping (i.e. on Windows).
|
||||
|
||||
When ``os.environ`` has been replaced (e.g. patched to a plain ``dict`` in tests), the
|
||||
requested case sensitivity is honored as-is.
|
||||
"""
|
||||
return os.name == 'nt' and isinstance(os.environ, os._Environ)
|
||||
|
||||
|
||||
class EnvSettingsSource(PydanticBaseEnvSettingsSource):
|
||||
"""
|
||||
Source class for loading settings values from environment variables.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
settings_cls: type[BaseSettings],
|
||||
case_sensitive: bool | None = None,
|
||||
env_prefix: str | None = None,
|
||||
env_prefix_target: EnvPrefixTarget | None = None,
|
||||
env_nested_delimiter: str | None = None,
|
||||
env_nested_max_split: int | None = None,
|
||||
env_ignore_empty: bool | None = None,
|
||||
env_parse_none_str: str | None = None,
|
||||
env_parse_enums: bool | None = None,
|
||||
_init_state: InitState | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
settings_cls,
|
||||
case_sensitive,
|
||||
env_prefix,
|
||||
env_prefix_target,
|
||||
env_ignore_empty,
|
||||
env_parse_none_str,
|
||||
env_parse_enums,
|
||||
_init_state,
|
||||
)
|
||||
self.env_nested_delimiter = (
|
||||
env_nested_delimiter if env_nested_delimiter is not None else self.config.get('env_nested_delimiter')
|
||||
)
|
||||
self.env_nested_max_split = (
|
||||
env_nested_max_split if env_nested_max_split is not None else self.config.get('env_nested_max_split')
|
||||
)
|
||||
self.maxsplit = (self.env_nested_max_split or 0) - 1
|
||||
self.env_prefix_len = len(self.env_prefix)
|
||||
|
||||
self.env_vars = self._load_env_vars()
|
||||
|
||||
def _load_env_vars(self) -> Mapping[str, str | None]:
|
||||
# On Windows, ``os.environ`` is case-insensitive: variable names are normalized to
|
||||
# upper-case and looked up regardless of case. Matching environment variables
|
||||
# case-sensitively is therefore impossible, so we fall back to case-insensitive
|
||||
# matching to avoid spurious "field required" errors (see #295). ``.env`` files do
|
||||
# preserve case and are handled by ``DotEnvSettingsSource``, which overrides this method.
|
||||
if self.case_sensitive and _environ_is_case_insensitive():
|
||||
self.case_sensitive = False
|
||||
return parse_env_vars(os.environ, self.case_sensitive, self.env_ignore_empty, self.env_parse_none_str)
|
||||
|
||||
def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
|
||||
"""
|
||||
Gets the value for field from environment variables and a flag to determine whether value is complex.
|
||||
|
||||
Args:
|
||||
field: The field.
|
||||
field_name: The field name.
|
||||
|
||||
Returns:
|
||||
A tuple that contains the value (`None` if not found), key, and
|
||||
a flag to determine whether value is complex.
|
||||
"""
|
||||
|
||||
env_val: str | None = None
|
||||
# `field_key` and `value_is_complex` are deliberately read after the loop, so B007's
|
||||
# "unused loop variable" report doesn't apply here.
|
||||
for field_key, env_name, value_is_complex in self._extract_field_info(field, field_name): # noqa: B007
|
||||
env_val = self.env_vars.get(env_name)
|
||||
if env_val is not None:
|
||||
break
|
||||
|
||||
return env_val, field_key, value_is_complex
|
||||
|
||||
def prepare_field_value(self, field_name: str, field: FieldInfo, value: Any, value_is_complex: bool) -> Any:
|
||||
"""
|
||||
Prepare value for the field.
|
||||
|
||||
* Extract value for nested field.
|
||||
* Deserialize value to python object for complex field.
|
||||
|
||||
Args:
|
||||
field: The field.
|
||||
field_name: The field name.
|
||||
|
||||
Returns:
|
||||
A tuple contains prepared value for the field.
|
||||
|
||||
Raises:
|
||||
ValuesError: When There is an error in deserializing value for complex field.
|
||||
"""
|
||||
is_complex, allow_parse_failure = self._field_is_complex(field)
|
||||
if self.env_parse_enums:
|
||||
enum_val = _annotation_enum_name_to_val(field.annotation, value)
|
||||
value = value if enum_val is None else enum_val
|
||||
|
||||
if is_complex or value_is_complex:
|
||||
if isinstance(value, EnvNoneType):
|
||||
return value
|
||||
if value is None:
|
||||
# field is complex but no value found so far, try explode_env_vars
|
||||
env_val_built = self.explode_env_vars(field_name, field, self.env_vars)
|
||||
if env_val_built:
|
||||
return env_val_built
|
||||
else:
|
||||
# field is complex and there's a value, decode that as JSON, then add explode_env_vars
|
||||
try:
|
||||
value = self.decode_complex_value(field_name, field, value)
|
||||
except ValueError:
|
||||
if not allow_parse_failure:
|
||||
raise
|
||||
|
||||
if isinstance(value, dict):
|
||||
return deep_update(value, self.explode_env_vars(field_name, field, self.env_vars))
|
||||
return value
|
||||
elif value is not None:
|
||||
# simplest case, field is not complex, we only need to add the value if it was found
|
||||
return self._coerce_env_val_strict(field, value)
|
||||
|
||||
# No value resolved for this field; `None` tells the caller to skip it.
|
||||
return None
|
||||
|
||||
def _matches_alias_path_head(self, field: FieldInfo | Any | None, key: str) -> bool:
|
||||
"""
|
||||
Whether ``key`` matches the first element of a multi-part ``AliasPath`` on ``field``.
|
||||
|
||||
When a nested field is aliased via e.g. ``AliasPath('path', 0)``, the env value found
|
||||
under ``path`` is a container that must be JSON-decoded before pydantic can navigate
|
||||
into it (see #670). This mirrors the complexity that ``_extract_field_info`` already
|
||||
reports for top-level ``AliasPath`` fields.
|
||||
|
||||
Args:
|
||||
field: The field (or raw type) matched for ``key``.
|
||||
key: The (case-normalized) env name segment matched for the field.
|
||||
|
||||
Returns:
|
||||
Whether ``key`` is the head of a multi-element ``AliasPath``.
|
||||
"""
|
||||
if not isinstance(field, FieldInfo):
|
||||
return False
|
||||
v_alias = field.validation_alias
|
||||
paths: list[tuple[str | int, ...]] = []
|
||||
if isinstance(v_alias, AliasPath):
|
||||
paths.append(tuple(v_alias.path))
|
||||
elif isinstance(v_alias, AliasChoices):
|
||||
paths.extend(tuple(choice.path) for choice in v_alias.choices if isinstance(choice, AliasPath))
|
||||
for path in paths:
|
||||
head = path[0]
|
||||
if len(path) > 1 and isinstance(head, str) and self._apply_case_sensitive(head) == key:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _field_is_complex(self, field: FieldInfo) -> tuple[bool, bool]:
|
||||
"""
|
||||
Find out if a field is complex, and if so whether JSON errors should be ignored
|
||||
"""
|
||||
annotation, metadata = _get_field_annotation_and_metadata(field)
|
||||
if self.field_is_complex(field):
|
||||
allow_parse_failure = False
|
||||
elif is_union_origin(get_origin(annotation)) and _union_is_complex(annotation, metadata, self._init_state):
|
||||
allow_parse_failure = True
|
||||
else:
|
||||
return False, False
|
||||
|
||||
return True, allow_parse_failure
|
||||
|
||||
# Default value of `case_sensitive` is `None`, because we don't want to break existing behavior.
|
||||
# We have to change the method to a non-static method and use
|
||||
# `self.case_sensitive` instead in V3.
|
||||
def next_field(
|
||||
self, field: FieldInfo | Any | None, key: str, case_sensitive: bool | None = None
|
||||
) -> FieldInfo | None:
|
||||
"""
|
||||
Find the field in a sub model by key(env name)
|
||||
|
||||
By having the following models:
|
||||
|
||||
```py
|
||||
class SubSubModel(BaseSettings):
|
||||
dvals: Dict
|
||||
|
||||
class SubModel(BaseSettings):
|
||||
vals: list[str]
|
||||
sub_sub_model: SubSubModel
|
||||
|
||||
class Cfg(BaseSettings):
|
||||
sub_model: SubModel
|
||||
```
|
||||
|
||||
Then:
|
||||
next_field(sub_model, 'vals') Returns the `vals` field of `SubModel` class
|
||||
next_field(sub_model, 'sub_sub_model') Returns `sub_sub_model` field of `SubModel` class
|
||||
|
||||
Args:
|
||||
field: The field.
|
||||
key: The key (env name).
|
||||
case_sensitive: Whether to search for key case sensitively.
|
||||
|
||||
Returns:
|
||||
Field if it finds the next field otherwise `None`.
|
||||
"""
|
||||
if not field:
|
||||
return None
|
||||
|
||||
annotation = field.annotation if isinstance(field, FieldInfo) else field
|
||||
for type_ in get_args(annotation):
|
||||
type_has_key = self.next_field(type_, key, case_sensitive)
|
||||
if type_has_key:
|
||||
return type_has_key
|
||||
if _lenient_issubclass(get_origin(annotation), dict):
|
||||
# get value type if it's a dict
|
||||
args = get_args(annotation)
|
||||
return args[-1] if args else None
|
||||
if is_model_class(annotation) or is_pydantic_dataclass(annotation): # type: ignore[arg-type]
|
||||
fields = _get_model_fields(annotation)
|
||||
model_config = _get_model_config(annotation)
|
||||
# `case_sensitive is None` is here to be compatible with the old behavior.
|
||||
# Has to be removed in V3.
|
||||
for field_name, f in fields.items():
|
||||
for _, env_name, _ in self._extract_field_info(f, field_name, model_config):
|
||||
if case_sensitive is None or case_sensitive:
|
||||
if field_name == key or env_name == key:
|
||||
return f
|
||||
elif field_name.lower() == key.lower() or env_name.lower() == key.lower():
|
||||
return f
|
||||
return None
|
||||
|
||||
def explode_env_vars(self, field_name: str, field: FieldInfo, env_vars: Mapping[str, str | None]) -> dict[str, Any]: # noqa: C901
|
||||
"""
|
||||
Process env_vars and extract the values of keys containing env_nested_delimiter into nested dictionaries.
|
||||
|
||||
This is applied to a single field, hence filtering by env_var prefix.
|
||||
|
||||
Args:
|
||||
field_name: The field name.
|
||||
field: The field.
|
||||
env_vars: Environment variables.
|
||||
|
||||
Returns:
|
||||
A dictionary contains extracted values from nested env values.
|
||||
"""
|
||||
if not self.env_nested_delimiter:
|
||||
return {}
|
||||
|
||||
ann = field.annotation
|
||||
is_dict = ann is dict or _lenient_issubclass(get_origin(ann), dict)
|
||||
|
||||
prefixes = [
|
||||
f'{env_name}{self.env_nested_delimiter}' for _, env_name, _ in self._extract_field_info(field, field_name)
|
||||
]
|
||||
result: dict[str, Any] = {}
|
||||
for env_name, env_val in env_vars.items():
|
||||
try:
|
||||
prefix = next(prefix for prefix in prefixes if env_name.startswith(prefix))
|
||||
except StopIteration:
|
||||
continue
|
||||
# we remove the prefix before splitting in case the prefix has characters in common with the delimiter
|
||||
env_name_without_prefix = env_name[len(prefix) :]
|
||||
*keys, last_key = env_name_without_prefix.split(self.env_nested_delimiter, self.maxsplit)
|
||||
env_var = result
|
||||
target_field: FieldInfo | None = field
|
||||
for key in keys:
|
||||
target_field = self.next_field(target_field, key, self.case_sensitive)
|
||||
if isinstance(env_var, dict):
|
||||
env_var = env_var.setdefault(key, {})
|
||||
|
||||
# get proper field with last_key
|
||||
target_field = self.next_field(target_field, last_key, self.case_sensitive)
|
||||
|
||||
# check if env_val maps to a complex field and if so, parse the env_val
|
||||
if (target_field or is_dict) and env_val:
|
||||
if isinstance(target_field, FieldInfo):
|
||||
is_complex, allow_json_failure = self._field_is_complex(target_field)
|
||||
if not is_complex and self._matches_alias_path_head(target_field, last_key):
|
||||
# The env value sits under an AliasPath head (e.g. AliasPath('path', 0));
|
||||
# decode it so pydantic can navigate into the container (see #670).
|
||||
is_complex, allow_json_failure = True, True
|
||||
if self.env_parse_enums:
|
||||
enum_val = _annotation_enum_name_to_val(target_field.annotation, env_val)
|
||||
env_val = env_val if enum_val is None else enum_val
|
||||
elif target_field:
|
||||
# target_field is a raw type (e.g. from dict value type annotation)
|
||||
is_complex = _annotation_is_complex(target_field, [], self._init_state)
|
||||
allow_json_failure = True
|
||||
else:
|
||||
# nested field type is dict
|
||||
is_complex, allow_json_failure = True, True
|
||||
if is_complex:
|
||||
try:
|
||||
field_info = target_field if isinstance(target_field, FieldInfo) else None
|
||||
env_val = self.decode_complex_value(last_key, field_info, env_val) # type: ignore
|
||||
except ValueError:
|
||||
if not allow_json_failure:
|
||||
raise
|
||||
if isinstance(env_var, dict) and (
|
||||
last_key not in env_var or not isinstance(env_val, EnvNoneType) or env_var[last_key] == {}
|
||||
):
|
||||
env_var[last_key] = self._coerce_env_val_strict(target_field, env_val)
|
||||
return result
|
||||
|
||||
def _coerce_env_val_strict(self, field: FieldInfo | None, value: Any) -> Any:
|
||||
"""
|
||||
Coerce environment string values based on field annotation if model config is `strict=True`
|
||||
or if the field annotation contains strict-annotated types (e.g. Optional[StrictBool]).
|
||||
|
||||
Args:
|
||||
field: The field.
|
||||
value: The value to coerce.
|
||||
|
||||
Returns:
|
||||
The coerced value if successful, otherwise the original value.
|
||||
"""
|
||||
try:
|
||||
should_coerce = self.config.get('strict')
|
||||
if isinstance(field, FieldInfo):
|
||||
annotation, metadata = _get_field_annotation_and_metadata(field)
|
||||
if not should_coerce:
|
||||
should_coerce = (
|
||||
is_union_origin(get_origin(annotation)) and _union_has_strict_types(annotation)
|
||||
) or _literal_has_numeric_enum(annotation)
|
||||
if should_coerce and isinstance(value, str) and isinstance(field, FieldInfo):
|
||||
if value == self.env_parse_none_str:
|
||||
return value
|
||||
has_json = any(isinstance(md, Json) for md in metadata) # type: ignore[misc]
|
||||
if not has_json and not _annotation_contains_types(annotation, (Json,), is_instance=True):
|
||||
try:
|
||||
return TypeAdapter(cast('Any', field.annotation)).validate_python(value)
|
||||
except ValidationError:
|
||||
# Try JSON decoding as fallback (e.g. 'true' -> True for StrictBool)
|
||||
decoded = json.loads(value)
|
||||
if not isinstance(decoded, str):
|
||||
return TypeAdapter(cast('Any', field.annotation)).validate_python(decoded)
|
||||
raise
|
||||
except (ValidationError, json.JSONDecodeError):
|
||||
# Allow validation error to be raised at time of instantiation
|
||||
pass
|
||||
return value
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f'{self.__class__.__name__}(env_nested_delimiter={self.env_nested_delimiter!r}, '
|
||||
f'env_prefix_len={self.env_prefix_len!r})'
|
||||
)
|
||||
|
||||
|
||||
__all__ = ['EnvSettingsSource']
|
||||
@@ -0,0 +1,365 @@
|
||||
from __future__ import annotations as _annotations
|
||||
|
||||
import warnings
|
||||
from collections.abc import Iterator, Mapping
|
||||
from functools import cached_property
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from pydantic.fields import FieldInfo
|
||||
|
||||
from ...exceptions import SettingsError
|
||||
from ..types import SecretVersion
|
||||
from ..utils import InitState, _parse_env_none_str, _validate_by_name_enabled
|
||||
from .env import EnvSettingsSource
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from google.auth import default as google_auth_default
|
||||
from google.auth.credentials import Credentials
|
||||
from google.cloud.secretmanager import SecretManagerServiceClient
|
||||
|
||||
from pydantic_settings.main import BaseSettings
|
||||
else:
|
||||
Credentials = None
|
||||
SecretManagerServiceClient = None
|
||||
google_auth_default = None
|
||||
|
||||
|
||||
def import_gcp_secret_manager() -> None:
|
||||
global Credentials
|
||||
global SecretManagerServiceClient
|
||||
global google_auth_default
|
||||
|
||||
try:
|
||||
from google.auth import default as google_auth_default
|
||||
from google.auth.credentials import Credentials
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.filterwarnings('ignore', category=FutureWarning)
|
||||
from google.cloud.secretmanager import SecretManagerServiceClient
|
||||
except ImportError as e: # pragma: no cover
|
||||
raise ImportError(
|
||||
'GCP Secret Manager dependencies are not installed, run `pip install pydantic-settings[gcp-secret-manager]`'
|
||||
) from e
|
||||
|
||||
|
||||
def _is_not_found_error(exc: Exception) -> bool:
|
||||
try:
|
||||
from google.api_core.exceptions import NotFound
|
||||
|
||||
return isinstance(exc, NotFound)
|
||||
except ImportError: # pragma: no cover
|
||||
return False
|
||||
|
||||
|
||||
class GoogleSecretManagerMapping(Mapping[str, str | None]):
|
||||
_loaded_secrets: dict[str, str | None]
|
||||
_secret_client: SecretManagerServiceClient
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
secret_client: SecretManagerServiceClient,
|
||||
project_id: str,
|
||||
case_sensitive: bool,
|
||||
env_parse_none_str: str | None = None,
|
||||
) -> None:
|
||||
self._loaded_secrets = {}
|
||||
self._secret_client = secret_client
|
||||
self._project_id = project_id
|
||||
self._case_sensitive = case_sensitive
|
||||
self._env_parse_none_str = env_parse_none_str
|
||||
|
||||
@property
|
||||
def _gcp_project_path(self) -> str:
|
||||
return self._secret_client.common_project_path(self._project_id)
|
||||
|
||||
def _select_case_insensitive_secret(self, lower_name: str, candidates: list[str]) -> str:
|
||||
if len(candidates) == 1:
|
||||
return candidates[0]
|
||||
|
||||
# Sort to ensure deterministic selection (prefer lowercase / ASCII last)
|
||||
candidates.sort()
|
||||
winner = candidates[-1]
|
||||
warnings.warn(
|
||||
f"Secret collision: Found multiple secrets {candidates} normalizing to '{lower_name}'. "
|
||||
f"Using '{winner}' for case-insensitive lookup.",
|
||||
UserWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return winner
|
||||
|
||||
@cached_property
|
||||
def _secret_name_map(self) -> dict[str, str]:
|
||||
mapping: dict[str, str] = {}
|
||||
# Group secrets by normalized name to detect collisions
|
||||
normalized_groups: dict[str, list[str]] = {}
|
||||
|
||||
secrets = self._secret_client.list_secrets(parent=self._gcp_project_path)
|
||||
for secret in secrets:
|
||||
name = self._secret_client.parse_secret_path(secret.name).get('secret', '')
|
||||
mapping[name] = name
|
||||
|
||||
if not self._case_sensitive:
|
||||
lower_name = name.lower()
|
||||
if lower_name not in normalized_groups:
|
||||
normalized_groups[lower_name] = []
|
||||
normalized_groups[lower_name].append(name)
|
||||
|
||||
if not self._case_sensitive:
|
||||
for lower_name, candidates in normalized_groups.items():
|
||||
mapping[lower_name] = self._select_case_insensitive_secret(lower_name, candidates)
|
||||
|
||||
return mapping
|
||||
|
||||
@property
|
||||
def _secret_names(self) -> list[str]:
|
||||
return list(self._secret_name_map.keys())
|
||||
|
||||
def _secret_version_path(self, key: str, version: str = 'latest') -> str:
|
||||
return self._secret_client.secret_version_path(self._project_id, key, version)
|
||||
|
||||
def _get_secret_value(self, gcp_secret_name: str, version: str = 'latest') -> str | None:
|
||||
try:
|
||||
value = self._secret_client.access_secret_version(
|
||||
name=self._secret_version_path(gcp_secret_name, version)
|
||||
).payload.data.decode('UTF-8')
|
||||
except Exception:
|
||||
return None
|
||||
return _parse_env_none_str(value, self._env_parse_none_str)
|
||||
|
||||
def _get_secret_value_or_raise(self, gcp_secret_name: str) -> str | None:
|
||||
try:
|
||||
value = self._secret_client.access_secret_version(
|
||||
name=self._secret_version_path(gcp_secret_name)
|
||||
).payload.data.decode('UTF-8')
|
||||
except Exception as e:
|
||||
if _is_not_found_error(e):
|
||||
raise KeyError(gcp_secret_name) from e
|
||||
return None
|
||||
return _parse_env_none_str(value, self._env_parse_none_str)
|
||||
|
||||
def __getitem__(self, key: str) -> str | None:
|
||||
if key in self._loaded_secrets:
|
||||
return self._loaded_secrets[key]
|
||||
|
||||
if self._case_sensitive:
|
||||
self._loaded_secrets[key] = self._get_secret_value_or_raise(key)
|
||||
return self._loaded_secrets[key]
|
||||
|
||||
gcp_secret_name = self._secret_name_map.get(key)
|
||||
if gcp_secret_name is None:
|
||||
gcp_secret_name = self._secret_name_map.get(key.lower())
|
||||
|
||||
if gcp_secret_name:
|
||||
self._loaded_secrets[key] = self._get_secret_value(gcp_secret_name)
|
||||
else:
|
||||
raise KeyError(key)
|
||||
|
||||
return self._loaded_secrets[key]
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._secret_names)
|
||||
|
||||
def __iter__(self) -> Iterator[str]:
|
||||
return iter(self._secret_names)
|
||||
|
||||
|
||||
class GoogleSecretManagerSettingsSource(EnvSettingsSource):
|
||||
_credentials: Credentials | None
|
||||
_secret_client: SecretManagerServiceClient | None
|
||||
_project_id: str | None
|
||||
_explicit_project_id: str | None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
settings_cls: type[BaseSettings],
|
||||
credentials: Credentials | None = None,
|
||||
project_id: str | None = None,
|
||||
env_prefix: str | None = None,
|
||||
env_parse_none_str: str | None = None,
|
||||
env_parse_enums: bool | None = None,
|
||||
secret_client: SecretManagerServiceClient | None = None,
|
||||
case_sensitive: bool | None = True,
|
||||
project_id_field: str = 'project_id',
|
||||
_init_state: InitState | None = None,
|
||||
) -> None:
|
||||
"""Settings source that reads secrets from Google Cloud Secret Manager.
|
||||
|
||||
Args:
|
||||
project_id: The GCP project to read secrets from. If not provided, it is
|
||||
resolved lazily (see below).
|
||||
project_id_field: The key, populated by a previous (higher priority)
|
||||
settings source, to use as the ``project_id`` when one is not passed
|
||||
explicitly. This must match the key used in ``current_state`` by the
|
||||
previous source (typically the field name or its preferred alias).
|
||||
For example, with ``some_field: str = Field(alias='GCP_PROJECT')``,
|
||||
previous env sources will expose the value under ``'GCP_PROJECT'``.
|
||||
Defaults to ``'project_id'``.
|
||||
|
||||
The ``project_id`` is resolved lazily in :meth:`__call__` rather than at
|
||||
construction time so that it can be sourced from settings resolved by previous
|
||||
sources (only available via ``current_state`` once the source is called).
|
||||
Resolution order is:
|
||||
|
||||
1. the explicit ``project_id`` argument
|
||||
2. the ``project_id_field`` value from previous settings sources
|
||||
3. ``google.auth.default()``
|
||||
"""
|
||||
# Import Google Packages if they haven't already been imported
|
||||
if SecretManagerServiceClient is None or Credentials is None or google_auth_default is None:
|
||||
import_gcp_secret_manager()
|
||||
|
||||
# Resolution is deferred to __call__ (see _resolve_gcp_project / _load_env_vars).
|
||||
# _explicit_project_id is the immutable user input; _project_id is the resolved
|
||||
# value and is written exclusively by _resolve_gcp_project.
|
||||
self._explicit_project_id = project_id
|
||||
self._project_id = None
|
||||
self._credentials = credentials
|
||||
self._secret_client = secret_client
|
||||
self._project_id_field = project_id_field
|
||||
self._env_vars_loaded = False
|
||||
|
||||
super().__init__(
|
||||
settings_cls,
|
||||
case_sensitive=case_sensitive,
|
||||
env_prefix=env_prefix,
|
||||
env_ignore_empty=False,
|
||||
env_parse_none_str=env_parse_none_str,
|
||||
env_parse_enums=env_parse_enums,
|
||||
_init_state=_init_state,
|
||||
)
|
||||
# __init__ is past; subsequent _load_env_vars() calls may now resolve the project.
|
||||
self._env_vars_loaded = True
|
||||
|
||||
def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
|
||||
"""Override get_field_value to get the secret value from GCP Secret Manager.
|
||||
Look for a SecretVersion metadata field to specify a particular SecretVersion.
|
||||
|
||||
Args:
|
||||
field: The field to get the value for
|
||||
field_name: The declared name of the field
|
||||
|
||||
Returns:
|
||||
A tuple of (value, key, value_is_complex), where `key` is the identifier used
|
||||
to populate the model (either the field name or an alias, depending on
|
||||
configuration).
|
||||
"""
|
||||
|
||||
secret_version = next((m.version for m in field.metadata if isinstance(m, SecretVersion)), None)
|
||||
|
||||
# If a secret version is specified, try to get that specific version of the secret from
|
||||
# GCP Secret Manager via the GoogleSecretManagerMapping. This allows different versions
|
||||
# of the same secret name to be retrieved independently and cached in the GoogleSecretManagerMapping
|
||||
if secret_version and isinstance(self.env_vars, GoogleSecretManagerMapping):
|
||||
for field_key, env_name, value_is_complex in self._extract_field_info(field, field_name):
|
||||
if self.case_sensitive:
|
||||
gcp_secret_name: str | None = env_name
|
||||
else:
|
||||
gcp_secret_name = self.env_vars._secret_name_map.get(env_name)
|
||||
if gcp_secret_name is None: # pragma: no cover
|
||||
# Unreachable in practice: `_extract_field_info` already lower-cases
|
||||
# `env_name` when case-insensitive. Kept as a defensive fallback.
|
||||
gcp_secret_name = self.env_vars._secret_name_map.get(env_name.lower())
|
||||
|
||||
if gcp_secret_name:
|
||||
env_val = self.env_vars._get_secret_value(gcp_secret_name, secret_version)
|
||||
if env_val is not None:
|
||||
# If validation by name is enabled, return field_name to allow multiple fields
|
||||
# with the same alias but different versions to be distinguished
|
||||
if _validate_by_name_enabled(self.config):
|
||||
return env_val, field_name, value_is_complex
|
||||
return env_val, field_key, value_is_complex
|
||||
|
||||
# If a secret version is specified but not found, we should not fall back to "latest" (default behavior)
|
||||
# as that would be incorrect. We return None to indicate the value was not found.
|
||||
return None, field_name, False
|
||||
|
||||
val, key, is_complex = super().get_field_value(field, field_name)
|
||||
|
||||
# If validation by name is enabled, we need to return the field_name as the key
|
||||
# without this being enabled, you cannot load two secrets with the same name but different versions
|
||||
if _validate_by_name_enabled(self.config) and val is not None:
|
||||
return val, field_name, is_complex
|
||||
return val, key, is_complex
|
||||
|
||||
def _resolve_gcp_project(self) -> None:
|
||||
"""Resolve the credentials, project_id and Secret Manager client.
|
||||
|
||||
``project_id`` is resolved, in order of precedence, from: the explicit
|
||||
``project_id`` argument, the ``project_id_field`` value from previous settings
|
||||
sources (``current_state``), and finally ``google.auth.default()``.
|
||||
"""
|
||||
if self._project_id is not None:
|
||||
return
|
||||
|
||||
project_id = self._explicit_project_id
|
||||
credentials = self._credentials
|
||||
|
||||
# Fall back to a value resolved by a previous (higher priority) settings source.
|
||||
if project_id is None:
|
||||
state_project_id = self.current_state.get(self._project_id_field)
|
||||
if isinstance(state_project_id, str):
|
||||
project_id = state_project_id
|
||||
|
||||
# Fall back to google.auth.default for whatever is still missing.
|
||||
# Credentials are only needed if we have to build a client ourselves — if the
|
||||
# caller supplied a pre-built secret_client we can skip the auth call entirely
|
||||
# when project_id is also known, avoiding an unnecessary RPC / file read.
|
||||
need_credentials = self._secret_client is None and credentials is None
|
||||
need_project = project_id is None
|
||||
if need_credentials or need_project:
|
||||
_creds, _project_id = google_auth_default()
|
||||
if need_credentials:
|
||||
credentials = _creds
|
||||
if need_project and isinstance(_project_id, str):
|
||||
project_id = _project_id
|
||||
|
||||
if project_id is None:
|
||||
raise AttributeError(
|
||||
'project_id is required to be specified either as an argument, via a previous '
|
||||
'settings source, or from google.auth.default. See '
|
||||
'https://google-auth.readthedocs.io/en/master/reference/google.auth.html#google.auth.default'
|
||||
)
|
||||
|
||||
self._credentials = credentials
|
||||
self._project_id = project_id
|
||||
if self._secret_client is None:
|
||||
self._secret_client = SecretManagerServiceClient(credentials=self._credentials)
|
||||
|
||||
def _load_env_vars(self) -> Mapping[str, str | None]:
|
||||
# During __init__ the previous sources have not run yet, so defer building the
|
||||
# mapping until __call__, where current_state (and thus project_id) is available.
|
||||
if not self._env_vars_loaded:
|
||||
return {}
|
||||
|
||||
self._resolve_gcp_project()
|
||||
# Unreachable in practice: `_resolve_gcp_project` either raises or sets both
|
||||
# attributes. Kept as a defensive fallback.
|
||||
if self._project_id is None or self._secret_client is None: # pragma: no cover
|
||||
raise SettingsError(
|
||||
'GoogleSecretManagerSettingsSource: could not determine GCP project_id or initialize the Secret Manager client. '
|
||||
'Pass project_id explicitly or ensure it is available via application default credentials or a previous settings source.'
|
||||
)
|
||||
return GoogleSecretManagerMapping(
|
||||
self._secret_client,
|
||||
project_id=self._project_id,
|
||||
case_sensitive=self.case_sensitive,
|
||||
env_parse_none_str=self.env_parse_none_str,
|
||||
)
|
||||
|
||||
def __call__(self) -> dict[str, Any]:
|
||||
# current_state is populated by previous sources right before __call__; (re)build
|
||||
# the mapping now so project_id can be sourced from them.
|
||||
self._env_vars_loaded = True
|
||||
self.env_vars = self._load_env_vars()
|
||||
return super().__call__()
|
||||
|
||||
def __repr__(self) -> str:
|
||||
# Prefer the resolved project_id, falling back to the explicit constructor
|
||||
# arg so the repr is informative before the source has been called.
|
||||
project_id = self._project_id if self._project_id is not None else self._explicit_project_id
|
||||
return (
|
||||
f'{self.__class__.__name__}(project_id={project_id!r}, env_nested_delimiter={self.env_nested_delimiter!r})'
|
||||
)
|
||||
|
||||
|
||||
__all__ = ['GoogleSecretManagerMapping', 'GoogleSecretManagerSettingsSource']
|
||||
@@ -0,0 +1,61 @@
|
||||
"""JSON file settings source."""
|
||||
|
||||
from __future__ import annotations as _annotations
|
||||
|
||||
import json
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
)
|
||||
|
||||
from ..base import ConfigFileSourceMixin, InitSettingsSource
|
||||
from ..types import DEFAULT_PATH, ConfigFileSourceType
|
||||
from ..utils import InitState
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic_settings.main import BaseSettings
|
||||
|
||||
from ..types import Traversable
|
||||
|
||||
|
||||
class JsonConfigSettingsSource(InitSettingsSource, ConfigFileSourceMixin):
|
||||
"""
|
||||
A source class that loads variables from a JSON file
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
settings_cls: type[BaseSettings],
|
||||
json_file: ConfigFileSourceType | None = DEFAULT_PATH,
|
||||
json_file_encoding: str | None = None,
|
||||
deep_merge: bool = False,
|
||||
_init_state: InitState | None = None,
|
||||
):
|
||||
self.json_file_path = json_file if json_file != DEFAULT_PATH else settings_cls.model_config.get('json_file')
|
||||
self.json_file_encoding = (
|
||||
json_file_encoding
|
||||
if json_file_encoding is not None
|
||||
else settings_cls.model_config.get('json_file_encoding')
|
||||
)
|
||||
self.json_data = self._read_files(self.json_file_path, deep_merge=deep_merge)
|
||||
super().__init__(settings_cls, self.json_data, _init_state=_init_state)
|
||||
|
||||
def _read_file(self, file_path: Path | Traversable) -> dict[str, Any]:
|
||||
# Default to UTF-8 rather than the locale encoding, matching `DotEnvSettingsSource`
|
||||
# and `SecretsSettingsSource`: RFC 8259 mandates UTF-8 for interchanged JSON, and
|
||||
# the locale default corrupts (or fails to decode) it on a non-UTF-8 Windows code page.
|
||||
encoding = self.json_file_encoding if self.json_file_encoding is not None else 'utf-8'
|
||||
with file_path.open(encoding=encoding) as json_file:
|
||||
content = json_file.read()
|
||||
# An empty (or whitespace-only) file is not valid JSON; treat it as an empty
|
||||
# mapping so it falls back to defaults, mirroring `YamlConfigSettingsSource`'s
|
||||
# `yaml.safe_load(...) or {}` (and `tomllib`, which returns `{}` for an empty file).
|
||||
return json.loads(content) if content.strip() else {}
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f'{self.__class__.__name__}(json_file={self.json_file_path})'
|
||||
|
||||
|
||||
__all__ = ['JsonConfigSettingsSource']
|
||||
+220
@@ -0,0 +1,220 @@
|
||||
import os
|
||||
import warnings
|
||||
from collections.abc import Iterator, Mapping
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Literal, Optional
|
||||
|
||||
from ...exceptions import SettingsError
|
||||
from ...utils import path_type_label
|
||||
from ..base import PydanticBaseSettingsSource
|
||||
from ..utils import InitState, parse_env_vars
|
||||
from .env import EnvSettingsSource
|
||||
from .secrets import SecretsSettingsSource
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ...main import BaseSettings
|
||||
from ...sources import PathType
|
||||
|
||||
|
||||
SECRETS_DIR_MAX_SIZE = 16 * 2**20 # 16 MiB seems to be a reasonable default
|
||||
|
||||
|
||||
class NestedSecretsSettingsSource(EnvSettingsSource):
|
||||
def __init__(
|
||||
self,
|
||||
file_secret_settings: PydanticBaseSettingsSource | SecretsSettingsSource,
|
||||
secrets_dir: Optional['PathType'] = None,
|
||||
secrets_dir_missing: Literal['ok', 'warn', 'error'] | None = None,
|
||||
secrets_dir_max_size: int | None = None,
|
||||
secrets_case_sensitive: bool | None = None,
|
||||
secrets_prefix: str | None = None,
|
||||
secrets_nested_delimiter: str | None = None,
|
||||
secrets_nested_subdir: bool | None = None,
|
||||
# args for compatibility with SecretsSettingsSource, don't use directly
|
||||
case_sensitive: bool | None = None,
|
||||
env_prefix: str | None = None,
|
||||
_init_state: InitState | None = None,
|
||||
) -> None:
|
||||
# We allow the first argument to be settings_cls like original
|
||||
# SecretsSettingsSource. However, it is recommended to pass
|
||||
# SecretsSettingsSource instance instead (as it is shown in usage examples),
|
||||
# otherwise `_secrets_dir` arg passed to Settings() constructor will be ignored.
|
||||
settings_cls: type[BaseSettings] = getattr(
|
||||
file_secret_settings,
|
||||
'settings_cls',
|
||||
file_secret_settings, # type: ignore[arg-type]
|
||||
)
|
||||
# config options
|
||||
conf = settings_cls.model_config
|
||||
self.secrets_dir: PathType | None = first_not_none(
|
||||
getattr(file_secret_settings, 'secrets_dir', None),
|
||||
secrets_dir,
|
||||
conf.get('secrets_dir'),
|
||||
)
|
||||
self.secrets_dir_missing: Literal['ok', 'warn', 'error'] = first_not_none(
|
||||
secrets_dir_missing,
|
||||
conf.get('secrets_dir_missing'),
|
||||
'warn',
|
||||
)
|
||||
if self.secrets_dir_missing not in ('ok', 'warn', 'error'):
|
||||
raise SettingsError(f'invalid secrets_dir_missing value: {self.secrets_dir_missing}')
|
||||
self.secrets_dir_max_size: int = first_not_none(
|
||||
secrets_dir_max_size,
|
||||
conf.get('secrets_dir_max_size'),
|
||||
SECRETS_DIR_MAX_SIZE,
|
||||
)
|
||||
self.case_sensitive: bool = first_not_none(
|
||||
secrets_case_sensitive,
|
||||
conf.get('secrets_case_sensitive'),
|
||||
case_sensitive,
|
||||
conf.get('case_sensitive'),
|
||||
False,
|
||||
)
|
||||
self.secrets_prefix: str = first_not_none(
|
||||
secrets_prefix,
|
||||
conf.get('secrets_prefix'),
|
||||
env_prefix,
|
||||
conf.get('env_prefix'),
|
||||
'',
|
||||
)
|
||||
|
||||
# nested options
|
||||
self.secrets_nested_delimiter: str | None = first_not_none(
|
||||
secrets_nested_delimiter,
|
||||
conf.get('secrets_nested_delimiter'),
|
||||
conf.get('env_nested_delimiter'),
|
||||
)
|
||||
self.secrets_nested_subdir: bool = first_not_none(
|
||||
secrets_nested_subdir,
|
||||
conf.get('secrets_nested_subdir'),
|
||||
False,
|
||||
)
|
||||
if self.secrets_nested_subdir:
|
||||
if secrets_nested_delimiter or conf.get('secrets_nested_delimiter'):
|
||||
raise SettingsError('Options secrets_nested_delimiter and secrets_nested_subdir are mutually exclusive')
|
||||
self.secrets_nested_delimiter = os.sep
|
||||
|
||||
# ensure valid secrets_path
|
||||
if self.secrets_dir is None:
|
||||
paths = []
|
||||
elif isinstance(self.secrets_dir, (Path, str)):
|
||||
paths = [self.secrets_dir]
|
||||
else:
|
||||
paths = list(self.secrets_dir)
|
||||
self.secrets_paths: list[Path] = [Path(p).expanduser().resolve() for p in paths]
|
||||
for path in self.secrets_paths:
|
||||
self.validate_secrets_path(path)
|
||||
|
||||
# construct parent, inheriting the init state of the wrapped source if not provided
|
||||
super().__init__(
|
||||
settings_cls,
|
||||
case_sensitive=self.case_sensitive,
|
||||
env_prefix=self.secrets_prefix,
|
||||
env_nested_delimiter=self.secrets_nested_delimiter,
|
||||
env_ignore_empty=False, # match SecretsSettingsSource behaviour
|
||||
env_parse_enums=True, # we can pass everything here, it will still behave as "True"
|
||||
env_parse_none_str=None, # match SecretsSettingsSource behaviour
|
||||
_init_state=_init_state if _init_state is not None else getattr(file_secret_settings, '_init_state', None),
|
||||
)
|
||||
self.env_parse_none_str = None # update manually because of None
|
||||
|
||||
def _load_env_vars(self) -> Mapping[str, str | None]:
|
||||
# Overridden so that the base implementation never runs: secrets come from files,
|
||||
# not ``os.environ``, so the case-insensitivity of the Windows environment must not
|
||||
# be applied here. NTFS preserves filename case, and the downgrade would silently
|
||||
# ignore ``secrets_case_sensitive=True``. ``DotEnvSettingsSource`` overrides this
|
||||
# method for the same reason.
|
||||
self.env_parse_none_str = None # match SecretsSettingsSource behaviour
|
||||
|
||||
# Normalize each directory before merging so that the last directory wins. Merging
|
||||
# raw filenames first would keep the earlier key's insertion position, so a later
|
||||
# ``TOKEN`` could not displace an earlier ``token`` once names are case-folded.
|
||||
secrets: dict[str, str | None] = {}
|
||||
for path in self.secrets_paths:
|
||||
secrets.update(
|
||||
parse_env_vars(
|
||||
self.load_secrets(path),
|
||||
self.case_sensitive,
|
||||
self.env_ignore_empty,
|
||||
self.env_parse_none_str,
|
||||
)
|
||||
)
|
||||
return secrets
|
||||
|
||||
def validate_secrets_path(self, path: Path) -> None:
|
||||
if not path.exists():
|
||||
if self.secrets_dir_missing == 'ok':
|
||||
pass
|
||||
elif self.secrets_dir_missing == 'warn':
|
||||
warnings.warn(f'directory "{path}" does not exist', stacklevel=2)
|
||||
elif self.secrets_dir_missing == 'error':
|
||||
raise SettingsError(f'directory "{path}" does not exist')
|
||||
else: # pragma: no cover
|
||||
raise ValueError # unreachable, checked before
|
||||
else:
|
||||
if not path.is_dir():
|
||||
raise SettingsError(f'secrets_dir must reference a directory, not a {path_type_label(path)}')
|
||||
secrets_dir_size = sum(f.stat().st_size for f in self._iter_secret_files(path))
|
||||
if secrets_dir_size > self.secrets_dir_max_size:
|
||||
raise SettingsError(f'secrets_dir size is above {self.secrets_dir_max_size} bytes')
|
||||
|
||||
@staticmethod
|
||||
def _iter_secret_files(path: Path) -> Iterator[Path]:
|
||||
"""Yield the secret files contained in ``path``.
|
||||
|
||||
``path`` is expected to already be resolved. The directory tree is walked
|
||||
explicitly so that symbolic links are handled safely:
|
||||
|
||||
* a file is only yielded if its real location stays within ``path``; entries
|
||||
that resolve outside of it (e.g. through a symbolic link) are skipped, so
|
||||
they neither contribute to the ``secrets_dir_max_size`` accounting nor get
|
||||
loaded;
|
||||
* each real directory is visited at most once, so cyclic or repeated
|
||||
symlinks cannot make the walk loop and inflate the size accounting or the
|
||||
number of loaded secrets.
|
||||
|
||||
Because the size check and the loader share this iterator, they always see
|
||||
the same set of files.
|
||||
"""
|
||||
seen_dirs: set[Path] = set()
|
||||
|
||||
def walk(directory: Path) -> Iterator[Path]:
|
||||
# Guard against symlink loops / a directory reachable through multiple
|
||||
# links being traversed more than once.
|
||||
resolved_dir = directory.resolve()
|
||||
if resolved_dir in seen_dirs:
|
||||
return
|
||||
seen_dirs.add(resolved_dir)
|
||||
try:
|
||||
entries = sorted(directory.iterdir())
|
||||
except OSError:
|
||||
return
|
||||
for entry in entries:
|
||||
resolved = entry.resolve()
|
||||
if resolved.is_dir():
|
||||
# Only descend into directories that stay within secrets_dir.
|
||||
# A symlinked directory pointing outside of ``path`` is not
|
||||
# followed at all, so we never walk (potentially large) external
|
||||
# trees and never read files from outside secrets_dir.
|
||||
if resolved == path or path in resolved.parents:
|
||||
yield from walk(entry)
|
||||
elif resolved.is_file() and path in resolved.parents:
|
||||
# Defense in depth: a file whose real location escapes
|
||||
# secrets_dir (e.g. a symlink pointing outside of ``path``) is
|
||||
# skipped from both the size accounting and the load.
|
||||
yield entry
|
||||
|
||||
yield from walk(path)
|
||||
|
||||
@classmethod
|
||||
def load_secrets(cls, path: Path) -> dict[str, str]:
|
||||
# Explicit encoding, matching SecretsSettingsSource: the locale default would
|
||||
# corrupt UTF-8 secrets on a non-UTF-8 Windows code page.
|
||||
return {str(p.relative_to(path)): p.read_text(encoding='utf-8').strip() for p in cls._iter_secret_files(path)}
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f'NestedSecretsSettingsSource(secrets_dir={self.secrets_dir!r})'
|
||||
|
||||
|
||||
def first_not_none(*objs: Any) -> Any:
|
||||
return next(filter(lambda o: o is not None, objs), None)
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Pyproject TOML file settings source."""
|
||||
|
||||
from __future__ import annotations as _annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
|
||||
from ..utils import InitState
|
||||
from .toml import TomlConfigSettingsSource
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pydantic_settings.main import BaseSettings
|
||||
|
||||
|
||||
class PyprojectTomlConfigSettingsSource(TomlConfigSettingsSource):
|
||||
"""
|
||||
A source class that loads variables from a `pyproject.toml` file.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
settings_cls: type[BaseSettings],
|
||||
toml_file: Path | None = None,
|
||||
_init_state: InitState | None = None,
|
||||
) -> None:
|
||||
self.toml_file_path = self._pick_pyproject_toml_file(
|
||||
toml_file, settings_cls.model_config.get('pyproject_toml_depth', 0)
|
||||
)
|
||||
self.toml_table_header: tuple[str, ...] = settings_cls.model_config.get(
|
||||
'pyproject_toml_table_header', ('tool', 'pydantic-settings')
|
||||
)
|
||||
self.toml_data = self._read_files(self.toml_file_path)
|
||||
for key in self.toml_table_header:
|
||||
self.toml_data = self.toml_data.get(key, {})
|
||||
super(TomlConfigSettingsSource, self).__init__(settings_cls, self.toml_data, _init_state=_init_state)
|
||||
|
||||
@staticmethod
|
||||
def _pick_pyproject_toml_file(provided: Path | None, depth: int) -> Path:
|
||||
"""Pick a `pyproject.toml` file path to use.
|
||||
|
||||
Args:
|
||||
provided: Explicit path provided when instantiating this class.
|
||||
depth: Number of directories up the tree to check for a pyproject.toml.
|
||||
|
||||
"""
|
||||
if provided:
|
||||
return provided.resolve()
|
||||
rv = Path.cwd() / 'pyproject.toml'
|
||||
count = 0
|
||||
if not rv.is_file():
|
||||
child = rv.parent.parent / 'pyproject.toml'
|
||||
while count < depth:
|
||||
if child.is_file():
|
||||
return child
|
||||
if str(child.parent) == rv.root:
|
||||
break # end discovery after checking system root once
|
||||
child = child.parent.parent / 'pyproject.toml'
|
||||
count += 1
|
||||
return rv
|
||||
|
||||
|
||||
__all__ = ['PyprojectTomlConfigSettingsSource']
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Secrets file settings source."""
|
||||
|
||||
from __future__ import annotations as _annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
)
|
||||
|
||||
from pydantic.fields import FieldInfo
|
||||
|
||||
from pydantic_settings.utils import _settings_debug_enabled, logger, path_type_label
|
||||
|
||||
from ...exceptions import SettingsError
|
||||
from ..base import PydanticBaseEnvSettingsSource
|
||||
from ..types import EnvPrefixTarget, PathType
|
||||
from ..utils import InitState
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pydantic_settings.main import BaseSettings
|
||||
|
||||
|
||||
class SecretsSettingsSource(PydanticBaseEnvSettingsSource):
|
||||
"""
|
||||
Source class for loading settings values from secret files.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
settings_cls: type[BaseSettings],
|
||||
secrets_dir: PathType | None = None,
|
||||
case_sensitive: bool | None = None,
|
||||
env_prefix: str | None = None,
|
||||
env_prefix_target: EnvPrefixTarget | None = None,
|
||||
env_ignore_empty: bool | None = None,
|
||||
env_parse_none_str: str | None = None,
|
||||
env_parse_enums: bool | None = None,
|
||||
_init_state: InitState | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
settings_cls,
|
||||
case_sensitive,
|
||||
env_prefix,
|
||||
env_prefix_target,
|
||||
env_ignore_empty,
|
||||
env_parse_none_str,
|
||||
env_parse_enums,
|
||||
_init_state,
|
||||
)
|
||||
self.secrets_dir = secrets_dir if secrets_dir is not None else self.config.get('secrets_dir')
|
||||
|
||||
def __call__(self) -> dict[str, Any]:
|
||||
"""
|
||||
Build fields from "secrets" files.
|
||||
"""
|
||||
secrets: dict[str, str | None] = {}
|
||||
|
||||
if self.secrets_dir is None:
|
||||
return secrets
|
||||
|
||||
secrets_dirs = [self.secrets_dir] if isinstance(self.secrets_dir, (str, os.PathLike)) else self.secrets_dir
|
||||
secrets_paths = [Path(p).expanduser() for p in secrets_dirs]
|
||||
self.secrets_paths = []
|
||||
|
||||
for path in secrets_paths:
|
||||
if not path.exists():
|
||||
# No `stacklevel`: this source is reached at different depths — via
|
||||
# `Settings()` (through `_settings_build_values`) or by instantiating
|
||||
# the source directly — so no fixed value attributes the warning to
|
||||
# user code in both cases.
|
||||
warnings.warn(f'directory "{path}" does not exist') # noqa: B028
|
||||
else:
|
||||
self.secrets_paths.append(path)
|
||||
|
||||
if not len(self.secrets_paths):
|
||||
return secrets
|
||||
|
||||
for path in self.secrets_paths:
|
||||
if not path.is_dir():
|
||||
raise SettingsError(f'secrets_dir must reference a directory, not a {path_type_label(path)}')
|
||||
|
||||
return super().__call__()
|
||||
|
||||
@classmethod
|
||||
def find_case_path(cls, dir_path: Path, file_name: str, case_sensitive: bool) -> Path | None:
|
||||
"""
|
||||
Find a file within path's directory matching filename, optionally ignoring case.
|
||||
|
||||
Args:
|
||||
dir_path: Directory path.
|
||||
file_name: File name.
|
||||
case_sensitive: Whether to search for file name case sensitively.
|
||||
|
||||
Returns:
|
||||
Whether file path or `None` if file does not exist in directory.
|
||||
"""
|
||||
for f in dir_path.iterdir():
|
||||
if f.name == file_name or (not case_sensitive and f.name.lower() == file_name.lower()):
|
||||
return f
|
||||
return None
|
||||
|
||||
def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
|
||||
"""
|
||||
Gets the value for field from secret file and a flag to determine whether value is complex.
|
||||
|
||||
Args:
|
||||
field: The field.
|
||||
field_name: The field name.
|
||||
|
||||
Returns:
|
||||
A tuple that contains the value (`None` if the file does not exist), key, and
|
||||
a flag to determine whether value is complex.
|
||||
"""
|
||||
|
||||
debug = _settings_debug_enabled() and logger.isEnabledFor(logging.DEBUG)
|
||||
|
||||
for field_key, env_name, value_is_complex in self._extract_field_info(field, field_name):
|
||||
# paths reversed to match the last-wins behaviour of `env_file`
|
||||
for secrets_path in reversed(self.secrets_paths):
|
||||
path = self.find_case_path(secrets_path, env_name, self.case_sensitive)
|
||||
if not path:
|
||||
# path does not exist, we currently don't return a warning for this
|
||||
if debug:
|
||||
logger.debug('Secret file not found, skipping: %s', (secrets_path / env_name).resolve())
|
||||
continue
|
||||
|
||||
if path.is_file():
|
||||
if debug:
|
||||
logger.debug('Loading secret file: %s', path.resolve())
|
||||
# Explicit encoding: `read_text()` would otherwise decode with the
|
||||
# locale encoding, so a UTF-8 secret is corrupted on a non-UTF-8
|
||||
# Windows code page (silently, when every byte happens to map).
|
||||
return path.read_text(encoding='utf-8').strip(), field_key, value_is_complex
|
||||
warnings.warn(
|
||||
f'attempted to load secret file "{path}" but found a {path_type_label(path)} instead.',
|
||||
stacklevel=4,
|
||||
)
|
||||
|
||||
return None, field_key, value_is_complex
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f'{self.__class__.__name__}(secrets_dir={self.secrets_dir!r})'
|
||||
@@ -0,0 +1,72 @@
|
||||
"""TOML file settings source."""
|
||||
|
||||
from __future__ import annotations as _annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
)
|
||||
|
||||
from ..base import ConfigFileSourceMixin, InitSettingsSource
|
||||
from ..types import DEFAULT_PATH, ConfigFileSourceType
|
||||
from ..utils import InitState
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tomllib
|
||||
|
||||
from pydantic_settings.main import BaseSettings
|
||||
|
||||
from ..types import Traversable
|
||||
else:
|
||||
tomllib = None
|
||||
|
||||
|
||||
def import_toml() -> None:
|
||||
global tomllib
|
||||
if tomllib is not None:
|
||||
return
|
||||
import tomllib
|
||||
|
||||
|
||||
class TomlConfigSettingsSource(InitSettingsSource, ConfigFileSourceMixin):
|
||||
"""
|
||||
A source class that loads variables from a TOML file
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
settings_cls: type[BaseSettings],
|
||||
toml_file: ConfigFileSourceType | None = DEFAULT_PATH,
|
||||
toml_table_header: tuple[str, ...] | None = None,
|
||||
deep_merge: bool = False,
|
||||
_init_state: InitState | None = None,
|
||||
):
|
||||
self.toml_file_path = toml_file if toml_file != DEFAULT_PATH else settings_cls.model_config.get('toml_file')
|
||||
self.toml_table_header = (
|
||||
toml_table_header
|
||||
if toml_table_header is not None
|
||||
else settings_cls.model_config.get('toml_table_header', ())
|
||||
)
|
||||
self.toml_data = self._read_files(self.toml_file_path, deep_merge=deep_merge)
|
||||
|
||||
if self._any_file_exists(self.toml_file_path):
|
||||
for key in self.toml_table_header:
|
||||
if key not in self.toml_data:
|
||||
raise KeyError(f'toml_table_header key "{key}" not found in {self.toml_file_path}')
|
||||
self.toml_data = self.toml_data[key]
|
||||
|
||||
super().__init__(settings_cls, self.toml_data, _init_state=_init_state)
|
||||
|
||||
def _read_file(self, file_path: Path | Traversable) -> dict[str, Any]:
|
||||
import_toml()
|
||||
with file_path.open(mode='rb') as toml_file:
|
||||
return tomllib.load(toml_file)
|
||||
|
||||
@classmethod
|
||||
def _any_file_exists(cls, paths: ConfigFileSourceType | None) -> bool:
|
||||
"""Check if any of the given file paths would be read by `_read_files`."""
|
||||
return bool(cls._existing_files(paths))
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f'{self.__class__.__name__}(toml_file={self.toml_file_path}, toml_table_header={self.toml_table_header})'
|
||||
@@ -0,0 +1,147 @@
|
||||
"""YAML file settings source."""
|
||||
|
||||
from __future__ import annotations as _annotations
|
||||
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
)
|
||||
|
||||
from ..base import ConfigFileSourceMixin, InitSettingsSource
|
||||
from ..types import DEFAULT_PATH, ConfigFileSourceType
|
||||
from ..utils import InitState
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from pydantic_settings.main import BaseSettings
|
||||
|
||||
from ..types import Traversable
|
||||
else:
|
||||
yaml = None
|
||||
|
||||
|
||||
def import_yaml() -> None:
|
||||
global yaml
|
||||
if yaml is not None:
|
||||
return
|
||||
try:
|
||||
import yaml
|
||||
except ImportError as e:
|
||||
raise ImportError('PyYAML is not installed, run `pip install pydantic-settings[yaml]`') from e
|
||||
|
||||
|
||||
class YamlConfigSettingsSource(InitSettingsSource, ConfigFileSourceMixin):
|
||||
"""
|
||||
A source class that loads variables from a yaml file
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
settings_cls: type[BaseSettings],
|
||||
yaml_file: ConfigFileSourceType | None = DEFAULT_PATH,
|
||||
yaml_file_encoding: str | None = None,
|
||||
yaml_config_section: str | None = None,
|
||||
deep_merge: bool = False,
|
||||
_init_state: InitState | None = None,
|
||||
):
|
||||
self.yaml_file_path = yaml_file if yaml_file != DEFAULT_PATH else settings_cls.model_config.get('yaml_file')
|
||||
self.yaml_file_encoding = (
|
||||
yaml_file_encoding
|
||||
if yaml_file_encoding is not None
|
||||
else settings_cls.model_config.get('yaml_file_encoding')
|
||||
)
|
||||
self.yaml_config_section = (
|
||||
yaml_config_section
|
||||
if yaml_config_section is not None
|
||||
else settings_cls.model_config.get('yaml_config_section')
|
||||
)
|
||||
self.yaml_data = self._read_files(self.yaml_file_path, deep_merge=deep_merge)
|
||||
|
||||
if self.yaml_config_section is not None:
|
||||
section_data = self._traverse_nested_section(
|
||||
self.yaml_data, self.yaml_config_section, self.yaml_config_section
|
||||
)
|
||||
# An empty section (`section:` with no children) parses to `None`; treat it as an
|
||||
# empty mapping so it falls back to defaults, mirroring `_read_file`'s `or {}` for
|
||||
# an empty whole file. A non-mapping section (e.g. a scalar) is a user error, so
|
||||
# raise a clear message instead of letting `InitSettingsSource` choke on `.keys()`.
|
||||
if section_data is None:
|
||||
section_data = {}
|
||||
elif not isinstance(section_data, dict):
|
||||
raise TypeError(
|
||||
f'yaml_config_section "{self.yaml_config_section}" in {self.yaml_file_path} '
|
||||
f'must be a mapping, got {type(section_data).__name__}.'
|
||||
)
|
||||
self.yaml_data = section_data
|
||||
super().__init__(settings_cls, self.yaml_data, _init_state=_init_state)
|
||||
|
||||
def _read_file(self, file_path: Path | Traversable) -> dict[str, Any]:
|
||||
import_yaml()
|
||||
# Default to UTF-8 rather than the locale encoding, matching `DotEnvSettingsSource`
|
||||
# and `SecretsSettingsSource`: the YAML spec requires UTF-8/UTF-16, and the locale
|
||||
# default corrupts (or fails to decode) it on a non-UTF-8 Windows code page.
|
||||
encoding = self.yaml_file_encoding if self.yaml_file_encoding is not None else 'utf-8'
|
||||
with file_path.open(encoding=encoding) as yaml_file:
|
||||
return yaml.safe_load(yaml_file) or {}
|
||||
|
||||
def _traverse_nested_section(
|
||||
self, data: dict[str, Any], section_path: str, original_path: str | None = None
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Traverse nested YAML sections using dot-notation path.
|
||||
|
||||
This method tries to match the longest possible key first before splitting on dots,
|
||||
allowing access to YAML keys that contain literal dot characters.
|
||||
|
||||
For example, with section_path="a.b.c", it will try:
|
||||
1. "a.b.c" as a literal key
|
||||
2. "a.b" as a key, then traverse to "c"
|
||||
3. "a" as a key, then traverse to "b.c"
|
||||
4. "a" as a key, then "b" as a key, then "c" as a key
|
||||
"""
|
||||
# Track the original path for error messages
|
||||
if original_path is None:
|
||||
original_path = section_path
|
||||
|
||||
# Only reject truly empty paths
|
||||
if not section_path:
|
||||
raise ValueError('yaml_config_section cannot be empty')
|
||||
|
||||
# Try the full path as a literal key first (even with leading/trailing/consecutive dots)
|
||||
try:
|
||||
return data[section_path]
|
||||
except KeyError:
|
||||
pass # Not a literal key, try splitting
|
||||
except TypeError as e:
|
||||
raise TypeError(
|
||||
f'yaml_config_section path "{original_path}" cannot be traversed in {self.yaml_file_path}. '
|
||||
f'An intermediate value is not a dictionary.'
|
||||
) from e
|
||||
|
||||
# If path contains no dots, we already tried it as a literal key above
|
||||
if '.' not in section_path:
|
||||
raise KeyError(f'yaml_config_section key "{original_path}" not found in {self.yaml_file_path}')
|
||||
|
||||
# Try progressively shorter prefixes (greedy left-to-right approach)
|
||||
parts = section_path.split('.')
|
||||
for i in range(len(parts) - 1, 0, -1):
|
||||
prefix = '.'.join(parts[:i])
|
||||
suffix = '.'.join(parts[i:])
|
||||
|
||||
if prefix in data:
|
||||
# Found the prefix as a literal key, now recursively traverse the suffix
|
||||
# The recursive call already raises a `TypeError` carrying `original_path`,
|
||||
# so let it propagate rather than re-wrapping it in an identical message.
|
||||
return self._traverse_nested_section(data[prefix], suffix, original_path)
|
||||
|
||||
# If we get here, no match was found
|
||||
raise KeyError(f'yaml_config_section key "{original_path}" not found in {self.yaml_file_path}')
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f'{self.__class__.__name__}(yaml_file={self.yaml_file_path})'
|
||||
|
||||
|
||||
__all__ = ['YamlConfigSettingsSource']
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Type definitions for pydantic-settings sources."""
|
||||
|
||||
from __future__ import annotations as _annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from importlib.resources.abc import Traversable as Traversable # noqa: PLC0414 (explicit re-export)
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Literal
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pydantic._internal._dataclasses import PydanticDataclass
|
||||
from pydantic.main import BaseModel
|
||||
|
||||
PydanticModel = PydanticDataclass | BaseModel
|
||||
else:
|
||||
PydanticModel = Any
|
||||
|
||||
|
||||
class EnvNoneType(str):
|
||||
pass
|
||||
|
||||
|
||||
class NoDecode:
|
||||
"""Annotation to prevent decoding of a field value."""
|
||||
|
||||
|
||||
class ForceDecode:
|
||||
"""Annotation to force decoding of a field value."""
|
||||
|
||||
|
||||
EnvPrefixTarget = Literal['variable', 'alias', 'all']
|
||||
DotenvType = Path | str | Sequence[Path | str]
|
||||
PathType = Path | str | Sequence[Path | str]
|
||||
# Config file sources (json/toml/yaml) additionally accept packaged resources exposed as a
|
||||
# `Traversable` (e.g. `importlib.resources.files(...).joinpath(...)`), including non-filesystem
|
||||
# ones such as a file inside a zip/wheel.
|
||||
# See https://github.com/pydantic/pydantic-settings/issues/299
|
||||
ConfigFileSourceType = Path | str | Traversable | Sequence[Path | str | Traversable]
|
||||
DotenvFiltering = Literal['match_prefix', 'only_existing']
|
||||
DEFAULT_PATH: PathType = Path()
|
||||
|
||||
# This is used as the default value for `_env_file` in the `BaseSettings` class and
|
||||
# `env_file` in `DotEnvSettingsSource` so that the default can be distinguished from `None`.
|
||||
# See the docstring of `BaseSettings` for more details.
|
||||
ENV_FILE_SENTINEL: DotenvType = Path()
|
||||
|
||||
|
||||
class _CliSubCommand:
|
||||
pass
|
||||
|
||||
|
||||
class _CliPositionalArg:
|
||||
pass
|
||||
|
||||
|
||||
class _CliVariadicArg:
|
||||
pass
|
||||
|
||||
|
||||
class _CliImplicitFlag:
|
||||
pass
|
||||
|
||||
|
||||
class _CliToggleFlag(_CliImplicitFlag):
|
||||
pass
|
||||
|
||||
|
||||
class _CliDualFlag(_CliImplicitFlag):
|
||||
pass
|
||||
|
||||
|
||||
class _CliExplicitFlag:
|
||||
pass
|
||||
|
||||
|
||||
class _CliUnknownArgs:
|
||||
pass
|
||||
|
||||
|
||||
class SecretVersion:
|
||||
def __init__(self, version: str) -> None:
|
||||
self.version = version
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f'{self.__class__.__name__}({self.version!r})'
|
||||
|
||||
|
||||
__all__ = [
|
||||
'DEFAULT_PATH',
|
||||
'ENV_FILE_SENTINEL',
|
||||
'ConfigFileSourceType',
|
||||
'DotenvFiltering',
|
||||
'DotenvType',
|
||||
'EnvNoneType',
|
||||
'EnvPrefixTarget',
|
||||
'ForceDecode',
|
||||
'NoDecode',
|
||||
'PathType',
|
||||
'PydanticModel',
|
||||
'SecretVersion',
|
||||
'_CliDualFlag',
|
||||
'_CliExplicitFlag',
|
||||
'_CliImplicitFlag',
|
||||
'_CliPositionalArg',
|
||||
'_CliSubCommand',
|
||||
'_CliToggleFlag',
|
||||
'_CliUnknownArgs',
|
||||
'_CliVariadicArg',
|
||||
]
|
||||
@@ -0,0 +1,515 @@
|
||||
"""Utility functions for pydantic-settings sources."""
|
||||
|
||||
from __future__ import annotations as _annotations
|
||||
|
||||
import os
|
||||
import warnings
|
||||
from collections import deque
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Set as AbstractSet
|
||||
from dataclasses import is_dataclass
|
||||
from enum import Enum
|
||||
from itertools import islice
|
||||
from pathlib import Path
|
||||
from typing import Any, TypedDict, TypeVar, cast, get_args, get_origin
|
||||
|
||||
from pydantic import BaseModel, Json, RootModel, Secret
|
||||
from pydantic._internal._utils import is_model_class
|
||||
from pydantic.dataclasses import is_pydantic_dataclass
|
||||
from pydantic.fields import FieldInfo
|
||||
from pydantic.types import Strict
|
||||
from typing_inspection import typing_objects
|
||||
from typing_inspection.introspection import is_union_origin
|
||||
|
||||
from ..exceptions import IncompleteFieldDefinitionWarning, SettingsError
|
||||
from ..utils import _lenient_issubclass
|
||||
from .types import EnvNoneType
|
||||
|
||||
|
||||
class InitState(TypedDict, total=False):
|
||||
"""State shared between settings sources during a single settings resolution."""
|
||||
|
||||
field_info_ids: set[int]
|
||||
"""The `id()`s of the incomplete `FieldInfo` instances that were already warned about."""
|
||||
|
||||
|
||||
def _warn_if_field_info_incomplete(field_info: FieldInfo, field_name: str, init_state: InitState) -> None:
|
||||
"""Warn if the field is incomplete, i.e. its annotation contains unresolved forward references.
|
||||
|
||||
An incomplete `FieldInfo` instance is unsafe to inspect — any of its attributes (annotation,
|
||||
aliases, metadata, default) may rely on the unresolved annotation, so settings sources may
|
||||
silently fail to resolve the field's value. Each instance is only warned about once per
|
||||
`init_state`, so that a field accessed by multiple sources during a single settings
|
||||
resolution doesn't emit duplicate warnings.
|
||||
"""
|
||||
if getattr(field_info, '_complete', True):
|
||||
return
|
||||
warned_ids = init_state.setdefault('field_info_ids', set())
|
||||
if id(field_info) in warned_ids:
|
||||
return
|
||||
warned_ids.add(id(field_info))
|
||||
warnings.warn( # noqa: B028 (called from many sources at differing depths, so no stacklevel is correct)
|
||||
f'Field {field_name!r} has an incomplete definition: its annotation contains an unresolved '
|
||||
'forward reference, so settings sources may fail to correctly resolve its value. '
|
||||
'Call `model_rebuild()` on the model where the field is defined, once all the referenced '
|
||||
'types are defined.',
|
||||
IncompleteFieldDefinitionWarning,
|
||||
)
|
||||
|
||||
|
||||
def _get_env_var_key(key: str, case_sensitive: bool = False) -> str:
|
||||
return key if case_sensitive else key.lower()
|
||||
|
||||
|
||||
def _parse_env_none_str(value: str | None, parse_none_str: str | None = None) -> str | EnvNoneType | None:
|
||||
return value if not (value == parse_none_str and parse_none_str is not None) else EnvNoneType(value)
|
||||
|
||||
|
||||
def parse_env_vars(
|
||||
env_vars: Mapping[str, str | None],
|
||||
case_sensitive: bool = False,
|
||||
ignore_empty: bool = False,
|
||||
parse_none_str: str | None = None,
|
||||
) -> Mapping[str, str | None]:
|
||||
return {
|
||||
_get_env_var_key(k, case_sensitive): _parse_env_none_str(v, parse_none_str)
|
||||
for k, v in env_vars.items()
|
||||
if not (ignore_empty and v == '')
|
||||
}
|
||||
|
||||
|
||||
def _resolve_config_file(file: str | os.PathLike[str], depth: int = 0, *, allow_fifo: bool = False) -> Path | None:
|
||||
"""Resolve a config file path, optionally searching parent directories of the cwd."""
|
||||
|
||||
file_path = Path(file).expanduser()
|
||||
|
||||
if file_path.is_file() or (allow_fifo and file_path.is_fifo()):
|
||||
return file_path
|
||||
|
||||
if depth <= 0 or file_path.is_absolute():
|
||||
return None
|
||||
|
||||
for parent in islice(Path.cwd().parents, depth):
|
||||
candidate = parent / file_path
|
||||
if candidate.is_file() or (allow_fifo and candidate.is_fifo()):
|
||||
return candidate
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _substitute_typevars(tp: Any, param_map: dict[Any, Any]) -> Any:
|
||||
"""Substitute TypeVars in a type annotation with concrete types from param_map."""
|
||||
if isinstance(tp, TypeVar) and tp in param_map:
|
||||
return param_map[tp]
|
||||
args = get_args(tp)
|
||||
if not args:
|
||||
return tp
|
||||
new_args = tuple(_substitute_typevars(arg, param_map) for arg in args)
|
||||
if new_args == args:
|
||||
return tp
|
||||
origin = get_origin(tp)
|
||||
if origin is not None:
|
||||
try:
|
||||
return origin[new_args]
|
||||
except TypeError:
|
||||
# types.UnionType and similar are not directly subscriptable,
|
||||
# reconstruct using | operator
|
||||
import functools
|
||||
import operator
|
||||
|
||||
return functools.reduce(operator.or_, new_args)
|
||||
# Unreachable in practice: an annotation with args always has an origin.
|
||||
# Kept as a defensive fallback.
|
||||
return tp # pragma: no cover
|
||||
|
||||
|
||||
def _resolve_type_alias(annotation: Any) -> Any:
|
||||
"""Resolve a TypeAliasType to its underlying value, substituting type params if parameterized.
|
||||
|
||||
Aliases of aliases are followed until a non-alias annotation is reached. This terminates because
|
||||
pydantic rejects aliases that circularly refer to themselves without going through another type.
|
||||
"""
|
||||
while True:
|
||||
if typing_objects.is_typealiastype(annotation):
|
||||
annotation = annotation.__value__
|
||||
continue
|
||||
origin = get_origin(annotation)
|
||||
if not typing_objects.is_typealiastype(origin):
|
||||
return annotation
|
||||
type_params = getattr(origin, '__type_params__', ())
|
||||
type_args = get_args(annotation)
|
||||
annotation = origin.__value__
|
||||
# Always true in practice: only a subscripted alias reaches here, and subscripting
|
||||
# requires type params.
|
||||
if type_params and type_args: # pragma: no branch
|
||||
# Not `strict=True`: a parameterized alias may supply fewer args than
|
||||
# params (e.g. type params with defaults), which is valid.
|
||||
annotation = _substitute_typevars(annotation, dict(zip(type_params, type_args, strict=False)))
|
||||
|
||||
|
||||
def _annotation_is_complex(annotation: Any, metadata: list[Any], init_state: InitState | None = None) -> bool:
|
||||
# If the model is a root model, the root annotation should be used to
|
||||
# evaluate the complexity.
|
||||
annotation = _resolve_type_alias(annotation)
|
||||
if annotation is not None and _lenient_issubclass(annotation, RootModel) and annotation is not RootModel:
|
||||
annotation = cast('type[RootModel[Any]]', annotation)
|
||||
root_field = annotation.model_fields['root']
|
||||
if init_state is not None:
|
||||
_warn_if_field_info_incomplete(root_field, f'{annotation.__name__}.root', init_state)
|
||||
root_annotation = root_field.annotation
|
||||
if root_annotation is not None: # pragma: no branch
|
||||
annotation = root_annotation
|
||||
|
||||
if any(isinstance(md, Json) for md in metadata): # type: ignore[misc]
|
||||
return False
|
||||
|
||||
origin = get_origin(annotation)
|
||||
|
||||
# Check if annotation is of the form Annotated[type, metadata].
|
||||
if typing_objects.is_annotated(origin):
|
||||
# Return result of recursive call on inner type.
|
||||
inner, *meta = get_args(annotation)
|
||||
return _annotation_is_complex(inner, meta, init_state)
|
||||
|
||||
if _lenient_issubclass(origin, Secret):
|
||||
return False
|
||||
|
||||
return (
|
||||
_annotation_is_complex_inner(annotation)
|
||||
or _annotation_is_complex_inner(origin)
|
||||
or hasattr(origin, '__pydantic_core_schema__')
|
||||
or hasattr(origin, '__get_pydantic_core_schema__')
|
||||
)
|
||||
|
||||
|
||||
def _get_field_annotation_and_metadata(field: FieldInfo) -> tuple[Any, list[Any]]:
|
||||
"""Resolve a field's type alias and collect its outer Annotated metadata."""
|
||||
annotation = _resolve_type_alias(field.annotation)
|
||||
metadata = field.metadata
|
||||
origin = get_origin(annotation)
|
||||
if typing_objects.is_annotated(origin):
|
||||
annotation, *meta = get_args(annotation)
|
||||
# Build a new list rather than extending in place: `field.metadata` is shared
|
||||
# across instantiations, subclasses and rebuilds, so mutating it would append
|
||||
# the alias metadata again on every call.
|
||||
metadata = [*metadata, *meta]
|
||||
return annotation, metadata
|
||||
|
||||
|
||||
def _get_field_metadata(field: FieldInfo) -> list[Any]:
|
||||
"""Collect a field's metadata, including the outer Annotated metadata of its type alias."""
|
||||
return _get_field_annotation_and_metadata(field)[1]
|
||||
|
||||
|
||||
def _annotation_is_complex_inner(annotation: type[Any] | None) -> bool:
|
||||
if _lenient_issubclass(annotation, (str, bytes)):
|
||||
return False
|
||||
|
||||
return _lenient_issubclass(
|
||||
annotation, (BaseModel, Mapping, Sequence, AbstractSet, tuple, set, frozenset, deque)
|
||||
) or is_dataclass(annotation)
|
||||
|
||||
|
||||
def _union_is_complex(
|
||||
annotation: type[Any] | None,
|
||||
metadata: list[Any],
|
||||
init_state: InitState | None = None,
|
||||
_seen: tuple[Any, ...] = (),
|
||||
) -> bool:
|
||||
"""Check if a union type contains any complex types."""
|
||||
for arg in get_args(annotation):
|
||||
if arg in _seen:
|
||||
continue
|
||||
# Track the original member: resolving a generic alias rebuilds its union,
|
||||
# but aliases with the same origin and arguments still compare equal.
|
||||
seen = (*_seen, arg)
|
||||
arg = _resolve_type_alias(arg)
|
||||
if _annotation_is_complex(arg, metadata, init_state):
|
||||
return True
|
||||
# _annotation_is_complex doesn't handle bare Union types, including those
|
||||
# exposed by a type alias or Annotated[Union[X, Y], ...]. Recurse, but only
|
||||
# if no Annotated metadata suppresses complexity (e.g. Json).
|
||||
inner = _unwrap_union_member(arg)
|
||||
if is_union_origin(get_origin(inner)) and _union_is_complex(inner, metadata, init_state, seen):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _unwrap_union_member(annotation: Any) -> Any:
|
||||
"""Strip Annotated layers and the type aliases they wrap from a union member.
|
||||
|
||||
Returns `None` if any layer's metadata suppresses complexity (e.g. Json).
|
||||
"""
|
||||
while typing_objects.is_annotated(get_origin(annotation)):
|
||||
annotation, *meta = get_args(annotation)
|
||||
if any(isinstance(md, Json) for md in meta): # type: ignore[misc]
|
||||
return None
|
||||
annotation = _resolve_type_alias(annotation)
|
||||
return annotation
|
||||
|
||||
|
||||
def _union_has_strict_types(annotation: type[Any] | None) -> bool:
|
||||
"""Check if a union type contains any strict-annotated types."""
|
||||
for arg in get_args(annotation):
|
||||
if typing_objects.is_annotated(get_origin(arg)):
|
||||
_, *meta = get_args(arg)
|
||||
if any(isinstance(m, Strict) for m in meta):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _is_subtype_of(annotation: Any, types: tuple[Any, ...]) -> bool:
|
||||
"""Check if a type annotation is a class that subclasses any of the specified types.
|
||||
|
||||
Text and bytes types never match, since they are sequences themselves, but are treated as
|
||||
scalar values by the settings sources.
|
||||
"""
|
||||
return not _lenient_issubclass(annotation, (str, bytes, bytearray)) and _lenient_issubclass(annotation, types)
|
||||
|
||||
|
||||
def _annotation_contains_types(
|
||||
annotation: Any,
|
||||
types: tuple[Any, ...],
|
||||
is_include_origin: bool = True,
|
||||
is_strip_annotated: bool = False,
|
||||
is_instance: bool = False,
|
||||
collect: set[Any] | None = None,
|
||||
is_subtype: bool = False,
|
||||
_seen_aliases: tuple[Any, ...] = (),
|
||||
) -> bool:
|
||||
"""Check if a type annotation contains any of the specified types.
|
||||
|
||||
With `is_subtype`, subclasses of the specified types match as well, so `tuple` matches
|
||||
`Sequence`, or `frozenset` matches `Set`.
|
||||
"""
|
||||
if is_strip_annotated:
|
||||
annotation = _strip_annotated(annotation)
|
||||
resolved = _resolve_type_alias(annotation)
|
||||
if resolved is not annotation:
|
||||
# An alias already on the current path is recursive, so stop there. Only the current path is
|
||||
# tracked, so sibling specializations of the same alias are still inspected.
|
||||
return annotation not in _seen_aliases and _annotation_contains_types(
|
||||
resolved,
|
||||
types,
|
||||
is_include_origin=is_include_origin,
|
||||
is_strip_annotated=is_strip_annotated,
|
||||
is_instance=is_instance,
|
||||
collect=collect,
|
||||
is_subtype=is_subtype,
|
||||
_seen_aliases=(*_seen_aliases, annotation),
|
||||
)
|
||||
if is_include_origin is True:
|
||||
origin = get_origin(annotation)
|
||||
if origin in types or (is_subtype and _is_subtype_of(origin, types)):
|
||||
if collect is None:
|
||||
return True
|
||||
collect.add(annotation)
|
||||
if is_instance and any(isinstance(origin, type_) for type_ in types):
|
||||
if collect is None:
|
||||
return True
|
||||
collect.add(annotation)
|
||||
for type_ in get_args(annotation):
|
||||
if (
|
||||
_annotation_contains_types(
|
||||
type_,
|
||||
types,
|
||||
is_include_origin=True,
|
||||
is_strip_annotated=is_strip_annotated,
|
||||
is_instance=is_instance,
|
||||
collect=collect,
|
||||
is_subtype=is_subtype,
|
||||
_seen_aliases=_seen_aliases,
|
||||
)
|
||||
and collect is None
|
||||
):
|
||||
return True
|
||||
if is_instance and any(isinstance(annotation, type_) for type_ in types):
|
||||
if collect is None:
|
||||
return True
|
||||
collect.add(annotation)
|
||||
if annotation in types or (is_subtype and _is_subtype_of(annotation, types)):
|
||||
if collect is not None:
|
||||
collect.add(annotation)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _get_union_members(annotation: Any) -> tuple[Any, ...]:
|
||||
"""Get the members of a union, flattening unions nested inside type aliases or `Annotated`.
|
||||
|
||||
A non-union annotation is returned as its only member.
|
||||
"""
|
||||
resolved = _strip_annotated(_resolve_type_alias(_strip_annotated(annotation)))
|
||||
if resolved is not annotation:
|
||||
return _get_union_members(resolved)
|
||||
if not is_union_origin(get_origin(annotation)):
|
||||
return (annotation,)
|
||||
return tuple(member for arg in get_args(annotation) for member in _get_union_members(arg))
|
||||
|
||||
|
||||
def _strip_annotated(annotation: Any) -> Any:
|
||||
if typing_objects.is_annotated(get_origin(annotation)):
|
||||
return annotation.__origin__
|
||||
return annotation
|
||||
|
||||
|
||||
def _annotation_enum_val_to_name(annotation: Any, value: Any) -> str | None:
|
||||
for type_ in (annotation, get_origin(annotation)):
|
||||
if _lenient_issubclass(type_, Enum):
|
||||
enum_type = cast('type[Enum]', type_)
|
||||
if value in enum_type.__members__.values():
|
||||
return enum_type(value).name
|
||||
for arg in get_args(annotation):
|
||||
enum_name = _annotation_enum_val_to_name(arg, value)
|
||||
if enum_name is not None:
|
||||
return enum_name
|
||||
return None
|
||||
|
||||
|
||||
def _annotation_enum_name_to_val(annotation: Any, name: Any) -> Any:
|
||||
for type_ in (annotation, get_origin(annotation)):
|
||||
if _lenient_issubclass(type_, Enum):
|
||||
enum_type = cast('type[Enum]', type_)
|
||||
if name in enum_type.__members__:
|
||||
return enum_type[name]
|
||||
for arg in get_args(annotation):
|
||||
enum_val = _annotation_enum_name_to_val(arg, name)
|
||||
if enum_val is not None:
|
||||
return enum_val
|
||||
return None
|
||||
|
||||
|
||||
def _literal_has_numeric_enum(annotation: type[Any] | None) -> bool:
|
||||
"""Check if annotation is a Literal type containing numeric Enum members (IntEnum, (int, Enum), (float, Enum))."""
|
||||
if typing_objects.is_literal(get_origin(annotation)):
|
||||
return any(isinstance(arg, (int, float)) and isinstance(arg, Enum) for arg in get_args(annotation))
|
||||
# Handle Annotated wrapping, e.g. Annotated[Literal[IntEnum.member], Field(...)]
|
||||
if typing_objects.is_annotated(get_origin(annotation)):
|
||||
inner = get_args(annotation)[0]
|
||||
return _literal_has_numeric_enum(inner)
|
||||
# Handle Union/Optional wrapping, e.g. Optional[Literal[IntEnum.member]]
|
||||
if is_union_origin(get_origin(annotation)):
|
||||
return any(_literal_has_numeric_enum(arg) for arg in get_args(annotation))
|
||||
return False
|
||||
|
||||
|
||||
def _get_model_fields(model_cls: type[Any]) -> dict[str, FieldInfo]:
|
||||
"""Get fields from a pydantic model or dataclass."""
|
||||
|
||||
if is_pydantic_dataclass(model_cls) and hasattr(model_cls, '__pydantic_fields__'):
|
||||
return model_cls.__pydantic_fields__
|
||||
if is_model_class(model_cls):
|
||||
return model_cls.model_fields
|
||||
raise SettingsError(f'Error: {model_cls.__name__} is not subclass of BaseModel or pydantic.dataclasses.dataclass')
|
||||
|
||||
|
||||
def _get_alias_names(
|
||||
field_name: str,
|
||||
field_info: FieldInfo,
|
||||
alias_path_args: dict[str, int | None] | None = None,
|
||||
case_sensitive: bool = True,
|
||||
populate_by_name: bool = False,
|
||||
validate_by_alias: bool = True,
|
||||
) -> tuple[tuple[str, ...], bool]:
|
||||
"""Get alias names for a field, handling alias paths and case sensitivity.
|
||||
|
||||
With `validate_by_alias=False` pydantic ignores the aliases, so only the field name is returned.
|
||||
|
||||
The first name is the key that sources pass the value to pydantic under.
|
||||
"""
|
||||
from pydantic import AliasChoices, AliasPath
|
||||
|
||||
alias_names: list[str] = []
|
||||
is_alias_path_only: bool = True
|
||||
if not validate_by_alias or not any((field_info.alias, field_info.validation_alias)):
|
||||
alias_names += [field_name]
|
||||
is_alias_path_only = False
|
||||
else:
|
||||
new_alias_paths: list[AliasPath] = []
|
||||
# Pydantic only validates input against `validation_alias` when it is set, so `alias` isn't an input name then
|
||||
alias = field_info.validation_alias if field_info.validation_alias is not None else field_info.alias
|
||||
if isinstance(alias, str):
|
||||
alias_names.append(alias)
|
||||
is_alias_path_only = False
|
||||
elif isinstance(alias, AliasChoices):
|
||||
for name in alias.choices:
|
||||
if isinstance(name, str):
|
||||
alias_names.append(name)
|
||||
is_alias_path_only = False
|
||||
else:
|
||||
new_alias_paths.append(name)
|
||||
else:
|
||||
# Can't be `None`, the check above ensures one of the aliases is set
|
||||
new_alias_paths.append(cast(AliasPath, alias))
|
||||
for alias_path in new_alias_paths:
|
||||
name = cast(str, alias_path.path[0])
|
||||
name = name.lower() if not case_sensitive else name
|
||||
if alias_path_args is not None:
|
||||
alias_path_args[name] = (
|
||||
alias_path.path[1] if len(alias_path.path) > 1 and isinstance(alias_path.path[1], int) else None
|
||||
)
|
||||
if not alias_names and is_alias_path_only:
|
||||
alias_names.append(name)
|
||||
if populate_by_name and field_name not in alias_names:
|
||||
alias_names.append(field_name)
|
||||
is_alias_path_only = False
|
||||
if not case_sensitive:
|
||||
alias_names = [alias_name.lower() for alias_name in alias_names]
|
||||
return tuple(dict.fromkeys(alias_names)), is_alias_path_only
|
||||
|
||||
|
||||
def _get_model_config(model: Any) -> Mapping[str, Any]:
|
||||
"""Get the config of a pydantic model or pydantic dataclass."""
|
||||
return getattr(model, 'model_config', None) or getattr(model, '__pydantic_config__', {})
|
||||
|
||||
|
||||
def _validate_by_alias_enabled(config: Mapping[str, Any]) -> bool:
|
||||
"""Whether fields can be populated by their aliases. pydantic defaults `validate_by_alias` to True."""
|
||||
return config.get('validate_by_alias') is not False
|
||||
|
||||
|
||||
def _validate_by_name_enabled(config: Mapping[str, Any]) -> bool:
|
||||
"""Whether fields can be populated by their name, resolved the same way pydantic resolves its config.
|
||||
|
||||
pydantic < 2.14 wrote the resolved `validate_by_name` back into `model_config`, but newer versions leave
|
||||
`model_config` as declared, so the `populate_by_name` fallback and the `validate_by_alias=False` implication
|
||||
have to be applied here.
|
||||
"""
|
||||
validate_by_name = config.get('validate_by_name')
|
||||
if validate_by_name is not None:
|
||||
return validate_by_name
|
||||
populate_by_name = config.get('populate_by_name')
|
||||
if populate_by_name is not None:
|
||||
return populate_by_name
|
||||
return config.get('validate_by_alias') is False
|
||||
|
||||
|
||||
def _is_function(obj: Any) -> bool:
|
||||
"""Check if an object is a function."""
|
||||
from types import BuiltinFunctionType, FunctionType
|
||||
|
||||
return isinstance(obj, (FunctionType, BuiltinFunctionType))
|
||||
|
||||
|
||||
__all__ = [
|
||||
'InitState',
|
||||
'_annotation_contains_types',
|
||||
'_annotation_enum_name_to_val',
|
||||
'_annotation_enum_val_to_name',
|
||||
'_annotation_is_complex',
|
||||
'_annotation_is_complex_inner',
|
||||
'_get_alias_names',
|
||||
'_get_env_var_key',
|
||||
'_get_model_fields',
|
||||
'_get_union_members',
|
||||
'_is_function',
|
||||
'_literal_has_numeric_enum',
|
||||
'_parse_env_none_str',
|
||||
'_resolve_config_file',
|
||||
'_resolve_type_alias',
|
||||
'_strip_annotated',
|
||||
'_union_has_strict_types',
|
||||
'_union_is_complex',
|
||||
'_warn_if_field_info_incomplete',
|
||||
'parse_env_vars',
|
||||
]
|
||||
@@ -0,0 +1,46 @@
|
||||
import logging
|
||||
import os
|
||||
import types
|
||||
from pathlib import Path
|
||||
from typing import Any, _Final, _GenericAlias # type: ignore [attr-defined]
|
||||
|
||||
logger = logging.getLogger('pydantic_settings')
|
||||
|
||||
_DEBUG_ENV_VAR = 'PYDANTIC_SETTINGS_DEBUG'
|
||||
|
||||
|
||||
def _settings_debug_enabled() -> bool:
|
||||
"""Whether settings source debugging is enabled via the `PYDANTIC_SETTINGS_DEBUG` env var."""
|
||||
return os.environ.get(_DEBUG_ENV_VAR, '').strip().lower() in ('1', 'true', 'yes', 'on')
|
||||
|
||||
|
||||
_PATH_TYPE_LABELS: tuple[tuple[str, str], ...] = (
|
||||
('is_dir', 'directory'),
|
||||
('is_file', 'file'),
|
||||
('is_mount', 'mount point'),
|
||||
('is_symlink', 'symlink'),
|
||||
('is_block_device', 'block device'),
|
||||
('is_char_device', 'char device'),
|
||||
('is_fifo', 'FIFO'),
|
||||
('is_socket', 'socket'),
|
||||
)
|
||||
|
||||
|
||||
def path_type_label(p: Path) -> str:
|
||||
"""
|
||||
Find out what sort of thing a path is.
|
||||
"""
|
||||
assert p.exists(), 'path does not exist'
|
||||
for method_name, name in _PATH_TYPE_LABELS:
|
||||
if getattr(p, method_name)():
|
||||
return name
|
||||
|
||||
return 'unknown' # pragma: no cover
|
||||
|
||||
|
||||
def _lenient_issubclass(cls: Any, class_or_tuple: Any) -> bool:
|
||||
return isinstance(cls, type) and issubclass(cls, class_or_tuple)
|
||||
|
||||
|
||||
_WithArgsTypes = (_GenericAlias, types.GenericAlias, types.UnionType)
|
||||
_typing_base: Any = _Final # pyright: ignore[reportAttributeAccessIssue]
|
||||
@@ -0,0 +1 @@
|
||||
VERSION = '2.16.1'
|
||||
Reference in New Issue
Block a user