Initial Commit
This commit is contained in:
Mark Gregory
@@ -0,0 +1,393 @@
|
||||
from __future__ import annotations as _annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
cast,
|
||||
get_args,
|
||||
get_origin,
|
||||
)
|
||||
|
||||
from pydantic import AliasChoices, AliasPath, Json, TypeAdapter, ValidationError
|
||||
from pydantic._internal._utils import deep_update, is_model_class
|
||||
from pydantic.dataclasses import is_pydantic_dataclass
|
||||
from pydantic.fields import FieldInfo
|
||||
from typing_inspection.introspection import is_union_origin
|
||||
|
||||
from ...utils import _lenient_issubclass
|
||||
from ..base import PydanticBaseEnvSettingsSource
|
||||
from ..types import EnvNoneType, EnvPrefixTarget
|
||||
from ..utils import (
|
||||
InitState,
|
||||
_annotation_contains_types,
|
||||
_annotation_enum_name_to_val,
|
||||
_annotation_is_complex,
|
||||
_get_field_annotation_and_metadata,
|
||||
_get_model_config,
|
||||
_get_model_fields,
|
||||
_literal_has_numeric_enum,
|
||||
_union_has_strict_types,
|
||||
_union_is_complex,
|
||||
parse_env_vars,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pydantic_settings.main import BaseSettings
|
||||
|
||||
|
||||
def _environ_is_case_insensitive() -> bool:
|
||||
"""Whether ``os.environ`` is the OS-backed, case-insensitive mapping (i.e. on Windows).
|
||||
|
||||
When ``os.environ`` has been replaced (e.g. patched to a plain ``dict`` in tests), the
|
||||
requested case sensitivity is honored as-is.
|
||||
"""
|
||||
return os.name == 'nt' and isinstance(os.environ, os._Environ)
|
||||
|
||||
|
||||
class EnvSettingsSource(PydanticBaseEnvSettingsSource):
|
||||
"""
|
||||
Source class for loading settings values from environment variables.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
settings_cls: type[BaseSettings],
|
||||
case_sensitive: bool | None = None,
|
||||
env_prefix: str | None = None,
|
||||
env_prefix_target: EnvPrefixTarget | None = None,
|
||||
env_nested_delimiter: str | None = None,
|
||||
env_nested_max_split: int | None = None,
|
||||
env_ignore_empty: bool | None = None,
|
||||
env_parse_none_str: str | None = None,
|
||||
env_parse_enums: bool | None = None,
|
||||
_init_state: InitState | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
settings_cls,
|
||||
case_sensitive,
|
||||
env_prefix,
|
||||
env_prefix_target,
|
||||
env_ignore_empty,
|
||||
env_parse_none_str,
|
||||
env_parse_enums,
|
||||
_init_state,
|
||||
)
|
||||
self.env_nested_delimiter = (
|
||||
env_nested_delimiter if env_nested_delimiter is not None else self.config.get('env_nested_delimiter')
|
||||
)
|
||||
self.env_nested_max_split = (
|
||||
env_nested_max_split if env_nested_max_split is not None else self.config.get('env_nested_max_split')
|
||||
)
|
||||
self.maxsplit = (self.env_nested_max_split or 0) - 1
|
||||
self.env_prefix_len = len(self.env_prefix)
|
||||
|
||||
self.env_vars = self._load_env_vars()
|
||||
|
||||
def _load_env_vars(self) -> Mapping[str, str | None]:
|
||||
# On Windows, ``os.environ`` is case-insensitive: variable names are normalized to
|
||||
# upper-case and looked up regardless of case. Matching environment variables
|
||||
# case-sensitively is therefore impossible, so we fall back to case-insensitive
|
||||
# matching to avoid spurious "field required" errors (see #295). ``.env`` files do
|
||||
# preserve case and are handled by ``DotEnvSettingsSource``, which overrides this method.
|
||||
if self.case_sensitive and _environ_is_case_insensitive():
|
||||
self.case_sensitive = False
|
||||
return parse_env_vars(os.environ, self.case_sensitive, self.env_ignore_empty, self.env_parse_none_str)
|
||||
|
||||
def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
|
||||
"""
|
||||
Gets the value for field from environment variables and a flag to determine whether value is complex.
|
||||
|
||||
Args:
|
||||
field: The field.
|
||||
field_name: The field name.
|
||||
|
||||
Returns:
|
||||
A tuple that contains the value (`None` if not found), key, and
|
||||
a flag to determine whether value is complex.
|
||||
"""
|
||||
|
||||
env_val: str | None = None
|
||||
# `field_key` and `value_is_complex` are deliberately read after the loop, so B007's
|
||||
# "unused loop variable" report doesn't apply here.
|
||||
for field_key, env_name, value_is_complex in self._extract_field_info(field, field_name): # noqa: B007
|
||||
env_val = self.env_vars.get(env_name)
|
||||
if env_val is not None:
|
||||
break
|
||||
|
||||
return env_val, field_key, value_is_complex
|
||||
|
||||
def prepare_field_value(self, field_name: str, field: FieldInfo, value: Any, value_is_complex: bool) -> Any:
|
||||
"""
|
||||
Prepare value for the field.
|
||||
|
||||
* Extract value for nested field.
|
||||
* Deserialize value to python object for complex field.
|
||||
|
||||
Args:
|
||||
field: The field.
|
||||
field_name: The field name.
|
||||
|
||||
Returns:
|
||||
A tuple contains prepared value for the field.
|
||||
|
||||
Raises:
|
||||
ValuesError: When There is an error in deserializing value for complex field.
|
||||
"""
|
||||
is_complex, allow_parse_failure = self._field_is_complex(field)
|
||||
if self.env_parse_enums:
|
||||
enum_val = _annotation_enum_name_to_val(field.annotation, value)
|
||||
value = value if enum_val is None else enum_val
|
||||
|
||||
if is_complex or value_is_complex:
|
||||
if isinstance(value, EnvNoneType):
|
||||
return value
|
||||
if value is None:
|
||||
# field is complex but no value found so far, try explode_env_vars
|
||||
env_val_built = self.explode_env_vars(field_name, field, self.env_vars)
|
||||
if env_val_built:
|
||||
return env_val_built
|
||||
else:
|
||||
# field is complex and there's a value, decode that as JSON, then add explode_env_vars
|
||||
try:
|
||||
value = self.decode_complex_value(field_name, field, value)
|
||||
except ValueError:
|
||||
if not allow_parse_failure:
|
||||
raise
|
||||
|
||||
if isinstance(value, dict):
|
||||
return deep_update(value, self.explode_env_vars(field_name, field, self.env_vars))
|
||||
return value
|
||||
elif value is not None:
|
||||
# simplest case, field is not complex, we only need to add the value if it was found
|
||||
return self._coerce_env_val_strict(field, value)
|
||||
|
||||
# No value resolved for this field; `None` tells the caller to skip it.
|
||||
return None
|
||||
|
||||
def _matches_alias_path_head(self, field: FieldInfo | Any | None, key: str) -> bool:
|
||||
"""
|
||||
Whether ``key`` matches the first element of a multi-part ``AliasPath`` on ``field``.
|
||||
|
||||
When a nested field is aliased via e.g. ``AliasPath('path', 0)``, the env value found
|
||||
under ``path`` is a container that must be JSON-decoded before pydantic can navigate
|
||||
into it (see #670). This mirrors the complexity that ``_extract_field_info`` already
|
||||
reports for top-level ``AliasPath`` fields.
|
||||
|
||||
Args:
|
||||
field: The field (or raw type) matched for ``key``.
|
||||
key: The (case-normalized) env name segment matched for the field.
|
||||
|
||||
Returns:
|
||||
Whether ``key`` is the head of a multi-element ``AliasPath``.
|
||||
"""
|
||||
if not isinstance(field, FieldInfo):
|
||||
return False
|
||||
v_alias = field.validation_alias
|
||||
paths: list[tuple[str | int, ...]] = []
|
||||
if isinstance(v_alias, AliasPath):
|
||||
paths.append(tuple(v_alias.path))
|
||||
elif isinstance(v_alias, AliasChoices):
|
||||
paths.extend(tuple(choice.path) for choice in v_alias.choices if isinstance(choice, AliasPath))
|
||||
for path in paths:
|
||||
head = path[0]
|
||||
if len(path) > 1 and isinstance(head, str) and self._apply_case_sensitive(head) == key:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _field_is_complex(self, field: FieldInfo) -> tuple[bool, bool]:
|
||||
"""
|
||||
Find out if a field is complex, and if so whether JSON errors should be ignored
|
||||
"""
|
||||
annotation, metadata = _get_field_annotation_and_metadata(field)
|
||||
if self.field_is_complex(field):
|
||||
allow_parse_failure = False
|
||||
elif is_union_origin(get_origin(annotation)) and _union_is_complex(annotation, metadata, self._init_state):
|
||||
allow_parse_failure = True
|
||||
else:
|
||||
return False, False
|
||||
|
||||
return True, allow_parse_failure
|
||||
|
||||
# Default value of `case_sensitive` is `None`, because we don't want to break existing behavior.
|
||||
# We have to change the method to a non-static method and use
|
||||
# `self.case_sensitive` instead in V3.
|
||||
def next_field(
|
||||
self, field: FieldInfo | Any | None, key: str, case_sensitive: bool | None = None
|
||||
) -> FieldInfo | None:
|
||||
"""
|
||||
Find the field in a sub model by key(env name)
|
||||
|
||||
By having the following models:
|
||||
|
||||
```py
|
||||
class SubSubModel(BaseSettings):
|
||||
dvals: Dict
|
||||
|
||||
class SubModel(BaseSettings):
|
||||
vals: list[str]
|
||||
sub_sub_model: SubSubModel
|
||||
|
||||
class Cfg(BaseSettings):
|
||||
sub_model: SubModel
|
||||
```
|
||||
|
||||
Then:
|
||||
next_field(sub_model, 'vals') Returns the `vals` field of `SubModel` class
|
||||
next_field(sub_model, 'sub_sub_model') Returns `sub_sub_model` field of `SubModel` class
|
||||
|
||||
Args:
|
||||
field: The field.
|
||||
key: The key (env name).
|
||||
case_sensitive: Whether to search for key case sensitively.
|
||||
|
||||
Returns:
|
||||
Field if it finds the next field otherwise `None`.
|
||||
"""
|
||||
if not field:
|
||||
return None
|
||||
|
||||
annotation = field.annotation if isinstance(field, FieldInfo) else field
|
||||
for type_ in get_args(annotation):
|
||||
type_has_key = self.next_field(type_, key, case_sensitive)
|
||||
if type_has_key:
|
||||
return type_has_key
|
||||
if _lenient_issubclass(get_origin(annotation), dict):
|
||||
# get value type if it's a dict
|
||||
args = get_args(annotation)
|
||||
return args[-1] if args else None
|
||||
if is_model_class(annotation) or is_pydantic_dataclass(annotation): # type: ignore[arg-type]
|
||||
fields = _get_model_fields(annotation)
|
||||
model_config = _get_model_config(annotation)
|
||||
# `case_sensitive is None` is here to be compatible with the old behavior.
|
||||
# Has to be removed in V3.
|
||||
for field_name, f in fields.items():
|
||||
for _, env_name, _ in self._extract_field_info(f, field_name, model_config):
|
||||
if case_sensitive is None or case_sensitive:
|
||||
if field_name == key or env_name == key:
|
||||
return f
|
||||
elif field_name.lower() == key.lower() or env_name.lower() == key.lower():
|
||||
return f
|
||||
return None
|
||||
|
||||
def explode_env_vars(self, field_name: str, field: FieldInfo, env_vars: Mapping[str, str | None]) -> dict[str, Any]: # noqa: C901
|
||||
"""
|
||||
Process env_vars and extract the values of keys containing env_nested_delimiter into nested dictionaries.
|
||||
|
||||
This is applied to a single field, hence filtering by env_var prefix.
|
||||
|
||||
Args:
|
||||
field_name: The field name.
|
||||
field: The field.
|
||||
env_vars: Environment variables.
|
||||
|
||||
Returns:
|
||||
A dictionary contains extracted values from nested env values.
|
||||
"""
|
||||
if not self.env_nested_delimiter:
|
||||
return {}
|
||||
|
||||
ann = field.annotation
|
||||
is_dict = ann is dict or _lenient_issubclass(get_origin(ann), dict)
|
||||
|
||||
prefixes = [
|
||||
f'{env_name}{self.env_nested_delimiter}' for _, env_name, _ in self._extract_field_info(field, field_name)
|
||||
]
|
||||
result: dict[str, Any] = {}
|
||||
for env_name, env_val in env_vars.items():
|
||||
try:
|
||||
prefix = next(prefix for prefix in prefixes if env_name.startswith(prefix))
|
||||
except StopIteration:
|
||||
continue
|
||||
# we remove the prefix before splitting in case the prefix has characters in common with the delimiter
|
||||
env_name_without_prefix = env_name[len(prefix) :]
|
||||
*keys, last_key = env_name_without_prefix.split(self.env_nested_delimiter, self.maxsplit)
|
||||
env_var = result
|
||||
target_field: FieldInfo | None = field
|
||||
for key in keys:
|
||||
target_field = self.next_field(target_field, key, self.case_sensitive)
|
||||
if isinstance(env_var, dict):
|
||||
env_var = env_var.setdefault(key, {})
|
||||
|
||||
# get proper field with last_key
|
||||
target_field = self.next_field(target_field, last_key, self.case_sensitive)
|
||||
|
||||
# check if env_val maps to a complex field and if so, parse the env_val
|
||||
if (target_field or is_dict) and env_val:
|
||||
if isinstance(target_field, FieldInfo):
|
||||
is_complex, allow_json_failure = self._field_is_complex(target_field)
|
||||
if not is_complex and self._matches_alias_path_head(target_field, last_key):
|
||||
# The env value sits under an AliasPath head (e.g. AliasPath('path', 0));
|
||||
# decode it so pydantic can navigate into the container (see #670).
|
||||
is_complex, allow_json_failure = True, True
|
||||
if self.env_parse_enums:
|
||||
enum_val = _annotation_enum_name_to_val(target_field.annotation, env_val)
|
||||
env_val = env_val if enum_val is None else enum_val
|
||||
elif target_field:
|
||||
# target_field is a raw type (e.g. from dict value type annotation)
|
||||
is_complex = _annotation_is_complex(target_field, [], self._init_state)
|
||||
allow_json_failure = True
|
||||
else:
|
||||
# nested field type is dict
|
||||
is_complex, allow_json_failure = True, True
|
||||
if is_complex:
|
||||
try:
|
||||
field_info = target_field if isinstance(target_field, FieldInfo) else None
|
||||
env_val = self.decode_complex_value(last_key, field_info, env_val) # type: ignore
|
||||
except ValueError:
|
||||
if not allow_json_failure:
|
||||
raise
|
||||
if isinstance(env_var, dict) and (
|
||||
last_key not in env_var or not isinstance(env_val, EnvNoneType) or env_var[last_key] == {}
|
||||
):
|
||||
env_var[last_key] = self._coerce_env_val_strict(target_field, env_val)
|
||||
return result
|
||||
|
||||
def _coerce_env_val_strict(self, field: FieldInfo | None, value: Any) -> Any:
|
||||
"""
|
||||
Coerce environment string values based on field annotation if model config is `strict=True`
|
||||
or if the field annotation contains strict-annotated types (e.g. Optional[StrictBool]).
|
||||
|
||||
Args:
|
||||
field: The field.
|
||||
value: The value to coerce.
|
||||
|
||||
Returns:
|
||||
The coerced value if successful, otherwise the original value.
|
||||
"""
|
||||
try:
|
||||
should_coerce = self.config.get('strict')
|
||||
if isinstance(field, FieldInfo):
|
||||
annotation, metadata = _get_field_annotation_and_metadata(field)
|
||||
if not should_coerce:
|
||||
should_coerce = (
|
||||
is_union_origin(get_origin(annotation)) and _union_has_strict_types(annotation)
|
||||
) or _literal_has_numeric_enum(annotation)
|
||||
if should_coerce and isinstance(value, str) and isinstance(field, FieldInfo):
|
||||
if value == self.env_parse_none_str:
|
||||
return value
|
||||
has_json = any(isinstance(md, Json) for md in metadata) # type: ignore[misc]
|
||||
if not has_json and not _annotation_contains_types(annotation, (Json,), is_instance=True):
|
||||
try:
|
||||
return TypeAdapter(cast('Any', field.annotation)).validate_python(value)
|
||||
except ValidationError:
|
||||
# Try JSON decoding as fallback (e.g. 'true' -> True for StrictBool)
|
||||
decoded = json.loads(value)
|
||||
if not isinstance(decoded, str):
|
||||
return TypeAdapter(cast('Any', field.annotation)).validate_python(decoded)
|
||||
raise
|
||||
except (ValidationError, json.JSONDecodeError):
|
||||
# Allow validation error to be raised at time of instantiation
|
||||
pass
|
||||
return value
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f'{self.__class__.__name__}(env_nested_delimiter={self.env_nested_delimiter!r}, '
|
||||
f'env_prefix_len={self.env_prefix_len!r})'
|
||||
)
|
||||
|
||||
|
||||
__all__ = ['EnvSettingsSource']
|
||||
Reference in New Issue
Block a user