Skip to content

Configuration

pluginlake.config

Root settings for pluginlake.

Provides Pydantic Settings-based configuration with environment variable support and a medallion-architecture storage layer structure:

All settings can be overridden via environment variables. See each class for the applicable prefix.

LogLevel

Bases: StrEnum

Supported log levels.

Source code in src/pluginlake/config.py
class LogLevel(StrEnum):
    """Supported log levels."""

    DEBUG = "DEBUG"
    INFO = "INFO"
    WARNING = "WARNING"
    ERROR = "ERROR"
    CRITICAL = "CRITICAL"

StorageLayer

Bases: StrEnum

Storage layers following the medallion architecture.

Attributes:

Name Type Description
RAW

Bronze layer — raw ingested data, stored as-is.

PROCESSED

Silver layer — cleaned and transformed data.

OUTPUT

Gold layer — curated, publication-ready datasets.

Source code in src/pluginlake/config.py
class StorageLayer(StrEnum):
    """Storage layers following the medallion architecture.

    Attributes:
        RAW: Bronze layer — raw ingested data, stored as-is.
        PROCESSED: Silver layer — cleaned and transformed data.
        OUTPUT: Gold layer — curated, publication-ready datasets.
    """

    RAW = "raw"
    PROCESSED = "processed"
    OUTPUT = "output"

Settings

Bases: BaseSettings

Base settings for pluginlake.

All settings can be overridden via environment variables prefixed with PLUGINLAKE_.

Source code in src/pluginlake/config.py
class Settings(BaseSettings):
    """Base settings for pluginlake.

    All settings can be overridden via environment variables
    prefixed with ``PLUGINLAKE_``.
    """

    model_config = SettingsConfigDict(
        env_prefix="PLUGINLAKE_",
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    debug: bool = False
    log_level: LogLevel = LogLevel.INFO
    verbose: bool = False

    @property
    def effective_log_level(self) -> int:
        """Return the effective log level, accounting for verbose/debug flags."""
        if self.debug or self.verbose:
            return logging.DEBUG
        return logging.getLevelNamesMapping()[self.log_level.value]

effective_log_level property

Return the effective log level, accounting for verbose/debug flags.

StorageSettings

Bases: BaseSettings

Storage layer configuration.

Defines the base data directory and the medallion-architecture layer structure (raw / processed / output).

Environment variables are prefixed with PLUGINLAKE_STORAGE_.

Examples:

>>> settings = StorageSettings(base_dir="/tmp/data")
>>> settings.raw_dir
PosixPath('/tmp/data/raw')
>>> settings.get_layer_path(StorageLayer.OUTPUT, "patients.parquet")
PosixPath('/tmp/data/output/patients.parquet')
Source code in src/pluginlake/config.py
class StorageSettings(BaseSettings):
    """Storage layer configuration.

    Defines the base data directory and the medallion-architecture
    layer structure (raw / processed / output).

    Environment variables are prefixed with ``PLUGINLAKE_STORAGE_``.

    Examples:
        >>> settings = StorageSettings(base_dir="/tmp/data")
        >>> settings.raw_dir
        PosixPath('/tmp/data/raw')
        >>> settings.get_layer_path(StorageLayer.OUTPUT, "patients.parquet")
        PosixPath('/tmp/data/output/patients.parquet')
    """

    model_config = SettingsConfigDict(
        env_prefix="PLUGINLAKE_STORAGE_",
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    base_dir: Path = Field(
        default=Path(".data"),
        description="Root directory for all storage layers.",
    )
    backend: Literal["local"] = Field(
        default="local",
        description="Storage backend type. Currently only 'local' is supported.",
    )

    @model_validator(mode="after")
    def _resolve_base_dir(self) -> "StorageSettings":
        """Resolve relative base_dir to an absolute path."""
        self.base_dir = self.base_dir.resolve()
        return self

    # -- Layer directory properties ------------------------------------------

    @property
    def raw_dir(self) -> Path:
        """Path to the raw (bronze) storage layer."""
        return self.base_dir / StorageLayer.RAW

    @property
    def processed_dir(self) -> Path:
        """Path to the processed (silver) storage layer."""
        return self.base_dir / StorageLayer.PROCESSED

    @property
    def output_dir(self) -> Path:
        """Path to the output (gold) storage layer."""
        return self.base_dir / StorageLayer.OUTPUT

    # -- Helper methods ------------------------------------------------------

    def get_layer_path(self, layer: StorageLayer, *parts: str) -> Path:
        """Return a path within a specific storage layer.

        Args:
            layer: The storage layer to build the path in.
            *parts: Additional path segments appended after the layer directory.

        Returns:
            Absolute path combining base_dir, layer, and any extra segments.
        """
        return self.base_dir / layer.value / Path(*parts) if parts else self.base_dir / layer.value

    def ensure_directories(self) -> None:
        """Create the base directory and all layer subdirectories.

        Raises:
            OSError: If directory creation fails due to permissions or
                other filesystem errors.
        """
        for layer in StorageLayer:
            (self.base_dir / layer.value).mkdir(parents=True, exist_ok=True)

    @property
    def layer_dirs(self) -> dict[StorageLayer, Path]:
        """Return a mapping of each storage layer to its directory path."""
        return {layer: self.base_dir / layer.value for layer in StorageLayer}

raw_dir property

Path to the raw (bronze) storage layer.

processed_dir property

Path to the processed (silver) storage layer.

output_dir property

Path to the output (gold) storage layer.

layer_dirs property

Return a mapping of each storage layer to its directory path.

get_layer_path(layer, *parts)

Return a path within a specific storage layer.

Parameters:

Name Type Description Default
layer StorageLayer

The storage layer to build the path in.

required
*parts str

Additional path segments appended after the layer directory.

()

Returns:

Type Description
Path

Absolute path combining base_dir, layer, and any extra segments.

Source code in src/pluginlake/config.py
def get_layer_path(self, layer: StorageLayer, *parts: str) -> Path:
    """Return a path within a specific storage layer.

    Args:
        layer: The storage layer to build the path in.
        *parts: Additional path segments appended after the layer directory.

    Returns:
        Absolute path combining base_dir, layer, and any extra segments.
    """
    return self.base_dir / layer.value / Path(*parts) if parts else self.base_dir / layer.value

ensure_directories()

Create the base directory and all layer subdirectories.

Raises:

Type Description
OSError

If directory creation fails due to permissions or other filesystem errors.

Source code in src/pluginlake/config.py
def ensure_directories(self) -> None:
    """Create the base directory and all layer subdirectories.

    Raises:
        OSError: If directory creation fails due to permissions or
            other filesystem errors.
    """
    for layer in StorageLayer:
        (self.base_dir / layer.value).mkdir(parents=True, exist_ok=True)

ServerSettings

Bases: BaseSettings

HTTP server configuration.

Environment variables are prefixed with PLUGINLAKE_SERVER_.

Source code in src/pluginlake/config.py
class ServerSettings(BaseSettings):
    """HTTP server configuration.

    Environment variables are prefixed with ``PLUGINLAKE_SERVER_``.
    """

    model_config = SettingsConfigDict(
        env_prefix="PLUGINLAKE_SERVER_",
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    host: str = Field(default="0.0.0.0", description="Bind address for the HTTP server.")  # noqa: S104
    port: int = Field(default=8000, description="Port for the HTTP server.")

PostgresSettings

Bases: BaseSettings

PostgreSQL connection configuration.

Environment variables are prefixed with PLUGINLAKE_POSTGRES_.

Source code in src/pluginlake/config.py
class PostgresSettings(BaseSettings):
    """PostgreSQL connection configuration.

    Environment variables are prefixed with ``PLUGINLAKE_POSTGRES_``.
    """

    model_config = SettingsConfigDict(
        env_prefix="PLUGINLAKE_POSTGRES_",
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    host: str = Field(default="localhost", description="PostgreSQL hostname.")
    port: int = Field(default=5432, description="PostgreSQL port.")
    user: SecretStr = Field(description="PostgreSQL user.")
    password: SecretStr = Field(description="PostgreSQL password.")
    db: str = Field(default="pluginlake", description="PostgreSQL database name.")

LogSettings

Bases: BaseSettings

Operational JSONL log configuration.

Environment variables are prefixed with PLUGINLAKE_LOG_.

Source code in src/pluginlake/config.py
class LogSettings(BaseSettings):
    """Operational JSONL log configuration.

    Environment variables are prefixed with ``PLUGINLAKE_LOG_``.
    """

    model_config = SettingsConfigDict(
        env_prefix="PLUGINLAKE_LOG_",
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    base_dir: Path = Field(
        default=Path(".data/logs"),
        description="Root directory for operational JSONL logs.",
    )

    @model_validator(mode="after")
    def _resolve_base_dir(self) -> "LogSettings":
        self.base_dir = self.base_dir.resolve()
        return self

    @property
    def query_log_dir(self) -> Path:
        """Path to the query/traffic log directory."""
        return self.base_dir / "query_log"

    @property
    def ingestion_log_dir(self) -> Path:
        """Path to the ingestion history log directory."""
        return self.base_dir / "ingestion_log"

    def ensure_directories(self) -> None:
        """Create both log subdirectories."""
        self.query_log_dir.mkdir(parents=True, exist_ok=True)
        self.ingestion_log_dir.mkdir(parents=True, exist_ok=True)

query_log_dir property

Path to the query/traffic log directory.

ingestion_log_dir property

Path to the ingestion history log directory.

ensure_directories()

Create both log subdirectories.

Source code in src/pluginlake/config.py
def ensure_directories(self) -> None:
    """Create both log subdirectories."""
    self.query_log_dir.mkdir(parents=True, exist_ok=True)
    self.ingestion_log_dir.mkdir(parents=True, exist_ok=True)

DagsterSettings

Bases: BaseSettings

Dagster orchestrator configuration.

Environment variables are prefixed with PLUGINLAKE_DAGSTER_.

Source code in src/pluginlake/config.py
class DagsterSettings(BaseSettings):
    """Dagster orchestrator configuration.

    Environment variables are prefixed with ``PLUGINLAKE_DAGSTER_``.
    """

    model_config = SettingsConfigDict(
        env_prefix="PLUGINLAKE_DAGSTER_",
        env_file=".env",
        env_file_encoding="utf-8",
        extra="ignore",
    )

    module: str = Field(
        default="pluginlake.definitions",
        description="Python module containing Dagster definitions.",
    )
    webserver_url: str = Field(
        default="http://localhost:3000",
        description="URL of the Dagster webserver (GraphQL API).",
    )
    pg_host: str = Field(default="localhost", description="Dagster PostgreSQL hostname.")
    pg_user: SecretStr = Field(description="Dagster PostgreSQL user.")
    pg_password: SecretStr = Field(description="Dagster PostgreSQL password.")
    pg_db: str = Field(default="dagster", description="Dagster PostgreSQL database name.")
    DAGSTER_HOME: Path = Field(default=Path.home(), description="Directory for Dagster to store instance data.")