Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions examples/pipeline/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
from abc import ABC
from dataclasses import dataclass
from pathlib import Path
from typing import Optional

from mashumaro import DataClassDictMixin

Expand All @@ -16,9 +17,10 @@ class ExecutionEnvironment:


class PipelineStep(Runnable, ABC):
def __init__(self, environment: ExecutionEnvironment, group_name: str) -> None:
def __init__(self, environment: ExecutionEnvironment, group_name: Optional[str]) -> None:
super().__init__()
self.environment = environment
self.output_dir = self.environment.output_dir / group_name
self.output_dir = self.environment.output_dir / group_name if group_name else self.environment.output_dir

@property
def project_root_dir(self) -> Path:
Expand All @@ -32,9 +34,7 @@ class MyConfig(DataClassDictMixin):

def main() -> None:
this_dir = Path(__file__).parent
my_config = MyConfig.from_dict(
json.loads(this_dir.joinpath("config.json").read_text())
)
my_config = MyConfig.from_dict(json.loads(this_dir.joinpath("config.json").read_text()))
loader = PipelineLoader[PipelineStep](my_config.pipeline, this_dir)
steps_references = loader.load_steps()
for step_reference in steps_references:
Expand Down
18 changes: 13 additions & 5 deletions src/py_app_dev/core/pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
Type,
TypeAlias,
TypeVar,
Union,
)

from mashumaro import DataClassDictMixin
Expand All @@ -37,7 +38,7 @@ class PipelineStepConfig(DataClassDictMixin):
config: Optional[Dict[str, Any]] = None


PipelineConfig: TypeAlias = OrderedDict[str, List[PipelineStepConfig]]
PipelineConfig: TypeAlias = Union[List[PipelineStepConfig], OrderedDict[str, List[PipelineStepConfig]]]

TPipelineStep = TypeVar("TPipelineStep")

Expand All @@ -52,7 +53,7 @@ class PipelineStep:
class PipelineStepReference(Generic[TPipelineStep]):
"""Once a Step is found, keep the Step class reference to be able to instantiate it later."""

group_name: str
group_name: Optional[str]
_class: Type[TPipelineStep]
config: Optional[Dict[str, Any]] = None

Expand All @@ -64,13 +65,20 @@ def __init__(self, pipeline_config: PipelineConfig, project_root_dir: Path) -> N

def load_steps(self) -> List[PipelineStepReference[TPipelineStep]]:
result = []
for group_name, steps_config in self.pipeline_config.items():
result.extend(self._load_steps(group_name, steps_config, self.project_root_dir))
if isinstance(self.pipeline_config, list):
# Handle List[PipelineStepConfig]
result.extend(self._load_steps(None, self.pipeline_config, self.project_root_dir))
elif isinstance(self.pipeline_config, OrderedDict):
# Handle OrderedDict[str, List[PipelineStepConfig]]
for group_name, steps_config in self.pipeline_config.items():
result.extend(self._load_steps(group_name, steps_config, self.project_root_dir))
else:
raise UserNotificationException("Invalid pipeline configuration. Expected a list or an ordered dictionary.")
return result

@staticmethod
def _load_steps(
group_name: str,
group_name: Optional[str],
steps_config: List[PipelineStepConfig],
project_root_dir: Path,
) -> List[PipelineStepReference[TPipelineStep]]:
Expand Down
62 changes: 51 additions & 11 deletions tests/test_pipeline.py
Original file line number Diff line number Diff line change
@@ -1,21 +1,18 @@
import textwrap
from pathlib import Path
from typing import List
from typing import Any, List, OrderedDict

import pytest

from py_app_dev.core.exceptions import UserNotificationException
from py_app_dev.core.pipeline import PipelineLoader, PipelineStep, PipelineStepConfig


def test_load_unknown_step():
with pytest.raises(UserNotificationException):
PipelineLoader[PipelineStep]._load_steps("install", [PipelineStepConfig(step="StepIDontExist")], Path("."))


def test_load_step_from_file(tmp_path: Path) -> None:
my_python_file = tmp_path / "my_python_file.py"
my_python_file.write_text(
@pytest.fixture
def my_python_file(tmp_path: Path) -> Path:
"""Fixture to create a temporary Python file with a sample step."""
python_file = tmp_path / "my_python_file.py"
python_file.write_text(
textwrap.dedent(
"""\
from typing import List
Expand All @@ -31,10 +28,19 @@ def get_outputs(self) -> List[Path]:
"""
)
)
return python_file


def test_load_unknown_step():
with pytest.raises(UserNotificationException):
PipelineLoader[PipelineStep]._load_steps("install", [PipelineStepConfig(step="StepIDontExist")], Path("."))


def test_load_step_from_file(my_python_file: Path) -> None:
result = PipelineLoader[PipelineStep]._load_steps(
"install",
[PipelineStepConfig(step="MyStep", file="my_python_file.py", config={"data": "value"})],
tmp_path,
[PipelineStepConfig(step="MyStep", file=str(my_python_file), config={"data": "value"})],
my_python_file.parent,
)
assert len(result) == 1
assert result[0].group_name == "install"
Expand All @@ -58,3 +64,37 @@ def test_load_module_step_builtin():
step_class_name = "MyCustomPipelineStep"
result = PipelineLoader[MyCustomPipelineStep]._load_module_step(module_name, step_class_name)
assert result == MyCustomPipelineStep


def test_load_pipeline_config_as_list(my_python_file: Path) -> None:
# Define the pipeline configuration as a list
pipeline_config = [PipelineStepConfig(step="MyStep", file=str(my_python_file), config={"data": "value"})]

loader = PipelineLoader[PipelineStep](pipeline_config, my_python_file)
steps = loader.load_steps()

assert len(steps) == 1
assert steps[0].group_name is None
assert steps[0]._class.__name__ == "MyStep"
assert steps[0].config == {"data": "value"}


def test_load_pipeline_config_as_ordereddict(my_python_file: Path) -> None:
# Define the pipeline configuration as an OrderedDict
pipeline_config = OrderedDict({"install": [PipelineStepConfig(step="MyStep", file=str(my_python_file), config={"data": "value"})]})

loader = PipelineLoader[PipelineStep](pipeline_config, my_python_file)
steps = loader.load_steps()

assert len(steps) == 1
assert steps[0].group_name == "install"
assert steps[0]._class.__name__ == "MyStep"
assert steps[0].config == {"data": "value"}


def test_invalid_pipeline_config() -> None:
# Define an invalid pipeline configuration
invalid_pipeline_config: Any = "InvalidConfig"

with pytest.raises(UserNotificationException, match="Invalid pipeline configuration"):
PipelineLoader[PipelineStep](invalid_pipeline_config, Path(".")).load_steps()