From 80a2992fb6e668aa5f3b921bf9d4acbf84a9fe80 Mon Sep 17 00:00:00 2001 From: "Kobagapu, Akshit (RDD-RM)" Date: Thu, 5 Dec 2024 11:01:13 +0100 Subject: [PATCH] feat: update PipelineConfig to support a Union of List and OrderedDict --- examples/pipeline/main.py | 10 +++--- src/py_app_dev/core/pipeline.py | 18 +++++++--- tests/test_pipeline.py | 62 +++++++++++++++++++++++++++------ 3 files changed, 69 insertions(+), 21 deletions(-) diff --git a/examples/pipeline/main.py b/examples/pipeline/main.py index 954ec8e..bf003f7 100644 --- a/examples/pipeline/main.py +++ b/examples/pipeline/main.py @@ -2,6 +2,7 @@ from abc import ABC from dataclasses import dataclass from pathlib import Path +from typing import Optional from mashumaro import DataClassDictMixin @@ -16,9 +17,10 @@ class ExecutionEnvironment: class PipelineStep(Runnable, ABC): - def __init__(self, environment: ExecutionEnvironment, group_name: str) -> None: + def __init__(self, environment: ExecutionEnvironment, group_name: Optional[str]) -> None: + super().__init__() self.environment = environment - self.output_dir = self.environment.output_dir / group_name + self.output_dir = self.environment.output_dir / group_name if group_name else self.environment.output_dir @property def project_root_dir(self) -> Path: @@ -32,9 +34,7 @@ class MyConfig(DataClassDictMixin): def main() -> None: this_dir = Path(__file__).parent - my_config = MyConfig.from_dict( - json.loads(this_dir.joinpath("config.json").read_text()) - ) + my_config = MyConfig.from_dict(json.loads(this_dir.joinpath("config.json").read_text())) loader = PipelineLoader[PipelineStep](my_config.pipeline, this_dir) steps_references = loader.load_steps() for step_reference in steps_references: diff --git a/src/py_app_dev/core/pipeline.py b/src/py_app_dev/core/pipeline.py index c0f9566..3b6547e 100644 --- a/src/py_app_dev/core/pipeline.py +++ b/src/py_app_dev/core/pipeline.py @@ -12,6 +12,7 @@ Type, TypeAlias, TypeVar, + Union, ) from mashumaro import DataClassDictMixin @@ -37,7 +38,7 @@ class PipelineStepConfig(DataClassDictMixin): config: Optional[Dict[str, Any]] = None -PipelineConfig: TypeAlias = OrderedDict[str, List[PipelineStepConfig]] +PipelineConfig: TypeAlias = Union[List[PipelineStepConfig], OrderedDict[str, List[PipelineStepConfig]]] TPipelineStep = TypeVar("TPipelineStep") @@ -52,7 +53,7 @@ class PipelineStep: class PipelineStepReference(Generic[TPipelineStep]): """Once a Step is found, keep the Step class reference to be able to instantiate it later.""" - group_name: str + group_name: Optional[str] _class: Type[TPipelineStep] config: Optional[Dict[str, Any]] = None @@ -64,13 +65,20 @@ def __init__(self, pipeline_config: PipelineConfig, project_root_dir: Path) -> N def load_steps(self) -> List[PipelineStepReference[TPipelineStep]]: result = [] - for group_name, steps_config in self.pipeline_config.items(): - result.extend(self._load_steps(group_name, steps_config, self.project_root_dir)) + if isinstance(self.pipeline_config, list): + # Handle List[PipelineStepConfig] + result.extend(self._load_steps(None, self.pipeline_config, self.project_root_dir)) + elif isinstance(self.pipeline_config, OrderedDict): + # Handle OrderedDict[str, List[PipelineStepConfig]] + for group_name, steps_config in self.pipeline_config.items(): + result.extend(self._load_steps(group_name, steps_config, self.project_root_dir)) + else: + raise UserNotificationException("Invalid pipeline configuration. Expected a list or an ordered dictionary.") return result @staticmethod def _load_steps( - group_name: str, + group_name: Optional[str], steps_config: List[PipelineStepConfig], project_root_dir: Path, ) -> List[PipelineStepReference[TPipelineStep]]: diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 0f0f4b6..96f6614 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -1,6 +1,6 @@ import textwrap from pathlib import Path -from typing import List +from typing import Any, List, OrderedDict import pytest @@ -8,14 +8,11 @@ from py_app_dev.core.pipeline import PipelineLoader, PipelineStep, PipelineStepConfig -def test_load_unknown_step(): - with pytest.raises(UserNotificationException): - PipelineLoader[PipelineStep]._load_steps("install", [PipelineStepConfig(step="StepIDontExist")], Path(".")) - - -def test_load_step_from_file(tmp_path: Path) -> None: - my_python_file = tmp_path / "my_python_file.py" - my_python_file.write_text( +@pytest.fixture +def my_python_file(tmp_path: Path) -> Path: + """Fixture to create a temporary Python file with a sample step.""" + python_file = tmp_path / "my_python_file.py" + python_file.write_text( textwrap.dedent( """\ from typing import List @@ -31,10 +28,19 @@ def get_outputs(self) -> List[Path]: """ ) ) + return python_file + + +def test_load_unknown_step(): + with pytest.raises(UserNotificationException): + PipelineLoader[PipelineStep]._load_steps("install", [PipelineStepConfig(step="StepIDontExist")], Path(".")) + + +def test_load_step_from_file(my_python_file: Path) -> None: result = PipelineLoader[PipelineStep]._load_steps( "install", - [PipelineStepConfig(step="MyStep", file="my_python_file.py", config={"data": "value"})], - tmp_path, + [PipelineStepConfig(step="MyStep", file=str(my_python_file), config={"data": "value"})], + my_python_file.parent, ) assert len(result) == 1 assert result[0].group_name == "install" @@ -58,3 +64,37 @@ def test_load_module_step_builtin(): step_class_name = "MyCustomPipelineStep" result = PipelineLoader[MyCustomPipelineStep]._load_module_step(module_name, step_class_name) assert result == MyCustomPipelineStep + + +def test_load_pipeline_config_as_list(my_python_file: Path) -> None: + # Define the pipeline configuration as a list + pipeline_config = [PipelineStepConfig(step="MyStep", file=str(my_python_file), config={"data": "value"})] + + loader = PipelineLoader[PipelineStep](pipeline_config, my_python_file) + steps = loader.load_steps() + + assert len(steps) == 1 + assert steps[0].group_name is None + assert steps[0]._class.__name__ == "MyStep" + assert steps[0].config == {"data": "value"} + + +def test_load_pipeline_config_as_ordereddict(my_python_file: Path) -> None: + # Define the pipeline configuration as an OrderedDict + pipeline_config = OrderedDict({"install": [PipelineStepConfig(step="MyStep", file=str(my_python_file), config={"data": "value"})]}) + + loader = PipelineLoader[PipelineStep](pipeline_config, my_python_file) + steps = loader.load_steps() + + assert len(steps) == 1 + assert steps[0].group_name == "install" + assert steps[0]._class.__name__ == "MyStep" + assert steps[0].config == {"data": "value"} + + +def test_invalid_pipeline_config() -> None: + # Define an invalid pipeline configuration + invalid_pipeline_config: Any = "InvalidConfig" + + with pytest.raises(UserNotificationException, match="Invalid pipeline configuration"): + PipelineLoader[PipelineStep](invalid_pipeline_config, Path(".")).load_steps()