From 488602cbcf357b3bc2a42865037f902032f2abc9 Mon Sep 17 00:00:00 2001 From: Timothe Jost <timothe.jost@wanadoo.fr> Date: Tue, 27 Feb 2024 19:04:35 +0100 Subject: [PATCH] removing single step feature (long overdue) --- src/pypelines/__init__.py | 2 +- src/pypelines/examples.py | 2 +- src/pypelines/pickle_backend.py | 2 +- src/pypelines/pipelines.py | 34 ++++++++++++++++----------------- src/pypelines/pipes.py | 12 ++++++------ src/pypelines/steps.py | 6 +++--- 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/pypelines/__init__.py b/src/pypelines/__init__.py index f3c2e9b..ba31a3e 100644 --- a/src/pypelines/__init__.py +++ b/src/pypelines/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.0.22" +__version__ = "0.0.23" from . import loggs from .pipes import * diff --git a/src/pypelines/examples.py b/src/pypelines/examples.py index 6f77e0e..f762fcd 100644 --- a/src/pypelines/examples.py +++ b/src/pypelines/examples.py @@ -7,7 +7,7 @@ example_pipeline = Pipeline("example") @example_pipeline.register_pipe class treated_videos(PicklePipe): - single_step = True + # single_step = True @stepmethod() def compress(self, session, video_codec="ffmpeg", extra="", compression_rate=0.5): diff --git a/src/pypelines/pickle_backend.py b/src/pypelines/pickle_backend.py index 50b3e2e..39a5f90 100644 --- a/src/pypelines/pickle_backend.py +++ b/src/pypelines/pickle_backend.py @@ -267,7 +267,7 @@ class PickleDiskObject(BaseDiskObject): class PicklePipe(BasePipe): - single_step = False + # single_step = False step_class = BaseStep disk_class = PickleDiskObject diff --git a/src/pypelines/pipelines.py b/src/pypelines/pipelines.py index 9e96349..39d48af 100644 --- a/src/pypelines/pipelines.py +++ b/src/pypelines/pipelines.py @@ -10,7 +10,7 @@ if TYPE_CHECKING: class Pipeline: use_celery = False - pipes: Dict[str, BasePipe] + pipes: Dict[str, "BasePipe"] def __init__(self, name: str, conf_path=None, use_celery=False): self.pipeline_name = name @@ -27,29 +27,29 @@ class Pipeline: instance = pipe_class(self) # attaches the instance itself to the pipeline, and to the dictionnary 'pipes' of the current pipeline - if instance.single_step: - # in case it's a single_step instance (speficied by the user, not auto detected) - # then we attach the step to the pipeline directly as a pipe, for ease of use. - step = list(instance.steps.values())[0] - self.pipes[instance.pipe_name] = step - # just add steps to the step instance serving as a pipe, so that it behaves - # similarly to a pipe for some pipelines function requiring this attribute to exist. - step.steps = instance.steps - setattr(self, instance.pipe_name, step) - else: - # in case it's a pipe, we attach it in a simple manner. - self.pipes[instance.pipe_name] = instance - setattr(self, instance.pipe_name, instance) + # if instance.single_step: + # # in case it's a single_step instance (speficied by the user, not auto detected) + # # then we attach the step to the pipeline directly as a pipe, for ease of use. + # step = list(instance.steps.values())[0] + # self.pipes[instance.pipe_name] = step + # # just add steps to the step instance serving as a pipe, so that it behaves + # # similarly to a pipe for some pipelines function requiring this attribute to exist. + # step.steps = instance.steps + # setattr(self, instance.pipe_name, step) + # else: + # in case it's a pipe, we attach it in a simple manner. + self.pipes[instance.pipe_name] = instance + setattr(self, instance.pipe_name, instance) self.resolved = False return pipe_class - def resolve_instance(self, instance_name: str) -> Type["BaseStep"]: + def resolve_instance(self, instance_name: str) -> "BaseStep": pipe_name, step_name = instance_name.split(".") try: pipe = self.pipes[pipe_name] - if pipe.single_step: - return pipe + # if pipe.single_step: + # return pipe return pipe.steps[step_name] except KeyError as exc: raise KeyError(f"No instance {instance_name} has been registered to the pipeline") from exc diff --git a/src/pypelines/pipes.py b/src/pypelines/pipes.py index 1fd2d0e..9dc41a8 100644 --- a/src/pypelines/pipes.py +++ b/src/pypelines/pipes.py @@ -20,7 +20,7 @@ class BasePipe(metaclass=ABCMeta): default_extra = None - single_step: bool = False # a flag to tell the initializer to bind the unique step of this pipe in place + # single_step: bool = False # a flag to tell the initializer to bind the unique step of this pipe in place # of the pipe itself, to the registered pipeline. step_class: Type[BaseStep] = BaseStep disk_class: Type[BaseDiskObject] = BaseDiskObject @@ -44,11 +44,11 @@ class BasePipe(metaclass=ABCMeta): f" { _steps = }" ) - if len(_steps) > 1 and self.single_step: - raise ValueError( - f"Cannot set single_step to True if you registered more than one step inside {self.pipe_name} class." - f" { _steps = }" - ) + # if len(_steps) > 1 and self.single_step: + # raise ValueError( + # f"Cannot set single_step to True if you registered more than one step inside {self.pipe_name} class." + # f" { _steps = }" + # ) number_of_steps_with_requirements = 0 for step in _steps.values(): diff --git a/src/pypelines/steps.py b/src/pypelines/steps.py index ab95ea4..211445b 100644 --- a/src/pypelines/steps.py +++ b/src/pypelines/steps.py @@ -97,9 +97,9 @@ class BaseStep: def full_name(self): return f"{self.pipe_name}.{self.step_name}" - @property - def single_step(self): - return self.pipe.single_step + # @property + # def single_step(self): + # return self.pipe.single_step def disk_step(self, session, extra=""): disk_object = self.get_disk_object(session, extra) -- GitLab