diff --git a/.coverage b/.coverage new file mode 100644 index 0000000000000000000000000000000000000000..c7b5fd8c2fd5aeaf44e3768d819c42ec925cddfe Binary files /dev/null and b/.coverage differ diff --git a/coverage.xml b/coverage.xml new file mode 100644 index 0000000000000000000000000000000000000000..8aa3a338ae52cef37a9147dd76541c79a2fb4cd7 --- /dev/null +++ b/coverage.xml @@ -0,0 +1,45 @@ +<?xml version="1.0" ?> +<coverage version="7.3.2" timestamp="1696514560568" lines-valid="13" lines-covered="5" line-rate="0.3846" branches-covered="0" branches-valid="0" branch-rate="0" complexity="0"> + <!-- Generated by coverage.py: https://coverage.readthedocs.io/en/7.3.2 --> + <!-- Based on https://raw.githubusercontent.com/cobertura/web/master/htdocs/xml/coverage-04.dtd --> + <sources> + <source>C:\Users\tjostmou\Documents\Python\__packages__\Pypelines</source> + </sources> + <packages> + <package name="pypelines" line-rate="1" branch-rate="0" complexity="0"> + <classes> + <class name="__init__.py" filename="pypelines/__init__.py" complexity="0" line-rate="1" branch-rate="0"> + <methods/> + <lines> + <line number="1" hits="1"/> + </lines> + </class> + </classes> + </package> + <package name="tests" line-rate="0.3333" branch-rate="0" complexity="0"> + <classes> + <class name="__init__.py" filename="tests/__init__.py" complexity="0" line-rate="1" branch-rate="0"> + <methods/> + <lines/> + </class> + <class name="tests.py" filename="tests/tests.py" complexity="0" line-rate="0.3333" branch-rate="0"> + <methods/> + <lines> + <line number="1" hits="1"/> + <line number="3" hits="1"/> + <line number="5" hits="1"/> + <line number="7" hits="1"/> + <line number="9" hits="0"/> + <line number="11" hits="0"/> + <line number="12" hits="0"/> + <line number="13" hits="0"/> + <line number="15" hits="0"/> + <line number="16" hits="0"/> + <line number="18" hits="0"/> + <line number="19" hits="0"/> + </lines> + </class> + </classes> + </package> + </packages> +</coverage> diff --git a/pypelines/__init__.py b/pypelines/__init__.py index 134c1697b67ddffe61701c0b7b617af4c42023d0..38f5db9aa3e3aa69faf0254954368e77a172d35c 100644 --- a/pypelines/__init__.py +++ b/pypelines/__init__.py @@ -1,2 +1,6 @@ __version__ = "0.0.1" +from .pipe import * +from .pipeline import * +from .step import * +from .versions import * \ No newline at end of file diff --git a/pypelines/__pycache__/__init__.cpython-311.pyc b/pypelines/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8550abbac6f7ee0b29e5f4d402f453d9c6cdd4e9 Binary files /dev/null and b/pypelines/__pycache__/__init__.cpython-311.pyc differ diff --git a/pypelines/__pycache__/examples.cpython-311.pyc b/pypelines/__pycache__/examples.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a86a6670afd5629a2944bd98be85e7c0ea8f4a56 Binary files /dev/null and b/pypelines/__pycache__/examples.cpython-311.pyc differ diff --git a/pypelines/__pycache__/loggs.cpython-311.pyc b/pypelines/__pycache__/loggs.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..59a5b8beef301a67600a51883a78eafa02fc3586 Binary files /dev/null and b/pypelines/__pycache__/loggs.cpython-311.pyc differ diff --git a/pypelines/__pycache__/multisession.cpython-311.pyc b/pypelines/__pycache__/multisession.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3a084f7de3b503704d14479f1d1512ff1d81e897 Binary files /dev/null and b/pypelines/__pycache__/multisession.cpython-311.pyc differ diff --git a/pypelines/__pycache__/pickle_backend.cpython-311.pyc b/pypelines/__pycache__/pickle_backend.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3b724cfb7ee20e8dbeae3727c4c62d0942f3a241 Binary files /dev/null and b/pypelines/__pycache__/pickle_backend.cpython-311.pyc differ diff --git a/pypelines/__pycache__/pipe.cpython-311.pyc b/pypelines/__pycache__/pipe.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5c73a501c7a2e462bc56295d5ffe27f1cbab1ea7 Binary files /dev/null and b/pypelines/__pycache__/pipe.cpython-311.pyc differ diff --git a/pypelines/__pycache__/pipeline.cpython-311.pyc b/pypelines/__pycache__/pipeline.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..caf0bc1b20115b3044272f07dc526daaa7680c37 Binary files /dev/null and b/pypelines/__pycache__/pipeline.cpython-311.pyc differ diff --git a/pypelines/__pycache__/step.cpython-311.pyc b/pypelines/__pycache__/step.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..820550bcc69593057ce4d311f66a9064ab58d1e1 Binary files /dev/null and b/pypelines/__pycache__/step.cpython-311.pyc differ diff --git a/pypelines/__pycache__/versions.cpython-311.pyc b/pypelines/__pycache__/versions.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cb02e1806733ca7950fff3eddc97958cdb7bb6bc Binary files /dev/null and b/pypelines/__pycache__/versions.cpython-311.pyc differ diff --git a/pypelines/examples.py b/pypelines/examples.py new file mode 100644 index 0000000000000000000000000000000000000000..eb6d392ae851db3cd680e7520f021fe9edd326fd --- /dev/null +++ b/pypelines/examples.py @@ -0,0 +1,20 @@ + +from .pickle_backend import PicklePipe +from .pipeline import BasePipeline +from .step import stepmethod + +class ExamplePipeline(BasePipeline): + ... + +example_pipeline = ExamplePipeline() + +@example_pipeline.register_pipe +class ExamplePipe(PicklePipe): + + @stepmethod() + def example_step1(self, argument1, optionnal_argument2 = "23"): + return {"argument1" : argument1, "optionnal_argument2" : optionnal_argument2} + + @stepmethod(requires = [example_step1]) + def example_step2(self, argument1, argument2): + return {"argument1" : argument1, "argument2" : argument2} diff --git a/pypelines/loggs.py b/pypelines/loggs.py new file mode 100644 index 0000000000000000000000000000000000000000..a1b907b26113d8ee94bbf8378f68b4fba0947dcb --- /dev/null +++ b/pypelines/loggs.py @@ -0,0 +1,47 @@ + +import logging +from functools import wraps + +class ContextFilter(logging.Filter): + """ + This is a filter which injects contextual information into the log. + """ + def __init__(self, context_msg): + self.context_msg = context_msg + + def filter(self, record): + record.msg = f"{self.context_msg} {record.msg}" + return True + +class LogContext: + def __init__(self, context_msg): + self.context_msg = context_msg + self.filter_was_added = False + + def __enter__(self): + self.root_logger = logging.getLogger() + for filter in self.root_logger.filters: + if getattr(filter, "context_msg" , "") == self.context_msg: + return + + self.filter_was_added = True + self.context_filter = ContextFilter(self.context_msg) + self.root_logger.addFilter(self.context_filter) + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.filter_was_added : + self.root_logger.removeFilter(self.context_filter) + +class LogSession(LogContext): + def __init__(self, session): + context_msg = "s#" + str(session.alias) + super().__init__(context_msg) + +def loggedmethod(func): + @wraps(func) + def wrapper(session, *args,**kwargs): + if kwargs.get("no_session_log", False) : + return func(*args,**kwargs) + with LogSession(session): + return func(session, *args,**kwargs) + return wrapper \ No newline at end of file diff --git a/pypelines/multisession.py b/pypelines/multisession.py index f3d62cd102acd7f2a453a914a253bb7462fb6d38..260619286d05a0144803aa3b33dfa585a060961a 100644 --- a/pypelines/multisession.py +++ b/pypelines/multisession.py @@ -2,5 +2,5 @@ class BaseMultisessionAccessor: - def __init__(self, parent_step): + def __init__(self, parent): pass \ No newline at end of file diff --git a/pypelines/pickle_backend.py b/pypelines/pickle_backend.py new file mode 100644 index 0000000000000000000000000000000000000000..26a1bee58acc1d27282739cddca23cc2426072d7 --- /dev/null +++ b/pypelines/pickle_backend.py @@ -0,0 +1,6 @@ +import pickle + +from .pipe import BasePipe + +class PicklePipe(BasePipe): + ... \ No newline at end of file diff --git a/pypelines/pipe.py b/pypelines/pipe.py index 0f8d8580429088dcd5d6a524a97def51919c847e..e3f27ecbe62322f60a75181982a6f54a6e2f70fc 100644 --- a/pypelines/pipe.py +++ b/pypelines/pipe.py @@ -1,11 +1,22 @@ -from . step import BaseStep, step +from . step import BaseStep from . multisession import BaseMultisessionAccessor -from typing import Callable, Type, Iterable +from functools import wraps + +from typing import Callable, Type, Iterable, Protocol, TYPE_CHECKING + +if TYPE_CHECKING: + from .pipeline import BasePipeline + +class OutputData(Protocol): + """Can be a mapping, iterable, single element, or None. + + This class is defined for typehints, and is not a real class useable at runtime""" class PipeMetaclass(type): def __new__(cls : Type, pipe_name : str, bases : Iterable[Type], attributes : dict) -> Type: + print(pipe_name, attributes) attributes["pipe_name"] = pipe_name steps = {} @@ -14,16 +25,16 @@ class PipeMetaclass(type): if getattr(attribute, "is_step", False): steps[name] = PipeMetaclass.make_step_attributes(attribute , pipe_name , name) + attributes["steps"] = steps + if len(attributes["steps"]) > 1 and attributes["single_step"]: raise ValueError(f"Cannot set single_step to True if you registered more than one step inside {pipe_name} class") - attributes["steps"] = steps - return super().__new__(cls, pipe_name, bases, attributes) @staticmethod def make_step_attributes(step : Callable, pipe_name : str, step_name : str) -> Callable: - + print(f"init of {pipe_name}") setattr(step, "pipe_name", pipe_name) setattr(step, "step_name", step_name) @@ -39,7 +50,7 @@ class BasePipe(metaclass = PipeMetaclass): step_class = BaseStep multisession_class = BaseMultisessionAccessor - def __init__(self, parent_pipeline : BasePipeline) -> None : + def __init__(self, parent_pipeline : "BasePipeline") -> None : self.multisession = self.multisession_class(self) self.pipeline = parent_pipeline @@ -62,10 +73,22 @@ class BasePipe(metaclass = PipeMetaclass): self.pipeline.pipes[self.pipe_name] = self setattr(self.pipeline, self.pipe_name, self) + self._make_wrapped_functions() + + def _make_wrapped_functions(self): + self.make_wrapped_save() + self.make_wrapped_load() + def __repr__(self) -> str: return f"<{self.__class__.__bases__[0].__name__}.{self.pipe_name} PipeObject>" - def file_getter(self, session, extra, version) -> | : + def make_wrapped_save(self): + self.save = self.dispatcher(self.file_saver) + + def make_wrapped_load(self): + self.load = self.dispatcher(self.file_loader) + + def file_getter(self, session, extra, version) -> OutputData : #finds file, opens it, and return data. #if it cannot find the file, it returns a IOError ... @@ -73,7 +96,8 @@ class BasePipe(metaclass = PipeMetaclass): def _check_version(self, step_name , found_version): #checks the found_version of the file is above or equal in the requirement order, to the step we are looking for - ... + #TODO + self.pipeline.get_requirement_stack(step_name) def step_version(self, step): #simply returns the current string of the version that is in . @@ -83,10 +107,10 @@ class BasePipe(metaclass = PipeMetaclass): #simply returns the version string of the file(s) that it found. ... - def file_saver(self, session, dumped_object, version ): + def file_saver(self, session, dumped_object, extra, version ): ... - def file_loader(self, session, dumped_object, version ): + def file_loader(self, session, extra, version ): ... def file_checker(self, session): @@ -96,7 +120,3 @@ class BasePipe(metaclass = PipeMetaclass): # the dispatcher must be return a wrapped function ... -class ExamplePipe(BasePipe): - - @step(requires = []) - def diff --git a/pypelines/pipeline.py b/pypelines/pipeline.py index 274307af9912155a0f1a0cc128ba10fa498a4f0b..ec6abf881a3719f77c0b3d2be5a3f8a36b624b27 100644 --- a/pypelines/pipeline.py +++ b/pypelines/pipeline.py @@ -1,13 +1,17 @@ +from typing import Callable, Type, Iterable, Protocol, TYPE_CHECKING - +if TYPE_CHECKING: + from .pipe import BasePipe class BasePipeline: pipes = {} - def register_pipe(self,pipe_class): + def register_pipe(self, pipe_class : type) -> type: + """Wrapper to instanciate and attache a a class inheriting from BasePipe it to the Pipeline instance. + The Wraper returns the class without changing it.""" pipe_class(self) return pipe_class diff --git a/pypelines/step.py b/pypelines/step.py index bcd0f0f4590bb9a7495445c398f2dba90373b472..18bc5ff98ec6557a533cd1df1506f96b92c0921e 100644 --- a/pypelines/step.py +++ b/pypelines/step.py @@ -1,6 +1,9 @@ from functools import wraps, partial, update_wrapper +from .loggs import loggedmethod +import logging +from typing import Callable -def step(requires = []): +def stepmethod(requires = []): # This method allows to register class methods inheriting of BasePipe as steps. # It basically just step an "is_step" stamp on the method that are defined as steps. # This stamp will later be used in the metaclass __new__ to set additionnal usefull attributes to those methods @@ -28,11 +31,9 @@ class BaseStep: self.requirement_stack = partial( self.pipeline.get_requirement_stack, instance = self ) self.step_version = partial( self.pipe.step_version, step = self ) - self.load = self._loading_wrapper(self.pipe.file_loader) - self.save = self._saving_wrapper(self.pipe.file_saver) - self.generate = self._generating_wrapper(self.step) update_wrapper(self, self.step) + self._make_wrapped_functions() def __call__(self, *args, **kwargs): return self.step(*args, **kwargs) @@ -47,13 +48,133 @@ class BaseStep: return self.pipe.dispatcher(self._version_wrapper(function, self.pipe.step_version)) def _generating_wrapper(self, function): - return self.pipe.dispatcher( - session_log_decorator - ) + return + + def _make_wrapped_functions(self): + self.make_wrapped_save() + self.make_wrapped_load() + self.make_wrapped_generate() + + def make_wrapped_save(self): + self.save = self._saving_wrapper(self.pipe.file_saver) + + def make_wrapped_load(self): + self.load = self._loading_wrapper(self.pipe.file_loader) + + def make_wrapped_generate(self): + self.generate = loggedmethod( + self._version_wrapper( + self.pipe.dispatcher( + self._loading_wrapper( + self._saving_wrapper( + self.pipe.pre_run_wrapper(self.step) + ) + ) + ) + ) + ) + def _version_wrapper(self, function_to_wrap, version_getter): @wraps(function_to_wrap) def wrapper(*args,**kwargs): version = version_getter(self) return function_to_wrap(*args, version=version, **kwargs) return wrapper + + def _loading_wrapper(self, func: Callable): + """ + Decorator to load instead of calculating if not refreshing and saved data exists + """ + + @wraps(func) + def wrap(session_details, *args, **kwargs): + """ + Decorator function + + Parameters + ---------- + *args : TYPE + DESCRIPTION. + **kwargs : TYPE + DESCRIPTION. + + Returns + ------- + TYPE + DESCRIPTION. + + """ + logger = logging.getLogger("load_pipeline") + + kwargs = kwargs.copy() + extra = kwargs.get("extra", None) + skipping = kwargs.pop("skip", False) + # we raise if file not found only if skipping is True + refresh = kwargs.get("refresh", False) + refresh_main_only = kwargs.get("refresh_main_only", False) + + if refresh_main_only: + # we set refresh true no matter what and then set + # refresh_main_only to False so that possible childs functions will never do this again + refresh = True + kwargs["refresh"] = False + kwargs["refresh_main_only"] = False + + if refresh and skipping: + raise ValueError( + """You tried to set refresh (or refresh_main_only) to True and skipping to True simultaneouly. + Stopped code to prevent mistakes : You probably set this by error as both have antagonistic effects. + (skipping passes without loading if file exists, refresh overwrites after generating output if file exists) + Please change arguments according to your clarified intention.""" + ) + + if not refresh: + if skipping and self.pipe.file_checker(session_details, extra): + logger.load_info( + f"File exists for {self.pipe_name}{'.' + extra if extra else ''}. Loading and processing have been skipped" + ) + return None + logger.debug(f"Trying to load saved data") + try: + result = self.pipe.file_loader(session_details, extra=extra) + logger.load_info( + f"Found and loaded {self.pipe_name}{'.' + extra if extra else ''} file. Processing has been skipped " + ) + return result + except IOError: + logger.load_info( + f"Could not find or load {self.pipe_name}{'.' + extra if extra else ''} saved file." + ) + + logger.load_info( + f"Performing the computation to generate {self.pipe_name}{'.' + extra if extra else ''}. Hold tight." + ) + return func(session_details, *args, **kwargs) + + return wrap + + def _saving_wrapper(self, func: Callable): + # decorator to load instead of calculating if not refreshing and saved data exists + @wraps(func) + def wrap(session_details, *args, **kwargs): + + logger = logging.getLogger("save_pipeline") + + kwargs = kwargs.copy() + extra = kwargs.get("extra", "") + save_pipeline = kwargs.pop("save_pipeline", True) + + + result = func(session_details, *args, **kwargs) + if session_details is not None: + if save_pipeline: + # we overwrite inside saver, if file exists and save_pipeline is True + self.pipe.file_checker(result, session_details, extra=extra) + else: + logger.warning( + f"Cannot guess data saving location for {self.pipe_name}: 'session_details' argument must be supplied." + ) + return result + + return wrap diff --git a/run_tests.py b/run_tests.py new file mode 100644 index 0000000000000000000000000000000000000000..3b8b94821e7ec4786c76eee9daa9fc45fa03a4e8 --- /dev/null +++ b/run_tests.py @@ -0,0 +1,15 @@ +import os + +# Executing "coverage run -m unittest discover" +print("Running tests with coverage...") +os.system("coverage run --source=pypelines -m unittest discover -s tests/") + +# Executing "coverage xml" +print("Generating XML report...") +os.system("coverage report") + +# Executing "pycobertura lcov --output report.lcov coverage.xml" +#print("Converting report to lcov format...") +#os.system("pycobertura lcov --output report.lcov coverage.xml") + +print("Done") \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/__pycache__/__init__.cpython-311.pyc b/tests/__pycache__/__init__.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..be96c6b190b406a5be0b54b97546fcd84cf2ba81 Binary files /dev/null and b/tests/__pycache__/__init__.cpython-311.pyc differ diff --git a/tests/__pycache__/tests.cpython-311.pyc b/tests/__pycache__/tests.cpython-311.pyc new file mode 100644 index 0000000000000000000000000000000000000000..443c11517a3b8b79ca849efedb82d20132fd6c2f Binary files /dev/null and b/tests/__pycache__/tests.cpython-311.pyc differ diff --git a/tests/instances.py b/tests/instances.py new file mode 100644 index 0000000000000000000000000000000000000000..c1aed34a6b5a3e179424b87aed8a7b31e5f82a42 --- /dev/null +++ b/tests/instances.py @@ -0,0 +1,13 @@ + +import pypelines + +pipeline_test_instance = pypelines.BasePipeline() + + + + + + +@pipeline_test_instance.register_pipe +class TestPipe(pypelines.BasePipe): + \ No newline at end of file diff --git a/tests/tests.py b/tests/tests.py new file mode 100644 index 0000000000000000000000000000000000000000..9a6103585fe14fc4e91817b0bfa14bfec5257919 --- /dev/null +++ b/tests/tests.py @@ -0,0 +1,21 @@ +import unittest, sys, os + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) + +import pypelines + +from . instances import pipeline_test_instance + +class TestVersions(unittest.TestCase): + + def setUp(self): + self.version_handler = pypelines.HashVersionHandler('version_example.py') + self.pipeline = pipeline_test_instance + + def test_function_hash(self): + self.assertEqual(self.version_handler.get_function_hash(), "ad2d1f4") + +if __name__ == '__main__': + unittest.main() + + diff --git a/pypelines/versions.json b/tests/versions_example.json similarity index 100% rename from pypelines/versions.json rename to tests/versions_example.json