Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
P
Pypelines
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Container registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
HaissLab
Data Management
Pypelines
Commits
5da6ca14
Commit
5da6ca14
authored
1 year ago
by
Timothe Jost
Browse files
Options
Downloads
Patches
Plain Diff
better type hinting
parent
487ab746
No related branches found
No related tags found
No related merge requests found
Pipeline
#124145
passed
1 year ago
Stage: test
Changes
4
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
src/pypelines/__init__.py
+1
-1
1 addition, 1 deletion
src/pypelines/__init__.py
src/pypelines/pipelines.py
+4
-3
4 additions, 3 deletions
src/pypelines/pipelines.py
src/pypelines/pipes.py
+17
-13
17 additions, 13 deletions
src/pypelines/pipes.py
src/pypelines/steps.py
+1
-0
1 addition, 0 deletions
src/pypelines/steps.py
with
23 additions
and
17 deletions
src/pypelines/__init__.py
+
1
−
1
View file @
5da6ca14
__version__
=
"
0.0.2
1
"
__version__
=
"
0.0.2
2
"
from
.
import
loggs
from
.pipes
import
*
...
...
This diff is collapsed.
Click to expand it.
src/pypelines/pipelines.py
+
4
−
3
View file @
5da6ca14
from
typing
import
Callable
,
Type
,
Iterable
,
Protocol
,
TYPE_CHECKING
from
typing
import
Callable
,
Type
,
Dict
,
Iterable
,
Protocol
,
TYPE_CHECKING
import
os
...
...
@@ -10,6 +10,7 @@ if TYPE_CHECKING:
class
Pipeline
:
use_celery
=
False
pipes
:
Dict
[
str
,
BasePipe
]
def
__init__
(
self
,
name
:
str
,
conf_path
=
None
,
use_celery
=
False
):
self
.
pipeline_name
=
name
...
...
@@ -20,7 +21,7 @@ class Pipeline:
if
use_celery
:
self
.
configure_celery
()
def
register_pipe
(
self
,
pipe_class
:
"
BasePipe
"
)
->
"
BasePipe
"
:
def
register_pipe
(
self
,
pipe_class
:
Type
[
"
BasePipe
"
]
)
->
Type
[
"
BasePipe
"
]
:
"""
Wrapper to instanciate and attache a a class inheriting from BasePipe it to the Pipeline instance.
The Wraper returns the class without changing it.
"""
instance
=
pipe_class
(
self
)
...
...
@@ -43,7 +44,7 @@ class Pipeline:
self
.
resolved
=
False
return
pipe_class
def
resolve_instance
(
self
,
instance_name
:
str
)
->
"
BaseStep
"
:
def
resolve_instance
(
self
,
instance_name
:
str
)
->
Type
[
"
BaseStep
"
]
:
pipe_name
,
step_name
=
instance_name
.
split
(
"
.
"
)
try
:
pipe
=
self
.
pipes
[
pipe_name
]
...
...
This diff is collapsed.
Click to expand it.
src/pypelines/pipes.py
+
17
−
13
View file @
5da6ca14
...
...
@@ -8,7 +8,8 @@ import inspect, hashlib
from
abc
import
ABCMeta
,
abstractmethod
from
typing
import
Callable
,
Type
,
Iterable
,
Protocol
,
TYPE_CHECKING
,
Literal
from
typing
import
Callable
,
Type
,
Iterable
,
Protocol
,
TYPE_CHECKING
,
Literal
,
Dict
from
types
import
MethodType
if
TYPE_CHECKING
:
from
.pipelines
import
Pipeline
...
...
@@ -21,38 +22,40 @@ class BasePipe(metaclass=ABCMeta):
single_step
:
bool
=
False
# a flag to tell the initializer to bind the unique step of this pipe in place
# of the pipe itself, to the registered pipeline.
step_class
:
BaseStep
=
BaseStep
disk_class
:
BaseDiskObject
=
BaseDiskObject
multisession_class
:
BaseMultisessionAccessor
=
BaseMultisessionAccessor
step_class
:
Type
[
BaseStep
]
=
BaseStep
disk_class
:
Type
[
BaseDiskObject
]
=
BaseDiskObject
multisession_class
:
Type
[
BaseMultisessionAccessor
]
=
BaseMultisessionAccessor
steps
:
Dict
[
str
,
BaseStep
]
def
__init__
(
self
,
parent_pipeline
:
"
Pipeline
"
)
->
None
:
self
.
pipeline
=
parent_pipeline
self
.
pipe_name
=
self
.
__class__
.
__name__
self
.
steps
=
{}
_steps
:
Dict
[
str
,
MethodType
]
=
{}
# this loop populates self.steps dictionnary from the instanciated (bound) step methods.
for
step_name
,
step
in
inspect
.
getmembers
(
self
,
predicate
=
inspect
.
ismethod
):
if
getattr
(
step
,
"
is_step
"
,
False
):
self
.
steps
[
step_name
]
=
step
_
steps
[
step_name
]
=
step
if
len
(
self
.
steps
)
<
1
:
if
len
(
_
steps
)
<
1
:
raise
ValueError
(
f
"
You should register at least one step class with @stepmethod in
{
self
.
pipe_name
}
class.
"
f
"
{
self
.
steps
=
}
"
f
"
{
_
steps
=
}
"
)
if
len
(
self
.
steps
)
>
1
and
self
.
single_step
:
if
len
(
_
steps
)
>
1
and
self
.
single_step
:
raise
ValueError
(
f
"
Cannot set single_step to True if you registered more than one step inside
{
self
.
pipe_name
}
class.
"
f
"
{
self
.
steps
=
}
"
f
"
{
_
steps
=
}
"
)
number_of_steps_with_requirements
=
0
for
step
in
self
.
steps
.
values
():
for
step
in
_
steps
.
values
():
if
len
(
step
.
requires
):
number_of_steps_with_requirements
+=
1
if
number_of_steps_with_requirements
<
len
(
self
.
steps
)
-
1
:
if
number_of_steps_with_requirements
<
len
(
_
steps
)
-
1
:
raise
ValueError
(
"
Steps of a single pipe must be linked in hierarchical order : Cannot have a single pipe with N steps
"
"
(N>1) and have no `requires` specification for at least N-1 steps.
"
...
...
@@ -60,7 +63,8 @@ class BasePipe(metaclass=ABCMeta):
# this loop populates self.steps and replacs the bound methods with usefull Step objects.
# They must inherit from BaseStep
for
step_name
,
step
in
self
.
steps
.
items
():
self
.
steps
=
{}
for
step_name
,
step
in
_steps
.
items
():
step
=
self
.
step_class
(
self
.
pipeline
,
self
,
step
)
# , step_name)
self
.
steps
[
step_name
]
=
step
# replace the bound_method by a step_class using that bound method,
# so that we attach the necessary components to it.
...
...
This diff is collapsed.
Click to expand it.
src/pypelines/steps.py
+
1
−
0
View file @
5da6ca14
...
...
@@ -53,6 +53,7 @@ def stepmethod(requires=[], version=None, do_dispatch=True, on_save_callbacks=[]
class
BaseStep
:
def
__init__
(
self
,
pipeline
:
"
Pipeline
"
,
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment