diff --git a/requirements.txt b/requirements.txt index 424d8dd9d27ac75ba5938a2ed7c881a83090ec23..738f099f61cdc6a016aa07cb3f49ff275c98d1aa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,5 @@ PyYAML>=6.0.1 PyYAML>=6.0.1 Requests>=2.31.0 tqdm>=4.66.1 +pint +rich diff --git a/src/one/__init__.py b/src/one/__init__.py index 62bffa1d3ad0ba5cae25570e3103f652ef73fba5..02426acad24fd8a937eb1ac3e1a8552761555bc0 100644 --- a/src/one/__init__.py +++ b/src/one/__init__.py @@ -1,5 +1,5 @@ """The Open Neurophysiology Environment (ONE) API""" -__version__ = "2.1.14" +__version__ = "2.1.15" from . import api from .api import ONE diff --git a/src/one/api.py b/src/one/api.py index e77d0d9022b425f33070d9f7c5765a4b525f8e13..4a372d447f1ede2cf4f7c6786972c94f6c57e834 100644 --- a/src/one/api.py +++ b/src/one/api.py @@ -68,26 +68,31 @@ class MultiSessionPlaceholder(pd.core.series.Series): *args, project="Adaptation", analysis_group="default", - data_path="", + remote_path="", data_repository=None, + mode="remote", + date="2000-01-01", **kwargs, ): super().__init__(*args, **kwargs) if data_repository is not None: if data_repository == "local": - data_path = one.params.get().LOCAL_ROOT + remote_path = one.params.get().LOCAL_ROOT else: - data_path = self._get_connector().alyx.rest("data-repository", "read", data_repository)["data_path"] - if data_path == "": + remote_path = self._get_connector().alyx.rest("data-repository", "read", data_repository)["data_path"] + + if remote_path == "": raise ValueError( "Data path cannot be empty string. Must either be obtained by supplying data_repository argument, " "or data_path directly" ) - data_path = os.path.normpath(data_path) + remote_path = os.path.normpath(remote_path) self["rel_path"] = os.path.join("multisession", analysis_group) - self["path"] = os.path.join(data_path, self["rel_path"]) + self["local_path"] = os.path.join(os.path.normpath(one.params.get().LOCAL_ROOT), self["rel_path"]) + self["remote_path"] = os.path.join(remote_path, self["rel_path"]) + self["path"] = self["remote_path"] if mode == "remote" else self["local_path"] self["alias"] = analysis_group self["u_alias"] = analysis_group self["projects"] = [project] diff --git a/src/one/files.py b/src/one/files.py index e42dba09dc654ab750971f31f49b6bc542930e35..bef8501cb99d594eb6754472df5944c5e2772b0f 100644 --- a/src/one/files.py +++ b/src/one/files.py @@ -226,7 +226,14 @@ class FileTransferManager: transfers_infos = [] for (source, destination), transfers in self.results.groupby(["source_volume", "destination_volume"]): free_space = shutil.disk_usage(destination).free * Quantity("bytes") - transfer_space = transfers["source_filesize"].sum() + + transfered_files = transfers[transfers["decision"] == "transfer"] + overwritten_files = transfers[transfers["decision"] == "overwrite"] + + transfer_space = ( + overwritten_files["destination_filesize"] - overwritten_files["source_filesize"] + ).sum() + transfered_files["source_filesize"].sum() + session_nb = len(transfers.session.unique()) files_nb = len(transfers) @@ -431,6 +438,8 @@ class FileTransferManager: destination_creation_date = destination_stat.st_birthtime destination_modification_date = destination_stat.st_mtime + destination_filesize = destination_stat.st_size * Quantity("bytes") + destination_date = max(destination_creation_date, destination_modification_date) if (difference := abs(source_date - destination_date)) < policies["close_dates_threshold"]: @@ -457,11 +466,13 @@ class FileTransferManager: decision = policies["no_file_exists"] warnings = "File is not existing on the destination, transfering is absolutely okay." destination_date = None + destination_filesize = 0 * Quantity("bytes") record = dict( source_filepath=source_filepath, destination_filepath=destination_filepath, source_filesize=source_filesize, + destination_filesize=destination_filesize, relative_filepath=relative_filepath, destination_exists=destination_exists, source_date=source_date, diff --git a/src/one/pd_accessors.py b/src/one/pd_accessors.py index 249bf25d728819832807988c8ba9306f9710a296..506bece9132a79e061128dbd476279f2b37f808c 100644 --- a/src/one/pd_accessors.py +++ b/src/one/pd_accessors.py @@ -8,12 +8,15 @@ from pathlib import Path class AlyxDataframeAcessorsRegistry: def __init__(self, pandas_obj) -> None: self.pandas_obj = pandas_obj - self._validate(self.pandas_obj) @property def datasets(self): return DatasetsDataframeAcessor(self.pandas_obj) + @property + def sessions(self): + return SessionsDataframeAccessor(self.pandas_obj) + @pd.api.extensions.register_series_accessor("alyx") class AlyxSeriesAcessorsRegistry: @@ -32,12 +35,44 @@ class AlyxSeriesAcessorsRegistry: def files(self): return FilesSeriesAccessor(self.pandas_obj) + @property + def session(self): + return SessionSeriesAccessor(self.pandas_obj) + class PlotSeriesAcessor: def __init__(self, pandas_obj) -> None: self.pandas_obj = pandas_obj +class SessionsDataframeAccessor: + + def __init__(self, pandas_obj) -> None: + self._obj: pd.DataFrame = pandas_obj + + def local_mode(self): + return self._obj.assign(path=self._obj["local_path"]) + + def remote_mode(self): + return self._obj.assign(path=self._obj["remote_path"]) + + +class SessionSeriesAccessor: + + def __init__(self, pandas_obj) -> None: + self._obj: pd.Series = pandas_obj + + def local_mode(self): + series = self._obj.copy() + series["path"] = series["local_path"] + return series + + def remote_mode(self): + series = self._obj.copy() + series["path"] = series["remote_path"] + return series + + class FilesSeriesAccessor: def __init__(self, pandas_obj) -> None: