diff --git a/.gitattributes b/.gitattributes index 40ca7edde186cf7747ca08fac2fde1c08246d9bb..ccb58707bf8ac8d5397e3891c621cbdf553be24c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ data/ filter=lfs diff=lfs merge=lfs -text data/**/*.csv filter=lfs diff=lfs merge=lfs -text +data/**/*.tsv filter=lfs diff=lfs merge=lfs -text diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3eb89e358441cf5cadbfa8cff786f7bb0fa251ab..60d46c3f0110a7dff9533cb1881d431f689b888a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -203,6 +203,15 @@ lint: --dir content/3.defense-systems/ --pfam public/pfam-a-hmm.csv --output data/list-systems.json + - df-wiki-cli meilisearch delete-all-documents refseq + - > + df-wiki-cli + meilisearch + --host ${MEILI_HOST} + --key ${MEILI_MASTER_KEY} + index-update + refseq + sys_id - > df-wiki-cli meilisearch diff --git a/packages/df-wiki-cli/df_wiki_cli/content/main.py b/packages/df-wiki-cli/df_wiki_cli/content/main.py index e516f383e563160386cfe3b495a8096ce347b532..ee09705cf6a667bf650c0267c240a94f86dcedff 100644 --- a/packages/df-wiki-cli/df_wiki_cli/content/main.py +++ b/packages/df-wiki-cli/df_wiki_cli/content/main.py @@ -264,7 +264,6 @@ def refseq( ), ], ): - with open(output, "w") as out, open(input, "r") as refseq_f: reader = csv.DictReader(refseq_f) fieldnames = reader.fieldnames @@ -275,5 +274,3 @@ def refseq( console.print(f"[green]{row['sys_id']} -> {result}") row["sys_id"] = result writer.writerow(row) - - \ No newline at end of file diff --git a/packages/df-wiki-cli/df_wiki_cli/meilisearch/__init__.py b/packages/df-wiki-cli/df_wiki_cli/meilisearch/__init__.py index d60b9ceb6e387e402981f78f1b95e0821d290e7a..3479787cd2b5187332b02065e26547b9b5854ccd 100644 --- a/packages/df-wiki-cli/df_wiki_cli/meilisearch/__init__.py +++ b/packages/df-wiki-cli/df_wiki_cli/meilisearch/__init__.py @@ -5,10 +5,14 @@ import json from typing import Annotated, List, Optional from pydantic import BaseModel, Field, BeforeValidator from enum import Enum +from rich.console import Console + + +console = Console() class RefSeqCsv(BaseModel): - id: str + sys_id: str replicon: str type: str subtype: str @@ -86,7 +90,9 @@ def update_refseq( row["accession_in_sys"] = split_on_comma(row["accession_in_sys"]) doc = RefSeqCsv(**row) documents.append(doc.model_dump(by_alias=True)) - index.add_documents_in_batches(documents, primary_key="id") + tasks = index.add_documents_in_batches(documents, primary_key="sys_id") + for task in tasks: + console.print(task) index.update_pagination_settings({"maxTotalHits": 1000000}) index.update_filterable_attributes( body=[ diff --git a/packages/df-wiki-cli/df_wiki_cli/meilisearch/main.py b/packages/df-wiki-cli/df_wiki_cli/meilisearch/main.py index 858e189d803cc336741dc29fbd0128657cfd1cdc..e7e3037766ce72d268bfdae830e4e52de3d34b03 100644 --- a/packages/df-wiki-cli/df_wiki_cli/meilisearch/main.py +++ b/packages/df-wiki-cli/df_wiki_cli/meilisearch/main.py @@ -5,6 +5,10 @@ from pathlib import Path from df_wiki_cli.meilisearch import update_refseq, update_structure, update_systems from enum import Enum from types import SimpleNamespace +from rich.console import Console + + +console = Console() app = typer.Typer() @@ -66,6 +70,21 @@ def delete_all_documents(ctx: typer.Context, id: str): index.delete_all_documents() +@app.command() +def indexes(ctx: typer.Context): + client = meilisearch.Client(ctx.obj.host, ctx.obj.key) + indexes = client.get_indexes()["results"] + for index in indexes: + print(index.uid) + + +@app.command() +def index_update(ctx: typer.Context, index: str, primary_key: str): + client = meilisearch.Client(ctx.obj.host, ctx.obj.key) + task = client.index(index).update(primary_key=primary_key) + console.print(task) + + @app.command() def task(ctx: typer.Context, id: str): client = meilisearch.Client(ctx.obj.host, ctx.obj.key) diff --git a/packages/df-wiki-cli/pyproject.toml b/packages/df-wiki-cli/pyproject.toml index 538eb535c3348574c06cd5ebdaf073d7bfd119dd..22f5ffe80cd5293c84c985efe6daf7e7e3c715e5 100644 --- a/packages/df-wiki-cli/pyproject.toml +++ b/packages/df-wiki-cli/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "df-wiki-cli" -version = "0.1.4" +version = "0.1.5" description = "" authors = ["Remi PLANEL <rplanel@pasteur.fr>"] readme = "README.md"