update refseq

.gitattributes

 data/ filter=lfs diff=lfs merge=lfs -text
 data/**/*.csv filter=lfs diff=lfs merge=lfs -text
+data/**/*.tsv filter=lfs diff=lfs merge=lfs -text

.gitlab-ci.yml

@@ -203,6 +203,15 @@ lint:
       --dir content/3.defense-systems/ 
       --pfam public/pfam-a-hmm.csv 
       --output data/list-systems.json
+    - df-wiki-cli meilisearch delete-all-documents refseq  
+    - >
+      df-wiki-cli
+      meilisearch 
+      --host ${MEILI_HOST} 
+      --key ${MEILI_MASTER_KEY}
+      index-update 
+      refseq 
+      sys_id
     - >
       df-wiki-cli
       meilisearch 

packages/df-wiki-cli/df_wiki_cli/content/main.py

@@ -264,7 +264,6 @@ def refseq(
         ),
     ],
 ):
-    
     with open(output, "w") as out, open(input, "r") as refseq_f:
         reader = csv.DictReader(refseq_f)
         fieldnames = reader.fieldnames
@@ -275,5 +274,3 @@ def refseq(
             console.print(f"[green]{row['sys_id']} ->  {result}")
             row["sys_id"] = result
             writer.writerow(row)
-
-        
\ No newline at end of file

packages/df-wiki-cli/df_wiki_cli/meilisearch/__init__.py

@@ -5,10 +5,14 @@ import json
 from typing import Annotated, List, Optional
 from pydantic import BaseModel, Field, BeforeValidator
 from enum import Enum
+from rich.console import Console
+
+
+console = Console()
 
 
 class RefSeqCsv(BaseModel):
-    id: str
+    sys_id: str
     replicon: str
     type: str
     subtype: str
@@ -86,7 +90,9 @@ def update_refseq(
             row["accession_in_sys"] = split_on_comma(row["accession_in_sys"])
             doc = RefSeqCsv(**row)
             documents.append(doc.model_dump(by_alias=True))
-        index.add_documents_in_batches(documents, primary_key="id")
+        tasks = index.add_documents_in_batches(documents, primary_key="sys_id")
+        for task in tasks:
+            console.print(task)
     index.update_pagination_settings({"maxTotalHits": 1000000})
     index.update_filterable_attributes(
         body=[

packages/df-wiki-cli/df_wiki_cli/meilisearch/main.py

@@ -5,6 +5,10 @@ from pathlib import Path
 from df_wiki_cli.meilisearch import update_refseq, update_structure, update_systems
 from enum import Enum
 from types import SimpleNamespace
+from rich.console import Console
+
+
+console = Console()
 
 app = typer.Typer()
 
@@ -66,6 +70,21 @@ def delete_all_documents(ctx: typer.Context, id: str):
     index.delete_all_documents()
 
 
+@app.command()
+def indexes(ctx: typer.Context):
+    client = meilisearch.Client(ctx.obj.host, ctx.obj.key)
+    indexes = client.get_indexes()["results"]
+    for index in indexes:
+        print(index.uid)
+
+
+@app.command()
+def index_update(ctx: typer.Context, index: str, primary_key: str):
+    client = meilisearch.Client(ctx.obj.host, ctx.obj.key)
+    task = client.index(index).update(primary_key=primary_key)
+    console.print(task)
+
+
 @app.command()
 def task(ctx: typer.Context, id: str):
     client = meilisearch.Client(ctx.obj.host, ctx.obj.key)

packages/df-wiki-cli/pyproject.toml

 [tool.poetry]
 name = "df-wiki-cli"
-version = "0.1.4"
+version = "0.1.5"
 description = ""
 authors = ["Remi  PLANEL <rplanel@pasteur.fr>"]
 readme = "README.md"