From d962b4a585cfb4a08e46b40427f6bbf6088a5e42 Mon Sep 17 00:00:00 2001
From: Remi PLANEL <rplanel@pasteur.fr>
Date: Thu, 11 Jan 2024 17:35:18 +0100
Subject: [PATCH] update refseq
---
.gitattributes | 1 +
.gitlab-ci.yml | 9 +++++++++
.../df-wiki-cli/df_wiki_cli/content/main.py | 3 ---
.../df_wiki_cli/meilisearch/__init__.py | 10 ++++++++--
.../df_wiki_cli/meilisearch/main.py | 19 +++++++++++++++++++
packages/df-wiki-cli/pyproject.toml | 2 +-
6 files changed, 38 insertions(+), 6 deletions(-)
diff --git a/.gitattributes b/.gitattributes
index 40ca7edd..ccb58707 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,2 +1,3 @@
data/ filter=lfs diff=lfs merge=lfs -text
data/**/*.csv filter=lfs diff=lfs merge=lfs -text
+data/**/*.tsv filter=lfs diff=lfs merge=lfs -text
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3eb89e35..60d46c3f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -203,6 +203,15 @@ lint:
--dir content/3.defense-systems/
--pfam public/pfam-a-hmm.csv
--output data/list-systems.json
+ - df-wiki-cli meilisearch delete-all-documents refseq
+ - >
+ df-wiki-cli
+ meilisearch
+ --host ${MEILI_HOST}
+ --key ${MEILI_MASTER_KEY}
+ index-update
+ refseq
+ sys_id
- >
df-wiki-cli
meilisearch
diff --git a/packages/df-wiki-cli/df_wiki_cli/content/main.py b/packages/df-wiki-cli/df_wiki_cli/content/main.py
index e516f383..ee09705c 100644
--- a/packages/df-wiki-cli/df_wiki_cli/content/main.py
+++ b/packages/df-wiki-cli/df_wiki_cli/content/main.py
@@ -264,7 +264,6 @@ def refseq(
),
],
):
-
with open(output, "w") as out, open(input, "r") as refseq_f:
reader = csv.DictReader(refseq_f)
fieldnames = reader.fieldnames
@@ -275,5 +274,3 @@ def refseq(
console.print(f"[green]{row['sys_id']} -> {result}")
row["sys_id"] = result
writer.writerow(row)
-
-
\ No newline at end of file
diff --git a/packages/df-wiki-cli/df_wiki_cli/meilisearch/__init__.py b/packages/df-wiki-cli/df_wiki_cli/meilisearch/__init__.py
index d60b9ceb..3479787c 100644
--- a/packages/df-wiki-cli/df_wiki_cli/meilisearch/__init__.py
+++ b/packages/df-wiki-cli/df_wiki_cli/meilisearch/__init__.py
@@ -5,10 +5,14 @@ import json
from typing import Annotated, List, Optional
from pydantic import BaseModel, Field, BeforeValidator
from enum import Enum
+from rich.console import Console
+
+
+console = Console()
class RefSeqCsv(BaseModel):
- id: str
+ sys_id: str
replicon: str
type: str
subtype: str
@@ -86,7 +90,9 @@ def update_refseq(
row["accession_in_sys"] = split_on_comma(row["accession_in_sys"])
doc = RefSeqCsv(**row)
documents.append(doc.model_dump(by_alias=True))
- index.add_documents_in_batches(documents, primary_key="id")
+ tasks = index.add_documents_in_batches(documents, primary_key="sys_id")
+ for task in tasks:
+ console.print(task)
index.update_pagination_settings({"maxTotalHits": 1000000})
index.update_filterable_attributes(
body=[
diff --git a/packages/df-wiki-cli/df_wiki_cli/meilisearch/main.py b/packages/df-wiki-cli/df_wiki_cli/meilisearch/main.py
index 858e189d..e7e30377 100644
--- a/packages/df-wiki-cli/df_wiki_cli/meilisearch/main.py
+++ b/packages/df-wiki-cli/df_wiki_cli/meilisearch/main.py
@@ -5,6 +5,10 @@ from pathlib import Path
from df_wiki_cli.meilisearch import update_refseq, update_structure, update_systems
from enum import Enum
from types import SimpleNamespace
+from rich.console import Console
+
+
+console = Console()
app = typer.Typer()
@@ -66,6 +70,21 @@ def delete_all_documents(ctx: typer.Context, id: str):
index.delete_all_documents()
+@app.command()
+def indexes(ctx: typer.Context):
+ client = meilisearch.Client(ctx.obj.host, ctx.obj.key)
+ indexes = client.get_indexes()["results"]
+ for index in indexes:
+ print(index.uid)
+
+
+@app.command()
+def index_update(ctx: typer.Context, index: str, primary_key: str):
+ client = meilisearch.Client(ctx.obj.host, ctx.obj.key)
+ task = client.index(index).update(primary_key=primary_key)
+ console.print(task)
+
+
@app.command()
def task(ctx: typer.Context, id: str):
client = meilisearch.Client(ctx.obj.host, ctx.obj.key)
diff --git a/packages/df-wiki-cli/pyproject.toml b/packages/df-wiki-cli/pyproject.toml
index 538eb535..22f5ffe8 100644
--- a/packages/df-wiki-cli/pyproject.toml
+++ b/packages/df-wiki-cli/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "df-wiki-cli"
-version = "0.1.4"
+version = "0.1.5"
description = ""
authors = ["Remi PLANEL <rplanel@pasteur.fr>"]
readme = "README.md"
--
GitLab