diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 074ec2041e6ec6e9900274746712488c8e078ea1..97f916458c5d28a6b8ef56af05c1a96743a6b64d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -26,14 +26,14 @@ cache:
stages:
+ - build-df-cli
+ - deploy-meilisearch
- zotero
- get-meili-key
- build
# - build-wiki
- delete-release
- - build-df-cli
- lint
- - deploy-meilisearch
- update-meilisearch-indexes
- deploy
- post-deploy
@@ -236,6 +236,21 @@ lint:
update
--file data/list-systems.json
--document systems
+ - >
+ df-wiki-cli
+ meilisearch
+ --host ${MEILI_HOST}
+ --key ${MEILI_MASTER_KEY}
+ delete-all-documents article
+ - >
+ df-wiki-cli
+ meilisearch
+ --host ${MEILI_HOST}
+ --key ${MEILI_MASTER_KEY}
+ update
+ --file zot-articles.json
+ --document article
+
allow_failure: false
@@ -293,28 +308,17 @@ set-meili-env:prod:
- if: $CI_COMMIT_BRANCH == "main"
##############################
-get-zotero:
+sync-zotero:
extends: .df-wiki-cli-run
stage: zotero
script:
- - df-wiki-cli articles --key ${ZOTERO_API_KEY} --output content/_data/_articles.json
+ - df-wiki-cli articles missing-doi --dir ./content/ --key ${ZOTERO_API_KEY}
+ - df-wiki-cli articles fetch-from-zotero --key ${ZOTERO_API_KEY} --output zot-articles.json
artifacts:
paths:
- - content/_data/_articles.json
+ - zot-articles.json
rules:
- - if: $CI_COMMIT_BRANCH == "main"
-
-# get-pfam:
-# extends: .df-wiki-cli-run
-# stage: get-data
-# script:
-# - df-wiki-cli pfam --output public/pfam-a-hmm.csv
-# artifacts:
-# paths:
-# - public/pfam-a-hmm.csv
- # rules:
- # - if: $CI_COMMIT_BRANCH == "main"
-
+ - if: $CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH == "rework-references"
################ BUILD ##########################
@@ -359,7 +363,7 @@ build:prod:wiki:
extends: .build
needs:
- set-meili-env:prod
- - get-zotero
+ - sync-zotero
# - get-pfam
variables:
BASE_URL: /wiki/
@@ -405,7 +409,7 @@ build:prod:wiki:
deploy:dev:
extends: .deploy
rules:
- - if: $CI_COMMIT_BRANCH == "dev" || $CI_COMMIT_BRANCH == "refseq-no-sys"
+ - if: $CI_COMMIT_BRANCH == "dev" || $CI_COMMIT_BRANCH == "rework-references"
needs:
- "build:dev:wiki"
when: manual
@@ -471,7 +475,7 @@ delete-helm-release:prod:
-.post-deploy:
+.create-structures-archives:
stage: post-deploy
image: harbor.pasteur.fr/kube-system/helm-kubectl:$HELM_VERSION
when: manual
@@ -488,8 +492,8 @@ delete-helm-release:prod:
- kubectl --namespace ${KUBE_NAMESPACE} exec ${WIKI_POD} -- bash -c "cd /usr/share/nginx/html/ && find ./ -name '*.cif' -exec tar -czvf /usr/share/nginx/html/df-all-cifs.tar.gz {} +"
-post-deploy:dev:
- extends: .post-deploy
+create-structures-archives:dev:
+ extends: .create-structures-archives
rules:
- if: $CI_COMMIT_BRANCH != "main"
needs:
@@ -505,8 +509,8 @@ post-deploy:dev:
url: "https://${HOST_DEV}"
-post-deploy:prod:
- extends: .post-deploy
+create-structures-archives:prod:
+ extends: .create-structures-archives
rules:
- if: $CI_COMMIT_BRANCH == "main"
needs:
diff --git a/components/content/ArticleDoi.vue b/components/content/ArticleDoi.vue
index fa7e1b86ef58c6e14c2bcb72738255237108d015..271556aec9e788ac11bdf1d0debe114434461aac 100644
--- a/components/content/ArticleDoi.vue
+++ b/components/content/ArticleDoi.vue
@@ -1,6 +1,6 @@
<script setup lang="ts">
import { useDisplay } from "vuetify";
-import { useArticlesStore } from '@/stores/articles'
+import type { WikiArticle } from '@/types/articles';
export interface Props {
index?: number;
@@ -19,10 +19,20 @@ const props = withDefaults(defineProps<Props>(), {
isRelevant: false,
});
-const { article } = useFetchArticle(props.doi);
+const article = ref<WikiArticle | undefined>(undefined)
const { mobile } = useDisplay();
const show = ref(false);
+onBeforeMount(async () => {
+ const { article: articleOnMounted } = await useFetchArticle(props.doi);
+ article.value = articleOnMounted.value
+})
+
+
+onMounted(async () => {
+ const { article: articleOnMounted } = await useFetchArticle(props.doi);
+ article.value = articleOnMounted.value
+})
const articleTitle = computed(() => {
return props?.title ?? article?.value?.title ?? props.doi;
@@ -32,16 +42,11 @@ const articleAbstract = computed(() => {
});
</script>
<template>
- <v-list-item :href="article?.href" :id="`ref-${props.doi}`" :target="article?.target" density="compact" color="transparent"
- class="px-1">
+ <v-list-item :href="article?.href" :id="`ref-${props.doi}`" :target="article?.target" density="compact"
+ color="transparent" class="px-1">
<template v-if="!mobile" #prepend>
<v-icon icon="md:star" :color="props.isRelevant ? 'info' : 'transparent'"></v-icon>
</template>
- <!-- <template v-if="!mobile" #append>
- <v-btn v-if="articleAbstract" size="x-small" variant="plain"
- :append-icon="show ? 'mdi-chevron-up' : 'mdi-chevron-down'" class="px-0"
- @click.stop.prevent="show = !show">Abstract</v-btn>
- </template> -->
<v-card flat color="transparent" density="compact" class="my-0 article-ref">
<v-card-item density="compact" class="pa-0">
<v-toolbar class="py-0 d-flex align-start article-toolbar" color="transparent" :height="20">
@@ -51,9 +56,6 @@ const articleAbstract = computed(() => {
:append-icon="show ? 'mdi-chevron-up' : 'mdi-chevron-down'" class="px-1 align-center"
@click.stop.prevent="show = !show">Abstract</v-btn>
</v-toolbar>
- <!-- <v-card-title class="py-0"><span class="font-weight-bold">{{
- articleTitle
- }}</span></v-card-title> -->
<v-card-subtitle class="py-0">
{{ article?.subtitle ?? "no authors" }}</v-card-subtitle>
<v-card-subtitle class="py-0">
diff --git a/components/content/ArticleReference.vue b/components/content/ArticleReference.vue
deleted file mode 100644
index 4c18ab75841fb8a95fffc3e4339220678fd32aba..0000000000000000000000000000000000000000
--- a/components/content/ArticleReference.vue
+++ /dev/null
@@ -1,55 +0,0 @@
-<script setup lang="ts">
-import { useDisplay } from "vuetify";
-
-export interface Props {
- index?: number;
- doi: string;
- divider?: boolean;
- enumerate?: boolean;
- title?: string;
- abstract?: string;
-}
-
-const props = withDefaults(defineProps<Props>(), {
- enumerate: true,
- divider: false,
-});
-const { article } = useFetchArticle(props.doi);
-const { mobile } = useDisplay();
-
-const articleTitle = computed(() => {
- return props?.title ?? article?.value?.title ?? props.doi;
-});
-
-const articleAuthorsString = computed(() => {
- // console.log(article.value)
- // console.log(props.doi)
- return article.value?.author?.length > 0 ? `${article.value.author[0].family} & al` : null
-})
-
-</script>
-<template>
- <v-list-item v-if="article" :href="article?.href" :id="`ref-${props.doi}`" :target="article?.target" density="compact"
- class="px-1">
- <template #prepend v-if="!mobile && enumerate">
- <v-avatar color="primary" size="small" density="compact" variant="tonal">
- {{ props?.index ?? "#" }}
- </v-avatar>
- </template>
-
- <template #title>
- <span class="text-subtitle-1 font-weight-bold">{{
- articleTitle
- }} </span>
-
- </template>
- <template #subtitle><span class="text-caption">{{ articleAuthorsString }}</span></template>
- <template #append v-if="!mobile">
- <span> {{ article?.containerTitle ?? "no containerTitle" }} ({{
- article?.year
- }})</span>
- </template>
-
- </v-list-item>
- <v-divider v-if="props.divider" inset></v-divider>
-</template>
\ No newline at end of file
diff --git a/components/content/RefArticle.vue b/components/content/RefArticle.vue
index 4242f444d04dc76146a45053479c1d6d0180dd9c..ee9181ccb05925b5b1e53c2a7a80c70e9c218e87 100644
--- a/components/content/RefArticle.vue
+++ b/components/content/RefArticle.vue
@@ -1,12 +1,23 @@
<script setup lang="ts">
import { useTheme } from "vuetify";
+import type { WikiArticle } from '@/types/articles';
+
+
const theme = useTheme();
export interface Props {
doi: string;
}
const props = withDefaults(defineProps<Props>(), {});
-
-const { article } = useFetchArticle(props.doi);
+const article = ref<WikiArticle | undefined>(undefined)
+// const { article } = useFetchArticle(props.doi);
+onBeforeMount(async () => {
+ const { article: articleOnMounted } = await useFetchArticle(props.doi);
+ article.value = articleOnMounted.value
+})
+onMounted(async () => {
+ const { article: articleOnMounted } = await useFetchArticle(props.doi);
+ article.value = articleOnMounted.value
+})
</script>
<template>
<!-- <v-chip v-if="article" variant="text" :href="`#ref-${props.doi}`" class="pa-0 font-italic">{{
diff --git a/composables/useFetchArticle.ts b/composables/useFetchArticle.ts
index 59c1e51851cd53369ccd27d4db85c17b86508d28..eef82b70629bd8487f6c421e0537733cf7068628 100644
--- a/composables/useFetchArticle.ts
+++ b/composables/useFetchArticle.ts
@@ -1,60 +1,22 @@
-import { useArticlesStore, type CslJson } from '../stores/articles'
-import { ref, computed, watchEffect, toValue } from "vue"
-// import { useFetch } from '#app';
-// import { useFetch } from "nuxt"
-import { type MaybeRef, useFetch } from '#imports'
-import article from "@/public/articles.json"
+import { ref, toValue } from "vue"
+import { StorageSerializers } from "@vueuse/core"
+import type { CslJson, WikiArticle } from '@/types/articles';
+import type { SearchParams } from 'meilisearch'
-export interface CrossrefArticle {
- DOI: string;
- issue: number;
- type: string;
- title: string[];
- author: Array<{ family: string; given: string }>;
- // "container-title-short": string;
- "short-container-title": string;
- "container-title": string;
- abstract?: string;
- published: {
- "date-parts": string[];
- };
- issued: {
- "date-parts": string[];
- };
-}
-
-
-
-export interface WikiArticle {
- DOI: string
- title: string
- subtitle: string
- author: Array<{ family: string; given: string }>
- containerTitle: string
- abstract: string
- year: string
- href: string
- target: string
- prependIcon: string
-}
-export interface RawArticle {
- message: CrossrefArticle
-
-}
-
-type SrcArticle = CrossrefArticle | CslJson
-
-export function useFetchArticle(doi: MaybeRef<string> = ref("")) {
- // const article = ref<Article>()
- // const rawArticle = ref<RawArticle>()
- const srcArticle = ref<SrcArticle | null>(null)
- const store = useArticlesStore()
- const pending = ref(false)
+export async function useFetchArticle(doi: string = "") {
const doiBaseUrl = ref(new URL("https://doi.org/"));
- const url = ref(new URL(`/works/${toValue(doi)}`, " https://api.crossref.org/").href);
- const article = ref()
- const zoteroArticles = ref()
-
+ // const article = ref<WikiArticle | undefined>(undefined)
+ const article = useSessionStorage<WikiArticle>(doi, null, { serializer: StorageSerializers.object })
+ const client = useMeiliSearchRef()
+ const index = ref("article")
+ const params = ref<SearchParams>({
+ facets: ["*"],
+ filter: [`DOI='${doi}'`],
+ limit: 1
+ })
+ function getReferenceUrl(doi: string) {
+ return new URL(doi, doiBaseUrl.value).href;
+ }
function toAuthorsString(authors: Array<{ family: string; given: string }>) {
return authors
.map((curr) => {
@@ -62,12 +24,7 @@ export function useFetchArticle(doi: MaybeRef<string> = ref("")) {
})
.join(", ");
}
-
- function getReferenceUrl(doi: string) {
- return new URL(doi, doiBaseUrl.value).href;
- }
-
- function zoteroArticleToArticle(zoteroArticle: CslJson) {
+ function zoteroArticleToArticle(zoteroArticle: CslJson): WikiArticle | undefined {
if (zoteroArticle != undefined) {
const {
DOI,
@@ -90,74 +47,38 @@ export function useFetchArticle(doi: MaybeRef<string> = ref("")) {
target: "_blank",
prependIcon: "mdi-newspaper-variant-outline",
}
- }
+ } else { return undefined }
+ }
+ if (!article.value) {
+ const { data, error } = await useAsyncData(
+ doi,
+ async () => {
+ return await client.index(toValue(index)).search<CslJson>("", toValue(params))
+ },
+ {
+ transform: function (data) {
+ if (data !== undefined && data?.hits.length >= 1) {
+ return zoteroArticleToArticle(data.hits[0])
+ }
+ }
+ }
+ )
- }
- function crossrefToArticle(article: CrossrefArticle): WikiArticle {
- const { title, DOI, type, "container-title": ct, "short-container-title": sct, abstract, author, issued } = article
- // let sanitizedAbstract = abstract
- const sanitizedAbstract = abstract ? /(?:\<jats\:p\>)?(.*)(?:\<\/jats\:p\>)?/.exec(abstract)?.[1] ?? '' : ''
- const sanitizedContainerTitle = sct?.length > 0 ? sct[0] : ct?.length > 0 ? ct[0] : ""
- return {
- title: title?.length > 0 ? title[0] : "",
- DOI,
- abstract: sanitizedAbstract,
- containerTitle: sanitizedContainerTitle,
- subtitle: toAuthorsString(author || []),
- author,
- year: issued?.["date-parts"][0][0] ?? '',
- href: getReferenceUrl(DOI),
- target: "_blank",
- prependIcon: "mdi-newspaper-variant-outline"
+ if (error.value) {
+ throw createError({
+ ...error.value,
+ statusMessage: `Could not fetch article ${doi}`
+ })
}
+ article.value = data.value
}
+ return { article }
+}
- function crossrefToCsl(article: CrossrefArticle): CslJson {
- const { title, DOI, type, "container-title": ct, "short-container-title": sct, abstract, author, issued } = article
- // let sanitizedAbstract = abstract
- const sanitizedAbstract = abstract ? /(?:\<jats\:p\>)?(.*)(?:\<\/jats\:p\>)?/.exec(abstract)?.[1] ?? '' : ''
- const sanitizedContainerTitle = sct?.length > 0 ? sct[0] : ct?.length > 0 ? ct[0] : ""
- return {
- title: title?.length > 0 ? title[0] : "",
- type,
- DOI,
- abstract: sanitizedAbstract,
- author,
- "container-title": sanitizedContainerTitle,
- issued
- }
- }
- if (store.articles.has(toValue(doi))) {
- const cslArticle = store.articles.get(toValue(doi))
- article.value = cslArticle ? zoteroArticleToArticle(cslArticle) : undefined
- }
- else {
- useFetch<RawArticle>(toValue(url), {
- lazy: true, server: false,
- }).then(({ data, pending: pendingUseFetch }) => {
- if (data.value?.message) {
- article.value = crossrefToArticle(data.value.message)
- store.add(crossrefToCsl(data.value.message))
- }
- pending.value = pendingUseFetch.value
- })
- }
- // const fetchCrossRef = () => {
- // useFetch<RawArticle>(toValue(url), {
- // lazy: true, server: false,
- // }).then(({ data, pending: pendingUseFetch }) => {
- // if (data.value?.message) {
- // srcArticle.value = data.value.message
- // }
- // pending.value = pendingUseFetch.value
- // })
- // }
- return { article, pending }
-}
diff --git a/content/3.defense-systems/paris.md b/content/3.defense-systems/paris.md
index 7aaafe5058f11deef029c573d18a4118bfa2909d..a09b72fca555cde404bc032bccbf8a9fd83f9ce0 100644
--- a/content/3.defense-systems/paris.md
+++ b/content/3.defense-systems/paris.md
@@ -23,7 +23,7 @@ PARIS (for Phage Anti-Restriction-Induced System) is a novel anti-phage system.
## Molecular mechanisms
-This system relies on an unknown [Abortive infection](/general-concepts/abortive-infection) mechanism to trigger growth arrest upon sensing a phage-encoded protein (Ocr). Interestingly, the Ocr protein has been found to inhibit R-M systems and BREX systems, making PARIS a suitable defense mechanism against RM resistant and/or BREX resistant phages :ref{doi=0.1016/j.chom.2022.02.018,10.1093/nar/gkaa510,10.1007/BF01036001}.Â
+This system relies on an unknown [Abortive infection](/general-concepts/abortive-infection) mechanism to trigger growth arrest upon sensing a phage-encoded protein (Ocr). Interestingly, the Ocr protein has been found to inhibit R-M systems and BREX systems, making PARIS a suitable defense mechanism against RM resistant and/or BREX resistant phages :ref{doi=10.1016/j.chom.2022.02.018,10.1093/nar/gkaa510,10.1007/BF01036001}.Â
## Example of genomic structure
diff --git a/packages/df-wiki-cli/df_wiki_cli/articles/__init__.py b/packages/df-wiki-cli/df_wiki_cli/articles/__init__.py
index 123615c1548e058679701b7ced37c10c0e567b34..7e90a068d8307dd3dee85e7616351191b9df27ad 100644
--- a/packages/df-wiki-cli/df_wiki_cli/articles/__init__.py
+++ b/packages/df-wiki-cli/df_wiki_cli/articles/__init__.py
@@ -1,9 +1,15 @@
import json
from pyzotero import zotero
from pathlib import Path
+from habanero import Crossref
+from rich.console import Console
+console = Console()
-def fetch_articles(key: str, batch_size: int = 100, output: Path = "articles.json"):
+
+def fetch_articles_from_zotero(
+ key: str, batch_size: int = 100, output: Path = "articles.json"
+):
zot = zotero.Zotero("5151022", "group", key)
collection = zot.collection("BSWL96X3")
tot_items = collection["meta"]["numItems"]
@@ -16,7 +22,7 @@ def fetch_articles(key: str, batch_size: int = 100, output: Path = "articles.jso
format="csljson",
limit=batch_size,
start=i,
- itemType="journalArticle",
+ itemType="journalArticle || Preprint",
sort="title",
)["items"]
for item in items:
@@ -26,3 +32,118 @@ def fetch_articles(key: str, batch_size: int = 100, output: Path = "articles.jso
json_object = json.dumps(items, indent=2)
with open(output, "w") as outfile:
outfile.write(json_object)
+
+
+def add_articles_to_zotero_from_doi(doi, key):
+ cr = Crossref(mailto="defense-finder@pasteur.fr")
+ res = cr.works(ids=[doi])
+
+ zot = zotero.Zotero("5151022", "group", key)
+
+ # pyzotero grabs the template dict from the server
+
+ # console.print(zitem)
+ message = res["message"]
+ console.print(f"add doi {doi}, type {message['type']}")
+ if message["type"] == "posted-content":
+ # {
+ # "itemType": "Preprint",
+ # "title": "",
+ # "creators": [{"creatorType": "author", "firstName": "", "lastName": ""}],
+ # "abstractNote": "",
+ # "genre": "",
+ # "repository": "",
+ # "archiveID": "",
+ # "place": "",
+ # "date": "",
+ # "series": "",
+ # "seriesNumber": "",
+ # "DOI": "",
+ # "citationKey": "",
+ # "url": "",
+ # "accessDate": "",
+ # "archive": "",
+ # "archiveLocation": "",
+ # "shortTitle": "",
+ # "language": "",
+ # "libraryCatalog": "",
+ # "callNumber": "",
+ # "rights": "",
+ # "extra": "",
+ # "tags": [],
+ # "collections": [],
+ # "relations": {},
+ # }
+ itemtype = "Preprint"
+ elif message["type"] == "journal-article":
+ # {
+ # 'itemType': 'journalArticle',
+ # 'title': '',
+ # 'creators': [{'creatorType': 'author', 'firstName': '', 'lastName': ''}],
+ # 'abstractNote': '',
+ # 'publicationTitle': '',
+ # 'volume': '',
+ # 'issue': '',
+ # 'pages': '',
+ # 'date': '',
+ # 'series': '',
+ # 'seriesTitle': '',
+ # 'seriesText': '',
+ # 'journalAbbreviation': '',
+ # 'language': '',
+ # 'DOI': '',
+ # 'ISSN': '',
+ # 'shortTitle': '',
+ # 'url': '',
+ # 'accessDate': '',
+ # 'archive': '',
+ # 'archiveLocation': '',
+ # 'libraryCatalog': '',
+ # 'callNumber': '',
+ # 'rights': '',
+ # 'extra': '',
+ # 'tags': [],
+ # 'collections': [],
+ # 'relations': {}
+ # }
+ itemtype = "journalArticle"
+ else:
+ raise NotImplementedError(f"type {message['type']} need to be implemented")
+ zitem = zot.item_template(itemtype)
+ # console.print(message)
+
+ # console.print(message["title"])
+ zitem["title"] = message["title"][0]
+ if "page" in message:
+ zitem["pages"] = message["page"]
+ if "abstract" in message:
+ zitem["abstractNote"] = message["abstract"]
+
+ if "container-title" in message and len(message["container-title"]) > 0:
+ zitem["publicationTitle"] = message["container-title"][0]
+ if "short-container-title" in message and len(message["short-container-title"]) > 0:
+ zitem["journalAbbreviation"] = message["short-container-title"][0]
+ zitem["creators"] = [
+ {
+ "creatorType": "author",
+ "firstName": author["given"],
+ "lastName": author["family"],
+ }
+ for author in message["author"]
+ ]
+ zitem["libraryCatalog"] = "DOI.org (Crossref)"
+ if "ISSN" in message:
+ zitem["ISSN"] = ", ".join(message["ISSN"])
+ zitem["url"] = message["resource"]["primary"]["URL"]
+ zitem["date"] = "/".join([str(d) for d in message["published"]["date-parts"][0]])
+ for key in ["DOI", "volume", "issue", "language"]:
+ if key in message:
+ zitem[key] = message[key]
+
+ items_to_add = [zitem]
+
+ zot.check_items(items_to_add)
+ # res = zot.create_items(items_to_add)
+ # new_item = res["successful"]["0"]
+ # zot.addto_collection("BSWL96X3", new_item)
+ console.print("[green]done")
diff --git a/packages/df-wiki-cli/df_wiki_cli/articles/main.py b/packages/df-wiki-cli/df_wiki_cli/articles/main.py
new file mode 100644
index 0000000000000000000000000000000000000000..5c4bc2c89d36ee484596680d75325873225f091b
--- /dev/null
+++ b/packages/df-wiki-cli/df_wiki_cli/articles/main.py
@@ -0,0 +1,98 @@
+import typer
+from typing_extensions import Annotated
+from pathlib import Path
+from df_wiki_cli.articles import (
+ add_articles_to_zotero_from_doi,
+ fetch_articles_from_zotero,
+)
+from pyzotero import zotero
+from habanero import Crossref
+from rich.console import Console
+import frontmatter
+import re
+import json
+
+app = typer.Typer()
+console = Console()
+
+
+@app.command()
+def fetch_from_zotero(
+ output: Annotated[
+ Path,
+ typer.Option(
+ exists=False,
+ file_okay=True,
+ writable=True,
+ ),
+ ] = "articles.json",
+ key: Annotated[str, typer.Option(help="Zotero api key")] = "",
+ batch_size: Annotated[
+ int, typer.Option(help="Number articles get per request")
+ ] = 100,
+):
+ """
+ Get articles metadata from Zotero collection
+
+ """
+ if key != "":
+ fetch_articles_from_zotero(key, batch_size, output)
+ else:
+ print("You must provide a zotero api key")
+ raise typer.Exit(code=1)
+
+
+@app.command()
+def missing_doi(
+ dir: Annotated[
+ Path,
+ typer.Option(exists=False, file_okay=False, readable=True, dir_okay=True),
+ ],
+ key: Annotated[str, typer.Option(help="Zotero api key")] = "",
+ batch_size: Annotated[
+ int, typer.Option(help="Number articles get per request")
+ ] = 100,
+):
+ # parse content to look at dois
+
+ # get current list of dois in zotero
+ zotero_list = "/tmp/zotero-dois.json"
+ fetch_articles_from_zotero(key, batch_size, zotero_list)
+ zotero_dois_set = set()
+ with open(zotero_list) as zotero_f:
+ zotero_data = json.load(zotero_f)
+ for d in zotero_data:
+ if "DOI" in d:
+ doi_lower = d["DOI"].lower()
+ zotero_dois_set.add(doi_lower)
+
+ dois_set = set()
+ for file in dir.rglob("*"):
+ if file.suffix == ".md":
+ console.rule(f"[bold blue]{file.name}", style="blue")
+ with open(file) as f:
+ metadata, content = frontmatter.parse(f.read())
+ if (
+ "relevantAbstracts" in metadata
+ and len(metadata["relevantAbstracts"]) > 0
+ ):
+ dois = [d["doi"].lower() for d in metadata["relevantAbstracts"]]
+ dois_set.update(dois)
+
+ # handle content
+ group = re.findall(r":ref{doi=(.*?)}", content)
+ for g in group:
+ splitted = [doi.lower() for doi in re.split(",", g)]
+ dois_set.update(splitted)
+
+ for doi in dois_set - zotero_dois_set:
+ add_articles_to_zotero_from_doi(doi, key)
+
+
+@app.command()
+def from_apicrossref(
+ doi: Annotated[str, typer.Option(help="DOI identifier")],
+ key: Annotated[str, typer.Option(help="Zotero api key")] = "",
+):
+
+ add_articles_to_zotero_from_doi(doi, key)
diff --git a/packages/df-wiki-cli/df_wiki_cli/main.py b/packages/df-wiki-cli/df_wiki_cli/main.py
index ace7d2e92bbc20738e98c10a7a882fe23523a121..c6992782b6e983a434513bf10a08235d7ce0cc56 100644
--- a/packages/df-wiki-cli/df_wiki_cli/main.py
+++ b/packages/df-wiki-cli/df_wiki_cli/main.py
@@ -1,15 +1,16 @@
import typer
from pathlib import Path
from typing_extensions import Annotated
-from df_wiki_cli.articles import fetch_articles
from df_wiki_cli.pfam import fetch_pfam
from df_wiki_cli.meilisearch import main as ms_main
from df_wiki_cli.content import main as content_main
+from df_wiki_cli.articles import main as articles_main
# from df_wiki_cli.ms import main as ms_main
app = typer.Typer()
app.add_typer(ms_main.app, name="meilisearch")
app.add_typer(content_main.app, name="content")
+app.add_typer(articles_main.app, name="articles")
@app.callback()
@@ -21,33 +22,6 @@ def callback():
"""
-
-@app.command()
-def articles(
- output: Annotated[
- Path,
- typer.Option(
- exists=False,
- file_okay=True,
- writable=True,
- ),
- ] = "articles.json",
- key: Annotated[str, typer.Option(help="Zotero api key")] = "",
- batch_size: Annotated[
- int, typer.Option(help="Number articles get per request")
- ] = 100,
-):
- """
- Get articles metadata from Zotero collection
-
- """
- if key != "":
- fetch_articles(key, batch_size, output)
- else:
- print("You must provide a zotero api key")
- raise typer.Exit(code=1)
-
-
@app.command()
def pfam(
output: Annotated[
diff --git a/packages/df-wiki-cli/df_wiki_cli/meilisearch/__init__.py b/packages/df-wiki-cli/df_wiki_cli/meilisearch/__init__.py
index 1111225625f07cbc4584ae571646350cbe00a998..5707df46949c4fd609e35340458fb61e1a5c30aa 100644
--- a/packages/df-wiki-cli/df_wiki_cli/meilisearch/__init__.py
+++ b/packages/df-wiki-cli/df_wiki_cli/meilisearch/__init__.py
@@ -308,6 +308,41 @@ def update_systems(
index.update_typo_tolerance({"enabled": False})
+def update_articles(
+ host: str,
+ key: str,
+ file: Path,
+ document: str,
+):
+ client = meilisearch.Client(host, key)
+ index = client.index(document.lower())
+ with open(file, "r") as jsonfile:
+ json_object = json.load(jsonfile)
+ for obj in json_object:
+ obj["ms_id"] = obj["id"].replace("/", "_")
+ tasks = index.add_documents_in_batches(json_object, primary_key="ms_id")
+ for task in tasks:
+ print(task)
+
+ pagination_settings_task = index.update_pagination_settings(
+ {"maxTotalHits": 100000}
+ )
+ print(pagination_settings_task)
+ attr_task = index.update_filterable_attributes(
+ body=[
+ "DOI",
+ ]
+ )
+ params = {
+ "maxValuesPerFacet": 1000000,
+ "sortFacetValuesBy": {"*": "count"},
+ }
+ index.update_faceting_settings(params)
+
+ print(attr_task)
+
+
+
def split_on_comma(str_val: str) -> List[str]:
for val in str_val.split(","):
yield val.strip()
diff --git a/packages/df-wiki-cli/df_wiki_cli/meilisearch/main.py b/packages/df-wiki-cli/df_wiki_cli/meilisearch/main.py
index f32ac8c3af221457efe03522b7e0bdf206f0c610..e6cb0e7b9c38fe0874f3efa988810e8494633c9f 100644
--- a/packages/df-wiki-cli/df_wiki_cli/meilisearch/main.py
+++ b/packages/df-wiki-cli/df_wiki_cli/meilisearch/main.py
@@ -3,10 +3,11 @@ import meilisearch
from typing_extensions import Annotated
from pathlib import Path
from df_wiki_cli.meilisearch import (
+ update_refseqtaxo,
+ update_articles,
update_refseq,
update_structure,
update_systems,
- update_refseqtaxo,
)
from enum import Enum
from types import SimpleNamespace
@@ -23,6 +24,7 @@ class Documents(str, Enum):
refseq = "refseq"
structure = "structure"
systems = "systems"
+ article = "article"
@app.callback()
@@ -69,6 +71,8 @@ def update(
update_structure(ctx.obj.host, ctx.obj.key, file, document)
if document == "systems":
update_systems(ctx.obj.host, ctx.obj.key, file, document)
+ if document == "article":
+ update_articles(ctx.obj.host, ctx.obj.key, file, document)
@app.command()
diff --git a/packages/df-wiki-cli/poetry.lock b/packages/df-wiki-cli/poetry.lock
index 3b10a841198f28079546217a59f1b62eabd5f82a..8d3b6e0c6dc2126973b96b586c8c0ad2b8250d32 100644
--- a/packages/df-wiki-cli/poetry.lock
+++ b/packages/df-wiki-cli/poetry.lock
@@ -373,6 +373,26 @@ ufo = ["fs (>=2.2.0,<3)"]
unicode = ["unicodedata2 (>=15.1.0)"]
woff = ["brotli (>=1.0.1)", "brotlicffi (>=0.8.0)", "zopfli (>=0.1.4)"]
+[[package]]
+name = "habanero"
+version = "1.2.6"
+description = "Low Level Client for Crossref Search API"
+optional = false
+python-versions = ">=3.7"
+files = [
+ {file = "habanero-1.2.6-py2.py3-none-any.whl", hash = "sha256:ccc57e7dc9cc1850961b4e6f85fc7749896cba6ddcc06ea15297dbad9af2b62e"},
+ {file = "habanero-1.2.6.tar.gz", hash = "sha256:b206d49f44f41c2289f0ad731f259a50d4376c747d8ecbb219a73874d45309d4"},
+]
+
+[package.dependencies]
+requests = ">=2.7.0"
+tqdm = "*"
+
+[package.extras]
+all = ["pytest"]
+bibtex = ["bibtexparser (>=2.0.0)"]
+test = ["pytest"]
+
[[package]]
name = "idna"
version = "3.4"
@@ -1297,6 +1317,26 @@ files = [
{file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"},
]
+[[package]]
+name = "tqdm"
+version = "4.66.2"
+description = "Fast, Extensible Progress Meter"
+optional = false
+python-versions = ">=3.7"
+files = [
+ {file = "tqdm-4.66.2-py3-none-any.whl", hash = "sha256:1ee4f8a893eb9bef51c6e35730cebf234d5d0b6bd112b0271e10ed7c24a02bd9"},
+ {file = "tqdm-4.66.2.tar.gz", hash = "sha256:6cd52cdf0fef0e0f543299cfc96fec90d7b8a7e88745f411ec33eb44d5ed3531"},
+]
+
+[package.dependencies]
+colorama = {version = "*", markers = "platform_system == \"Windows\""}
+
+[package.extras]
+dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"]
+notebook = ["ipywidgets (>=6)"]
+slack = ["slack-sdk"]
+telegram = ["requests"]
+
[[package]]
name = "typer"
version = "0.9.0"
@@ -1378,4 +1418,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p
[metadata]
lock-version = "2.0"
python-versions = ">=3.11,<3.13"
-content-hash = "c424b174806be51ccfd8655e9f31c5ac7e1d8a7a362ba99f69746b2e71a59174"
+content-hash = "d526b2f2acfadf5a8eccfae5e902ab702eaedd8ca88ec310d2f6c4dab88bea8a"
diff --git a/packages/df-wiki-cli/pyproject.toml b/packages/df-wiki-cli/pyproject.toml
index 22f5ffe80cd5293c84c985efe6daf7e7e3c715e5..d0a3341443d8102a33b93a44f44f7b16797bb3ed 100644
--- a/packages/df-wiki-cli/pyproject.toml
+++ b/packages/df-wiki-cli/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "df-wiki-cli"
-version = "0.1.5"
+version = "0.1.6"
description = ""
authors = ["Remi PLANEL <rplanel@pasteur.fr>"]
readme = "README.md"
@@ -18,6 +18,7 @@ pydantic = "^2.4.2"
pydantic-yaml = "^1.2.0"
python-frontmatter = "^1.0.1"
matplotlib = "^3.8.2"
+habanero = "^1.2.6"
[tool.poetry.group.dev.dependencies]
diff --git a/stores/articles.ts b/stores/articles.ts
index 1ba95fd2d0584f76814bd6818b4ab794bf0f534a..a28c03bc198d9a37f77cf10e1831765f45fcfc05 100644
--- a/stores/articles.ts
+++ b/stores/articles.ts
@@ -1,24 +1,8 @@
import { defineStore } from 'pinia'
import { ref } from 'vue'
// import jsonArticles from '@/assets/articles.json'
+import type { CslJson } from "@/types/articles"
-export interface CslJson {
- // id: string
- type: string
- title: string
- "container-title": string
- // page: string,
- // volume: string,
- abstract: string
- // URL: string
- DOI: string
- // journalAbbreviation: string
- // language: string
- author: Array<{ family: string, given: string }>
- issued: {
- "date-parts": Array<string>
- },
-}
export const useArticlesStore = defineStore('articles', () => {
const articleMap = new Map([])
diff --git a/types/articles.ts b/types/articles.ts
new file mode 100644
index 0000000000000000000000000000000000000000..92f2f10debb602df6ceac169d3b7cc74b202ebfe
--- /dev/null
+++ b/types/articles.ts
@@ -0,0 +1,55 @@
+export interface WikiArticle {
+ DOI: string
+ title: string
+ subtitle: string
+ author: Array<{ family: string; given: string }>
+ containerTitle: string
+ abstract: string
+ year: string
+ href: string
+ target: string
+ prependIcon: string
+}
+
+export interface CslJson {
+ // id: string
+ type: string
+ title: string
+ "container-title": string
+ // page: string,
+ // volume: string,
+ abstract: string
+ // URL: string
+ DOI: string
+ // journalAbbreviation: string
+ // language: string
+ author: Array<{ family: string, given: string }>
+ issued: {
+ "date-parts": Array<string>
+ },
+}
+
+export interface CrossrefArticle {
+ DOI: string;
+ issue: number;
+ type: string;
+ title: string[];
+ author: Array<{ family: string; given: string }>;
+ // "container-title-short": string;
+ "short-container-title": string;
+ "container-title": string;
+ abstract?: string;
+ published: {
+ "date-parts": string[];
+ };
+ issued: {
+ "date-parts": string[];
+ };
+}
+
+export interface RawArticle {
+ message: CrossrefArticle
+
+}
+
+export type SrcArticle = CrossrefArticle | CslJson