Skip to content
Snippets Groups Projects
Select Git revision
  • main default protected
  • dev protected
  • tclabby-main-patch-92350
  • tclabby-main-patch-26246
  • operon-struct-type
  • operon-struct-type-article
  • remove-duplicate-structure-Lamassu-Fam_PDDEXK
  • operon-struct-type-article-update
  • system-distribution-plot
  • update-article-auto-sections
  • genome-context-system
  • select-columns
  • mao
  • ftesson_abip2
  • ftesson_abia
  • ftesson_abij
  • ftesson_abiz
  • ftesson_abie
  • ftesson_abir
  • ftesson_abii
20 results
Blame History Permalink
.gitlab-ci.yml 14.16 KiB 
workflow:
  rules:
  - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    when: never
  - when: always

# Functions that should be executed before the build script is run

variables:
  HELM_VERSION: "3.13.3"
  IMAGE_NAME: "df-wiki"
  # dev
  HOST_DEV: 'defense-finder.dev.pasteur.cloud'
  MEILI_HOST_DEV: 'defense-finder-meilisearch.dev.pasteur.cloud'
  # prod
  HOST_PROD: 'defense-finder.pasteur.cloud'
  MEILI_HOST_PROD: 'defense-finder-meilisearch.pasteur.cloud'
  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"



cache:
  paths:
    - node_modules/
    - .output/public
    

stages:
  - build-df-cli
  - zotero
  - get-meili-key
  - build
  # - build-wiki
  - delete-release
  - lint
  - deploy-meilisearch
  - update-meilisearch-indexes
  - deploy
  - post-deploy

.docker-login: &docker-login
  - i=0; while [ "$i" -lt 12 ]; do docker info && break; sleep 5; i=$(( i + 1 )) ; done
  - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY



# Build df-wiki-cli package


.df-wiki-cli-run:
  image: python:3.11-bullseye
  cache:                      # Pip's cache doesn't store the python packages
    paths:                    # https://pip.pypa.io/en/stable/topics/caching/
      - .cache/pip
  before_script:
    - pip install df-wiki-cli --index-url https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.pasteur.fr/api/v4/projects/5222/packages/pypi/simple

build:df-wiki-cli:
  image: python:3.11-bullseye
  stage: build-df-cli
  before_script:
    - cd packages/df-wiki-cli/
    - pip install poetry
    - poetry install
    - source `poetry env info --path`/bin/activate

  script:
    - echo "This is the build stage"
    - poetry config repositories.gitlab "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi"
    - poetry config http-basic.gitlab gitlab-ci-token "$CI_JOB_TOKEN"
    - echo "Repository gitlab configured ..."
    - poetry build
    - echo "Build done ..."
    - poetry publish --repository gitlab --skip-existing
    - echo "Publishing done!"
  when: manual
  allow_failure: true
################ DEPLOY MEILISEARCH #################

.deploy:meilisearch:
  stage: deploy-meilisearch
  image: harbor.pasteur.fr/kube-system/helm-kubectl:$HELM_VERSION
  variables:
    CI_DEBUG_TRACE: "false"
    ENV: development
  before_script:
    - helm dependency update deploy/meilisearch/ 
  script: 
    - >
      helm upgrade --install 
      ${CI_PROJECT_NAME}-${CI_ENVIRONMENT_NAME}-meilisearch 
      ./deploy/meilisearch 
      --namespace=${KUBE_NAMESPACE}
      --set meilisearch.ingress.hosts[0]=${PUBLIC_URL}
      --set meilisearch.environment.MEILI_MASTER_KEY=${MEILI_MASTER_KEY}
      --set meilisearch.environment.MEILI_ENV=${ENV:-development}
      --set env=${ENV:-development}
      --values deploy/meilisearch/values.yaml
      --values deploy/meilisearch/values.${ENV:-development}.yaml
    # wait for it to start
    # - MEILI_POD=$(kubectl -n=${KUBE_NAMESPACE} get po -l app.kubernetes.io\/instance=${CI_PROJECT_NAME}-${CI_ENVIRONMENT_NAME}-meilisearch,app.kubernetes.io\/name=meilisearch --output jsonpath='{.items[0].metadata.name}')
    # - |
    #   until kubectl -n=${KUBE_NAMESPACE} wait --for=condition=ready pod ${MEILI_POD} --timeout=1s
    #   do
    #     date
    #     sleep 1
    #     kubectl -n=${KUBE_NAMESPACE} get po
    #   done
  when: manual


deploy:meilisearch:dev:
  extends: .deploy:meilisearch
  rules:
    - if: $CI_COMMIT_BRANCH != "main"
  variables:
    CI_DEBUG_TRACE: "true"
    ENV: "development"
    PUBLIC_URL: "$MEILI_HOST_DEV"
    KUBE_NAMESPACE: "defense-finder-dev"

  environment:
      name: k8sdev-01
      url: "https://${MEILI_HOST_DEV}"


deploy:meilisearch:prod:
  extends: .deploy:meilisearch
  rules:
    - if: $CI_COMMIT_BRANCH == "main"
  variables:
    CI_DEBUG_TRACE: "false"
    ENV: "production"
    PUBLIC_URL: $MEILI_HOST_PROD
    KUBE_NAMESPACE: "defense-finder-prod"
  environment:
      name: k8sprod-02
      url: "https://${MEILI_HOST_PROD}"


############### DELETE RELEASE

delete-meili-helm-release:
  rules:
    - if: $CI_COMMIT_BRANCH != "main"
  stage: delete-release
  when: manual
  image: harbor.pasteur.fr/kube-system/helm-kubectl:$HELM_VERSION
  variables:
    GIT_STRATEGY: none # important to not checkout source when branch is deleted
    NAMESPACE: "defense-finder-dev"
  environment:
    name: "k8sdev-01"
    action: stop
  script:
    - echo "Removing $CI_PROJECT_NAME-$CI_ENVIRONMENT_NAME-meilisearch"
    - helm delete -n ${NAMESPACE} ${CI_PROJECT_NAME}-${CI_ENVIRONMENT_NAME}-meilisearch 


delete-meili-helm-release:prod:
  rules:
    - if: $CI_COMMIT_BRANCH == "main"
  stage: delete-release
  when: manual
  image: harbor.pasteur.fr/kube-system/helm-kubectl:$HELM_VERSION
  variables:
    GIT_STRATEGY: none # important to not checkout source when branch is deleted
    NAMESPACE: "defense-finder-prod"
  environment:
    name: "k8sprod-02"
    action: stop
  script:
    - echo "Removing $CI_PROJECT_NAME-$CI_ENVIRONMENT_NAME-meilisearch"
    - helm delete -n ${NAMESPACE} ${CI_PROJECT_NAME}-${CI_ENVIRONMENT_NAME}-meilisearch 


# lint

lint:
  extends: .df-wiki-cli-run
  stage: lint
  script:
    - cd content/3.defense-systems
    - find . -name '*.md' ! -name '0.index.md' | sort | xargs -I {} df-wiki-cli content lint --file {}
  when: manual

# Update Meili search indexes + generate some file that goes to meilisearch
.update-meilisearch-index:
  extends: .df-wiki-cli-run
  stage: update-meilisearch-indexes
  variables:
    MEILI_HOST: "http://localhost:7700"
  script:
    # - rm data/list-systems.json
    - >
      df-wiki-cli
      meilisearch 
      --host ${MEILI_HOST} 
      --key ${MEILI_MASTER_KEY}
      delete-all-documents refseq
    - >
      df-wiki-cli 
      content systems 
      --dir content/3.defense-systems/ 
      --pfam public/pfam-a-hmm.csv 
      --output data/list-systems.json
    - >
      df-wiki-cli
      meilisearch 
      --host ${MEILI_HOST} 
      --key ${MEILI_MASTER_KEY}
      update
      --file data/refseq_res.csv
      --document refseq
    - >
      df-wiki-cli
      meilisearch 
      --host ${MEILI_HOST} 
      --key ${MEILI_MASTER_KEY}
      update
      --file data/refseq_res.csv
      --document refseqtaxo
    - >
      df-wiki-cli
      meilisearch 
      --host ${MEILI_HOST} 
      --key ${MEILI_MASTER_KEY}
      update
      --file data/all_predictions_statistics_clean.csv
      --document structure
    - >
      df-wiki-cli
      meilisearch 
      --host ${MEILI_HOST} 
      --key ${MEILI_MASTER_KEY}
      update
      --file data/list-systems.json
      --document systems
    - > 
      df-wiki-cli
      meilisearch 
      --host ${MEILI_HOST} 
      --key ${MEILI_MASTER_KEY}
      delete-all-documents article
    - >
      df-wiki-cli
      meilisearch 
      --host ${MEILI_HOST} 
      --key ${MEILI_MASTER_KEY}
      update
      --file zot-articles.json
      --document article



  allow_failure: false
  when: manual


update-meilisearch-index:dev:
  rules:
    - if: $CI_COMMIT_BRANCH != "main"
  extends: .update-meilisearch-index
  variables:
    MEILI_HOST: "https://${MEILI_HOST_DEV}"


update-meilisearch-index:prod:
  rules:
    - if: $CI_COMMIT_BRANCH == "main"
  extends: .update-meilisearch-index
  variables:
    MEILI_HOST: "https://${MEILI_HOST_PROD}"

############# get-meili-key ###############

.set-meili-env:
  extends: .df-wiki-cli-run
  stage: get-meili-key
  variables:
    MEILI_HOST: "http://localhost:7700"
  script:
    - > 
      df-wiki-cli 
      meilisearch 
      --key "${MEILI_MASTER_KEY}" 
      --host ${MEILI_HOST} 
      get-env-var
      --output build.env
  artifacts:
      reports:
        dotenv: build.env
  allow_failure: false


set-meili-env:dev:
  extends: .set-meili-env
  variables:
    MEILI_HOST: "https://${MEILI_HOST_DEV}"
  rules:
    - if: $CI_COMMIT_BRANCH != "main"  

set-meili-env:prod:
  extends: .set-meili-env
  variables:
    MEILI_HOST: "https://${MEILI_HOST_PROD}"
  rules:
    - if: $CI_COMMIT_BRANCH == "main"  

##############################
sync-zotero:
  extends: .df-wiki-cli-run
  stage: zotero
  script:
    - df-wiki-cli articles missing-doi --dir ./content/ --key ${ZOTERO_API_KEY}
    - df-wiki-cli articles fetch-from-zotero --key ${ZOTERO_API_KEY} --output zot-articles.json
  artifacts:
    paths:
      - zot-articles.json
  rules:
    - if: $CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH == "rework-references"


################ BUILD ##########################

.build:
  stage: build
  image: docker:24
  variables:
    CONTEXT: "."
    DOCKERFILE: "Dockerfile"
    BASE_URL: /
    MEILI_HOST: "http://localhost:7700"
  before_script:
    - *docker-login
  script:
    - >
      docker buildx build --pull -t "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHORT_SHA" 
      --build-arg "BASE_URL=$BASE_URL" 
      --build-arg "MEILI_HOST=$MEILI_HOST" 
      --build-arg "MEILI_API_KEY=$MEILI_API_KEY"
      -f $DOCKERFILE $CONTEXT
    - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHORT_SHA"



build:dev:wiki:
  extends: .build
  needs:
    - set-meili-env:dev
    # - get-pfam
  variables:
    BASE_URL: /wiki/
    HOST_URL: https://${HOST_DEV}
  before_script:
    - *docker-login
    # - "sed -i 's/MEILISEARCH_API_KEY/${$MEILI_API_KEY}/g' nuxt.config.ts"
  rules:
    - if: $CI_COMMIT_BRANCH != "main"  


build:prod:wiki:
  extends: .build
  needs:
    - set-meili-env:prod
    - sync-zotero
    # - get-pfam
  variables:
    BASE_URL: /wiki/
    HOST_URL: https://${HOST_PROD}

  rules:
    - if: $CI_COMMIT_BRANCH == "main"  




################ DEPLOY ##########################
.deploy:
  stage: deploy
  image: harbor.pasteur.fr/kube-system/helm-kubectl:$HELM_VERSION
  variables:
    CI_DEBUG_TRACE: "false"
    TEAM_ID: "df"
  script:
    - >
      helm upgrade --install $CI_PROJECT_NAME-$CI_ENVIRONMENT_NAME ./deploy/df-wiki --namespace=${KUBE_NAMESPACE}
      --set registry.image=${CI_REGISTRY_IMAGE}
      --set registry.username=${DEPLOY_USER}
      --set registry.password=${DEPLOY_TOKEN}
      --set registry.host=${CI_REGISTRY}
      --set ingress.hosts[0].host="${PUBLIC_URL}"
      --set imagePullSecrets[0].name="registry-pull-secret-${CI_COMMIT_REF_SLUG}"
      --set image.repository="$CI_REGISTRY_IMAGE/$IMAGE_NAME"
      --set image.tag="$CI_COMMIT_SHORT_SHA"
      --set env="${ENV:-development}"
      --values deploy/df-wiki/values.yaml
      --values deploy/df-wiki/values.${ENV:-development}.yaml
  after_script:
    - sleep 30
    - kubectl --namespace ${KUBE_NAMESPACE} wait pod -l "app.kubernetes.io/name=df-wiki" --for condition=Ready --timeout=600s 
    - echo "Wiki pod is ready"
    - WIKI_POD=$(kubectl --namespace ${KUBE_NAMESPACE} get pods -l "app.kubernetes.io/name=df-wiki" --output jsonpath='{.items[0].metadata.name}')
    - echo ${WIKI_POD}
    - kubectl --namespace ${KUBE_NAMESPACE} exec ${WIKI_POD} -- rsync -avz /public-website/ /usr/share/nginx/html/
    
    

deploy:dev:
  extends: .deploy
  rules:
    - if: $CI_COMMIT_BRANCH == "dev" || $CI_COMMIT_BRANCH == "rework-references"
  needs:
    - "build:dev:wiki"
  when: manual
  variables:
    NODE_ENV: "development"
    KUBE_NAMESPACE: "defense-finder-dev"
    PUBLIC_URL: "${HOST_DEV}"
    CI_DEBUG_TRACE: "true"
    ENV: "development"
  environment:
    name: k8sdev-01
    url: "https://${HOST_DEV}"


deploy:prod:
  extends: .deploy
  rules:
    - if: $CI_COMMIT_BRANCH == "main"
  needs:
    - "build:prod:wiki"
  when: manual
  variables:
    NODE_ENV: "production"
    KUBE_NAMESPACE: "defense-finder-prod"
    PUBLIC_URL: "${HOST_PROD}"
    CI_DEBUG_TRACE: "false"
    ENV: "production"
  environment:
    name: k8sprod-02
    url: "https://${HOST_PROD}"

delete-helm-release:dev:
  rules:
    - if: $CI_COMMIT_BRANCH != "main"
  stage: delete-release
  when: manual
  image: harbor.pasteur.fr/kube-system/helm-kubectl:$HELM_VERSION
  variables:
    GIT_STRATEGY: none # important to not checkout source when branch is deleted
    NAMESPACE: "defense-finder-dev"
  environment:
    name: "k8sdev-01"
    action: stop
  script:
    - echo "Removing $CI_PROJECT_NAME-$CI_ENVIRONMENT_NAME"
    - helm delete -n ${NAMESPACE} $CI_PROJECT_NAME-$CI_ENVIRONMENT_NAME

delete-helm-release:prod:
  rules:
    - if: $CI_COMMIT_BRANCH == "main"
  stage: delete-release
  when: manual
  image: harbor.pasteur.fr/kube-system/helm-kubectl:$HELM_VERSION
  variables:
    GIT_STRATEGY: none # important to not checkout source when branch is deleted
    NAMESPACE: "defense-finder-prod"
  environment:
    name: "k8sprod-02"
    action: stop
  script:
    - echo "Removing $CI_PROJECT_NAME-$CI_ENVIRONMENT_NAME"
    - helm delete -n ${NAMESPACE} $CI_PROJECT_NAME-$CI_ENVIRONMENT_NAME



.create-structures-archives:
  stage: post-deploy
  image: harbor.pasteur.fr/kube-system/helm-kubectl:$HELM_VERSION
  when: manual
  variables:
    CI_DEBUG_TRACE: "false"
    TEAM_ID: "df"
  script:
    - kubectl --namespace ${KUBE_NAMESPACE} wait pod -l "app.kubernetes.io/name=df-wiki" --for condition=Ready --timeout=600s 
    - echo "Wiki pod is ready"
    - WIKI_POD=$(kubectl --namespace ${KUBE_NAMESPACE} get pods -l "app.kubernetes.io/name=df-wiki" --output jsonpath='{.items[0].metadata.name}')
    - echo ${WIKI_POD}
    # - kubectl --namespace ${KUBE_NAMESPACE} exec ${WIKI_POD} -- rsync -avz /public-website/ /usr/share/nginx/html/
    - kubectl --namespace ${KUBE_NAMESPACE} exec ${WIKI_POD} -- bash -c "cd /usr/share/nginx/html/ && find ./ -name '*.pdb' -exec tar -czvf /usr/share/nginx/html/df-all-pdbs.tar.gz {} +"
    - kubectl --namespace ${KUBE_NAMESPACE} exec ${WIKI_POD} -- bash -c "cd /usr/share/nginx/html/ && find ./ -name '*.cif' -exec tar -czvf /usr/share/nginx/html/df-all-cifs.tar.gz {} +"


create-structures-archives:dev:
  extends: .create-structures-archives
  rules:
    - if: $CI_COMMIT_BRANCH != "main"  
  needs:
    - "deploy:dev"
  variables:
    NODE_ENV: "development"
    KUBE_NAMESPACE: "defense-finder-dev"
    PUBLIC_URL: "${HOST_DEV}"
    CI_DEBUG_TRACE: "true"
    ENV: "development"
  environment:
    name: k8sdev-01
    url: "https://${HOST_DEV}"


create-structures-archives:prod:
  extends: .create-structures-archives
  rules:
    - if: $CI_COMMIT_BRANCH == "main"  
  needs:
    - "deploy:prod"
  variables:
    NODE_ENV: "production"
    KUBE_NAMESPACE: "defense-finder-prod"
    PUBLIC_URL: "${HOST_PROD}"
    CI_DEBUG_TRACE: "false"
    ENV: "production"
  environment:
    name: k8sprod-02
    url: "https://${HOST_PROD}"