From f81a5d25c749d70a2e08da74c31bcb49dcb63bee Mon Sep 17 00:00:00 2001
From: Henry Chen <1474479+chenhunghan@users.noreply.github.com>
Date: Mon, 7 Aug 2023 22:11:47 +0300
Subject: [PATCH] Fixes for falcon model(s) (#40)
Signed-off-by: Hung-Han (Henry) Chen <chenhungh@gmail.com>
---
examples/values/falcon-40b.yaml | 2 +-
examples/values/falcon-7b.yaml | 4 +++-
get_llm.py | 5 ++++-
3 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/examples/values/falcon-40b.yaml b/examples/values/falcon-40b.yaml
index 8fed4f1..1c309d3 100644
--- a/examples/values/falcon-40b.yaml
+++ b/examples/values/falcon-40b.yaml
@@ -3,7 +3,7 @@ deployment:
image: quay.io/chenhunghan/ialacol:latest
env:
DEFAULT_MODEL_HG_REPO_ID: TheBloke/WizardLM-Uncensored-Falcon-40B-GGML
- DEFAULT_MODEL_FILE: wizard-falcon40b.ggmlv3.q4_K_S.bin
+ DEFAULT_MODEL_FILE: wizardlm-uncensored-falcon-40b.ggccv1.q4_0.bin
resources:
{}
cache:
diff --git a/examples/values/falcon-7b.yaml b/examples/values/falcon-7b.yaml
index 1b59128..d7b8d48 100644
--- a/examples/values/falcon-7b.yaml
+++ b/examples/values/falcon-7b.yaml
@@ -3,7 +3,9 @@ deployment:
image: quay.io/chenhunghan/ialacol:latest
env:
DEFAULT_MODEL_HG_REPO_ID: TheBloke/WizardLM-Uncensored-Falcon-7B-GGML
- DEFAULT_MODEL_FILE: wizard-falcon-7b.ggmlv3.q4_1.bin
+ DEFAULT_MODEL_FILE: wizardlm-7b-uncensored.ggccv1.q4_0.bin
+ # the file name of the model file is missing `falcon` so we need to set the model type
+ MODE_TYPE: falcon
resources:
{}
cache:
diff --git a/get_llm.py b/get_llm.py
index eef6bd5..f112c98 100644
--- a/get_llm.py
+++ b/get_llm.py
@@ -38,7 +38,10 @@ async def get_llm(
ctransformer_model_type = "dolly-v2"
if "stablelm" in body.model:
ctransformer_model_type = "gpt_neox"
-
+
+ MODE_TYPE = get_env("MODE_TYPE", "")
+ if len(MODE_TYPE) > 0:
+ ctransformer_model_type = MODE_TYPE
MODELS_FOLDER = get_env("MODELS_FOLDER", "models")
return AutoModelForCausalLM.from_pretrained(
--
GitLab