Support download tokenizer automatically.

Author: quic-zhanweiw

samples/genie/python/GenieAPIClient.py

@@ -18,6 +18,7 @@
 parser = argparse.ArgumentParser()
 parser.add_argument("--stream", action="store_true")
 parser.add_argument("--prompt", default="你好", type=str)
+parser.add_argument("--model", default="IBM-Granite-v3.1-8B", type=str)
 args = parser.parse_args()
 
 client = OpenAI(base_url="http://" + HOST + ":" + PORT + "/v1", api_key="123")
@@ -28,8 +29,7 @@
 messages = [{"role": "system", "content": "You are a math teacher who teaches algebra."}, {"role": "user", "content": args.prompt}]
 extra_body = {"size": 4096, "seed": 146, "temp": 1.5, "top_k": 13, "top_p": 0.6, "penalty_last_n": 64, "penalty_repeat": 1.3}
 
-model_name = "IBM-Granite-v3.1-8B"
-# model_name = "Phi-3.5-mini"
+model_name = args.model
 
 if args.stream:
     response = client.chat.completions.create(model=model_name, stream=True, messages=messages, extra_body=extra_body)

samples/genie/python/GenieAPIService.py

@@ -20,6 +20,7 @@
 
 sys.path.append("python")
 
+import utils.install as install
 import stable_diffusion_v2_1.stable_diffusion_v2_1 as stable_diffusion
 
 sys.stdin.reconfigure(encoding='utf-8')
@@ -429,13 +430,37 @@ def shutdown():
 
 app.add_api_route('/shutdown', shutdown, methods=['GET'])
 
+def download():
+    PATH_Tokenizer = APP_PATH + "\\models\\IBM-Granite-v3.1-8B\\tokenizer.json"
+    URL_Tokenizer = "https://gitee.com/hf-models/granite-3.1-8b-base/raw/main/tokenizer.json"
+
+    if not os.path.exists(PATH_Tokenizer):
+        install.download_url(URL_Tokenizer, PATH_Tokenizer)
+
+    PATH_Tokenizer = APP_PATH + "\\models\\Phi-3.5-mini\\tokenizer.json"
+    URL_Tokenizer = "https://gitee.com/hf-models/Phi-3.5-mini-instruct/raw/main/tokenizer.json"
+
+    if not os.path.exists(PATH_Tokenizer):
+        install.download_url(URL_Tokenizer, PATH_Tokenizer)
+
+        import re
+        with open(PATH_Tokenizer, 'r', encoding='utf-8') as f:
+            content = f.read()
+        pattern = r',\s*{\s*"type":\s*"Strip",\s*"content":\s*"\s*",\s*"start":\s*\d+,\s*"stop":\s*\d+\s*}'
+        new_content = re.sub(pattern, '', content)
+        with open(PATH_Tokenizer, 'w', encoding='utf-8') as f:
+            f.write(new_content)
+
+
 def model_load(model_name):
     global llm, _current_model, _model_list, _max_query_times
     find_model = False
 
     if model_name.lower() in _current_model.lower():
         return False
 
+    download()
+    
     if _model_list is None:
         model_root = APP_PATH + "models"
         _model_list = [f for f in os.listdir(model_root) if os.path.isdir(os.path.join(model_root, f))]

samples/python/utils/install.py

@@ -107,9 +107,6 @@ def verify_package(url, filepath, filesize, desc=None, fail=None):
         else:
             is_continue = True
 
-    print("If the download speed is too slow, you can download them from one of below URLs and install it manually before you run this 'setup.py'.")
-    print("https://softwarecenter.qualcomm.com/#/catalog/item/Qualcomm_AI_Runtime_SDK")
-    print("https://qpm.qualcomm.com/#/main/tools/details/Qualcomm_AI_Runtime_SDK")
     print()
 
     if is_continue:
@@ -404,6 +401,11 @@ def install_qai_sdk(version):
                f"If the downloading speed is too slow, please download it manually from {QNN_DOWNLOAD_URL} and install it. " + TEXT_RUN_SCRIPT_AGAIN
         fail = f"Failed to download file from {url}: \n\t1. Please try again a few times.\n\t2. If still doesn't work, please try to download it manually"\
                f" from {QNN_DOWNLOAD_URL} and install it. " + TEXT_RUN_SCRIPT_AGAIN
+
+        print("If the download speed is too slow, you can download them from one of below URLs and install it manually before you run this 'setup.py'.")
+        print("https://softwarecenter.qualcomm.com/#/catalog/item/Qualcomm_AI_Runtime_SDK")
+        print("https://qpm.qualcomm.com/#/main/tools/details/Qualcomm_AI_Runtime_SDK")
+
         ret = download_url(url, qnn_zip_path, desc=desc, fail=fail)
         # install_clean(QNN_SDK_ROOT, zip_name)
         if not ret: