[tokenizer] Update import script for huggingface_hub api change (#2850)

deepjavalibrary · Apr 26, 2024 · 13df6ea · 13df6ea
1 parent 052fc3f
commit 13df6ea
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/extensions/tokenizers/src/main/python/huggingface_models.py b/extensions/tokenizers/src/main/python/huggingface_models.py
@@ -16,7 +16,7 @@
 from argparse import Namespace
 from typing import List
 
-from huggingface_hub import HfApi, ModelSearchArguments
+from huggingface_hub import HfApi
 from huggingface_hub import hf_hub_download
 from huggingface_hub.hf_api import ModelInfo
 
@@ -27,7 +27,7 @@
     "ForMultipleChoice": "text-classification",
     "ForMaskedLM": "fill-mask",
 }
-LANGUAGES = ModelSearchArguments().language
+LANGUAGES = HfApi().get_model_tags()["language"]
 
 
 def get_lang_tags(model_info):

diff --git a/extensions/tokenizers/src/main/python/requirements.txt b/extensions/tokenizers/src/main/python/requirements.txt
@@ -1,4 +1,4 @@
 huggingface_hub
 transformers
-torch==1.11.0
+torch
 protobuf==3.20.2