Fix jieba bug (#163)

aigc-apps · Aug 23, 2024 · 312bac4 · 312bac4
1 parent 49961e4
commit 312bac4
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/pai_rag/utils/tokenizer.py b/src/pai_rag/utils/tokenizer.py
@@ -14,7 +14,7 @@
 ## PUT in utils file and add stopword in TRIE structure.
 def jieba_tokenizer(text: str) -> List[str]:
     tokens = []
-    for w in jieba.lcut(text):
+    for w in jieba.cut(text):
         token = w.lower()
         if not stop_trie.match(token):
             tokens.append(token)