immediately return initialed tiktokenizer instance and remove dead code in usage of tiktokenizer (#17957)

2025-04-30 16:07:20 +08:00
parent bcc95e520b
commit 12c96b93d9
5 changed files with 18 additions and 73 deletions
--- a/api/pyproject.toml
+++ b/api/pyproject.toml
@@ -77,7 +77,7 @@ dependencies = [
    "sentry-sdk[flask]~=1.44.1",
    "sqlalchemy~=2.0.29",
    "starlette==0.41.0",
-    "tiktoken~=0.8.0",
+    "tiktoken~=0.9.0",
    "tokenizers~=0.15.0",
    "transformers~=4.35.0",
    "unstructured[docx,epub,md,ppt,pptx]~=0.16.1",