require tiktoken 0.8.0

markhofm · markhofm · commit c9642a050a07 · 2025-02-02T21:58:50.000+01:00
diff --git a/pyproject.toml b/pyproject.toml
@@ -42,7 +42,7 @@ dependencies = [
     "accelerate>=0.33.0",
     "sentencepiece>=0.2.0",
     "sentence-transformers>=2.5.1",
-    "tiktoken>=0.7.0",
+    "tiktoken>=0.8.0",
     "tika>=2.6.0",
     "beautifulsoup4>=4.12.3",
     "colorama>=0.4.6",
