added comments

AK3847 · AK3847 · commit bd84220012c0 · 2024-01-18T23:59:11.000+05:30
diff --git a/tokens.py b/tokens.py
@@ -1,4 +1,5 @@
 from utils import *
+#user-defined function to convert length of a code file into tokens
 def tokencount(content):
     encoding=tiktoken.get_encoding("cl100k_base")
     token_num=len(encoding.encode(content))

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`from utils import *`
	`2`	`+#user-defined function to convert length of a code file into tokens`
`2`	`3`	`def tokencount(content):`
`3`	`4`	`encoding=tiktoken.get_encoding("cl100k_base")`
`4`	`5`	`token_num=len(encoding.encode(content))`