| 123456789 |
- # import tiktoken
- # from typing import Union
- # def tokenize(text: str, model: str = 'gpt-3.5-turbo') -> Union[int, str]:
- # encoding = tiktoken.encoding_for_model(model)
- # encoded = encoding.encode(text)
- # num_tokens = len(encoded)
-
- # return num_tokens, encoded
|
