class CodeChunker(BaseChunker):
8192
)"cl100k_base"
)def __init__(
self,
chunk_size: int = 8192,
model_name: str = 'cl100k_base',
remove_image: Optional[bool] = True
):
def count_tokens(self, text: str):
def _split_oversized(self, line: str):
def chunk(self, content: List[str]):