Token Chunking

from praisonaiagents import Agent

agent = Agent(
    instructions="Answer questions from documents.",
    knowledge={
        "sources": ["document.pdf"],
        "chunker": {
            "type": "token",
            "chunk_size": 256,
            "chunk_overlap": 50
        }
    }
)

response = agent.start("Summarize the main points")

from praisonaiagents.knowledge.chunking import Chunking

chunker = Chunking(
    chunker_type="token",
    chunk_size=256,
    chunk_overlap=50,
    tokenizer_or_token_counter="gpt2"
)

chunks = chunker.chunk("Your document text here...")
for chunk in chunks:
    print(f"Tokens: {chunk.token_count}")
    print(chunk.text[:100])