Editing Long Context and Memory in LLMs (section)

== <span style="color: #FFFFFF;">Applying</span> ==
'''Long-context document QA with sliding summary memory:'''
<syntaxhighlight lang="python">
from openai import OpenAI
from anthropic import Anthropic
import tiktoken

client = OpenAI()
enc = tiktoken.encoding_for_model("gpt-4o")

def chunk_document(text: str, chunk_size: int = 8000, overlap: int = 500) -> list[str]:
    """Split document into overlapping chunks."""
    tokens = enc.encode(text)
    chunks = []
    for i in range(0, len(tokens), chunk_size - overlap):
        chunk_tokens = tokens[i:i + chunk_size]
        chunks.append(enc.decode(chunk_tokens))
    return chunks

def answer_with_long_context(document: str, question: str) -> str:
    chunks = chunk_document(document)
    # Map phase: extract relevant info from each chunk
    relevant_excerpts = []
    for chunk in chunks:
        response = client.chat.completions.create(
            model="gpt-4o-mini",
            messages=[{"role":"user", "content": f"From the following text, extract any information relevant to: '{question}'\nText: {chunk}\nIf nothing relevant, respond 'NONE'."}]
        )
        excerpt = response.choices[0].message.content
        if excerpt != "NONE":
            relevant_excerpts.append(excerpt)
    # Reduce phase: synthesize extracted excerpts into final answer
    combined = "\n\n".join(relevant_excerpts)
    final = client.chat.completions.create(
        model="gpt-4o",
        messages=[{"role":"user", "content": f"Based on these excerpts, answer: {question}\n\nExcerpts:\n{combined}"}]
    )
    return final.choices[0].message.content
</syntaxhighlight>

; Long context strategy by use case
: '''Full book comprehension''' → Gemini 1.5 Pro (1M tokens), Claude 3.5 Sonnet (200k)
: '''Long-document QA''' → RAG with chunking + cross-encoder reranker
: '''Multi-session agent memory''' → Conversation summary + vector DB (episodic memory)
: '''Code repository understanding''' → Tree-sitter parsing + selective context, CodeGraph
: '''Long conversations''' → Progressive summarization of older turns into rolling summary
</div>

<div style="background-color: #8B4500; color: #FFFFFF; padding: 20px; border-radius: 8px; margin-bottom: 15px;">