RAG/Retrieval Pipeline — agent-core to Exo Mapping

# agent-core pattern
class EmbeddingProvider(ABC):
    @abstractmethod
    async def embed(self, text: str) -> list[float]: ...
    @abstractmethod
    async def embed_batch(self, texts: list[str]) -> list[list[float]]: ...
    @property
    @abstractmethod
    def dimension(self) -> int: ...

from openjiuwen.core.rag import (
    OpenAIEmbeddingProvider,
    VectorStore,
    VectorRetriever,
    SparseRetriever,
    HybridRetriever,
)

embeddings = OpenAIEmbeddingProvider(api_key="sk-...", model="text-embedding-3-small")
store = VectorStore.create("pgvector", dsn="postgresql://...")
dense = VectorRetriever(embeddings=embeddings, store=store)
sparse = SparseRetriever()
sparse.index(chunks)
hybrid = HybridRetriever(dense=dense, sparse=sparse, vector_weight=0.6)

results = await hybrid.retrieve("How does authentication work?", top_k=10)

from exo.retrieval import (
    OpenAIEmbeddings,
    HybridRetriever,
    VectorRetriever,
    SparseRetriever,
)
from exo.retrieval.backends.pgvector import PgVectorStore

embeddings = OpenAIEmbeddings(api_key="sk-...", model="text-embedding-3-small")
store = PgVectorStore(dsn="postgresql://...", dimensions=1536)
await store.initialize()

dense = VectorRetriever(embeddings=embeddings, store=store)
sparse = SparseRetriever()
await sparse.index(chunks)
hybrid = HybridRetriever(
    vector_retriever=dense,
    sparse_retriever=sparse,
    vector_weight=0.6,
)

results = await hybrid.retrieve("How does authentication work?", top_k=10)

from openjiuwen.core.rag import (
    OpenAIEmbeddingProvider,
    VectorStore,
    Document,
)

embeddings = OpenAIEmbeddingProvider(api_key="sk-...")
store = VectorStore.create("chroma", path="./chroma_data")

doc = Document(id="doc-1", content=open("paper.txt").read())
chunks = doc.chunk(size=500, overlap=50)  # inline chunking
vectors = await embeddings.embed_batch([c.content for c in chunks])
await store.add(chunks, vectors)

from exo.retrieval import (
    OpenAIEmbeddings,
    Document,
    CharacterChunker,
)
from exo.retrieval.backends.chroma import ChromaVectorStore

embeddings = OpenAIEmbeddings(api_key="sk-...")
store = ChromaVectorStore(collection_name="papers", path="./chroma_data")
chunker = CharacterChunker(chunk_size=500, chunk_overlap=50)

doc = Document(id="doc-1", content=open("paper.txt").read(), metadata={})
chunks = chunker.chunk(doc)
vectors = await embeddings.embed_batch([c.content for c in chunks])
await store.add(chunks, vectors)

from openjiuwen.core.rag import (
    AgenticRetriever,
    QueryRewriter,
    VectorRetriever,
)

rewriter = QueryRewriter(model="gpt-4o")
base = VectorRetriever(embeddings=emb, store=store)
agentic = AgenticRetriever(
    base_retriever=base,
    rewriter=rewriter,
    model="gpt-4o",
    max_rounds=3,
)
results = await agentic.retrieve("What are the auth options?")

from exo.retrieval import (
    AgenticRetriever,
    QueryRewriter,
    VectorRetriever,
)

rewriter = QueryRewriter(model="openai:gpt-4o")
base = VectorRetriever(embeddings=emb, store=store)
agentic = AgenticRetriever(
    base_retriever=base,
    rewriter=rewriter,
    model="openai:gpt-4o",  # provider:model format
    max_rounds=3,
    sufficiency_threshold=0.7,
)
results = await agentic.retrieve("What are the auth options?")

from exo.retrieval import (
    retrieve_tool,
    index_tool,
    VectorRetriever,
    CharacterChunker,
    OpenAIEmbeddings,
    InMemoryVectorStore,
)
from exo.agent import Agent

embeddings = OpenAIEmbeddings(api_key="sk-...")
store = InMemoryVectorStore()
retriever = VectorRetriever(embeddings=embeddings, store=store)
chunker = CharacterChunker()

agent = Agent(
    name="rag-agent",
    model="openai:gpt-4o",
    tools=[
        retrieve_tool(retriever),
        index_tool(chunker, store, embeddings),
    ],
)