chinnam.AI — RAG & Multi-Agent Content Platform

datasource db {
  provider   = "postgresql"
  url        = env("DATABASE_URL")
  extensions = [vector]  // Enable pgvector extension
}

// Blog posts — the final output
model Post {
  id        String   @id @default(cuid())
  title     String
  slug      String   @unique
  content   String   @db.Text     // Markdown content
  excerpt   String?  @db.Text
  published Boolean  @default(false)
  tags      String[]
  createdAt DateTime @default(now())
  updatedAt DateTime @updatedAt
}

// RAG source — chunked research paper text
model PaperChunk {
  id         String                         @id @default(cuid())
  paperId    String
  title      String                         // Paper title
  authors    String
  arxivId    String                         // arXiv ID for dedup
  chunkIndex Int                            // Position within paper
  section    String    @default("")      // Section heading
  content    String    @db.Text           // Chunk text (~800 tokens)
  embedding  Unsupported("vector(1536)")  // OpenAI embedding
  createdAt  DateTime  @default(now())

  @@index([paperId])
  @@index([arxivId])
}

// Extracted figures with Claude Vision descriptions
model PaperImage {
  id          String                         @id @default(cuid())
  imageData   Bytes                          // Binary image data
  contentType String                         // "image/png"
  description String    @db.Text           // Searchable description
  embedding   Unsupported("vector(1536)")  // Embed the description
  // ... paperId, arxivId, pageNumber, imageIndex
}

// Extracted data tables in markdown format
model PaperTable {
  markdown    String    @db.Text           // Ready-to-use markdown table
  description String    @db.Text           // Searchable description
  embedding   Unsupported("vector(1536)")  // Embed the description
  // ... paperId, arxivId, pageNumber, tableIndex
}

SELECT
  id, title, authors, section, content,
  1 - (embedding <=> '[0.023, -0.041, ...]'::vector) AS similarity
FROM "PaperChunk"
WHERE 1 - (embedding <=> '[...]'::vector) > 0.25   -- similarity threshold
ORDER BY embedding <=> '[...]'::vector              -- nearest first
LIMIT 8                                              -- top-k results

# Section-aware chunking: detects paper headings to preserve context
CHUNK_SIZE = 800       # tokens per chunk
CHUNK_OVERLAP = 150    # overlapping tokens between chunks

# Regex detects: "1. Introduction", "3.1 Model Architecture",
#                "I. INTRODUCTION", "Abstract"
SECTION_HEADING_RE = re.compile(
    r'\n('
    r'(?:1?\d)\.\d+\s+[A-Z][a-zA-Z]+(?:\s+[a-zA-Z\-:,]+){1,10}'
    r'|(?:1?\d)\.\s+[A-Z][a-zA-Z]+(?:\s+[a-zA-Z\-:,]+){0,10}'
    r'|[IVX]+\.\s*[A-Z][^\n]{2,60}'
    r'|Abstract(?:\s*[\u2014\u2013\-])?'
    r')\n'
)

# Stop at References — no useful content beyond this point
FILTER_SECTIONS_RE = re.compile(
    r'(?i)^(?:\d+\.?\s*)?(?:references|bibliography|appendix)'
)

def chunk_text(text, chunk_size=CHUNK_SIZE, overlap=CHUNK_OVERLAP):
    """Split text into overlapping chunks using sentence boundaries."""
    enc = tiktoken.get_encoding("cl100k_base")
    tokens = enc.encode(text)

    chunks = []
    start = 0
    while start < len(tokens):
        end = min(start + chunk_size, len(tokens))
        chunk_text = enc.decode(tokens[start:end])

        # Try to end at sentence boundary (past 50% mark)
        if end < len(tokens):
            last_period = chunk_text.rfind(". ")
            last_newline = chunk_text.rfind("\n")
            split_point = max(last_period, last_newline)
            if split_point > len(chunk_text) * 0.5:
                chunk_text = chunk_text[:split_point + 1]

        chunks.append(chunk_text.strip())
        start = end - overlap  # Overlap for context continuity

    return chunks

import OpenAI from "openai"
import { prisma } from "@/lib/db"

// Generate query embedding using OpenAI
async function generateEmbedding(text: string): Promise<number[]> {
  const response = await getOpenAIClient().embeddings.create({
    model: "text-embedding-3-small",
    input: text,
  })
  return response.data[0].embedding  // 1536-dimensional vector
}

// Semantic search against pgvector with cosine similarity
export async function searchPapers(
  query: string,
  topK: number = 10,
  similarityThreshold: number = 0.3
): Promise<SearchResult[]> {
  const embedding = await generateEmbedding(query)
  const embeddingStr = `[${embedding.join(",")}]`

  // Raw SQL for pgvector cosine similarity
  const results = await prisma.$queryRawUnsafe(`
    SELECT id, title, authors, "arxivId", section, content,
      1 - (embedding <=> '${embeddingStr}'::vector) as similarity
    FROM "PaperChunk"
    WHERE 1 - (embedding <=> '${embeddingStr}'::vector) > ${similarityThreshold}
    ORDER BY embedding <=> '${embeddingStr}'::vector
    LIMIT ${topK}
  `)

  return results
}

// Run all 3 searches in parallel — latency = max(calls), not sum(calls)
const [sources, images, tables] = await Promise.all([
  searchPapers(state.topic, 8, 0.25),   // 8 chunks, threshold 0.25
  searchImages(state.topic, 4, 0.3),   // 4 figures, threshold 0.3
  searchTables(state.topic, 3, 0.3),   // 3 tables, threshold 0.3
])

export interface PipelineState {
  topic: string
  researchSummary: string
  sources: Array<{
    title: string; authors: string; arxivId: string
    section: string; content: string; similarity: number
  }>
  images: Array<{ id: string; title: string; description: string; similarity: number }>
  tables: Array<{ id: string; markdown: string; description: string; similarity: number }>
  draft: string
  factCheckResults: { verified: string[]; issues: string[]; suggestions: string[] }
  finalArticle: string
  metadata: { title: string; excerpt: string; tags: string[] }
  status: "researching" | "drafting" | "fact-checking" | "complete" | "error"
}

const RESEARCH_SYSTEM_PROMPT = `You are a Research Agent specializing in AI/ML topics.
Given a topic and relevant paper excerpts, you must:
1. Identify the key concepts, techniques, and findings
2. Organize findings into logical themes
3. Note contrasting viewpoints or approaches
4. Highlight practical implications and code-worthy examples
5. Cite sources by title and authors`

export async function runResearchAgent(state: PipelineState) {
  // 1. Parallel vector search across 3 content types
  const [sources, images, tables] = await Promise.all([
    searchPapers(state.topic, 8, 0.25),
    searchImages(state.topic, 4, 0.3),
    searchTables(state.topic, 3, 0.3),
  ])

  // 2. Build context string with source attribution
  const sourcesContext = sources.map((s, i) =>
    `[Source ${i+1}] "${s.title}" by ${s.authors}` +
    `${s.section ? ` [Section: ${s.section}]` : ""}` +
    `\n${s.content}`
  ).join("\n\n---\n\n")

  // 3. Send to Claude for structured research summary
  const response = await anthropic.messages.create({
    model: "claude-haiku-4-5-20251001",
    max_tokens: 2000,
    system: RESEARCH_SYSTEM_PROMPT,
    messages: [{ role: "user", content:
      `Topic: ${state.topic}\n\nPaper excerpts:\n\n${sourcesContext}`
    }],
  })

  state.researchSummary = response.content[0].text
  return state
}

const DRAFTING_SYSTEM_PROMPT = `You are a Technical Blog Drafting Agent.
Writing Guidelines:
- Start with a compelling introduction (WHY this matters)
- Progressive complexity: simple → advanced
- Include practical Python code examples
- Include relevant figures: ![Desc](/api/images/IMAGE_ID)
- Output metadata: ---METADATA--- TITLE / EXCERPT / TAGS`

export async function runDraftingAgent(state: PipelineState) {
  // Build multi-modal context: text + images + tables
  const imagesContext = state.images.map((img, i) =>
    `[Figure ${i+1}] From "${img.title}"\nURL: /api/images/${img.id}\nDesc: ${img.description}`
  ).join("\n\n")

  const tablesContext = state.tables.map((tbl, i) =>
    `[Table ${i+1}] From "${tbl.title}"\nMarkdown:\n${tbl.markdown}`
  ).join("\n\n")

  const response = await anthropic.messages.create({
    model: "claude-haiku-4-5-20251001",
    max_tokens: 4000,
    system: DRAFTING_SYSTEM_PROMPT,
    messages: [{ role: "user", content:
      `Topic: ${state.topic}\nResearch:\n${state.researchSummary}` +
      `\n\nFigures:\n${imagesContext}\n\nTables:\n${tablesContext}`
    }],
  })

  // Parse metadata from structured output
  const [article, meta] = response.content[0].text.split("---METADATA---")
  state.draft = article.trim()
  state.metadata = parseMetadata(meta)  // TITLE, EXCERPT, TAGS

  return state
}

const FACTCHECK_SYSTEM_PROMPT = `You are a Fact-Check Agent.
For each claim in the article:
1. Check if supported by research sources
2. Flag potential hallucinations or inaccuracies
3. Verify code examples are syntactically correct
4. Suggest improvements for clarity
Output: ## Verified Claims / ## Issues Found / ## Suggestions / ## Overall Assessment`

export async function runFactCheckAgent(state: PipelineState) {
  const response = await anthropic.messages.create({
    model: "claude-haiku-4-5-20251001",
    max_tokens: 1500,
    system: FACTCHECK_SYSTEM_PROMPT,
    messages: [{ role: "user", content:
      `Review this draft:\n\n${state.draft}\n\nSources:\n${sourcesContext}`
    }],
  })

  // Parse structured output into verified/issues/suggestions arrays
  const sections = response.content[0].text.split("##")
  for (const section of sections) {
    if (section.includes("Verified Claims"))
      verified.push(...extractBullets(section))
    else if (section.includes("Issues Found"))
      issues.push(...extractBullets(section))
    else if (section.includes("Suggestions"))
      suggestions.push(...extractBullets(section))
  }

  state.factCheckResults = { verified, issues, suggestions }
  return state
}

export const maxDuration = 60  // Vercel timeout: 60s per step

export async function POST(request: Request) {
  if (!authorize(request)) return unauthorized()

  const { step, topic, state: prevState } = await request.json()

  // ─── Step 1: Research ───────────────────────────
  if (step === "research") {
    const state = createInitialState(topic)
    await runResearchAgent(state)
    return NextResponse.json({
      step: "research",
      state: { topic, researchSummary, sources, images, tables },
      stats: { sourcesFound, knowledgeBaseSize },
    })
  }

  // ─── Step 2: Draft ──────────────────────────────
  if (step === "draft") {
    const state = rebuildState(topic, prevState)  // From previous step
    await runDraftingAgent(state)
    return NextResponse.json({
      step: "draft",
      state: { ...prevState, draft, metadata },
    })
  }

  // ─── Step 3: Fact-Check ─────────────────────────
  if (step === "fact-check") {
    const state = rebuildState(topic, prevState)
    await runFactCheckAgent(state)
    return NextResponse.json({
      step: "fact-check",
      status: "complete",
      metadata, article, factCheck, stats,
    })
  }
}

// Supports two auth methods:
// 1. Admin UI: x-admin-password header (human users)
// 2. External API: x-api-key header (programmatic access)
function authorize(request: Request): boolean {
  const apiKey = request.headers.get("x-api-key")
  const adminPassword = request.headers.get("x-admin-password")
  return (
    adminPassword === process.env.ADMIN_PASSWORD ||
    apiKey === process.env.API_SECRET_KEY
  )
}

// Frontend orchestrates 3 sequential API calls
// Each call passes the accumulated state to the next
const handleGenerate = async () => {
  setCurrentStep("researching")   // Update UI: Research ●○○○

  // Step 1: Research — search knowledge base
  const researchData = await callStep("Research", {
    step: "research",
    topic: topic.trim(),
  })

  setCurrentStep("drafting")      // Update UI: Research ✓ Draft ●○○

  // Step 2: Draft — pass research state forward
  const draftData = await callStep("Drafting", {
    step: "draft",
    topic: topic.trim(),
    state: researchData.state,     // ← accumulated state
  })

  setCurrentStep("fact-checking")  // Update UI: Research ✓ Draft ✓ FC ●○

  // Step 3: Fact-Check — validate draft against sources
  const factCheckData = await callStep("Fact-check", {
    step: "fact-check",
    topic: topic.trim(),
    state: draftData.state,        // ← accumulated state
  })

  // Pre-fill editable form for human review before publishing
  setEditedTitle(factCheckData.metadata.title)
  setEditedContent(factCheckData.article)
  setCurrentStep("complete")       // Update UI: All ✓
}

EMBEDDING_MODEL = "text-embedding-3-small"
BATCH_SIZE = 50  # OpenAI allows up to 2048 inputs per request

def generate_embeddings(texts: list[str]) -> list[list[float]]:
    """Batch embedding generation via OpenAI API."""
    client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
    response = client.embeddings.create(
        model=EMBEDDING_MODEL,
        input=texts,
    )
    return [item.embedding for item in response.data]

def upload_to_neon(fresh=False):
    """Embed chunks and upload to pgvector with deduplication."""
    conn = psycopg2.connect(os.environ["DATABASE_URL"])
    cur = conn.cursor()

    # Deduplication: skip existing arxiv_id + chunk_index pairs
    cur.execute('SELECT "arxivId", "chunkIndex" FROM "PaperChunk"')
    existing = {(row[0], row[1]) for row in cur.fetchall()}
    new_chunks = [c for c in chunks
                  if (c["arxiv_id"], c["chunk_index"]) not in existing]

    # Batch process: embed + insert
    for i in range(0, len(new_chunks), BATCH_SIZE):
        batch = new_chunks[i : i + BATCH_SIZE]
        texts = [c["content"].replace("\x00", "") for c in batch]
        embeddings = generate_embeddings(texts)

        for chunk, embedding in zip(batch, embeddings):
            embedding_str = "[" + ",".join(str(x) for x in embedding) + "]"
            cur.execute("""
                INSERT INTO "PaperChunk"
                  (id, "paperId", title, authors, "arxivId",
                   "chunkIndex", section, content, embedding, "createdAt")
                VALUES (gen_random_uuid()::text, %s, %s, %s, %s,
                        %s, %s, %s, %s::vector, NOW())
            """, (chunk["paper_id"], ..., embedding_str))

        conn.commit()
        time.sleep(1)  # Rate limit for OpenAI API

"""
Usage:
  python run_ingestion.py          # Run all 5 steps
  python run_ingestion.py --step 2 # Chunk only
  python run_ingestion.py --fresh  # Clear DB and re-ingest
"""

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--step", type=int, choices=[1,2,3,4,5])
    parser.add_argument("--fresh", action="store_true")
    args = parser.parse_args()

    if args.step in (None, 1): download_papers()    # arXiv API → PDFs
    if args.step in (None, 2): process_papers()      # PDF → section-aware chunks
    if args.step in (None, 3): process_images()      # PDF → images + Claude Vision
    if args.step in (None, 4): process_tables()      # PDF → markdown tables
    if args.step in (None, 5):                        # Embed + upload all
        upload_to_neon(fresh=args.fresh)
        upload_images_to_neon()
        upload_tables_to_neon()

const callStep = async (step: string, body: Record<string, unknown>) => {
  const response = await fetch("/api/generate", {
    method: "POST",
    headers: {
      "Content-Type": "application/json",
      "x-admin-password": password,
    },
    body: JSON.stringify(body),
  })

  // Detect Vercel timeout — returns HTML, not JSON
  const contentType = response.headers.get("content-type") || ""
  if (!contentType.includes("application/json")) {
    const text = await response.text()
    throw new Error(
      text.includes("FUNCTION_INVOCATION_TIMEOUT")
        ? `${step} agent timed out. Try a simpler topic.`
        : `Server error: ${text.slice(0, 100)}`
    )
  }

  return await response.json()
}

// Frontend holds state between sequential API calls
const researchData = await callStep("research", { topic })
// researchData.state = { topic, researchSummary, sources, images, tables }

const draftData = await callStep("draft", {
  topic,
  state: researchData.state  // ← pass accumulated state forward
})

const factCheckData = await callStep("fact-check", {
  topic,
  state: draftData.state     // ← pass accumulated state forward
})

// Server is STATELESS — no sessions, no DB persistence for pipeline state
// Each API call receives prev state in body, returns updated state in response

from langgraph.graph import StateGraph, END
from typing import TypedDict

# 1. Define shared state — all agents read/write to this
class PipelineState(TypedDict):
    topic: str
    research: str
    draft: str
    fact_check: dict
    revision_count: int
    next_step: str  # Supervisor sets this

# 2. Define agent functions (nodes)
def research_agent(state: PipelineState) -> PipelineState:
    # Query pgvector, generate research summary
    state["research"] = call_claude(RESEARCH_PROMPT, state["topic"])
    return state

def drafting_agent(state: PipelineState) -> PipelineState:
    # Generate article from research
    state["draft"] = call_claude(DRAFT_PROMPT, state["research"])
    return state

def fact_check_agent(state: PipelineState) -> PipelineState:
    # Validate claims against sources
    result = call_claude(FACTCHECK_PROMPT, state["draft"])
    state["fact_check"] = parse_fact_check(result)
    return state

def supervisor(state: PipelineState) -> PipelineState:
    # Dynamic routing based on state
    if not state.get("research"):
        state["next_step"] = "research"
    elif not state.get("draft"):
        state["next_step"] = "draft"
    elif not state.get("fact_check"):
        state["next_step"] = "fact_check"
    elif state["fact_check"]["issues"] and state["revision_count"] < 2:
        state["next_step"] = "draft"       # ← LOOP BACK to revise!
        state["revision_count"] += 1
    else:
        state["next_step"] = "end"
    return state

# 3. Build the graph
graph = StateGraph(PipelineState)
graph.add_node("research", research_agent)
graph.add_node("draft", drafting_agent)
graph.add_node("fact_check", fact_check_agent)
graph.add_node("supervisor", supervisor)

# 4. Define edges — all agents report back to supervisor
graph.set_entry_point("supervisor")
graph.add_edge("research", "supervisor")
graph.add_edge("draft", "supervisor")
graph.add_edge("fact_check", "supervisor")

# 5. Conditional routing — supervisor decides next agent
graph.add_conditional_edges(
    "supervisor",
    lambda state: state["next_step"],
    {
        "research": "research",
        "draft": "draft",
        "fact_check": "fact_check",
        "end": END,
    }
)

# 6. Compile and run
app = graph.compile()
result = app.invoke({"topic": "RAG systems", "revision_count": 0})

# Producer — Research Agent publishes result
from kafka import KafkaProducer
import json

producer = KafkaProducer(value_serializer=lambda v: json.dumps(v).encode())

def research_agent(topic):
    summary = call_claude(RESEARCH_PROMPT, topic)
    producer.send("research.done", {
        "topic": topic,
        "research_summary": summary,
        "sources": sources,
    })

# Consumer — Draft Agent subscribes to research events
from kafka import KafkaConsumer

consumer = KafkaConsumer("research.done")
for message in consumer:
    data = json.loads(message.value)
    draft = call_claude(DRAFT_PROMPT, data["research_summary"])
    producer.send("draft.done", {
        **data,
        "draft": draft,
    })

import anthropic

client = anthropic.Anthropic()

# Define agents as tools the LLM can invoke
tools = [
    {
        "name": "research_agent",
        "description": "Search knowledge base and summarize research papers on a topic",
        "input_schema": {
            "type": "object",
            "properties": { "topic": { "type": "string" } },
        }
    },
    {
        "name": "drafting_agent",
        "description": "Write a technical blog article from research summary",
        "input_schema": {
            "type": "object",
            "properties": { "research_summary": { "type": "string" } },
        }
    },
    {
        "name": "fact_check_agent",
        "description": "Validate article claims against source papers",
        "input_schema": {
            "type": "object",
            "properties": { "article": { "type": "string" } },
        }
    },
]

# Supervisor LLM decides which agents to call and in what order
response = client.messages.create(
    model="claude-sonnet-4-5-20250929",
    max_tokens=4096,
    system="You are a content pipeline supervisor. Use the available tools to research, draft, and fact-check an article. If fact-check finds issues, revise the draft.",
    tools=tools,
    messages=[{"role": "user", "content": "Write an article about RAG systems"}]
)

# The LLM will return tool_use blocks — you execute them
# and send results back in a tool_result message (agentic loop)