LLM vs Agents

def attention(Q, K, V):
    # Q, K, V are projections of the input embeddings
    # d_k is the dimension of the key vectors

    scores = matmul(Q, K.transpose()) / sqrt(d_k)
    weights = softmax(scores)  # Normalize to sum to 1
    output = matmul(weights, V)

    return output

def generate(prompt, max_tokens):
    tokens = tokenize(prompt)

    for _ in range(max_tokens):
        # Full forward pass through all transformer layers
        logits = model.forward(tokens)

        # Get probability distribution for next token
        next_token_probs = softmax(logits[-1])

        # Sample from distribution (details in next section)
        next_token = sample(next_token_probs)

        tokens.append(next_token)

        if next_token == END_OF_SEQUENCE:
            break

    return detokenize(tokens)

logits = [2.0, 1.0, 0.5, 0.1]  # Raw model output

# Temperature = 1.0 (default)
probs_t1 = softmax(logits / 1.0)  # [0.48, 0.18, 0.11, 0.07, ...]

# Temperature = 0.3 (more deterministic)
probs_t03 = softmax(logits / 0.3)  # [0.89, 0.07, 0.02, 0.01, ...]

# Temperature = 1.5 (more random)
probs_t15 = softmax(logits / 1.5)  # [0.35, 0.21, 0.16, 0.12, ...]

# Top-k = 5: Only consider 5 most likely tokens
probs = [0.3, 0.25, 0.15, 0.1, 0.08, 0.05, 0.03, 0.02, 0.02, ...]
top_k_probs = [0.3, 0.25, 0.15, 0.1, 0.08, 0, 0, 0, 0, ...]
# Renormalize: [0.34, 0.28, 0.17, 0.11, 0.09]

# Top-p = 0.9: Include tokens until cumulative prob > 0.9
probs = [0.4, 0.3, 0.15, 0.08, 0.04, 0.02, 0.01, ...]
cumsum = [0.4, 0.7, 0.85, 0.93, ...]  # Stop at 0.93 > 0.9
nucleus = [0.4, 0.3, 0.15, 0.08]  # 4 tokens selected
# Renormalize: [0.43, 0.32, 0.16, 0.09]

// Turn 1
request_1 = {
  messages: [
    {role: "user", content: "What is Rust?"}
  ]
}

// Turn 2 - ENTIRE history is sent again
request_2 = {
  messages: [
    {role: "user", content: "What is Rust?"},
    {role: "assistant", content: "Rust is a systems programming language..."},
    {role: "user", content: "How does ownership work?"}
  ]
}

// Turn 3 - History keeps growing
request_3 = {
  messages: [
    {role: "user", content: "What is Rust?"},
    {role: "assistant", content: "Rust is a systems programming language..."},
    {role: "user", content: "How does ownership work?"},
    {role: "assistant", content: "Ownership is Rust's approach to memory..."},
    {role: "user", content: "Show me an example"}
  ]
}

Question: What is the population of the capital of France?

Thought: I need to find the capital of France first, then look up its population.
Action: search("capital of France")
Observation: Paris is the capital of France.

Thought: Now I know the capital is Paris. I need to find its population.
Action: search("population of Paris")
Observation: The population of Paris is approximately 2.1 million in the city proper.

Thought: I have the answer. Paris is the capital with 2.1 million people.
Action: finish("The capital of France is Paris, with a population of about 2.1 million")

// Step 1: Planning phase
Planner Input: "Create a report on Q3 sales by region"
Planner Output: [
  "1. Query database for Q3 sales data",
  "2. Group results by region",
  "3. Calculate summary statistics per region",
  "4. Generate visualization",
  "5. Write narrative summary",
  "6. Compile into report format"
]

// Step 2: Execution phase (per step)
Executor Input: "Execute step 1: Query database for Q3 sales data"
Executor Output: {action: "sql_query", query: "SELECT * FROM sales WHERE quarter='Q3'"}

// Step 3: Observation and replanning if needed
Observation: Query returned 0 rows - table name is 'transactions' not 'sales'
Replanner: [Update step 1 to use correct table name, continue from step 1]

// MRKL architecture
modules = {
  "calculator": CalculatorModule(),      // Symbolic math
  "search": SearchModule(),               // Information retrieval
  "code_exec": PythonREPL(),              // Code execution
  "database": SQLModule(),                // Structured data
  "llm": LLMModule()                      // General reasoning
}

// Router decides which module handles each part
query = "What's 15% of the revenue from last quarter?"

router_decision = [
  ("database", "SELECT revenue FROM quarterly_reports WHERE quarter='last'"),
  ("calculator", "0.15 * {database_result}")
]

// Toolformer-style generation (tool calls embedded in text)
Input: "The Eiffel Tower is [QA("height of Eiffel Tower")->330 meters] tall
        and was completed in [QA("Eiffel Tower completion year")->1889]."

// Model learns to:
// 1. Recognize when external information is needed
// 2. Insert appropriate API calls
// 3. Continue generation using API responses

// Tool definitions provided to the model
tools = [
  {
    name: "search_web",
    description: "Search the web for current information",
    parameters: {
      type: "object",
      properties: {
        query: {type: "string", description: "Search query"},
        num_results: {type: "integer", default: 5}
      },
      required: ["query"]
    }
  },
  {
    name: "read_file",
    description: "Read contents of a file",
    parameters: {
      type: "object",
      properties: {
        path: {type: "string", description: "File path"}
      },
      required: ["path"]
    }
  }
]

// Model output (structured, not free-form text)
{
  "function_call": {
    "name": "search_web",
    "arguments": {"query": "latest rust 1.75 features", "num_results": 3}
  }
}

// Conversation memory with summarization
class ConversationMemory:
    def __init__(self, max_tokens=4000):
        self.messages = []
        self.summary = ""

    def add(self, message):
        self.messages.append(message)

        # If approaching limit, summarize older messages
        if self.token_count() > self.max_tokens:
            old_messages = self.messages[:-5]  # Keep last 5 intact
            self.summary = summarize(self.summary, old_messages)
            self.messages = self.messages[-5:]

    def get_context(self):
        if self.summary:
            return [{"role": "system", "content": f"Previous context: {self.summary}"}]
                   + self.messages
        return self.messages

// Episodic memory structure
episode = {
  id: "ep_20240115_001",
  timestamp: 1705334400,
  trigger: "User asked to debug authentication issue",
  actions: [
    "Searched codebase for auth-related files",
    "Found bug in token validation",
    "Applied fix and tested"
  ],
  outcome: "success",
  learnings: ["Always check token expiry logic first for auth issues"],
  embedding: vector_768d  // For similarity search
}

// Retrieval: find similar past experiences
relevant_episodes = vector_search(
  query_embedding=embed(current_situation),
  filter={"outcome": "success"},
  top_k=3
)

// Semantic memory as knowledge graph + vector store
knowledge_base = {
  entities: [
    {type: "project", name: "auth-service", language: "rust", ...},
    {type: "api", name: "/api/login", method: "POST", ...},
    {type: "preference", user: "alice", setting: "verbose_output", value: true}
  ],

  relations: [
    {from: "auth-service", relation: "exposes", to: "/api/login"},
    {from: "alice", relation: "owns", to: "auth-service"}
  ],

  facts: [  // Vector-indexed for retrieval
    {content: "Auth tokens expire after 24 hours", embedding: [...]},
    {content: "Rate limit is 100 requests per minute", embedding: [...]}
  ]
}

// Working memory for a multi-step task
working_memory = {
  goal: "Generate quarterly report for Q3 2024",

  plan: [
    {step: 1, action: "fetch_data", status: "complete", result: "..."},
    {step: 2, action: "analyze", status: "in_progress"},
    {step: 3, action: "visualize", status: "pending"},
    {step: 4, action: "write_summary", status: "pending"}
  ],

  scratchpad: {
    total_revenue: 1250000,
    regions_processed: ["NA", "EU"],
    pending_questions: ["Should APAC be included?"]
  },

  context_files: ["/data/q3_sales.csv", "/templates/report.md"]
}

async function agentLoop(goal: string, maxIterations: number = 10) {
  let state = initializeState(goal);
  let iteration = 0;

  while (iteration < maxIterations) {
    iteration++;

    // 1. Observe: Gather current context
    const observation = await gatherContext(state);

    // 2. Think: LLM decides next action (INFERENCE CALL)
    const decision = await llm.generate({
      system: AGENT_SYSTEM_PROMPT,
      messages: formatMessages(state, observation),
      tools: AVAILABLE_TOOLS
    });

    // 3. Check for completion
    if (decision.action === "finish") {
      return {success: true, result: decision.result, iterations: iteration};
    }

    // 4. Act: Execute the chosen tool
    const result = await executeTool(decision.action, decision.arguments);

    // 5. Update state with result
    state = updateState(state, decision, result);

    // 6. Optional: Reflect on progress (ANOTHER INFERENCE CALL)
    if (iteration % 3 === 0) {
      state = await reflectOnProgress(state);
    }
  }

  return {success: false, error: "Max iterations exceeded", state};
}

const explanation = await llm.generate({
  system: "You are a code explanation assistant.",
  messages: [{
    role: "user",
    content: `Explain this function:

${codeSnippet}`
  }]
});

// No tools, no iteration, no external data needed
return explanation;

const result = await agent.run({
  goal: "Find the top 3 competitors to Acme Corp and summarize their pricing",
  tools: [webSearch, readPage, extractData, writeReport],
  maxIterations: 15
});

// Agent will:
// 1. Search for "Acme Corp competitors"
// 2. Read several pages to identify competitors
// 3. Search for each competitor's pricing page
// 4. Extract pricing information
// 5. Synthesize findings into summary
// Each step may require multiple sub-steps and verification

// Raw LLM approach - simple and efficient
async function reviewCode(diff: string): Promise<Review> {
  const response = await llm.generate({
    system: `You are a senior code reviewer. Analyze the diff for:
      - Bugs or logic errors
      - Security issues
      - Performance concerns
      - Style violations
      Provide specific, actionable feedback.`,
    messages: [{
      role: "user",
      content: `Review this diff:

${diff}`
    }],
    temperature: 0.3  // Low temperature for consistent, focused output
  });

  return parseReview(response);
}

// Single LLM call, ~2000 tokens, <2 seconds
// No agent overhead needed

// Agent approach - necessary for investigation
const investigation = await agent.run({
  goal: "Investigate and fix the broken login page",

  tools: {
    read_logs: (filter) => fetchLogs("auth-service", filter),
    search_code: (query) => grepCodebase(query),
    read_file: (path) => readFile(path),
    run_test: (testName) => executeTest(testName),
    git_blame: (file, line) => getBlameInfo(file, line),
    propose_fix: (file, changes) => createPatch(file, changes)
  },

  maxIterations: 20
});

// Typical execution trace:
// 1. read_logs({level: "error", service: "auth", last: "1h"})
//    -> Found: "TypeError: Cannot read property 'token' of undefined"
// 2. search_code("property 'token'")
//    -> Found references in auth/validate.ts, auth/session.ts
// 3. read_file("auth/validate.ts")
//    -> Identified null check missing on line 47
// 4. git_blame("auth/validate.ts", 47)
//    -> Changed 2 days ago in commit abc123
// 5. run_test("auth.validate.test")
//    -> Confirmed: test fails with null session
// 6. propose_fix("auth/validate.ts", {line: 47, change: "add null check"})

// 12 LLM calls, ~15,000 tokens, ~30 seconds
// Could not be done without agent capabilities

// Hybrid approach - agent orchestrates, raw LLM processes
async function analyzeCustomerFeedback(feedbackBatch: Feedback[]) {
  // Agent handles orchestration and decision-making
  const agent = createAgent({
    goal: "Analyze customer feedback and generate insights report",

    tools: {
      // This tool uses RAW LLM for each item - no agent overhead
      classify_feedback: async (items: Feedback[]) => {
        return Promise.all(items.map(item =>
          llm.generate({
            messages: [{role: "user", content: `Classify: ${item.text}`}],
            temperature: 0.1  // Deterministic classification
          })
        ));
      },

      // Raw LLM for summarization
      summarize_category: async (category: string, items: Feedback[]) => {
        return llm.generate({
          messages: [{
            role: "user",
            content: `Summarize these ${category} feedback items: ${JSON.stringify(items)}`
          }]
        });
      },

      // Actual tool actions
      save_to_db: (report) => database.insert("reports", report),
      notify_team: (channel, message) => slack.post(channel, message)
    }
  });

  return agent.run();
}

// Agent makes ~5 orchestration calls
// Raw LLM makes ~100 classification calls (in parallel)
// Best of both worlds: intelligent orchestration + efficient processing