RAG vs Fine-Tuning vs Prompting: When to Use Each for AI-Powered Developer Tools

// Prompting: changes behavior, not knowledge
// - Shapes: output format, reasoning style, constraints, persona
// - Cannot: add knowledge the model doesn't have
// - Speed: immediate, zero cost
// - Best for: format control, behavior shaping, few-shot examples

const prompt = `You are a code reviewer. When reviewing code:
1. First identify the bug or issue
2. Explain why it is a problem
3. Provide the corrected code
4. Explain why the fix works
Format: use the ISSUE/WHY/FIX/EXPLANATION structure.`;

// RAG: retrieves dynamic knowledge at inference time
// - Shapes: what the model knows for this specific request
// - Cannot: change how the model reasons or writes
// - Speed: 100-500ms retrieval latency
// - Best for: documentation Q&A, codebase-specific knowledge, recent information

async function ragQuery(userQuestion) {
 const relevantDocs = await vectorStore.similaritySearch(userQuestion, { k: 5 });
 return llm.complete(SYSTEM_PROMPT + relevantDocs.map(d => d.content).join('\n') + userQuestion);
}

// Fine-tuning: permanently changes model weights
// - Shapes: writing style, output format, domain-specific patterns
// - Cannot: add knowledge to the model reliably (use RAG for that)
// - Speed: hours-days to train, inference same as base
// - Best for: consistent format, specialized writing style, classification tasks

import Anthropic from '@anthropic-ai/sdk';
import { VectorStore } from './vector-store';

const client = new Anthropic();
const docs = new VectorStore();

// Index your codebase documentation
async function indexDocumentation() {
 const files = await glob('docs/**/*.md');
 for (const file of files) {
 const content = await fs.readFile(file, 'utf-8');
 const chunks = splitIntoChunks(content, { size: 512, overlap: 50 });
 for (const chunk of chunks) {
 await docs.upsert({
 id: `${file}-${chunk.index}`,
 vector: await embed(chunk.text),
 metadata: { source: file, text: chunk.text }
 });
 }
 }
}

// Answer question with retrieved context
async function answerDocQuestion(question) {
 // 1. Retrieve relevant chunks
 const results = await docs.similaritySearch(question, {
 k: 5,
 threshold: 0.7 // Only use high-confidence matches
 });

 if (results.length === 0) {
 return "I don't have documentation for that topic.";
 }

 // 2. Build context-augmented prompt
 const context = results.map(r =>
 `Source: ${r.metadata.source}\n${r.metadata.text}`
 ).join('\n---\n');

 // 3. Generate answer grounded in retrieved docs
 const response = await client.messages.create({
 model: 'claude-sonnet-4-5',
 max_tokens: 1024,
 system: `You are a documentation assistant. Answer based ONLY on the provided context.
 If the answer is not in the context, say so explicitly.
 Always cite which source document you used.`,
 messages: [{
 role: 'user',
 content: `Context:\n${context}\n\nQuestion: ${question}`
 }]
 });

 return response.content[0].text;
}

// Fine-tuning WORKS for:

// 1. Consistent output format
// Base model: sometimes returns JSON, sometimes prose, sometimes both
// Fine-tuned: always returns exactly your JSON schema
// Training data: 500-2000 examples of (input, expected_format_output)

// 2. Domain-specific code style
// Base model: generates generic Python
// Fine-tuned: generates your team's exact conventions, naming patterns, error handling style
// Training data: 1000+ examples from your codebase

// 3. High-volume classification
// Base model: needs prompt explanation every time (expensive)
// Fine-tuned: classifies correctly with minimal prompt (cheap per inference)

// Fine-tuning DOES NOT work for:

// 1. Adding new factual knowledge (hallucination risk increases)
// Use RAG for knowledge, fine-tuning for style

// 2. Fixing reasoning errors on your domain
// If Claude gets math wrong in your domain, fine-tuning won't fix the reasoning
// Use chain-of-thought prompting instead

// 3. Teaching the model about recent events
// Model cannot learn new events from fine-tuning reliably
// Use RAG for anything time-sensitive

// Most production AI developer tools use all three:

class AICodeReviewer {
 async review(prDiff) {
 // Layer 1: Prompting — establishes review behavior and format
 const systemPrompt = `You are a senior code reviewer.
 Focus on: security vulnerabilities, performance issues, error handling.
 Format: use JSON with { issues: [{line, severity, description, fix}] }`;

 // Layer 2: RAG — retrieves team-specific conventions
 const conventions = await this.codebaseIndex.search(
 "coding conventions and patterns for " + inferLanguage(prDiff)
 );

 // Layer 3: Fine-tuned model — trained on team's historical review comments
 // Uses company-fine-tuned Claude that writes in team's review style
 const response = await this.fineTunedClient.create({
 model: 'ft:claude-sonnet-4-5:company:code-reviewer:abc123',
 system: systemPrompt + "\n\nTeam conventions:\n" + conventions,
 messages: [{ role: 'user', content: prDiff }]
 });

 return JSON.parse(response.content[0].text);
 }
}