[{"data":1,"prerenderedAt":842},["ShallowReactive",2],{"navigation_docs_en":3,"\u002Fen\u002Fai-engineering\u002Fevaluate-ai-systems\u002Fch04-3-design-your-evaluation-pipeline":141,"\u002Fen\u002Fai-engineering\u002Fevaluate-ai-systems\u002Fch04-3-design-your-evaluation-pipeline-surround":837},[4],{"title":5,"icon":6,"path":7,"stem":8,"children":9,"page":45},"AI Engineering",null,"\u002Fen\u002Fai-engineering","en\u002F1.ai-engineering",[10,46,77,114],{"title":11,"icon":12,"path":13,"stem":14,"children":15,"page":45},"Introduction to Building AI Applications with Foundation Models","i-lucide-brain-circuit","\u002Fen\u002Fai-engineering\u002Fintro","en\u002F1.ai-engineering\u002F1.intro",[16,20,25,30,35,40],{"title":11,"path":17,"stem":18,"icon":19},"\u002Fen\u002Fai-engineering\u002Fintro\u002Fch01","en\u002F1.ai-engineering\u002F1.intro\u002Fch01","i-lucide-sparkles",{"title":21,"path":22,"stem":23,"icon":24},"The Rise of AI Engineering","\u002Fen\u002Fai-engineering\u002Fintro\u002Fch011-the-rise-of-ai-engineering","en\u002F1.ai-engineering\u002F1.intro\u002Fch011-the-rise-of-ai-engineering","i-lucide-history",{"title":26,"path":27,"stem":28,"icon":29},"Foundation Model Use Cases","\u002Fen\u002Fai-engineering\u002Fintro\u002Fch012-foundation-model-use-cases","en\u002F1.ai-engineering\u002F1.intro\u002Fch012-foundation-model-use-cases","i-lucide-layout-grid",{"title":31,"path":32,"stem":33,"icon":34},"Planning AI Applications","\u002Fen\u002Fai-engineering\u002Fintro\u002Fch013-planning-ai-applications","en\u002F1.ai-engineering\u002F1.intro\u002Fch013-planning-ai-applications","i-lucide-clipboard-list",{"title":36,"path":37,"stem":38,"icon":39},"The AI Engineering Stack","\u002Fen\u002Fai-engineering\u002Fintro\u002Fch014-the-ai-engineering-stack","en\u002F1.ai-engineering\u002F1.intro\u002Fch014-the-ai-engineering-stack","i-lucide-layers",{"title":41,"path":42,"stem":43,"icon":44},"Summary","\u002Fen\u002Fai-engineering\u002Fintro\u002Fch015-summary","en\u002F1.ai-engineering\u002F1.intro\u002Fch015-summary","i-lucide-flag",false,{"title":47,"icon":6,"path":48,"stem":49,"children":50,"page":45},"Understanding Foundation Models","\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models","en\u002F1.ai-engineering\u002F2.understanding-foundation-models",[51,54,59,64,69,74],{"title":47,"path":52,"stem":53,"icon":12},"\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models\u002Fch02","en\u002F1.ai-engineering\u002F2.understanding-foundation-models\u002Fch02",{"title":55,"path":56,"stem":57,"icon":58},"Training Data","\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models\u002Fch02-1-training-data","en\u002F1.ai-engineering\u002F2.understanding-foundation-models\u002Fch02-1-training-data","i-lucide-database",{"title":60,"path":61,"stem":62,"icon":63},"Modeling","\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models\u002Fch02-2-modeling","en\u002F1.ai-engineering\u002F2.understanding-foundation-models\u002Fch02-2-modeling","i-lucide-network",{"title":65,"path":66,"stem":67,"icon":68},"Post-Training","\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models\u002Fch02-3-post-training","en\u002F1.ai-engineering\u002F2.understanding-foundation-models\u002Fch02-3-post-training","i-lucide-sliders-horizontal",{"title":70,"path":71,"stem":72,"icon":73},"Sampling","\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models\u002Fch02-4-sampling","en\u002F1.ai-engineering\u002F2.understanding-foundation-models\u002Fch02-4-sampling","i-lucide-dices",{"title":41,"path":75,"stem":76,"icon":44},"\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models\u002Fch02-5-summary","en\u002F1.ai-engineering\u002F2.understanding-foundation-models\u002Fch02-5-summary",{"title":78,"path":79,"stem":80,"children":81,"page":45},"Evaluation Methodology","\u002Fen\u002Fai-engineering\u002Fevaluation-methodology","en\u002F1.ai-engineering\u002F3.evaluation-methodology",[82,86,91,96,101,106,111],{"title":78,"path":83,"stem":84,"icon":85},"\u002Fen\u002Fai-engineering\u002Fevaluation-methodology\u002Fch03","en\u002F1.ai-engineering\u002F3.evaluation-methodology\u002Fch03","i-lucide-clipboard-check",{"title":87,"path":88,"stem":89,"icon":90},"Challenges of Evaluating Foundation Models","\u002Fen\u002Fai-engineering\u002Fevaluation-methodology\u002Fch03-1-challenges-of-evaluating-foundation-models","en\u002F1.ai-engineering\u002F3.evaluation-methodology\u002Fch03-1-challenges-of-evaluating-foundation-models","i-lucide-shield-alert",{"title":92,"path":93,"stem":94,"icon":95},"Understanding Language Modeling Metrics","\u002Fen\u002Fai-engineering\u002Fevaluation-methodology\u002Fch03-2-understanding-language-modeling-metrics","en\u002F1.ai-engineering\u002F3.evaluation-methodology\u002Fch03-2-understanding-language-modeling-metrics","i-lucide-sigma",{"title":97,"path":98,"stem":99,"icon":100},"Exact Evaluation","\u002Fen\u002Fai-engineering\u002Fevaluation-methodology\u002Fch03-3-exact-evaluation","en\u002F1.ai-engineering\u002F3.evaluation-methodology\u002Fch03-3-exact-evaluation","i-lucide-check-check",{"title":102,"path":103,"stem":104,"icon":105},"AI as a Judge","\u002Fen\u002Fai-engineering\u002Fevaluation-methodology\u002Fch03-4-ai-as-a-judge","en\u002F1.ai-engineering\u002F3.evaluation-methodology\u002Fch03-4-ai-as-a-judge","i-lucide-scale",{"title":107,"path":108,"stem":109,"icon":110},"Ranking Models with Comparative Evaluation","\u002Fen\u002Fai-engineering\u002Fevaluation-methodology\u002Fch03-5-ranking-models-with-comparative-evaluation","en\u002F1.ai-engineering\u002F3.evaluation-methodology\u002Fch03-5-ranking-models-with-comparative-evaluation","i-lucide-trophy",{"title":41,"path":112,"stem":113,"icon":44},"\u002Fen\u002Fai-engineering\u002Fevaluation-methodology\u002Fch03-6-summary","en\u002F1.ai-engineering\u002F3.evaluation-methodology\u002Fch03-6-summary",{"title":115,"icon":116,"path":117,"stem":118,"children":119,"page":45},"Evaluate AI Systems","i-lucide-binary","\u002Fen\u002Fai-engineering\u002Fevaluate-ai-systems","en\u002F1.ai-engineering\u002F4.evaluate-ai-systems",[120,123,128,133,138],{"title":115,"path":121,"stem":122,"icon":116},"\u002Fen\u002Fai-engineering\u002Fevaluate-ai-systems\u002Fch04","en\u002F1.ai-engineering\u002F4.evaluate-ai-systems\u002Fch04",{"title":124,"path":125,"stem":126,"icon":127},"Evaluation Criteria","\u002Fen\u002Fai-engineering\u002Fevaluate-ai-systems\u002Fch04-1-evaluation-criteria","en\u002F1.ai-engineering\u002F4.evaluate-ai-systems\u002Fch04-1-evaluation-criteria","i-lucide-check-circle-2",{"title":129,"path":130,"stem":131,"icon":132},"Model Selection","\u002Fen\u002Fai-engineering\u002Fevaluate-ai-systems\u002Fch04-2-model-selection","en\u002F1.ai-engineering\u002F4.evaluate-ai-systems\u002Fch04-2-model-selection","i-lucide-cpu",{"title":134,"path":135,"stem":136,"icon":137},"Design Your Evaluation Pipeline","\u002Fen\u002Fai-engineering\u002Fevaluate-ai-systems\u002Fch04-3-design-your-evaluation-pipeline","en\u002F1.ai-engineering\u002F4.evaluate-ai-systems\u002Fch04-3-design-your-evaluation-pipeline","i-lucide-workflow",{"title":41,"path":139,"stem":140,"icon":44},"\u002Fen\u002Fai-engineering\u002Fevaluate-ai-systems\u002Fch04-4-summary","en\u002F1.ai-engineering\u002F4.evaluate-ai-systems\u002Fch04-4-summary",{"id":142,"title":134,"body":143,"description":831,"extension":832,"links":6,"meta":833,"navigation":834,"path":135,"seo":835,"stem":136,"__hash__":836},"docs_en\u002Fen\u002F1.ai-engineering\u002F4.evaluate-ai-systems\u002Fch04-3-design-your-evaluation-pipeline.md",{"type":144,"value":145,"toc":805},"minimark",[146,156,161,165,170,173,190,194,209,213,235,240,248,267,270,274,277,382,386,406,410,413,464,466,470,474,477,481,484,504,514,579,584,588,591,645,656,658,662,665],[147,148,149,153],"u-page-hero",{},[150,151,134],"template",{"v-slot:title":152},"",[150,154,155],{"v-slot:description":152},"The success of an AI application hinges on your ability to reliably differentiate good outcomes from bad ones. Building a production-grade evaluation pipeline is the single most important investment in AI engineering.",[157,158,160],"h2",{"id":159},"step-1-multi-level-system-evaluation","Step 1: Multi-Level System Evaluation",[162,163,164],"p",{},"Real-world AI applications are multi-step distributed systems. Evaluating only the final output obscures failure boundaries.",[166,167,169],"h3",{"id":168},"component-level-vs-end-to-end-evaluation","Component-Level vs. End-to-End Evaluation",[162,171,172],{},"Consider an application that extracts an applicant's current employer from a PDF resume:",[174,175,176,184],"ol",{},[177,178,179,183],"li",{},[180,181,182],"strong",{},"Step 1 (Ingestion)",": Extract raw text from the PDF layout.",[177,185,186,189],{},[180,187,188],{},"Step 2 (Extraction)",": Extract the employer name from the parsed text.",[191,192,193],"warning",{},"If the application outputs the wrong employer, did Step 1 fail (OCR formatting truncation) or did Step 2 fail (model hallucination)?",[195,196,197,203],"ul",{},[177,198,199,202],{},[180,200,201],{},"Evaluate Step 1",": Measure string similarity between OCR output and ground-truth text.",[177,204,205,208],{},[180,206,207],{},"Evaluate Step 2",": Measure extraction accuracy given gold-standard parsed text.",[166,210,212],{"id":211},"turn-based-vs-task-based-evaluation","Turn-Based vs. Task-Based Evaluation",[214,215,216,226],"card-group",{},[217,218,221,222,225],"card",{"icon":219,"title":220},"i-lucide-message-square","Turn-Based Evaluation","Assesses the quality, safety, and conciseness of ",[180,223,224],{},"individual response messages"," at each step of an interaction.",[217,227,230,231,234],{"icon":228,"title":229},"i-lucide-target","Task-Based Evaluation","Evaluates whether the user's ",[180,232,233],{},"overarching objective"," was achieved (e.g., bug resolved, booking completed) and how many turns it required.",[236,237,239],"h4",{"id":238},"example-twenty-questions-benchmark","Example: Twenty Questions Benchmark",[162,241,242,243,247],{},"In BIG-bench's ",[244,245,246],"code",{},"twenty_questions"," benchmark, two model instances play the guessing game:",[195,249,250,258,261],{},[177,251,252,253,257],{},"Model A picks a concept (e.g., ",[254,255,256],"em",{},"apple",").",[177,259,260],{},"Model B asks yes\u002Fno questions to deduce the concept.",[177,262,263,266],{},[180,264,265],{},"Score",": Binary task success combined with the number of turns taken to solve it.",[268,269],"hr",{},[157,271,273],{"id":272},"step-2-establish-unambiguous-guidelines","Step 2: Establish Unambiguous Guidelines",[162,275,276],{},"An ambiguous rubric generates noisy scores that mislead development.",[278,279,280,375],"note",{},[166,281,283,284,374],{"id":282},"correctness-neq-quality","Correctness ",[285,286,289,313],"span",{"className":287},[288],"katex",[285,290,293],{"className":291},[292],"katex-mathml",[294,295,297],"math",{"xmlns":296},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML",[298,299,300,308],"semantics",{},[301,302,303],"mrow",{},[304,305,307],"mo",{"mathvariant":306},"normal","≠",[309,310,312],"annotation",{"encoding":311},"application\u002Fx-tex","\\neq",[285,314,318],{"className":315,"ariaHidden":317},[316],"katex-html","true",[285,319,322,327],{"className":320},[321],"base",[285,323],{"className":324,"style":326},[325],"strut","height:0.8889em;vertical-align:-0.1944em;",[285,328,331,365,370],{"className":329},[330],"mrel",[285,332,334],{"className":333},[330],[285,335,339],{"className":336},[337,338],"mord","vbox",[285,340,343],{"className":341},[342],"thinbox",[285,344,347,350,361],{"className":345},[346],"rlap",[285,348],{"className":349,"style":326},[325],[285,351,354],{"className":352},[353],"inner",[285,355,357],{"className":356},[337],[285,358,360],{"className":359},[330],"",[285,362],{"className":363},[364],"fix",[285,366],{"className":367},[368,369],"mspace","nobreak",[285,371,373],{"className":372},[330],"="," Quality",[162,376,377,378,381],{},"When LinkedIn deployed its AI-powered Job Assessment assistant, they discovered that the response ",[254,379,380],{},"\"You are a terrible fit\""," was technically correct, but completely unhelpful. A high-quality response must explain the qualification gap and provide actionable steps to close it.",[166,383,385],{"id":384},"concrete-scoring-rubrics","Concrete Scoring Rubrics",[174,387,388,394,400],{},[177,389,390,393],{},[180,391,392],{},"Relevance",": Does the completion address the user's specific prompt?",[177,395,396,399],{},[180,397,398],{},"Factual Consistency",": Is the completion fully supported by the provided source context?",[177,401,402,405],{},[180,403,404],{},"Safety & Policy",": Does the response respect content safety and tone guidelines?",[166,407,409],{"id":408},"tying-technical-metrics-to-business-outcomes","Tying Technical Metrics to Business Outcomes",[162,411,412],{},"Map model evaluation scores directly to operational thresholds:",[414,415,416,430],"table",{},[417,418,419],"thead",{},[420,421,422,427],"tr",{},[423,424,426],"th",{"align":425},"left","Factual Consistency Score",[423,428,429],{"align":425},"Operational Automation Capability",[431,432,433,444,454],"tbody",{},[420,434,435,441],{},[436,437,438],"td",{"align":425},[180,439,440],{},"80%",[436,442,443],{"align":425},"Safe for product recommendations; unusable for billing",[420,445,446,451],{},[436,447,448],{"align":425},[180,449,450],{},"90%",[436,452,453],{"align":425},"Automates ~50% of Tier-1 customer support tickets",[420,455,456,461],{},[436,457,458],{"align":425},[180,459,460],{},"98%",[436,462,463],{"align":425},"Automates ~90% of requests with minimal human escalation",[268,465],{},[157,467,469],{"id":468},"step-3-methods-datasets-and-slicing","Step 3: Methods, Datasets, and Slicing",[166,471,473],{"id":472},"combining-fast-classifiers-with-deep-judges","Combining Fast Classifiers with Deep Judges",[162,475,476],{},"A balanced evaluation architecture combines lightweight classifiers (Perspective API, DeBERTa entailment models) running over 100% of production traffic with deeper LLM-as-a-judge reviews over a sampled 1–5% cohort.",[166,478,480],{"id":479},"slice-based-evaluation","Slice-Based Evaluation",[162,482,483],{},"Aggregated scores hide fatal vulnerabilities. Slicing separates evaluation datasets into distinct cohorts:",[195,485,486,492,498],{},[177,487,488,491],{},[180,489,490],{},"Input Length",": Short queries vs. multi-thousand token contexts.",[177,493,494,497],{},[180,495,496],{},"User Tiers",": Free tier vs. enterprise accounts.",[177,499,500,503],{},[180,501,502],{},"Error-Prone Scenarios",": Prompts with typos, noisy OCR text, or out-of-scope queries.",[505,506,507,511],"caution",{},[166,508,510],{"id":509},"simpsons-paradox-in-model-evaluation","Simpson's Paradox in Model Evaluation",[162,512,513],{},"Aggregated benchmarks can lead to false conclusions where Model B wins overall despite losing across every individual subgroup.",[414,515,516,529],{},[417,517,518],{},[420,519,520,523,526],{},[423,521,522],{"align":425},"Cohort",[423,524,525],{"align":425},"Model A (Accuracy)",[423,527,528],{"align":425},"Model B (Accuracy)",[431,530,531,547,563],{},[420,532,533,538,544],{},[436,534,535],{"align":425},[180,536,537],{},"Group 1 (Simple Queries)",[436,539,540,543],{"align":425},[180,541,542],{},"93%"," (81 \u002F 87)",[436,545,546],{"align":425},"87% (234 \u002F 270)",[420,548,549,554,560],{},[436,550,551],{"align":425},[180,552,553],{},"Group 2 (Complex Queries)",[436,555,556,559],{"align":425},[180,557,558],{},"73%"," (192 \u002F 263)",[436,561,562],{"align":425},"69% (55 \u002F 80)",[420,564,565,570,573],{},[436,566,567],{"align":425},[180,568,569],{},"Overall Aggregate",[436,571,572],{"align":425},"78% (273 \u002F 350)",[436,574,575,578],{"align":425},[180,576,577],{},"83%"," (289 \u002F 350)",[162,580,581],{},[254,582,583],{},"Model B wins overall solely due to a skewed sample distribution across simple and complex cohorts.",[166,585,587],{"id":586},"sizing-your-evaluation-dataset","Sizing Your Evaluation Dataset",[162,589,590],{},"How large must an evaluation dataset be to verify that a new model or prompt is genuinely better?",[414,592,593,603],{},[417,594,595],{},[420,596,597,600],{},[423,598,599],{"align":425},"Target Score Difference",[423,601,602],{"align":425},"Sample Size Needed (95% Confidence)",[431,604,605,615,625,635],{},[420,606,607,612],{},[436,608,609],{"align":425},[180,610,611],{},"30% difference",[436,613,614],{"align":425},"~10 examples",[420,616,617,622],{},[436,618,619],{"align":425},[180,620,621],{},"10% difference",[436,623,624],{"align":425},"~100 examples",[420,626,627,632],{},[436,628,629],{"align":425},[180,630,631],{},"3% difference",[436,633,634],{"align":425},"~1,000 examples",[420,636,637,642],{},[436,638,639],{"align":425},[180,640,641],{},"1% difference",[436,643,644],{"align":425},"~10,000 examples",[646,647,648,651,652,655],"tip",{},[180,649,650],{},"The 3× \u002F 10× Rule",": For every 3× decrease in score margin you wish to reliably detect, the required sample size increases by ",[180,653,654],{},"10×",".",[268,657],{},[157,659,661],{"id":660},"step-4-evaluating-the-evaluator-iteration","Step 4: Evaluating the Evaluator & Iteration",[162,663,664],{},"Your evaluation harness must itself be monitored and tested:",[666,667,669,673,676,680,744,748,798,802],"steps",{"level":668},"3",[166,670,672],{"id":671},"calibration-ground-truth-alignment","Calibration & Ground Truth Alignment",[162,674,675],{},"Do prompts that receive a 5\u002F5 from the AI judge actually correlate with customer satisfaction and task completion?",[166,677,679],{"id":678},"reproducibility-variance","Reproducibility & Variance",[162,681,682,683,743],{},"Run the pipeline multiple times with temperature ",[285,684,686,708],{"className":685},[288],[285,687,689],{"className":688},[292],[294,690,691],{"xmlns":296},[298,692,693,705],{},[301,694,695,699,701],{},[696,697,698],"mi",{},"T",[304,700,373],{},[702,703,704],"mn",{},"0",[309,706,707],{"encoding":311},"T=0",[285,709,711,733],{"className":710,"ariaHidden":317},[316],[285,712,714,718,723,727,730],{"className":713},[321],[285,715],{"className":716,"style":717},[325],"height:0.6833em;",[285,719,698],{"className":720,"style":722},[337,721],"mathnormal","margin-right:0.1389em;",[285,724],{"className":725,"style":726},[368],"margin-right:0.2778em;",[285,728,373],{"className":729},[330],[285,731],{"className":732,"style":726},[368],[285,734,736,740],{"className":735},[321],[285,737],{"className":738,"style":739},[325],"height:0.6444em;",[285,741,704],{"className":742},[337],". If scores fluctuate on identical inputs, the rubric or prompt is under-specified.",[166,745,747],{"id":746},"metric-de-correlation","Metric De-Correlation",[162,749,750,751,797],{},"Ensure selected metrics test distinct axes. If two metrics exhibit a correlation ",[285,752,754,772],{"className":753},[288],[285,755,757],{"className":756},[292],[294,758,759],{"xmlns":296},[298,760,761,769],{},[301,762,763,766],{},[304,764,765],{},">",[702,767,768],{},"0.9",[309,770,771],{"encoding":311},">0.9",[285,773,775,788],{"className":774,"ariaHidden":317},[316],[285,776,778,782,785],{"className":777},[321],[285,779],{"className":780,"style":781},[325],"height:0.5782em;vertical-align:-0.0391em;",[285,783,765],{"className":784},[330],[285,786],{"className":787,"style":726},[368],[285,789,791,794],{"className":790},[321],[285,792],{"className":793,"style":739},[325],[285,795,768],{"className":796},[337],", drop one to save compute and eliminate bias.",[166,799,801],{"id":800},"experiment-tracking","Experiment Tracking",[162,803,804],{},"Log full evaluation runs in version control — including rubric definitions, judge model snapshots, temperatures, and input datasets.",{"title":152,"searchDepth":806,"depth":806,"links":807},2,[808,813,819,825],{"id":159,"depth":806,"text":160,"children":809},[810,812],{"id":168,"depth":811,"text":169},3,{"id":211,"depth":811,"text":212},{"id":272,"depth":806,"text":273,"children":814},[815,817,818],{"id":282,"depth":811,"text":816},"Correctness ≠\\neq= Quality",{"id":384,"depth":811,"text":385},{"id":408,"depth":811,"text":409},{"id":468,"depth":806,"text":469,"children":820},[821,822,823,824],{"id":472,"depth":811,"text":473},{"id":479,"depth":811,"text":480},{"id":509,"depth":811,"text":510},{"id":586,"depth":811,"text":587},{"id":660,"depth":806,"text":661,"children":826},[827,828,829,830],{"id":671,"depth":811,"text":672},{"id":678,"depth":811,"text":679},{"id":746,"depth":811,"text":747},{"id":800,"depth":811,"text":801},"A practical guide to architecting production evaluation pipelines, defining scoring rubrics, slicing datasets, and avoiding Simpson's paradox.","md",{},{"icon":137},{"title":134,"description":831},"Im1LEehjp3uvZsCks3uLRpq3qjVdKB5KwGWMl57rak8",[838,840],{"title":129,"path":130,"stem":131,"description":839,"icon":132,"children":-1},"A strategic guide to selecting foundation models, comparing self-hosting vs model APIs, and critically navigating public benchmarks and leaderboards.",{"title":41,"path":139,"stem":140,"description":841,"icon":44,"children":-1},"A recap of system evaluation principles, model selection trade-offs, and practical guidelines for building production evaluation harnesses.",1789413992558]