[{"data":1,"prerenderedAt":432},["ShallowReactive",2],{"navigation_docs_en":3,"\u002Fen\u002Fai-engineering\u002Fevaluate-ai-systems\u002Fch04":141,"\u002Fen\u002Fai-engineering\u002Fevaluate-ai-systems\u002Fch04-surround":427},[4],{"title":5,"icon":6,"path":7,"stem":8,"children":9,"page":45},"AI Engineering",null,"\u002Fen\u002Fai-engineering","en\u002F1.ai-engineering",[10,46,77,114],{"title":11,"icon":12,"path":13,"stem":14,"children":15,"page":45},"Introduction to Building AI Applications with Foundation Models","i-lucide-brain-circuit","\u002Fen\u002Fai-engineering\u002Fintro","en\u002F1.ai-engineering\u002F1.intro",[16,20,25,30,35,40],{"title":11,"path":17,"stem":18,"icon":19},"\u002Fen\u002Fai-engineering\u002Fintro\u002Fch01","en\u002F1.ai-engineering\u002F1.intro\u002Fch01","i-lucide-sparkles",{"title":21,"path":22,"stem":23,"icon":24},"The Rise of AI Engineering","\u002Fen\u002Fai-engineering\u002Fintro\u002Fch011-the-rise-of-ai-engineering","en\u002F1.ai-engineering\u002F1.intro\u002Fch011-the-rise-of-ai-engineering","i-lucide-history",{"title":26,"path":27,"stem":28,"icon":29},"Foundation Model Use Cases","\u002Fen\u002Fai-engineering\u002Fintro\u002Fch012-foundation-model-use-cases","en\u002F1.ai-engineering\u002F1.intro\u002Fch012-foundation-model-use-cases","i-lucide-layout-grid",{"title":31,"path":32,"stem":33,"icon":34},"Planning AI Applications","\u002Fen\u002Fai-engineering\u002Fintro\u002Fch013-planning-ai-applications","en\u002F1.ai-engineering\u002F1.intro\u002Fch013-planning-ai-applications","i-lucide-clipboard-list",{"title":36,"path":37,"stem":38,"icon":39},"The AI Engineering Stack","\u002Fen\u002Fai-engineering\u002Fintro\u002Fch014-the-ai-engineering-stack","en\u002F1.ai-engineering\u002F1.intro\u002Fch014-the-ai-engineering-stack","i-lucide-layers",{"title":41,"path":42,"stem":43,"icon":44},"Summary","\u002Fen\u002Fai-engineering\u002Fintro\u002Fch015-summary","en\u002F1.ai-engineering\u002F1.intro\u002Fch015-summary","i-lucide-flag",false,{"title":47,"icon":6,"path":48,"stem":49,"children":50,"page":45},"Understanding Foundation Models","\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models","en\u002F1.ai-engineering\u002F2.understanding-foundation-models",[51,54,59,64,69,74],{"title":47,"path":52,"stem":53,"icon":12},"\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models\u002Fch02","en\u002F1.ai-engineering\u002F2.understanding-foundation-models\u002Fch02",{"title":55,"path":56,"stem":57,"icon":58},"Training Data","\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models\u002Fch02-1-training-data","en\u002F1.ai-engineering\u002F2.understanding-foundation-models\u002Fch02-1-training-data","i-lucide-database",{"title":60,"path":61,"stem":62,"icon":63},"Modeling","\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models\u002Fch02-2-modeling","en\u002F1.ai-engineering\u002F2.understanding-foundation-models\u002Fch02-2-modeling","i-lucide-network",{"title":65,"path":66,"stem":67,"icon":68},"Post-Training","\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models\u002Fch02-3-post-training","en\u002F1.ai-engineering\u002F2.understanding-foundation-models\u002Fch02-3-post-training","i-lucide-sliders-horizontal",{"title":70,"path":71,"stem":72,"icon":73},"Sampling","\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models\u002Fch02-4-sampling","en\u002F1.ai-engineering\u002F2.understanding-foundation-models\u002Fch02-4-sampling","i-lucide-dices",{"title":41,"path":75,"stem":76,"icon":44},"\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models\u002Fch02-5-summary","en\u002F1.ai-engineering\u002F2.understanding-foundation-models\u002Fch02-5-summary",{"title":78,"path":79,"stem":80,"children":81,"page":45},"Evaluation Methodology","\u002Fen\u002Fai-engineering\u002Fevaluation-methodology","en\u002F1.ai-engineering\u002F3.evaluation-methodology",[82,86,91,96,101,106,111],{"title":78,"path":83,"stem":84,"icon":85},"\u002Fen\u002Fai-engineering\u002Fevaluation-methodology\u002Fch03","en\u002F1.ai-engineering\u002F3.evaluation-methodology\u002Fch03","i-lucide-clipboard-check",{"title":87,"path":88,"stem":89,"icon":90},"Challenges of Evaluating Foundation Models","\u002Fen\u002Fai-engineering\u002Fevaluation-methodology\u002Fch03-1-challenges-of-evaluating-foundation-models","en\u002F1.ai-engineering\u002F3.evaluation-methodology\u002Fch03-1-challenges-of-evaluating-foundation-models","i-lucide-shield-alert",{"title":92,"path":93,"stem":94,"icon":95},"Understanding Language Modeling Metrics","\u002Fen\u002Fai-engineering\u002Fevaluation-methodology\u002Fch03-2-understanding-language-modeling-metrics","en\u002F1.ai-engineering\u002F3.evaluation-methodology\u002Fch03-2-understanding-language-modeling-metrics","i-lucide-sigma",{"title":97,"path":98,"stem":99,"icon":100},"Exact Evaluation","\u002Fen\u002Fai-engineering\u002Fevaluation-methodology\u002Fch03-3-exact-evaluation","en\u002F1.ai-engineering\u002F3.evaluation-methodology\u002Fch03-3-exact-evaluation","i-lucide-check-check",{"title":102,"path":103,"stem":104,"icon":105},"AI as a Judge","\u002Fen\u002Fai-engineering\u002Fevaluation-methodology\u002Fch03-4-ai-as-a-judge","en\u002F1.ai-engineering\u002F3.evaluation-methodology\u002Fch03-4-ai-as-a-judge","i-lucide-scale",{"title":107,"path":108,"stem":109,"icon":110},"Ranking Models with Comparative Evaluation","\u002Fen\u002Fai-engineering\u002Fevaluation-methodology\u002Fch03-5-ranking-models-with-comparative-evaluation","en\u002F1.ai-engineering\u002F3.evaluation-methodology\u002Fch03-5-ranking-models-with-comparative-evaluation","i-lucide-trophy",{"title":41,"path":112,"stem":113,"icon":44},"\u002Fen\u002Fai-engineering\u002Fevaluation-methodology\u002Fch03-6-summary","en\u002F1.ai-engineering\u002F3.evaluation-methodology\u002Fch03-6-summary",{"title":115,"icon":116,"path":117,"stem":118,"children":119,"page":45},"Evaluate AI Systems","i-lucide-binary","\u002Fen\u002Fai-engineering\u002Fevaluate-ai-systems","en\u002F1.ai-engineering\u002F4.evaluate-ai-systems",[120,123,128,133,138],{"title":115,"path":121,"stem":122,"icon":116},"\u002Fen\u002Fai-engineering\u002Fevaluate-ai-systems\u002Fch04","en\u002F1.ai-engineering\u002F4.evaluate-ai-systems\u002Fch04",{"title":124,"path":125,"stem":126,"icon":127},"Evaluation Criteria","\u002Fen\u002Fai-engineering\u002Fevaluate-ai-systems\u002Fch04-1-evaluation-criteria","en\u002F1.ai-engineering\u002F4.evaluate-ai-systems\u002Fch04-1-evaluation-criteria","i-lucide-check-circle-2",{"title":129,"path":130,"stem":131,"icon":132},"Model Selection","\u002Fen\u002Fai-engineering\u002Fevaluate-ai-systems\u002Fch04-2-model-selection","en\u002F1.ai-engineering\u002F4.evaluate-ai-systems\u002Fch04-2-model-selection","i-lucide-cpu",{"title":134,"path":135,"stem":136,"icon":137},"Design Your Evaluation Pipeline","\u002Fen\u002Fai-engineering\u002Fevaluate-ai-systems\u002Fch04-3-design-your-evaluation-pipeline","en\u002F1.ai-engineering\u002F4.evaluate-ai-systems\u002Fch04-3-design-your-evaluation-pipeline","i-lucide-workflow",{"title":41,"path":139,"stem":140,"icon":44},"\u002Fen\u002Fai-engineering\u002Fevaluate-ai-systems\u002Fch04-4-summary","en\u002F1.ai-engineering\u002F4.evaluate-ai-systems\u002Fch04-4-summary",{"id":142,"title":115,"body":143,"description":421,"extension":422,"links":6,"meta":423,"navigation":424,"path":121,"seo":425,"stem":122,"__hash__":426},"docs_en\u002Fen\u002F1.ai-engineering\u002F4.evaluate-ai-systems\u002Fch04.md",{"type":144,"value":145,"toc":408},"minimark",[146,161,166,170,178,182,185,241,245,248,255,262,288,292],[147,148,149,153],"u-page-hero",{},[150,151,115],"template",{"v-slot:title":152},"",[150,154,155,156,160],{"v-slot:description":152},"A model is only useful if it works for its intended purpose. While Chapter 3 establishes the foundational methods of automatic evaluation, this chapter focuses on how to apply them directly to ",[157,158,159],"strong",{},"evaluating models for your specific applications",".",[162,163,165],"h2",{"id":164},"the-application-first-evaluation-mindset","The Application-First Evaluation Mindset",[167,168,169],"p",{},"Models cannot be evaluated in isolation. A model that scores exceptionally well on a generic benchmark may still fail in production if it doesn't meet the precise operational, behavioral, and economic constraints of your use case.",[171,172,173,174,177],"note",{},"You must evaluate models in the ",[157,175,176],{},"context of your application",". The goal is not identifying an abstract \"best overall model,\" but finding the model that reliably satisfies your domain requirements within acceptable latency and cost boundaries.",[162,179,181],{"id":180},"three-pillars-of-system-evaluation","Three Pillars of System Evaluation",[167,183,184],{},"This chapter breaks down system evaluation into three structured components:",[186,187,188,208,224],"card-group",{},[189,190,192,193,196,197,200,201,204,205,160],"card",{"icon":127,"title":191},"1. Defining Evaluation Criteria","How to establish concrete metrics for your application: ",[157,194,195],{},"domain-specific capability",", ",[157,198,199],{},"generation capability"," (including factual consistency and hallucination detection), ",[157,202,203],{},"instruction-following fidelity",", and ",[157,206,207],{},"cost & latency budgets",[189,209,211,212,215,216,219,220,223],{"icon":132,"title":210},"2. Model Selection & Benchmarks","Navigating thousands of candidate models: evaluating ",[157,213,214],{},"benchmark credibility",", interpreting ",[157,217,218],{},"public leaderboards",", and resolving the ",[157,221,222],{},"self-hosting vs. proprietary model API"," trade-off.",[189,225,227,228,196,231,196,234,237,238,160],{"icon":137,"title":226},"3. Evaluation Pipelines","Architecting an automated evaluation harness that guides your system over time: ",[157,229,230],{},"curating evaluation datasets",[157,232,233],{},"aggregating metrics",[157,235,236],{},"slice-based evaluation",", and implementing ",[157,239,240],{},"continuous CI\u002FCD evaluation",[162,242,244],{"id":243},"the-model-selection-dilemma","The Model Selection Dilemma",[167,246,247],{},"With the rapid expansion of the foundation model landscape, engineering teams face critical architectural questions when choosing candidate models:",[249,250,251,252,160],"warning",{},"Thousands of benchmarks have been introduced to evaluate models across diverse criteria. However, public benchmarks and aggregate leaderboards often suffer from ",[157,253,254],{},"data contamination, narrow task alignment, and misleading score inflation",[167,256,257,258,261],{},"A question many teams must revisit repeatedly is ",[157,259,260],{},"deployment architecture",":",[186,263,264,272,280],{},[189,265,268,269,160],{"icon":266,"title":267},"i-lucide-server","Self-Hosted Models","Deploying open source models on private infrastructure for ",[157,270,271],{},"maximum privacy, latency control, and custom fine-tuning",[189,273,276,277,160],{"icon":274,"title":275},"i-lucide-cloud","Proprietary APIs","Consuming frontier closed models via API for ",[157,278,279],{},"rapid iteration, zero infrastructure overhead, and cutting-edge reasoning",[189,281,284,285,160],{"icon":282,"title":283},"i-lucide-boxes","Hosted Open Source APIs","Leveraging third-party providers hosting open weights — combining ",[157,286,287],{},"open-source flexibility with serverless API convenience",[162,289,291],{"id":290},"detailed-chapter-structure-subsections","Detailed Chapter Structure & Subsections",[293,294,296,304,307,335,341,344,364,370,373,399,405],"steps",{"level":295},"3",[297,298,300],"h3",{"id":299},"evaluation-criteria",[301,302,124],"a",{"href":303},".\u002Fch04-1-evaluation-criteria",[167,305,306],{},"Core qualitative and quantitative dimensions for assessing model outputs:",[308,309,310,317,323,329],"ul",{},[311,312,313,316],"li",{},[157,314,315],{},"Domain-Specific Capability",": Measuring expertise in math, coding, science, summarization, and specialized domain knowledge.",[311,318,319,322],{},[157,320,321],{},"Generation Capability",": Assessing factual consistency, detecting hallucinations, and scoring fluency and coherence.",[311,324,325,328],{},[157,326,327],{},"Instruction-Following Capability",": Testing negative constraints, format adherence (JSON, YAML), and complex system prompts.",[311,330,331,334],{},[157,332,333],{},"Cost and Latency",": Analyzing token economics, time-to-first-token (TTFT), throughput, and price-performance trade-offs.",[297,336,338],{"id":337},"model-selection",[301,339,129],{"href":340},".\u002Fch04-2-model-selection",[167,342,343],{},"Selecting the optimal foundation model for your workload:",[308,345,346,352,358],{},[311,347,348,351],{},[157,349,350],{},"Benchmarks",": Understanding standardized suites (MMLU, GSM8k, HumanEval), benchmark saturation, and data leakage.",[311,353,354,357],{},[157,355,356],{},"Public Leaderboards",": Interpreting crowd-sourced Elo systems (LMSYS Chatbot Arena) and static automated leaderboards.",[311,359,360,363],{},[157,361,362],{},"Self-Hosting Versus Model API",": Comparing total cost of ownership (TCO), governance, operational complexity, and vendor lock-in.",[297,365,367],{"id":366},"design-your-evaluation-pipeline",[301,368,134],{"href":369},".\u002Fch04-3-design-your-evaluation-pipeline",[167,371,372],{},"Engineering an automated, reliable evaluation infrastructure:",[308,374,375,381,387,393],{},[311,376,377,380],{},[157,378,379],{},"Create an Evaluation Dataset",": Assembling gold-standard datasets, synthetic generation, and hard edge-case curation.",[311,382,383,386],{},[157,384,385],{},"Aggregate Evaluation Results",": Combining diverse signals, computing micro vs. macro scores, and setting release gates.",[311,388,389,392],{},[157,390,391],{},"Slice-Based Evaluation",": Identifying blind spots by slicing performance across input length, user segments, and failure categories.",[311,394,395,398],{},[157,396,397],{},"Continuous Evaluation",": Automating evals within CI\u002FCD pipelines and monitoring production drifts post-deployment.",[297,400,402],{"id":401},"summary",[301,403,41],{"href":404},".\u002Fch04-4-summary",[167,406,407],{},"Synthesizing the evaluation lifecycle: transitioning from offline experimentation to production-grade reliability.",{"title":152,"searchDepth":409,"depth":409,"links":410},2,[411,412,413,414],{"id":164,"depth":409,"text":165},{"id":180,"depth":409,"text":181},{"id":243,"depth":409,"text":244},{"id":290,"depth":409,"text":291,"children":415},[416,418,419,420],{"id":299,"depth":417,"text":124},3,{"id":337,"depth":417,"text":129},{"id":366,"depth":417,"text":134},{"id":401,"depth":417,"text":41},"How to define evaluation criteria, navigate benchmarks for model selection, and architect production evaluation pipelines for AI applications.","md",{},{"icon":116},{"title":115,"description":421},"joShCbmcEog11uiIyepqGCJS59xiTTXtinFk4PmSakc",[428,430],{"title":41,"path":112,"stem":113,"description":429,"icon":44,"children":-1},"A recap of why foundation-model evaluation is hard and how language-modeling metrics, exact methods, AI as a judge, and comparative ranking fit together.",{"title":124,"path":125,"stem":126,"description":431,"icon":127,"children":-1},"How to define and calculate criteria for evaluating AI applications, including domain capabilities, factual consistency, safety, instruction-following, and cost-latency tradeoffs.",1789413992178]