[{"data":1,"prerenderedAt":4210},["ShallowReactive",2],{"navigation_docs_en":3,"\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models\u002Fch02-4-sampling":77,"\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models\u002Fch02-4-sampling-surround":4205},[4],{"title":5,"icon":6,"path":7,"stem":8,"children":9,"page":45},"AI Engineering",null,"\u002Fen\u002Fai-engineering","en\u002F1.ai-engineering",[10,46],{"title":11,"icon":12,"path":13,"stem":14,"children":15,"page":45},"Introduction to Building AI Applications with Foundation Models","i-lucide-brain-circuit","\u002Fen\u002Fai-engineering\u002Fintro","en\u002F1.ai-engineering\u002F1.intro",[16,20,25,30,35,40],{"title":11,"path":17,"stem":18,"icon":19},"\u002Fen\u002Fai-engineering\u002Fintro\u002Fch01","en\u002F1.ai-engineering\u002F1.intro\u002Fch01","i-lucide-sparkles",{"title":21,"path":22,"stem":23,"icon":24},"The Rise of AI Engineering","\u002Fen\u002Fai-engineering\u002Fintro\u002Fch011-the-rise-of-ai-engineering","en\u002F1.ai-engineering\u002F1.intro\u002Fch011-the-rise-of-ai-engineering","i-lucide-history",{"title":26,"path":27,"stem":28,"icon":29},"Foundation Model Use Cases","\u002Fen\u002Fai-engineering\u002Fintro\u002Fch012-foundation-model-use-cases","en\u002F1.ai-engineering\u002F1.intro\u002Fch012-foundation-model-use-cases","i-lucide-layout-grid",{"title":31,"path":32,"stem":33,"icon":34},"Planning AI Applications","\u002Fen\u002Fai-engineering\u002Fintro\u002Fch013-planning-ai-applications","en\u002F1.ai-engineering\u002F1.intro\u002Fch013-planning-ai-applications","i-lucide-clipboard-list",{"title":36,"path":37,"stem":38,"icon":39},"The AI Engineering Stack","\u002Fen\u002Fai-engineering\u002Fintro\u002Fch014-the-ai-engineering-stack","en\u002F1.ai-engineering\u002F1.intro\u002Fch014-the-ai-engineering-stack","i-lucide-layers",{"title":41,"path":42,"stem":43,"icon":44},"Summary","\u002Fen\u002Fai-engineering\u002Fintro\u002Fch015-summary","en\u002F1.ai-engineering\u002F1.intro\u002Fch015-summary","i-lucide-flag",false,{"title":47,"icon":6,"path":48,"stem":49,"children":50,"page":45},"Understanding Foundation Models","\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models","en\u002F1.ai-engineering\u002F2.understanding-foundation-models",[51,54,59,64,69,74],{"title":47,"path":52,"stem":53,"icon":12},"\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models\u002Fch02","en\u002F1.ai-engineering\u002F2.understanding-foundation-models\u002Fch02",{"title":55,"path":56,"stem":57,"icon":58},"Training Data","\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models\u002Fch02-1-training-data","en\u002F1.ai-engineering\u002F2.understanding-foundation-models\u002Fch02-1-training-data","i-lucide-database",{"title":60,"path":61,"stem":62,"icon":63},"Modeling","\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models\u002Fch02-2-modeling","en\u002F1.ai-engineering\u002F2.understanding-foundation-models\u002Fch02-2-modeling","i-lucide-network",{"title":65,"path":66,"stem":67,"icon":68},"Post-Training","\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models\u002Fch02-3-post-training","en\u002F1.ai-engineering\u002F2.understanding-foundation-models\u002Fch02-3-post-training","i-lucide-sliders-horizontal",{"title":70,"path":71,"stem":72,"icon":73},"Sampling","\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models\u002Fch02-4-sampling","en\u002F1.ai-engineering\u002F2.understanding-foundation-models\u002Fch02-4-sampling","i-lucide-dices",{"title":41,"path":75,"stem":76,"icon":44},"\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models\u002Fch02-5-summary","en\u002F1.ai-engineering\u002F2.understanding-foundation-models\u002Fch02-5-summary",{"id":78,"title":70,"body":79,"description":4199,"extension":4200,"links":6,"meta":4201,"navigation":4202,"path":71,"seo":4203,"stem":72,"__hash__":4204},"docs_en\u002Fen\u002F1.ai-engineering\u002F2.understanding-foundation-models\u002Fch02-4-sampling.md",{"type":80,"value":81,"toc":4153},"minimark",[82,97,102,111,115,118,122,125,144,151,156,160,167,179,182,186,193,199,203,206,210,620,1100,1104,1107,1110,1114,1117,1123,1446,1451,1625,1912,1925,1928,1934,1943,1946,1957,1960,1964,1980,1998,2001,2007,2011,2014,2026,2030,2486,2510,2513,2517,2520,2532,2541,2544,2547,2553,2557,2560,2569,2573,2585,2605,2608,2611,2615,2618,2625,2647,2658,2664,2668,2671,2675,2678,2718,2946,2949,3182,3193,3197,3217,3226,3229,3238,3242,3245,3254,3257,3263,3274,3278,3281,3297,3309,3321,3325,3328,3343,3346,3349,3353,3356,3361,3369,3374,3380,3385,3398,3402,3408,3412,3421,3424,3428,3454,3465,3468,3475,3481,3485,3489,3492,3495,3498,3502,3514,3517,3520,3524,3527,3536,3539,3542,3546,3552,3555,3561,3565,3578,3586,3589,3593,3605,3621,3627,3631,3634,3637,3641,3648,3655,3658,3669,3672,3683,3686,3705,3708,3711,3723,3726,3732,3736,3739,3743,3746,3765,3768,3771,3774,3777,3780,3789,3815,3818,3821,3824,3827,3831,3840,3843,3849,3852,3858,3862,3871,3877,3881,3884,3902,3906,3915,3918,3927,3938,3941,3944,3950,3960,3963,3966,3969,4149],[83,84,85,89],"u-page-hero",{},[86,87,70],"template",{"v-slot:title":88},"",[86,90,91,92,96],{"v-slot:description":88},"A model constructs its outputs through ",[93,94,95],"strong",{},"sampling",". Sampling strategies and variables such as temperature, top-k, and top-p shape creativity, consistency, cost, structured outputs, and hallucination behavior.",[98,99,101],"h2",{"id":100},"what-sampling-controls","What Sampling Controls",[103,104,105,106,110],"p",{},"This section discusses different sampling strategies and ",[107,108,109],"em",{},"sampling variables",", including temperature, top-k, and top-p. It'll then explore how to sample multiple outputs to improve a model's performance. We'll also see how the sampling process can be modified to get models to generate responses that follow certain formats and constraints.",[112,113,114],"note",{},"Sampling makes AI's outputs probabilistic. Understanding this probabilistic nature is important for handling AI's behaviors, such as inconsistency and hallucination.",[103,116,117],{},"This section ends with a deep dive into what this probabilistic nature means and how to work with it.",[98,119,121],{"id":120},"sampling-fundamentals","Sampling Fundamentals",[103,123,124],{},"Given an input, a neural network produces an output by first computing the probabilities of possible outcomes.",[126,127,128,134,139],"card-group",{},[129,130,133],"card",{"icon":131,"title":132},"i-lucide-tags","Classification Model","Possible outcomes are the available classes. If a model is trained to classify whether an email is spam or not, there are only two possible outcomes: spam and not spam.",[129,135,138],{"icon":136,"title":137},"i-lucide-mail-warning","Decision Threshold","The model might compute that the probability of the email being spam is 90%, and not spam is 10%. If any email with a spam probability higher than 50% should be marked as spam, an email with a 90% spam probability will be marked as spam.",[129,140,143],{"icon":141,"title":142},"i-lucide-languages","Language Model","To generate the next token, the model first computes the probability distribution over all tokens in the vocabulary.",[103,145,146],{},[147,148],"img",{"alt":149,"src":150},"Figure 2-14. To generate the next token, the language model first computes the probability distribution over all tokens in the vocabulary.",".\u002Fmedia\u002Ffig-2-14.png",[152,153,154],"blockquote",{},[103,155,149],{},[98,157,159],{"id":158},"greedy-sampling-versus-distribution-sampling","Greedy Sampling Versus Distribution Sampling",[103,161,162,163,166],{},"When working with possible outcomes of different probabilities, a common strategy is to pick the outcome with the highest probability. Always picking the most likely outcome is called ",[107,164,165],{},"greedy sampling",".",[126,168,169,174],{},[129,170,173],{"icon":171,"title":172},"i-lucide-check","Works for Classification","If the model thinks that an email is more likely to be spam than not spam, it makes sense to mark it as spam.",[129,175,178],{"icon":176,"title":177},"i-lucide-snooze","Boring for Language","For a language model, greedy sampling creates boring outputs. Imagine a model that, for whatever question you ask, always responds with the most common words.",[103,180,181],{},"Instead of always picking the next most likely token, the model can sample the next token according to the probability distribution over all possible values. Given the context of \"My favorite color is ...\" as shown in Figure 2-14, if \"red\" has a 30% chance of being the next token and \"green\" has a 50% chance, \"red\" will be picked 30% of the time, and \"green\" 50% of the time.",[98,183,185],{"id":184},"logits-probabilities-and-softmax","Logits, Probabilities, and Softmax",[103,187,188,189,192],{},"How does a model compute these probabilities? Given an input, a neural network outputs a logit vector. Each ",[107,190,191],{},"logit"," corresponds to one possible value. In the case of a language model, each logit corresponds to one token in the model's vocabulary. The logit vector size is the size of the vocabulary.",[103,194,195],{},[147,196],{"alt":197,"src":198},"Figure 2-15. For each input, a language model produces a logit vector. Each logit corresponds to a token in the vocabulary.",".\u002Fmedia\u002Ffig-2-15.png",[152,200,201],{},[103,202,197],{},[103,204,205],{},"While larger logits correspond to higher probabilities, logits don't represent probabilities. Logits don't sum up to one. Logits can even be negative, while probabilities have to be non-negative.",[207,208,209],"tip",{},"To convert logits to probabilities, a softmax layer is often used.",[103,211,212,213,470,471,546,547,619],{},"Let's say the model has a vocabulary of N and the logit vector is ",[214,215,218,281],"span",{"className":216},[217],"katex",[214,219,222],{"className":220},[221],"katex-mathml",[223,224,226],"math",{"xmlns":225},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML",[227,228,229,276],"semantics",{},[230,231,232,237,248,252,259,261,264,266,273],"mrow",{},[233,234,236],"mo",{"stretchy":235},"false","[",[238,239,240,244],"msub",{},[241,242,243],"mi",{},"x",[245,246,247],"mn",{},"1",[233,249,251],{"separator":250},"true",",",[238,253,254,256],{},[241,255,243],{},[245,257,258],{},"2",[233,260,251],{"separator":250},[233,262,263],{},"…",[233,265,251],{"separator":250},[238,267,268,270],{},[241,269,243],{},[241,271,272],{},"N",[233,274,275],{"stretchy":235},"]",[277,278,280],"annotation",{"encoding":279},"application\u002Fx-tex","[x_1, x_2, \\ldots, x_N]",[214,282,285],{"className":283,"ariaHidden":250},[284],"katex-html",[214,286,289,294,298,356,360,365,405,408,411,415,418,421,424,466],{"className":287},[288],"base",[214,290],{"className":291,"style":293},[292],"strut","height:1em;vertical-align:-0.25em;",[214,295,236],{"className":296},[297],"mopen",[214,299,302,306],{"className":300},[301],"mord",[214,303,243],{"className":304},[301,305],"mathnormal",[214,307,310],{"className":308},[309],"msupsub",[214,311,315,347],{"className":312},[313,314],"vlist-t","vlist-t2",[214,316,319,342],{"className":317},[318],"vlist-r",[214,320,324],{"className":321,"style":323},[322],"vlist","height:0.3011em;",[214,325,327,332],{"style":326},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[214,328],{"className":329,"style":331},[330],"pstrut","height:2.7em;",[214,333,339],{"className":334},[335,336,337,338],"sizing","reset-size6","size3","mtight",[214,340,247],{"className":341},[301,338],[214,343,346],{"className":344},[345],"vlist-s","​",[214,348,350],{"className":349},[318],[214,351,354],{"className":352,"style":353},[322],"height:0.15em;",[214,355],{},[214,357,251],{"className":358},[359],"mpunct",[214,361],{"className":362,"style":364},[363],"mspace","margin-right:0.1667em;",[214,366,368,371],{"className":367},[301],[214,369,243],{"className":370},[301,305],[214,372,374],{"className":373},[309],[214,375,377,397],{"className":376},[313,314],[214,378,380,394],{"className":379},[318],[214,381,383],{"className":382,"style":323},[322],[214,384,385,388],{"style":326},[214,386],{"className":387,"style":331},[330],[214,389,391],{"className":390},[335,336,337,338],[214,392,258],{"className":393},[301,338],[214,395,346],{"className":396},[345],[214,398,400],{"className":399},[318],[214,401,403],{"className":402,"style":353},[322],[214,404],{},[214,406,251],{"className":407},[359],[214,409],{"className":410,"style":364},[363],[214,412,263],{"className":413},[414],"minner",[214,416],{"className":417,"style":364},[363],[214,419,251],{"className":420},[359],[214,422],{"className":423,"style":364},[363],[214,425,427,430],{"className":426},[301],[214,428,243],{"className":429},[301,305],[214,431,433],{"className":432},[309],[214,434,436,458],{"className":435},[313,314],[214,437,439,455],{"className":438},[318],[214,440,443],{"className":441,"style":442},[322],"height:0.3283em;",[214,444,445,448],{"style":326},[214,446],{"className":447,"style":331},[330],[214,449,451],{"className":450},[335,336,337,338],[214,452,272],{"className":453,"style":454},[301,305,338],"margin-right:0.109em;",[214,456,346],{"className":457},[345],[214,459,461],{"className":460},[318],[214,462,464],{"className":463,"style":353},[322],[214,465],{},[214,467,275],{"className":468},[469],"mclose",". The probability for the ",[214,472,474,500],{"className":473},[217],[214,475,477],{"className":476},[221],[223,478,479],{"xmlns":225},[227,480,481,497],{},[230,482,483],{},[484,485,486,489],"msup",{},[241,487,488],{},"i",[230,490,491,494],{},[241,492,493],{},"t",[241,495,496],{},"h",[277,498,499],{"encoding":279},"i^{th}",[214,501,503],{"className":502,"ariaHidden":250},[284],[214,504,506,510],{"className":505},[288],[214,507],{"className":508,"style":509},[292],"height:0.8491em;",[214,511,513,516],{"className":512},[301],[214,514,488],{"className":515},[301,305],[214,517,519],{"className":518},[309],[214,520,522],{"className":521},[313],[214,523,525],{"className":524},[318],[214,526,528],{"className":527,"style":509},[322],[214,529,531,534],{"style":530},"top:-3.063em;margin-right:0.05em;",[214,532],{"className":533,"style":331},[330],[214,535,537],{"className":536},[335,336,337,338],[214,538,540,543],{"className":539},[301,338],[214,541,493],{"className":542},[301,305,338],[214,544,496],{"className":545},[301,305,338]," token, ",[214,548,550,568],{"className":549},[217],[214,551,553],{"className":552},[221],[223,554,555],{"xmlns":225},[227,556,557,565],{},[230,558,559],{},[238,560,561,563],{},[241,562,103],{},[241,564,488],{},[277,566,567],{"encoding":279},"p_i",[214,569,571],{"className":570,"ariaHidden":250},[284],[214,572,574,578],{"className":573},[288],[214,575],{"className":576,"style":577},[292],"height:0.625em;vertical-align:-0.1944em;",[214,579,581,584],{"className":580},[301],[214,582,103],{"className":583},[301,305],[214,585,587],{"className":586},[309],[214,588,590,611],{"className":589},[313,314],[214,591,593,608],{"className":592},[318],[214,594,597],{"className":595,"style":596},[322],"height:0.3117em;",[214,598,599,602],{"style":326},[214,600],{"className":601,"style":331},[330],[214,603,605],{"className":604},[335,336,337,338],[214,606,488],{"className":607},[301,305,338],[214,609,346],{"className":610},[345],[214,612,614],{"className":613},[318],[214,615,617],{"className":616,"style":353},[322],[214,618],{},", is computed as follows:",[103,621,622],{},[214,623,625,698],{"className":624},[217],[214,626,628],{"className":627},[221],[223,629,630],{"xmlns":225},[227,631,632,695],{},[230,633,634,640,643,647,650,656,659,661],{},[238,635,636,638],{},[241,637,103],{},[241,639,488],{},[233,641,642],{},"=",[644,645,646],"mtext",{},"softmax",[233,648,649],{"stretchy":235},"(",[238,651,652,654],{},[241,653,243],{},[241,655,488],{},[233,657,658],{"stretchy":235},")",[233,660,642],{},[662,663,664,675],"mfrac",{},[484,665,666,669],{},[241,667,668],{},"e",[238,670,671,673],{},[241,672,243],{},[241,674,488],{},[230,676,677,685],{},[238,678,679,682],{},[233,680,681],{},"∑",[241,683,684],{},"j",[484,686,687,689],{},[241,688,668],{},[238,690,691,693],{},[241,692,243],{},[241,694,684],{},[277,696,697],{"encoding":279},"p_i = \\text{softmax}(x_i) = \\frac{e^{x_i}}{\\sum_j e^{x_j}}",[214,699,701,758,826],{"className":700,"ariaHidden":250},[284],[214,702,704,707,747,751,755],{"className":703},[288],[214,705],{"className":706,"style":577},[292],[214,708,710,713],{"className":709},[301],[214,711,103],{"className":712},[301,305],[214,714,716],{"className":715},[309],[214,717,719,739],{"className":718},[313,314],[214,720,722,736],{"className":721},[318],[214,723,725],{"className":724,"style":596},[322],[214,726,727,730],{"style":326},[214,728],{"className":729,"style":331},[330],[214,731,733],{"className":732},[335,336,337,338],[214,734,488],{"className":735},[301,305,338],[214,737,346],{"className":738},[345],[214,740,742],{"className":741},[318],[214,743,745],{"className":744,"style":353},[322],[214,746],{},[214,748],{"className":749,"style":750},[363],"margin-right:0.2778em;",[214,752,642],{"className":753},[754],"mrel",[214,756],{"className":757,"style":750},[363],[214,759,761,764,771,774,814,817,820,823],{"className":760},[288],[214,762],{"className":763,"style":293},[292],[214,765,768],{"className":766},[301,767],"text",[214,769,646],{"className":770},[301],[214,772,649],{"className":773},[297],[214,775,777,780],{"className":776},[301],[214,778,243],{"className":779},[301,305],[214,781,783],{"className":782},[309],[214,784,786,806],{"className":785},[313,314],[214,787,789,803],{"className":788},[318],[214,790,792],{"className":791,"style":596},[322],[214,793,794,797],{"style":326},[214,795],{"className":796,"style":331},[330],[214,798,800],{"className":799},[335,336,337,338],[214,801,488],{"className":802},[301,305,338],[214,804,346],{"className":805},[345],[214,807,809],{"className":808},[318],[214,810,812],{"className":811,"style":353},[322],[214,813],{},[214,815,658],{"className":816},[469],[214,818],{"className":819,"style":750},[363],[214,821,642],{"className":822},[754],[214,824],{"className":825,"style":750},[363],[214,827,829,833],{"className":828},[288],[214,830],{"className":831,"style":832},[292],"height:1.5885em;vertical-align:-0.6775em;",[214,834,836,840,1097],{"className":835},[301],[214,837],{"className":838},[297,839],"nulldelimiter",[214,841,843],{"className":842},[662],[214,844,846,1088],{"className":845},[313,314],[214,847,849,1085],{"className":848},[318],[214,850,853,993,1004],{"className":851,"style":852},[322],"height:0.911em;",[214,854,856,860],{"style":855},"top:-2.6447em;",[214,857],{"className":858,"style":859},[330],"height:3em;",[214,861,863],{"className":862},[335,336,337,338],[214,864,866,917,921],{"className":865},[301,338],[214,867,870,876],{"className":868},[869,338],"mop",[214,871,681],{"className":872,"style":875},[869,873,874,338],"op-symbol","small-op","position:relative;top:0em;",[214,877,879],{"className":878},[309],[214,880,882,908],{"className":881},[313,314],[214,883,885,905],{"className":884},[318],[214,886,889],{"className":887,"style":888},[322],"height:0.1496em;",[214,890,892,896],{"style":891},"top:-2.1786em;margin-left:0em;margin-right:0.0714em;",[214,893],{"className":894,"style":895},[330],"height:2.5em;",[214,897,901],{"className":898},[335,899,900,338],"reset-size3","size1",[214,902,684],{"className":903,"style":904},[301,305,338],"margin-right:0.0572em;",[214,906,346],{"className":907},[345],[214,909,911],{"className":910},[318],[214,912,915],{"className":913,"style":914},[322],"height:0.4603em;",[214,916],{},[214,918],{"className":919,"style":920},[363,338],"margin-right:0.1952em;",[214,922,924,927],{"className":923},[301,338],[214,925,668],{"className":926},[301,305,338],[214,928,930],{"className":929},[309],[214,931,933],{"className":932},[313],[214,934,936],{"className":935},[318],[214,937,940],{"className":938,"style":939},[322],"height:0.779em;",[214,941,943,946],{"style":942},"top:-2.9714em;margin-right:0.0714em;",[214,944],{"className":945,"style":895},[330],[214,947,949],{"className":948},[335,899,900,338],[214,950,952],{"className":951},[301,338],[214,953,955,958],{"className":954},[301,338],[214,956,243],{"className":957},[301,305,338],[214,959,961],{"className":960},[309],[214,962,964,984],{"className":963},[313,314],[214,965,967,981],{"className":966},[318],[214,968,971],{"className":969,"style":970},[322],"height:0.3448em;",[214,972,974,978],{"style":973},"top:-2.3448em;margin-left:0em;margin-right:0.1em;",[214,975],{"className":976,"style":977},[330],"height:2.6595em;",[214,979,684],{"className":980,"style":904},[301,305,338],[214,982,346],{"className":983},[345],[214,985,987],{"className":986},[318],[214,988,991],{"className":989,"style":990},[322],"height:0.5092em;",[214,992],{},[214,994,996,999],{"style":995},"top:-3.23em;",[214,997],{"className":998,"style":859},[330],[214,1000],{"className":1001,"style":1003},[1002],"frac-line","border-bottom-width:0.04em;",[214,1005,1007,1010],{"style":1006},"top:-3.394em;",[214,1008],{"className":1009,"style":859},[330],[214,1011,1013],{"className":1012},[335,336,337,338],[214,1014,1016],{"className":1015},[301,338],[214,1017,1019,1022],{"className":1018},[301,338],[214,1020,668],{"className":1021},[301,305,338],[214,1023,1025],{"className":1024},[309],[214,1026,1028],{"className":1027},[313],[214,1029,1031],{"className":1030},[318],[214,1032,1035],{"className":1033,"style":1034},[322],"height:0.7385em;",[214,1036,1038,1041],{"style":1037},"top:-2.931em;margin-right:0.0714em;",[214,1039],{"className":1040,"style":895},[330],[214,1042,1044],{"className":1043},[335,899,900,338],[214,1045,1047],{"className":1046},[301,338],[214,1048,1050,1053],{"className":1049},[301,338],[214,1051,243],{"className":1052},[301,305,338],[214,1054,1056],{"className":1055},[309],[214,1057,1059,1076],{"className":1058},[313,314],[214,1060,1062,1073],{"className":1061},[318],[214,1063,1065],{"className":1064,"style":970},[322],[214,1066,1067,1070],{"style":973},[214,1068],{"className":1069,"style":977},[330],[214,1071,488],{"className":1072},[301,305,338],[214,1074,346],{"className":1075},[345],[214,1077,1079],{"className":1078},[318],[214,1080,1083],{"className":1081,"style":1082},[322],"height:0.3147em;",[214,1084],{},[214,1086,346],{"className":1087},[345],[214,1089,1091],{"className":1090},[318],[214,1092,1095],{"className":1093,"style":1094},[322],"height:0.6775em;",[214,1096],{},[214,1098],{"className":1099},[469,839],[98,1101,1103],{"id":1102},"sampling-strategies","Sampling Strategies",[103,1105,1106],{},"The right sampling strategy can make a model generate responses more suitable for your application. One sampling strategy can make the model generate more creative responses, whereas another strategy can make its generations more predictable.",[112,1108,1109],{},"Many different sampling strategies have been introduced to nudge models toward responses with specific attributes. You can also design your own sampling strategy, though this typically requires access to the model's logits.",[98,1111,1113],{"id":1112},"temperature","Temperature",[103,1115,1116],{},"One problem with sampling the next token according to the probability distribution is that the model can be less creative. In the previous example, common colors like \"red\", \"green\", \"purple\", and so on have the highest probabilities. The language model's answer ends up sounding like that of a five-year-old: \"My favorite color is green\". Because \"the\" has a low probability, the model has a low chance of generating a creative sentence such as \"My favorite color is the color of a still lake on a spring morning\".",[207,1118,1119,1120,1122],{},"To redistribute the probabilities of the possible values, you can sample with a ",[107,1121,1112],{},". A higher temperature reduces the probabilities of common tokens and increases the probabilities of rarer tokens. This enables models to create more creative responses.",[103,1124,1125,1126,1157,1158,1226,1227,1374,1375,166],{},"Temperature is a constant used to adjust the logits before the softmax transformation. Logits are divided by temperature. For a given temperature ",[214,1127,1129,1143],{"className":1128},[217],[214,1130,1132],{"className":1131},[221],[223,1133,1134],{"xmlns":225},[227,1135,1136,1141],{},[230,1137,1138],{},[241,1139,1140],{},"T",[277,1142,1140],{"encoding":279},[214,1144,1146],{"className":1145,"ariaHidden":250},[284],[214,1147,1149,1153],{"className":1148},[288],[214,1150],{"className":1151,"style":1152},[292],"height:0.6833em;",[214,1154,1140],{"className":1155,"style":1156},[301,305],"margin-right:0.1389em;",", the adjusted logit for the ",[214,1159,1161,1182],{"className":1160},[217],[214,1162,1164],{"className":1163},[221],[223,1165,1166],{"xmlns":225},[227,1167,1168,1180],{},[230,1169,1170],{},[484,1171,1172,1174],{},[241,1173,488],{},[230,1175,1176,1178],{},[241,1177,493],{},[241,1179,496],{},[277,1181,499],{"encoding":279},[214,1183,1185],{"className":1184,"ariaHidden":250},[284],[214,1186,1188,1191],{"className":1187},[288],[214,1189],{"className":1190,"style":509},[292],[214,1192,1194,1197],{"className":1193},[301],[214,1195,488],{"className":1196},[301,305],[214,1198,1200],{"className":1199},[309],[214,1201,1203],{"className":1202},[313],[214,1204,1206],{"className":1205},[318],[214,1207,1209],{"className":1208,"style":509},[322],[214,1210,1211,1214],{"style":530},[214,1212],{"className":1213,"style":331},[330],[214,1215,1217],{"className":1216},[335,336,337,338],[214,1218,1220,1223],{"className":1219},[301,338],[214,1221,493],{"className":1222},[301,305,338],[214,1224,496],{"className":1225},[301,305,338]," token is ",[214,1228,1230,1252],{"className":1229},[217],[214,1231,1233],{"className":1232},[221],[223,1234,1235],{"xmlns":225},[227,1236,1237,1249],{},[230,1238,1239],{},[662,1240,1241,1247],{},[238,1242,1243,1245],{},[241,1244,243],{},[241,1246,488],{},[241,1248,1140],{},[277,1250,1251],{"encoding":279},"\\frac{x_i}{T}",[214,1253,1255],{"className":1254,"ariaHidden":250},[284],[214,1256,1258,1262],{"className":1257},[288],[214,1259],{"className":1260,"style":1261},[292],"height:1.0565em;vertical-align:-0.345em;",[214,1263,1265,1268,1371],{"className":1264},[301],[214,1266],{"className":1267},[297,839],[214,1269,1271],{"className":1270},[662],[214,1272,1274,1362],{"className":1273},[313,314],[214,1275,1277,1359],{"className":1276},[318],[214,1278,1281,1296,1304],{"className":1279,"style":1280},[322],"height:0.7115em;",[214,1282,1284,1287],{"style":1283},"top:-2.655em;",[214,1285],{"className":1286,"style":859},[330],[214,1288,1290],{"className":1289},[335,336,337,338],[214,1291,1293],{"className":1292},[301,338],[214,1294,1140],{"className":1295,"style":1156},[301,305,338],[214,1297,1298,1301],{"style":995},[214,1299],{"className":1300,"style":859},[330],[214,1302],{"className":1303,"style":1003},[1002],[214,1305,1307,1310],{"style":1306},"top:-3.4101em;",[214,1308],{"className":1309,"style":859},[330],[214,1311,1313],{"className":1312},[335,336,337,338],[214,1314,1316],{"className":1315},[301,338],[214,1317,1319,1322],{"className":1318},[301,338],[214,1320,243],{"className":1321},[301,305,338],[214,1323,1325],{"className":1324},[309],[214,1326,1328,1350],{"className":1327},[313,314],[214,1329,1331,1347],{"className":1330},[318],[214,1332,1335],{"className":1333,"style":1334},[322],"height:0.3281em;",[214,1336,1338,1341],{"style":1337},"top:-2.357em;margin-left:0em;margin-right:0.0714em;",[214,1339],{"className":1340,"style":895},[330],[214,1342,1344],{"className":1343},[335,899,900,338],[214,1345,488],{"className":1346},[301,305,338],[214,1348,346],{"className":1349},[345],[214,1351,1353],{"className":1352},[318],[214,1354,1357],{"className":1355,"style":1356},[322],"height:0.143em;",[214,1358],{},[214,1360,346],{"className":1361},[345],[214,1363,1365],{"className":1364},[318],[214,1366,1369],{"className":1367,"style":1368},[322],"height:0.345em;",[214,1370],{},[214,1372],{"className":1373},[469,839],". Softmax is then applied on this adjusted logit instead of on ",[214,1376,1378,1396],{"className":1377},[217],[214,1379,1381],{"className":1380},[221],[223,1382,1383],{"xmlns":225},[227,1384,1385,1393],{},[230,1386,1387],{},[238,1388,1389,1391],{},[241,1390,243],{},[241,1392,488],{},[277,1394,1395],{"encoding":279},"x_i",[214,1397,1399],{"className":1398,"ariaHidden":250},[284],[214,1400,1402,1406],{"className":1401},[288],[214,1403],{"className":1404,"style":1405},[292],"height:0.5806em;vertical-align:-0.15em;",[214,1407,1409,1412],{"className":1408},[301],[214,1410,243],{"className":1411},[301,305],[214,1413,1415],{"className":1414},[309],[214,1416,1418,1438],{"className":1417},[313,314],[214,1419,1421,1435],{"className":1420},[318],[214,1422,1424],{"className":1423,"style":596},[322],[214,1425,1426,1429],{"style":326},[214,1427],{"className":1428,"style":331},[330],[214,1430,1432],{"className":1431},[335,336,337,338],[214,1433,488],{"className":1434},[301,305,338],[214,1436,346],{"className":1437},[345],[214,1439,1441],{"className":1440},[318],[214,1442,1444],{"className":1443,"style":353},[322],[214,1445],{},[1447,1448,1450],"h3",{"id":1449},"temperature-example","Temperature Example",[103,1452,1453,1454,1506,1507,1536,1537,1566,1567,1536,1597,166],{},"Imagine that we have a model that has only two possible outputs: A and B. The logits computed from the last layer are ",[214,1455,1457,1479],{"className":1456},[217],[214,1458,1460],{"className":1459},[221],[223,1461,1462],{"xmlns":225},[227,1463,1464,1476],{},[230,1465,1466,1468,1470,1472,1474],{},[233,1467,236],{"stretchy":235},[245,1469,247],{},[233,1471,251],{"separator":250},[245,1473,258],{},[233,1475,275],{"stretchy":235},[277,1477,1478],{"encoding":279},"[1, 2]",[214,1480,1482],{"className":1481,"ariaHidden":250},[284],[214,1483,1485,1488,1491,1494,1497,1500,1503],{"className":1484},[288],[214,1486],{"className":1487,"style":293},[292],[214,1489,236],{"className":1490},[297],[214,1492,247],{"className":1493},[301],[214,1495,251],{"className":1496},[359],[214,1498],{"className":1499,"style":364},[363],[214,1501,258],{"className":1502},[301],[214,1504,275],{"className":1505},[469],". The logit for ",[214,1508,1510,1524],{"className":1509},[217],[214,1511,1513],{"className":1512},[221],[223,1514,1515],{"xmlns":225},[227,1516,1517,1522],{},[230,1518,1519],{},[241,1520,1521],{},"A",[277,1523,1521],{"encoding":279},[214,1525,1527],{"className":1526,"ariaHidden":250},[284],[214,1528,1530,1533],{"className":1529},[288],[214,1531],{"className":1532,"style":1152},[292],[214,1534,1521],{"className":1535},[301,305]," is ",[214,1538,1540,1553],{"className":1539},[217],[214,1541,1543],{"className":1542},[221],[223,1544,1545],{"xmlns":225},[227,1546,1547,1551],{},[230,1548,1549],{},[245,1550,247],{},[277,1552,247],{"encoding":279},[214,1554,1556],{"className":1555,"ariaHidden":250},[284],[214,1557,1559,1563],{"className":1558},[288],[214,1560],{"className":1561,"style":1562},[292],"height:0.6444em;",[214,1564,247],{"className":1565},[301]," and ",[214,1568,1570,1584],{"className":1569},[217],[214,1571,1573],{"className":1572},[221],[223,1574,1575],{"xmlns":225},[227,1576,1577,1582],{},[230,1578,1579],{},[241,1580,1581],{},"B",[277,1583,1581],{"encoding":279},[214,1585,1587],{"className":1586,"ariaHidden":250},[284],[214,1588,1590,1593],{"className":1589},[288],[214,1591],{"className":1592,"style":1152},[292],[214,1594,1581],{"className":1595,"style":1596},[301,305],"margin-right:0.0502em;",[214,1598,1600,1613],{"className":1599},[217],[214,1601,1603],{"className":1602},[221],[223,1604,1605],{"xmlns":225},[227,1606,1607,1611],{},[230,1608,1609],{},[245,1610,258],{},[277,1612,258],{"encoding":279},[214,1614,1616],{"className":1615,"ariaHidden":250},[284],[214,1617,1619,1622],{"className":1618},[288],[214,1620],{"className":1621,"style":1562},[292],[214,1623,258],{"className":1624},[301],[126,1626,1627,1716],{},[129,1628,1631,1632,1686,1687,1715],{"icon":1629,"title":1630},"i-lucide-thermometer","Temperature 1","Without using temperature, which is equivalent to using the temperature of 1, the softmax probabilities are ",[214,1633,1635,1659],{"className":1634},[217],[214,1636,1638],{"className":1637},[221],[223,1639,1640],{"xmlns":225},[227,1641,1642,1656],{},[230,1643,1644,1646,1649,1651,1654],{},[233,1645,236],{"stretchy":235},[245,1647,1648],{},"0.27",[233,1650,251],{"separator":250},[245,1652,1653],{},"0.73",[233,1655,275],{"stretchy":235},[277,1657,1658],{"encoding":279},"[0.27, 0.73]",[214,1660,1662],{"className":1661,"ariaHidden":250},[284],[214,1663,1665,1668,1671,1674,1677,1680,1683],{"className":1664},[288],[214,1666],{"className":1667,"style":293},[292],[214,1669,236],{"className":1670},[297],[214,1672,1648],{"className":1673},[301],[214,1675,251],{"className":1676},[359],[214,1678],{"className":1679,"style":364},[363],[214,1681,1653],{"className":1682},[301],[214,1684,275],{"className":1685},[469],". The model picks ",[214,1688,1690,1703],{"className":1689},[217],[214,1691,1693],{"className":1692},[221],[223,1694,1695],{"xmlns":225},[227,1696,1697,1701],{},[230,1698,1699],{},[241,1700,1581],{},[277,1702,1581],{"encoding":279},[214,1704,1706],{"className":1705,"ariaHidden":250},[284],[214,1707,1709,1712],{"className":1708},[288],[214,1710],{"className":1711,"style":1152},[292],[214,1713,1581],{"className":1714,"style":1596},[301,305]," 73% of the time.",[129,1717,1720,1721,1827,1828,1882,1883,1911],{"icon":1718,"title":1719},"i-lucide-thermometer-snowflake","Temperature 0.5","With ",[214,1722,1724,1767],{"className":1723},[217],[214,1725,1727],{"className":1726},[221],[223,1728,1729],{"xmlns":225},[227,1730,1731,1764],{},[230,1732,1733,1735,1737,1740,1742,1744,1747,1750,1752,1755,1757,1759,1761],{},[241,1734,493],{},[241,1736,668],{},[241,1738,1739],{},"m",[241,1741,103],{},[241,1743,668],{},[241,1745,1746],{},"r",[241,1748,1749],{},"a",[241,1751,493],{},[241,1753,1754],{},"u",[241,1756,1746],{},[241,1758,668],{},[233,1760,642],{},[245,1762,1763],{},"0.5",[277,1765,1766],{"encoding":279},"temperature = 0.5",[214,1768,1770,1818],{"className":1769,"ariaHidden":250},[284],[214,1771,1773,1777,1780,1783,1786,1789,1794,1797,1800,1803,1806,1809,1812,1815],{"className":1772},[288],[214,1774],{"className":1775,"style":1776},[292],"height:0.8095em;vertical-align:-0.1944em;",[214,1778,493],{"className":1779},[301,305],[214,1781,668],{"className":1782},[301,305],[214,1784,1739],{"className":1785},[301,305],[214,1787,103],{"className":1788},[301,305],[214,1790,1793],{"className":1791,"style":1792},[301,305],"margin-right:0.0278em;","er",[214,1795,1749],{"className":1796},[301,305],[214,1798,493],{"className":1799},[301,305],[214,1801,1754],{"className":1802},[301,305],[214,1804,1746],{"className":1805,"style":1792},[301,305],[214,1807,668],{"className":1808},[301,305],[214,1810],{"className":1811,"style":750},[363],[214,1813,642],{"className":1814},[754],[214,1816],{"className":1817,"style":750},[363],[214,1819,1821,1824],{"className":1820},[288],[214,1822],{"className":1823,"style":1562},[292],[214,1825,1763],{"className":1826},[301],", the probabilities are ",[214,1829,1831,1855],{"className":1830},[217],[214,1832,1834],{"className":1833},[221],[223,1835,1836],{"xmlns":225},[227,1837,1838,1852],{},[230,1839,1840,1842,1845,1847,1850],{},[233,1841,236],{"stretchy":235},[245,1843,1844],{},"0.12",[233,1846,251],{"separator":250},[245,1848,1849],{},"0.88",[233,1851,275],{"stretchy":235},[277,1853,1854],{"encoding":279},"[0.12, 0.88]",[214,1856,1858],{"className":1857,"ariaHidden":250},[284],[214,1859,1861,1864,1867,1870,1873,1876,1879],{"className":1860},[288],[214,1862],{"className":1863,"style":293},[292],[214,1865,236],{"className":1866},[297],[214,1868,1844],{"className":1869},[301],[214,1871,251],{"className":1872},[359],[214,1874],{"className":1875,"style":364},[363],[214,1877,1849],{"className":1878},[301],[214,1880,275],{"className":1881},[469],". The model now picks ",[214,1884,1886,1899],{"className":1885},[217],[214,1887,1889],{"className":1888},[221],[223,1890,1891],{"xmlns":225},[227,1892,1893,1897],{},[230,1894,1895],{},[241,1896,1581],{},[277,1898,1581],{"encoding":279},[214,1900,1902],{"className":1901,"ariaHidden":250},[284],[214,1903,1905,1908],{"className":1904},[288],[214,1906],{"className":1907,"style":1152},[292],[214,1909,1581],{"className":1910,"style":1596},[301,305]," 88% of the time.",[1913,1914,1915,1916],"warning",{},"The higher the temperature, the less likely it is that the model is going to pick the most obvious value, making the model's outputs more creative but potentially less coherent. The lower the temperature, the more likely it is that the model is going to pick the most obvious value, making the model's output more consistent but potentially more boring.",[1917,1918,1919],"sup",{},[1749,1920,247],{"href":1921,"ariaDescribedBy":1922,"dataFootnoteRef":88,"id":1924},"#user-content-fn-24",[1923],"footnote-label","user-content-fnref-24",[103,1926,1927],{},"Figure 2-16 shows the softmax probabilities for tokens A and B at different temperatures. As the temperature gets closer to 0, the probability that the model picks token B becomes closer to 1. In our example, for a temperature below 0.1, the model almost always outputs B. As the temperature increases, the probability that token A is picked increases while the probability that token B is picked decreases.",[103,1929,1930],{},[147,1931],{"alt":1932,"src":1933},"Figure 2-16. The softmax probabilities for tokens A and B at different temperatures, given their logits being 1, 2.",".\u002Fmedia\u002Ffig-2-16.png",[152,1935,1936],{},[103,1937,1938,1939,1942],{},"Figure 2-16. The softmax probabilities for tokens A and B at different temperatures, given their logits being ",[214,1940,1941],{},"1, 2",". Without setting the temperature value, which is equivalent to using the temperature of 1, the softmax probability of B would be 73%.",[103,1944,1945],{},"Model providers typically limit the temperature to be between 0 and 2. If you own your model, you can use any non-negative temperature. A temperature of 0.7 is often recommended for creative use cases, as it balances creativity and predictability, but you should experiment and find the temperature that works best for you.",[112,1947,1948,1949,1956],{},"It's common practice to set the temperature to 0 for the model's outputs to be more consistent. Technically, temperature can never be 0 -- logits can't be divided by 0. In practice, when we set the temperature to 0, the model just picks the token with the largest logit,",[1917,1950,1951],{},[1749,1952,258],{"href":1953,"ariaDescribedBy":1954,"dataFootnoteRef":88,"id":1955},"#user-content-fn-25",[1923],"user-content-fnref-25"," without doing logit adjustment and softmax calculation.",[207,1958,1959],{},"A common debugging technique when working with an AI model is to look at the probabilities this model computes for given inputs. For example, if the probabilities look random, the model hasn't learned much.",[98,1961,1963],{"id":1962},"logprobs","Logprobs",[103,1965,1966,1967,1972,1973,1975,1976,1979],{},"Many model providers return probabilities generated by their models as ",[1749,1968,1962],{"href":1969,"rel":1970},"https:\u002F\u002Fdevelopers.openai.com\u002Fcookbook\u002Fexamples\u002Fusing_logprobs",[1971],"nofollow",". ",[107,1974,1963],{},", short for ",[107,1977,1978],{},"log probabilities",", are probabilities in the log scale.",[112,1981,1982,1983,1988,1989,1997],{},"Log scale is preferred when working with a neural network's probabilities because it helps reduce the ",[1749,1984,1987],{"href":1985,"rel":1986},"https:\u002F\u002Fen.wikipedia.org\u002Fwiki\u002FArithmetic_underflow",[1971],"underflow"," problem.",[1917,1990,1991],{},[1749,1992,1996],{"href":1993,"ariaDescribedBy":1994,"dataFootnoteRef":88,"id":1995},"#user-content-fn-26",[1923],"user-content-fnref-26","3"," A language model might be working with a vocabulary size of 100,000, which means the probabilities for many of the tokens can be too small to be represented by a machine. The small numbers might be rounded down to 0. Log scale helps reduce this problem.",[103,1999,2000],{},"Figure 2-17 shows the workflow of how logits, probabilities, and logprobs are computed.",[103,2002,2003],{},[147,2004],{"alt":2005,"src":2006},"Figure 2-17. How logits, probabilities, and logprobs are computed.",".\u002Fmedia\u002Ffig-2-17.png",[152,2008,2009],{},[103,2010,2005],{},[103,2012,2013],{},"As you'll see throughout the book, logprobs are useful for building applications, especially for classification, evaluating applications, and understanding how models work under the hood.",[1913,2015,2016,2017,2025],{},"As of this writing, many model providers don't expose their models' logprobs, or if they do, the logprobs API is limited.",[1917,2018,2019],{},[1749,2020,2024],{"href":2021,"ariaDescribedBy":2022,"dataFootnoteRef":88,"id":2023},"#user-content-fn-27",[1923],"user-content-fnref-27","4"," The limited logprobs API is likely due to security reasons as a model's exposed logprobs make it easier for others to replicate the model.",[98,2027,2029],{"id":2028},"top-k","Top-k",[103,2031,2032,2034,2035,2193,2194,2485],{},[107,2033,2029],{}," is a sampling strategy to reduce the computation workload without sacrificing too much of the model's response diversity. Recall that a softmax layer is used to compute the probability distribution over all possible values. Softmax requires two passes over all possible values: one to perform the exponential sum ",[214,2036,2038,2066],{"className":2037},[217],[214,2039,2041],{"className":2040},[221],[223,2042,2043],{"xmlns":225},[227,2044,2045,2063],{},[230,2046,2047,2053],{},[238,2048,2049,2051],{},[233,2050,681],{},[241,2052,684],{},[484,2054,2055,2057],{},[241,2056,668],{},[238,2058,2059,2061],{},[241,2060,243],{},[241,2062,684],{},[277,2064,2065],{"encoding":279},"\\sum_j e^{x_j}",[214,2067,2069],{"className":2068,"ariaHidden":250},[284],[214,2070,2072,2076,2119,2122],{"className":2071},[288],[214,2073],{"className":2074,"style":2075},[292],"height:1.1858em;vertical-align:-0.4358em;",[214,2077,2079,2082],{"className":2078},[869],[214,2080,681],{"className":2081,"style":875},[869,873,874],[214,2083,2085],{"className":2084},[309],[214,2086,2088,2110],{"className":2087},[313,314],[214,2089,2091,2107],{"className":2090},[318],[214,2092,2095],{"className":2093,"style":2094},[322],"height:0.162em;",[214,2096,2098,2101],{"style":2097},"top:-2.4003em;margin-left:0em;margin-right:0.05em;",[214,2099],{"className":2100,"style":331},[330],[214,2102,2104],{"className":2103},[335,336,337,338],[214,2105,684],{"className":2106,"style":904},[301,305,338],[214,2108,346],{"className":2109},[345],[214,2111,2113],{"className":2112},[318],[214,2114,2117],{"className":2115,"style":2116},[322],"height:0.4358em;",[214,2118],{},[214,2120],{"className":2121,"style":364},[363],[214,2123,2125,2128],{"className":2124},[301],[214,2126,668],{"className":2127},[301,305],[214,2129,2131],{"className":2130},[309],[214,2132,2134],{"className":2133},[313],[214,2135,2137],{"className":2136},[318],[214,2138,2141],{"className":2139,"style":2140},[322],"height:0.6644em;",[214,2142,2143,2146],{"style":530},[214,2144],{"className":2145,"style":331},[330],[214,2147,2149],{"className":2148},[335,336,337,338],[214,2150,2152],{"className":2151},[301,338],[214,2153,2155,2158],{"className":2154},[301,338],[214,2156,243],{"className":2157},[301,305,338],[214,2159,2161],{"className":2160},[309],[214,2162,2164,2184],{"className":2163},[313,314],[214,2165,2167,2181],{"className":2166},[318],[214,2168,2170],{"className":2169,"style":1334},[322],[214,2171,2172,2175],{"style":1337},[214,2173],{"className":2174,"style":895},[330],[214,2176,2178],{"className":2177},[335,899,900,338],[214,2179,684],{"className":2180,"style":904},[301,305,338],[214,2182,346],{"className":2183},[345],[214,2185,2187],{"className":2186},[318],[214,2188,2191],{"className":2189,"style":2190},[322],"height:0.2819em;",[214,2192],{},", and one to perform ",[214,2195,2197,2239],{"className":2196},[217],[214,2198,2200],{"className":2199},[221],[223,2201,2202],{"xmlns":225},[227,2203,2204,2236],{},[230,2205,2206],{},[662,2207,2208,2218],{},[484,2209,2210,2212],{},[241,2211,668],{},[238,2213,2214,2216],{},[241,2215,243],{},[241,2217,488],{},[230,2219,2220,2226],{},[238,2221,2222,2224],{},[233,2223,681],{},[241,2225,684],{},[484,2227,2228,2230],{},[241,2229,668],{},[238,2231,2232,2234],{},[241,2233,243],{},[241,2235,684],{},[277,2237,2238],{"encoding":279},"\\frac{e^{x_i}}{\\sum_j e^{x_j}}",[214,2240,2242],{"className":2241,"ariaHidden":250},[284],[214,2243,2245,2248],{"className":2244},[288],[214,2246],{"className":2247,"style":832},[292],[214,2249,2251,2254,2482],{"className":2250},[301],[214,2252],{"className":2253},[297,839],[214,2255,2257],{"className":2256},[662],[214,2258,2260,2474],{"className":2259},[313,314],[214,2261,2263,2471],{"className":2262},[318],[214,2264,2266,2386,2394],{"className":2265,"style":852},[322],[214,2267,2268,2271],{"style":855},[214,2269],{"className":2270,"style":859},[330],[214,2272,2274],{"className":2273},[335,336,337,338],[214,2275,2277,2317,2320],{"className":2276},[301,338],[214,2278,2280,2283],{"className":2279},[869,338],[214,2281,681],{"className":2282,"style":875},[869,873,874,338],[214,2284,2286],{"className":2285},[309],[214,2287,2289,2309],{"className":2288},[313,314],[214,2290,2292,2306],{"className":2291},[318],[214,2293,2295],{"className":2294,"style":888},[322],[214,2296,2297,2300],{"style":891},[214,2298],{"className":2299,"style":895},[330],[214,2301,2303],{"className":2302},[335,899,900,338],[214,2304,684],{"className":2305,"style":904},[301,305,338],[214,2307,346],{"className":2308},[345],[214,2310,2312],{"className":2311},[318],[214,2313,2315],{"className":2314,"style":914},[322],[214,2316],{},[214,2318],{"className":2319,"style":920},[363,338],[214,2321,2323,2326],{"className":2322},[301,338],[214,2324,668],{"className":2325},[301,305,338],[214,2327,2329],{"className":2328},[309],[214,2330,2332],{"className":2331},[313],[214,2333,2335],{"className":2334},[318],[214,2336,2338],{"className":2337,"style":939},[322],[214,2339,2340,2343],{"style":942},[214,2341],{"className":2342,"style":895},[330],[214,2344,2346],{"className":2345},[335,899,900,338],[214,2347,2349],{"className":2348},[301,338],[214,2350,2352,2355],{"className":2351},[301,338],[214,2353,243],{"className":2354},[301,305,338],[214,2356,2358],{"className":2357},[309],[214,2359,2361,2378],{"className":2360},[313,314],[214,2362,2364,2375],{"className":2363},[318],[214,2365,2367],{"className":2366,"style":970},[322],[214,2368,2369,2372],{"style":973},[214,2370],{"className":2371,"style":977},[330],[214,2373,684],{"className":2374,"style":904},[301,305,338],[214,2376,346],{"className":2377},[345],[214,2379,2381],{"className":2380},[318],[214,2382,2384],{"className":2383,"style":990},[322],[214,2385],{},[214,2387,2388,2391],{"style":995},[214,2389],{"className":2390,"style":859},[330],[214,2392],{"className":2393,"style":1003},[1002],[214,2395,2396,2399],{"style":1006},[214,2397],{"className":2398,"style":859},[330],[214,2400,2402],{"className":2401},[335,336,337,338],[214,2403,2405],{"className":2404},[301,338],[214,2406,2408,2411],{"className":2407},[301,338],[214,2409,668],{"className":2410},[301,305,338],[214,2412,2414],{"className":2413},[309],[214,2415,2417],{"className":2416},[313],[214,2418,2420],{"className":2419},[318],[214,2421,2423],{"className":2422,"style":1034},[322],[214,2424,2425,2428],{"style":1037},[214,2426],{"className":2427,"style":895},[330],[214,2429,2431],{"className":2430},[335,899,900,338],[214,2432,2434],{"className":2433},[301,338],[214,2435,2437,2440],{"className":2436},[301,338],[214,2438,243],{"className":2439},[301,305,338],[214,2441,2443],{"className":2442},[309],[214,2444,2446,2463],{"className":2445},[313,314],[214,2447,2449,2460],{"className":2448},[318],[214,2450,2452],{"className":2451,"style":970},[322],[214,2453,2454,2457],{"style":973},[214,2455],{"className":2456,"style":977},[330],[214,2458,488],{"className":2459},[301,305,338],[214,2461,346],{"className":2462},[345],[214,2464,2466],{"className":2465},[318],[214,2467,2469],{"className":2468,"style":1082},[322],[214,2470],{},[214,2472,346],{"className":2473},[345],[214,2475,2477],{"className":2476},[318],[214,2478,2480],{"className":2479,"style":1094},[322],[214,2481],{},[214,2483],{"className":2484},[469,839]," for each value. For a language model with a large vocabulary, this process is computationally expensive.",[2487,2488,2489,2493,2496,2500,2503,2507],"steps",{"level":1996},[1447,2490,2492],{"id":2491},"compute-logits","Compute Logits",[103,2494,2495],{},"After the model has computed the logits, pick the top-k logits.",[1447,2497,2499],{"id":2498},"apply-softmax-only-to-top-k","Apply Softmax Only to Top-k",[103,2501,2502],{},"Perform softmax over these top-k logits only.",[1447,2504,2506],{"id":2505},"sample-from-top-values","Sample From Top Values",[103,2508,2509],{},"The model then samples from these top values. Depending on how diverse you want your application to be, k can be anywhere from 50 to 500 -- much smaller than a model's vocabulary size.",[103,2511,2512],{},"A smaller k value makes the text more predictable but less interesting, as the model is limited to a smaller set of likely words.",[98,2514,2516],{"id":2515},"top-p","Top-p",[103,2518,2519],{},"In top-k sampling, the number of values considered is fixed to k. However, this number should change depending on the situation.",[126,2521,2522,2527],{},[129,2523,2526],{"icon":2524,"title":2525},"i-lucide-check-circle","Narrow Prompt","Given the prompt \"Do you like music? Answer with only yes or no.\", the number of values considered should be two: yes and no.",[129,2528,2531],{"icon":2529,"title":2530},"i-lucide-circle-help","Open-Ended Prompt","Given the prompt \"What's the meaning of life?\", the number of values considered should be much larger.",[103,2533,2534,2536,2537,2540],{},[107,2535,2516],{},", also known as ",[107,2538,2539],{},"nucleus sampling",", allows for a more dynamic selection of values to be sampled from. In top-p sampling, the model sums the probabilities of the most likely next values in descending order and stops when the sum reaches p. Only the values within this cumulative probability are considered.",[112,2542,2543],{},"Common values for top-p (nucleus) sampling in language models typically range from 0.9 to 0.95. A top-p value of 0.9 means that the model will consider the smallest set of values whose cumulative probability exceeds 90%.",[103,2545,2546],{},"Let's say the probabilities of all tokens are as shown in Figure 2-18. If top-p is 90%, only \"yes\" and \"maybe\" will be considered, as their cumulative probability is greater than 90%. If top-p is 99%, then \"yes\", \"maybe\", and \"no\" are considered.",[103,2548,2549],{},[147,2550],{"alt":2551,"src":2552},"Figure 2-18. Example token probabilities.",".\u002Fmedia\u002Ffig-2-18.png",[152,2554,2555],{},[103,2556,2551],{},[103,2558,2559],{},"Unlike top-k, top-p doesn't necessarily reduce the softmax computation load. Its benefit is that because it focuses only on the set of most relevant values for each context, it allows outputs to be more contextually appropriate. In theory, there don't seem to be a lot of benefits to top-p sampling. However, in practice, top-p sampling has proven to work well, causing its popularity to rise.",[103,2561,2562,2563,2568],{},"A related sampling strategy is ",[1749,2564,2567],{"href":2565,"rel":2566},"https:\u002F\u002Fgithub.com\u002Fhuggingface\u002Ftransformers\u002Fissues\u002F27670",[1971],"min-p",", where you set the minimum probability that a token must reach to be considered during sampling.",[98,2570,2572],{"id":2571},"stopping-condition","Stopping Condition",[103,2574,2575,2576,2584],{},"An autoregressive language model generates sequences of tokens by generating one token after another. A long output sequence takes more time, costs more compute (money),",[1917,2577,2578],{},[1749,2579,2583],{"href":2580,"ariaDescribedBy":2581,"dataFootnoteRef":88,"id":2582},"#user-content-fn-28",[1923],"user-content-fnref-28","5"," and can sometimes annoy users. We might want to set a condition for the model to stop the sequence.",[126,2586,2587,2592],{},[129,2588,2591],{"icon":2589,"title":2590},"i-lucide-scissors","Fixed Token Limit","One easy method is to ask models to stop generating after a fixed number of tokens. The downside is that the output is likely to be cut off mid-sentence.",[129,2593,2596,2597,2600,2601,2604],{"icon":2594,"title":2595},"i-lucide-octagon","Stop Tokens or Words","Another method is to use ",[107,2598,2599],{},"stop tokens"," or ",[107,2602,2603],{},"stop words",". For example, you can ask a model to stop generating when it encounters the end-of-sequence token.",[103,2606,2607],{},"Stopping conditions are helpful to keep latency and costs down.",[1913,2609,2610],{},"The downside of early stopping is that if you want models to generate outputs in a certain format, premature stopping can cause outputs to be malformatted. If you ask the model to generate JSON, early stopping can cause the output JSON to be missing things like closing brackets, making the generated JSON hard to parse.",[98,2612,2614],{"id":2613},"test-time-compute","Test Time Compute",[103,2616,2617],{},"The last section discussed how a model might sample the next token. This section discusses how a model might sample the whole output.",[103,2619,2620,2621,2624],{},"One simple way to improve a model's response quality is ",[107,2622,2623],{},"test time compute",": instead of generating only one response per query, you generate multiple responses to increase the chance of good responses.",[126,2626,2627,2631,2642],{},[129,2628,2630],{"icon":73,"title":2629},"Best of N","Randomly generate multiple outputs and pick one that works best.",[129,2632,2635,2636,2641],{"icon":2633,"title":2634},"i-lucide-git-branch","Beam Search","Instead of generating all outputs independently, which might include many less promising candidates, use ",[1749,2637,2640],{"href":2638,"rel":2639},"https:\u002F\u002Fen.wikipedia.org\u002Fwiki\u002FBeam_search",[1971],"beam search"," to generate a fixed number of most promising candidates, the beam, at each step of sequence generation.",[129,2643,2646],{"icon":2644,"title":2645},"i-lucide-shuffle","Diverse Samples","A more diverse set of options is more likely to yield better candidates. If you use the same model to generate different options, it's often a good practice to vary the model's sampling variables.",[1913,2648,2649,2650],{},"Although you can usually expect some model performance improvement by sampling multiple outputs, it's expensive. On average, generating two outputs costs approximately twice as much as generating one.",[1917,2651,2652],{},[1749,2653,2657],{"href":2654,"ariaDescribedBy":2655,"dataFootnoteRef":88,"id":2656},"#user-content-fn-29",[1923],"user-content-fnref-29","6",[112,2659,2660,2661,2663],{},"I use the term ",[107,2662,2623],{}," to be consistent with the existing literature, even though several early reviewers protested that this term is confusing. In AI research, test time is typically used to refer to inference because researchers mostly only do inference to test a model. However, this technique can be applied to models in production in general. It's test time compute because the number of outputs you can sample is determined by how much compute you can allocate to each inference call.",[98,2665,2667],{"id":2666},"selecting-the-best-output","Selecting the Best Output",[103,2669,2670],{},"To pick the best output, you can either show users multiple outputs and let them choose the one that works best for them, or you can devise a method to select the best one.",[1447,2672,2674],{"id":2673},"highest-probability","Highest Probability",[103,2676,2677],{},"One selection method is to pick the output with the highest probability. A language model's output is a sequence of tokens, and each token has a probability computed by the model. The probability of an output is the product of the probabilities of all tokens in the output.",[103,2679,2680,2681,2685,2686,1536,2689,2692,2693,2696,2697,1536,2699,2702,2703,2696,2706,1566,2708,1536,2710,2713,2714,2717],{},"Consider the sequence of tokens ",[2682,2683,2684],"code",{},"[\"I\", \"love\", \"food\"]",". If the probability for ",[2682,2687,2688],{},"\"I\"",[2682,2690,2691],{},"0.2",", the probability for ",[2682,2694,2695],{},"\"love\""," given ",[2682,2698,2688],{},[2682,2700,2701],{},"0.1",", and the probability for ",[2682,2704,2705],{},"\"food\"",[2682,2707,2688],{},[2682,2709,2695],{},[2682,2711,2712],{},"0.3",", the sequence's probability is: ",[2682,2715,2716],{},"0.2 x 0.1 x 0.3 = 0.006",". Mathematically, this can be denoted as follows:",[103,2719,2720],{},[214,2721,2723,2788],{"className":2722},[217],[214,2724,2726],{"className":2725},[221],[223,2727,2728],{"xmlns":225},[227,2729,2730,2785],{},[230,2731,2732,2734,2736,2739,2741,2743,2745,2747,2750,2752,2755,2757,2759,2762,2765,2767,2769,2771,2773,2775,2778,2780,2783],{},[241,2733,103],{},[233,2735,649],{"stretchy":235},[644,2737,2738],{},"I love food",[233,2740,658],{"stretchy":235},[233,2742,642],{},[241,2744,103],{},[233,2746,649],{"stretchy":235},[644,2748,2749],{},"I",[233,2751,658],{"stretchy":235},[233,2753,2754],{},"×",[241,2756,103],{},[233,2758,649],{"stretchy":235},[644,2760,2761],{},"love",[233,2763,2764],{},"∣",[644,2766,2749],{},[233,2768,658],{"stretchy":235},[233,2770,2754],{},[241,2772,103],{},[233,2774,649],{"stretchy":235},[644,2776,2777],{},"food",[233,2779,2764],{},[644,2781,2782],{},"I, love",[233,2784,658],{"stretchy":235},[277,2786,2787],{"encoding":279},"p(\\text{I love food}) = p(\\text{I}) \\times p(\\text{love} \\mid \\text{I}) \\times p(\\text{food} \\mid \\text{I, love})",[214,2789,2791,2821,2853,2880,2904,2931],{"className":2790,"ariaHidden":250},[284],[214,2792,2794,2797,2800,2803,2809,2812,2815,2818],{"className":2793},[288],[214,2795],{"className":2796,"style":293},[292],[214,2798,103],{"className":2799},[301,305],[214,2801,649],{"className":2802},[297],[214,2804,2806],{"className":2805},[301,767],[214,2807,2738],{"className":2808},[301],[214,2810,658],{"className":2811},[469],[214,2813],{"className":2814,"style":750},[363],[214,2816,642],{"className":2817},[754],[214,2819],{"className":2820,"style":750},[363],[214,2822,2824,2827,2830,2833,2839,2842,2846,2850],{"className":2823},[288],[214,2825],{"className":2826,"style":293},[292],[214,2828,103],{"className":2829},[301,305],[214,2831,649],{"className":2832},[297],[214,2834,2836],{"className":2835},[301,767],[214,2837,2749],{"className":2838},[301],[214,2840,658],{"className":2841},[469],[214,2843],{"className":2844,"style":2845},[363],"margin-right:0.2222em;",[214,2847,2754],{"className":2848},[2849],"mbin",[214,2851],{"className":2852,"style":2845},[363],[214,2854,2856,2859,2862,2865,2871,2874,2877],{"className":2855},[288],[214,2857],{"className":2858,"style":293},[292],[214,2860,103],{"className":2861},[301,305],[214,2863,649],{"className":2864},[297],[214,2866,2868],{"className":2867},[301,767],[214,2869,2761],{"className":2870},[301],[214,2872],{"className":2873,"style":750},[363],[214,2875,2764],{"className":2876},[754],[214,2878],{"className":2879,"style":750},[363],[214,2881,2883,2886,2892,2895,2898,2901],{"className":2882},[288],[214,2884],{"className":2885,"style":293},[292],[214,2887,2889],{"className":2888},[301,767],[214,2890,2749],{"className":2891},[301],[214,2893,658],{"className":2894},[469],[214,2896],{"className":2897,"style":2845},[363],[214,2899,2754],{"className":2900},[2849],[214,2902],{"className":2903,"style":2845},[363],[214,2905,2907,2910,2913,2916,2922,2925,2928],{"className":2906},[288],[214,2908],{"className":2909,"style":293},[292],[214,2911,103],{"className":2912},[301,305],[214,2914,649],{"className":2915},[297],[214,2917,2919],{"className":2918},[301,767],[214,2920,2777],{"className":2921},[301],[214,2923],{"className":2924,"style":750},[363],[214,2926,2764],{"className":2927},[754],[214,2929],{"className":2930,"style":750},[363],[214,2932,2934,2937,2943],{"className":2933},[288],[214,2935],{"className":2936,"style":293},[292],[214,2938,2940],{"className":2939},[301,767],[214,2941,2782],{"className":2942},[301],[214,2944,658],{"className":2945},[469],[103,2947,2948],{},"Remember that it's easier to work with probabilities on a log scale. The logarithm of a product is equal to a sum of logarithms, so the logprob of a sequence of tokens is the sum of the logprob of all tokens in the sequence:",[103,2950,2951],{},[214,2952,2954,3014],{"className":2953},[217],[214,2955,2957],{"className":2956},[221],[223,2958,2959],{"xmlns":225},[227,2960,2961,3011],{},[230,2962,2963,2966,2968,2970,2972,2974,2976,2978,2980,2982,2985,2987,2989,2991,2993,2995,2997,2999,3001,3003,3005,3007,3009],{},[644,2964,2965],{},"logprob",[233,2967,649],{"stretchy":235},[644,2969,2738],{},[233,2971,658],{"stretchy":235},[233,2973,642],{},[644,2975,2965],{},[233,2977,649],{"stretchy":235},[644,2979,2749],{},[233,2981,658],{"stretchy":235},[233,2983,2984],{},"+",[644,2986,2965],{},[233,2988,649],{"stretchy":235},[644,2990,2761],{},[233,2992,2764],{},[644,2994,2749],{},[233,2996,658],{"stretchy":235},[233,2998,2984],{},[644,3000,2965],{},[233,3002,649],{"stretchy":235},[644,3004,2777],{},[233,3006,2764],{},[644,3008,2782],{},[233,3010,658],{"stretchy":235},[277,3012,3013],{"encoding":279},"\\text{logprob}(\\text{I love food}) = \\text{logprob}(\\text{I}) + \\text{logprob}(\\text{love} \\mid \\text{I}) + \\text{logprob}(\\text{food} \\mid \\text{I, love})",[214,3015,3017,3050,3083,3113,3137,3167],{"className":3016,"ariaHidden":250},[284],[214,3018,3020,3023,3029,3032,3038,3041,3044,3047],{"className":3019},[288],[214,3021],{"className":3022,"style":293},[292],[214,3024,3026],{"className":3025},[301,767],[214,3027,2965],{"className":3028},[301],[214,3030,649],{"className":3031},[297],[214,3033,3035],{"className":3034},[301,767],[214,3036,2738],{"className":3037},[301],[214,3039,658],{"className":3040},[469],[214,3042],{"className":3043,"style":750},[363],[214,3045,642],{"className":3046},[754],[214,3048],{"className":3049,"style":750},[363],[214,3051,3053,3056,3062,3065,3071,3074,3077,3080],{"className":3052},[288],[214,3054],{"className":3055,"style":293},[292],[214,3057,3059],{"className":3058},[301,767],[214,3060,2965],{"className":3061},[301],[214,3063,649],{"className":3064},[297],[214,3066,3068],{"className":3067},[301,767],[214,3069,2749],{"className":3070},[301],[214,3072,658],{"className":3073},[469],[214,3075],{"className":3076,"style":2845},[363],[214,3078,2984],{"className":3079},[2849],[214,3081],{"className":3082,"style":2845},[363],[214,3084,3086,3089,3095,3098,3104,3107,3110],{"className":3085},[288],[214,3087],{"className":3088,"style":293},[292],[214,3090,3092],{"className":3091},[301,767],[214,3093,2965],{"className":3094},[301],[214,3096,649],{"className":3097},[297],[214,3099,3101],{"className":3100},[301,767],[214,3102,2761],{"className":3103},[301],[214,3105],{"className":3106,"style":750},[363],[214,3108,2764],{"className":3109},[754],[214,3111],{"className":3112,"style":750},[363],[214,3114,3116,3119,3125,3128,3131,3134],{"className":3115},[288],[214,3117],{"className":3118,"style":293},[292],[214,3120,3122],{"className":3121},[301,767],[214,3123,2749],{"className":3124},[301],[214,3126,658],{"className":3127},[469],[214,3129],{"className":3130,"style":2845},[363],[214,3132,2984],{"className":3133},[2849],[214,3135],{"className":3136,"style":2845},[363],[214,3138,3140,3143,3149,3152,3158,3161,3164],{"className":3139},[288],[214,3141],{"className":3142,"style":293},[292],[214,3144,3146],{"className":3145},[301,767],[214,3147,2965],{"className":3148},[301],[214,3150,649],{"className":3151},[297],[214,3153,3155],{"className":3154},[301,767],[214,3156,2777],{"className":3157},[301],[214,3159],{"className":3160,"style":750},[363],[214,3162,2764],{"className":3163},[754],[214,3165],{"className":3166,"style":750},[363],[214,3168,3170,3173,3179],{"className":3169},[288],[214,3171],{"className":3172,"style":293},[292],[214,3174,3176],{"className":3175},[301,767],[214,3177,2782],{"className":3178},[301],[214,3180,658],{"className":3181},[469],[207,3183,3184,3185],{},"With summing, longer sequences are likely to have a lower total logprob because logprob values are usually negative. To avoid biasing toward short sequences, you can use the average logprob by dividing the sum of a sequence by its length. After sampling multiple outputs, you pick the one with the highest average logprob. As of this writing, this is what the OpenAI API uses.",[1917,3186,3187],{},[1749,3188,3192],{"href":3189,"ariaDescribedBy":3190,"dataFootnoteRef":88,"id":3191},"#user-content-fn-30",[1923],"user-content-fnref-30","7",[1447,3194,3196],{"id":3195},"reward-models-and-verifiers","Reward Models and Verifiers",[103,3198,3199,3200,1566,3205,3210,3211,3216],{},"Another selection method is to use a reward model to score each output, as discussed in the previous section. Recall that both ",[1749,3201,3204],{"href":3202,"rel":3203},"https:\u002F\u002Fmultithreaded.stitchfix.com\u002Fblog\u002F2023\u002F03\u002F06\u002Fexpert-in-the-loop-generative-ai-at-stitch-fix\u002F",[1971],"Stitch Fix",[1749,3206,3209],{"href":3207,"rel":3208},"https:\u002F\u002Fengineering.grab.com\u002Fllm-powered-data-classification",[1971],"Grab"," pick the outputs given high scores by their reward models or verifiers. ",[1749,3212,3215],{"href":3213,"rel":3214},"https:\u002F\u002Fengblog.nextdoor.com\u002Flet-ai-entertain-you-increasing-user-engagement-with-generative-ai-and-rejection-sampling-50a402264f56",[1971],"Nextdoor"," found that using a reward model was the key factor in improving their application's performance (2023).",[103,3218,3219,3220,3225],{},"OpenAI also trained verifiers to help their models pick the best solutions to math problems (",[1749,3221,3224],{"href":3222,"rel":3223},"https:\u002F\u002Farxiv.org\u002Fpdf\u002F2110.14168",[1971],"Cobbe et al., 2021","). They found that using a verifier significantly boosted the model performance.",[112,3227,3228],{},"The use of verifiers resulted in approximately the same performance boost as a 30x model size increase. This means that a 100-million-parameter model that uses a verifier can perform on par with a 3-billion-parameter model that doesn't use a verifier.",[103,3230,3231,3232,3237],{},"DeepMind further proves the value of test time compute, arguing that scaling test time compute, e.g. allocating more compute to generate more outputs during inference, can be more efficient than scaling model parameters (",[1749,3233,3236],{"href":3234,"rel":3235},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2408.03314",[1971],"Snell et al., 2024","). The same paper asks an interesting question: If an LLM is allowed to use a fixed but nontrivial amount of inference-time compute, how much can it improve its performance on a challenging prompt?",[98,3239,3241],{"id":3240},"how-much-test-time-compute-helps","How Much Test Time Compute Helps",[103,3243,3244],{},"In OpenAI's experiment, sampling more outputs led to better performance, but only up to a certain point. In this experiment, that point was 400 outputs. Beyond this point, performance decreases, as shown in Figure 2-19. They hypothesized that as the number of sampled outputs increases, the chance of finding adversarial outputs that can fool the verifier also increases.",[103,3246,3247,3248,3253],{},"However, a Stanford experiment showed a different conclusion. \"Monkey Business\" (",[1749,3249,3252],{"href":3250,"rel":3251},"https:\u002F\u002Fscalingintelligence.stanford.edu\u002Fblogs\u002Fmonkeys\u002F",[1971],"Brown et al., 2024",") finds that the number of problems solved often increases log-linearly as the number of samples increases from 1 to 10,000.",[1913,3255,3256],{},"While it's interesting to think about whether test time compute can be scaled indefinitely, I don't believe anyone in production samples 400 or 10,000 different outputs for each input. The cost would be astronomical.",[103,3258,3259],{},[147,3260],{"alt":3261,"src":3262},"Figure 2-19. OpenAI (2021) found that sampling more outputs led to better performance, but only up to 400 outputs.",".\u002Fmedia\u002Ffig-2-19.png",[152,3264,3265],{},[103,3266,3267,3268,3273],{},"Figure 2-19. ",[1749,3269,3272],{"href":3270,"rel":3271},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2110.14168",[1971],"OpenAI"," (2021) found that sampling more outputs led to better performance, but only up to 400 outputs.",[1447,3275,3277],{"id":3276},"application-specific-selection","Application-Specific Selection",[103,3279,3280],{},"You can also use application-specific heuristics to select the best response.",[126,3282,3283,3288,3292],{},[129,3284,3287],{"icon":3285,"title":3286},"i-lucide-align-left","Shorter Responses","If your application benefits from shorter responses, you can pick the shortest candidate.",[129,3289,3291],{"icon":58,"title":3290},"Valid SQL","If your application converts natural language to SQL queries, you can get the model to keep on generating outputs until it generates a valid SQL query.",[129,3293,3296],{"icon":3294,"title":3295},"i-lucide-timer","Lower Latency","Kittipat Kampa, head of AI at TIFIN, told me that his team asks their model to generate multiple responses in parallel and show the user the first response that is completed and valid.",[103,3298,3299,3300,3308],{},"Picking out the most common output among a set of outputs can be especially useful for tasks that expect exact answers.",[1917,3301,3302],{},[1749,3303,3307],{"href":3304,"ariaDescribedBy":3305,"dataFootnoteRef":88,"id":3306},"#user-content-fn-31",[1923],"user-content-fnref-31","8"," For example, given a math problem, the model can solve it multiple times and pick the most frequent answer as its final solution. Similarly, for a multiple-choice question, a model can pick the most frequent output option. This is what Google did when evaluating Gemini on the MMLU benchmark. They sampled 32 outputs for each question. This allowed the model to achieve a higher score than what it would've achieved with only one output per question.",[103,3310,3311,3312,3320],{},"A model is considered robust if it doesn't dramatically change its outputs with small variations in the input. The less robust a model is, the more you can benefit from sampling multiple outputs.",[1917,3313,3314],{},[1749,3315,3319],{"href":3316,"ariaDescribedBy":3317,"dataFootnoteRef":88,"id":3318},"#user-content-fn-32",[1923],"user-content-fnref-32","9"," For one project, we used AI to extract certain information from an image of the product. We found that for the same image, our model could read the information only half of the time. For the other half, the model said that the image was too blurry or the text was too small to read. However, by trying three times with each image, the model was able to extract the correct information for most images.",[98,3322,3324],{"id":3323},"structured-outputs","Structured Outputs",[103,3326,3327],{},"Often, in production, you need models to generate outputs following certain formats. Structured outputs are crucial for the following two scenarios:",[126,3329,3330,3335],{},[129,3331,3334],{"icon":3332,"title":3333},"i-lucide-braces","Tasks Requiring Structure","The most common category is semantic parsing: converting natural language into a structured, machine-readable format. Text-to-SQL is an example where outputs must be valid SQL queries.",[129,3336,3339,3340,166],{"icon":3337,"title":3338},"i-lucide-workflow","Downstream Applications","The task itself might not need structured outputs, but downstream applications might need parsable formats, such as a JSON document with keys like ",[2682,3341,3342],{},"{\"title\": [TITLE], \"body\": [EMAIL BODY]}",[103,3344,3345],{},"Semantic parsing allows users to interact with APIs using a natural language, e.g. English. For example, text-to-PostgreSQL allows users to query a Postgres database using English queries such as \"What's the average monthly revenue over the last 6 months\" instead of writing it in PostgreSQL.",[1913,3347,3348],{},"This is especially important for agentic workflows where a model's outputs are often passed as inputs into tools that the model can use, as discussed in Chapter 6.",[1447,3350,3352],{"id":3351},"text-to-regex-example","Text-to-Regex Example",[103,3354,3355],{},"This is an example of a prompt for GPT-4o to do text-to-regex. The outputs are actual outputs generated by GPT-4o:",[103,3357,3358],{},[93,3359,3360],{},"System prompt",[3362,3363,3367],"pre",{"className":3364,"code":3366,"language":767,"meta":88},[3365],"language-text","Given an item, create a regex that represents all the ways the item can be written. Return only the regex.\n\nExample:\n\nUS phone number -> \\+?1?\\s?(\\()?(\\d{3})(?(1)\\))[-.\\s]?(\\d{3})[-.\\s]? (\\d{4})\n",[2682,3368,3366],{"__ignoreMap":88},[103,3370,3371],{},[93,3372,3373],{},"User prompt",[3362,3375,3378],{"className":3376,"code":3377,"language":767,"meta":88},[3365],"Email address ->\n",[2682,3379,3377],{"__ignoreMap":88},[103,3381,3382],{},[93,3383,3384],{},"GPT-4o",[3362,3386,3390],{"className":3387,"code":3388,"language":3389,"meta":88,"style":88},"language-regex shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}\n","regex",[2682,3391,3392],{"__ignoreMap":88},[214,3393,3396],{"class":3394,"line":3395},"line",1,[214,3397,3388],{},[103,3399,3400],{},[93,3401,3373],{},[3362,3403,3406],{"className":3404,"code":3405,"language":767,"meta":88},[3365],"Dates ->\n",[2682,3407,3405],{"__ignoreMap":88},[103,3409,3410],{},[93,3411,3384],{},[3362,3413,3415],{"className":3387,"code":3414,"language":3389,"meta":88,"style":88},"(?:\\d{1,2}[\\\u002F\\-\\.])(?:\\d{1,2}[\\\u002F\\-\\.])?\\d{2,4}\n",[2682,3416,3417],{"__ignoreMap":88},[214,3418,3419],{"class":3394,"line":3395},[214,3420,3414],{},[103,3422,3423],{},"Other categories of tasks in this scenario include classification where the outputs have to be valid classes.",[98,3425,3427],{"id":3426},"structured-output-tooling","Structured Output Tooling",[103,3429,3430,3431,3436,3437,3436,3442,3447,3448,3453],{},"Frameworks that support structured outputs include ",[1749,3432,3435],{"href":3433,"rel":3434},"https:\u002F\u002Fgithub.com\u002Fguidance-ai\u002Fguidance",[1971],"guidance",", ",[1749,3438,3441],{"href":3439,"rel":3440},"https:\u002F\u002Fgithub.com\u002Fdottxt-ai\u002Foutlines",[1971],"outlines",[1749,3443,3446],{"href":3444,"rel":3445},"https:\u002F\u002Fgithub.com\u002Finstructor-ai\u002Finstructor",[1971],"instructor",", and ",[1749,3449,3452],{"href":3450,"rel":3451},"https:\u002F\u002Fgithub.com\u002Fggerganov\u002Fllama.cpp\u002Fdiscussions\u002F177",[1971],"llama.cpp",". Each model provider might also use their own techniques to improve their models' ability to generate structured outputs.",[103,3455,3456,3457,3464],{},"OpenAI was the first model provider to introduce ",[1749,3458,3461],{"href":3459,"rel":3460},"https:\u002F\u002Fdevelopers.openai.com\u002Fapi\u002Fdocs\u002Fguides\u002Fstructured-outputs",[1971],[107,3462,3463],{},"JSON mode"," in their text generation API.",[1913,3466,3467],{},"An API's JSON mode typically guarantees only that the outputs are valid JSON -- not the content of the JSON objects. The otherwise valid generated JSONs can also be truncated, and thus not parsable, if the generation stops too soon, such as when it reaches the maximum output token length. However, if the max token length is set too long, the model's responses become both too slow and expensive.",[103,3469,3470,3474],{},[1749,3471,3473],{"href":3472},"#index_split_001.html_p125","Figure 2-20"," shows two examples of using guidance to generate outputs constrained to a set of options and a regex.",[103,3476,3477],{},[147,3478],{"alt":3479,"src":3480},"Figure 2-20. Using guidance to generate constrained outputs.",".\u002Fmedia\u002Ffig-2-20.png",[152,3482,3483],{},[103,3484,3479],{},[98,3486,3488],{"id":3487},"five-ways-to-guide-structured-outputs","Five Ways to Guide Structured Outputs",[103,3490,3491],{},"You can guide a model to generate structured outputs at different layers of the AI stack: prompting, post-processing, test time compute, constrained sampling, and finetuning.",[112,3493,3494],{},"The first three are more like bandages. They work best if the model is already pretty good at generating structured outputs and just needs a little nudge. For intensive treatment, you need constrained sampling and finetuning.",[103,3496,3497],{},"Test time compute has just been discussed in the previous section -- keep on generating outputs until one fits the expected format. This section focuses on the other four approaches.",[1447,3499,3501],{"id":3500},"prompting","Prompting",[103,3503,3504,3505,3513],{},"Prompting is the first line of action for structured outputs. You can instruct a model to generate outputs in any format. However, whether a model can follow this instruction depends on the model's instruction-following capability, discussed in Chapter 4, and the clarity of the instruction, discussed in Chapter 5. While models are getting increasingly good at following instructions, there's no guarantee that they'll always follow your instructions.",[1917,3506,3507],{},[1749,3508,3512],{"href":3509,"ariaDescribedBy":3510,"dataFootnoteRef":88,"id":3511},"#user-content-fn-33",[1923],"user-content-fnref-33","10"," A few percentage points of invalid model outputs can still be unacceptable for many applications.",[207,3515,3516],{},"To increase the percentage of valid outputs, some people use AI to validate and\u002For correct the output of the original prompt. This is an example of the AI as a judge approach discussed in Chapter 3.",[103,3518,3519],{},"This means that for each output, there will be at least two model queries: one to generate the output and one to validate it. While the added validation layer can significantly improve the validity of the outputs, the extra cost and latency incurred by the extra validation queries can make this approach too expensive for some.",[1447,3521,3523],{"id":3522},"post-processing","Post-processing",[103,3525,3526],{},"Post-processing is simple and cheap but can work surprisingly well. During my time teaching, I noticed that students tended to make very similar mistakes. When I started working with foundation models, I noticed the same thing. A model tends to repeat similar mistakes across queries. This means if you find the common mistakes a model makes, you can potentially write a script to correct them.",[112,3528,3529,3530,3535],{},"For example, if the generated JSON object misses a closing bracket, manually add that bracket. LinkedIn's defensive YAML parser increased the percentage of correct YAML outputs from 90% to 99.99% (",[1749,3531,3534],{"href":3532,"rel":3533},"https:\u002F\u002Fwww.linkedin.com\u002Fblog\u002Fengineering\u002Fgenerative-ai\u002Fmusings-on-building-a-generative-ai-product?_l=en_US",[1971],"Bottaro and Ramgopal, 2020",").",[207,3537,3538],{},"JSON and YAML are common text formats. LinkedIn found that their underlying model, GPT-4, worked with both, but they chose YAML as their output format because it is less verbose, and hence requires fewer output tokens than JSON (Bottaro and Ramgopal, 2020).",[103,3540,3541],{},"Post-processing works only if the mistakes are easy to fix. This usually happens if a model's outputs are already mostly correctly formatted, with occasional small errors.",[1447,3543,3545],{"id":3544},"constrained-sampling","Constrained Sampling",[103,3547,3548,3551],{},[107,3549,3550],{},"Constrained sampling"," is a technique for guiding the generation of text toward certain constraints. It is typically followed by structured output tools.",[103,3553,3554],{},"At a high level, to generate a token, the model samples among values that meet the constraints. Recall that to generate a token, your model first outputs a logit vector, each logit corresponding to one possible token. Constrained sampling filters this logit vector to keep only the tokens that meet the constraints. It then samples from these valid tokens.",[103,3556,3557],{},[147,3558],{"alt":3559,"src":3560},"Figure 2-21. Filter out logits that don't meet the constraints in order to sample only among valid outputs.",".\u002Fmedia\u002Ffig-2-21.png",[152,3562,3563],{},[103,3564,3559],{},[103,3566,3567,3568,3571,3572,3574,3575,166],{},"In the example in Figure 2-21, the constraint is straightforward to filter for. However, most cases aren't that straightforward. You need to have a grammar that specifies what is and isn't allowed at each step. For example, JSON grammar dictates that after ",[2682,3569,3570],{},"{",", you can't have another ",[2682,3573,3570],{}," unless it's part of a string, as in ",[2682,3576,3577],{},"{\"key\": \"{{string}}\"}",[1913,3579,3580,3581,3535],{},"Building out that grammar and incorporating it into the sampling process is nontrivial. Because each output format -- JSON, YAML, regex, CSV, and so on -- needs its own grammar, constrained sampling is less generalizable. Its use is limited to the formats whose grammars are supported by external tools or by your team. Grammar verification can also increase generation latency (",[1749,3582,3585],{"href":3583,"rel":3584},"https:\u002F\u002Fblog.dottxt.ai\u002Fhow-fast-cfg.html",[1971],"Brandon T. Willard, 2024",[103,3587,3588],{},"Some are against constrained sampling because they believe the resources needed for constrained sampling are better invested in training models to become better at following instructions.",[1447,3590,3592],{"id":3591},"finetuning","Finetuning",[103,3594,3595,3596,3604],{},"Finetuning a model on examples following your desirable format is the most effective and general approach to get models to generate outputs in this format.",[1917,3597,3598],{},[1749,3599,3603],{"href":3600,"ariaDescribedBy":3601,"dataFootnoteRef":88,"id":3602},"#user-content-fn-34",[1923],"user-content-fnref-34","11"," It can work with any expected format. While simple finetuning doesn't guarantee that the model will always output the expected format, it is much more reliable than prompting.",[103,3606,3607,3608,3616,3617,3620],{},"For certain tasks, you can guarantee the output format by modifying the model's architecture before finetuning. For example, for classification, you can append a classifier head to the foundation model's architecture to make sure that the model outputs only one of the pre-specified classes. The architecture looks like Figure 2-22.",[1917,3609,3610],{},[1749,3611,3615],{"href":3612,"ariaDescribedBy":3613,"dataFootnoteRef":88,"id":3614},"#user-content-fn-35",[1923],"user-content-fnref-35","12"," This approach is also called ",[107,3618,3619],{},"feature-based transfer"," and is discussed more with other transfer learning techniques in Chapter 7.",[103,3622,3623],{},[147,3624],{"alt":3625,"src":3626},"Figure 2-22. Adding a classifier head to your base model to turn it into a classifier. In this example, the classifier works with three classes.",".\u002Fmedia\u002Ffig-2-22.png",[152,3628,3629],{},[103,3630,3625],{},[103,3632,3633],{},"During finetuning, you can retrain the whole model end-to-end or part of the model, such as this classifier head. End-to-end training requires more resources, but promises better performance.",[112,3635,3636],{},"We need techniques for structured outputs because of the assumption that the model, by itself, isn't capable of generating structured outputs. However, as models become more powerful, we can expect them to get better at following instructions. I suspect that in the future, it'll be easier to get models to output exactly what we need with minimal prompting, and these techniques will become less important.",[98,3638,3640],{"id":3639},"the-probabilistic-nature-of-ai","The Probabilistic Nature of AI",[103,3642,3643,3644,3647],{},"The way AI models sample their responses makes them ",[107,3645,3646],{},"probabilistic",". Imagine that you want to know what's the best cuisine in the world. If you ask your friend this question twice, a minute apart, your friend's answers both times should be the same. If you ask an AI model the same question twice, its answer can change.",[112,3649,3650,3651,3654],{},"If an AI model thinks that Vietnamese cuisine has a 70% chance of being the best cuisine in the world and Italian cuisine has a 30% chance, it'll answer \"Vietnamese cuisine\" 70% of the time and \"Italian cuisine\" 30% of the time. The opposite of probabilistic is ",[107,3652,3653],{},"deterministic",", when the outcome can be determined without any random variation.",[103,3656,3657],{},"This probabilistic nature can cause inconsistency and hallucinations.",[126,3659,3660,3664],{},[129,3661,3663],{"icon":2644,"title":3662},"Inconsistency","A model generates very different responses for the same or slightly different prompts.",[129,3665,3668],{"icon":3666,"title":3667},"i-lucide-cloud-alert","Hallucination","A model gives a response that isn't grounded in facts.",[103,3670,3671],{},"Imagine if someone on the internet wrote an essay about how all US presidents are aliens, and this essay was included in the training data. The model later will probabilistically output that the current US president is an alien. From the perspective of someone who doesn't believe that US presidents are aliens, the model is making this up.",[1913,3673,3674,3675],{},"Foundation models are usually trained using a large amount of data. They are aggregations of the opinions of the masses, containing within them, literally, a world of possibilities. Anything with a non-zero probability, no matter how far-fetched or wrong, can be generated by AI.",[1917,3676,3677],{},[1749,3678,3682],{"href":3679,"ariaDescribedBy":3680,"dataFootnoteRef":88,"id":3681},"#user-content-fn-36",[1923],"user-content-fnref-36","13",[103,3684,3685],{},"This characteristic makes building AI applications both exciting and challenging. Many of the AI engineering efforts, as we'll see in this book, aim to harness and mitigate this probabilistic nature.",[126,3687,3688,3692],{},[129,3689,3691],{"icon":19,"title":3690},"Great for Creativity","What is creativity but the ability to explore beyond the common paths -- to think outside the box? AI can brainstorm limitless ideas and generate never-before-seen designs.",[129,3693,3696,3697],{"icon":3694,"title":3695},"i-lucide-triangle-alert","Painful for Reliability","This same probabilistic nature can be a pain for everything else.",[1917,3698,3699],{},[1749,3700,3704],{"href":3701,"ariaDescribedBy":3702,"dataFootnoteRef":88,"id":3703},"#user-content-fn-37",[1923],"user-content-fnref-37","14",[98,3706,3662],{"id":3707},"inconsistency",[103,3709,3710],{},"Model inconsistency manifests in two scenarios:",[126,3712,3713,3718],{},[129,3714,3717],{"icon":3715,"title":3716},"i-lucide-copy","Same Input, Different Outputs","Giving the model the same prompt twice leads to two very different responses.",[129,3719,3722],{"icon":3720,"title":3721},"i-lucide-pencil-ruler","Slightly Different Input, Drastic Output","Giving the model a slightly different prompt, such as accidentally capitalizing a letter, can lead to a very different output.",[103,3724,3725],{},"Figure 2-23 shows an example of me trying to use ChatGPT to score essays. The same prompt gave me two different scores when I ran it twice: 3\u002F5 and 5\u002F5.",[103,3727,3728],{},[147,3729],{"alt":3730,"src":3731},"Figure 2-23. The same input can produce different outputs in the same model.",".\u002Fmedia\u002Ffig-2-23.png",[152,3733,3734],{},[103,3735,3730],{},[103,3737,3738],{},"Inconsistency can create a jarring user experience. In human-to-human communication, we expect a certain level of consistency. Imagine a person giving you a different name every time you see them. Similarly, users expect a certain level of consistency when communicating with AI.",[1447,3740,3742],{"id":3741},"mitigating-inconsistency","Mitigating Inconsistency",[103,3744,3745],{},"For the same input, different outputs scenario, there are multiple approaches to mitigate inconsistency.",[126,3747,3748,3752,3756],{},[129,3749,3751],{"icon":58,"title":3750},"Cache Answers","Cache the answer so that the next time the same question is asked, the same answer is returned.",[129,3753,3755],{"icon":68,"title":3754},"Fix Sampling Variables","Fix the model's sampling variables, such as temperature, top-p, and top-k values.",[129,3757,3760,3761,3764],{"icon":3758,"title":3759},"i-lucide-sprout","Fix the Seed","Fix the ",[107,3762,3763],{},"seed"," variable, which you can think of as the starting point for the random number generator used for sampling the next token.",[1913,3766,3767],{},"Even if you fix all these variables, there's no guarantee that your model will be consistent 100% of the time. The hardware the model runs the output generation on can also impact the output, as different machines have different ways of executing the same instruction and can handle different ranges of numbers.",[103,3769,3770],{},"If you host your models, you have some control over the hardware you use. However, if you use a model API provider like OpenAI or Google, it's up to these providers to give you any control.",[103,3772,3773],{},"Fixing the output generation settings is a good practice, but it doesn't inspire trust in the system. Imagine a teacher who gives you consistent scores only if that teacher sits in one particular room. If that teacher sits in a different room, that teacher's scores for you will be wild.",[103,3775,3776],{},"The second scenario -- slightly different input, drastically different outputs -- is more challenging. Fixing the model's output generation variables is still a good practice, but it won't force the model to generate the same outputs for different inputs. It is, however, possible to get models to generate responses closer to what you want with carefully crafted prompts, discussed in Chapter 5, and a memory system, discussed in Chapter 6.",[98,3778,3667],{"id":3779},"hallucination",[103,3781,3782,3783,3788],{},"Hallucinations are fatal for tasks that depend on factuality. If you're asking AI to help you explain the pros and cons of a vaccine, you don't want AI to be pseudo-scientific. In June 2023, a law firm was ",[1749,3784,3787],{"href":3785,"rel":3786},"https:\u002F\u002Fwww.cbsnews.com\u002Fnews\u002Fchatgpt-judge-fines-lawyers-who-used-ai\u002F",[1971],"fined for submitting fictitious legal research to court",". They had used ChatGPT to prepare their case, unaware of ChatGPT's tendency to hallucinate.",[103,3790,3791,3792,3797,3798,3803,3804,3809,3810,3535],{},"While hallucination became a prominent issue with the rise of LLMs, hallucination was a common phenomenon for generative models even before the term foundation model and the transformer architecture were introduced. Hallucination in the context of text generation was mentioned as early as 2016 (",[1749,3793,3796],{"href":3794,"rel":3795},"https:\u002F\u002Faclanthology.org\u002FC16-1103\u002F",[1971],"Goyal et al., 2016","). Detecting and measuring hallucinations has been a staple in natural language generation (NLG) since then (see ",[1749,3799,3802],{"href":3800,"rel":3801},"https:\u002F\u002Fopenreview.net\u002Fforum?id=SkxJ-309FQ",[1971],"Lee et al., 2018","; ",[1749,3805,3808],{"href":3806,"rel":3807},"https:\u002F\u002Faclanthology.org\u002FP19-1256\u002F",[1971],"Nie et al., 2019","; and ",[1749,3811,3814],{"href":3812,"rel":3813},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2011.02593",[1971],"Zhou et al., 2020",[112,3816,3817],{},"This section focuses on explaining why hallucinations happen. How to detect and measure evaluation is discussed in Chapter 4.",[103,3819,3820],{},"If inconsistency arises from randomness in the sampling process, the cause of hallucination is more nuanced. The sampling process alone doesn't sufficiently explain it. A model samples outputs from all probable options. But how does something never seen before become a probable option?",[1913,3822,3823],{},"A model can output something that is believed to have never been seen before in the training data. We can't say this for sure because it's impossible to comb through the training data to verify whether it contains an idea. Our ability to construct something so complex that we can no longer understand it is both a blessing and a curse.",[103,3825,3826],{},"It's hard to devise a way to eliminate hallucinations without understanding why hallucinations occur in the first place. There are currently two hypotheses about why language models hallucinate.",[98,3828,3830],{"id":3829},"hypothesis-1-self-delusion","Hypothesis 1: Self-Delusion",[103,3832,3833,3834,3839],{},"The first hypothesis, originally expressed by ",[1749,3835,3838],{"href":3836,"rel":3837},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2110.10819#deepmind",[1971],"Ortega et al. at DeepMind in 2021",", is that a language model hallucinates because it can't differentiate between the data it's given and the data it generates.",[103,3841,3842],{},"Imagine that you give the model the prompt: \"Who's Chip Huyen?\" and the first sentence the model generates is: \"Chip Huyen is an architect.\" The next token the model generates will be conditioned on the sequence: \"Who's Chip Huyen? Chip Huyen is an architect.\" The model treats \"Chip Huyen is an architect.\", something it produced, the same way it treats a given fact.",[1913,3844,3845,3846,166],{},"Starting with a generated sequence slightly out of the ordinary, the model can expand upon it and generate outrageously wrong facts. Ortega and the other authors called hallucinations a form of ",[107,3847,3848],{},"self-delusion",[103,3850,3851],{},"Figure 2-24 shows an example of self-delusion by the model LLaVA-v1.5-7B. I asked the model to identify ingredients listed on the product's label in the image, which is a bottle of shampoo. In its response, the model convinces itself that the product in the image is a bottle of milk, then continues to include milk in the list of ingredients extracted from the product's label.",[103,3853,3854],{},[147,3855],{"alt":3856,"src":3857},"Figure 2-24. An example of self-delusion by LLaVA-v1.5-7B.",".\u002Fmedia\u002Ffig-2-24.png",[152,3859,3860],{},[103,3861,3856],{},[103,3863,3864,3865,3870],{},"Zhang et al. (2023) call this phenomenon ",[1749,3866,3869],{"href":3867,"rel":3868},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2305.13534",[1971],"snowballing hallucinations",". After making an incorrect assumption, a model can continue hallucinating to justify the initial wrong assumption. Interestingly, the authors show that initial wrong assumptions can cause the model to make mistakes on questions it would otherwise be able to answer correctly, as shown in Figure 2-25.",[103,3872,3873],{},[147,3874],{"alt":3875,"src":3876},"Figure 2-25. An initial incorrect assumption can cause the model to claim that 9677 is divisible by 13, even if it knows this isn't true.",".\u002Fmedia\u002Ffig-2-25.png",[152,3878,3879],{},[103,3880,3875],{},[103,3882,3883],{},"The DeepMind paper showed that hallucinations can be mitigated by two techniques.",[126,3885,3886,3898],{},[129,3887,3890,3891,3894,3895,166],{"icon":3888,"title":3889},"i-lucide-bot","Separate Observations and Actions","From reinforcement learning, the model is made to differentiate between user-provided prompts, called ",[107,3892,3893],{},"observations about the world",", and tokens generated by the model, called the model's ",[107,3896,3897],{},"actions",[129,3899,3901],{"icon":58,"title":3900},"Add Factual Signals","From supervised learning, factual and counterfactual signals are included in the training data.",[98,3903,3905],{"id":3904},"hypothesis-2-mismatched-internal-knowledge","Hypothesis 2: Mismatched Internal Knowledge",[103,3907,3908,3909,3914],{},"The second hypothesis is that hallucination is caused by the mismatch between the model's internal knowledge and the labeler's internal knowledge. This view was first argued by ",[1749,3910,3913],{"href":3911,"rel":3912},"https:\u002F\u002Fwww.alignmentforum.org\u002Fposts\u002FBgoKdAzogxmgkuuAt\u002Fbehavior-cloning-is-miscalibrated",[1971],"Leo Gao",", an OpenAI researcher.",[103,3916,3917],{},"During SFT, models are trained to mimic responses written by labelers. If these responses use the knowledge that the labelers have but the model doesn't have, we're effectively teaching the model to hallucinate. In theory, if labelers can include the knowledge they use with each response they write so that the model knows that the responses aren't made up, we can perhaps teach the model to use only what it knows. However, this is impossible in practice.",[103,3919,3920,3921,3926],{},"In April 2023, John Schulman, an OpenAI co-founder, expressed the same view in his ",[1749,3922,3925],{"href":3923,"rel":3924},"https:\u002F\u002Fwww.youtube.com\u002Fwatch?v=hhiLw5Q_UFg",[1971],"UC Berkeley talk",". Schulman also believes that LLMs know if they know something, which, in itself, is a big claim. If this belief is true, hallucinations can be fixed by forcing a model to give answers based on only the information it knows.",[126,3928,3929,3934],{},[129,3930,3933],{"icon":3931,"title":3932},"i-lucide-search-check","Verification","For each response, ask the model to retrieve the sources it bases this response on.",[129,3935,3937],{"icon":68,"title":3936},"Reinforcement Learning","A better reward function that punishes a model more for making things up can help mitigate hallucinations.",[103,3939,3940],{},"Remember that the reward model is trained using only comparisons -- response A is better than response B -- without an explanation of why A is better.",[1913,3942,3943],{},"In that same talk, Schulman mentioned that OpenAI found that RLHF helps with reducing hallucinations. However, the InstructGPT paper shows that RLHF made hallucination worse, as shown in Figure 2-26. Even though RLHF seemed to worsen hallucinations for InstructGPT, it improved other aspects, and overall, human labelers prefer the RLHF model over the SFT alone model.",[103,3945,3946],{},[147,3947],{"alt":3948,"src":3949},"Figure 2-26. Hallucination is worse for the model that uses both RLHF and SFT (InstructGPT) compared to the same model that uses only SFT (Ouyang et al., 2022).",".\u002Fmedia\u002Ffig-2-26.png",[152,3951,3952],{},[103,3953,3954,3955,3535],{},"Figure 2-26. Hallucination is worse for the model that uses both RLHF and SFT (InstructGPT) compared to the same model that uses only SFT (",[1749,3956,3959],{"href":3957,"rel":3958},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2203.02155",[1971],"Ouyang et al., 2022",[103,3961,3962],{},"Based on the assumption that a foundation model knows what it knows, some people try to reduce hallucination with prompts, such as adding \"Answer as truthfully as possible, and if you're unsure of the answer, say, 'Sorry, I don't know.'\" Asking models for concise responses also seems to help with hallucinations -- the fewer tokens a model has to generate, the less chance it has to make things up. Prompting and context construction techniques in Chapters 5 and 6 can also help mitigate hallucinations.",[112,3964,3965],{},"The two hypotheses discussed complement each other. The self-delusion hypothesis focuses on how self-supervision causes hallucinations, whereas the mismatched internal knowledge hypothesis focuses on how supervision causes hallucinations.",[103,3967,3968],{},"If we can't stop hallucinations altogether, can we at least detect when a model hallucinates so that we won't serve those hallucinated responses to users? Detecting hallucinations isn't that straightforward either -- think about how hard it is for us to detect when another human is lying or making things up. But people have tried. We discuss how to detect and measure hallucinations in Chapter 4.",[3970,3971,3974,3979],"section",{"className":3972,"dataFootnotes":88},[3973],"footnotes",[98,3975,3978],{"className":3976,"id":1923},[3977],"sr-only","Footnotes",[3980,3981,3982,3994,4008,4017,4037,4046,4055,4070,4084,4093,4102,4111,4126,4140],"ol",{},[3983,3984,3986,3987],"li",{"id":3985},"user-content-fn-24","A visual image I have in mind when thinking about temperature, which isn't entirely scientific, is that a higher temperature causes the probability distribution to be more chaotic, which enables lower-probability tokens to surface. ",[1749,3988,3993],{"href":3989,"ariaLabel":3990,"className":3991,"dataFootnoteBackref":88},"#user-content-fnref-24","Back to reference 1",[3992],"data-footnote-backref","↩",[3983,3995,3997,3998,1972,4003],{"id":3996},"user-content-fn-25","Performing an ",[1749,3999,4002],{"href":4000,"rel":4001},"https:\u002F\u002Fen.wikipedia.org\u002Fwiki\u002FArg_max",[1971],"arg max function",[1749,4004,3993],{"href":4005,"ariaLabel":4006,"className":4007,"dataFootnoteBackref":88},"#user-content-fnref-25","Back to reference 2",[3992],[3983,4009,4011,4012],{"id":4010},"user-content-fn-26","The underflow problem occurs when a number is too small to be represented in a given format, leading to it being rounded down to zero. ",[1749,4013,3993],{"href":4014,"ariaLabel":4015,"className":4016,"dataFootnoteBackref":88},"#user-content-fnref-26","Back to reference 3",[3992],[3983,4018,4020,4021,4025,4026,4031,4032],{"id":4019},"user-content-fn-27","To be more specific, as of this writing, OpenAI API only shows you the ",[1749,4022,1962],{"href":4023,"rel":4024},"https:\u002F\u002Foreil.ly\u002FjWEsP",[1971]," of up to the 20 most likely tokens. It used to let you get the logprobs of arbitrary user-provided text but discontinued this in ",[1749,4027,4030],{"href":4028,"rel":4029},"https:\u002F\u002Fx.com\u002Fxuanalogue\u002Fstatus\u002F1707757449900437984",[1971],"September 2023",". Anthropic doesn't expose its models' logprobs. ",[1749,4033,3993],{"href":4034,"ariaLabel":4035,"className":4036,"dataFootnoteBackref":88},"#user-content-fnref-27","Back to reference 4",[3992],[3983,4038,4040,4041],{"id":4039},"user-content-fn-28","Paid model APIs often charge per number of output tokens. ",[1749,4042,3993],{"href":4043,"ariaLabel":4044,"className":4045,"dataFootnoteBackref":88},"#user-content-fnref-28","Back to reference 5",[3992],[3983,4047,4049,4050],{"id":4048},"user-content-fn-29","There are things you can do to reduce the cost of generating multiple outputs for the same input. For example, the input might only be processed once and reused for all outputs. ",[1749,4051,3993],{"href":4052,"ariaLabel":4053,"className":4054,"dataFootnoteBackref":88},"#user-content-fnref-29","Back to reference 6",[3992],[3983,4056,4058,4059,4064,4065],{"id":4057},"user-content-fn-30","As of this writing, in the OpenAI API, you can set the parameter ",[1749,4060,4063],{"href":4061,"rel":4062},"https:\u002F\u002Foreil.ly\u002FXYugZ",[1971],"best_of"," to a specific value, say 10, to ask OpenAI models to return the output with the highest average logprob out of 10 different outputs. ",[1749,4066,3993],{"href":4067,"ariaLabel":4068,"className":4069,"dataFootnoteBackref":88},"#user-content-fnref-30","Back to reference 7",[3992],[3983,4071,4073,4078,4079],{"id":4072},"user-content-fn-31",[1749,4074,4077],{"href":4075,"rel":4076},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2203.11171",[1971],"Wang et al. (2023)"," called this approach self-consistency. ",[1749,4080,3993],{"href":4081,"ariaLabel":4082,"className":4083,"dataFootnoteBackref":88},"#user-content-fnref-31","Back to reference 8",[3992],[3983,4085,4087,4088],{"id":4086},"user-content-fn-32","The optimal thing to do with a brittle model, however, is to swap it out for another. ",[1749,4089,3993],{"href":4090,"ariaLabel":4091,"className":4092,"dataFootnoteBackref":88},"#user-content-fnref-32","Back to reference 9",[3992],[3983,4094,4096,4097],{"id":4095},"user-content-fn-33","As of this writing, depending on the application and the model, I've seen the percentage of correctly generated JSON objects anywhere between 0% and up to the high 90%. ",[1749,4098,3993],{"href":4099,"ariaLabel":4100,"className":4101,"dataFootnoteBackref":88},"#user-content-fnref-33","Back to reference 10",[3992],[3983,4103,4105,4106],{"id":4104},"user-content-fn-34","Training a model from scratch on data following the desirable format works too, but this book isn't about developing models from scratch. ",[1749,4107,3993],{"href":4108,"ariaLabel":4109,"className":4110,"dataFootnoteBackref":88},"#user-content-fnref-34","Back to reference 11",[3992],[3983,4112,4114,4115,4120,4121],{"id":4113},"user-content-fn-35","Some finetuning services do this for you automatically. ",[1749,4116,4119],{"href":4117,"rel":4118},"https:\u002F\u002Fcommunity.openai.com\u002Ft\u002Fhow-to-build-a-conversation-classifier-with-the-new-fine-tuning-job-api-and-gpt3-5\u002F341075",[1971],"OpenAI's finetuning services"," used to let you add a classifier head when training, but as I write, this feature has been disabled. ",[1749,4122,3993],{"href":4123,"ariaLabel":4124,"className":4125,"dataFootnoteBackref":88},"#user-content-fnref-35","Back to reference 12",[3992],[3983,4127,4129,4130,1972,4135],{"id":4128},"user-content-fn-36","As the meme says, ",[1749,4131,4134],{"href":4132,"rel":4133},"https:\u002F\u002Fx.com\u002FOxfordDiplomat\u002Fstatus\u002F1424388443010998277?lang=en",[1971],"the chances are low, but never zero",[1749,4136,3993],{"href":4137,"ariaLabel":4138,"className":4139,"dataFootnoteBackref":88},"#user-content-fnref-36","Back to reference 13",[3992],[3983,4141,4143,4144],{"id":4142},"user-content-fn-37","In December 2023, I went over three months' worth of customer support requests for an AI company I advised and found that one-fifth of the questions were about handling the inconsistency of AI models. In a panel I participated in with Drew Houston (CEO of Dropbox) and Harrison Chase (CEO of LangChain) in July 2023, we all agreed that hallucination is the biggest blocker for many AI enterprise use cases. ",[1749,4145,3993],{"href":4146,"ariaLabel":4147,"className":4148,"dataFootnoteBackref":88},"#user-content-fnref-37","Back to reference 14",[3992],[4150,4151,4152],"style",{},"html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":88,"searchDepth":4154,"depth":4154,"links":4155},2,[4156,4157,4158,4159,4160,4161,4165,4166,4171,4172,4173,4174,4178,4181,4184,4185,4191,4192,4195,4196,4197,4198],{"id":100,"depth":4154,"text":101},{"id":120,"depth":4154,"text":121},{"id":158,"depth":4154,"text":159},{"id":184,"depth":4154,"text":185},{"id":1102,"depth":4154,"text":1103},{"id":1112,"depth":4154,"text":1113,"children":4162},[4163],{"id":1449,"depth":4164,"text":1450},3,{"id":1962,"depth":4154,"text":1963},{"id":2028,"depth":4154,"text":2029,"children":4167},[4168,4169,4170],{"id":2491,"depth":4164,"text":2492},{"id":2498,"depth":4164,"text":2499},{"id":2505,"depth":4164,"text":2506},{"id":2515,"depth":4154,"text":2516},{"id":2571,"depth":4154,"text":2572},{"id":2613,"depth":4154,"text":2614},{"id":2666,"depth":4154,"text":2667,"children":4175},[4176,4177],{"id":2673,"depth":4164,"text":2674},{"id":3195,"depth":4164,"text":3196},{"id":3240,"depth":4154,"text":3241,"children":4179},[4180],{"id":3276,"depth":4164,"text":3277},{"id":3323,"depth":4154,"text":3324,"children":4182},[4183],{"id":3351,"depth":4164,"text":3352},{"id":3426,"depth":4154,"text":3427},{"id":3487,"depth":4154,"text":3488,"children":4186},[4187,4188,4189,4190],{"id":3500,"depth":4164,"text":3501},{"id":3522,"depth":4164,"text":3523},{"id":3544,"depth":4164,"text":3545},{"id":3591,"depth":4164,"text":3592},{"id":3639,"depth":4154,"text":3640},{"id":3707,"depth":4154,"text":3662,"children":4193},[4194],{"id":3741,"depth":4164,"text":3742},{"id":3779,"depth":4154,"text":3667},{"id":3829,"depth":4154,"text":3830},{"id":3904,"depth":4154,"text":3905},{"id":1923,"depth":4154,"text":3978},"How sampling strategies, test-time compute, structured outputs, and probabilistic behavior shape AI model responses.","md",{},{"icon":73},{"title":70,"description":4199},"xCLIOR7J8ythPyO7L2st1eWHKwnGAKFxSNANruLW99g",[4206,4208],{"title":65,"path":66,"stem":67,"description":4207,"icon":68,"children":-1},"How supervised finetuning, preference finetuning, reward models, and RLHF make pre-trained models usable.",{"title":41,"path":75,"stem":76,"description":4209,"icon":44,"children":-1},"A recap of how training data, modeling choices, post-training, and sampling shape foundation model behavior.",1779363442215]