[{"data":1,"prerenderedAt":16569},["ShallowReactive",2],{"writing-\u002Fwriting\u002Fhow-many-gpus-to-serve-an-llm":3},{"id":4,"title":5,"body":6,"date":16561,"description":16562,"extension":16563,"meta":16564,"navigation":12653,"path":16565,"seo":16566,"stem":16567,"__hash__":16568},"writing\u002Fwriting\u002F06.how-many-gpus-to-serve-an-llm.md","How Many GPUs Do You Need to Serve an LLM?",{"type":7,"value":8,"toc":16545},"minimark",[9,13,20,26,31,34,351,626,629,648,652,655,1108,1308,1316,1319,1322,1640,1644,1653,1664,1668,1671,1676,1679,1804,1840,1844,1847,1853,2185,2188,2195,2201,2248,2268,2274,2471,2474,2600,2708,3021,3024,3225,3228,3472,3679,3682,3948,3965,3969,4052,4402,4730,5043,5251,5523,5786,6174,8612,8619,8623,8656,8661,8870,8968,9347,9731,10002,10328,10427,12146,12149,12156,12160,12163,12414,12417,12553,12617,12621,12624,14297,14301,14304,14310,14445,14451,14454,14460,14651,14912,14915,15240,15389,15401,15404,15765,15772,15776,15998,16001,16005,16541],[10,11,12],"p",{},"You have a model and some serving constraints to meet. How many GPUs does it take to serve? You can answer that on a napkin, using numbers you already know. It comes down to two questions.",[10,14,15,19],{},[16,17,18],"em",{},"Does it fit?"," The weights and the KV cache have to live in GPU memory. This is the easy one, and where most people stop.",[10,21,22,25],{},[16,23,24],{},"Can it keep up?"," The GPUs have to push tokens fast enough to hit your rate without missing the latency target. This is where the GPU count usually comes from.",[27,28,30],"h2",{"id":29},"the-answer-in-one-line","The Answer in One Line",[10,32,33],{},"Call the three bills memory, prefill, and decode. The model has to fit, and it has to keep up:",[35,36,39],"span",{"className":37},[38],"katex-display",[35,40,43,118],{"className":41},[42],"katex",[35,44,47],{"className":45},[46],"katex-mathml",[48,49,52],"math",{"xmlns":50,"display":51},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML","block",[53,54,55,113],"semantics",{},[56,57,58,62,66,69,72,78,87,90,93,100,103,110],"mrow",{},[59,60,61],"mi",{},"G",[63,64,65],"mo",{},"=",[59,67,68],{},"max",[63,70,71],{},"⁡",[63,73,77],{"fence":74,"stretchy":75,"minsize":76,"maxsize":76},"false","true","1.2em","(",[79,80,81,83],"msub",{},[59,82,61],{},[84,85,86],"mtext",{},"fit",[63,88,89],{"separator":75},",",[84,91,92],{}," ",[79,94,95,97],{},[59,96,61],{},[84,98,99],{},"prefill",[63,101,102],{},"+",[79,104,105,107],{},[59,106,61],{},[84,108,109],{},"decode",[63,111,112],{"fence":74,"stretchy":75,"minsize":76,"maxsize":76},")",[114,115,117],"annotation",{"encoding":116},"application\u002Fx-tex","G = \\max\\big(G_\\text{fit},\\ G_\\text{prefill} + G_\\text{decode}\\big)",[35,119,122,148,296],{"className":120,"ariaHidden":75},[121],"katex-html",[35,123,126,131,136,141,145],{"className":124},[125],"base",[35,127],{"className":128,"style":130},[129],"strut","height:0.6833em;",[35,132,61],{"className":133},[134,135],"mord","mathnormal",[35,137],{"className":138,"style":140},[139],"mspace","margin-right:0.2778em;",[35,142,65],{"className":143},[144],"mrel",[35,146],{"className":147,"style":140},[139],[35,149,151,155,159,163,171,231,235,238,241,285,289,293],{"className":150},[125],[35,152],{"className":153,"style":154},[129],"height:1.2em;vertical-align:-0.35em;",[35,156,68],{"className":157},[158],"mop",[35,160],{"className":161,"style":162},[139],"margin-right:0.1667em;",[35,164,166],{"className":165},[134],[35,167,77],{"className":168},[169,170],"delimsizing","size1",[35,172,174,177],{"className":173},[134],[35,175,61],{"className":176},[134,135],[35,178,181],{"className":179},[180],"msupsub",[35,182,186,222],{"className":183},[184,185],"vlist-t","vlist-t2",[35,187,190,217],{"className":188},[189],"vlist-r",[35,191,195],{"className":192,"style":194},[193],"vlist","height:0.3361em;",[35,196,198,203],{"style":197},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[35,199],{"className":200,"style":202},[201],"pstrut","height:2.7em;",[35,204,210],{"className":205},[206,207,208,209],"sizing","reset-size6","size3","mtight",[35,211,214],{"className":212},[134,213,209],"text",[35,215,86],{"className":216},[134,209],[35,218,221],{"className":219},[220],"vlist-s","​",[35,223,225],{"className":224},[189],[35,226,229],{"className":227,"style":228},[193],"height:0.15em;",[35,230],{},[35,232,89],{"className":233},[234],"mpunct",[35,236,92],{"className":237},[139],[35,239],{"className":240,"style":162},[139],[35,242,244,247],{"className":243},[134],[35,245,61],{"className":246},[134,135],[35,248,250],{"className":249},[180],[35,251,253,276],{"className":252},[184,185],[35,254,256,273],{"className":255},[189],[35,257,259],{"className":258,"style":194},[193],[35,260,261,264],{"style":197},[35,262],{"className":263,"style":202},[201],[35,265,267],{"className":266},[206,207,208,209],[35,268,270],{"className":269},[134,213,209],[35,271,99],{"className":272},[134,209],[35,274,221],{"className":275},[220],[35,277,279],{"className":278},[189],[35,280,283],{"className":281,"style":282},[193],"height:0.2861em;",[35,284],{},[35,286],{"className":287,"style":288},[139],"margin-right:0.2222em;",[35,290,102],{"className":291},[292],"mbin",[35,294],{"className":295,"style":288},[139],[35,297,299,302,345],{"className":298},[125],[35,300],{"className":301,"style":154},[129],[35,303,305,308],{"className":304},[134],[35,306,61],{"className":307},[134,135],[35,309,311],{"className":310},[180],[35,312,314,337],{"className":313},[184,185],[35,315,317,334],{"className":316},[189],[35,318,320],{"className":319,"style":194},[193],[35,321,322,325],{"style":197},[35,323],{"className":324,"style":202},[201],[35,326,328],{"className":327},[206,207,208,209],[35,329,331],{"className":330},[134,213,209],[35,332,109],{"className":333},[134,209],[35,335,221],{"className":336},[220],[35,338,340],{"className":339},[189],[35,341,343],{"className":342,"style":228},[193],[35,344],{},[35,346,348],{"className":347},[134],[35,349,112],{"className":350},[169,170],[352,353,354,371],"table",{},[355,356,357],"thead",{},[358,359,360,365,368],"tr",{},[361,362,364],"th",{"align":363},"left","Bill",[361,366,367],{"align":363},"Question",[361,369,370],{"align":363},"Bottleneck",[372,373,374,459,543],"tbody",{},[358,375,376,453,456],{},[377,378,379],"td",{"align":363},[35,380,382,400],{"className":381},[42],[35,383,385],{"className":384},[46],[48,386,387],{"xmlns":50},[53,388,389,397],{},[56,390,391],{},[79,392,393,395],{},[59,394,61],{},[84,396,86],{},[114,398,399],{"encoding":116},"G_\\text{fit}",[35,401,403],{"className":402,"ariaHidden":75},[121],[35,404,406,410],{"className":405},[125],[35,407],{"className":408,"style":409},[129],"height:0.8333em;vertical-align:-0.15em;",[35,411,413,416],{"className":412},[134],[35,414,61],{"className":415},[134,135],[35,417,419],{"className":418},[180],[35,420,422,445],{"className":421},[184,185],[35,423,425,442],{"className":424},[189],[35,426,428],{"className":427,"style":194},[193],[35,429,430,433],{"style":197},[35,431],{"className":432,"style":202},[201],[35,434,436],{"className":435},[206,207,208,209],[35,437,439],{"className":438},[134,213,209],[35,440,86],{"className":441},[134,209],[35,443,221],{"className":444},[220],[35,446,448],{"className":447},[189],[35,449,451],{"className":450,"style":228},[193],[35,452],{},[377,454,455],{"align":363},"Do weights and KV cache fit?",[377,457,458],{"align":363},"GPU memory capacity",[358,460,461,537,540],{},[377,462,463],{"align":363},[35,464,466,484],{"className":465},[42],[35,467,469],{"className":468},[46],[48,470,471],{"xmlns":50},[53,472,473,481],{},[56,474,475],{},[79,476,477,479],{},[59,478,61],{},[84,480,99],{},[114,482,483],{"encoding":116},"G_\\text{prefill}",[35,485,487],{"className":486,"ariaHidden":75},[121],[35,488,490,494],{"className":489},[125],[35,491],{"className":492,"style":493},[129],"height:0.9694em;vertical-align:-0.2861em;",[35,495,497,500],{"className":496},[134],[35,498,61],{"className":499},[134,135],[35,501,503],{"className":502},[180],[35,504,506,529],{"className":505},[184,185],[35,507,509,526],{"className":508},[189],[35,510,512],{"className":511,"style":194},[193],[35,513,514,517],{"style":197},[35,515],{"className":516,"style":202},[201],[35,518,520],{"className":519},[206,207,208,209],[35,521,523],{"className":522},[134,213,209],[35,524,99],{"className":525},[134,209],[35,527,221],{"className":528},[220],[35,530,532],{"className":531},[189],[35,533,535],{"className":534,"style":282},[193],[35,536],{},[377,538,539],{"align":363},"Can we process prompts fast enough?",[377,541,542],{"align":363},"FLOPs",[358,544,545,620,623],{},[377,546,547],{"align":363},[35,548,550,568],{"className":549},[42],[35,551,553],{"className":552},[46],[48,554,555],{"xmlns":50},[53,556,557,565],{},[56,558,559],{},[79,560,561,563],{},[59,562,61],{},[84,564,109],{},[114,566,567],{"encoding":116},"G_\\text{decode}",[35,569,571],{"className":570,"ariaHidden":75},[121],[35,572,574,577],{"className":573},[125],[35,575],{"className":576,"style":409},[129],[35,578,580,583],{"className":579},[134],[35,581,61],{"className":582},[134,135],[35,584,586],{"className":585},[180],[35,587,589,612],{"className":588},[184,185],[35,590,592,609],{"className":591},[189],[35,593,595],{"className":594,"style":194},[193],[35,596,597,600],{"style":197},[35,598],{"className":599,"style":202},[201],[35,601,603],{"className":602},[206,207,208,209],[35,604,606],{"className":605},[134,213,209],[35,607,109],{"className":608},[134,209],[35,610,221],{"className":611},[220],[35,613,615],{"className":614},[189],[35,616,618],{"className":617,"style":228},[193],[35,619],{},[377,621,622],{"align":363},"Can we generate tokens fast enough?",[377,624,625],{"align":363},"HBM bandwidth",[10,627,628],{},"For the running example below, Qwen3-8B on H200s at 2 req\u002Fs with 4K input and 1K output, the bills come out to about 0.7 cards to fit, 0.3 for prefill, and 0.5 for decode. That rounds to one H200. Push the same workload to 20 req\u002Fs and it becomes about eight.",[630,631,632,639],"content-alert",{},[633,634,636],"template",{"v-slot:title":635},"",[10,637,638],{},"Want to run your own numbers?",[10,640,641,642,647],{},"The ",[643,644,646],"a",{"href":645},"#run-the-napkin","calculator below"," uses the same formulas as the article. Change the constants at the top, run it, and it prints the memory, prefill, decode, and final GPU bills separately.",[27,649,651],{"id":650},"what-you-need","What You Need",[10,653,654],{},"You need three groups of inputs:",[656,657,658,770,918],"ul",{},[659,660,661,665,666,696,697,769],"li",{},[662,663,664],"strong",{},"Model:"," total parameters ",[35,667,669,683],{"className":668},[42],[35,670,672],{"className":671},[46],[48,673,674],{"xmlns":50},[53,675,676,681],{},[56,677,678],{},[59,679,680],{},"P",[114,682,680],{"encoding":116},[35,684,686],{"className":685,"ariaHidden":75},[121],[35,687,689,692],{"className":688},[125],[35,690],{"className":691,"style":130},[129],[35,693,680],{"className":694,"style":695},[134,135],"margin-right:0.1389em;",", active parameters ",[35,698,700,718],{"className":699},[42],[35,701,703],{"className":702},[46],[48,704,705],{"xmlns":50},[53,706,707,715],{},[56,708,709],{},[79,710,711,713],{},[59,712,680],{},[59,714,643],{},[114,716,717],{"encoding":116},"P_a",[35,719,721],{"className":720,"ariaHidden":75},[121],[35,722,724,727],{"className":723},[125],[35,725],{"className":726,"style":409},[129],[35,728,730,733],{"className":729},[134],[35,731,680],{"className":732,"style":695},[134,135],[35,734,736],{"className":735},[180],[35,737,739,761],{"className":738},[184,185],[35,740,742,758],{"className":741},[189],[35,743,746],{"className":744,"style":745},[193],"height:0.1514em;",[35,747,749,752],{"style":748},"top:-2.55em;margin-left:-0.1389em;margin-right:0.05em;",[35,750],{"className":751,"style":202},[201],[35,753,755],{"className":754},[206,207,208,209],[35,756,643],{"className":757},[134,135,209],[35,759,221],{"className":760},[220],[35,762,764],{"className":763},[189],[35,765,767],{"className":766,"style":228},[193],[35,768],{},", precision, and attention architecture.",[659,771,772,775,776,853,854,887,888,917],{},[662,773,774],{},"Hardware:"," memory per GPU ",[35,777,779,799],{"className":778},[42],[35,780,782],{"className":781},[46],[48,783,784],{"xmlns":50},[53,785,786,796],{},[56,787,788],{},[79,789,790,793],{},[59,791,792],{},"M",[84,794,795],{},"gpu",[114,797,798],{"encoding":116},"M_\\text{gpu}",[35,800,802],{"className":801,"ariaHidden":75},[121],[35,803,805,808],{"className":804},[125],[35,806],{"className":807,"style":493},[129],[35,809,811,815],{"className":810},[134],[35,812,792],{"className":813,"style":814},[134,135],"margin-right:0.109em;",[35,816,818],{"className":817},[180],[35,819,821,845],{"className":820},[184,185],[35,822,824,842],{"className":823},[189],[35,825,827],{"className":826,"style":745},[193],[35,828,830,833],{"style":829},"top:-2.55em;margin-left:-0.109em;margin-right:0.05em;",[35,831],{"className":832,"style":202},[201],[35,834,836],{"className":835},[206,207,208,209],[35,837,839],{"className":838},[134,213,209],[35,840,795],{"className":841},[134,209],[35,843,221],{"className":844},[220],[35,846,848],{"className":847},[189],[35,849,851],{"className":850,"style":282},[193],[35,852],{},", bandwidth ",[35,855,857,872],{"className":856},[42],[35,858,860],{"className":859},[46],[48,861,862],{"xmlns":50},[53,863,864,869],{},[56,865,866],{},[84,867,868],{},"BW",[114,870,871],{"encoding":116},"\\text{BW}",[35,873,875],{"className":874,"ariaHidden":75},[121],[35,876,878,881],{"className":877},[125],[35,879],{"className":880,"style":130},[129],[35,882,884],{"className":883},[134,213],[35,885,868],{"className":886},[134],", and peak compute ",[35,889,891,905],{"className":890},[42],[35,892,894],{"className":893},[46],[48,895,896],{"xmlns":50},[53,897,898,903],{},[56,899,900],{},[59,901,902],{},"F",[114,904,902],{"encoding":116},[35,906,908],{"className":907,"ariaHidden":75},[121],[35,909,911,914],{"className":910},[125],[35,912],{"className":913,"style":130},[129],[35,915,902],{"className":916,"style":695},[134,135],".",[659,919,920,923,924,954,955,1031,1032,1107],{},[662,921,922],{},"Demand:"," request rate ",[35,925,927,941],{"className":926},[42],[35,928,930],{"className":929},[46],[48,931,932],{"xmlns":50},[53,933,934,939],{},[56,935,936],{},[59,937,938],{},"R",[114,940,938],{"encoding":116},[35,942,944],{"className":943,"ariaHidden":75},[121],[35,945,947,950],{"className":946},[125],[35,948],{"className":949,"style":130},[129],[35,951,938],{"className":952,"style":953},[134,135],"margin-right:0.0077em;",", input length ",[35,956,958,978],{"className":957},[42],[35,959,961],{"className":960},[46],[48,962,963],{"xmlns":50},[53,964,965,975],{},[56,966,967],{},[79,968,969,972],{},[59,970,971],{},"L",[84,973,974],{},"in",[114,976,977],{"encoding":116},"L_\\text{in}",[35,979,981],{"className":980,"ariaHidden":75},[121],[35,982,984,987],{"className":983},[125],[35,985],{"className":986,"style":409},[129],[35,988,990,993],{"className":989},[134],[35,991,971],{"className":992},[134,135],[35,994,996],{"className":995},[180],[35,997,999,1023],{"className":998},[184,185],[35,1000,1002,1020],{"className":1001},[189],[35,1003,1006],{"className":1004,"style":1005},[193],"height:0.3175em;",[35,1007,1008,1011],{"style":197},[35,1009],{"className":1010,"style":202},[201],[35,1012,1014],{"className":1013},[206,207,208,209],[35,1015,1017],{"className":1016},[134,213,209],[35,1018,974],{"className":1019},[134,209],[35,1021,221],{"className":1022},[220],[35,1024,1026],{"className":1025},[189],[35,1027,1029],{"className":1028,"style":228},[193],[35,1030],{},", output length ",[35,1033,1035,1054],{"className":1034},[42],[35,1036,1038],{"className":1037},[46],[48,1039,1040],{"xmlns":50},[53,1041,1042,1051],{},[56,1043,1044],{},[79,1045,1046,1048],{},[59,1047,971],{},[84,1049,1050],{},"out",[114,1052,1053],{"encoding":116},"L_\\text{out}",[35,1055,1057],{"className":1056,"ariaHidden":75},[121],[35,1058,1060,1063],{"className":1059},[125],[35,1061],{"className":1062,"style":409},[129],[35,1064,1066,1069],{"className":1065},[134],[35,1067,971],{"className":1068},[134,135],[35,1070,1072],{"className":1071},[180],[35,1073,1075,1099],{"className":1074},[184,185],[35,1076,1078,1096],{"className":1077},[189],[35,1079,1082],{"className":1080,"style":1081},[193],"height:0.2806em;",[35,1083,1084,1087],{"style":197},[35,1085],{"className":1086,"style":202},[201],[35,1088,1090],{"className":1089},[206,207,208,209],[35,1091,1093],{"className":1092},[134,213,209],[35,1094,1050],{"className":1095},[134,209],[35,1097,221],{"className":1098},[220],[35,1100,1102],{"className":1101},[189],[35,1103,1105],{"className":1104,"style":228},[193],[35,1106],{},", and TPOT target.",[10,1109,1110,1179,1180,1208,1209,1237,1238,1307],{},[35,1111,1113,1130],{"className":1112},[42],[35,1114,1116],{"className":1115},[46],[48,1117,1118],{"xmlns":50},[53,1119,1120,1128],{},[56,1121,1122],{},[79,1123,1124,1126],{},[59,1125,680],{},[59,1127,643],{},[114,1129,717],{"encoding":116},[35,1131,1133],{"className":1132,"ariaHidden":75},[121],[35,1134,1136,1139],{"className":1135},[125],[35,1137],{"className":1138,"style":409},[129],[35,1140,1142,1145],{"className":1141},[134],[35,1143,680],{"className":1144,"style":695},[134,135],[35,1146,1148],{"className":1147},[180],[35,1149,1151,1171],{"className":1150},[184,185],[35,1152,1154,1168],{"className":1153},[189],[35,1155,1157],{"className":1156,"style":745},[193],[35,1158,1159,1162],{"style":748},[35,1160],{"className":1161,"style":202},[201],[35,1163,1165],{"className":1164},[206,207,208,209],[35,1166,643],{"className":1167},[134,135,209],[35,1169,221],{"className":1170},[220],[35,1172,1174],{"className":1173},[189],[35,1175,1177],{"className":1176,"style":228},[193],[35,1178],{}," is equal to ",[35,1181,1183,1196],{"className":1182},[42],[35,1184,1186],{"className":1185},[46],[48,1187,1188],{"xmlns":50},[53,1189,1190,1194],{},[56,1191,1192],{},[59,1193,680],{},[114,1195,680],{"encoding":116},[35,1197,1199],{"className":1198,"ariaHidden":75},[121],[35,1200,1202,1205],{"className":1201},[125],[35,1203],{"className":1204,"style":130},[129],[35,1206,680],{"className":1207,"style":695},[134,135]," for a dense model. For an MoE, ",[35,1210,1212,1225],{"className":1211},[42],[35,1213,1215],{"className":1214},[46],[48,1216,1217],{"xmlns":50},[53,1218,1219,1223],{},[56,1220,1221],{},[59,1222,680],{},[114,1224,680],{"encoding":116},[35,1226,1228],{"className":1227,"ariaHidden":75},[121],[35,1229,1231,1234],{"className":1230},[125],[35,1232],{"className":1233,"style":130},[129],[35,1235,680],{"className":1236,"style":695},[134,135]," is the total resident parameter count, while ",[35,1239,1241,1258],{"className":1240},[42],[35,1242,1244],{"className":1243},[46],[48,1245,1246],{"xmlns":50},[53,1247,1248,1256],{},[56,1249,1250],{},[79,1251,1252,1254],{},[59,1253,680],{},[59,1255,643],{},[114,1257,717],{"encoding":116},[35,1259,1261],{"className":1260,"ariaHidden":75},[121],[35,1262,1264,1267],{"className":1263},[125],[35,1265],{"className":1266,"style":409},[129],[35,1268,1270,1273],{"className":1269},[134],[35,1271,680],{"className":1272,"style":695},[134,135],[35,1274,1276],{"className":1275},[180],[35,1277,1279,1299],{"className":1278},[184,185],[35,1280,1282,1296],{"className":1281},[189],[35,1283,1285],{"className":1284,"style":745},[193],[35,1286,1287,1290],{"style":748},[35,1288],{"className":1289,"style":202},[201],[35,1291,1293],{"className":1292},[206,207,208,209],[35,1294,643],{"className":1295},[134,135,209],[35,1297,221],{"className":1298},[220],[35,1300,1302],{"className":1301},[189],[35,1303,1305],{"className":1304,"style":228},[193],[35,1306],{}," is the active parameter count per token. Precision sets bytes per parameter: 2 for bf16, which we use throughout, or 1 for fp8.",[10,1309,1310,1311,1315],{},"For hardware, an H200 has 141 GB of HBM, 4.8 TB\u002Fs of bandwidth, and about 990 dense bf16 TFLOPS ",[1312,1313],"content-cite",{"n":1314},"1",". The datasheet's larger bf16 number assumes sparsity; for dense weights, use half.",[10,1317,1318],{},"TPOT means time per output token. In the examples below, the decode SLO is 50 ms\u002Ftoken. If the model writes 1,000 output tokens, that is a 50 second decode window.",[10,1320,1321],{},"The running anchor is the case I'll carry through the memory, prefill, and decode bills, before varying the parameters at the end:",[35,1323,1325],{"className":1324},[38],[35,1326,1328,1400],{"className":1327},[42],[35,1329,1331],{"className":1330},[46],[48,1332,1333],{"xmlns":50,"display":51},[53,1334,1335,1397],{},[56,1336,1337,1339,1341,1345,1348,1350,1353,1359,1361,1364,1366,1368,1374,1376,1379,1381,1383,1386,1388,1391,1394],{},[59,1338,938],{},[63,1340,65],{},[1342,1343,1344],"mn",{},"2",[84,1346,1347],{}," req\u002Fs",[63,1349,89],{"separator":75},[139,1351],{"width":1352},"1em",[79,1354,1355,1357],{},[59,1356,971],{},[84,1358,974],{},[63,1360,65],{},[1342,1362,1363],{},"4,000",[63,1365,89],{"separator":75},[139,1367],{"width":1352},[79,1369,1370,1372],{},[59,1371,971],{},[84,1373,1050],{},[63,1375,65],{},[1342,1377,1378],{},"1,000",[63,1380,89],{"separator":75},[139,1382],{"width":1352},[59,1384,1385],{},"t",[63,1387,65],{},[1342,1389,1390],{},"50",[84,1392,1393],{}," ms",[59,1395,917],{"mathvariant":1396},"normal",[114,1398,1399],{"encoding":116},"R = 2\\text{ req\u002Fs}, \\quad L_\\text{in}=4{,}000, \\quad L_\\text{out}=1{,}000, \\quad t=50\\text{ ms}.",[35,1401,1403,1421,1499,1581,1621],{"className":1402,"ariaHidden":75},[121],[35,1404,1406,1409,1412,1415,1418],{"className":1405},[125],[35,1407],{"className":1408,"style":130},[129],[35,1410,938],{"className":1411,"style":953},[134,135],[35,1413],{"className":1414,"style":140},[139],[35,1416,65],{"className":1417},[144],[35,1419],{"className":1420,"style":140},[139],[35,1422,1424,1428,1431,1437,1440,1444,1447,1490,1493,1496],{"className":1423},[125],[35,1425],{"className":1426,"style":1427},[129],"height:1em;vertical-align:-0.25em;",[35,1429,1344],{"className":1430},[134],[35,1432,1434],{"className":1433},[134,213],[35,1435,1347],{"className":1436},[134],[35,1438,89],{"className":1439},[234],[35,1441],{"className":1442,"style":1443},[139],"margin-right:1em;",[35,1445],{"className":1446,"style":162},[139],[35,1448,1450,1453],{"className":1449},[134],[35,1451,971],{"className":1452},[134,135],[35,1454,1456],{"className":1455},[180],[35,1457,1459,1482],{"className":1458},[184,185],[35,1460,1462,1479],{"className":1461},[189],[35,1463,1465],{"className":1464,"style":1005},[193],[35,1466,1467,1470],{"style":197},[35,1468],{"className":1469,"style":202},[201],[35,1471,1473],{"className":1472},[206,207,208,209],[35,1474,1476],{"className":1475},[134,213,209],[35,1477,974],{"className":1478},[134,209],[35,1480,221],{"className":1481},[220],[35,1483,1485],{"className":1484},[189],[35,1486,1488],{"className":1487,"style":228},[193],[35,1489],{},[35,1491],{"className":1492,"style":140},[139],[35,1494,65],{"className":1495},[144],[35,1497],{"className":1498,"style":140},[139],[35,1500,1502,1506,1510,1516,1520,1523,1526,1529,1572,1575,1578],{"className":1501},[125],[35,1503],{"className":1504,"style":1505},[129],"height:0.8778em;vertical-align:-0.1944em;",[35,1507,1509],{"className":1508},[134],"4",[35,1511,1513],{"className":1512},[134],[35,1514,89],{"className":1515},[234],[35,1517,1519],{"className":1518},[134],"000",[35,1521,89],{"className":1522},[234],[35,1524],{"className":1525,"style":1443},[139],[35,1527],{"className":1528,"style":162},[139],[35,1530,1532,1535],{"className":1531},[134],[35,1533,971],{"className":1534},[134,135],[35,1536,1538],{"className":1537},[180],[35,1539,1541,1564],{"className":1540},[184,185],[35,1542,1544,1561],{"className":1543},[189],[35,1545,1547],{"className":1546,"style":1081},[193],[35,1548,1549,1552],{"style":197},[35,1550],{"className":1551,"style":202},[201],[35,1553,1555],{"className":1554},[206,207,208,209],[35,1556,1558],{"className":1557},[134,213,209],[35,1559,1050],{"className":1560},[134,209],[35,1562,221],{"className":1563},[220],[35,1565,1567],{"className":1566},[189],[35,1568,1570],{"className":1569,"style":228},[193],[35,1571],{},[35,1573],{"className":1574,"style":140},[139],[35,1576,65],{"className":1577},[144],[35,1579],{"className":1580,"style":140},[139],[35,1582,1584,1588,1591,1597,1600,1603,1606,1609,1612,1615,1618],{"className":1583},[125],[35,1585],{"className":1586,"style":1587},[129],"height:0.8389em;vertical-align:-0.1944em;",[35,1589,1314],{"className":1590},[134],[35,1592,1594],{"className":1593},[134],[35,1595,89],{"className":1596},[234],[35,1598,1519],{"className":1599},[134],[35,1601,89],{"className":1602},[234],[35,1604],{"className":1605,"style":1443},[139],[35,1607],{"className":1608,"style":162},[139],[35,1610,1385],{"className":1611},[134,135],[35,1613],{"className":1614,"style":140},[139],[35,1616,65],{"className":1617},[144],[35,1619],{"className":1620,"style":140},[139],[35,1622,1624,1628,1631,1637],{"className":1623},[125],[35,1625],{"className":1626,"style":1627},[129],"height:0.6444em;",[35,1629,1390],{"className":1630},[134],[35,1632,1634],{"className":1633},[134,213],[35,1635,1393],{"className":1636},[134],[35,1638,917],{"className":1639},[134],[27,1641,1643],{"id":1642},"two-questions-three-resources","Two Questions, Three Resources",[10,1645,1646,1647,1649,1650,1652],{},"The two questions turn into three resource bills. ",[16,1648,18],{}," is about capacity: the weights and the live KV cache have to sit in GPU memory at the same time. ",[16,1651,24],{}," is about throughput, but generation does not have a single bottleneck from start to finish.",[10,1654,1655,1656,1659,1660,1663],{},"Prefill reads the prompt all at once, runs large matrix multiplies, and is compute-bound ",[1312,1657],{"n":1658},"8",". Decode writes one token at a time. Serving engines batch many streams together, but each decode step still streams weights and KV cache from HBM, so decode is bandwidth-bound ",[1312,1661],{"n":1662},"10",". That split is why the final count has one memory bill and two throughput bills.",[27,1665,1667],{"id":1666},"step-1-does-it-fit","Step 1: Does It Fit?",[10,1669,1670],{},"Capacity has two parts: the weights, which are fixed, and the KV cache, which grows with the number of live tokens. Start with the fixed part.",[1672,1673,1675],"h3",{"id":1674},"the-weights","The Weights",[10,1677,1678],{},"The weights are the easy part. Multiply parameters by bytes per parameter:",[35,1680,1682],{"className":1681},[38],[35,1683,1685,1714],{"className":1684},[42],[35,1686,1688],{"className":1687},[46],[48,1689,1690],{"xmlns":50,"display":51},[53,1691,1692,1711],{},[56,1693,1694,1701,1703,1705,1708],{},[79,1695,1696,1698],{},[59,1697,792],{},[84,1699,1700],{},"weights",[63,1702,65],{},[59,1704,680],{},[63,1706,1707],{},"×",[59,1709,1710],{},"b",[114,1712,1713],{"encoding":116},"M_\\text{weights} = P \\times b",[35,1715,1717,1775,1794],{"className":1716,"ariaHidden":75},[121],[35,1718,1720,1723,1766,1769,1772],{"className":1719},[125],[35,1721],{"className":1722,"style":493},[129],[35,1724,1726,1729],{"className":1725},[134],[35,1727,792],{"className":1728,"style":814},[134,135],[35,1730,1732],{"className":1731},[180],[35,1733,1735,1758],{"className":1734},[184,185],[35,1736,1738,1755],{"className":1737},[189],[35,1739,1741],{"className":1740,"style":194},[193],[35,1742,1743,1746],{"style":829},[35,1744],{"className":1745,"style":202},[201],[35,1747,1749],{"className":1748},[206,207,208,209],[35,1750,1752],{"className":1751},[134,213,209],[35,1753,1700],{"className":1754},[134,209],[35,1756,221],{"className":1757},[220],[35,1759,1761],{"className":1760},[189],[35,1762,1764],{"className":1763,"style":282},[193],[35,1765],{},[35,1767],{"className":1768,"style":140},[139],[35,1770,65],{"className":1771},[144],[35,1773],{"className":1774,"style":140},[139],[35,1776,1778,1782,1785,1788,1791],{"className":1777},[125],[35,1779],{"className":1780,"style":1781},[129],"height:0.7667em;vertical-align:-0.0833em;",[35,1783,680],{"className":1784,"style":695},[134,135],[35,1786],{"className":1787,"style":288},[139],[35,1789,1707],{"className":1790},[292],[35,1792],{"className":1793,"style":288},[139],[35,1795,1797,1801],{"className":1796},[125],[35,1798],{"className":1799,"style":1800},[129],"height:0.6944em;",[35,1802,1710],{"className":1803},[134,135],[10,1805,1806,1807,1835,1836,1839],{},"Here, ",[35,1808,1810,1823],{"className":1809},[42],[35,1811,1813],{"className":1812},[46],[48,1814,1815],{"xmlns":50},[53,1816,1817,1821],{},[56,1818,1819],{},[59,1820,1710],{},[114,1822,1710],{"encoding":116},[35,1824,1826],{"className":1825,"ariaHidden":75},[121],[35,1827,1829,1832],{"className":1828},[125],[35,1830],{"className":1831,"style":1800},[129],[35,1833,1710],{"className":1834},[134,135]," is bytes per parameter. Qwen3-8B is 16 GB at bf16, and the 30B-A3B MoE is 60 GB. That MoE number uses the ",[16,1837,1838],{},"total"," parameter count, not the 3B active count, because every expert has to be resident even though a given token routes through only a few.",[1672,1841,1843],{"id":1842},"the-kv-cache","The KV Cache",[10,1845,1846],{},"The KV cache is the term you cannot read off the model's parameter count. During generation, each token's keys and values are stored so later tokens can attend to them. The bytes per token depend on the attention design: number of layers, KV heads, head dimension, and precision.",[10,1848,1849,1850,1852],{},"For standard grouped-query attention ",[1312,1851],{"n":1344},", the expression includes both keys and values. The per-token cache is:",[35,1854,1856],{"className":1855},[38],[35,1857,1859,1915],{"className":1858},[42],[35,1860,1862],{"className":1861},[46],[48,1863,1864],{"xmlns":50,"display":51},[53,1865,1866,1912],{},[56,1867,1868,1875,1877,1879,1881,1889,1891,1898,1900,1908,1910],{},[79,1869,1870,1872],{},[59,1871,792],{},[84,1873,1874],{},"kv\u002Ftok",[63,1876,65],{},[1342,1878,1344],{},[63,1880,1707],{},[79,1882,1883,1886],{},[59,1884,1885],{},"n",[84,1887,1888],{},"layers",[63,1890,1707],{},[79,1892,1893,1895],{},[59,1894,1885],{},[84,1896,1897],{},"kv heads",[63,1899,1707],{},[79,1901,1902,1905],{},[59,1903,1904],{},"d",[84,1906,1907],{},"head",[63,1909,1707],{},[59,1911,1710],{},[114,1913,1914],{"encoding":116},"M_\\text{kv\u002Ftok} = 2 \\times n_\\text{layers} \\times n_\\text{kv heads} \\times d_\\text{head} \\times b",[35,1916,1918,1980,1999,2058,2117,2176],{"className":1917,"ariaHidden":75},[121],[35,1919,1921,1925,1971,1974,1977],{"className":1920},[125],[35,1922],{"className":1923,"style":1924},[129],"height:1.0385em;vertical-align:-0.3552em;",[35,1926,1928,1931],{"className":1927},[134],[35,1929,792],{"className":1930,"style":814},[134,135],[35,1932,1934],{"className":1933},[180],[35,1935,1937,1962],{"className":1936},[184,185],[35,1938,1940,1959],{"className":1939},[189],[35,1941,1944],{"className":1942,"style":1943},[193],"height:0.3448em;",[35,1945,1947,1950],{"style":1946},"top:-2.5198em;margin-left:-0.109em;margin-right:0.05em;",[35,1948],{"className":1949,"style":202},[201],[35,1951,1953],{"className":1952},[206,207,208,209],[35,1954,1956],{"className":1955},[134,213,209],[35,1957,1874],{"className":1958},[134,209],[35,1960,221],{"className":1961},[220],[35,1963,1965],{"className":1964},[189],[35,1966,1969],{"className":1967,"style":1968},[193],"height:0.3552em;",[35,1970],{},[35,1972],{"className":1973,"style":140},[139],[35,1975,65],{"className":1976},[144],[35,1978],{"className":1979,"style":140},[139],[35,1981,1983,1987,1990,1993,1996],{"className":1982},[125],[35,1984],{"className":1985,"style":1986},[129],"height:0.7278em;vertical-align:-0.0833em;",[35,1988,1344],{"className":1989},[134],[35,1991],{"className":1992,"style":288},[139],[35,1994,1707],{"className":1995},[292],[35,1997],{"className":1998,"style":288},[139],[35,2000,2002,2006,2049,2052,2055],{"className":2001},[125],[35,2003],{"className":2004,"style":2005},[129],"height:0.8694em;vertical-align:-0.2861em;",[35,2007,2009,2012],{"className":2008},[134],[35,2010,1885],{"className":2011},[134,135],[35,2013,2015],{"className":2014},[180],[35,2016,2018,2041],{"className":2017},[184,185],[35,2019,2021,2038],{"className":2020},[189],[35,2022,2024],{"className":2023,"style":194},[193],[35,2025,2026,2029],{"style":197},[35,2027],{"className":2028,"style":202},[201],[35,2030,2032],{"className":2031},[206,207,208,209],[35,2033,2035],{"className":2034},[134,213,209],[35,2036,1888],{"className":2037},[134,209],[35,2039,221],{"className":2040},[220],[35,2042,2044],{"className":2043},[189],[35,2045,2047],{"className":2046,"style":282},[193],[35,2048],{},[35,2050],{"className":2051,"style":288},[139],[35,2053,1707],{"className":2054},[292],[35,2056],{"className":2057,"style":288},[139],[35,2059,2061,2065,2108,2111,2114],{"className":2060},[125],[35,2062],{"className":2063,"style":2064},[129],"height:0.7333em;vertical-align:-0.15em;",[35,2066,2068,2071],{"className":2067},[134],[35,2069,1885],{"className":2070},[134,135],[35,2072,2074],{"className":2073},[180],[35,2075,2077,2100],{"className":2076},[184,185],[35,2078,2080,2097],{"className":2079},[189],[35,2081,2083],{"className":2082,"style":194},[193],[35,2084,2085,2088],{"style":197},[35,2086],{"className":2087,"style":202},[201],[35,2089,2091],{"className":2090},[206,207,208,209],[35,2092,2094],{"className":2093},[134,213,209],[35,2095,1897],{"className":2096},[134,209],[35,2098,221],{"className":2099},[220],[35,2101,2103],{"className":2102},[189],[35,2104,2106],{"className":2105,"style":228},[193],[35,2107],{},[35,2109],{"className":2110,"style":288},[139],[35,2112,1707],{"className":2113},[292],[35,2115],{"className":2116,"style":288},[139],[35,2118,2120,2124,2167,2170,2173],{"className":2119},[125],[35,2121],{"className":2122,"style":2123},[129],"height:0.8444em;vertical-align:-0.15em;",[35,2125,2127,2130],{"className":2126},[134],[35,2128,1904],{"className":2129},[134,135],[35,2131,2133],{"className":2132},[180],[35,2134,2136,2159],{"className":2135},[184,185],[35,2137,2139,2156],{"className":2138},[189],[35,2140,2142],{"className":2141,"style":194},[193],[35,2143,2144,2147],{"style":197},[35,2145],{"className":2146,"style":202},[201],[35,2148,2150],{"className":2149},[206,207,208,209],[35,2151,2153],{"className":2152},[134,213,209],[35,2154,1907],{"className":2155},[134,209],[35,2157,221],{"className":2158},[220],[35,2160,2162],{"className":2161},[189],[35,2163,2165],{"className":2164,"style":228},[193],[35,2166],{},[35,2168],{"className":2169,"style":288},[139],[35,2171,1707],{"className":2172},[292],[35,2174],{"className":2175,"style":288},[139],[35,2177,2179,2182],{"className":2178},[125],[35,2180],{"className":2181,"style":1800},[129],[35,2183,1710],{"className":2184},[134,135],[10,2186,2187],{},"The leading 2 is for keys and values. So yes, the KV-per-token number can be calculated, but you need architectural details: layer count, number of KV heads, head dimension, and dtype. If the model uses a hybrid or compressed attention scheme, use its actual cache layout instead of this standard-GQA formula.",[10,2189,2190,2191,2194],{},"Qwen3's dense line is a clean example: 8 KV heads and head dim 128 across sizes. That is 4 KB per layer at bf16 ",[1312,2192],{"n":2193},"6",". The cache is then 4 KB times the layer count. The 8B has 36 layers, so 144 KB per token; the 0.6B has 28 layers, so 112 KB. Despite a more than 10x parameter gap, their KV caches are in the same ballpark.",[10,2196,2197,2200],{},[16,2198,2199],{},"However",", nobody keeps layer counts in their head, so a rule of thumb helps. For standard dense GQA at bf16, the table below gives a ceiling; use it for any model whose attention you have not checked.",[352,2202,2203,2214],{},[355,2204,2205],{},[358,2206,2207,2211],{},[361,2208,2210],{"align":2209},"center","Model size",[361,2212,2213],{"align":2209},"KV cache per token",[372,2215,2216,2224,2232,2240],{},[358,2217,2218,2221],{},[377,2219,2220],{"align":2209},"0.6B",[377,2222,2223],{"align":2209},"~112 KB",[358,2225,2226,2229],{},[377,2227,2228],{"align":2209},"8B",[377,2230,2231],{"align":2209},"~144 KB",[358,2233,2234,2237],{},[377,2235,2236],{"align":2209},"32B",[377,2238,2239],{"align":2209},"~256 KB",[358,2241,2242,2245],{},[377,2243,2244],{"align":2209},"72B",[377,2246,2247],{"align":2209},"~320 KB",[2249,2250,2252,2255,2262],"content-collapsible",{"title":2251},"What if my model isn't standard GQA?",[10,2253,2254],{},"For most modern architectures, the table still works as a conservative ceiling. Newer attention variants usually shrink the KV cache. The savings vary a lot, though: the gap swings more than an order of magnitude depending on the trick.",[10,2256,2257,2258,2261],{},"Qwen3.5's Gated DeltaNet hybrid ",[1312,2259],{"n":2260},"3"," runs full attention on just one in four layers. The other three use linear attention, whose state does not grow with the sequence. So the model lands around a quarter of the standard-GQA figure.",[10,2263,2264,2265,2267],{},"DeepSeek's multi-head latent attention takes the other route: it keeps every layer, but compresses each layer's per-head keys and values into a single shared latent vector, cutting per-token KV storage by over 90% ",[1312,2266],{"n":2260},". One cuts how many layers cache, the other how much each layer caches.",[10,2269,2270,2271,2273],{},"The table is per token; multiply it by average sequence length and requests in flight to get total KV bytes in memory ",[1312,2272],{"n":1509},":",[35,2275,2277],{"className":2276},[38],[35,2278,2280,2317],{"className":2279},[42],[35,2281,2283],{"className":2282},[46],[48,2284,2285],{"xmlns":50,"display":51},[53,2286,2287,2314],{},[56,2288,2289,2296,2298,2304,2306,2309,2311],{},[79,2290,2291,2293],{},[59,2292,792],{},[84,2294,2295],{},"kv",[63,2297,65],{},[79,2299,2300,2302],{},[59,2301,792],{},[84,2303,1874],{},[63,2305,1707],{},[84,2307,2308],{},"seq len",[63,2310,1707],{},[84,2312,2313],{},"concurrency",[114,2315,2316],{"encoding":116},"M_\\text{kv} = M_\\text{kv\u002Ftok} \\times \\text{seq len} \\times \\text{concurrency}",[35,2318,2320,2378,2436,2458],{"className":2319,"ariaHidden":75},[121],[35,2321,2323,2326,2369,2372,2375],{"className":2322},[125],[35,2324],{"className":2325,"style":409},[129],[35,2327,2329,2332],{"className":2328},[134],[35,2330,792],{"className":2331,"style":814},[134,135],[35,2333,2335],{"className":2334},[180],[35,2336,2338,2361],{"className":2337},[184,185],[35,2339,2341,2358],{"className":2340},[189],[35,2342,2344],{"className":2343,"style":194},[193],[35,2345,2346,2349],{"style":829},[35,2347],{"className":2348,"style":202},[201],[35,2350,2352],{"className":2351},[206,207,208,209],[35,2353,2355],{"className":2354},[134,213,209],[35,2356,2295],{"className":2357},[134,209],[35,2359,221],{"className":2360},[220],[35,2362,2364],{"className":2363},[189],[35,2365,2367],{"className":2366,"style":228},[193],[35,2368],{},[35,2370],{"className":2371,"style":140},[139],[35,2373,65],{"className":2374},[144],[35,2376],{"className":2377,"style":140},[139],[35,2379,2381,2384,2427,2430,2433],{"className":2380},[125],[35,2382],{"className":2383,"style":1924},[129],[35,2385,2387,2390],{"className":2386},[134],[35,2388,792],{"className":2389,"style":814},[134,135],[35,2391,2393],{"className":2392},[180],[35,2394,2396,2419],{"className":2395},[184,185],[35,2397,2399,2416],{"className":2398},[189],[35,2400,2402],{"className":2401,"style":1943},[193],[35,2403,2404,2407],{"style":1946},[35,2405],{"className":2406,"style":202},[201],[35,2408,2410],{"className":2409},[206,207,208,209],[35,2411,2413],{"className":2412},[134,213,209],[35,2414,1874],{"className":2415},[134,209],[35,2417,221],{"className":2418},[220],[35,2420,2422],{"className":2421},[189],[35,2423,2425],{"className":2424,"style":1968},[193],[35,2426],{},[35,2428],{"className":2429,"style":288},[139],[35,2431,1707],{"className":2432},[292],[35,2434],{"className":2435,"style":288},[139],[35,2437,2439,2443,2449,2452,2455],{"className":2438},[125],[35,2440],{"className":2441,"style":2442},[129],"height:0.8889em;vertical-align:-0.1944em;",[35,2444,2446],{"className":2445},[134,213],[35,2447,2308],{"className":2448},[134],[35,2450],{"className":2451,"style":288},[139],[35,2453,1707],{"className":2454},[292],[35,2456],{"className":2457,"style":288},[139],[35,2459,2461,2465],{"className":2460},[125],[35,2462],{"className":2463,"style":2464},[129],"height:0.625em;vertical-align:-0.1944em;",[35,2466,2468],{"className":2467},[134,213],[35,2469,2313],{"className":2470},[134],[10,2472,2473],{},"Concurrency comes from Little's Law: requests in flight equal rate times latency. For generation, napkin latency is output tokens times TPOT; add prefill and queueing if those are meaningful fractions of your SLO:",[35,2475,2477],{"className":2476},[38],[35,2478,2480,2508],{"className":2479},[42],[35,2481,2483],{"className":2482},[46],[48,2484,2485],{"xmlns":50,"display":51},[53,2486,2487,2505],{},[56,2488,2489,2492,2495,2501,2503],{},[84,2490,2491],{},"latency",[63,2493,2494],{},"≈",[79,2496,2497,2499],{},[59,2498,971],{},[84,2500,1050],{},[63,2502,1707],{},[59,2504,1385],{},[114,2506,2507],{"encoding":116},"\\text{latency} \\approx L_\\text{out} \\times t",[35,2509,2511,2532,2590],{"className":2510,"ariaHidden":75},[121],[35,2512,2514,2517,2523,2526,2529],{"className":2513},[125],[35,2515],{"className":2516,"style":2442},[129],[35,2518,2520],{"className":2519},[134,213],[35,2521,2491],{"className":2522},[134],[35,2524],{"className":2525,"style":140},[139],[35,2527,2494],{"className":2528},[144],[35,2530],{"className":2531,"style":140},[139],[35,2533,2535,2538,2581,2584,2587],{"className":2534},[125],[35,2536],{"className":2537,"style":409},[129],[35,2539,2541,2544],{"className":2540},[134],[35,2542,971],{"className":2543},[134,135],[35,2545,2547],{"className":2546},[180],[35,2548,2550,2573],{"className":2549},[184,185],[35,2551,2553,2570],{"className":2552},[189],[35,2554,2556],{"className":2555,"style":1081},[193],[35,2557,2558,2561],{"style":197},[35,2559],{"className":2560,"style":202},[201],[35,2562,2564],{"className":2563},[206,207,208,209],[35,2565,2567],{"className":2566},[134,213,209],[35,2568,1050],{"className":2569},[134,209],[35,2571,221],{"className":2572},[220],[35,2574,2576],{"className":2575},[189],[35,2577,2579],{"className":2578,"style":228},[193],[35,2580],{},[35,2582],{"className":2583,"style":288},[139],[35,2585,1707],{"className":2586},[292],[35,2588],{"className":2589,"style":288},[139],[35,2591,2593,2597],{"className":2592},[125],[35,2594],{"className":2595,"style":2596},[129],"height:0.6151em;",[35,2598,1385],{"className":2599},[134,135],[10,2601,2602,2603,2707],{},"For the anchor, TPOT is 50 ms and ",[35,2604,2606,2628],{"className":2605},[42],[35,2607,2609],{"className":2608},[46],[48,2610,2611],{"xmlns":50},[53,2612,2613,2625],{},[56,2614,2615,2621,2623],{},[79,2616,2617,2619],{},[59,2618,971],{},[84,2620,1050],{},[63,2622,65],{},[1342,2624,1378],{},[114,2626,2627],{"encoding":116},"L_\\text{out}=1{,}000",[35,2629,2631,2689],{"className":2630,"ariaHidden":75},[121],[35,2632,2634,2637,2680,2683,2686],{"className":2633},[125],[35,2635],{"className":2636,"style":409},[129],[35,2638,2640,2643],{"className":2639},[134],[35,2641,971],{"className":2642},[134,135],[35,2644,2646],{"className":2645},[180],[35,2647,2649,2672],{"className":2648},[184,185],[35,2650,2652,2669],{"className":2651},[189],[35,2653,2655],{"className":2654,"style":1081},[193],[35,2656,2657,2660],{"style":197},[35,2658],{"className":2659,"style":202},[201],[35,2661,2663],{"className":2662},[206,207,208,209],[35,2664,2666],{"className":2665},[134,213,209],[35,2667,1050],{"className":2668},[134,209],[35,2670,221],{"className":2671},[220],[35,2673,2675],{"className":2674},[189],[35,2676,2678],{"className":2677,"style":228},[193],[35,2679],{},[35,2681],{"className":2682,"style":140},[139],[35,2684,65],{"className":2685},[144],[35,2687],{"className":2688,"style":140},[139],[35,2690,2692,2695,2698,2704],{"className":2691},[125],[35,2693],{"className":2694,"style":1587},[129],[35,2696,1314],{"className":2697},[134],[35,2699,2701],{"className":2700},[134],[35,2702,89],{"className":2703},[234],[35,2705,1519],{"className":2706},[134],", so each request occupies the system for about 50 seconds. At 2 req\u002Fs, Little's Law gives 100 requests in flight. The average sequence length during decode is:",[35,2709,2711],{"className":2710},[38],[35,2712,2714,2772],{"className":2713},[42],[35,2715,2717],{"className":2716},[46],[48,2718,2719],{"xmlns":50,"display":51},[53,2720,2721,2769],{},[56,2722,2723,2731,2733,2739,2741,2747,2750,2752,2754,2756,2758,2760,2762,2764,2766],{},[2724,2725,2726,2728],"mover",{"accent":75},[59,2727,1885],{},[63,2729,2730],{},"ˉ",[63,2732,65],{},[79,2734,2735,2737],{},[59,2736,971],{},[84,2738,974],{},[63,2740,102],{},[79,2742,2743,2745],{},[59,2744,971],{},[84,2746,1050],{},[59,2748,2749],{"mathvariant":1396},"\u002F",[1342,2751,1344],{},[63,2753,65],{},[1342,2755,1363],{},[63,2757,102],{},[1342,2759,1378],{},[59,2761,2749],{"mathvariant":1396},[1342,2763,1344],{},[63,2765,65],{},[1342,2767,2768],{},"4,500.",[114,2770,2771],{"encoding":116},"\\bar n = L_\\text{in} + L_\\text{out}\u002F2 = 4{,}000 + 1{,}000\u002F2 = 4{,}500.",[35,2773,2775,2827,2885,2947,2974,3002],{"className":2774,"ariaHidden":75},[121],[35,2776,2778,2782,2818,2821,2824],{"className":2777},[125],[35,2779],{"className":2780,"style":2781},[129],"height:0.5678em;",[35,2783,2786],{"className":2784},[134,2785],"accent",[35,2787,2789],{"className":2788},[184],[35,2790,2792],{"className":2791},[189],[35,2793,2795,2805],{"className":2794,"style":2781},[193],[35,2796,2798,2802],{"style":2797},"top:-3em;",[35,2799],{"className":2800,"style":2801},[201],"height:3em;",[35,2803,1885],{"className":2804},[134,135],[35,2806,2807,2810],{"style":2797},[35,2808],{"className":2809,"style":2801},[201],[35,2811,2815],{"className":2812,"style":2814},[2813],"accent-body","left:-0.25em;",[35,2816,2730],{"className":2817},[134],[35,2819],{"className":2820,"style":140},[139],[35,2822,65],{"className":2823},[144],[35,2825],{"className":2826,"style":140},[139],[35,2828,2830,2833,2876,2879,2882],{"className":2829},[125],[35,2831],{"className":2832,"style":409},[129],[35,2834,2836,2839],{"className":2835},[134],[35,2837,971],{"className":2838},[134,135],[35,2840,2842],{"className":2841},[180],[35,2843,2845,2868],{"className":2844},[184,185],[35,2846,2848,2865],{"className":2847},[189],[35,2849,2851],{"className":2850,"style":1005},[193],[35,2852,2853,2856],{"style":197},[35,2854],{"className":2855,"style":202},[201],[35,2857,2859],{"className":2858},[206,207,208,209],[35,2860,2862],{"className":2861},[134,213,209],[35,2863,974],{"className":2864},[134,209],[35,2866,221],{"className":2867},[220],[35,2869,2871],{"className":2870},[189],[35,2872,2874],{"className":2873,"style":228},[193],[35,2875],{},[35,2877],{"className":2878,"style":288},[139],[35,2880,102],{"className":2881},[292],[35,2883],{"className":2884,"style":288},[139],[35,2886,2888,2891,2934,2938,2941,2944],{"className":2887},[125],[35,2889],{"className":2890,"style":1427},[129],[35,2892,2894,2897],{"className":2893},[134],[35,2895,971],{"className":2896},[134,135],[35,2898,2900],{"className":2899},[180],[35,2901,2903,2926],{"className":2902},[184,185],[35,2904,2906,2923],{"className":2905},[189],[35,2907,2909],{"className":2908,"style":1081},[193],[35,2910,2911,2914],{"style":197},[35,2912],{"className":2913,"style":202},[201],[35,2915,2917],{"className":2916},[206,207,208,209],[35,2918,2920],{"className":2919},[134,213,209],[35,2921,1050],{"className":2922},[134,209],[35,2924,221],{"className":2925},[220],[35,2927,2929],{"className":2928},[189],[35,2930,2932],{"className":2931,"style":228},[193],[35,2933],{},[35,2935,2937],{"className":2936},[134],"\u002F2",[35,2939],{"className":2940,"style":140},[139],[35,2942,65],{"className":2943},[144],[35,2945],{"className":2946,"style":140},[139],[35,2948,2950,2953,2956,2962,2965,2968,2971],{"className":2949},[125],[35,2951],{"className":2952,"style":1587},[129],[35,2954,1509],{"className":2955},[134],[35,2957,2959],{"className":2958},[134],[35,2960,89],{"className":2961},[234],[35,2963,1519],{"className":2964},[134],[35,2966],{"className":2967,"style":288},[139],[35,2969,102],{"className":2970},[292],[35,2972],{"className":2973,"style":288},[139],[35,2975,2977,2980,2983,2989,2993,2996,2999],{"className":2976},[125],[35,2978],{"className":2979,"style":1427},[129],[35,2981,1314],{"className":2982},[134],[35,2984,2986],{"className":2985},[134],[35,2987,89],{"className":2988},[234],[35,2990,2992],{"className":2991},[134],"000\u002F2",[35,2994],{"className":2995,"style":140},[139],[35,2997,65],{"className":2998},[144],[35,3000],{"className":3001,"style":140},[139],[35,3003,3005,3008,3011,3017],{"className":3004},[125],[35,3006],{"className":3007,"style":1587},[129],[35,3009,1509],{"className":3010},[134],[35,3012,3014],{"className":3013},[134],[35,3015,89],{"className":3016},[234],[35,3018,3020],{"className":3019},[134],"500.",[10,3022,3023],{},"Qwen3-8B has 16 GB of bf16 weights and a standard-GQA cache of 144 KB\u002Ftoken, so:",[35,3025,3027],{"className":3026},[38],[35,3028,3030,3076],{"className":3029},[42],[35,3031,3033],{"className":3032},[46],[48,3034,3035],{"xmlns":50,"display":51},[53,3036,3037,3073],{},[56,3038,3039,3045,3047,3050,3053,3055,3058,3060,3063,3065,3068,3071],{},[79,3040,3041,3043],{},[59,3042,792],{},[84,3044,2295],{},[63,3046,2494],{},[1342,3048,3049],{},"144",[84,3051,3052],{}," KB",[63,3054,1707],{},[1342,3056,3057],{},"4,500",[63,3059,1707],{},[1342,3061,3062],{},"100",[63,3064,2494],{},[1342,3066,3067],{},"65",[84,3069,3070],{}," GB",[59,3072,917],{"mathvariant":1396},[114,3074,3075],{"encoding":116},"M_\\text{kv} \\approx 144\\text{ KB} \\times 4{,}500 \\times 100 \\approx 65\\text{ GB}.",[35,3077,3079,3137,3161,3189,3207],{"className":3078,"ariaHidden":75},[121],[35,3080,3082,3085,3128,3131,3134],{"className":3081},[125],[35,3083],{"className":3084,"style":409},[129],[35,3086,3088,3091],{"className":3087},[134],[35,3089,792],{"className":3090,"style":814},[134,135],[35,3092,3094],{"className":3093},[180],[35,3095,3097,3120],{"className":3096},[184,185],[35,3098,3100,3117],{"className":3099},[189],[35,3101,3103],{"className":3102,"style":194},[193],[35,3104,3105,3108],{"style":829},[35,3106],{"className":3107,"style":202},[201],[35,3109,3111],{"className":3110},[206,207,208,209],[35,3112,3114],{"className":3113},[134,213,209],[35,3115,2295],{"className":3116},[134,209],[35,3118,221],{"className":3119},[220],[35,3121,3123],{"className":3122},[189],[35,3124,3126],{"className":3125,"style":228},[193],[35,3127],{},[35,3129],{"className":3130,"style":140},[139],[35,3132,2494],{"className":3133},[144],[35,3135],{"className":3136,"style":140},[139],[35,3138,3140,3143,3146,3152,3155,3158],{"className":3139},[125],[35,3141],{"className":3142,"style":1781},[129],[35,3144,3049],{"className":3145},[134],[35,3147,3149],{"className":3148},[134,213],[35,3150,3052],{"className":3151},[134],[35,3153],{"className":3154,"style":288},[139],[35,3156,1707],{"className":3157},[292],[35,3159],{"className":3160,"style":288},[139],[35,3162,3164,3167,3170,3176,3180,3183,3186],{"className":3163},[125],[35,3165],{"className":3166,"style":1587},[129],[35,3168,1509],{"className":3169},[134],[35,3171,3173],{"className":3172},[134],[35,3174,89],{"className":3175},[234],[35,3177,3179],{"className":3178},[134],"500",[35,3181],{"className":3182,"style":288},[139],[35,3184,1707],{"className":3185},[292],[35,3187],{"className":3188,"style":288},[139],[35,3190,3192,3195,3198,3201,3204],{"className":3191},[125],[35,3193],{"className":3194,"style":1627},[129],[35,3196,3062],{"className":3197},[134],[35,3199],{"className":3200,"style":140},[139],[35,3202,2494],{"className":3203},[144],[35,3205],{"className":3206,"style":140},[139],[35,3208,3210,3213,3216,3222],{"className":3209},[125],[35,3211],{"className":3212,"style":130},[129],[35,3214,3067],{"className":3215},[134],[35,3217,3219],{"className":3218},[134,213],[35,3220,3070],{"className":3221},[134],[35,3223,917],{"className":3224},[134],[10,3226,3227],{},"The memory you need to fit is weights plus KV cache, with about 15% extra for CUDA context, fragmentation, and activation buffers:",[35,3229,3231],{"className":3230},[38],[35,3232,3234,3277],{"className":3233},[42],[35,3235,3237],{"className":3236},[46],[48,3238,3239],{"xmlns":50,"display":51},[53,3240,3241,3274],{},[56,3242,3243,3249,3251,3253,3259,3261,3267,3269,3271],{},[79,3244,3245,3247],{},[59,3246,792],{},[84,3248,86],{},[63,3250,65],{},[63,3252,77],{"stretchy":74},[79,3254,3255,3257],{},[59,3256,792],{},[84,3258,1700],{},[63,3260,102],{},[79,3262,3263,3265],{},[59,3264,792],{},[84,3266,2295],{},[63,3268,112],{"stretchy":74},[63,3270,1707],{},[1342,3272,3273],{},"1.15",[114,3275,3276],{"encoding":116},"M_\\text{fit} = (M_\\text{weights} + M_\\text{kv}) \\times 1.15",[35,3278,3280,3338,3401,3463],{"className":3279,"ariaHidden":75},[121],[35,3281,3283,3286,3329,3332,3335],{"className":3282},[125],[35,3284],{"className":3285,"style":409},[129],[35,3287,3289,3292],{"className":3288},[134],[35,3290,792],{"className":3291,"style":814},[134,135],[35,3293,3295],{"className":3294},[180],[35,3296,3298,3321],{"className":3297},[184,185],[35,3299,3301,3318],{"className":3300},[189],[35,3302,3304],{"className":3303,"style":194},[193],[35,3305,3306,3309],{"style":829},[35,3307],{"className":3308,"style":202},[201],[35,3310,3312],{"className":3311},[206,207,208,209],[35,3313,3315],{"className":3314},[134,213,209],[35,3316,86],{"className":3317},[134,209],[35,3319,221],{"className":3320},[220],[35,3322,3324],{"className":3323},[189],[35,3325,3327],{"className":3326,"style":228},[193],[35,3328],{},[35,3330],{"className":3331,"style":140},[139],[35,3333,65],{"className":3334},[144],[35,3336],{"className":3337,"style":140},[139],[35,3339,3341,3345,3349,3392,3395,3398],{"className":3340},[125],[35,3342],{"className":3343,"style":3344},[129],"height:1.0361em;vertical-align:-0.2861em;",[35,3346,77],{"className":3347},[3348],"mopen",[35,3350,3352,3355],{"className":3351},[134],[35,3353,792],{"className":3354,"style":814},[134,135],[35,3356,3358],{"className":3357},[180],[35,3359,3361,3384],{"className":3360},[184,185],[35,3362,3364,3381],{"className":3363},[189],[35,3365,3367],{"className":3366,"style":194},[193],[35,3368,3369,3372],{"style":829},[35,3370],{"className":3371,"style":202},[201],[35,3373,3375],{"className":3374},[206,207,208,209],[35,3376,3378],{"className":3377},[134,213,209],[35,3379,1700],{"className":3380},[134,209],[35,3382,221],{"className":3383},[220],[35,3385,3387],{"className":3386},[189],[35,3388,3390],{"className":3389,"style":282},[193],[35,3391],{},[35,3393],{"className":3394,"style":288},[139],[35,3396,102],{"className":3397},[292],[35,3399],{"className":3400,"style":288},[139],[35,3402,3404,3407,3450,3454,3457,3460],{"className":3403},[125],[35,3405],{"className":3406,"style":1427},[129],[35,3408,3410,3413],{"className":3409},[134],[35,3411,792],{"className":3412,"style":814},[134,135],[35,3414,3416],{"className":3415},[180],[35,3417,3419,3442],{"className":3418},[184,185],[35,3420,3422,3439],{"className":3421},[189],[35,3423,3425],{"className":3424,"style":194},[193],[35,3426,3427,3430],{"style":829},[35,3428],{"className":3429,"style":202},[201],[35,3431,3433],{"className":3432},[206,207,208,209],[35,3434,3436],{"className":3435},[134,213,209],[35,3437,2295],{"className":3438},[134,209],[35,3440,221],{"className":3441},[220],[35,3443,3445],{"className":3444},[189],[35,3446,3448],{"className":3447,"style":228},[193],[35,3449],{},[35,3451,112],{"className":3452},[3453],"mclose",[35,3455],{"className":3456,"style":288},[139],[35,3458,1707],{"className":3459},[292],[35,3461],{"className":3462,"style":288},[139],[35,3464,3466,3469],{"className":3465},[125],[35,3467],{"className":3468,"style":1627},[129],[35,3470,3273],{"className":3471},[134],[35,3473,3475],{"className":3474},[38],[35,3476,3478,3517],{"className":3477},[42],[35,3479,3481],{"className":3480},[46],[48,3482,3483],{"xmlns":50,"display":51},[53,3484,3485,3514],{},[56,3486,3487,3493,3495,3501,3504,3506,3508],{},[79,3488,3489,3491],{},[59,3490,61],{},[84,3492,86],{},[63,3494,65],{},[79,3496,3497,3499],{},[59,3498,792],{},[84,3500,86],{},[84,3502,3503],{}," ",[59,3505,2749],{"mathvariant":1396},[84,3507,3503],{},[79,3509,3510,3512],{},[59,3511,792],{},[84,3513,795],{},[114,3515,3516],{"encoding":116},"G_\\text{fit} = M_\\text{fit} \\,\u002F\\, M_\\text{gpu}",[35,3518,3520,3578],{"className":3519,"ariaHidden":75},[121],[35,3521,3523,3526,3569,3572,3575],{"className":3522},[125],[35,3524],{"className":3525,"style":409},[129],[35,3527,3529,3532],{"className":3528},[134],[35,3530,61],{"className":3531},[134,135],[35,3533,3535],{"className":3534},[180],[35,3536,3538,3561],{"className":3537},[184,185],[35,3539,3541,3558],{"className":3540},[189],[35,3542,3544],{"className":3543,"style":194},[193],[35,3545,3546,3549],{"style":197},[35,3547],{"className":3548,"style":202},[201],[35,3550,3552],{"className":3551},[206,207,208,209],[35,3553,3555],{"className":3554},[134,213,209],[35,3556,86],{"className":3557},[134,209],[35,3559,221],{"className":3560},[220],[35,3562,3564],{"className":3563},[189],[35,3565,3567],{"className":3566,"style":228},[193],[35,3568],{},[35,3570],{"className":3571,"style":140},[139],[35,3573,65],{"className":3574},[144],[35,3576],{"className":3577,"style":140},[139],[35,3579,3581,3584,3627,3630,3633,3636],{"className":3580},[125],[35,3582],{"className":3583,"style":3344},[129],[35,3585,3587,3590],{"className":3586},[134],[35,3588,792],{"className":3589,"style":814},[134,135],[35,3591,3593],{"className":3592},[180],[35,3594,3596,3619],{"className":3595},[184,185],[35,3597,3599,3616],{"className":3598},[189],[35,3600,3602],{"className":3601,"style":194},[193],[35,3603,3604,3607],{"style":829},[35,3605],{"className":3606,"style":202},[201],[35,3608,3610],{"className":3609},[206,207,208,209],[35,3611,3613],{"className":3612},[134,213,209],[35,3614,86],{"className":3615},[134,209],[35,3617,221],{"className":3618},[220],[35,3620,3622],{"className":3621},[189],[35,3623,3625],{"className":3624,"style":228},[193],[35,3626],{},[35,3628],{"className":3629,"style":162},[139],[35,3631,2749],{"className":3632},[134],[35,3634],{"className":3635,"style":162},[139],[35,3637,3639,3642],{"className":3638},[134],[35,3640,792],{"className":3641,"style":814},[134,135],[35,3643,3645],{"className":3644},[180],[35,3646,3648,3671],{"className":3647},[184,185],[35,3649,3651,3668],{"className":3650},[189],[35,3652,3654],{"className":3653,"style":745},[193],[35,3655,3656,3659],{"style":829},[35,3657],{"className":3658,"style":202},[201],[35,3660,3662],{"className":3661},[206,207,208,209],[35,3663,3665],{"className":3664},[134,213,209],[35,3666,795],{"className":3667},[134,209],[35,3669,221],{"className":3670},[220],[35,3672,3674],{"className":3673},[189],[35,3675,3677],{"className":3676,"style":282},[193],[35,3678],{},[10,3680,3681],{},"Dividing by 141 GB of H200 memory gives the fit bill:",[35,3683,3685],{"className":3684},[38],[35,3686,3688,3744],{"className":3687},[42],[35,3689,3691],{"className":3690},[46],[48,3692,3693],{"xmlns":50,"display":51},[53,3694,3695,3741],{},[56,3696,3697,3703,3705,3736,3738],{},[79,3698,3699,3701],{},[59,3700,61],{},[84,3702,86],{},[63,3704,2494],{},[3706,3707,3708,3729],"mfrac",{},[56,3709,3710,3712,3715,3717,3719,3721,3723,3725,3727],{},[63,3711,77],{"stretchy":74},[1342,3713,3714],{},"16",[84,3716,3070],{},[63,3718,102],{},[1342,3720,3067],{},[84,3722,3070],{},[63,3724,112],{"stretchy":74},[63,3726,1707],{},[1342,3728,3273],{},[56,3730,3731,3734],{},[1342,3732,3733],{},"141",[84,3735,3070],{},[63,3737,2494],{},[1342,3739,3740],{},"0.7",[114,3742,3743],{"encoding":116},"G_\\text{fit} \\approx \\frac{(16\\text{ GB} + 65\\text{ GB}) \\times 1.15}{141\\text{ GB}} \\approx 0.7",[35,3745,3747,3805,3939],{"className":3746,"ariaHidden":75},[121],[35,3748,3750,3753,3796,3799,3802],{"className":3749},[125],[35,3751],{"className":3752,"style":409},[129],[35,3754,3756,3759],{"className":3755},[134],[35,3757,61],{"className":3758},[134,135],[35,3760,3762],{"className":3761},[180],[35,3763,3765,3788],{"className":3764},[184,185],[35,3766,3768,3785],{"className":3767},[189],[35,3769,3771],{"className":3770,"style":194},[193],[35,3772,3773,3776],{"style":197},[35,3774],{"className":3775,"style":202},[201],[35,3777,3779],{"className":3778},[206,207,208,209],[35,3780,3782],{"className":3781},[134,213,209],[35,3783,86],{"className":3784},[134,209],[35,3786,221],{"className":3787},[220],[35,3789,3791],{"className":3790},[189],[35,3792,3794],{"className":3793,"style":228},[193],[35,3795],{},[35,3797],{"className":3798,"style":140},[139],[35,3800,2494],{"className":3801},[144],[35,3803],{"className":3804,"style":140},[139],[35,3806,3808,3812,3930,3933,3936],{"className":3807},[125],[35,3809],{"className":3810,"style":3811},[129],"height:2.113em;vertical-align:-0.686em;",[35,3813,3815,3819,3927],{"className":3814},[134],[35,3816],{"className":3817},[3348,3818],"nulldelimiter",[35,3820,3822],{"className":3821},[3706],[35,3823,3825,3918],{"className":3824},[184,185],[35,3826,3828,3915],{"className":3827},[189],[35,3829,3832,3850,3861],{"className":3830,"style":3831},[193],"height:1.427em;",[35,3833,3835,3838],{"style":3834},"top:-2.314em;",[35,3836],{"className":3837,"style":2801},[201],[35,3839,3841,3844],{"className":3840},[134],[35,3842,3733],{"className":3843},[134],[35,3845,3847],{"className":3846},[134,213],[35,3848,3070],{"className":3849},[134],[35,3851,3853,3856],{"style":3852},"top:-3.23em;",[35,3854],{"className":3855,"style":2801},[201],[35,3857],{"className":3858,"style":3860},[3859],"frac-line","border-bottom-width:0.04em;",[35,3862,3864,3867],{"style":3863},"top:-3.677em;",[35,3865],{"className":3866,"style":2801},[201],[35,3868,3870,3873,3876,3882,3885,3888,3891,3894,3900,3903,3906,3909,3912],{"className":3869},[134],[35,3871,77],{"className":3872},[3348],[35,3874,3714],{"className":3875},[134],[35,3877,3879],{"className":3878},[134,213],[35,3880,3070],{"className":3881},[134],[35,3883],{"className":3884,"style":288},[139],[35,3886,102],{"className":3887},[292],[35,3889],{"className":3890,"style":288},[139],[35,3892,3067],{"className":3893},[134],[35,3895,3897],{"className":3896},[134,213],[35,3898,3070],{"className":3899},[134],[35,3901,112],{"className":3902},[3453],[35,3904],{"className":3905,"style":288},[139],[35,3907,1707],{"className":3908},[292],[35,3910],{"className":3911,"style":288},[139],[35,3913,3273],{"className":3914},[134],[35,3916,221],{"className":3917},[220],[35,3919,3921],{"className":3920},[189],[35,3922,3925],{"className":3923,"style":3924},[193],"height:0.686em;",[35,3926],{},[35,3928],{"className":3929},[3453,3818],[35,3931],{"className":3932,"style":140},[139],[35,3934,2494],{"className":3935},[144],[35,3937],{"className":3938,"style":140},[139],[35,3940,3942,3945],{"className":3941},[125],[35,3943],{"className":3944,"style":1627},[129],[35,3946,3740],{"className":3947},[134],[3949,3950,3957,3958,3964],"aside",{"className":3951},[3952,3953,3954,3955,3956],"border-border","text-muted-foreground","my-6","border-l-2","pl-4","\n  ",[662,3959,3963],{"className":3960},[3961,3962],"text-foreground","font-medium","Takeaway:","\n  Fitting the model is weights plus live KV cache. KV cache scales with traffic shape and attention design, not just parameter count.\n",[27,3966,3968],{"id":3967},"step-2a-prefill-is-a-compute-bill","Step 2a: Prefill Is a Compute Bill",[10,3970,3971,3972,3974,3975,4003,4004,4049,4050,917],{},"Prefill reads the prompt. Every input token is processed together in one large matrix-multiply batch, so compute units stay saturated and the phase is compute-bound ",[1312,3973],{"n":1658},". That makes the GPU bill a FLOP budget: demand is rate times tokens-per-request times FLOPs-per-token, and supply is MFU times peak compute ",[35,3976,3978,3991],{"className":3977},[42],[35,3979,3981],{"className":3980},[46],[48,3982,3983],{"xmlns":50},[53,3984,3985,3989],{},[56,3986,3987],{},[59,3988,902],{},[114,3990,902],{"encoding":116},[35,3992,3994],{"className":3993,"ariaHidden":75},[121],[35,3995,3997,4000],{"className":3996},[125],[35,3998],{"className":3999,"style":130},[129],[35,4001,902],{"className":4002,"style":695},[134,135],", with MFU ",[35,4005,4007,4024],{"className":4006},[42],[35,4008,4010],{"className":4009},[46],[48,4011,4012],{"xmlns":50},[53,4013,4014,4021],{},[56,4015,4016,4018],{},[63,4017,2494],{},[1342,4019,4020],{},"0.5",[114,4022,4023],{"encoding":116},"\\approx 0.5",[35,4025,4027,4040],{"className":4026,"ariaHidden":75},[121],[35,4028,4030,4034,4037],{"className":4029},[125],[35,4031],{"className":4032,"style":4033},[129],"height:0.4831em;",[35,4035,2494],{"className":4036},[144],[35,4038],{"className":4039,"style":140},[139],[35,4041,4043,4046],{"className":4042},[125],[35,4044],{"className":4045,"style":1627},[129],[35,4047,4020],{"className":4048},[134]," ",[1312,4051],{"n":1658},[10,4053,4054,4055,4131,4132,4207,4208,4401],{},"The per-token cost ",[35,4056,4058,4077],{"className":4057},[42],[35,4059,4061],{"className":4060},[46],[48,4062,4063],{"xmlns":50},[53,4064,4065,4074],{},[56,4066,4067],{},[79,4068,4069,4072],{},[59,4070,4071],{},"C",[84,4073,974],{},[114,4075,4076],{"encoding":116},"C_\\text{in}",[35,4078,4080],{"className":4079,"ariaHidden":75},[121],[35,4081,4083,4086],{"className":4082},[125],[35,4084],{"className":4085,"style":409},[129],[35,4087,4089,4093],{"className":4088},[134],[35,4090,4071],{"className":4091,"style":4092},[134,135],"margin-right:0.0715em;",[35,4094,4096],{"className":4095},[180],[35,4097,4099,4123],{"className":4098},[184,185],[35,4100,4102,4120],{"className":4101},[189],[35,4103,4105],{"className":4104,"style":1005},[193],[35,4106,4108,4111],{"style":4107},"top:-2.55em;margin-left:-0.0715em;margin-right:0.05em;",[35,4109],{"className":4110,"style":202},[201],[35,4112,4114],{"className":4113},[206,207,208,209],[35,4115,4117],{"className":4116},[134,213,209],[35,4118,974],{"className":4119},[134,209],[35,4121,221],{"className":4122},[220],[35,4124,4126],{"className":4125},[189],[35,4127,4129],{"className":4128,"style":228},[193],[35,4130],{}," has two terms. The weight term (",[35,4133,4135,4155],{"className":4134},[42],[35,4136,4138],{"className":4137},[46],[48,4139,4140],{"xmlns":50},[53,4141,4142,4152],{},[56,4143,4144,4146],{},[1342,4145,1344],{},[79,4147,4148,4150],{},[59,4149,680],{},[59,4151,643],{},[114,4153,4154],{"encoding":116},"2P_a",[35,4156,4158],{"className":4157,"ariaHidden":75},[121],[35,4159,4161,4164,4167],{"className":4160},[125],[35,4162],{"className":4163,"style":409},[129],[35,4165,1344],{"className":4166},[134],[35,4168,4170,4173],{"className":4169},[134],[35,4171,680],{"className":4172,"style":695},[134,135],[35,4174,4176],{"className":4175},[180],[35,4177,4179,4199],{"className":4178},[184,185],[35,4180,4182,4196],{"className":4181},[189],[35,4183,4185],{"className":4184,"style":745},[193],[35,4186,4187,4190],{"style":748},[35,4188],{"className":4189,"style":202},[201],[35,4191,4193],{"className":4192},[206,207,208,209],[35,4194,643],{"className":4195},[134,135,209],[35,4197,221],{"className":4198},[220],[35,4200,4202],{"className":4201},[189],[35,4203,4205],{"className":4204,"style":228},[193],[35,4206],{},") is one multiply-add per active parameter. The attention term (",[35,4209,4211,4250],{"className":4210},[42],[35,4212,4214],{"className":4213},[46],[48,4215,4216],{"xmlns":50},[53,4217,4218,4247],{},[56,4219,4220,4222,4224,4230,4232,4239,4241],{},[1342,4221,1509],{},[84,4223,3503],{},[79,4225,4226,4228],{},[59,4227,1885],{},[84,4229,1888],{},[84,4231,3503],{},[79,4233,4234,4236],{},[59,4235,1904],{},[84,4237,4238],{},"hidden",[84,4240,3503],{},[79,4242,4243,4245],{},[59,4244,971],{},[84,4246,974],{},[114,4248,4249],{"encoding":116},"4\\,n_\\text{layers}\\,d_\\text{hidden}\\,L_\\text{in}",[35,4251,4253],{"className":4252,"ariaHidden":75},[121],[35,4254,4256,4260,4263,4266,4309,4312,4355,4358],{"className":4255},[125],[35,4257],{"className":4258,"style":4259},[129],"height:0.9805em;vertical-align:-0.2861em;",[35,4261,1509],{"className":4262},[134],[35,4264],{"className":4265,"style":162},[139],[35,4267,4269,4272],{"className":4268},[134],[35,4270,1885],{"className":4271},[134,135],[35,4273,4275],{"className":4274},[180],[35,4276,4278,4301],{"className":4277},[184,185],[35,4279,4281,4298],{"className":4280},[189],[35,4282,4284],{"className":4283,"style":194},[193],[35,4285,4286,4289],{"style":197},[35,4287],{"className":4288,"style":202},[201],[35,4290,4292],{"className":4291},[206,207,208,209],[35,4293,4295],{"className":4294},[134,213,209],[35,4296,1888],{"className":4297},[134,209],[35,4299,221],{"className":4300},[220],[35,4302,4304],{"className":4303},[189],[35,4305,4307],{"className":4306,"style":282},[193],[35,4308],{},[35,4310],{"className":4311,"style":162},[139],[35,4313,4315,4318],{"className":4314},[134],[35,4316,1904],{"className":4317},[134,135],[35,4319,4321],{"className":4320},[180],[35,4322,4324,4347],{"className":4323},[184,185],[35,4325,4327,4344],{"className":4326},[189],[35,4328,4330],{"className":4329,"style":194},[193],[35,4331,4332,4335],{"style":197},[35,4333],{"className":4334,"style":202},[201],[35,4336,4338],{"className":4337},[206,207,208,209],[35,4339,4341],{"className":4340},[134,213,209],[35,4342,4238],{"className":4343},[134,209],[35,4345,221],{"className":4346},[220],[35,4348,4350],{"className":4349},[189],[35,4351,4353],{"className":4352,"style":228},[193],[35,4354],{},[35,4356],{"className":4357,"style":162},[139],[35,4359,4361,4364],{"className":4360},[134],[35,4362,971],{"className":4363},[134,135],[35,4365,4367],{"className":4366},[180],[35,4368,4370,4393],{"className":4369},[184,185],[35,4371,4373,4390],{"className":4372},[189],[35,4374,4376],{"className":4375,"style":1005},[193],[35,4377,4378,4381],{"style":197},[35,4379],{"className":4380,"style":202},[201],[35,4382,4384],{"className":4383},[206,207,208,209],[35,4385,4387],{"className":4386},[134,213,209],[35,4388,974],{"className":4389},[134,209],[35,4391,221],{"className":4392},[220],[35,4394,4396],{"className":4395},[189],[35,4397,4399],{"className":4398,"style":228},[193],[35,4400],{},") comes from scoring each query against every prior key and summing weighted values across all layers:",[35,4403,4405],{"className":4404},[38],[35,4406,4408,4464],{"className":4407},[42],[35,4409,4411],{"className":4410},[46],[48,4412,4413],{"xmlns":50,"display":51},[53,4414,4415,4461],{},[56,4416,4417,4423,4425,4427,4433,4435,4437,4439,4445,4447,4453,4455],{},[79,4418,4419,4421],{},[59,4420,4071],{},[84,4422,974],{},[63,4424,65],{},[1342,4426,1344],{},[79,4428,4429,4431],{},[59,4430,680],{},[59,4432,643],{},[63,4434,102],{},[1342,4436,1509],{},[84,4438,3503],{},[79,4440,4441,4443],{},[59,4442,1885],{},[84,4444,1888],{},[84,4446,3503],{},[79,4448,4449,4451],{},[59,4450,1904],{},[84,4452,4238],{},[84,4454,3503],{},[79,4456,4457,4459],{},[59,4458,971],{},[84,4460,974],{},[114,4462,4463],{"encoding":116},"C_\\text{in} = 2P_a + 4\\,n_\\text{layers}\\,d_\\text{hidden}\\,L_\\text{in}",[35,4465,4467,4525,4583],{"className":4466,"ariaHidden":75},[121],[35,4468,4470,4473,4516,4519,4522],{"className":4469},[125],[35,4471],{"className":4472,"style":409},[129],[35,4474,4476,4479],{"className":4475},[134],[35,4477,4071],{"className":4478,"style":4092},[134,135],[35,4480,4482],{"className":4481},[180],[35,4483,4485,4508],{"className":4484},[184,185],[35,4486,4488,4505],{"className":4487},[189],[35,4489,4491],{"className":4490,"style":1005},[193],[35,4492,4493,4496],{"style":4107},[35,4494],{"className":4495,"style":202},[201],[35,4497,4499],{"className":4498},[206,207,208,209],[35,4500,4502],{"className":4501},[134,213,209],[35,4503,974],{"className":4504},[134,209],[35,4506,221],{"className":4507},[220],[35,4509,4511],{"className":4510},[189],[35,4512,4514],{"className":4513,"style":228},[193],[35,4515],{},[35,4517],{"className":4518,"style":140},[139],[35,4520,65],{"className":4521},[144],[35,4523],{"className":4524,"style":140},[139],[35,4526,4528,4531,4534,4574,4577,4580],{"className":4527},[125],[35,4529],{"className":4530,"style":409},[129],[35,4532,1344],{"className":4533},[134],[35,4535,4537,4540],{"className":4536},[134],[35,4538,680],{"className":4539,"style":695},[134,135],[35,4541,4543],{"className":4542},[180],[35,4544,4546,4566],{"className":4545},[184,185],[35,4547,4549,4563],{"className":4548},[189],[35,4550,4552],{"className":4551,"style":745},[193],[35,4553,4554,4557],{"style":748},[35,4555],{"className":4556,"style":202},[201],[35,4558,4560],{"className":4559},[206,207,208,209],[35,4561,643],{"className":4562},[134,135,209],[35,4564,221],{"className":4565},[220],[35,4567,4569],{"className":4568},[189],[35,4570,4572],{"className":4571,"style":228},[193],[35,4573],{},[35,4575],{"className":4576,"style":288},[139],[35,4578,102],{"className":4579},[292],[35,4581],{"className":4582,"style":288},[139],[35,4584,4586,4589,4592,4595,4638,4641,4684,4687],{"className":4585},[125],[35,4587],{"className":4588,"style":4259},[129],[35,4590,1509],{"className":4591},[134],[35,4593],{"className":4594,"style":162},[139],[35,4596,4598,4601],{"className":4597},[134],[35,4599,1885],{"className":4600},[134,135],[35,4602,4604],{"className":4603},[180],[35,4605,4607,4630],{"className":4606},[184,185],[35,4608,4610,4627],{"className":4609},[189],[35,4611,4613],{"className":4612,"style":194},[193],[35,4614,4615,4618],{"style":197},[35,4616],{"className":4617,"style":202},[201],[35,4619,4621],{"className":4620},[206,207,208,209],[35,4622,4624],{"className":4623},[134,213,209],[35,4625,1888],{"className":4626},[134,209],[35,4628,221],{"className":4629},[220],[35,4631,4633],{"className":4632},[189],[35,4634,4636],{"className":4635,"style":282},[193],[35,4637],{},[35,4639],{"className":4640,"style":162},[139],[35,4642,4644,4647],{"className":4643},[134],[35,4645,1904],{"className":4646},[134,135],[35,4648,4650],{"className":4649},[180],[35,4651,4653,4676],{"className":4652},[184,185],[35,4654,4656,4673],{"className":4655},[189],[35,4657,4659],{"className":4658,"style":194},[193],[35,4660,4661,4664],{"style":197},[35,4662],{"className":4663,"style":202},[201],[35,4665,4667],{"className":4666},[206,207,208,209],[35,4668,4670],{"className":4669},[134,213,209],[35,4671,4238],{"className":4672},[134,209],[35,4674,221],{"className":4675},[220],[35,4677,4679],{"className":4678},[189],[35,4680,4682],{"className":4681,"style":228},[193],[35,4683],{},[35,4685],{"className":4686,"style":162},[139],[35,4688,4690,4693],{"className":4689},[134],[35,4691,971],{"className":4692},[134,135],[35,4694,4696],{"className":4695},[180],[35,4697,4699,4722],{"className":4698},[184,185],[35,4700,4702,4719],{"className":4701},[189],[35,4703,4705],{"className":4704,"style":1005},[193],[35,4706,4707,4710],{"style":197},[35,4708],{"className":4709,"style":202},[201],[35,4711,4713],{"className":4712},[206,207,208,209],[35,4714,4716],{"className":4715},[134,213,209],[35,4717,974],{"className":4718},[134,209],[35,4720,221],{"className":4721},[220],[35,4723,4725],{"className":4724},[189],[35,4726,4728],{"className":4727,"style":228},[193],[35,4729],{},[35,4731,4733],{"className":4732},[38],[35,4734,4736,4789],{"className":4735},[42],[35,4737,4739],{"className":4738},[46],[48,4740,4741],{"xmlns":50,"display":51},[53,4742,4743,4786],{},[56,4744,4745,4751,4753,4784],{},[79,4746,4747,4749],{},[59,4748,61],{},[84,4750,99],{},[63,4752,65],{},[3706,4754,4755,4775],{},[56,4756,4757,4759,4761,4767,4769],{},[59,4758,938],{},[63,4760,1707],{},[79,4762,4763,4765],{},[59,4764,971],{},[84,4766,974],{},[63,4768,1707],{},[79,4770,4771,4773],{},[59,4772,4071],{},[84,4774,974],{},[56,4776,4777,4780,4782],{},[84,4778,4779],{},"MFU",[63,4781,1707],{},[59,4783,902],{},[59,4785,917],{"mathvariant":1396},[114,4787,4788],{"encoding":116},"G_\\text{prefill} = \\frac{R \\times L_\\text{in} \\times C_\\text{in}}{\\text{MFU} \\times F}.",[35,4790,4792,4850],{"className":4791,"ariaHidden":75},[121],[35,4793,4795,4798,4841,4844,4847],{"className":4794},[125],[35,4796],{"className":4797,"style":493},[129],[35,4799,4801,4804],{"className":4800},[134],[35,4802,61],{"className":4803},[134,135],[35,4805,4807],{"className":4806},[180],[35,4808,4810,4833],{"className":4809},[184,185],[35,4811,4813,4830],{"className":4812},[189],[35,4814,4816],{"className":4815,"style":194},[193],[35,4817,4818,4821],{"style":197},[35,4819],{"className":4820,"style":202},[201],[35,4822,4824],{"className":4823},[206,207,208,209],[35,4825,4827],{"className":4826},[134,213,209],[35,4828,99],{"className":4829},[134,209],[35,4831,221],{"className":4832},[220],[35,4834,4836],{"className":4835},[189],[35,4837,4839],{"className":4838,"style":282},[193],[35,4840],{},[35,4842],{"className":4843,"style":140},[139],[35,4845,65],{"className":4846},[144],[35,4848],{"className":4849,"style":140},[139],[35,4851,4853,4857,5040],{"className":4852},[125],[35,4854],{"className":4855,"style":4856},[129],"height:2.1297em;vertical-align:-0.7693em;",[35,4858,4860,4863,5037],{"className":4859},[134],[35,4861],{"className":4862},[3348,3818],[35,4864,4866],{"className":4865},[3706],[35,4867,4869,5028],{"className":4868},[184,185],[35,4870,4872,5025],{"className":4871},[189],[35,4873,4876,4902,4910],{"className":4874,"style":4875},[193],"height:1.3603em;",[35,4877,4878,4881],{"style":3834},[35,4879],{"className":4880,"style":2801},[201],[35,4882,4884,4890,4893,4896,4899],{"className":4883},[134],[35,4885,4887],{"className":4886},[134,213],[35,4888,4779],{"className":4889},[134],[35,4891],{"className":4892,"style":288},[139],[35,4894,1707],{"className":4895},[292],[35,4897],{"className":4898,"style":288},[139],[35,4900,902],{"className":4901,"style":695},[134,135],[35,4903,4904,4907],{"style":3852},[35,4905],{"className":4906,"style":2801},[201],[35,4908],{"className":4909,"style":3860},[3859],[35,4911,4912,4915],{"style":3863},[35,4913],{"className":4914,"style":2801},[201],[35,4916,4918,4921,4924,4927,4930,4973,4976,4979,4982],{"className":4917},[134],[35,4919,938],{"className":4920,"style":953},[134,135],[35,4922],{"className":4923,"style":288},[139],[35,4925,1707],{"className":4926},[292],[35,4928],{"className":4929,"style":288},[139],[35,4931,4933,4936],{"className":4932},[134],[35,4934,971],{"className":4935},[134,135],[35,4937,4939],{"className":4938},[180],[35,4940,4942,4965],{"className":4941},[184,185],[35,4943,4945,4962],{"className":4944},[189],[35,4946,4948],{"className":4947,"style":1005},[193],[35,4949,4950,4953],{"style":197},[35,4951],{"className":4952,"style":202},[201],[35,4954,4956],{"className":4955},[206,207,208,209],[35,4957,4959],{"className":4958},[134,213,209],[35,4960,974],{"className":4961},[134,209],[35,4963,221],{"className":4964},[220],[35,4966,4968],{"className":4967},[189],[35,4969,4971],{"className":4970,"style":228},[193],[35,4972],{},[35,4974],{"className":4975,"style":288},[139],[35,4977,1707],{"className":4978},[292],[35,4980],{"className":4981,"style":288},[139],[35,4983,4985,4988],{"className":4984},[134],[35,4986,4071],{"className":4987,"style":4092},[134,135],[35,4989,4991],{"className":4990},[180],[35,4992,4994,5017],{"className":4993},[184,185],[35,4995,4997,5014],{"className":4996},[189],[35,4998,5000],{"className":4999,"style":1005},[193],[35,5001,5002,5005],{"style":4107},[35,5003],{"className":5004,"style":202},[201],[35,5006,5008],{"className":5007},[206,207,208,209],[35,5009,5011],{"className":5010},[134,213,209],[35,5012,974],{"className":5013},[134,209],[35,5015,221],{"className":5016},[220],[35,5018,5020],{"className":5019},[189],[35,5021,5023],{"className":5022,"style":228},[193],[35,5024],{},[35,5026,221],{"className":5027},[220],[35,5029,5031],{"className":5030},[189],[35,5032,5035],{"className":5033,"style":5034},[193],"height:0.7693em;",[35,5036],{},[35,5038],{"className":5039},[3453,3818],[35,5041,917],{"className":5042},[134],[10,5044,5045,5046,5143,5144,5250],{},"For Qwen3-8B (",[35,5047,5049,5072],{"className":5048},[42],[35,5050,5052],{"className":5051},[46],[48,5053,5054],{"xmlns":50},[53,5055,5056,5069],{},[56,5057,5058,5064,5066],{},[79,5059,5060,5062],{},[59,5061,1885],{},[84,5063,1888],{},[63,5065,65],{},[1342,5067,5068],{},"36",[114,5070,5071],{"encoding":116},"n_\\text{layers}=36",[35,5073,5075,5134],{"className":5074,"ariaHidden":75},[121],[35,5076,5078,5082,5125,5128,5131],{"className":5077},[125],[35,5079],{"className":5080,"style":5081},[129],"height:0.7167em;vertical-align:-0.2861em;",[35,5083,5085,5088],{"className":5084},[134],[35,5086,1885],{"className":5087},[134,135],[35,5089,5091],{"className":5090},[180],[35,5092,5094,5117],{"className":5093},[184,185],[35,5095,5097,5114],{"className":5096},[189],[35,5098,5100],{"className":5099,"style":194},[193],[35,5101,5102,5105],{"style":197},[35,5103],{"className":5104,"style":202},[201],[35,5106,5108],{"className":5107},[206,207,208,209],[35,5109,5111],{"className":5110},[134,213,209],[35,5112,1888],{"className":5113},[134,209],[35,5115,221],{"className":5116},[220],[35,5118,5120],{"className":5119},[189],[35,5121,5123],{"className":5122,"style":282},[193],[35,5124],{},[35,5126],{"className":5127,"style":140},[139],[35,5129,65],{"className":5130},[144],[35,5132],{"className":5133,"style":140},[139],[35,5135,5137,5140],{"className":5136},[125],[35,5138],{"className":5139,"style":1627},[129],[35,5141,5068],{"className":5142},[134],", ",[35,5145,5147,5170],{"className":5146},[42],[35,5148,5150],{"className":5149},[46],[48,5151,5152],{"xmlns":50},[53,5153,5154,5167],{},[56,5155,5156,5162,5164],{},[79,5157,5158,5160],{},[59,5159,1904],{},[84,5161,4238],{},[63,5163,65],{},[1342,5165,5166],{},"4,096",[114,5168,5169],{"encoding":116},"d_\\text{hidden}=4{,}096",[35,5171,5173,5231],{"className":5172,"ariaHidden":75},[121],[35,5174,5176,5179,5222,5225,5228],{"className":5175},[125],[35,5177],{"className":5178,"style":2123},[129],[35,5180,5182,5185],{"className":5181},[134],[35,5183,1904],{"className":5184},[134,135],[35,5186,5188],{"className":5187},[180],[35,5189,5191,5214],{"className":5190},[184,185],[35,5192,5194,5211],{"className":5193},[189],[35,5195,5197],{"className":5196,"style":194},[193],[35,5198,5199,5202],{"style":197},[35,5200],{"className":5201,"style":202},[201],[35,5203,5205],{"className":5204},[206,207,208,209],[35,5206,5208],{"className":5207},[134,213,209],[35,5209,4238],{"className":5210},[134,209],[35,5212,221],{"className":5213},[220],[35,5215,5217],{"className":5216},[189],[35,5218,5220],{"className":5219,"style":228},[193],[35,5221],{},[35,5223],{"className":5224,"style":140},[139],[35,5226,65],{"className":5227},[144],[35,5229],{"className":5230,"style":140},[139],[35,5232,5234,5237,5240,5246],{"className":5233},[125],[35,5235],{"className":5236,"style":1587},[129],[35,5238,1509],{"className":5239},[134],[35,5241,5243],{"className":5242},[134],[35,5244,89],{"className":5245},[234],[35,5247,5249],{"className":5248},[134],"096","), the weight term is 16B FLOPs\u002Ftoken and the attention term adds 2.4B at a 4K prompt:",[35,5252,5254],{"className":5253},[38],[35,5255,5257,5312],{"className":5256},[42],[35,5258,5260],{"className":5259},[46],[48,5261,5262],{"xmlns":50,"display":51},[53,5263,5264,5309],{},[56,5265,5266,5272,5274,5276,5278,5280,5283,5285,5287,5289,5291,5293,5295,5297,5299,5301,5304,5307],{},[79,5267,5268,5270],{},[59,5269,4071],{},[84,5271,974],{},[63,5273,65],{},[1342,5275,1344],{},[63,5277,1707],{},[1342,5279,1658],{},[84,5281,5282],{},"B",[63,5284,102],{},[1342,5286,1509],{},[63,5288,1707],{},[1342,5290,5068],{},[63,5292,1707],{},[1342,5294,5166],{},[63,5296,1707],{},[1342,5298,1363],{},[63,5300,2494],{},[1342,5302,5303],{},"18.4",[84,5305,5306],{}," B FLOPs\u002Ftoken",[59,5308,917],{"mathvariant":1396},[114,5310,5311],{"encoding":116},"C_\\text{in} = 2 \\times 8\\text{B} + 4 \\times 36 \\times 4{,}096 \\times 4{,}000 \\approx 18.4\\text{ B FLOPs\u002Ftoken}.",[35,5313,5315,5373,5391,5415,5433,5451,5478,5505],{"className":5314,"ariaHidden":75},[121],[35,5316,5318,5321,5364,5367,5370],{"className":5317},[125],[35,5319],{"className":5320,"style":409},[129],[35,5322,5324,5327],{"className":5323},[134],[35,5325,4071],{"className":5326,"style":4092},[134,135],[35,5328,5330],{"className":5329},[180],[35,5331,5333,5356],{"className":5332},[184,185],[35,5334,5336,5353],{"className":5335},[189],[35,5337,5339],{"className":5338,"style":1005},[193],[35,5340,5341,5344],{"style":4107},[35,5342],{"className":5343,"style":202},[201],[35,5345,5347],{"className":5346},[206,207,208,209],[35,5348,5350],{"className":5349},[134,213,209],[35,5351,974],{"className":5352},[134,209],[35,5354,221],{"className":5355},[220],[35,5357,5359],{"className":5358},[189],[35,5360,5362],{"className":5361,"style":228},[193],[35,5363],{},[35,5365],{"className":5366,"style":140},[139],[35,5368,65],{"className":5369},[144],[35,5371],{"className":5372,"style":140},[139],[35,5374,5376,5379,5382,5385,5388],{"className":5375},[125],[35,5377],{"className":5378,"style":1986},[129],[35,5380,1344],{"className":5381},[134],[35,5383],{"className":5384,"style":288},[139],[35,5386,1707],{"className":5387},[292],[35,5389],{"className":5390,"style":288},[139],[35,5392,5394,5397,5400,5406,5409,5412],{"className":5393},[125],[35,5395],{"className":5396,"style":1781},[129],[35,5398,1658],{"className":5399},[134],[35,5401,5403],{"className":5402},[134,213],[35,5404,5282],{"className":5405},[134],[35,5407],{"className":5408,"style":288},[139],[35,5410,102],{"className":5411},[292],[35,5413],{"className":5414,"style":288},[139],[35,5416,5418,5421,5424,5427,5430],{"className":5417},[125],[35,5419],{"className":5420,"style":1986},[129],[35,5422,1509],{"className":5423},[134],[35,5425],{"className":5426,"style":288},[139],[35,5428,1707],{"className":5429},[292],[35,5431],{"className":5432,"style":288},[139],[35,5434,5436,5439,5442,5445,5448],{"className":5435},[125],[35,5437],{"className":5438,"style":1986},[129],[35,5440,5068],{"className":5441},[134],[35,5443],{"className":5444,"style":288},[139],[35,5446,1707],{"className":5447},[292],[35,5449],{"className":5450,"style":288},[139],[35,5452,5454,5457,5460,5466,5469,5472,5475],{"className":5453},[125],[35,5455],{"className":5456,"style":1587},[129],[35,5458,1509],{"className":5459},[134],[35,5461,5463],{"className":5462},[134],[35,5464,89],{"className":5465},[234],[35,5467,5249],{"className":5468},[134],[35,5470],{"className":5471,"style":288},[139],[35,5473,1707],{"className":5474},[292],[35,5476],{"className":5477,"style":288},[139],[35,5479,5481,5484,5487,5493,5496,5499,5502],{"className":5480},[125],[35,5482],{"className":5483,"style":1587},[129],[35,5485,1509],{"className":5486},[134],[35,5488,5490],{"className":5489},[134],[35,5491,89],{"className":5492},[234],[35,5494,1519],{"className":5495},[134],[35,5497],{"className":5498,"style":140},[139],[35,5500,2494],{"className":5501},[144],[35,5503],{"className":5504,"style":140},[139],[35,5506,5508,5511,5514,5520],{"className":5507},[125],[35,5509],{"className":5510,"style":1427},[129],[35,5512,5303],{"className":5513},[134],[35,5515,5517],{"className":5516},[134,213],[35,5518,5306],{"className":5519},[134],[35,5521,917],{"className":5522},[134],[35,5524,5526],{"className":5525},[38],[35,5527,5529,5582],{"className":5528},[42],[35,5530,5532],{"className":5531},[46],[48,5533,5534],{"xmlns":50,"display":51},[53,5535,5536,5579],{},[56,5537,5538,5544,5546,5574,5576],{},[79,5539,5540,5542],{},[59,5541,61],{},[84,5543,99],{},[63,5545,65],{},[3706,5547,5548,5562],{},[56,5549,5550,5552,5554,5556,5558,5560],{},[1342,5551,1344],{},[63,5553,1707],{},[1342,5555,1363],{},[63,5557,1707],{},[1342,5559,5303],{},[84,5561,5282],{},[56,5563,5564,5566,5568,5571],{},[1342,5565,4020],{},[63,5567,1707],{},[1342,5569,5570],{},"990",[84,5572,5573],{},"T",[63,5575,2494],{},[1342,5577,5578],{},"0.30.",[114,5580,5581],{"encoding":116},"G_\\text{prefill} = \\frac{2 \\times 4{,}000 \\times 18.4\\text{B}}{0.5 \\times 990\\text{T}} \\approx 0.30.",[35,5583,5585,5643,5777],{"className":5584,"ariaHidden":75},[121],[35,5586,5588,5591,5634,5637,5640],{"className":5587},[125],[35,5589],{"className":5590,"style":493},[129],[35,5592,5594,5597],{"className":5593},[134],[35,5595,61],{"className":5596},[134,135],[35,5598,5600],{"className":5599},[180],[35,5601,5603,5626],{"className":5602},[184,185],[35,5604,5606,5623],{"className":5605},[189],[35,5607,5609],{"className":5608,"style":194},[193],[35,5610,5611,5614],{"style":197},[35,5612],{"className":5613,"style":202},[201],[35,5615,5617],{"className":5616},[206,207,208,209],[35,5618,5620],{"className":5619},[134,213,209],[35,5621,99],{"className":5622},[134,209],[35,5624,221],{"className":5625},[220],[35,5627,5629],{"className":5628},[189],[35,5630,5632],{"className":5631,"style":282},[193],[35,5633],{},[35,5635],{"className":5636,"style":140},[139],[35,5638,65],{"className":5639},[144],[35,5641],{"className":5642,"style":140},[139],[35,5644,5646,5649,5768,5771,5774],{"className":5645},[125],[35,5647],{"className":5648,"style":4856},[129],[35,5650,5652,5655,5765],{"className":5651},[134],[35,5653],{"className":5654},[3348,3818],[35,5656,5658],{"className":5657},[3706],[35,5659,5661,5757],{"className":5660},[184,185],[35,5662,5664,5754],{"className":5663},[189],[35,5665,5667,5696,5704],{"className":5666,"style":4875},[193],[35,5668,5669,5672],{"style":3834},[35,5670],{"className":5671,"style":2801},[201],[35,5673,5675,5678,5681,5684,5687,5690],{"className":5674},[134],[35,5676,4020],{"className":5677},[134],[35,5679],{"className":5680,"style":288},[139],[35,5682,1707],{"className":5683},[292],[35,5685],{"className":5686,"style":288},[139],[35,5688,5570],{"className":5689},[134],[35,5691,5693],{"className":5692},[134,213],[35,5694,5573],{"className":5695},[134],[35,5697,5698,5701],{"style":3852},[35,5699],{"className":5700,"style":2801},[201],[35,5702],{"className":5703,"style":3860},[3859],[35,5705,5706,5709],{"style":3863},[35,5707],{"className":5708,"style":2801},[201],[35,5710,5712,5715,5718,5721,5724,5727,5733,5736,5739,5742,5745,5748],{"className":5711},[134],[35,5713,1344],{"className":5714},[134],[35,5716],{"className":5717,"style":288},[139],[35,5719,1707],{"className":5720},[292],[35,5722],{"className":5723,"style":288},[139],[35,5725,1509],{"className":5726},[134],[35,5728,5730],{"className":5729},[134],[35,5731,89],{"className":5732},[234],[35,5734,1519],{"className":5735},[134],[35,5737],{"className":5738,"style":288},[139],[35,5740,1707],{"className":5741},[292],[35,5743],{"className":5744,"style":288},[139],[35,5746,5303],{"className":5747},[134],[35,5749,5751],{"className":5750},[134,213],[35,5752,5282],{"className":5753},[134],[35,5755,221],{"className":5756},[220],[35,5758,5760],{"className":5759},[189],[35,5761,5763],{"className":5762,"style":5034},[193],[35,5764],{},[35,5766],{"className":5767},[3453,3818],[35,5769],{"className":5770,"style":140},[139],[35,5772,2494],{"className":5773},[144],[35,5775],{"className":5776,"style":140},[139],[35,5778,5780,5783],{"className":5779},[125],[35,5781],{"className":5782,"style":1627},[129],[35,5784,5578],{"className":5785},[134],[10,5787,5788,5789,5938,5939,6080,6081,6173],{},"Below ",[35,5790,5792,5822],{"className":5791},[42],[35,5793,5795],{"className":5794},[46],[48,5796,5797],{"xmlns":50},[53,5798,5799,5819],{},[56,5800,5801,5807,5809,5811,5813],{},[79,5802,5803,5805],{},[59,5804,971],{},[84,5806,974],{},[63,5808,2494],{},[1342,5810,2193],{},[84,5812,3503],{},[79,5814,5815,5817],{},[59,5816,1904],{},[84,5818,4238],{},[114,5820,5821],{"encoding":116},"L_\\text{in} \\approx 6\\,d_\\text{hidden}",[35,5823,5825,5883],{"className":5824,"ariaHidden":75},[121],[35,5826,5828,5831,5874,5877,5880],{"className":5827},[125],[35,5829],{"className":5830,"style":409},[129],[35,5832,5834,5837],{"className":5833},[134],[35,5835,971],{"className":5836},[134,135],[35,5838,5840],{"className":5839},[180],[35,5841,5843,5866],{"className":5842},[184,185],[35,5844,5846,5863],{"className":5845},[189],[35,5847,5849],{"className":5848,"style":1005},[193],[35,5850,5851,5854],{"style":197},[35,5852],{"className":5853,"style":202},[201],[35,5855,5857],{"className":5856},[206,207,208,209],[35,5858,5860],{"className":5859},[134,213,209],[35,5861,974],{"className":5862},[134,209],[35,5864,221],{"className":5865},[220],[35,5867,5869],{"className":5868},[189],[35,5870,5872],{"className":5871,"style":228},[193],[35,5873],{},[35,5875],{"className":5876,"style":140},[139],[35,5878,2494],{"className":5879},[144],[35,5881],{"className":5882,"style":140},[139],[35,5884,5886,5889,5892,5895],{"className":5885},[125],[35,5887],{"className":5888,"style":2123},[129],[35,5890,2193],{"className":5891},[134],[35,5893],{"className":5894,"style":162},[139],[35,5896,5898,5901],{"className":5897},[134],[35,5899,1904],{"className":5900},[134,135],[35,5902,5904],{"className":5903},[180],[35,5905,5907,5930],{"className":5906},[184,185],[35,5908,5910,5927],{"className":5909},[189],[35,5911,5913],{"className":5912,"style":194},[193],[35,5914,5915,5918],{"style":197},[35,5916],{"className":5917,"style":202},[201],[35,5919,5921],{"className":5920},[206,207,208,209],[35,5922,5924],{"className":5923},[134,213,209],[35,5925,4238],{"className":5926},[134,209],[35,5928,221],{"className":5929},[220],[35,5931,5933],{"className":5932},[189],[35,5934,5936],{"className":5935,"style":228},[193],[35,5937],{}," the weight term dominates and ",[35,5940,5942,5970],{"className":5941},[42],[35,5943,5945],{"className":5944},[46],[48,5946,5947],{"xmlns":50},[53,5948,5949,5967],{},[56,5950,5951,5957,5959,5961],{},[79,5952,5953,5955],{},[59,5954,4071],{},[84,5956,974],{},[63,5958,2494],{},[1342,5960,1344],{},[79,5962,5963,5965],{},[59,5964,680],{},[59,5966,643],{},[114,5968,5969],{"encoding":116},"C_\\text{in} \\approx 2P_a",[35,5971,5973,6031],{"className":5972,"ariaHidden":75},[121],[35,5974,5976,5979,6022,6025,6028],{"className":5975},[125],[35,5977],{"className":5978,"style":409},[129],[35,5980,5982,5985],{"className":5981},[134],[35,5983,4071],{"className":5984,"style":4092},[134,135],[35,5986,5988],{"className":5987},[180],[35,5989,5991,6014],{"className":5990},[184,185],[35,5992,5994,6011],{"className":5993},[189],[35,5995,5997],{"className":5996,"style":1005},[193],[35,5998,5999,6002],{"style":4107},[35,6000],{"className":6001,"style":202},[201],[35,6003,6005],{"className":6004},[206,207,208,209],[35,6006,6008],{"className":6007},[134,213,209],[35,6009,974],{"className":6010},[134,209],[35,6012,221],{"className":6013},[220],[35,6015,6017],{"className":6016},[189],[35,6018,6020],{"className":6019,"style":228},[193],[35,6021],{},[35,6023],{"className":6024,"style":140},[139],[35,6026,2494],{"className":6027},[144],[35,6029],{"className":6030,"style":140},[139],[35,6032,6034,6037,6040],{"className":6033},[125],[35,6035],{"className":6036,"style":409},[129],[35,6038,1344],{"className":6039},[134],[35,6041,6043,6046],{"className":6042},[134],[35,6044,680],{"className":6045,"style":695},[134,135],[35,6047,6049],{"className":6048},[180],[35,6050,6052,6072],{"className":6051},[184,185],[35,6053,6055,6069],{"className":6054},[189],[35,6056,6058],{"className":6057,"style":745},[193],[35,6059,6060,6063],{"style":748},[35,6061],{"className":6062,"style":202},[201],[35,6064,6066],{"className":6065},[206,207,208,209],[35,6067,643],{"className":6068},[134,135,209],[35,6070,221],{"className":6071},[220],[35,6073,6075],{"className":6074},[189],[35,6076,6078],{"className":6077,"style":228},[193],[35,6079],{},"; above it, attention takes over and total prefill compute scales as ",[35,6082,6084,6105],{"className":6083},[42],[35,6085,6087],{"className":6086},[46],[48,6088,6089],{"xmlns":50},[53,6090,6091,6102],{},[56,6092,6093],{},[6094,6095,6096,6098,6100],"msubsup",{},[59,6097,971],{},[84,6099,974],{},[1342,6101,1344],{},[114,6103,6104],{"encoding":116},"L_\\text{in}^2",[35,6106,6108],{"className":6107,"ariaHidden":75},[121],[35,6109,6111,6115],{"className":6110},[125],[35,6112],{"className":6113,"style":6114},[129],"height:1.0786em;vertical-align:-0.2645em;",[35,6116,6118,6121],{"className":6117},[134],[35,6119,971],{"className":6120},[134,135],[35,6122,6124],{"className":6123},[180],[35,6125,6127,6164],{"className":6126},[184,185],[35,6128,6130,6161],{"className":6129},[189],[35,6131,6134,6149],{"className":6132,"style":6133},[193],"height:0.8141em;",[35,6135,6137,6140],{"style":6136},"top:-2.4355em;margin-left:0em;margin-right:0.05em;",[35,6138],{"className":6139,"style":202},[201],[35,6141,6143],{"className":6142},[206,207,208,209],[35,6144,6146],{"className":6145},[134,213,209],[35,6147,974],{"className":6148},[134,209],[35,6150,6152,6155],{"style":6151},"top:-3.063em;margin-right:0.05em;",[35,6153],{"className":6154,"style":202},[201],[35,6156,6158],{"className":6157},[206,207,208,209],[35,6159,1344],{"className":6160},[134,209],[35,6162,221],{"className":6163},[220],[35,6165,6167],{"className":6166},[189],[35,6168,6171],{"className":6169,"style":6170},[193],"height:0.2645em;",[35,6172],{},". That crossover is roughly 6K tokens for a 0.6B model, up to 50K for 72B.",[2249,6175,6177,6515,7509,8131],{"title":6176},"Where the per-token cost comes from",[10,6178,6179,6182,6183,6252,6253,6327,6328,6420,6421,6514],{},[662,6180,6181],{},"The weight cost."," A forward pass multiplies the token against every active weight in the model, and each weight is one multiply and one add: 2 FLOPs. So ",[35,6184,6186,6203],{"className":6185},[42],[35,6187,6189],{"className":6188},[46],[48,6190,6191],{"xmlns":50},[53,6192,6193,6201],{},[56,6194,6195],{},[79,6196,6197,6199],{},[59,6198,680],{},[59,6200,643],{},[114,6202,717],{"encoding":116},[35,6204,6206],{"className":6205,"ariaHidden":75},[121],[35,6207,6209,6212],{"className":6208},[125],[35,6210],{"className":6211,"style":409},[129],[35,6213,6215,6218],{"className":6214},[134],[35,6216,680],{"className":6217,"style":695},[134,135],[35,6219,6221],{"className":6220},[180],[35,6222,6224,6244],{"className":6223},[184,185],[35,6225,6227,6241],{"className":6226},[189],[35,6228,6230],{"className":6229,"style":745},[193],[35,6231,6232,6235],{"style":748},[35,6233],{"className":6234,"style":202},[201],[35,6236,6238],{"className":6237},[206,207,208,209],[35,6239,643],{"className":6240},[134,135,209],[35,6242,221],{"className":6243},[220],[35,6245,6247],{"className":6246},[189],[35,6248,6250],{"className":6249,"style":228},[193],[35,6251],{}," active parameters cost ",[35,6254,6256,6275],{"className":6255},[42],[35,6257,6259],{"className":6258},[46],[48,6260,6261],{"xmlns":50},[53,6262,6263,6273],{},[56,6264,6265,6267],{},[1342,6266,1344],{},[79,6268,6269,6271],{},[59,6270,680],{},[59,6272,643],{},[114,6274,4154],{"encoding":116},[35,6276,6278],{"className":6277,"ariaHidden":75},[121],[35,6279,6281,6284,6287],{"className":6280},[125],[35,6282],{"className":6283,"style":409},[129],[35,6285,1344],{"className":6286},[134],[35,6288,6290,6293],{"className":6289},[134],[35,6291,680],{"className":6292,"style":695},[134,135],[35,6294,6296],{"className":6295},[180],[35,6297,6299,6319],{"className":6298},[184,185],[35,6300,6302,6316],{"className":6301},[189],[35,6303,6305],{"className":6304,"style":745},[193],[35,6306,6307,6310],{"style":748},[35,6308],{"className":6309,"style":202},[201],[35,6311,6313],{"className":6312},[206,207,208,209],[35,6314,643],{"className":6315},[134,135,209],[35,6317,221],{"className":6318},[220],[35,6320,6322],{"className":6321},[189],[35,6323,6325],{"className":6324,"style":228},[193],[35,6326],{}," per token. A dense model hits all of them (",[35,6329,6331,6353],{"className":6330},[42],[35,6332,6334],{"className":6333},[46],[48,6335,6336],{"xmlns":50},[53,6337,6338,6350],{},[56,6339,6340,6346,6348],{},[79,6341,6342,6344],{},[59,6343,680],{},[59,6345,643],{},[63,6347,65],{},[59,6349,680],{},[114,6351,6352],{"encoding":116},"P_a = P",[35,6354,6356,6411],{"className":6355,"ariaHidden":75},[121],[35,6357,6359,6362,6402,6405,6408],{"className":6358},[125],[35,6360],{"className":6361,"style":409},[129],[35,6363,6365,6368],{"className":6364},[134],[35,6366,680],{"className":6367,"style":695},[134,135],[35,6369,6371],{"className":6370},[180],[35,6372,6374,6394],{"className":6373},[184,185],[35,6375,6377,6391],{"className":6376},[189],[35,6378,6380],{"className":6379,"style":745},[193],[35,6381,6382,6385],{"style":748},[35,6383],{"className":6384,"style":202},[201],[35,6386,6388],{"className":6387},[206,207,208,209],[35,6389,643],{"className":6390},[134,135,209],[35,6392,221],{"className":6393},[220],[35,6395,6397],{"className":6396},[189],[35,6398,6400],{"className":6399,"style":228},[193],[35,6401],{},[35,6403],{"className":6404,"style":140},[139],[35,6406,65],{"className":6407},[144],[35,6409],{"className":6410,"style":140},[139],[35,6412,6414,6417],{"className":6413},[125],[35,6415],{"className":6416,"style":130},[129],[35,6418,680],{"className":6419,"style":695},[134,135],"); an MoE routes each token through only a few experts, so ",[35,6422,6424,6447],{"className":6423},[42],[35,6425,6427],{"className":6426},[46],[48,6428,6429],{"xmlns":50},[53,6430,6431,6444],{},[56,6432,6433,6439,6442],{},[79,6434,6435,6437],{},[59,6436,680],{},[59,6438,643],{},[63,6440,6441],{},"≪",[59,6443,680],{},[114,6445,6446],{"encoding":116},"P_a \\ll P",[35,6448,6450,6505],{"className":6449,"ariaHidden":75},[121],[35,6451,6453,6456,6496,6499,6502],{"className":6452},[125],[35,6454],{"className":6455,"style":409},[129],[35,6457,6459,6462],{"className":6458},[134],[35,6460,680],{"className":6461,"style":695},[134,135],[35,6463,6465],{"className":6464},[180],[35,6466,6468,6488],{"className":6467},[184,185],[35,6469,6471,6485],{"className":6470},[189],[35,6472,6474],{"className":6473,"style":745},[193],[35,6475,6476,6479],{"style":748},[35,6477],{"className":6478,"style":202},[201],[35,6480,6482],{"className":6481},[206,207,208,209],[35,6483,643],{"className":6484},[134,135,209],[35,6486,221],{"className":6487},[220],[35,6489,6491],{"className":6490},[189],[35,6492,6494],{"className":6493,"style":228},[193],[35,6495],{},[35,6497],{"className":6498,"style":140},[139],[35,6500,6441],{"className":6501},[144],[35,6503],{"className":6504,"style":140},[139],[35,6506,6508,6511],{"className":6507},[125],[35,6509],{"className":6510,"style":130},[129],[35,6512,680],{"className":6513,"style":695},[134,135]," and prefill is proportionally cheaper.",[10,6516,6517,6520,6521,6733,6734,6808,6809,6874,6875,6947,6948,7033,7034,7107,7108,7171,7172,7322,7323,7357,7358,7508],{},[662,6518,6519],{},"The attention cost."," Attention is ",[35,6522,6524,6569],{"className":6523},[42],[35,6525,6527],{"className":6526},[46],[48,6528,6529],{"xmlns":50},[53,6530,6531,6566],{},[56,6532,6533,6536,6538,6541,6550,6552,6561,6563],{},[84,6534,6535],{},"softmax",[63,6537,77],{"stretchy":74},[59,6539,6540],{},"Q",[6542,6543,6544,6547],"msup",{},[59,6545,6546],{},"K",[59,6548,6549],{"mathvariant":1396},"⊤",[59,6551,2749],{"mathvariant":1396},[6553,6554,6555],"msqrt",{},[79,6556,6557,6559],{},[59,6558,1904],{},[84,6560,1907],{},[63,6562,112],{"stretchy":74},[59,6564,6565],{},"V",[114,6567,6568],{"encoding":116},"\\text{softmax}(QK^\\top \u002F \\sqrt{d_\\text{head}})V",[35,6570,6572],{"className":6571,"ariaHidden":75},[121],[35,6573,6575,6579,6585,6588,6591,6621,6624,6727,6730],{"className":6574},[125],[35,6576],{"className":6577,"style":6578},[129],"height:1.1072em;vertical-align:-0.25em;",[35,6580,6582],{"className":6581},[134,213],[35,6583,6535],{"className":6584},[134],[35,6586,77],{"className":6587},[3348],[35,6589,6540],{"className":6590},[134,135],[35,6592,6594,6597],{"className":6593},[134],[35,6595,6546],{"className":6596,"style":4092},[134,135],[35,6598,6600],{"className":6599},[180],[35,6601,6603],{"className":6602},[184],[35,6604,6606],{"className":6605},[189],[35,6607,6610],{"className":6608,"style":6609},[193],"height:0.8491em;",[35,6611,6612,6615],{"style":6151},[35,6613],{"className":6614,"style":202},[201],[35,6616,6618],{"className":6617},[206,207,208,209],[35,6619,6549],{"className":6620},[134,209],[35,6622,2749],{"className":6623},[134],[35,6625,6628],{"className":6626},[134,6627],"sqrt",[35,6629,6631,6718],{"className":6630},[184,185],[35,6632,6634,6715],{"className":6633},[189],[35,6635,6638,6692],{"className":6636,"style":6637},[193],"height:0.8572em;",[35,6639,6642,6645],{"className":6640,"style":2797},[6641],"svg-align",[35,6643],{"className":6644,"style":2801},[201],[35,6646,6649],{"className":6647,"style":6648},[134],"padding-left:0.833em;",[35,6650,6652,6655],{"className":6651},[134],[35,6653,1904],{"className":6654},[134,135],[35,6656,6658],{"className":6657},[180],[35,6659,6661,6684],{"className":6660},[184,185],[35,6662,6664,6681],{"className":6663},[189],[35,6665,6667],{"className":6666,"style":194},[193],[35,6668,6669,6672],{"style":197},[35,6670],{"className":6671,"style":202},[201],[35,6673,6675],{"className":6674},[206,207,208,209],[35,6676,6678],{"className":6677},[134,213,209],[35,6679,1907],{"className":6680},[134,209],[35,6682,221],{"className":6683},[220],[35,6685,6687],{"className":6686},[189],[35,6688,6690],{"className":6689,"style":228},[193],[35,6691],{},[35,6693,6695,6698],{"style":6694},"top:-2.8172em;",[35,6696],{"className":6697,"style":2801},[201],[35,6699,6703],{"className":6700,"style":6702},[6701],"hide-tail","min-width:0.853em;height:1.08em;",[6704,6705,6711],"svg",{"xmlns":6706,"width":6707,"height":6708,"viewBox":6709,"preserveAspectRatio":6710},"http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","400em","1.08em","0 0 400000 1080","xMinYMin slice",[6712,6713],"path",{"d":6714},"M95,702\nc-2.7,0,-7.17,-2.7,-13.5,-8c-5.8,-5.3,-9.5,-10,-9.5,-14\nc0,-2,0.3,-3.3,1,-4c1.3,-2.7,23.83,-20.7,67.5,-54\nc44.2,-33.3,65.8,-50.3,66.5,-51c1.3,-1.3,3,-2,5,-2c4.7,0,8.7,3.3,12,10\ns173,378,173,378c0.7,0,35.3,-71,104,-213c68.7,-142,137.5,-285,206.5,-429\nc69,-144,104.5,-217.7,106.5,-221\nl0 -0\nc5.3,-9.3,12,-14,20,-14\nH400000v40H845.2724\ns-225.272,467,-225.272,467s-235,486,-235,486c-2.7,4.7,-9,7,-19,7\nc-6,0,-10,-1,-12,-3s-194,-422,-194,-422s-65,47,-65,47z\nM834 80h400000v40h-400000z",[35,6716,221],{"className":6717},[220],[35,6719,6721],{"className":6720},[189],[35,6722,6725],{"className":6723,"style":6724},[193],"height:0.1828em;",[35,6726],{},[35,6728,112],{"className":6729},[3453],[35,6731,6565],{"className":6732,"style":288},[134,135],". Q, K, and V come from weight projections, already counted above in ",[35,6735,6737,6756],{"className":6736},[42],[35,6738,6740],{"className":6739},[46],[48,6741,6742],{"xmlns":50},[53,6743,6744,6754],{},[56,6745,6746,6748],{},[1342,6747,1344],{},[79,6749,6750,6752],{},[59,6751,680],{},[59,6753,643],{},[114,6755,4154],{"encoding":116},[35,6757,6759],{"className":6758,"ariaHidden":75},[121],[35,6760,6762,6765,6768],{"className":6761},[125],[35,6763],{"className":6764,"style":409},[129],[35,6766,1344],{"className":6767},[134],[35,6769,6771,6774],{"className":6770},[134],[35,6772,680],{"className":6773,"style":695},[134,135],[35,6775,6777],{"className":6776},[180],[35,6778,6780,6800],{"className":6779},[184,185],[35,6781,6783,6797],{"className":6782},[189],[35,6784,6786],{"className":6785,"style":745},[193],[35,6787,6788,6791],{"style":748},[35,6789],{"className":6790,"style":202},[201],[35,6792,6794],{"className":6793},[206,207,208,209],[35,6795,643],{"className":6796},[134,135,209],[35,6798,221],{"className":6799},[220],[35,6801,6803],{"className":6802},[189],[35,6804,6806],{"className":6805,"style":228},[193],[35,6807],{},". The ",[35,6810,6812,6832],{"className":6811},[42],[35,6813,6815],{"className":6814},[46],[48,6816,6817],{"xmlns":50},[53,6818,6819,6829],{},[56,6820,6821,6823],{},[59,6822,6540],{},[6542,6824,6825,6827],{},[59,6826,6546],{},[59,6828,6549],{"mathvariant":1396},[114,6830,6831],{"encoding":116},"QK^\\top",[35,6833,6835],{"className":6834,"ariaHidden":75},[121],[35,6836,6838,6842,6845],{"className":6837},[125],[35,6839],{"className":6840,"style":6841},[129],"height:1.0435em;vertical-align:-0.1944em;",[35,6843,6540],{"className":6844},[134,135],[35,6846,6848,6851],{"className":6847},[134],[35,6849,6546],{"className":6850,"style":4092},[134,135],[35,6852,6854],{"className":6853},[180],[35,6855,6857],{"className":6856},[184],[35,6858,6860],{"className":6859},[189],[35,6861,6863],{"className":6862,"style":6609},[193],[35,6864,6865,6868],{"style":6151},[35,6866],{"className":6867,"style":202},[201],[35,6869,6871],{"className":6870},[206,207,208,209],[35,6872,6549],{"className":6873},[134,209]," term scores each token's query against every token's key. With ",[35,6876,6878,6895],{"className":6877},[42],[35,6879,6881],{"className":6880},[46],[48,6882,6883],{"xmlns":50},[53,6884,6885,6893],{},[56,6886,6887],{},[79,6888,6889,6891],{},[59,6890,971],{},[84,6892,974],{},[114,6894,977],{"encoding":116},[35,6896,6898],{"className":6897,"ariaHidden":75},[121],[35,6899,6901,6904],{"className":6900},[125],[35,6902],{"className":6903,"style":409},[129],[35,6905,6907,6910],{"className":6906},[134],[35,6908,971],{"className":6909},[134,135],[35,6911,6913],{"className":6912},[180],[35,6914,6916,6939],{"className":6915},[184,185],[35,6917,6919,6936],{"className":6918},[189],[35,6920,6922],{"className":6921,"style":1005},[193],[35,6923,6924,6927],{"style":197},[35,6925],{"className":6926,"style":202},[201],[35,6928,6930],{"className":6929},[206,207,208,209],[35,6931,6933],{"className":6932},[134,213,209],[35,6934,974],{"className":6935},[134,209],[35,6937,221],{"className":6938},[220],[35,6940,6942],{"className":6941},[189],[35,6943,6945],{"className":6944,"style":228},[193],[35,6946],{}," tokens, that is ",[35,6949,6951,6970],{"className":6950},[42],[35,6952,6954],{"className":6953},[46],[48,6955,6956],{"xmlns":50},[53,6957,6958,6968],{},[56,6959,6960],{},[6094,6961,6962,6964,6966],{},[59,6963,971],{},[84,6965,974],{},[1342,6967,1344],{},[114,6969,6104],{"encoding":116},[35,6971,6973],{"className":6972,"ariaHidden":75},[121],[35,6974,6976,6979],{"className":6975},[125],[35,6977],{"className":6978,"style":6114},[129],[35,6980,6982,6985],{"className":6981},[134],[35,6983,971],{"className":6984},[134,135],[35,6986,6988],{"className":6987},[180],[35,6989,6991,7025],{"className":6990},[184,185],[35,6992,6994,7022],{"className":6993},[189],[35,6995,6997,7011],{"className":6996,"style":6133},[193],[35,6998,6999,7002],{"style":6136},[35,7000],{"className":7001,"style":202},[201],[35,7003,7005],{"className":7004},[206,207,208,209],[35,7006,7008],{"className":7007},[134,213,209],[35,7009,974],{"className":7010},[134,209],[35,7012,7013,7016],{"style":6151},[35,7014],{"className":7015,"style":202},[201],[35,7017,7019],{"className":7018},[206,207,208,209],[35,7020,1344],{"className":7021},[134,209],[35,7023,221],{"className":7024},[220],[35,7026,7028],{"className":7027},[189],[35,7029,7031],{"className":7030,"style":6170},[193],[35,7032],{}," score pairs. Each score is a dot product over ",[35,7035,7037,7055],{"className":7036},[42],[35,7038,7040],{"className":7039},[46],[48,7041,7042],{"xmlns":50},[53,7043,7044,7052],{},[56,7045,7046],{},[79,7047,7048,7050],{},[59,7049,1904],{},[84,7051,1907],{},[114,7053,7054],{"encoding":116},"d_\\text{head}",[35,7056,7058],{"className":7057,"ariaHidden":75},[121],[35,7059,7061,7064],{"className":7060},[125],[35,7062],{"className":7063,"style":2123},[129],[35,7065,7067,7070],{"className":7066},[134],[35,7068,1904],{"className":7069},[134,135],[35,7071,7073],{"className":7072},[180],[35,7074,7076,7099],{"className":7075},[184,185],[35,7077,7079,7096],{"className":7078},[189],[35,7080,7082],{"className":7081,"style":194},[193],[35,7083,7084,7087],{"style":197},[35,7085],{"className":7086,"style":202},[201],[35,7088,7090],{"className":7089},[206,207,208,209],[35,7091,7093],{"className":7092},[134,213,209],[35,7094,1907],{"className":7095},[134,209],[35,7097,221],{"className":7098},[220],[35,7100,7102],{"className":7101},[189],[35,7103,7105],{"className":7104,"style":228},[193],[35,7106],{}," dimensions, so per head ",[35,7109,7111,7130],{"className":7110},[42],[35,7112,7114],{"className":7113},[46],[48,7115,7116],{"xmlns":50},[53,7117,7118,7128],{},[56,7119,7120,7122],{},[59,7121,6540],{},[6542,7123,7124,7126],{},[59,7125,6546],{},[59,7127,6549],{"mathvariant":1396},[114,7129,6831],{"encoding":116},[35,7131,7133],{"className":7132,"ariaHidden":75},[121],[35,7134,7136,7139,7142],{"className":7135},[125],[35,7137],{"className":7138,"style":6841},[129],[35,7140,6540],{"className":7141},[134,135],[35,7143,7145,7148],{"className":7144},[134],[35,7146,6546],{"className":7147,"style":4092},[134,135],[35,7149,7151],{"className":7150},[180],[35,7152,7154],{"className":7153},[184],[35,7155,7157],{"className":7156},[189],[35,7158,7160],{"className":7159,"style":6609},[193],[35,7161,7162,7165],{"style":6151},[35,7163],{"className":7164,"style":202},[201],[35,7166,7168],{"className":7167},[206,207,208,209],[35,7169,6549],{"className":7170},[134,209]," costs ",[35,7173,7175,7207],{"className":7174},[42],[35,7176,7178],{"className":7177},[46],[48,7179,7180],{"xmlns":50},[53,7181,7182,7204],{},[56,7183,7184,7186,7188,7194,7196],{},[1342,7185,1344],{},[84,7187,3503],{},[79,7189,7190,7192],{},[59,7191,1904],{},[84,7193,1907],{},[84,7195,3503],{},[6094,7197,7198,7200,7202],{},[59,7199,971],{},[84,7201,974],{},[1342,7203,1344],{},[114,7205,7206],{"encoding":116},"2\\,d_\\text{head}\\,L_\\text{in}^2",[35,7208,7210],{"className":7209,"ariaHidden":75},[121],[35,7211,7213,7216,7219,7222,7265,7268],{"className":7212},[125],[35,7214],{"className":7215,"style":6114},[129],[35,7217,1344],{"className":7218},[134],[35,7220],{"className":7221,"style":162},[139],[35,7223,7225,7228],{"className":7224},[134],[35,7226,1904],{"className":7227},[134,135],[35,7229,7231],{"className":7230},[180],[35,7232,7234,7257],{"className":7233},[184,185],[35,7235,7237,7254],{"className":7236},[189],[35,7238,7240],{"className":7239,"style":194},[193],[35,7241,7242,7245],{"style":197},[35,7243],{"className":7244,"style":202},[201],[35,7246,7248],{"className":7247},[206,207,208,209],[35,7249,7251],{"className":7250},[134,213,209],[35,7252,1907],{"className":7253},[134,209],[35,7255,221],{"className":7256},[220],[35,7258,7260],{"className":7259},[189],[35,7261,7263],{"className":7262,"style":228},[193],[35,7264],{},[35,7266],{"className":7267,"style":162},[139],[35,7269,7271,7274],{"className":7270},[134],[35,7272,971],{"className":7273},[134,135],[35,7275,7277],{"className":7276},[180],[35,7278,7280,7314],{"className":7279},[184,185],[35,7281,7283,7311],{"className":7282},[189],[35,7284,7286,7300],{"className":7285,"style":6133},[193],[35,7287,7288,7291],{"style":6136},[35,7289],{"className":7290,"style":202},[201],[35,7292,7294],{"className":7293},[206,207,208,209],[35,7295,7297],{"className":7296},[134,213,209],[35,7298,974],{"className":7299},[134,209],[35,7301,7302,7305],{"style":6151},[35,7303],{"className":7304,"style":202},[201],[35,7306,7308],{"className":7307},[206,207,208,209],[35,7309,1344],{"className":7310},[134,209],[35,7312,221],{"className":7313},[220],[35,7315,7317],{"className":7316},[189],[35,7318,7320],{"className":7319,"style":6170},[193],[35,7321],{}," FLOPs. The ",[35,7324,7326,7342],{"className":7325},[42],[35,7327,7329],{"className":7328},[46],[48,7330,7331],{"xmlns":50},[53,7332,7333,7339],{},[56,7334,7335,7337],{},[63,7336,1707],{},[59,7338,6565],{},[114,7340,7341],{"encoding":116},"\\times V",[35,7343,7345],{"className":7344,"ariaHidden":75},[121],[35,7346,7348,7351,7354],{"className":7347},[125],[35,7349],{"className":7350,"style":1781},[129],[35,7352,1707],{"className":7353},[134],[35,7355,6565],{"className":7356,"style":288},[134,135]," term then uses those scores as weights to mix the value vectors. It has the same shape, so it costs the same. That is ",[35,7359,7361,7393],{"className":7360},[42],[35,7362,7364],{"className":7363},[46],[48,7365,7366],{"xmlns":50},[53,7367,7368,7390],{},[56,7369,7370,7372,7374,7380,7382],{},[1342,7371,1509],{},[84,7373,3503],{},[79,7375,7376,7378],{},[59,7377,1904],{},[84,7379,1907],{},[84,7381,3503],{},[6094,7383,7384,7386,7388],{},[59,7385,971],{},[84,7387,974],{},[1342,7389,1344],{},[114,7391,7392],{"encoding":116},"4\\,d_\\text{head}\\,L_\\text{in}^2",[35,7394,7396],{"className":7395,"ariaHidden":75},[121],[35,7397,7399,7402,7405,7408,7451,7454],{"className":7398},[125],[35,7400],{"className":7401,"style":6114},[129],[35,7403,1509],{"className":7404},[134],[35,7406],{"className":7407,"style":162},[139],[35,7409,7411,7414],{"className":7410},[134],[35,7412,1904],{"className":7413},[134,135],[35,7415,7417],{"className":7416},[180],[35,7418,7420,7443],{"className":7419},[184,185],[35,7421,7423,7440],{"className":7422},[189],[35,7424,7426],{"className":7425,"style":194},[193],[35,7427,7428,7431],{"style":197},[35,7429],{"className":7430,"style":202},[201],[35,7432,7434],{"className":7433},[206,207,208,209],[35,7435,7437],{"className":7436},[134,213,209],[35,7438,1907],{"className":7439},[134,209],[35,7441,221],{"className":7442},[220],[35,7444,7446],{"className":7445},[189],[35,7447,7449],{"className":7448,"style":228},[193],[35,7450],{},[35,7452],{"className":7453,"style":162},[139],[35,7455,7457,7460],{"className":7456},[134],[35,7458,971],{"className":7459},[134,135],[35,7461,7463],{"className":7462},[180],[35,7464,7466,7500],{"className":7465},[184,185],[35,7467,7469,7497],{"className":7468},[189],[35,7470,7472,7486],{"className":7471,"style":6133},[193],[35,7473,7474,7477],{"style":6136},[35,7475],{"className":7476,"style":202},[201],[35,7478,7480],{"className":7479},[206,207,208,209],[35,7481,7483],{"className":7482},[134,213,209],[35,7484,974],{"className":7485},[134,209],[35,7487,7488,7491],{"style":6151},[35,7489],{"className":7490,"style":202},[201],[35,7492,7494],{"className":7493},[206,207,208,209],[35,7495,1344],{"className":7496},[134,209],[35,7498,221],{"className":7499},[220],[35,7501,7503],{"className":7502},[189],[35,7504,7506],{"className":7505,"style":6170},[193],[35,7507],{}," per head.",[10,7510,7511,7512,7720,7721,7793,7794,7867,7868,7940,7941,917],{},"Summing across heads uses ",[35,7513,7515,7551],{"className":7514},[42],[35,7516,7518],{"className":7517},[46],[48,7519,7520],{"xmlns":50},[53,7521,7522,7548],{},[56,7523,7524,7531,7534,7540,7542],{},[79,7525,7526,7528],{},[59,7527,1885],{},[84,7529,7530],{},"heads",[63,7532,7533],{},"⋅",[79,7535,7536,7538],{},[59,7537,1904],{},[84,7539,1907],{},[63,7541,65],{},[79,7543,7544,7546],{},[59,7545,1904],{},[84,7547,4238],{},[114,7549,7550],{"encoding":116},"n_\\text{heads} \\cdot d_\\text{head} = d_\\text{hidden}",[35,7552,7554,7613,7671],{"className":7553,"ariaHidden":75},[121],[35,7555,7557,7561,7604,7607,7610],{"className":7556},[125],[35,7558],{"className":7559,"style":7560},[129],"height:0.5945em;vertical-align:-0.15em;",[35,7562,7564,7567],{"className":7563},[134],[35,7565,1885],{"className":7566},[134,135],[35,7568,7570],{"className":7569},[180],[35,7571,7573,7596],{"className":7572},[184,185],[35,7574,7576,7593],{"className":7575},[189],[35,7577,7579],{"className":7578,"style":194},[193],[35,7580,7581,7584],{"style":197},[35,7582],{"className":7583,"style":202},[201],[35,7585,7587],{"className":7586},[206,207,208,209],[35,7588,7590],{"className":7589},[134,213,209],[35,7591,7530],{"className":7592},[134,209],[35,7594,221],{"className":7595},[220],[35,7597,7599],{"className":7598},[189],[35,7600,7602],{"className":7601,"style":228},[193],[35,7603],{},[35,7605],{"className":7606,"style":288},[139],[35,7608,7533],{"className":7609},[292],[35,7611],{"className":7612,"style":288},[139],[35,7614,7616,7619,7662,7665,7668],{"className":7615},[125],[35,7617],{"className":7618,"style":2123},[129],[35,7620,7622,7625],{"className":7621},[134],[35,7623,1904],{"className":7624},[134,135],[35,7626,7628],{"className":7627},[180],[35,7629,7631,7654],{"className":7630},[184,185],[35,7632,7634,7651],{"className":7633},[189],[35,7635,7637],{"className":7636,"style":194},[193],[35,7638,7639,7642],{"style":197},[35,7640],{"className":7641,"style":202},[201],[35,7643,7645],{"className":7644},[206,207,208,209],[35,7646,7648],{"className":7647},[134,213,209],[35,7649,1907],{"className":7650},[134,209],[35,7652,221],{"className":7653},[220],[35,7655,7657],{"className":7656},[189],[35,7658,7660],{"className":7659,"style":228},[193],[35,7661],{},[35,7663],{"className":7664,"style":140},[139],[35,7666,65],{"className":7667},[144],[35,7669],{"className":7670,"style":140},[139],[35,7672,7674,7677],{"className":7673},[125],[35,7675],{"className":7676,"style":2123},[129],[35,7678,7680,7683],{"className":7679},[134],[35,7681,1904],{"className":7682},[134,135],[35,7684,7686],{"className":7685},[180],[35,7687,7689,7712],{"className":7688},[184,185],[35,7690,7692,7709],{"className":7691},[189],[35,7693,7695],{"className":7694,"style":194},[193],[35,7696,7697,7700],{"style":197},[35,7698],{"className":7699,"style":202},[201],[35,7701,7703],{"className":7702},[206,207,208,209],[35,7704,7706],{"className":7705},[134,213,209],[35,7707,4238],{"className":7708},[134,209],[35,7710,221],{"className":7711},[220],[35,7713,7715],{"className":7714},[189],[35,7716,7718],{"className":7717,"style":228},[193],[35,7719],{},", collapsing the per-head ",[35,7722,7724,7741],{"className":7723},[42],[35,7725,7727],{"className":7726},[46],[48,7728,7729],{"xmlns":50},[53,7730,7731,7739],{},[56,7732,7733],{},[79,7734,7735,7737],{},[59,7736,1904],{},[84,7738,1907],{},[114,7740,7054],{"encoding":116},[35,7742,7744],{"className":7743,"ariaHidden":75},[121],[35,7745,7747,7750],{"className":7746},[125],[35,7748],{"className":7749,"style":2123},[129],[35,7751,7753,7756],{"className":7752},[134],[35,7754,1904],{"className":7755},[134,135],[35,7757,7759],{"className":7758},[180],[35,7760,7762,7785],{"className":7761},[184,185],[35,7763,7765,7782],{"className":7764},[189],[35,7766,7768],{"className":7767,"style":194},[193],[35,7769,7770,7773],{"style":197},[35,7771],{"className":7772,"style":202},[201],[35,7774,7776],{"className":7775},[206,207,208,209],[35,7777,7779],{"className":7778},[134,213,209],[35,7780,1907],{"className":7781},[134,209],[35,7783,221],{"className":7784},[220],[35,7786,7788],{"className":7787},[189],[35,7789,7791],{"className":7790,"style":228},[193],[35,7792],{}," into a single ",[35,7795,7797,7815],{"className":7796},[42],[35,7798,7800],{"className":7799},[46],[48,7801,7802],{"xmlns":50},[53,7803,7804,7812],{},[56,7805,7806],{},[79,7807,7808,7810],{},[59,7809,1904],{},[84,7811,4238],{},[114,7813,7814],{"encoding":116},"d_\\text{hidden}",[35,7816,7818],{"className":7817,"ariaHidden":75},[121],[35,7819,7821,7824],{"className":7820},[125],[35,7822],{"className":7823,"style":2123},[129],[35,7825,7827,7830],{"className":7826},[134],[35,7828,1904],{"className":7829},[134,135],[35,7831,7833],{"className":7832},[180],[35,7834,7836,7859],{"className":7835},[184,185],[35,7837,7839,7856],{"className":7838},[189],[35,7840,7842],{"className":7841,"style":194},[193],[35,7843,7844,7847],{"style":197},[35,7845],{"className":7846,"style":202},[201],[35,7848,7850],{"className":7849},[206,207,208,209],[35,7851,7853],{"className":7852},[134,213,209],[35,7854,4238],{"className":7855},[134,209],[35,7857,221],{"className":7858},[220],[35,7860,7862],{"className":7861},[189],[35,7863,7865],{"className":7864,"style":228},[193],[35,7866],{},". GQA shrinks the key and value side, but it keeps every query head, so the full ",[35,7869,7871,7888],{"className":7870},[42],[35,7872,7874],{"className":7873},[46],[48,7875,7876],{"xmlns":50},[53,7877,7878,7886],{},[56,7879,7880],{},[79,7881,7882,7884],{},[59,7883,1904],{},[84,7885,4238],{},[114,7887,7814],{"encoding":116},[35,7889,7891],{"className":7890,"ariaHidden":75},[121],[35,7892,7894,7897],{"className":7893},[125],[35,7895],{"className":7896,"style":2123},[129],[35,7898,7900,7903],{"className":7899},[134],[35,7901,1904],{"className":7902},[134,135],[35,7904,7906],{"className":7905},[180],[35,7907,7909,7932],{"className":7908},[184,185],[35,7910,7912,7929],{"className":7911},[189],[35,7913,7915],{"className":7914,"style":194},[193],[35,7916,7917,7920],{"style":197},[35,7918],{"className":7919,"style":202},[201],[35,7921,7923],{"className":7922},[206,207,208,209],[35,7924,7926],{"className":7925},[134,213,209],[35,7927,4238],{"className":7928},[134,209],[35,7930,221],{"className":7931},[220],[35,7933,7935],{"className":7934},[189],[35,7936,7938],{"className":7937,"style":228},[193],[35,7939],{}," still applies. Across all layers, the per-token cost is ",[35,7942,7944,7981],{"className":7943},[42],[35,7945,7947],{"className":7946},[46],[48,7948,7949],{"xmlns":50},[53,7950,7951,7979],{},[56,7952,7953,7955,7957,7963,7965,7971,7973],{},[1342,7954,1509],{},[84,7956,3503],{},[79,7958,7959,7961],{},[59,7960,1885],{},[84,7962,1888],{},[84,7964,3503],{},[79,7966,7967,7969],{},[59,7968,1904],{},[84,7970,4238],{},[84,7972,3503],{},[79,7974,7975,7977],{},[59,7976,971],{},[84,7978,974],{},[114,7980,4249],{"encoding":116},[35,7982,7984],{"className":7983,"ariaHidden":75},[121],[35,7985,7987,7990,7993,7996,8039,8042,8085,8088],{"className":7986},[125],[35,7988],{"className":7989,"style":4259},[129],[35,7991,1509],{"className":7992},[134],[35,7994],{"className":7995,"style":162},[139],[35,7997,7999,8002],{"className":7998},[134],[35,8000,1885],{"className":8001},[134,135],[35,8003,8005],{"className":8004},[180],[35,8006,8008,8031],{"className":8007},[184,185],[35,8009,8011,8028],{"className":8010},[189],[35,8012,8014],{"className":8013,"style":194},[193],[35,8015,8016,8019],{"style":197},[35,8017],{"className":8018,"style":202},[201],[35,8020,8022],{"className":8021},[206,207,208,209],[35,8023,8025],{"className":8024},[134,213,209],[35,8026,1888],{"className":8027},[134,209],[35,8029,221],{"className":8030},[220],[35,8032,8034],{"className":8033},[189],[35,8035,8037],{"className":8036,"style":282},[193],[35,8038],{},[35,8040],{"className":8041,"style":162},[139],[35,8043,8045,8048],{"className":8044},[134],[35,8046,1904],{"className":8047},[134,135],[35,8049,8051],{"className":8050},[180],[35,8052,8054,8077],{"className":8053},[184,185],[35,8055,8057,8074],{"className":8056},[189],[35,8058,8060],{"className":8059,"style":194},[193],[35,8061,8062,8065],{"style":197},[35,8063],{"className":8064,"style":202},[201],[35,8066,8068],{"className":8067},[206,207,208,209],[35,8069,8071],{"className":8070},[134,213,209],[35,8072,4238],{"className":8073},[134,209],[35,8075,221],{"className":8076},[220],[35,8078,8080],{"className":8079},[189],[35,8081,8083],{"className":8082,"style":228},[193],[35,8084],{},[35,8086],{"className":8087,"style":162},[139],[35,8089,8091,8094],{"className":8090},[134],[35,8092,971],{"className":8093},[134,135],[35,8095,8097],{"className":8096},[180],[35,8098,8100,8123],{"className":8099},[184,185],[35,8101,8103,8120],{"className":8102},[189],[35,8104,8106],{"className":8105,"style":1005},[193],[35,8107,8108,8111],{"style":197},[35,8109],{"className":8110,"style":202},[201],[35,8112,8114],{"className":8113},[206,207,208,209],[35,8115,8117],{"className":8116},[134,213,209],[35,8118,974],{"className":8119},[134,209],[35,8121,221],{"className":8122},[220],[35,8124,8126],{"className":8125},[189],[35,8127,8129],{"className":8128,"style":228},[193],[35,8130],{},[10,8132,8133,8136,8137,8237,8238,8241,8242,8462,8463,8611],{},[662,8134,8135],{},"The crossover."," A standard layer has roughly ",[35,8138,8140,8165],{"className":8139},[42],[35,8141,8143],{"className":8142},[46],[48,8144,8145],{"xmlns":50},[53,8146,8147,8162],{},[56,8148,8149,8152,8154],{},[1342,8150,8151],{},"12",[84,8153,3503],{},[6094,8155,8156,8158,8160],{},[59,8157,1904],{},[84,8159,4238],{},[1342,8161,1344],{},[114,8163,8164],{"encoding":116},"12\\,d_\\text{hidden}^2",[35,8166,8168],{"className":8167,"ariaHidden":75},[121],[35,8169,8171,8175,8178,8181],{"className":8170},[125],[35,8172],{"className":8173,"style":8174},[129],"height:1.0972em;vertical-align:-0.2831em;",[35,8176,8151],{"className":8177},[134],[35,8179],{"className":8180,"style":162},[139],[35,8182,8184,8187],{"className":8183},[134],[35,8185,1904],{"className":8186},[134,135],[35,8188,8190],{"className":8189},[180],[35,8191,8193,8228],{"className":8192},[184,185],[35,8194,8196,8225],{"className":8195},[189],[35,8197,8199,8214],{"className":8198,"style":6133},[193],[35,8200,8202,8205],{"style":8201},"top:-2.4169em;margin-left:0em;margin-right:0.05em;",[35,8203],{"className":8204,"style":202},[201],[35,8206,8208],{"className":8207},[206,207,208,209],[35,8209,8211],{"className":8210},[134,213,209],[35,8212,4238],{"className":8213},[134,209],[35,8215,8216,8219],{"style":6151},[35,8217],{"className":8218,"style":202},[201],[35,8220,8222],{"className":8221},[206,207,208,209],[35,8223,1344],{"className":8224},[134,209],[35,8226,221],{"className":8227},[220],[35,8229,8231],{"className":8230},[189],[35,8232,8235],{"className":8233,"style":8234},[193],"height:0.2831em;",[35,8236],{}," parameters; the FFN dominates ",[1312,8239],{"n":8240},"7",". So ",[35,8243,8245,8288],{"className":8244},[42],[35,8246,8248],{"className":8247},[46],[48,8249,8250],{"xmlns":50},[53,8251,8252,8285],{},[56,8253,8254,8256,8262,8264,8267,8269,8275,8277],{},[1342,8255,1344],{},[79,8257,8258,8260],{},[59,8259,680],{},[59,8261,643],{},[63,8263,2494],{},[1342,8265,8266],{},"24",[84,8268,3503],{},[79,8270,8271,8273],{},[59,8272,1885],{},[84,8274,1888],{},[84,8276,3503],{},[6094,8278,8279,8281,8283],{},[59,8280,1904],{},[84,8282,4238],{},[1342,8284,1344],{},[114,8286,8287],{"encoding":116},"2P_a \\approx 24\\,n_\\text{layers}\\,d_\\text{hidden}^2",[35,8289,8291,8349],{"className":8290,"ariaHidden":75},[121],[35,8292,8294,8297,8300,8340,8343,8346],{"className":8293},[125],[35,8295],{"className":8296,"style":409},[129],[35,8298,1344],{"className":8299},[134],[35,8301,8303,8306],{"className":8302},[134],[35,8304,680],{"className":8305,"style":695},[134,135],[35,8307,8309],{"className":8308},[180],[35,8310,8312,8332],{"className":8311},[184,185],[35,8313,8315,8329],{"className":8314},[189],[35,8316,8318],{"className":8317,"style":745},[193],[35,8319,8320,8323],{"style":748},[35,8321],{"className":8322,"style":202},[201],[35,8324,8326],{"className":8325},[206,207,208,209],[35,8327,643],{"className":8328},[134,135,209],[35,8330,221],{"className":8331},[220],[35,8333,8335],{"className":8334},[189],[35,8336,8338],{"className":8337,"style":228},[193],[35,8339],{},[35,8341],{"className":8342,"style":140},[139],[35,8344,2494],{"className":8345},[144],[35,8347],{"className":8348,"style":140},[139],[35,8350,8352,8356,8359,8362,8405,8408],{"className":8351},[125],[35,8353],{"className":8354,"style":8355},[129],"height:1.1002em;vertical-align:-0.2861em;",[35,8357,8266],{"className":8358},[134],[35,8360],{"className":8361,"style":162},[139],[35,8363,8365,8368],{"className":8364},[134],[35,8366,1885],{"className":8367},[134,135],[35,8369,8371],{"className":8370},[180],[35,8372,8374,8397],{"className":8373},[184,185],[35,8375,8377,8394],{"className":8376},[189],[35,8378,8380],{"className":8379,"style":194},[193],[35,8381,8382,8385],{"style":197},[35,8383],{"className":8384,"style":202},[201],[35,8386,8388],{"className":8387},[206,207,208,209],[35,8389,8391],{"className":8390},[134,213,209],[35,8392,1888],{"className":8393},[134,209],[35,8395,221],{"className":8396},[220],[35,8398,8400],{"className":8399},[189],[35,8401,8403],{"className":8402,"style":282},[193],[35,8404],{},[35,8406],{"className":8407,"style":162},[139],[35,8409,8411,8414],{"className":8410},[134],[35,8412,1904],{"className":8413},[134,135],[35,8415,8417],{"className":8416},[180],[35,8418,8420,8454],{"className":8419},[184,185],[35,8421,8423,8451],{"className":8422},[189],[35,8424,8426,8440],{"className":8425,"style":6133},[193],[35,8427,8428,8431],{"style":8201},[35,8429],{"className":8430,"style":202},[201],[35,8432,8434],{"className":8433},[206,207,208,209],[35,8435,8437],{"className":8436},[134,213,209],[35,8438,4238],{"className":8439},[134,209],[35,8441,8442,8445],{"style":6151},[35,8443],{"className":8444,"style":202},[201],[35,8446,8448],{"className":8447},[206,207,208,209],[35,8449,1344],{"className":8450},[134,209],[35,8452,221],{"className":8453},[220],[35,8455,8457],{"className":8456},[189],[35,8458,8460],{"className":8459,"style":8234},[193],[35,8461],{},". Setting the weight cost equal to the attention cost gives ",[35,8464,8466,8495],{"className":8465},[42],[35,8467,8469],{"className":8468},[46],[48,8470,8471],{"xmlns":50},[53,8472,8473,8493],{},[56,8474,8475,8481,8483,8485,8487],{},[79,8476,8477,8479],{},[59,8478,971],{},[84,8480,974],{},[63,8482,2494],{},[1342,8484,2193],{},[84,8486,3503],{},[79,8488,8489,8491],{},[59,8490,1904],{},[84,8492,4238],{},[114,8494,5821],{"encoding":116},[35,8496,8498,8556],{"className":8497,"ariaHidden":75},[121],[35,8499,8501,8504,8547,8550,8553],{"className":8500},[125],[35,8502],{"className":8503,"style":409},[129],[35,8505,8507,8510],{"className":8506},[134],[35,8508,971],{"className":8509},[134,135],[35,8511,8513],{"className":8512},[180],[35,8514,8516,8539],{"className":8515},[184,185],[35,8517,8519,8536],{"className":8518},[189],[35,8520,8522],{"className":8521,"style":1005},[193],[35,8523,8524,8527],{"style":197},[35,8525],{"className":8526,"style":202},[201],[35,8528,8530],{"className":8529},[206,207,208,209],[35,8531,8533],{"className":8532},[134,213,209],[35,8534,974],{"className":8535},[134,209],[35,8537,221],{"className":8538},[220],[35,8540,8542],{"className":8541},[189],[35,8543,8545],{"className":8544,"style":228},[193],[35,8546],{},[35,8548],{"className":8549,"style":140},[139],[35,8551,2494],{"className":8552},[144],[35,8554],{"className":8555,"style":140},[139],[35,8557,8559,8562,8565,8568],{"className":8558},[125],[35,8560],{"className":8561,"style":2123},[129],[35,8563,2193],{"className":8564},[134],[35,8566],{"className":8567,"style":162},[139],[35,8569,8571,8574],{"className":8570},[134],[35,8572,1904],{"className":8573},[134,135],[35,8575,8577],{"className":8576},[180],[35,8578,8580,8603],{"className":8579},[184,185],[35,8581,8583,8600],{"className":8582},[189],[35,8584,8586],{"className":8585,"style":194},[193],[35,8587,8588,8591],{"style":197},[35,8589],{"className":8590,"style":202},[201],[35,8592,8594],{"className":8593},[206,207,208,209],[35,8595,8597],{"className":8596},[134,213,209],[35,8598,4238],{"className":8599},[134,209],[35,8601,221],{"className":8602},[220],[35,8604,8606],{"className":8605},[189],[35,8607,8609],{"className":8608,"style":228},[193],[35,8610],{},": depth cancels, and the crossover is just six times the model's width. Above that, the attention term dominates the per-token prefill FLOPs.",[3949,8613,3957,8615,8618],{"className":8614},[3952,3953,3954,3955,3956],[662,8616,3963],{"className":8617},[3961,3962],"\n  Prefill is a compute bill. Long prompts and dense active parameters move this term.\n",[27,8620,8622],{"id":8621},"step-2b-decode-is-a-bandwidth-bill","Step 2b: Decode Is a Bandwidth Bill",[10,8624,8625,8626,8655],{},"Decode writes the answer one token at a time. Each output token depends on the one before it, so a single request cannot be parallelised across time. Serving systems recover parallelism by batching requests instead, so one decode step advances ",[35,8627,8629,8642],{"className":8628},[42],[35,8630,8632],{"className":8631},[46],[48,8633,8634],{"xmlns":50},[53,8635,8636,8640],{},[56,8637,8638],{},[59,8639,5282],{},[114,8641,5282],{"encoding":116},[35,8643,8645],{"className":8644,"ariaHidden":75},[121],[35,8646,8648,8651],{"className":8647},[125],[35,8649],{"className":8650,"style":130},[129],[35,8652,5282],{"className":8653,"style":8654},[134,135],"margin-right:0.0502em;"," active streams by one token.",[10,8657,8658,8659,917],{},"Each decode step streams bytes from HBM. It reads the weights once across the batch, and it reads the KV cache once per stream. The arithmetic is small compared to this traffic, so decode is usually bandwidth-bound ",[1312,8660],{"n":1662},[10,8662,8663,8664,8722,8723,4049,8767,8770,8771,8799,8800,8829,8830,8869],{},"A single GPU has usable bandwidth ",[35,8665,8667,8686],{"className":8666},[42],[35,8668,8670],{"className":8669},[46],[48,8671,8672],{"xmlns":50},[53,8673,8674,8683],{},[56,8675,8676,8679,8681],{},[84,8677,8678],{},"MBU",[63,8680,7533],{},[84,8682,868],{},[114,8684,8685],{"encoding":116},"\\text{MBU}\\cdot\\text{BW}",[35,8687,8689,8710],{"className":8688,"ariaHidden":75},[121],[35,8690,8692,8695,8701,8704,8707],{"className":8691},[125],[35,8693],{"className":8694,"style":130},[129],[35,8696,8698],{"className":8697},[134,213],[35,8699,8678],{"className":8700},[134],[35,8702],{"className":8703,"style":288},[139],[35,8705,7533],{"className":8706},[292],[35,8708],{"className":8709,"style":288},[139],[35,8711,8713,8716],{"className":8712},[125],[35,8714],{"className":8715,"style":130},[129],[35,8717,8719],{"className":8718},[134,213],[35,8720,868],{"className":8721},[134],", where MBU is model-bandwidth utilization. We use MBU ",[35,8724,8726,8743],{"className":8725},[42],[35,8727,8729],{"className":8728},[46],[48,8730,8731],{"xmlns":50},[53,8732,8733,8740],{},[56,8734,8735,8737],{},[63,8736,2494],{},[1342,8738,8739],{},"0.6",[114,8741,8742],{"encoding":116},"\\approx 0.6",[35,8744,8746,8758],{"className":8745,"ariaHidden":75},[121],[35,8747,8749,8752,8755],{"className":8748},[125],[35,8750],{"className":8751,"style":4033},[129],[35,8753,2494],{"className":8754},[144],[35,8756],{"className":8757,"style":140},[139],[35,8759,8761,8764],{"className":8760},[125],[35,8762],{"className":8763,"style":1627},[129],[35,8765,8739],{"className":8766},[134],[1312,8768],{"n":8769},"9",". But the latency target has first claim on that bandwidth: to produce one token every ",[35,8772,8774,8787],{"className":8773},[42],[35,8775,8777],{"className":8776},[46],[48,8778,8779],{"xmlns":50},[53,8780,8781,8785],{},[56,8782,8783],{},[59,8784,1385],{},[114,8786,1385],{"encoding":116},[35,8788,8790],{"className":8789,"ariaHidden":75},[121],[35,8791,8793,8796],{"className":8792},[125],[35,8794],{"className":8795,"style":2596},[129],[35,8797,1385],{"className":8798},[134,135]," seconds, the card has to read ",[35,8801,8803,8817],{"className":8802},[42],[35,8804,8806],{"className":8805},[46],[48,8807,8808],{"xmlns":50},[53,8809,8810,8815],{},[56,8811,8812],{},[59,8813,8814],{},"W",[114,8816,8814],{"encoding":116},[35,8818,8820],{"className":8819,"ariaHidden":75},[121],[35,8821,8823,8826],{"className":8822},[125],[35,8824],{"className":8825,"style":130},[129],[35,8827,8814],{"className":8828,"style":695},[134,135]," bytes of weights, consuming ",[35,8831,8833,8851],{"className":8832},[42],[35,8834,8836],{"className":8835},[46],[48,8837,8838],{"xmlns":50},[53,8839,8840,8848],{},[56,8841,8842,8844,8846],{},[59,8843,8814],{},[59,8845,2749],{"mathvariant":1396},[59,8847,1385],{},[114,8849,8850],{"encoding":116},"W\u002Ft",[35,8852,8854],{"className":8853,"ariaHidden":75},[121],[35,8855,8857,8860,8863,8866],{"className":8856},[125],[35,8858],{"className":8859,"style":1427},[129],[35,8861,8814],{"className":8862,"style":695},[134,135],[35,8864,2749],{"className":8865},[134],[35,8867,1385],{"className":8868},[134,135]," bandwidth. Subtract that fixed weight read from usable bandwidth to get what remains for KV traffic:",[35,8871,8873],{"className":8872},[38],[35,8874,8876,8905],{"className":8875},[42],[35,8877,8879],{"className":8878},[46],[48,8880,8881],{"xmlns":50,"display":51},[53,8882,8883,8902],{},[56,8884,8885,8887,8889,8891,8894,8896,8898,8900],{},[84,8886,8678],{},[63,8888,7533],{},[84,8890,868],{},[63,8892,8893],{},"−",[59,8895,8814],{},[59,8897,2749],{"mathvariant":1396},[59,8899,1385],{},[59,8901,917],{"mathvariant":1396},[114,8903,8904],{"encoding":116},"\\text{MBU}\\cdot\\text{BW} - W\u002Ft.",[35,8906,8908,8929,8950],{"className":8907,"ariaHidden":75},[121],[35,8909,8911,8914,8920,8923,8926],{"className":8910},[125],[35,8912],{"className":8913,"style":130},[129],[35,8915,8917],{"className":8916},[134,213],[35,8918,8678],{"className":8919},[134],[35,8921],{"className":8922,"style":288},[139],[35,8924,7533],{"className":8925},[292],[35,8927],{"className":8928,"style":288},[139],[35,8930,8932,8935,8941,8944,8947],{"className":8931},[125],[35,8933],{"className":8934,"style":1781},[129],[35,8936,8938],{"className":8937},[134,213],[35,8939,868],{"className":8940},[134],[35,8942],{"className":8943,"style":288},[139],[35,8945,8893],{"className":8946},[292],[35,8948],{"className":8949,"style":288},[139],[35,8951,8953,8956,8959,8962,8965],{"className":8952},[125],[35,8954],{"className":8955,"style":1427},[129],[35,8957,8814],{"className":8958,"style":695},[134,135],[35,8960,2749],{"className":8961},[134],[35,8963,1385],{"className":8964},[134,135],[35,8966,917],{"className":8967},[134],[10,8969,8970,8971,1031,8999,9071,9072,9272,9273,9346],{},"With request rate ",[35,8972,8974,8987],{"className":8973},[42],[35,8975,8977],{"className":8976},[46],[48,8978,8979],{"xmlns":50},[53,8980,8981,8985],{},[56,8982,8983],{},[59,8984,938],{},[114,8986,938],{"encoding":116},[35,8988,8990],{"className":8989,"ariaHidden":75},[121],[35,8991,8993,8996],{"className":8992},[125],[35,8994],{"className":8995,"style":130},[129],[35,8997,938],{"className":8998,"style":953},[134,135],[35,9000,9002,9019],{"className":9001},[42],[35,9003,9005],{"className":9004},[46],[48,9006,9007],{"xmlns":50},[53,9008,9009,9017],{},[56,9010,9011],{},[79,9012,9013,9015],{},[59,9014,971],{},[84,9016,1050],{},[114,9018,1053],{"encoding":116},[35,9020,9022],{"className":9021,"ariaHidden":75},[121],[35,9023,9025,9028],{"className":9024},[125],[35,9026],{"className":9027,"style":409},[129],[35,9029,9031,9034],{"className":9030},[134],[35,9032,971],{"className":9033},[134,135],[35,9035,9037],{"className":9036},[180],[35,9038,9040,9063],{"className":9039},[184,185],[35,9041,9043,9060],{"className":9042},[189],[35,9044,9046],{"className":9045,"style":1081},[193],[35,9047,9048,9051],{"style":197},[35,9049],{"className":9050,"style":202},[201],[35,9052,9054],{"className":9053},[206,207,208,209],[35,9055,9057],{"className":9056},[134,213,209],[35,9058,1050],{"className":9059},[134,209],[35,9061,221],{"className":9062},[220],[35,9064,9066],{"className":9065},[189],[35,9067,9069],{"className":9068,"style":228},[193],[35,9070],{},", average context ",[35,9073,9075,9113],{"className":9074},[42],[35,9076,9078],{"className":9077},[46],[48,9079,9080],{"xmlns":50},[53,9081,9082,9110],{},[56,9083,9084,9090,9092,9098,9100,9106,9108],{},[2724,9085,9086,9088],{"accent":75},[59,9087,1885],{},[63,9089,2730],{},[63,9091,65],{},[79,9093,9094,9096],{},[59,9095,971],{},[84,9097,974],{},[63,9099,102],{},[79,9101,9102,9104],{},[59,9103,971],{},[84,9105,1050],{},[59,9107,2749],{"mathvariant":1396},[1342,9109,1344],{},[114,9111,9112],{"encoding":116},"\\bar n = L_\\text{in} + L_\\text{out}\u002F2",[35,9114,9116,9162,9220],{"className":9115,"ariaHidden":75},[121],[35,9117,9119,9122,9153,9156,9159],{"className":9118},[125],[35,9120],{"className":9121,"style":2781},[129],[35,9123,9125],{"className":9124},[134,2785],[35,9126,9128],{"className":9127},[184],[35,9129,9131],{"className":9130},[189],[35,9132,9134,9142],{"className":9133,"style":2781},[193],[35,9135,9136,9139],{"style":2797},[35,9137],{"className":9138,"style":2801},[201],[35,9140,1885],{"className":9141},[134,135],[35,9143,9144,9147],{"style":2797},[35,9145],{"className":9146,"style":2801},[201],[35,9148,9150],{"className":9149,"style":2814},[2813],[35,9151,2730],{"className":9152},[134],[35,9154],{"className":9155,"style":140},[139],[35,9157,65],{"className":9158},[144],[35,9160],{"className":9161,"style":140},[139],[35,9163,9165,9168,9211,9214,9217],{"className":9164},[125],[35,9166],{"className":9167,"style":409},[129],[35,9169,9171,9174],{"className":9170},[134],[35,9172,971],{"className":9173},[134,135],[35,9175,9177],{"className":9176},[180],[35,9178,9180,9203],{"className":9179},[184,185],[35,9181,9183,9200],{"className":9182},[189],[35,9184,9186],{"className":9185,"style":1005},[193],[35,9187,9188,9191],{"style":197},[35,9189],{"className":9190,"style":202},[201],[35,9192,9194],{"className":9193},[206,207,208,209],[35,9195,9197],{"className":9196},[134,213,209],[35,9198,974],{"className":9199},[134,209],[35,9201,221],{"className":9202},[220],[35,9204,9206],{"className":9205},[189],[35,9207,9209],{"className":9208,"style":228},[193],[35,9210],{},[35,9212],{"className":9213,"style":288},[139],[35,9215,102],{"className":9216},[292],[35,9218],{"className":9219,"style":288},[139],[35,9221,9223,9226,9269],{"className":9222},[125],[35,9224],{"className":9225,"style":1427},[129],[35,9227,9229,9232],{"className":9228},[134],[35,9230,971],{"className":9231},[134,135],[35,9233,9235],{"className":9234},[180],[35,9236,9238,9261],{"className":9237},[184,185],[35,9239,9241,9258],{"className":9240},[189],[35,9242,9244],{"className":9243,"style":1081},[193],[35,9245,9246,9249],{"style":197},[35,9247],{"className":9248,"style":202},[201],[35,9250,9252],{"className":9251},[206,207,208,209],[35,9253,9255],{"className":9254},[134,213,209],[35,9256,1050],{"className":9257},[134,209],[35,9259,221],{"className":9260},[220],[35,9262,9264],{"className":9263},[189],[35,9265,9267],{"className":9266,"style":228},[193],[35,9268],{},[35,9270,2937],{"className":9271},[134],", and per-token cache ",[35,9274,9276,9294],{"className":9275},[42],[35,9277,9279],{"className":9278},[46],[48,9280,9281],{"xmlns":50},[53,9282,9283,9291],{},[56,9284,9285],{},[79,9286,9287,9289],{},[59,9288,792],{},[84,9290,1874],{},[114,9292,9293],{"encoding":116},"M_\\text{kv\u002Ftok}",[35,9295,9297],{"className":9296,"ariaHidden":75},[121],[35,9298,9300,9303],{"className":9299},[125],[35,9301],{"className":9302,"style":1924},[129],[35,9304,9306,9309],{"className":9305},[134],[35,9307,792],{"className":9308,"style":814},[134,135],[35,9310,9312],{"className":9311},[180],[35,9313,9315,9338],{"className":9314},[184,185],[35,9316,9318,9335],{"className":9317},[189],[35,9319,9321],{"className":9320,"style":1943},[193],[35,9322,9323,9326],{"style":1946},[35,9324],{"className":9325,"style":202},[201],[35,9327,9329],{"className":9328},[206,207,208,209],[35,9330,9332],{"className":9331},[134,213,209],[35,9333,1874],{"className":9334},[134,209],[35,9336,221],{"className":9337},[220],[35,9339,9341],{"className":9340},[189],[35,9342,9344],{"className":9343,"style":1968},[193],[35,9345],{},", the decode bill is:",[35,9348,9350],{"className":9349},[38],[35,9351,9353,9421],{"className":9352},[42],[35,9354,9356],{"className":9355},[46],[48,9357,9358],{"xmlns":50,"display":51},[53,9359,9360,9418],{},[56,9361,9362,9368,9370,9416],{},[79,9363,9364,9366],{},[59,9365,61],{},[84,9367,109],{},[63,9369,65],{},[3706,9371,9372,9400],{},[56,9373,9374,9376,9378,9384,9386,9392,9394],{},[59,9375,938],{},[63,9377,1707],{},[79,9379,9380,9382],{},[59,9381,971],{},[84,9383,1050],{},[63,9385,1707],{},[2724,9387,9388,9390],{"accent":75},[59,9389,1885],{},[63,9391,2730],{},[84,9393,3503],{},[79,9395,9396,9398],{},[59,9397,792],{},[84,9399,1874],{},[56,9401,9402,9404,9406,9408,9410,9412,9414],{},[84,9403,8678],{},[63,9405,7533],{},[84,9407,868],{},[63,9409,8893],{},[59,9411,8814],{},[59,9413,2749],{"mathvariant":1396},[59,9415,1385],{},[59,9417,917],{"mathvariant":1396},[114,9419,9420],{"encoding":116},"G_\\text{decode} = \\frac{R \\times L_\\text{out} \\times \\bar n\\, M_\\text{kv\u002Ftok}}{\\text{MBU}\\cdot\\text{BW} - W\u002Ft}.",[35,9422,9424,9482],{"className":9423,"ariaHidden":75},[121],[35,9425,9427,9430,9473,9476,9479],{"className":9426},[125],[35,9428],{"className":9429,"style":409},[129],[35,9431,9433,9436],{"className":9432},[134],[35,9434,61],{"className":9435},[134,135],[35,9437,9439],{"className":9438},[180],[35,9440,9442,9465],{"className":9441},[184,185],[35,9443,9445,9462],{"className":9444},[189],[35,9446,9448],{"className":9447,"style":194},[193],[35,9449,9450,9453],{"style":197},[35,9451],{"className":9452,"style":202},[201],[35,9454,9456],{"className":9455},[206,207,208,209],[35,9457,9459],{"className":9458},[134,213,209],[35,9460,109],{"className":9461},[134,209],[35,9463,221],{"className":9464},[220],[35,9466,9468],{"className":9467},[189],[35,9469,9471],{"className":9470,"style":228},[193],[35,9472],{},[35,9474],{"className":9475,"style":140},[139],[35,9477,65],{"className":9478},[144],[35,9480],{"className":9481,"style":140},[139],[35,9483,9485,9489,9728],{"className":9484},[125],[35,9486],{"className":9487,"style":9488},[129],"height:2.3645em;vertical-align:-0.936em;",[35,9490,9492,9495,9725],{"className":9491},[134],[35,9493],{"className":9494},[3348,3818],[35,9496,9498],{"className":9497},[3706],[35,9499,9501,9716],{"className":9500},[184,185],[35,9502,9504,9713],{"className":9503},[189],[35,9505,9508,9555,9563],{"className":9506,"style":9507},[193],"height:1.4285em;",[35,9509,9510,9513],{"style":3834},[35,9511],{"className":9512,"style":2801},[201],[35,9514,9516,9522,9525,9528,9531,9537,9540,9543,9546,9549,9552],{"className":9515},[134],[35,9517,9519],{"className":9518},[134,213],[35,9520,8678],{"className":9521},[134],[35,9523],{"className":9524,"style":288},[139],[35,9526,7533],{"className":9527},[292],[35,9529],{"className":9530,"style":288},[139],[35,9532,9534],{"className":9533},[134,213],[35,9535,868],{"className":9536},[134],[35,9538],{"className":9539,"style":288},[139],[35,9541,8893],{"className":9542},[292],[35,9544],{"className":9545,"style":288},[139],[35,9547,8814],{"className":9548,"style":695},[134,135],[35,9550,2749],{"className":9551},[134],[35,9553,1385],{"className":9554},[134,135],[35,9556,9557,9560],{"style":3852},[35,9558],{"className":9559,"style":2801},[201],[35,9561],{"className":9562,"style":3860},[3859],[35,9564,9566,9569],{"style":9565},"top:-3.7452em;",[35,9567],{"className":9568,"style":2801},[201],[35,9570,9572,9575,9578,9581,9584,9627,9630,9633,9636,9667,9670],{"className":9571},[134],[35,9573,938],{"className":9574,"style":953},[134,135],[35,9576],{"className":9577,"style":288},[139],[35,9579,1707],{"className":9580},[292],[35,9582],{"className":9583,"style":288},[139],[35,9585,9587,9590],{"className":9586},[134],[35,9588,971],{"className":9589},[134,135],[35,9591,9593],{"className":9592},[180],[35,9594,9596,9619],{"className":9595},[184,185],[35,9597,9599,9616],{"className":9598},[189],[35,9600,9602],{"className":9601,"style":1081},[193],[35,9603,9604,9607],{"style":197},[35,9605],{"className":9606,"style":202},[201],[35,9608,9610],{"className":9609},[206,207,208,209],[35,9611,9613],{"className":9612},[134,213,209],[35,9614,1050],{"className":9615},[134,209],[35,9617,221],{"className":9618},[220],[35,9620,9622],{"className":9621},[189],[35,9623,9625],{"className":9624,"style":228},[193],[35,9626],{},[35,9628],{"className":9629,"style":288},[139],[35,9631,1707],{"className":9632},[292],[35,9634],{"className":9635,"style":288},[139],[35,9637,9639],{"className":9638},[134,2785],[35,9640,9642],{"className":9641},[184],[35,9643,9645],{"className":9644},[189],[35,9646,9648,9656],{"className":9647,"style":2781},[193],[35,9649,9650,9653],{"style":2797},[35,9651],{"className":9652,"style":2801},[201],[35,9654,1885],{"className":9655},[134,135],[35,9657,9658,9661],{"style":2797},[35,9659],{"className":9660,"style":2801},[201],[35,9662,9664],{"className":9663,"style":2814},[2813],[35,9665,2730],{"className":9666},[134],[35,9668],{"className":9669,"style":162},[139],[35,9671,9673,9676],{"className":9672},[134],[35,9674,792],{"className":9675,"style":814},[134,135],[35,9677,9679],{"className":9678},[180],[35,9680,9682,9705],{"className":9681},[184,185],[35,9683,9685,9702],{"className":9684},[189],[35,9686,9688],{"className":9687,"style":1943},[193],[35,9689,9690,9693],{"style":1946},[35,9691],{"className":9692,"style":202},[201],[35,9694,9696],{"className":9695},[206,207,208,209],[35,9697,9699],{"className":9698},[134,213,209],[35,9700,1874],{"className":9701},[134,209],[35,9703,221],{"className":9704},[220],[35,9706,9708],{"className":9707},[189],[35,9709,9711],{"className":9710,"style":1968},[193],[35,9712],{},[35,9714,221],{"className":9715},[220],[35,9717,9719],{"className":9718},[189],[35,9720,9723],{"className":9721,"style":9722},[193],"height:0.936em;",[35,9724],{},[35,9726],{"className":9727},[3453,3818],[35,9729,917],{"className":9730},[134],[10,9732,9733,9734,9793,9794,9886,9887,9948,9949,2273],{},"For Qwen3-8B, ",[35,9735,9737,9757],{"className":9736},[42],[35,9738,9740],{"className":9739},[46],[48,9741,9742],{"xmlns":50},[53,9743,9744,9754],{},[56,9745,9746,9748,9750,9752],{},[59,9747,8814],{},[63,9749,65],{},[1342,9751,3714],{},[84,9753,3070],{},[114,9755,9756],{"encoding":116},"W=16\\text{ GB}",[35,9758,9760,9778],{"className":9759,"ariaHidden":75},[121],[35,9761,9763,9766,9769,9772,9775],{"className":9762},[125],[35,9764],{"className":9765,"style":130},[129],[35,9767,8814],{"className":9768,"style":695},[134,135],[35,9770],{"className":9771,"style":140},[139],[35,9773,65],{"className":9774},[144],[35,9776],{"className":9777,"style":140},[139],[35,9779,9781,9784,9787],{"className":9780},[125],[35,9782],{"className":9783,"style":130},[129],[35,9785,3714],{"className":9786},[134],[35,9788,9790],{"className":9789},[134,213],[35,9791,3070],{"className":9792},[134]," and ",[35,9795,9797,9819],{"className":9796},[42],[35,9798,9800],{"className":9799},[46],[48,9801,9802],{"xmlns":50},[53,9803,9804,9816],{},[56,9805,9806,9812,9814],{},[2724,9807,9808,9810],{"accent":75},[59,9809,1885],{},[63,9811,2730],{},[63,9813,65],{},[1342,9815,3057],{},[114,9817,9818],{"encoding":116},"\\bar n=4{,}500",[35,9820,9822,9868],{"className":9821,"ariaHidden":75},[121],[35,9823,9825,9828,9859,9862,9865],{"className":9824},[125],[35,9826],{"className":9827,"style":2781},[129],[35,9829,9831],{"className":9830},[134,2785],[35,9832,9834],{"className":9833},[184],[35,9835,9837],{"className":9836},[189],[35,9838,9840,9848],{"className":9839,"style":2781},[193],[35,9841,9842,9845],{"style":2797},[35,9843],{"className":9844,"style":2801},[201],[35,9846,1885],{"className":9847},[134,135],[35,9849,9850,9853],{"style":2797},[35,9851],{"className":9852,"style":2801},[201],[35,9854,9856],{"className":9855,"style":2814},[2813],[35,9857,2730],{"className":9858},[134],[35,9860],{"className":9861,"style":140},[139],[35,9863,65],{"className":9864},[144],[35,9866],{"className":9867,"style":140},[139],[35,9869,9871,9874,9877,9883],{"className":9870},[125],[35,9872],{"className":9873,"style":1587},[129],[35,9875,1509],{"className":9876},[134],[35,9878,9880],{"className":9879},[134],[35,9881,89],{"className":9882},[234],[35,9884,3179],{"className":9885},[134],". On an H200, usable bandwidth is ",[35,9888,9890,9912],{"className":9889},[42],[35,9891,9893],{"className":9892},[46],[48,9894,9895],{"xmlns":50},[53,9896,9897,9909],{},[56,9898,9899,9901,9903,9906],{},[1342,9900,8739],{},[63,9902,1707],{},[1342,9904,9905],{},"4.8",[84,9907,9908],{}," TB\u002Fs",[114,9910,9911],{"encoding":116},"0.6 \\times 4.8\\text{ TB\u002Fs}",[35,9913,9915,9933],{"className":9914,"ariaHidden":75},[121],[35,9916,9918,9921,9924,9927,9930],{"className":9917},[125],[35,9919],{"className":9920,"style":1986},[129],[35,9922,8739],{"className":9923},[134],[35,9925],{"className":9926,"style":288},[139],[35,9928,1707],{"className":9929},[292],[35,9931],{"className":9932,"style":288},[139],[35,9934,9936,9939,9942],{"className":9935},[125],[35,9937],{"className":9938,"style":1427},[129],[35,9940,9905],{"className":9941},[134],[35,9943,9945],{"className":9944},[134,213],[35,9946,9908],{"className":9947},[134],", and the weight read consumes ",[35,9950,9952,9974],{"className":9951},[42],[35,9953,9955],{"className":9954},[46],[48,9956,9957],{"xmlns":50},[53,9958,9959,9971],{},[56,9960,9961,9963,9965,9967,9969],{},[1342,9962,3714],{},[84,9964,3070],{},[59,9966,2749],{"mathvariant":1396},[1342,9968,1390],{},[84,9970,1393],{},[114,9972,9973],{"encoding":116},"16\\text{ GB}\u002F50\\text{ ms}",[35,9975,9977],{"className":9976,"ariaHidden":75},[121],[35,9978,9980,9983,9986,9992,9996],{"className":9979},[125],[35,9981],{"className":9982,"style":1427},[129],[35,9984,3714],{"className":9985},[134],[35,9987,9989],{"className":9988},[134,213],[35,9990,3070],{"className":9991},[134],[35,9993,9995],{"className":9994},[134],"\u002F50",[35,9997,9999],{"className":9998},[134,213],[35,10000,1393],{"className":10001},[134],[35,10003,10005],{"className":10004},[38],[35,10006,10008,10075],{"className":10007},[42],[35,10009,10011],{"className":10010},[46],[48,10012,10013],{"xmlns":50,"display":51},[53,10014,10015,10072],{},[56,10016,10017,10023,10025,10067,10069],{},[79,10018,10019,10021],{},[59,10020,61],{},[84,10022,109],{},[63,10024,65],{},[3706,10026,10027,10045],{},[56,10028,10029,10031,10033,10035,10037,10039,10041,10043],{},[1342,10030,1344],{},[63,10032,1707],{},[1342,10034,1378],{},[63,10036,1707],{},[1342,10038,3057],{},[63,10040,1707],{},[1342,10042,3049],{},[84,10044,3052],{},[56,10046,10047,10049,10051,10053,10055,10057,10059,10061,10063,10065],{},[1342,10048,8739],{},[63,10050,1707],{},[1342,10052,9905],{},[84,10054,9908],{},[63,10056,8893],{},[1342,10058,3714],{},[84,10060,3070],{},[59,10062,2749],{"mathvariant":1396},[1342,10064,1390],{},[84,10066,1393],{},[63,10068,2494],{},[1342,10070,10071],{},"0.52.",[114,10073,10074],{"encoding":116},"G_\\text{decode} = \\frac{2 \\times 1{,}000 \\times 4{,}500 \\times 144\\text{ KB}}{0.6 \\times 4.8\\text{ TB\u002Fs} - 16\\text{ GB}\u002F50\\text{ ms}} \\approx 0.52.",[35,10076,10078,10136,10319],{"className":10077,"ariaHidden":75},[121],[35,10079,10081,10084,10127,10130,10133],{"className":10080},[125],[35,10082],{"className":10083,"style":409},[129],[35,10085,10087,10090],{"className":10086},[134],[35,10088,61],{"className":10089},[134,135],[35,10091,10093],{"className":10092},[180],[35,10094,10096,10119],{"className":10095},[184,185],[35,10097,10099,10116],{"className":10098},[189],[35,10100,10102],{"className":10101,"style":194},[193],[35,10103,10104,10107],{"style":197},[35,10105],{"className":10106,"style":202},[201],[35,10108,10110],{"className":10109},[206,207,208,209],[35,10111,10113],{"className":10112},[134,213,209],[35,10114,109],{"className":10115},[134,209],[35,10117,221],{"className":10118},[220],[35,10120,10122],{"className":10121},[189],[35,10123,10125],{"className":10124,"style":228},[193],[35,10126],{},[35,10128],{"className":10129,"style":140},[139],[35,10131,65],{"className":10132},[144],[35,10134],{"className":10135,"style":140},[139],[35,10137,10139,10143,10310,10313,10316],{"className":10138},[125],[35,10140],{"className":10141,"style":10142},[129],"height:2.2963em;vertical-align:-0.936em;",[35,10144,10146,10149,10307],{"className":10145},[134],[35,10147],{"className":10148},[3348,3818],[35,10150,10152],{"className":10151},[3706],[35,10153,10155,10299],{"className":10154},[184,185],[35,10156,10158,10296],{"className":10157},[189],[35,10159,10161,10217,10225],{"className":10160,"style":4875},[193],[35,10162,10163,10166],{"style":3834},[35,10164],{"className":10165,"style":2801},[201],[35,10167,10169,10172,10175,10178,10181,10184,10190,10193,10196,10199,10202,10208,10211],{"className":10168},[134],[35,10170,8739],{"className":10171},[134],[35,10173],{"className":10174,"style":288},[139],[35,10176,1707],{"className":10177},[292],[35,10179],{"className":10180,"style":288},[139],[35,10182,9905],{"className":10183},[134],[35,10185,10187],{"className":10186},[134,213],[35,10188,9908],{"className":10189},[134],[35,10191],{"className":10192,"style":288},[139],[35,10194,8893],{"className":10195},[292],[35,10197],{"className":10198,"style":288},[139],[35,10200,3714],{"className":10201},[134],[35,10203,10205],{"className":10204},[134,213],[35,10206,3070],{"className":10207},[134],[35,10209,9995],{"className":10210},[134],[35,10212,10214],{"className":10213},[134,213],[35,10215,1393],{"className":10216},[134],[35,10218,10219,10222],{"style":3852},[35,10220],{"className":10221,"style":2801},[201],[35,10223],{"className":10224,"style":3860},[3859],[35,10226,10227,10230],{"style":3863},[35,10228],{"className":10229,"style":2801},[201],[35,10231,10233,10236,10239,10242,10245,10248,10254,10257,10260,10263,10266,10269,10275,10278,10281,10284,10287,10290],{"className":10232},[134],[35,10234,1344],{"className":10235},[134],[35,10237],{"className":10238,"style":288},[139],[35,10240,1707],{"className":10241},[292],[35,10243],{"className":10244,"style":288},[139],[35,10246,1314],{"className":10247},[134],[35,10249,10251],{"className":10250},[134],[35,10252,89],{"className":10253},[234],[35,10255,1519],{"className":10256},[134],[35,10258],{"className":10259,"style":288},[139],[35,10261,1707],{"className":10262},[292],[35,10264],{"className":10265,"style":288},[139],[35,10267,1509],{"className":10268},[134],[35,10270,10272],{"className":10271},[134],[35,10273,89],{"className":10274},[234],[35,10276,3179],{"className":10277},[134],[35,10279],{"className":10280,"style":288},[139],[35,10282,1707],{"className":10283},[292],[35,10285],{"className":10286,"style":288},[139],[35,10288,3049],{"className":10289},[134],[35,10291,10293],{"className":10292},[134,213],[35,10294,3052],{"className":10295},[134],[35,10297,221],{"className":10298},[220],[35,10300,10302],{"className":10301},[189],[35,10303,10305],{"className":10304,"style":9722},[193],[35,10306],{},[35,10308],{"className":10309},[3453,3818],[35,10311],{"className":10312,"style":140},[139],[35,10314,2494],{"className":10315},[144],[35,10317],{"className":10318,"style":140},[139],[35,10320,10322,10325],{"className":10321},[125],[35,10323],{"className":10324,"style":1627},[129],[35,10326,10071],{"className":10327},[134],[10,10329,10330,10331,10369,10370,10426],{},"If ",[35,10332,10334,10351],{"className":10333},[42],[35,10335,10337],{"className":10336},[46],[48,10338,10339],{"xmlns":50},[53,10340,10341,10349],{},[56,10342,10343,10345,10347],{},[59,10344,8814],{},[59,10346,2749],{"mathvariant":1396},[59,10348,1385],{},[114,10350,8850],{"encoding":116},[35,10352,10354],{"className":10353,"ariaHidden":75},[121],[35,10355,10357,10360,10363,10366],{"className":10356},[125],[35,10358],{"className":10359,"style":1427},[129],[35,10361,8814],{"className":10362,"style":695},[134,135],[35,10364,2749],{"className":10365},[134],[35,10367,1385],{"className":10368},[134,135]," approaches ",[35,10371,10373,10390],{"className":10372},[42],[35,10374,10376],{"className":10375},[46],[48,10377,10378],{"xmlns":50},[53,10379,10380,10388],{},[56,10381,10382,10384,10386],{},[84,10383,8678],{},[63,10385,7533],{},[84,10387,868],{},[114,10389,8685],{"encoding":116},[35,10391,10393,10414],{"className":10392,"ariaHidden":75},[121],[35,10394,10396,10399,10405,10408,10411],{"className":10395},[125],[35,10397],{"className":10398,"style":130},[129],[35,10400,10402],{"className":10401},[134,213],[35,10403,8678],{"className":10404},[134],[35,10406],{"className":10407,"style":288},[139],[35,10409,7533],{"className":10410},[292],[35,10412],{"className":10413,"style":288},[139],[35,10415,10417,10420],{"className":10416},[125],[35,10418],{"className":10419,"style":130},[129],[35,10421,10423],{"className":10422},[134,213],[35,10424,868],{"className":10425},[134],", the GPU cuont explodes. We cannot hit the TPOT target without splitting the model across GPUs to shrink the per-GPU weight read (and each split adding all-reduce network overhead per step).",[2249,10428,10430,10610,10917,11156,11425,11767],{"title":10429},"Where the decode GPU count comes from",[10,10431,10432,10433,10461,10462,10490,10491,10552,10553,10609],{},"Each decode step reads the weights once across the batch (",[35,10434,10436,10449],{"className":10435},[42],[35,10437,10439],{"className":10438},[46],[48,10440,10441],{"xmlns":50},[53,10442,10443,10447],{},[56,10444,10445],{},[59,10446,8814],{},[114,10448,8814],{"encoding":116},[35,10450,10452],{"className":10451,"ariaHidden":75},[121],[35,10453,10455,10458],{"className":10454},[125],[35,10456],{"className":10457,"style":130},[129],[35,10459,8814],{"className":10460,"style":695},[134,135]," bytes) and the KV history once per stream. With ",[35,10463,10465,10478],{"className":10464},[42],[35,10466,10468],{"className":10467},[46],[48,10469,10470],{"xmlns":50},[53,10471,10472,10476],{},[56,10473,10474],{},[59,10475,5282],{},[114,10477,5282],{"encoding":116},[35,10479,10481],{"className":10480,"ariaHidden":75},[121],[35,10482,10484,10487],{"className":10483},[125],[35,10485],{"className":10486,"style":130},[129],[35,10488,5282],{"className":10489,"style":8654},[134,135]," streams, average context ",[35,10492,10494,10512],{"className":10493},[42],[35,10495,10497],{"className":10496},[46],[48,10498,10499],{"xmlns":50},[53,10500,10501,10509],{},[56,10502,10503],{},[2724,10504,10505,10507],{"accent":75},[59,10506,1885],{},[63,10508,2730],{},[114,10510,10511],{"encoding":116},"\\bar n",[35,10513,10515],{"className":10514,"ariaHidden":75},[121],[35,10516,10518,10521],{"className":10517},[125],[35,10519],{"className":10520,"style":2781},[129],[35,10522,10524],{"className":10523},[134,2785],[35,10525,10527],{"className":10526},[184],[35,10528,10530],{"className":10529},[189],[35,10531,10533,10541],{"className":10532,"style":2781},[193],[35,10534,10535,10538],{"style":2797},[35,10536],{"className":10537,"style":2801},[201],[35,10539,1885],{"className":10540},[134,135],[35,10542,10543,10546],{"style":2797},[35,10544],{"className":10545,"style":2801},[201],[35,10547,10549],{"className":10548,"style":2814},[2813],[35,10550,2730],{"className":10551},[134],", and effective bandwidth ",[35,10554,10556,10573],{"className":10555},[42],[35,10557,10559],{"className":10558},[46],[48,10560,10561],{"xmlns":50},[53,10562,10563,10571],{},[56,10564,10565,10567,10569],{},[84,10566,8678],{},[63,10568,7533],{},[84,10570,868],{},[114,10572,8685],{"encoding":116},[35,10574,10576,10597],{"className":10575,"ariaHidden":75},[121],[35,10577,10579,10582,10588,10591,10594],{"className":10578},[125],[35,10580],{"className":10581,"style":130},[129],[35,10583,10585],{"className":10584},[134,213],[35,10586,8678],{"className":10587},[134],[35,10589],{"className":10590,"style":288},[139],[35,10592,7533],{"className":10593},[292],[35,10595],{"className":10596,"style":288},[139],[35,10598,10600,10603],{"className":10599},[125],[35,10601],{"className":10602,"style":130},[129],[35,10604,10606],{"className":10605},[134,213],[35,10607,868],{"className":10608},[134],", the step time is:",[35,10611,10613],{"className":10612},[38],[35,10614,10616,10673],{"className":10615},[42],[35,10617,10619],{"className":10618},[46],[48,10620,10621],{"xmlns":50,"display":51},[53,10622,10623,10670],{},[56,10624,10625,10632,10634,10668],{},[79,10626,10627,10629],{},[59,10628,1385],{},[84,10630,10631],{},"step",[63,10633,65],{},[3706,10635,10636,10660],{},[56,10637,10638,10640,10642,10644,10646,10652,10654],{},[59,10639,8814],{},[63,10641,102],{},[59,10643,5282],{},[84,10645,3503],{},[2724,10647,10648,10650],{"accent":75},[59,10649,1885],{},[63,10651,2730],{},[84,10653,3503],{},[79,10655,10656,10658],{},[59,10657,792],{},[84,10659,1874],{},[56,10661,10662,10664,10666],{},[84,10663,8678],{},[63,10665,7533],{},[84,10667,868],{},[59,10669,917],{"mathvariant":1396},[114,10671,10672],{"encoding":116},"t_\\text{step} = \\frac{W + B\\,\\bar n\\, M_\\text{kv\u002Ftok}}{\\text{MBU}\\cdot\\text{BW}}.",[35,10674,10676,10735],{"className":10675,"ariaHidden":75},[121],[35,10677,10679,10683,10726,10729,10732],{"className":10678},[125],[35,10680],{"className":10681,"style":10682},[129],"height:0.9012em;vertical-align:-0.2861em;",[35,10684,10686,10689],{"className":10685},[134],[35,10687,1385],{"className":10688},[134,135],[35,10690,10692],{"className":10691},[180],[35,10693,10695,10718],{"className":10694},[184,185],[35,10696,10698,10715],{"className":10697},[189],[35,10699,10701],{"className":10700,"style":1081},[193],[35,10702,10703,10706],{"style":197},[35,10704],{"className":10705,"style":202},[201],[35,10707,10709],{"className":10708},[206,207,208,209],[35,10710,10712],{"className":10711},[134,213,209],[35,10713,10631],{"className":10714},[134,209],[35,10716,221],{"className":10717},[220],[35,10719,10721],{"className":10720},[189],[35,10722,10724],{"className":10723,"style":282},[193],[35,10725],{},[35,10727],{"className":10728,"style":140},[139],[35,10730,65],{"className":10731},[144],[35,10733],{"className":10734,"style":140},[139],[35,10736,10738,10742,10914],{"className":10737},[125],[35,10739],{"className":10740,"style":10741},[129],"height:2.1145em;vertical-align:-0.686em;",[35,10743,10745,10748,10911],{"className":10744},[134],[35,10746],{"className":10747},[3348,3818],[35,10749,10751],{"className":10750},[3706],[35,10752,10754,10903],{"className":10753},[184,185],[35,10755,10757,10900],{"className":10756},[189],[35,10758,10760,10789,10797],{"className":10759,"style":9507},[193],[35,10761,10762,10765],{"style":3834},[35,10763],{"className":10764,"style":2801},[201],[35,10766,10768,10774,10777,10780,10783],{"className":10767},[134],[35,10769,10771],{"className":10770},[134,213],[35,10772,8678],{"className":10773},[134],[35,10775],{"className":10776,"style":288},[139],[35,10778,7533],{"className":10779},[292],[35,10781],{"className":10782,"style":288},[139],[35,10784,10786],{"className":10785},[134,213],[35,10787,868],{"className":10788},[134],[35,10790,10791,10794],{"style":3852},[35,10792],{"className":10793,"style":2801},[201],[35,10795],{"className":10796,"style":3860},[3859],[35,10798,10799,10802],{"style":9565},[35,10800],{"className":10801,"style":2801},[201],[35,10803,10805,10808,10811,10814,10817,10820,10823,10854,10857],{"className":10804},[134],[35,10806,8814],{"className":10807,"style":695},[134,135],[35,10809],{"className":10810,"style":288},[139],[35,10812,102],{"className":10813},[292],[35,10815],{"className":10816,"style":288},[139],[35,10818,5282],{"className":10819,"style":8654},[134,135],[35,10821],{"className":10822,"style":162},[139],[35,10824,10826],{"className":10825},[134,2785],[35,10827,10829],{"className":10828},[184],[35,10830,10832],{"className":10831},[189],[35,10833,10835,10843],{"className":10834,"style":2781},[193],[35,10836,10837,10840],{"style":2797},[35,10838],{"className":10839,"style":2801},[201],[35,10841,1885],{"className":10842},[134,135],[35,10844,10845,10848],{"style":2797},[35,10846],{"className":10847,"style":2801},[201],[35,10849,10851],{"className":10850,"style":2814},[2813],[35,10852,2730],{"className":10853},[134],[35,10855],{"className":10856,"style":162},[139],[35,10858,10860,10863],{"className":10859},[134],[35,10861,792],{"className":10862,"style":814},[134,135],[35,10864,10866],{"className":10865},[180],[35,10867,10869,10892],{"className":10868},[184,185],[35,10870,10872,10889],{"className":10871},[189],[35,10873,10875],{"className":10874,"style":1943},[193],[35,10876,10877,10880],{"style":1946},[35,10878],{"className":10879,"style":202},[201],[35,10881,10883],{"className":10882},[206,207,208,209],[35,10884,10886],{"className":10885},[134,213,209],[35,10887,1874],{"className":10888},[134,209],[35,10890,221],{"className":10891},[220],[35,10893,10895],{"className":10894},[189],[35,10896,10898],{"className":10897,"style":1968},[193],[35,10899],{},[35,10901,221],{"className":10902},[220],[35,10904,10906],{"className":10905},[189],[35,10907,10909],{"className":10908,"style":3924},[193],[35,10910],{},[35,10912],{"className":10913},[3453,3818],[35,10915,917],{"className":10916},[134],[10,10918,10919,10920,11017,11018,11046,11047,11126,11127,11155],{},"Require ",[35,10921,10923,10946],{"className":10922},[42],[35,10924,10926],{"className":10925},[46],[48,10927,10928],{"xmlns":50},[53,10929,10930,10943],{},[56,10931,10932,10938,10941],{},[79,10933,10934,10936],{},[59,10935,1385],{},[84,10937,10631],{},[63,10939,10940],{},"≤",[59,10942,1385],{},[114,10944,10945],{"encoding":116},"t_\\text{step} \\le t",[35,10947,10949,11008],{"className":10948,"ariaHidden":75},[121],[35,10950,10952,10956,10999,11002,11005],{"className":10951},[125],[35,10953],{"className":10954,"style":10955},[129],"height:0.9221em;vertical-align:-0.2861em;",[35,10957,10959,10962],{"className":10958},[134],[35,10960,1385],{"className":10961},[134,135],[35,10963,10965],{"className":10964},[180],[35,10966,10968,10991],{"className":10967},[184,185],[35,10969,10971,10988],{"className":10970},[189],[35,10972,10974],{"className":10973,"style":1081},[193],[35,10975,10976,10979],{"style":197},[35,10977],{"className":10978,"style":202},[201],[35,10980,10982],{"className":10981},[206,207,208,209],[35,10983,10985],{"className":10984},[134,213,209],[35,10986,10631],{"className":10987},[134,209],[35,10989,221],{"className":10990},[220],[35,10992,10994],{"className":10993},[189],[35,10995,10997],{"className":10996,"style":282},[193],[35,10998],{},[35,11000],{"className":11001,"style":140},[139],[35,11003,10940],{"className":11004},[144],[35,11006],{"className":11007,"style":140},[139],[35,11009,11011,11014],{"className":11010},[125],[35,11012],{"className":11013,"style":2596},[129],[35,11015,1385],{"className":11016},[134,135]," and solve for ",[35,11019,11021,11034],{"className":11020},[42],[35,11022,11024],{"className":11023},[46],[48,11025,11026],{"xmlns":50},[53,11027,11028,11032],{},[56,11029,11030],{},[59,11031,5282],{},[114,11033,5282],{"encoding":116},[35,11035,11037],{"className":11036,"ariaHidden":75},[121],[35,11038,11040,11043],{"className":11039},[125],[35,11041],{"className":11042,"style":130},[129],[35,11044,5282],{"className":11045,"style":8654},[134,135],". The step's byte budget is ",[35,11048,11050,11072],{"className":11049},[42],[35,11051,11053],{"className":11052},[46],[48,11054,11055],{"xmlns":50},[53,11056,11057,11069],{},[56,11058,11059,11061,11063,11065,11067],{},[84,11060,8678],{},[63,11062,7533],{},[84,11064,868],{},[63,11066,7533],{},[59,11068,1385],{},[114,11070,11071],{"encoding":116},"\\text{MBU}\\cdot\\text{BW}\\cdot t",[35,11073,11075,11096,11117],{"className":11074,"ariaHidden":75},[121],[35,11076,11078,11081,11087,11090,11093],{"className":11077},[125],[35,11079],{"className":11080,"style":130},[129],[35,11082,11084],{"className":11083},[134,213],[35,11085,8678],{"className":11086},[134],[35,11088],{"className":11089,"style":288},[139],[35,11091,7533],{"className":11092},[292],[35,11094],{"className":11095,"style":288},[139],[35,11097,11099,11102,11108,11111,11114],{"className":11098},[125],[35,11100],{"className":11101,"style":130},[129],[35,11103,11105],{"className":11104},[134,213],[35,11106,868],{"className":11107},[134],[35,11109],{"className":11110,"style":288},[139],[35,11112,7533],{"className":11113},[292],[35,11115],{"className":11116,"style":288},[139],[35,11118,11120,11123],{"className":11119},[125],[35,11121],{"className":11122,"style":2596},[129],[35,11124,1385],{"className":11125},[134,135],"; subtract the fixed weight read ",[35,11128,11130,11143],{"className":11129},[42],[35,11131,11133],{"className":11132},[46],[48,11134,11135],{"xmlns":50},[53,11136,11137,11141],{},[56,11138,11139],{},[59,11140,8814],{},[114,11142,8814],{"encoding":116},[35,11144,11146],{"className":11145,"ariaHidden":75},[121],[35,11147,11149,11152],{"className":11148},[125],[35,11150],{"className":11151,"style":130},[129],[35,11153,8814],{"className":11154,"style":695},[134,135]," and divide by the per-stream KV cost:",[35,11157,11159],{"className":11158},[38],[35,11160,11162,11214],{"className":11161},[42],[35,11163,11165],{"className":11164},[46],[48,11166,11167],{"xmlns":50,"display":51},[53,11168,11169,11211],{},[56,11170,11171,11173,11175,11209],{},[59,11172,5282],{},[63,11174,10940],{},[3706,11176,11177,11193],{},[56,11178,11179,11181,11183,11185,11187,11189,11191],{},[84,11180,8678],{},[63,11182,7533],{},[84,11184,868],{},[63,11186,7533],{},[59,11188,1385],{},[63,11190,8893],{},[59,11192,8814],{},[56,11194,11195,11201,11203],{},[2724,11196,11197,11199],{"accent":75},[59,11198,1885],{},[63,11200,2730],{},[84,11202,3503],{},[79,11204,11205,11207],{},[59,11206,792],{},[84,11208,1874],{},[59,11210,917],{"mathvariant":1396},[114,11212,11213],{"encoding":116},"B \\le \\frac{\\text{MBU}\\cdot\\text{BW}\\cdot t - W}{\\bar n\\, M_\\text{kv\u002Ftok}}.",[35,11215,11217,11236],{"className":11216,"ariaHidden":75},[121],[35,11218,11220,11224,11227,11230,11233],{"className":11219},[125],[35,11221],{"className":11222,"style":11223},[129],"height:0.8193em;vertical-align:-0.136em;",[35,11225,5282],{"className":11226,"style":8654},[134,135],[35,11228],{"className":11229,"style":140},[139],[35,11231,10940],{"className":11232},[144],[35,11234],{"className":11235,"style":140},[139],[35,11237,11239,11243,11422],{"className":11238},[125],[35,11240],{"className":11241,"style":11242},[129],"height:2.4015em;vertical-align:-1.0412em;",[35,11244,11246,11249,11419],{"className":11245},[134],[35,11247],{"className":11248},[3348,3818],[35,11250,11252],{"className":11251},[3706],[35,11253,11255,11410],{"className":11254},[184,185],[35,11256,11258,11407],{"className":11257},[189],[35,11259,11261,11346,11354],{"className":11260,"style":4875},[193],[35,11262,11263,11266],{"style":3834},[35,11264],{"className":11265,"style":2801},[201],[35,11267,11269,11300,11303],{"className":11268},[134],[35,11270,11272],{"className":11271},[134,2785],[35,11273,11275],{"className":11274},[184],[35,11276,11278],{"className":11277},[189],[35,11279,11281,11289],{"className":11280,"style":2781},[193],[35,11282,11283,11286],{"style":2797},[35,11284],{"className":11285,"style":2801},[201],[35,11287,1885],{"className":11288},[134,135],[35,11290,11291,11294],{"style":2797},[35,11292],{"className":11293,"style":2801},[201],[35,11295,11297],{"className":11296,"style":2814},[2813],[35,11298,2730],{"className":11299},[134],[35,11301],{"className":11302,"style":162},[139],[35,11304,11306,11309],{"className":11305},[134],[35,11307,792],{"className":11308,"style":814},[134,135],[35,11310,11312],{"className":11311},[180],[35,11313,11315,11338],{"className":11314},[184,185],[35,11316,11318,11335],{"className":11317},[189],[35,11319,11321],{"className":11320,"style":1943},[193],[35,11322,11323,11326],{"style":1946},[35,11324],{"className":11325,"style":202},[201],[35,11327,11329],{"className":11328},[206,207,208,209],[35,11330,11332],{"className":11331},[134,213,209],[35,11333,1874],{"className":11334},[134,209],[35,11336,221],{"className":11337},[220],[35,11339,11341],{"className":11340},[189],[35,11342,11344],{"className":11343,"style":1968},[193],[35,11345],{},[35,11347,11348,11351],{"style":3852},[35,11349],{"className":11350,"style":2801},[201],[35,11352],{"className":11353,"style":3860},[3859],[35,11355,11356,11359],{"style":3863},[35,11357],{"className":11358,"style":2801},[201],[35,11360,11362,11368,11371,11374,11377,11383,11386,11389,11392,11395,11398,11401,11404],{"className":11361},[134],[35,11363,11365],{"className":11364},[134,213],[35,11366,8678],{"className":11367},[134],[35,11369],{"className":11370,"style":288},[139],[35,11372,7533],{"className":11373},[292],[35,11375],{"className":11376,"style":288},[139],[35,11378,11380],{"className":11379},[134,213],[35,11381,868],{"className":11382},[134],[35,11384],{"className":11385,"style":288},[139],[35,11387,7533],{"className":11388},[292],[35,11390],{"className":11391,"style":288},[139],[35,11393,1385],{"className":11394},[134,135],[35,11396],{"className":11397,"style":288},[139],[35,11399,8893],{"className":11400},[292],[35,11402],{"className":11403,"style":288},[139],[35,11405,8814],{"className":11406,"style":695},[134,135],[35,11408,221],{"className":11409},[220],[35,11411,11413],{"className":11412},[189],[35,11414,11417],{"className":11415,"style":11416},[193],"height:1.0412em;",[35,11418],{},[35,11420],{"className":11421},[3453,3818],[35,11423,917],{"className":11424},[134],[10,11426,11427,11428,11456,11457,11485,11486,11514,11515,11610,11611,11738,11739,2273],{},"One decode step advances all ",[35,11429,11431,11444],{"className":11430},[42],[35,11432,11434],{"className":11433},[46],[48,11435,11436],{"xmlns":50},[53,11437,11438,11442],{},[56,11439,11440],{},[59,11441,5282],{},[114,11443,5282],{"encoding":116},[35,11445,11447],{"className":11446,"ariaHidden":75},[121],[35,11448,11450,11453],{"className":11449},[125],[35,11451],{"className":11452,"style":130},[129],[35,11454,5282],{"className":11455,"style":8654},[134,135]," streams by one token, so the GPU produces ",[35,11458,11460,11473],{"className":11459},[42],[35,11461,11463],{"className":11462},[46],[48,11464,11465],{"xmlns":50},[53,11466,11467,11471],{},[56,11468,11469],{},[59,11470,5282],{},[114,11472,5282],{"encoding":116},[35,11474,11476],{"className":11475,"ariaHidden":75},[121],[35,11477,11479,11482],{"className":11478},[125],[35,11480],{"className":11481,"style":130},[129],[35,11483,5282],{"className":11484,"style":8654},[134,135]," tokens in ",[35,11487,11489,11502],{"className":11488},[42],[35,11490,11492],{"className":11491},[46],[48,11493,11494],{"xmlns":50},[53,11495,11496,11500],{},[56,11497,11498],{},[59,11499,1385],{},[114,11501,1385],{"encoding":116},[35,11503,11505],{"className":11504,"ariaHidden":75},[121],[35,11506,11508,11511],{"className":11507},[125],[35,11509],{"className":11510,"style":2596},[129],[35,11512,1385],{"className":11513},[134,135]," seconds. Demand is ",[35,11516,11518,11540],{"className":11517},[42],[35,11519,11521],{"className":11520},[46],[48,11522,11523],{"xmlns":50},[53,11524,11525,11537],{},[56,11526,11527,11529,11531],{},[59,11528,938],{},[63,11530,1707],{},[79,11532,11533,11535],{},[59,11534,971],{},[84,11536,1050],{},[114,11538,11539],{"encoding":116},"R \\times L_\\text{out}",[35,11541,11543,11561],{"className":11542,"ariaHidden":75},[121],[35,11544,11546,11549,11552,11555,11558],{"className":11545},[125],[35,11547],{"className":11548,"style":1781},[129],[35,11550,938],{"className":11551,"style":953},[134,135],[35,11553],{"className":11554,"style":288},[139],[35,11556,1707],{"className":11557},[292],[35,11559],{"className":11560,"style":288},[139],[35,11562,11564,11567],{"className":11563},[125],[35,11565],{"className":11566,"style":409},[129],[35,11568,11570,11573],{"className":11569},[134],[35,11571,971],{"className":11572},[134,135],[35,11574,11576],{"className":11575},[180],[35,11577,11579,11602],{"className":11578},[184,185],[35,11580,11582,11599],{"className":11581},[189],[35,11583,11585],{"className":11584,"style":1081},[193],[35,11586,11587,11590],{"style":197},[35,11588],{"className":11589,"style":202},[201],[35,11591,11593],{"className":11592},[206,207,208,209],[35,11594,11596],{"className":11595},[134,213,209],[35,11597,1050],{"className":11598},[134,209],[35,11600,221],{"className":11601},[220],[35,11603,11605],{"className":11604},[189],[35,11606,11608],{"className":11607,"style":228},[193],[35,11609],{}," tokens per second, so GPU count is ",[35,11612,11614,11644],{"className":11613},[42],[35,11615,11617],{"className":11616},[46],[48,11618,11619],{"xmlns":50},[53,11620,11621,11641],{},[56,11622,11623,11625,11627,11633,11635,11637,11639],{},[59,11624,938],{},[63,11626,1707],{},[79,11628,11629,11631],{},[59,11630,971],{},[84,11632,1050],{},[63,11634,1707],{},[59,11636,1385],{},[59,11638,2749],{"mathvariant":1396},[59,11640,5282],{},[114,11642,11643],{"encoding":116},"R \\times L_\\text{out} \\times t \u002F B",[35,11645,11647,11665,11723],{"className":11646,"ariaHidden":75},[121],[35,11648,11650,11653,11656,11659,11662],{"className":11649},[125],[35,11651],{"className":11652,"style":1781},[129],[35,11654,938],{"className":11655,"style":953},[134,135],[35,11657],{"className":11658,"style":288},[139],[35,11660,1707],{"className":11661},[292],[35,11663],{"className":11664,"style":288},[139],[35,11666,11668,11671,11714,11717,11720],{"className":11667},[125],[35,11669],{"className":11670,"style":409},[129],[35,11672,11674,11677],{"className":11673},[134],[35,11675,971],{"className":11676},[134,135],[35,11678,11680],{"className":11679},[180],[35,11681,11683,11706],{"className":11682},[184,185],[35,11684,11686,11703],{"className":11685},[189],[35,11687,11689],{"className":11688,"style":1081},[193],[35,11690,11691,11694],{"style":197},[35,11692],{"className":11693,"style":202},[201],[35,11695,11697],{"className":11696},[206,207,208,209],[35,11698,11700],{"className":11699},[134,213,209],[35,11701,1050],{"className":11702},[134,209],[35,11704,221],{"className":11705},[220],[35,11707,11709],{"className":11708},[189],[35,11710,11712],{"className":11711,"style":228},[193],[35,11713],{},[35,11715],{"className":11716,"style":288},[139],[35,11718,1707],{"className":11719},[292],[35,11721],{"className":11722,"style":288},[139],[35,11724,11726,11729,11732,11735],{"className":11725},[125],[35,11727],{"className":11728,"style":1427},[129],[35,11730,1385],{"className":11731},[134,135],[35,11733,2749],{"className":11734},[134],[35,11736,5282],{"className":11737,"style":8654},[134,135],". Substitute the batch ceiling above and cancel ",[35,11740,11742,11755],{"className":11741},[42],[35,11743,11745],{"className":11744},[46],[48,11746,11747],{"xmlns":50},[53,11748,11749,11753],{},[56,11750,11751],{},[59,11752,1385],{},[114,11754,1385],{"encoding":116},[35,11756,11758],{"className":11757,"ariaHidden":75},[121],[35,11759,11761,11764],{"className":11760},[125],[35,11762],{"className":11763,"style":2596},[129],[35,11765,1385],{"className":11766},[134,135],[35,11768,11770],{"className":11769},[38],[35,11771,11773,11840],{"className":11772},[42],[35,11774,11776],{"className":11775},[46],[48,11777,11778],{"xmlns":50,"display":51},[53,11779,11780,11838],{},[56,11781,11782,11788,11790,11836],{},[79,11783,11784,11786],{},[59,11785,61],{},[84,11787,109],{},[63,11789,65],{},[3706,11791,11792,11820],{},[56,11793,11794,11796,11798,11804,11806,11812,11814],{},[59,11795,938],{},[63,11797,1707],{},[79,11799,11800,11802],{},[59,11801,971],{},[84,11803,1050],{},[63,11805,1707],{},[2724,11807,11808,11810],{"accent":75},[59,11809,1885],{},[63,11811,2730],{},[84,11813,3503],{},[79,11815,11816,11818],{},[59,11817,792],{},[84,11819,1874],{},[56,11821,11822,11824,11826,11828,11830,11832,11834],{},[84,11823,8678],{},[63,11825,7533],{},[84,11827,868],{},[63,11829,8893],{},[59,11831,8814],{},[59,11833,2749],{"mathvariant":1396},[59,11835,1385],{},[59,11837,917],{"mathvariant":1396},[114,11839,9420],{"encoding":116},[35,11841,11843,11901],{"className":11842,"ariaHidden":75},[121],[35,11844,11846,11849,11892,11895,11898],{"className":11845},[125],[35,11847],{"className":11848,"style":409},[129],[35,11850,11852,11855],{"className":11851},[134],[35,11853,61],{"className":11854},[134,135],[35,11856,11858],{"className":11857},[180],[35,11859,11861,11884],{"className":11860},[184,185],[35,11862,11864,11881],{"className":11863},[189],[35,11865,11867],{"className":11866,"style":194},[193],[35,11868,11869,11872],{"style":197},[35,11870],{"className":11871,"style":202},[201],[35,11873,11875],{"className":11874},[206,207,208,209],[35,11876,11878],{"className":11877},[134,213,209],[35,11879,109],{"className":11880},[134,209],[35,11882,221],{"className":11883},[220],[35,11885,11887],{"className":11886},[189],[35,11888,11890],{"className":11889,"style":228},[193],[35,11891],{},[35,11893],{"className":11894,"style":140},[139],[35,11896,65],{"className":11897},[144],[35,11899],{"className":11900,"style":140},[139],[35,11902,11904,11907,12143],{"className":11903},[125],[35,11905],{"className":11906,"style":9488},[129],[35,11908,11910,11913,12140],{"className":11909},[134],[35,11911],{"className":11912},[3348,3818],[35,11914,11916],{"className":11915},[3706],[35,11917,11919,12132],{"className":11918},[184,185],[35,11920,11922,12129],{"className":11921},[189],[35,11923,11925,11972,11980],{"className":11924,"style":9507},[193],[35,11926,11927,11930],{"style":3834},[35,11928],{"className":11929,"style":2801},[201],[35,11931,11933,11939,11942,11945,11948,11954,11957,11960,11963,11966,11969],{"className":11932},[134],[35,11934,11936],{"className":11935},[134,213],[35,11937,8678],{"className":11938},[134],[35,11940],{"className":11941,"style":288},[139],[35,11943,7533],{"className":11944},[292],[35,11946],{"className":11947,"style":288},[139],[35,11949,11951],{"className":11950},[134,213],[35,11952,868],{"className":11953},[134],[35,11955],{"className":11956,"style":288},[139],[35,11958,8893],{"className":11959},[292],[35,11961],{"className":11962,"style":288},[139],[35,11964,8814],{"className":11965,"style":695},[134,135],[35,11967,2749],{"className":11968},[134],[35,11970,1385],{"className":11971},[134,135],[35,11973,11974,11977],{"style":3852},[35,11975],{"className":11976,"style":2801},[201],[35,11978],{"className":11979,"style":3860},[3859],[35,11981,11982,11985],{"style":9565},[35,11983],{"className":11984,"style":2801},[201],[35,11986,11988,11991,11994,11997,12000,12043,12046,12049,12052,12083,12086],{"className":11987},[134],[35,11989,938],{"className":11990,"style":953},[134,135],[35,11992],{"className":11993,"style":288},[139],[35,11995,1707],{"className":11996},[292],[35,11998],{"className":11999,"style":288},[139],[35,12001,12003,12006],{"className":12002},[134],[35,12004,971],{"className":12005},[134,135],[35,12007,12009],{"className":12008},[180],[35,12010,12012,12035],{"className":12011},[184,185],[35,12013,12015,12032],{"className":12014},[189],[35,12016,12018],{"className":12017,"style":1081},[193],[35,12019,12020,12023],{"style":197},[35,12021],{"className":12022,"style":202},[201],[35,12024,12026],{"className":12025},[206,207,208,209],[35,12027,12029],{"className":12028},[134,213,209],[35,12030,1050],{"className":12031},[134,209],[35,12033,221],{"className":12034},[220],[35,12036,12038],{"className":12037},[189],[35,12039,12041],{"className":12040,"style":228},[193],[35,12042],{},[35,12044],{"className":12045,"style":288},[139],[35,12047,1707],{"className":12048},[292],[35,12050],{"className":12051,"style":288},[139],[35,12053,12055],{"className":12054},[134,2785],[35,12056,12058],{"className":12057},[184],[35,12059,12061],{"className":12060},[189],[35,12062,12064,12072],{"className":12063,"style":2781},[193],[35,12065,12066,12069],{"style":2797},[35,12067],{"className":12068,"style":2801},[201],[35,12070,1885],{"className":12071},[134,135],[35,12073,12074,12077],{"style":2797},[35,12075],{"className":12076,"style":2801},[201],[35,12078,12080],{"className":12079,"style":2814},[2813],[35,12081,2730],{"className":12082},[134],[35,12084],{"className":12085,"style":162},[139],[35,12087,12089,12092],{"className":12088},[134],[35,12090,792],{"className":12091,"style":814},[134,135],[35,12093,12095],{"className":12094},[180],[35,12096,12098,12121],{"className":12097},[184,185],[35,12099,12101,12118],{"className":12100},[189],[35,12102,12104],{"className":12103,"style":1943},[193],[35,12105,12106,12109],{"style":1946},[35,12107],{"className":12108,"style":202},[201],[35,12110,12112],{"className":12111},[206,207,208,209],[35,12113,12115],{"className":12114},[134,213,209],[35,12116,1874],{"className":12117},[134,209],[35,12119,221],{"className":12120},[220],[35,12122,12124],{"className":12123},[189],[35,12125,12127],{"className":12126,"style":1968},[193],[35,12128],{},[35,12130,221],{"className":12131},[220],[35,12133,12135],{"className":12134},[189],[35,12136,12138],{"className":12137,"style":9722},[193],[35,12139],{},[35,12141],{"className":12142},[3453,3818],[35,12144,917],{"className":12145},[134],[10,12147,12148],{},"MoE models trip people up here. A 30B-A3B serves like a 3B model in prefill because only 3B parameters activate per token. Decode does not get the same discount. It is bandwidth-bound: what matters is bytes streamed from HBM per step, not FLOPs. All experts have to live in memory, and under load different requests pull different expert subsets simultaneously. By the time the batch is served, nearly the full pool has been read.",[3949,12150,3957,12152,12155],{"className":12151},[3952,3953,3954,3955,3956],[662,12153,3963],{"className":12154},[3961,3962],"\n  Decode is a bandwidth bill. It rises with output tokens and with the average KV cache each token has to read.\n",[27,12157,12159],{"id":12158},"step-3-combine-the-bills","Step 3: Combine the Bills",[10,12161,12162],{},"Now take the larger of the memory bill and the throughput bill.",[35,12164,12166],{"className":12165},[38],[35,12167,12169,12216],{"className":12168},[42],[35,12170,12172],{"className":12171},[46],[48,12173,12174],{"xmlns":50,"display":51},[53,12175,12176,12214],{},[56,12177,12178,12180,12182,12184,12186,12188,12194,12196,12198,12204,12206,12212],{},[59,12179,61],{},[63,12181,65],{},[59,12183,68],{},[63,12185,71],{},[63,12187,77],{"fence":74,"stretchy":75,"minsize":76,"maxsize":76},[79,12189,12190,12192],{},[59,12191,61],{},[84,12193,86],{},[63,12195,89],{"separator":75},[84,12197,92],{},[79,12199,12200,12202],{},[59,12201,61],{},[84,12203,99],{},[63,12205,102],{},[79,12207,12208,12210],{},[59,12209,61],{},[84,12211,109],{},[63,12213,112],{"fence":74,"stretchy":75,"minsize":76,"maxsize":76},[114,12215,117],{"encoding":116},[35,12217,12219,12237,12359],{"className":12218,"ariaHidden":75},[121],[35,12220,12222,12225,12228,12231,12234],{"className":12221},[125],[35,12223],{"className":12224,"style":130},[129],[35,12226,61],{"className":12227},[134,135],[35,12229],{"className":12230,"style":140},[139],[35,12232,65],{"className":12233},[144],[35,12235],{"className":12236,"style":140},[139],[35,12238,12240,12243,12246,12249,12255,12298,12301,12304,12307,12350,12353,12356],{"className":12239},[125],[35,12241],{"className":12242,"style":154},[129],[35,12244,68],{"className":12245},[158],[35,12247],{"className":12248,"style":162},[139],[35,12250,12252],{"className":12251},[134],[35,12253,77],{"className":12254},[169,170],[35,12256,12258,12261],{"className":12257},[134],[35,12259,61],{"className":12260},[134,135],[35,12262,12264],{"className":12263},[180],[35,12265,12267,12290],{"className":12266},[184,185],[35,12268,12270,12287],{"className":12269},[189],[35,12271,12273],{"className":12272,"style":194},[193],[35,12274,12275,12278],{"style":197},[35,12276],{"className":12277,"style":202},[201],[35,12279,12281],{"className":12280},[206,207,208,209],[35,12282,12284],{"className":12283},[134,213,209],[35,12285,86],{"className":12286},[134,209],[35,12288,221],{"className":12289},[220],[35,12291,12293],{"className":12292},[189],[35,12294,12296],{"className":12295,"style":228},[193],[35,12297],{},[35,12299,89],{"className":12300},[234],[35,12302,92],{"className":12303},[139],[35,12305],{"className":12306,"style":162},[139],[35,12308,12310,12313],{"className":12309},[134],[35,12311,61],{"className":12312},[134,135],[35,12314,12316],{"className":12315},[180],[35,12317,12319,12342],{"className":12318},[184,185],[35,12320,12322,12339],{"className":12321},[189],[35,12323,12325],{"className":12324,"style":194},[193],[35,12326,12327,12330],{"style":197},[35,12328],{"className":12329,"style":202},[201],[35,12331,12333],{"className":12332},[206,207,208,209],[35,12334,12336],{"className":12335},[134,213,209],[35,12337,99],{"className":12338},[134,209],[35,12340,221],{"className":12341},[220],[35,12343,12345],{"className":12344},[189],[35,12346,12348],{"className":12347,"style":282},[193],[35,12349],{},[35,12351],{"className":12352,"style":288},[139],[35,12354,102],{"className":12355},[292],[35,12357],{"className":12358,"style":288},[139],[35,12360,12362,12365,12408],{"className":12361},[125],[35,12363],{"className":12364,"style":154},[129],[35,12366,12368,12371],{"className":12367},[134],[35,12369,61],{"className":12370},[134,135],[35,12372,12374],{"className":12373},[180],[35,12375,12377,12400],{"className":12376},[184,185],[35,12378,12380,12397],{"className":12379},[189],[35,12381,12383],{"className":12382,"style":194},[193],[35,12384,12385,12388],{"style":197},[35,12386],{"className":12387,"style":202},[201],[35,12389,12391],{"className":12390},[206,207,208,209],[35,12392,12394],{"className":12393},[134,213,209],[35,12395,109],{"className":12396},[134,209],[35,12398,221],{"className":12399},[220],[35,12401,12403],{"className":12402},[189],[35,12404,12406],{"className":12405,"style":228},[193],[35,12407],{},[35,12409,12411],{"className":12410},[134],[35,12412,112],{"className":12413},[169,170],[10,12415,12416],{},"For our example, Qwen3-8B at 2 req\u002Fs, 4K input, 1K output, and 50 ms TPOT is a one-card H200 job before operational margin. The three bills collapse to:",[35,12418,12420],{"className":12419},[38],[35,12421,12423,12466],{"className":12422},[42],[35,12424,12426],{"className":12425},[46],[48,12427,12428],{"xmlns":50,"display":51},[53,12429,12430,12463],{},[56,12431,12432,12434,12436,12438,12440,12442,12444,12446,12448,12451,12453,12456,12458,12460],{},[59,12433,61],{},[63,12435,65],{},[59,12437,68],{},[63,12439,71],{},[63,12441,77],{"stretchy":74},[1342,12443,3740],{},[63,12445,89],{"separator":75},[84,12447,92],{},[1342,12449,12450],{},"0.30",[63,12452,102],{},[1342,12454,12455],{},"0.52",[63,12457,112],{"stretchy":74},[63,12459,2494],{},[1342,12461,12462],{},"0.8.",[114,12464,12465],{"encoding":116},"G = \\max(0.7,\\ 0.30 + 0.52) \\approx 0.8.",[35,12467,12469,12487,12523,12544],{"className":12468,"ariaHidden":75},[121],[35,12470,12472,12475,12478,12481,12484],{"className":12471},[125],[35,12473],{"className":12474,"style":130},[129],[35,12476,61],{"className":12477},[134,135],[35,12479],{"className":12480,"style":140},[139],[35,12482,65],{"className":12483},[144],[35,12485],{"className":12486,"style":140},[139],[35,12488,12490,12493,12496,12499,12502,12505,12508,12511,12514,12517,12520],{"className":12489},[125],[35,12491],{"className":12492,"style":1427},[129],[35,12494,68],{"className":12495},[158],[35,12497,77],{"className":12498},[3348],[35,12500,3740],{"className":12501},[134],[35,12503,89],{"className":12504},[234],[35,12506,92],{"className":12507},[139],[35,12509],{"className":12510,"style":162},[139],[35,12512,12450],{"className":12513},[134],[35,12515],{"className":12516,"style":288},[139],[35,12518,102],{"className":12519},[292],[35,12521],{"className":12522,"style":288},[139],[35,12524,12526,12529,12532,12535,12538,12541],{"className":12525},[125],[35,12527],{"className":12528,"style":1427},[129],[35,12530,12455],{"className":12531},[134],[35,12533,112],{"className":12534},[3453],[35,12536],{"className":12537,"style":140},[139],[35,12539,2494],{"className":12540},[144],[35,12542],{"className":12543,"style":140},[139],[35,12545,12547,12550],{"className":12546},[125],[35,12548],{"className":12549,"style":1627},[129],[35,12551,12462],{"className":12552},[134],[10,12554,12555,12556,12616],{},"The workload shape determines which compute term leads. Long prompts and short answers, like RAG, coding, multi-turn chat, and agents, push prefill up. Long outputs push ",[35,12557,12559,12576],{"className":12558},[42],[35,12560,12562],{"className":12561},[46],[48,12563,12564],{"xmlns":50},[53,12565,12566,12574],{},[56,12567,12568],{},[2724,12569,12570,12572],{"accent":75},[59,12571,1885],{},[63,12573,2730],{},[114,12575,10511],{"encoding":116},[35,12577,12579],{"className":12578,"ariaHidden":75},[121],[35,12580,12582,12585],{"className":12581},[125],[35,12583],{"className":12584,"style":2781},[129],[35,12586,12588],{"className":12587},[134,2785],[35,12589,12591],{"className":12590},[184],[35,12592,12594],{"className":12593},[189],[35,12595,12597,12605],{"className":12596,"style":2781},[193],[35,12598,12599,12602],{"style":2797},[35,12600],{"className":12601,"style":2801},[201],[35,12603,1885],{"className":12604},[134,135],[35,12606,12607,12610],{"style":2797},[35,12608],{"className":12609,"style":2801},[201],[35,12611,12613],{"className":12612,"style":2814},[2813],[35,12614,2730],{"className":12615},[134]," up and shift the cost into decode and the KV cache, as in reasoning models and long-form generation.",[27,12618,12620],{"id":12619},"run-the-napkin","Run the Napkin",[10,12622,12623],{},"If you want to adapt this to your own workload, start here. The script below uses the same formulas as the walkthrough. Change the constants at the top for your model, hardware, and traffic shape; it prints the memory, prefill, decode, and final bills separately, not just the rounded GPU count.",[2249,12625,12627],{"title":12626},"Show the script",[12628,12629,12633],"pre",{"className":12630,"code":12631,"language":12632,"meta":635,"style":635},"language-python shiki shiki-themes github-dark","import math\n\n# -----------------------------\n# Model\n# -----------------------------\nTOTAL_PARAM = 8e9      # total resident parameters\nACTIVE_PARAM = 8e9     # active parameters per token\nPRECISION = \"bf16\"     # bf16 = 2 bytes, fp8 = 1 byte\n\n# Standard-GQA KV cache details.\n# For hybrid\u002FMLA\u002Fetc., set KV_PER_TOKEN_BYTES directly.\nNUM_LAYERS = 36\nNUM_KV_HEADS = 8\nHEAD_DIM = 128\nKV_PER_TOKEN_BYTES = None  # Default: 2 * layers * kv_heads * head_dim * bytes\n\n# Prefill attention term.\nD_HIDDEN = 4096\n\n# -----------------------------\n# Traffic\n# -----------------------------\nREQ_RATE = 2         # requests \u002F second\nCTX_IN = 4_000       # input tokens \u002F request\nCTX_OUT = 1_000      # output tokens \u002F request\nTPOT_S = 0.050       # time per output token target, in seconds\n\n# -----------------------------\n# Hardware and utilization\n# -----------------------------\nACCELERATOR = \"h200\"\nMFU = 0.5            # model FLOPs util during prefill\nMBU = 0.6            # model bandwidth util during decode\nOVERHEAD = 1.15      # CUDA context, fragmentation, etc.\n\nPRECISION_BYTES = {\n    \"bf16\": 2,\n    \"fp8\": 1,\n}\n\nACCELERATORS = {\n    \"h200\": {\n        \"mem_bytes\": 141e9,\n        \"bw_bytes_s\": 4.8e12,\n        \"flops_s\": 990e12,  # dense bf16 FLOPs, not sparsity-assisted peak\n    },\n}\n\nb = PRECISION_BYTES[PRECISION]\nhw = ACCELERATORS[ACCELERATOR]\n\nweights_bytes = TOTAL_PARAM * b\nkv_per_token_bytes = (\n    KV_PER_TOKEN_BYTES\n    if KV_PER_TOKEN_BYTES is not None\n    else 2 * NUM_LAYERS * NUM_KV_HEADS * HEAD_DIM * b\n)\n\navg_seq_len = CTX_IN + CTX_OUT \u002F 2\nrequest_latency_s = CTX_OUT * TPOT_S\nconcurrency = REQ_RATE * request_latency_s\n\nkv_bytes = kv_per_token_bytes * avg_seq_len * concurrency\ng_fit = (weights_bytes + kv_bytes) * OVERHEAD \u002F hw[\"mem_bytes\"]\n\nc_in = 2 * ACTIVE_PARAM + 4 * NUM_LAYERS * D_HIDDEN * CTX_IN\ng_prefill = REQ_RATE * CTX_IN * c_in \u002F (MFU * hw[\"flops_s\"])\n\nusable_bw = MBU * hw[\"bw_bytes_s\"]\nweight_bw = weights_bytes \u002F TPOT_S\nremaining_bw = usable_bw - weight_bw\nif remaining_bw \u003C= 0:\n    raise ValueError(\n        \"One GPU cannot meet this TPOT; \"\n        \"split the model or relax TPOT.\"\n    )\n\ng_decode = REQ_RATE * CTX_OUT * avg_seq_len * kv_per_token_bytes \u002F remaining_bw\n\ng_throughput = g_prefill + g_decode\ng_final = max(g_fit, g_throughput)\n\n\ndef gb(x):\n    return x \u002F 1e9\n\n\ndef kb(x):\n    return x \u002F 1e3\n\n\nprint(\"=== Inputs ===\")\nprint(f\"accelerator: {ACCELERATOR}, precision: {PRECISION}\")\nprint(\n    f\"rate: {REQ_RATE:g} req\u002Fs, input: {CTX_IN:,} tok, \"\n    f\"output: {CTX_OUT:,} tok, TPOT: {TPOT_S*1000:.0f} ms\"\n)\nprint()\n\nprint(\"=== Memory ===\")\nprint(f\"weights: {gb(weights_bytes):.1f} GB\")\nprint(f\"KV\u002Ftoken: {kb(kv_per_token_bytes):.0f} KB\")\nprint(f\"avg sequence length: {avg_seq_len:,.0f} tokens\")\nprint(f\"concurrency: {concurrency:,.0f} requests\")\nprint(f\"resident KV cache: {gb(kv_bytes):.1f} GB\")\nprint(f\"G_fit: {g_fit:.2f}\")\nprint()\n\nprint(\"=== Prefill ===\")\nprint(f\"C_in: {c_in\u002F1e9:.1f}B FLOPs\u002Ftoken\")\nprint(f\"G_prefill: {g_prefill:.2f}\")\nprint()\n\nprint(\"=== Decode ===\")\nprint(f\"usable bandwidth: {gb(usable_bw):.0f} GB\u002Fs\")\nprint(f\"weight bandwidth at TPOT: {gb(weight_bw):.0f} GB\u002Fs\")\nprint(f\"bandwidth left for KV: {gb(remaining_bw):.0f} GB\u002Fs\")\nprint(f\"G_decode: {g_decode:.2f}\")\nprint()\n\nprint(\"=== Final ===\")\nprint(f\"throughput bill: {g_prefill:.2f} + {g_decode:.2f} = {g_throughput:.2f}\")\nprint(f\"final bill: max({g_fit:.2f}, {g_throughput:.2f}) = {g_final:.2f}\")\nprint(f\"rounded GPUs: {math.ceil(g_final)}\")\n","python",[12634,12635,12636,12648,12655,12662,12668,12673,12689,12702,12717,12722,12728,12734,12745,12756,12767,12781,12786,12792,12803,12808,12813,12819,12824,12838,12852,12866,12880,12885,12890,12896,12901,12912,12925,12938,12952,12957,12968,12982,12994,13000,13005,13015,13024,13037,13050,13067,13073,13078,13083,13102,13119,13124,13141,13152,13158,13176,13203,13209,13214,13237,13252,13268,13273,13295,13326,13331,13365,13401,13406,13426,13441,13458,13476,13488,13494,13500,13506,13511,13538,13543,13559,13573,13578,13583,13596,13610,13615,13620,13630,13642,13647,13652,13665,13692,13699,13731,13765,13770,13778,13783,13795,13823,13849,13876,13901,13926,13952,13959,13964,13976,14007,14032,14039,14044,14056,14082,14107,14132,14157,14164,14169,14181,14228,14274],"code",{"__ignoreMap":635},[35,12637,12640,12644],{"class":12638,"line":12639},"line",1,[35,12641,12643],{"class":12642},"snl16","import",[35,12645,12647],{"class":12646},"s95oV"," math\n",[35,12649,12651],{"class":12638,"line":12650},2,[35,12652,12654],{"emptyLinePlaceholder":12653},true,"\n",[35,12656,12658],{"class":12638,"line":12657},3,[35,12659,12661],{"class":12660},"sAwPA","# -----------------------------\n",[35,12663,12665],{"class":12638,"line":12664},4,[35,12666,12667],{"class":12660},"# Model\n",[35,12669,12671],{"class":12638,"line":12670},5,[35,12672,12661],{"class":12660},[35,12674,12676,12680,12683,12686],{"class":12638,"line":12675},6,[35,12677,12679],{"class":12678},"sDLfK","TOTAL_PARAM",[35,12681,12682],{"class":12642}," =",[35,12684,12685],{"class":12678}," 8e9",[35,12687,12688],{"class":12660},"      # total resident parameters\n",[35,12690,12692,12695,12697,12699],{"class":12638,"line":12691},7,[35,12693,12694],{"class":12678},"ACTIVE_PARAM",[35,12696,12682],{"class":12642},[35,12698,12685],{"class":12678},[35,12700,12701],{"class":12660},"     # active parameters per token\n",[35,12703,12705,12708,12710,12714],{"class":12638,"line":12704},8,[35,12706,12707],{"class":12678},"PRECISION",[35,12709,12682],{"class":12642},[35,12711,12713],{"class":12712},"sU2Wk"," \"bf16\"",[35,12715,12716],{"class":12660},"     # bf16 = 2 bytes, fp8 = 1 byte\n",[35,12718,12720],{"class":12638,"line":12719},9,[35,12721,12654],{"emptyLinePlaceholder":12653},[35,12723,12725],{"class":12638,"line":12724},10,[35,12726,12727],{"class":12660},"# Standard-GQA KV cache details.\n",[35,12729,12731],{"class":12638,"line":12730},11,[35,12732,12733],{"class":12660},"# For hybrid\u002FMLA\u002Fetc., set KV_PER_TOKEN_BYTES directly.\n",[35,12735,12737,12740,12742],{"class":12638,"line":12736},12,[35,12738,12739],{"class":12678},"NUM_LAYERS",[35,12741,12682],{"class":12642},[35,12743,12744],{"class":12678}," 36\n",[35,12746,12748,12751,12753],{"class":12638,"line":12747},13,[35,12749,12750],{"class":12678},"NUM_KV_HEADS",[35,12752,12682],{"class":12642},[35,12754,12755],{"class":12678}," 8\n",[35,12757,12759,12762,12764],{"class":12638,"line":12758},14,[35,12760,12761],{"class":12678},"HEAD_DIM",[35,12763,12682],{"class":12642},[35,12765,12766],{"class":12678}," 128\n",[35,12768,12770,12773,12775,12778],{"class":12638,"line":12769},15,[35,12771,12772],{"class":12678},"KV_PER_TOKEN_BYTES",[35,12774,12682],{"class":12642},[35,12776,12777],{"class":12678}," None",[35,12779,12780],{"class":12660},"  # Default: 2 * layers * kv_heads * head_dim * bytes\n",[35,12782,12784],{"class":12638,"line":12783},16,[35,12785,12654],{"emptyLinePlaceholder":12653},[35,12787,12789],{"class":12638,"line":12788},17,[35,12790,12791],{"class":12660},"# Prefill attention term.\n",[35,12793,12795,12798,12800],{"class":12638,"line":12794},18,[35,12796,12797],{"class":12678},"D_HIDDEN",[35,12799,12682],{"class":12642},[35,12801,12802],{"class":12678}," 4096\n",[35,12804,12806],{"class":12638,"line":12805},19,[35,12807,12654],{"emptyLinePlaceholder":12653},[35,12809,12811],{"class":12638,"line":12810},20,[35,12812,12661],{"class":12660},[35,12814,12816],{"class":12638,"line":12815},21,[35,12817,12818],{"class":12660},"# Traffic\n",[35,12820,12822],{"class":12638,"line":12821},22,[35,12823,12661],{"class":12660},[35,12825,12827,12830,12832,12835],{"class":12638,"line":12826},23,[35,12828,12829],{"class":12678},"REQ_RATE",[35,12831,12682],{"class":12642},[35,12833,12834],{"class":12678}," 2",[35,12836,12837],{"class":12660},"         # requests \u002F second\n",[35,12839,12841,12844,12846,12849],{"class":12638,"line":12840},24,[35,12842,12843],{"class":12678},"CTX_IN",[35,12845,12682],{"class":12642},[35,12847,12848],{"class":12678}," 4_000",[35,12850,12851],{"class":12660},"       # input tokens \u002F request\n",[35,12853,12855,12858,12860,12863],{"class":12638,"line":12854},25,[35,12856,12857],{"class":12678},"CTX_OUT",[35,12859,12682],{"class":12642},[35,12861,12862],{"class":12678}," 1_000",[35,12864,12865],{"class":12660},"      # output tokens \u002F request\n",[35,12867,12869,12872,12874,12877],{"class":12638,"line":12868},26,[35,12870,12871],{"class":12678},"TPOT_S",[35,12873,12682],{"class":12642},[35,12875,12876],{"class":12678}," 0.050",[35,12878,12879],{"class":12660},"       # time per output token target, in seconds\n",[35,12881,12883],{"class":12638,"line":12882},27,[35,12884,12654],{"emptyLinePlaceholder":12653},[35,12886,12888],{"class":12638,"line":12887},28,[35,12889,12661],{"class":12660},[35,12891,12893],{"class":12638,"line":12892},29,[35,12894,12895],{"class":12660},"# Hardware and utilization\n",[35,12897,12899],{"class":12638,"line":12898},30,[35,12900,12661],{"class":12660},[35,12902,12904,12907,12909],{"class":12638,"line":12903},31,[35,12905,12906],{"class":12678},"ACCELERATOR",[35,12908,12682],{"class":12642},[35,12910,12911],{"class":12712}," \"h200\"\n",[35,12913,12915,12917,12919,12922],{"class":12638,"line":12914},32,[35,12916,4779],{"class":12678},[35,12918,12682],{"class":12642},[35,12920,12921],{"class":12678}," 0.5",[35,12923,12924],{"class":12660},"            # model FLOPs util during prefill\n",[35,12926,12928,12930,12932,12935],{"class":12638,"line":12927},33,[35,12929,8678],{"class":12678},[35,12931,12682],{"class":12642},[35,12933,12934],{"class":12678}," 0.6",[35,12936,12937],{"class":12660},"            # model bandwidth util during decode\n",[35,12939,12941,12944,12946,12949],{"class":12638,"line":12940},34,[35,12942,12943],{"class":12678},"OVERHEAD",[35,12945,12682],{"class":12642},[35,12947,12948],{"class":12678}," 1.15",[35,12950,12951],{"class":12660},"      # CUDA context, fragmentation, etc.\n",[35,12953,12955],{"class":12638,"line":12954},35,[35,12956,12654],{"emptyLinePlaceholder":12653},[35,12958,12960,12963,12965],{"class":12638,"line":12959},36,[35,12961,12962],{"class":12678},"PRECISION_BYTES",[35,12964,12682],{"class":12642},[35,12966,12967],{"class":12646}," {\n",[35,12969,12971,12974,12977,12979],{"class":12638,"line":12970},37,[35,12972,12973],{"class":12712},"    \"bf16\"",[35,12975,12976],{"class":12646},": ",[35,12978,1344],{"class":12678},[35,12980,12981],{"class":12646},",\n",[35,12983,12985,12988,12990,12992],{"class":12638,"line":12984},38,[35,12986,12987],{"class":12712},"    \"fp8\"",[35,12989,12976],{"class":12646},[35,12991,1314],{"class":12678},[35,12993,12981],{"class":12646},[35,12995,12997],{"class":12638,"line":12996},39,[35,12998,12999],{"class":12646},"}\n",[35,13001,13003],{"class":12638,"line":13002},40,[35,13004,12654],{"emptyLinePlaceholder":12653},[35,13006,13008,13011,13013],{"class":12638,"line":13007},41,[35,13009,13010],{"class":12678},"ACCELERATORS",[35,13012,12682],{"class":12642},[35,13014,12967],{"class":12646},[35,13016,13018,13021],{"class":12638,"line":13017},42,[35,13019,13020],{"class":12712},"    \"h200\"",[35,13022,13023],{"class":12646},": {\n",[35,13025,13027,13030,13032,13035],{"class":12638,"line":13026},43,[35,13028,13029],{"class":12712},"        \"mem_bytes\"",[35,13031,12976],{"class":12646},[35,13033,13034],{"class":12678},"141e9",[35,13036,12981],{"class":12646},[35,13038,13040,13043,13045,13048],{"class":12638,"line":13039},44,[35,13041,13042],{"class":12712},"        \"bw_bytes_s\"",[35,13044,12976],{"class":12646},[35,13046,13047],{"class":12678},"4.8e12",[35,13049,12981],{"class":12646},[35,13051,13053,13056,13058,13061,13064],{"class":12638,"line":13052},45,[35,13054,13055],{"class":12712},"        \"flops_s\"",[35,13057,12976],{"class":12646},[35,13059,13060],{"class":12678},"990e12",[35,13062,13063],{"class":12646},",  ",[35,13065,13066],{"class":12660},"# dense bf16 FLOPs, not sparsity-assisted peak\n",[35,13068,13070],{"class":12638,"line":13069},46,[35,13071,13072],{"class":12646},"    },\n",[35,13074,13076],{"class":12638,"line":13075},47,[35,13077,12999],{"class":12646},[35,13079,13081],{"class":12638,"line":13080},48,[35,13082,12654],{"emptyLinePlaceholder":12653},[35,13084,13086,13089,13091,13094,13097,13099],{"class":12638,"line":13085},49,[35,13087,13088],{"class":12646},"b ",[35,13090,65],{"class":12642},[35,13092,13093],{"class":12678}," PRECISION_BYTES",[35,13095,13096],{"class":12646},"[",[35,13098,12707],{"class":12678},[35,13100,13101],{"class":12646},"]\n",[35,13103,13105,13108,13110,13113,13115,13117],{"class":12638,"line":13104},50,[35,13106,13107],{"class":12646},"hw ",[35,13109,65],{"class":12642},[35,13111,13112],{"class":12678}," ACCELERATORS",[35,13114,13096],{"class":12646},[35,13116,12906],{"class":12678},[35,13118,13101],{"class":12646},[35,13120,13122],{"class":12638,"line":13121},51,[35,13123,12654],{"emptyLinePlaceholder":12653},[35,13125,13127,13130,13132,13135,13138],{"class":12638,"line":13126},52,[35,13128,13129],{"class":12646},"weights_bytes ",[35,13131,65],{"class":12642},[35,13133,13134],{"class":12678}," TOTAL_PARAM",[35,13136,13137],{"class":12642}," *",[35,13139,13140],{"class":12646}," b\n",[35,13142,13144,13147,13149],{"class":12638,"line":13143},53,[35,13145,13146],{"class":12646},"kv_per_token_bytes ",[35,13148,65],{"class":12642},[35,13150,13151],{"class":12646}," (\n",[35,13153,13155],{"class":12638,"line":13154},54,[35,13156,13157],{"class":12678},"    KV_PER_TOKEN_BYTES\n",[35,13159,13161,13164,13167,13170,13173],{"class":12638,"line":13160},55,[35,13162,13163],{"class":12642},"    if",[35,13165,13166],{"class":12678}," KV_PER_TOKEN_BYTES",[35,13168,13169],{"class":12642}," is",[35,13171,13172],{"class":12642}," not",[35,13174,13175],{"class":12678}," None\n",[35,13177,13179,13182,13184,13186,13189,13191,13194,13196,13199,13201],{"class":12638,"line":13178},56,[35,13180,13181],{"class":12642},"    else",[35,13183,12834],{"class":12678},[35,13185,13137],{"class":12642},[35,13187,13188],{"class":12678}," NUM_LAYERS",[35,13190,13137],{"class":12642},[35,13192,13193],{"class":12678}," NUM_KV_HEADS",[35,13195,13137],{"class":12642},[35,13197,13198],{"class":12678}," HEAD_DIM",[35,13200,13137],{"class":12642},[35,13202,13140],{"class":12646},[35,13204,13206],{"class":12638,"line":13205},57,[35,13207,13208],{"class":12646},")\n",[35,13210,13212],{"class":12638,"line":13211},58,[35,13213,12654],{"emptyLinePlaceholder":12653},[35,13215,13217,13220,13222,13225,13228,13231,13234],{"class":12638,"line":13216},59,[35,13218,13219],{"class":12646},"avg_seq_len ",[35,13221,65],{"class":12642},[35,13223,13224],{"class":12678}," CTX_IN",[35,13226,13227],{"class":12642}," +",[35,13229,13230],{"class":12678}," CTX_OUT",[35,13232,13233],{"class":12642}," \u002F",[35,13235,13236],{"class":12678}," 2\n",[35,13238,13240,13243,13245,13247,13249],{"class":12638,"line":13239},60,[35,13241,13242],{"class":12646},"request_latency_s ",[35,13244,65],{"class":12642},[35,13246,13230],{"class":12678},[35,13248,13137],{"class":12642},[35,13250,13251],{"class":12678}," TPOT_S\n",[35,13253,13255,13258,13260,13263,13265],{"class":12638,"line":13254},61,[35,13256,13257],{"class":12646},"concurrency ",[35,13259,65],{"class":12642},[35,13261,13262],{"class":12678}," REQ_RATE",[35,13264,13137],{"class":12642},[35,13266,13267],{"class":12646}," request_latency_s\n",[35,13269,13271],{"class":12638,"line":13270},62,[35,13272,12654],{"emptyLinePlaceholder":12653},[35,13274,13276,13279,13281,13284,13287,13290,13292],{"class":12638,"line":13275},63,[35,13277,13278],{"class":12646},"kv_bytes ",[35,13280,65],{"class":12642},[35,13282,13283],{"class":12646}," kv_per_token_bytes ",[35,13285,13286],{"class":12642},"*",[35,13288,13289],{"class":12646}," avg_seq_len ",[35,13291,13286],{"class":12642},[35,13293,13294],{"class":12646}," concurrency\n",[35,13296,13298,13301,13303,13306,13308,13311,13313,13316,13318,13321,13324],{"class":12638,"line":13297},64,[35,13299,13300],{"class":12646},"g_fit ",[35,13302,65],{"class":12642},[35,13304,13305],{"class":12646}," (weights_bytes ",[35,13307,102],{"class":12642},[35,13309,13310],{"class":12646}," kv_bytes) ",[35,13312,13286],{"class":12642},[35,13314,13315],{"class":12678}," OVERHEAD",[35,13317,13233],{"class":12642},[35,13319,13320],{"class":12646}," hw[",[35,13322,13323],{"class":12712},"\"mem_bytes\"",[35,13325,13101],{"class":12646},[35,13327,13329],{"class":12638,"line":13328},65,[35,13330,12654],{"emptyLinePlaceholder":12653},[35,13332,13334,13337,13339,13341,13343,13346,13348,13351,13353,13355,13357,13360,13362],{"class":12638,"line":13333},66,[35,13335,13336],{"class":12646},"c_in ",[35,13338,65],{"class":12642},[35,13340,12834],{"class":12678},[35,13342,13137],{"class":12642},[35,13344,13345],{"class":12678}," ACTIVE_PARAM",[35,13347,13227],{"class":12642},[35,13349,13350],{"class":12678}," 4",[35,13352,13137],{"class":12642},[35,13354,13188],{"class":12678},[35,13356,13137],{"class":12642},[35,13358,13359],{"class":12678}," D_HIDDEN",[35,13361,13137],{"class":12642},[35,13363,13364],{"class":12678}," CTX_IN\n",[35,13366,13368,13371,13373,13375,13377,13379,13381,13384,13386,13389,13391,13393,13395,13398],{"class":12638,"line":13367},67,[35,13369,13370],{"class":12646},"g_prefill ",[35,13372,65],{"class":12642},[35,13374,13262],{"class":12678},[35,13376,13137],{"class":12642},[35,13378,13224],{"class":12678},[35,13380,13137],{"class":12642},[35,13382,13383],{"class":12646}," c_in ",[35,13385,2749],{"class":12642},[35,13387,13388],{"class":12646}," (",[35,13390,4779],{"class":12678},[35,13392,13137],{"class":12642},[35,13394,13320],{"class":12646},[35,13396,13397],{"class":12712},"\"flops_s\"",[35,13399,13400],{"class":12646},"])\n",[35,13402,13404],{"class":12638,"line":13403},68,[35,13405,12654],{"emptyLinePlaceholder":12653},[35,13407,13409,13412,13414,13417,13419,13421,13424],{"class":12638,"line":13408},69,[35,13410,13411],{"class":12646},"usable_bw ",[35,13413,65],{"class":12642},[35,13415,13416],{"class":12678}," MBU",[35,13418,13137],{"class":12642},[35,13420,13320],{"class":12646},[35,13422,13423],{"class":12712},"\"bw_bytes_s\"",[35,13425,13101],{"class":12646},[35,13427,13429,13432,13434,13437,13439],{"class":12638,"line":13428},70,[35,13430,13431],{"class":12646},"weight_bw ",[35,13433,65],{"class":12642},[35,13435,13436],{"class":12646}," weights_bytes ",[35,13438,2749],{"class":12642},[35,13440,13251],{"class":12678},[35,13442,13444,13447,13449,13452,13455],{"class":12638,"line":13443},71,[35,13445,13446],{"class":12646},"remaining_bw ",[35,13448,65],{"class":12642},[35,13450,13451],{"class":12646}," usable_bw ",[35,13453,13454],{"class":12642},"-",[35,13456,13457],{"class":12646}," weight_bw\n",[35,13459,13461,13464,13467,13470,13473],{"class":12638,"line":13460},72,[35,13462,13463],{"class":12642},"if",[35,13465,13466],{"class":12646}," remaining_bw ",[35,13468,13469],{"class":12642},"\u003C=",[35,13471,13472],{"class":12678}," 0",[35,13474,13475],{"class":12646},":\n",[35,13477,13479,13482,13485],{"class":12638,"line":13478},73,[35,13480,13481],{"class":12642},"    raise",[35,13483,13484],{"class":12678}," ValueError",[35,13486,13487],{"class":12646},"(\n",[35,13489,13491],{"class":12638,"line":13490},74,[35,13492,13493],{"class":12712},"        \"One GPU cannot meet this TPOT; \"\n",[35,13495,13497],{"class":12638,"line":13496},75,[35,13498,13499],{"class":12712},"        \"split the model or relax TPOT.\"\n",[35,13501,13503],{"class":12638,"line":13502},76,[35,13504,13505],{"class":12646},"    )\n",[35,13507,13509],{"class":12638,"line":13508},77,[35,13510,12654],{"emptyLinePlaceholder":12653},[35,13512,13514,13517,13519,13521,13523,13525,13527,13529,13531,13533,13535],{"class":12638,"line":13513},78,[35,13515,13516],{"class":12646},"g_decode ",[35,13518,65],{"class":12642},[35,13520,13262],{"class":12678},[35,13522,13137],{"class":12642},[35,13524,13230],{"class":12678},[35,13526,13137],{"class":12642},[35,13528,13289],{"class":12646},[35,13530,13286],{"class":12642},[35,13532,13283],{"class":12646},[35,13534,2749],{"class":12642},[35,13536,13537],{"class":12646}," remaining_bw\n",[35,13539,13541],{"class":12638,"line":13540},79,[35,13542,12654],{"emptyLinePlaceholder":12653},[35,13544,13546,13549,13551,13554,13556],{"class":12638,"line":13545},80,[35,13547,13548],{"class":12646},"g_throughput ",[35,13550,65],{"class":12642},[35,13552,13553],{"class":12646}," g_prefill ",[35,13555,102],{"class":12642},[35,13557,13558],{"class":12646}," g_decode\n",[35,13560,13562,13565,13567,13570],{"class":12638,"line":13561},81,[35,13563,13564],{"class":12646},"g_final ",[35,13566,65],{"class":12642},[35,13568,13569],{"class":12678}," max",[35,13571,13572],{"class":12646},"(g_fit, g_throughput)\n",[35,13574,13576],{"class":12638,"line":13575},82,[35,13577,12654],{"emptyLinePlaceholder":12653},[35,13579,13581],{"class":12638,"line":13580},83,[35,13582,12654],{"emptyLinePlaceholder":12653},[35,13584,13586,13589,13593],{"class":12638,"line":13585},84,[35,13587,13588],{"class":12642},"def",[35,13590,13592],{"class":13591},"svObZ"," gb",[35,13594,13595],{"class":12646},"(x):\n",[35,13597,13599,13602,13605,13607],{"class":12638,"line":13598},85,[35,13600,13601],{"class":12642},"    return",[35,13603,13604],{"class":12646}," x ",[35,13606,2749],{"class":12642},[35,13608,13609],{"class":12678}," 1e9\n",[35,13611,13613],{"class":12638,"line":13612},86,[35,13614,12654],{"emptyLinePlaceholder":12653},[35,13616,13618],{"class":12638,"line":13617},87,[35,13619,12654],{"emptyLinePlaceholder":12653},[35,13621,13623,13625,13628],{"class":12638,"line":13622},88,[35,13624,13588],{"class":12642},[35,13626,13627],{"class":13591}," kb",[35,13629,13595],{"class":12646},[35,13631,13633,13635,13637,13639],{"class":12638,"line":13632},89,[35,13634,13601],{"class":12642},[35,13636,13604],{"class":12646},[35,13638,2749],{"class":12642},[35,13640,13641],{"class":12678}," 1e3\n",[35,13643,13645],{"class":12638,"line":13644},90,[35,13646,12654],{"emptyLinePlaceholder":12653},[35,13648,13650],{"class":12638,"line":13649},91,[35,13651,12654],{"emptyLinePlaceholder":12653},[35,13653,13655,13658,13660,13663],{"class":12638,"line":13654},92,[35,13656,13657],{"class":12678},"print",[35,13659,77],{"class":12646},[35,13661,13662],{"class":12712},"\"=== Inputs ===\"",[35,13664,13208],{"class":12646},[35,13666,13668,13670,13672,13675,13678,13681,13684,13687,13690],{"class":12638,"line":13667},93,[35,13669,13657],{"class":12678},[35,13671,77],{"class":12646},[35,13673,13674],{"class":12642},"f",[35,13676,13677],{"class":12712},"\"accelerator: ",[35,13679,13680],{"class":12678},"{ACCELERATOR}",[35,13682,13683],{"class":12712},", precision: ",[35,13685,13686],{"class":12678},"{PRECISION}",[35,13688,13689],{"class":12712},"\"",[35,13691,13208],{"class":12646},[35,13693,13695,13697],{"class":12638,"line":13694},94,[35,13696,13657],{"class":12678},[35,13698,13487],{"class":12646},[35,13700,13702,13705,13708,13711,13714,13717,13720,13723,13726,13728],{"class":12638,"line":13701},95,[35,13703,13704],{"class":12642},"    f",[35,13706,13707],{"class":12712},"\"rate: ",[35,13709,13710],{"class":12678},"{REQ_RATE",[35,13712,13713],{"class":12642},":g",[35,13715,13716],{"class":12678},"}",[35,13718,13719],{"class":12712}," req\u002Fs, input: ",[35,13721,13722],{"class":12678},"{CTX_IN",[35,13724,13725],{"class":12642},":,",[35,13727,13716],{"class":12678},[35,13729,13730],{"class":12712}," tok, \"\n",[35,13732,13734,13736,13739,13742,13744,13746,13749,13752,13754,13757,13760,13762],{"class":12638,"line":13733},96,[35,13735,13704],{"class":12642},[35,13737,13738],{"class":12712},"\"output: ",[35,13740,13741],{"class":12678},"{CTX_OUT",[35,13743,13725],{"class":12642},[35,13745,13716],{"class":12678},[35,13747,13748],{"class":12712}," tok, TPOT: ",[35,13750,13751],{"class":12678},"{TPOT_S",[35,13753,13286],{"class":12642},[35,13755,13756],{"class":12678},"1000",[35,13758,13759],{"class":12642},":.0f",[35,13761,13716],{"class":12678},[35,13763,13764],{"class":12712}," ms\"\n",[35,13766,13768],{"class":12638,"line":13767},97,[35,13769,13208],{"class":12646},[35,13771,13773,13775],{"class":12638,"line":13772},98,[35,13774,13657],{"class":12678},[35,13776,13777],{"class":12646},"()\n",[35,13779,13781],{"class":12638,"line":13780},99,[35,13782,12654],{"emptyLinePlaceholder":12653},[35,13784,13786,13788,13790,13793],{"class":12638,"line":13785},100,[35,13787,13657],{"class":12678},[35,13789,77],{"class":12646},[35,13791,13792],{"class":12712},"\"=== Memory ===\"",[35,13794,13208],{"class":12646},[35,13796,13798,13800,13802,13804,13807,13810,13813,13816,13818,13821],{"class":12638,"line":13797},101,[35,13799,13657],{"class":12678},[35,13801,77],{"class":12646},[35,13803,13674],{"class":12642},[35,13805,13806],{"class":12712},"\"weights: ",[35,13808,13809],{"class":12678},"{",[35,13811,13812],{"class":12646},"gb(weights_bytes)",[35,13814,13815],{"class":12642},":.1f",[35,13817,13716],{"class":12678},[35,13819,13820],{"class":12712}," GB\"",[35,13822,13208],{"class":12646},[35,13824,13826,13828,13830,13832,13835,13837,13840,13842,13844,13847],{"class":12638,"line":13825},102,[35,13827,13657],{"class":12678},[35,13829,77],{"class":12646},[35,13831,13674],{"class":12642},[35,13833,13834],{"class":12712},"\"KV\u002Ftoken: ",[35,13836,13809],{"class":12678},[35,13838,13839],{"class":12646},"kb(kv_per_token_bytes)",[35,13841,13759],{"class":12642},[35,13843,13716],{"class":12678},[35,13845,13846],{"class":12712}," KB\"",[35,13848,13208],{"class":12646},[35,13850,13852,13854,13856,13858,13861,13863,13866,13869,13871,13874],{"class":12638,"line":13851},103,[35,13853,13657],{"class":12678},[35,13855,77],{"class":12646},[35,13857,13674],{"class":12642},[35,13859,13860],{"class":12712},"\"avg sequence length: ",[35,13862,13809],{"class":12678},[35,13864,13865],{"class":12646},"avg_seq_len",[35,13867,13868],{"class":12642},":,.0f",[35,13870,13716],{"class":12678},[35,13872,13873],{"class":12712}," tokens\"",[35,13875,13208],{"class":12646},[35,13877,13879,13881,13883,13885,13888,13890,13892,13894,13896,13899],{"class":12638,"line":13878},104,[35,13880,13657],{"class":12678},[35,13882,77],{"class":12646},[35,13884,13674],{"class":12642},[35,13886,13887],{"class":12712},"\"concurrency: ",[35,13889,13809],{"class":12678},[35,13891,2313],{"class":12646},[35,13893,13868],{"class":12642},[35,13895,13716],{"class":12678},[35,13897,13898],{"class":12712}," requests\"",[35,13900,13208],{"class":12646},[35,13902,13904,13906,13908,13910,13913,13915,13918,13920,13922,13924],{"class":12638,"line":13903},105,[35,13905,13657],{"class":12678},[35,13907,77],{"class":12646},[35,13909,13674],{"class":12642},[35,13911,13912],{"class":12712},"\"resident KV cache: ",[35,13914,13809],{"class":12678},[35,13916,13917],{"class":12646},"gb(kv_bytes)",[35,13919,13815],{"class":12642},[35,13921,13716],{"class":12678},[35,13923,13820],{"class":12712},[35,13925,13208],{"class":12646},[35,13927,13929,13931,13933,13935,13938,13940,13943,13946,13948,13950],{"class":12638,"line":13928},106,[35,13930,13657],{"class":12678},[35,13932,77],{"class":12646},[35,13934,13674],{"class":12642},[35,13936,13937],{"class":12712},"\"G_fit: ",[35,13939,13809],{"class":12678},[35,13941,13942],{"class":12646},"g_fit",[35,13944,13945],{"class":12642},":.2f",[35,13947,13716],{"class":12678},[35,13949,13689],{"class":12712},[35,13951,13208],{"class":12646},[35,13953,13955,13957],{"class":12638,"line":13954},107,[35,13956,13657],{"class":12678},[35,13958,13777],{"class":12646},[35,13960,13962],{"class":12638,"line":13961},108,[35,13963,12654],{"emptyLinePlaceholder":12653},[35,13965,13967,13969,13971,13974],{"class":12638,"line":13966},109,[35,13968,13657],{"class":12678},[35,13970,77],{"class":12646},[35,13972,13973],{"class":12712},"\"=== Prefill ===\"",[35,13975,13208],{"class":12646},[35,13977,13979,13981,13983,13985,13988,13990,13993,13995,13998,14000,14002,14005],{"class":12638,"line":13978},110,[35,13980,13657],{"class":12678},[35,13982,77],{"class":12646},[35,13984,13674],{"class":12642},[35,13986,13987],{"class":12712},"\"C_in: ",[35,13989,13809],{"class":12678},[35,13991,13992],{"class":12646},"c_in",[35,13994,2749],{"class":12642},[35,13996,13997],{"class":12678},"1e9",[35,13999,13815],{"class":12642},[35,14001,13716],{"class":12678},[35,14003,14004],{"class":12712},"B FLOPs\u002Ftoken\"",[35,14006,13208],{"class":12646},[35,14008,14010,14012,14014,14016,14019,14021,14024,14026,14028,14030],{"class":12638,"line":14009},111,[35,14011,13657],{"class":12678},[35,14013,77],{"class":12646},[35,14015,13674],{"class":12642},[35,14017,14018],{"class":12712},"\"G_prefill: ",[35,14020,13809],{"class":12678},[35,14022,14023],{"class":12646},"g_prefill",[35,14025,13945],{"class":12642},[35,14027,13716],{"class":12678},[35,14029,13689],{"class":12712},[35,14031,13208],{"class":12646},[35,14033,14035,14037],{"class":12638,"line":14034},112,[35,14036,13657],{"class":12678},[35,14038,13777],{"class":12646},[35,14040,14042],{"class":12638,"line":14041},113,[35,14043,12654],{"emptyLinePlaceholder":12653},[35,14045,14047,14049,14051,14054],{"class":12638,"line":14046},114,[35,14048,13657],{"class":12678},[35,14050,77],{"class":12646},[35,14052,14053],{"class":12712},"\"=== Decode ===\"",[35,14055,13208],{"class":12646},[35,14057,14059,14061,14063,14065,14068,14070,14073,14075,14077,14080],{"class":12638,"line":14058},115,[35,14060,13657],{"class":12678},[35,14062,77],{"class":12646},[35,14064,13674],{"class":12642},[35,14066,14067],{"class":12712},"\"usable bandwidth: ",[35,14069,13809],{"class":12678},[35,14071,14072],{"class":12646},"gb(usable_bw)",[35,14074,13759],{"class":12642},[35,14076,13716],{"class":12678},[35,14078,14079],{"class":12712}," GB\u002Fs\"",[35,14081,13208],{"class":12646},[35,14083,14085,14087,14089,14091,14094,14096,14099,14101,14103,14105],{"class":12638,"line":14084},116,[35,14086,13657],{"class":12678},[35,14088,77],{"class":12646},[35,14090,13674],{"class":12642},[35,14092,14093],{"class":12712},"\"weight bandwidth at TPOT: ",[35,14095,13809],{"class":12678},[35,14097,14098],{"class":12646},"gb(weight_bw)",[35,14100,13759],{"class":12642},[35,14102,13716],{"class":12678},[35,14104,14079],{"class":12712},[35,14106,13208],{"class":12646},[35,14108,14110,14112,14114,14116,14119,14121,14124,14126,14128,14130],{"class":12638,"line":14109},117,[35,14111,13657],{"class":12678},[35,14113,77],{"class":12646},[35,14115,13674],{"class":12642},[35,14117,14118],{"class":12712},"\"bandwidth left for KV: ",[35,14120,13809],{"class":12678},[35,14122,14123],{"class":12646},"gb(remaining_bw)",[35,14125,13759],{"class":12642},[35,14127,13716],{"class":12678},[35,14129,14079],{"class":12712},[35,14131,13208],{"class":12646},[35,14133,14135,14137,14139,14141,14144,14146,14149,14151,14153,14155],{"class":12638,"line":14134},118,[35,14136,13657],{"class":12678},[35,14138,77],{"class":12646},[35,14140,13674],{"class":12642},[35,14142,14143],{"class":12712},"\"G_decode: ",[35,14145,13809],{"class":12678},[35,14147,14148],{"class":12646},"g_decode",[35,14150,13945],{"class":12642},[35,14152,13716],{"class":12678},[35,14154,13689],{"class":12712},[35,14156,13208],{"class":12646},[35,14158,14160,14162],{"class":12638,"line":14159},119,[35,14161,13657],{"class":12678},[35,14163,13777],{"class":12646},[35,14165,14167],{"class":12638,"line":14166},120,[35,14168,12654],{"emptyLinePlaceholder":12653},[35,14170,14172,14174,14176,14179],{"class":12638,"line":14171},121,[35,14173,13657],{"class":12678},[35,14175,77],{"class":12646},[35,14177,14178],{"class":12712},"\"=== Final ===\"",[35,14180,13208],{"class":12646},[35,14182,14184,14186,14188,14190,14193,14195,14197,14199,14201,14204,14206,14208,14210,14212,14215,14217,14220,14222,14224,14226],{"class":12638,"line":14183},122,[35,14185,13657],{"class":12678},[35,14187,77],{"class":12646},[35,14189,13674],{"class":12642},[35,14191,14192],{"class":12712},"\"throughput bill: ",[35,14194,13809],{"class":12678},[35,14196,14023],{"class":12646},[35,14198,13945],{"class":12642},[35,14200,13716],{"class":12678},[35,14202,14203],{"class":12712}," + ",[35,14205,13809],{"class":12678},[35,14207,14148],{"class":12646},[35,14209,13945],{"class":12642},[35,14211,13716],{"class":12678},[35,14213,14214],{"class":12712}," = ",[35,14216,13809],{"class":12678},[35,14218,14219],{"class":12646},"g_throughput",[35,14221,13945],{"class":12642},[35,14223,13716],{"class":12678},[35,14225,13689],{"class":12712},[35,14227,13208],{"class":12646},[35,14229,14231,14233,14235,14237,14240,14242,14244,14246,14248,14250,14252,14254,14256,14258,14261,14263,14266,14268,14270,14272],{"class":12638,"line":14230},123,[35,14232,13657],{"class":12678},[35,14234,77],{"class":12646},[35,14236,13674],{"class":12642},[35,14238,14239],{"class":12712},"\"final bill: max(",[35,14241,13809],{"class":12678},[35,14243,13942],{"class":12646},[35,14245,13945],{"class":12642},[35,14247,13716],{"class":12678},[35,14249,5143],{"class":12712},[35,14251,13809],{"class":12678},[35,14253,14219],{"class":12646},[35,14255,13945],{"class":12642},[35,14257,13716],{"class":12678},[35,14259,14260],{"class":12712},") = ",[35,14262,13809],{"class":12678},[35,14264,14265],{"class":12646},"g_final",[35,14267,13945],{"class":12642},[35,14269,13716],{"class":12678},[35,14271,13689],{"class":12712},[35,14273,13208],{"class":12646},[35,14275,14277,14279,14281,14283,14286,14288,14291,14293,14295],{"class":12638,"line":14276},124,[35,14278,13657],{"class":12678},[35,14280,77],{"class":12646},[35,14282,13674],{"class":12642},[35,14284,14285],{"class":12712},"\"rounded GPUs: ",[35,14287,13809],{"class":12678},[35,14289,14290],{"class":12646},"math.ceil(g_final)",[35,14292,13716],{"class":12678},[35,14294,13689],{"class":12712},[35,14296,13208],{"class":12646},[27,14298,14300],{"id":14299},"turning-the-knobs","Turning the Knobs",[10,14302,14303],{},"We demonstrated the mechanics through the working example. Now we change one input at a time and watch which bill moves. Each case below is one constant changed at the top of the script.",[10,14305,14306,14309],{},[662,14307,14308],{},"Raise the rate."," Keep the dense 8B and the 4K-in, 1K-out shape, but push demand from 2 to 20 requests\u002Fsec. The load-dependent terms scale almost linearly: prefill grows to about 3 H200s, decode to about 5, and the KV-heavy fit floor to about 6.",[35,14311,14313],{"className":14312},[38],[35,14314,14316,14358],{"className":14315},[42],[35,14317,14319],{"className":14318},[46],[48,14320,14321],{"xmlns":50,"display":51},[53,14322,14323,14355],{},[56,14324,14325,14327,14329,14331,14333,14335,14337,14339,14341,14343,14345,14348,14350,14352],{},[59,14326,61],{},[63,14328,2494],{},[59,14330,68],{},[63,14332,71],{},[63,14334,77],{"stretchy":74},[1342,14336,2193],{},[63,14338,89],{"separator":75},[84,14340,92],{},[1342,14342,2260],{},[63,14344,102],{},[1342,14346,14347],{},"5",[63,14349,112],{"stretchy":74},[63,14351,2494],{},[1342,14353,14354],{},"8.",[114,14356,14357],{"encoding":116},"G \\approx \\max(6,\\ 3 + 5) \\approx 8.",[35,14359,14361,14379,14415,14436],{"className":14360,"ariaHidden":75},[121],[35,14362,14364,14367,14370,14373,14376],{"className":14363},[125],[35,14365],{"className":14366,"style":130},[129],[35,14368,61],{"className":14369},[134,135],[35,14371],{"className":14372,"style":140},[139],[35,14374,2494],{"className":14375},[144],[35,14377],{"className":14378,"style":140},[139],[35,14380,14382,14385,14388,14391,14394,14397,14400,14403,14406,14409,14412],{"className":14381},[125],[35,14383],{"className":14384,"style":1427},[129],[35,14386,68],{"className":14387},[158],[35,14389,77],{"className":14390},[3348],[35,14392,2193],{"className":14393},[134],[35,14395,89],{"className":14396},[234],[35,14398,92],{"className":14399},[139],[35,14401],{"className":14402,"style":162},[139],[35,14404,2260],{"className":14405},[134],[35,14407],{"className":14408,"style":288},[139],[35,14410,102],{"className":14411},[292],[35,14413],{"className":14414,"style":288},[139],[35,14416,14418,14421,14424,14427,14430,14433],{"className":14417},[125],[35,14419],{"className":14420,"style":1427},[129],[35,14422,14347],{"className":14423},[134],[35,14425,112],{"className":14426},[3453],[35,14428],{"className":14429,"style":140},[139],[35,14431,2494],{"className":14432},[144],[35,14434],{"className":14435,"style":140},[139],[35,14437,14439,14442],{"className":14438},[125],[35,14440],{"className":14441,"style":1627},[129],[35,14443,14354],{"className":14444},[134],[10,14446,14447,14450],{},[662,14448,14449],{},"Change the shape."," Back to 2 requests\u002Fsec on the dense 8B, but vary the request shape. With 16K-token prompts and 1K-token answers, prefill rises because the prompt is longer, and decode rises because the average context is now 16.5K tokens. The bills are about 2 H200s for fit, 1.6 for prefill, and 1.9 for decode, so the answer is roughly four cards.",[10,14452,14453],{},"With 8K-token prompts and 8K-token answers, prefill is still under one H200, but decode jumps to about 11. Every one of the 8,000 output tokens rereads a cache averaging 12K tokens. The resident KV cache also grows to roughly the same size, so the answer is about 12 cards, with decode and KV cache as the binding terms.",[10,14455,14456,14459],{},[662,14457,14458],{},"Swap the model."," Return to the original 2 requests\u002Fsec, 4K-in, 1K-out workload, but trade the dense 8B for Qwen3-30B-A3B. The MoE moves two numbers in opposite directions.",[10,14461,14462,14463,2273],{},"For prefill, 4K tokens sit well below the attention crossover for this model, so ",[35,14464,14466,14502],{"className":14465},[42],[35,14467,14469],{"className":14468},[46],[48,14470,14471],{"xmlns":50},[53,14472,14473,14499],{},[56,14474,14475,14481,14483,14485,14487,14493,14495,14497],{},[79,14476,14477,14479],{},[59,14478,4071],{},[84,14480,974],{},[63,14482,2494],{},[1342,14484,1344],{},[63,14486,1707],{},[79,14488,14489,14491],{},[59,14490,680],{},[59,14492,643],{},[63,14494,65],{},[1342,14496,2193],{},[84,14498,5282],{},[114,14500,14501],{"encoding":116},"C_\\text{in} \\approx 2 \\times P_a = 6\\text{B}",[35,14503,14505,14563,14581,14636],{"className":14504,"ariaHidden":75},[121],[35,14506,14508,14511,14554,14557,14560],{"className":14507},[125],[35,14509],{"className":14510,"style":409},[129],[35,14512,14514,14517],{"className":14513},[134],[35,14515,4071],{"className":14516,"style":4092},[134,135],[35,14518,14520],{"className":14519},[180],[35,14521,14523,14546],{"className":14522},[184,185],[35,14524,14526,14543],{"className":14525},[189],[35,14527,14529],{"className":14528,"style":1005},[193],[35,14530,14531,14534],{"style":4107},[35,14532],{"className":14533,"style":202},[201],[35,14535,14537],{"className":14536},[206,207,208,209],[35,14538,14540],{"className":14539},[134,213,209],[35,14541,974],{"className":14542},[134,209],[35,14544,221],{"className":14545},[220],[35,14547,14549],{"className":14548},[189],[35,14550,14552],{"className":14551,"style":228},[193],[35,14553],{},[35,14555],{"className":14556,"style":140},[139],[35,14558,2494],{"className":14559},[144],[35,14561],{"className":14562,"style":140},[139],[35,14564,14566,14569,14572,14575,14578],{"className":14565},[125],[35,14567],{"className":14568,"style":1986},[129],[35,14570,1344],{"className":14571},[134],[35,14573],{"className":14574,"style":288},[139],[35,14576,1707],{"className":14577},[292],[35,14579],{"className":14580,"style":288},[139],[35,14582,14584,14587,14627,14630,14633],{"className":14583},[125],[35,14585],{"className":14586,"style":409},[129],[35,14588,14590,14593],{"className":14589},[134],[35,14591,680],{"className":14592,"style":695},[134,135],[35,14594,14596],{"className":14595},[180],[35,14597,14599,14619],{"className":14598},[184,185],[35,14600,14602,14616],{"className":14601},[189],[35,14603,14605],{"className":14604,"style":745},[193],[35,14606,14607,14610],{"style":748},[35,14608],{"className":14609,"style":202},[201],[35,14611,14613],{"className":14612},[206,207,208,209],[35,14614,643],{"className":14615},[134,135,209],[35,14617,221],{"className":14618},[220],[35,14620,14622],{"className":14621},[189],[35,14623,14625],{"className":14624,"style":228},[193],[35,14626],{},[35,14628],{"className":14629,"style":140},[139],[35,14631,65],{"className":14632},[144],[35,14634],{"className":14635,"style":140},[139],[35,14637,14639,14642,14645],{"className":14638},[125],[35,14640],{"className":14641,"style":130},[129],[35,14643,2193],{"className":14644},[134],[35,14646,14648],{"className":14647},[134,213],[35,14649,5282],{"className":14650},[134],[35,14652,14654],{"className":14653},[38],[35,14655,14657,14708],{"className":14656},[42],[35,14658,14660],{"className":14659},[46],[48,14661,14662],{"xmlns":50,"display":51},[53,14663,14664,14705],{},[56,14665,14666,14672,14674,14700,14702],{},[79,14667,14668,14670],{},[59,14669,61],{},[84,14671,99],{},[63,14673,2494],{},[3706,14675,14676,14690],{},[56,14677,14678,14680,14682,14684,14686,14688],{},[1342,14679,1344],{},[63,14681,1707],{},[1342,14683,1363],{},[63,14685,1707],{},[1342,14687,2193],{},[84,14689,5282],{},[56,14691,14692,14694,14696,14698],{},[1342,14693,4020],{},[63,14695,1707],{},[1342,14697,5570],{},[84,14699,5573],{},[63,14701,2494],{},[1342,14703,14704],{},"0.1.",[114,14706,14707],{"encoding":116},"G_\\text{prefill} \\approx \\frac{2 \\times 4{,}000 \\times 6\\text{B}}{0.5 \\times 990\\text{T}} \\approx 0.1.",[35,14709,14711,14769,14903],{"className":14710,"ariaHidden":75},[121],[35,14712,14714,14717,14760,14763,14766],{"className":14713},[125],[35,14715],{"className":14716,"style":493},[129],[35,14718,14720,14723],{"className":14719},[134],[35,14721,61],{"className":14722},[134,135],[35,14724,14726],{"className":14725},[180],[35,14727,14729,14752],{"className":14728},[184,185],[35,14730,14732,14749],{"className":14731},[189],[35,14733,14735],{"className":14734,"style":194},[193],[35,14736,14737,14740],{"style":197},[35,14738],{"className":14739,"style":202},[201],[35,14741,14743],{"className":14742},[206,207,208,209],[35,14744,14746],{"className":14745},[134,213,209],[35,14747,99],{"className":14748},[134,209],[35,14750,221],{"className":14751},[220],[35,14753,14755],{"className":14754},[189],[35,14756,14758],{"className":14757,"style":282},[193],[35,14759],{},[35,14761],{"className":14762,"style":140},[139],[35,14764,2494],{"className":14765},[144],[35,14767],{"className":14768,"style":140},[139],[35,14770,14772,14775,14894,14897,14900],{"className":14771},[125],[35,14773],{"className":14774,"style":4856},[129],[35,14776,14778,14781,14891],{"className":14777},[134],[35,14779],{"className":14780},[3348,3818],[35,14782,14784],{"className":14783},[3706],[35,14785,14787,14883],{"className":14786},[184,185],[35,14788,14790,14880],{"className":14789},[189],[35,14791,14793,14822,14830],{"className":14792,"style":4875},[193],[35,14794,14795,14798],{"style":3834},[35,14796],{"className":14797,"style":2801},[201],[35,14799,14801,14804,14807,14810,14813,14816],{"className":14800},[134],[35,14802,4020],{"className":14803},[134],[35,14805],{"className":14806,"style":288},[139],[35,14808,1707],{"className":14809},[292],[35,14811],{"className":14812,"style":288},[139],[35,14814,5570],{"className":14815},[134],[35,14817,14819],{"className":14818},[134,213],[35,14820,5573],{"className":14821},[134],[35,14823,14824,14827],{"style":3852},[35,14825],{"className":14826,"style":2801},[201],[35,14828],{"className":14829,"style":3860},[3859],[35,14831,14832,14835],{"style":3863},[35,14833],{"className":14834,"style":2801},[201],[35,14836,14838,14841,14844,14847,14850,14853,14859,14862,14865,14868,14871,14874],{"className":14837},[134],[35,14839,1344],{"className":14840},[134],[35,14842],{"className":14843,"style":288},[139],[35,14845,1707],{"className":14846},[292],[35,14848],{"className":14849,"style":288},[139],[35,14851,1509],{"className":14852},[134],[35,14854,14856],{"className":14855},[134],[35,14857,89],{"className":14858},[234],[35,14860,1519],{"className":14861},[134],[35,14863],{"className":14864,"style":288},[139],[35,14866,1707],{"className":14867},[292],[35,14869],{"className":14870,"style":288},[139],[35,14872,2193],{"className":14873},[134],[35,14875,14877],{"className":14876},[134,213],[35,14878,5282],{"className":14879},[134],[35,14881,221],{"className":14882},[220],[35,14884,14886],{"className":14885},[189],[35,14887,14889],{"className":14888,"style":5034},[193],[35,14890],{},[35,14892],{"className":14893},[3453,3818],[35,14895],{"className":14896,"style":140},[139],[35,14898,2494],{"className":14899},[144],[35,14901],{"className":14902,"style":140},[139],[35,14904,14906,14909],{"className":14905},[125],[35,14907],{"className":14908,"style":1627},[129],[35,14910,14704],{"className":14911},[134],[10,14913,14914],{},"Decode does not get that discount. Under load the batch touches nearly the full expert pool, so the weight-read term uses all 60 GB:",[35,14916,14918],{"className":14917},[38],[35,14919,14921,14988],{"className":14920},[42],[35,14922,14924],{"className":14923},[46],[48,14925,14926],{"xmlns":50,"display":51},[53,14927,14928,14985],{},[56,14929,14930,14936,14938,14981,14983],{},[79,14931,14932,14934],{},[59,14933,61],{},[84,14935,109],{},[63,14937,65],{},[3706,14939,14940,14958],{},[56,14941,14942,14944,14946,14948,14950,14952,14954,14956],{},[1342,14943,1344],{},[63,14945,1707],{},[1342,14947,1378],{},[63,14949,1707],{},[1342,14951,3057],{},[63,14953,1707],{},[1342,14955,3049],{},[84,14957,3052],{},[56,14959,14960,14962,14964,14966,14968,14970,14973,14975,14977,14979],{},[1342,14961,8739],{},[63,14963,1707],{},[1342,14965,9905],{},[84,14967,9908],{},[63,14969,8893],{},[1342,14971,14972],{},"60",[84,14974,3070],{},[59,14976,2749],{"mathvariant":1396},[1342,14978,1390],{},[84,14980,1393],{},[63,14982,2494],{},[1342,14984,12462],{},[114,14986,14987],{"encoding":116},"G_\\text{decode} = \\frac{2 \\times 1{,}000 \\times 4{,}500 \\times 144\\text{ KB}}{0.6 \\times 4.8\\text{ TB\u002Fs} - 60\\text{ GB}\u002F50\\text{ ms}} \\approx 0.8.",[35,14989,14991,15049,15231],{"className":14990,"ariaHidden":75},[121],[35,14992,14994,14997,15040,15043,15046],{"className":14993},[125],[35,14995],{"className":14996,"style":409},[129],[35,14998,15000,15003],{"className":14999},[134],[35,15001,61],{"className":15002},[134,135],[35,15004,15006],{"className":15005},[180],[35,15007,15009,15032],{"className":15008},[184,185],[35,15010,15012,15029],{"className":15011},[189],[35,15013,15015],{"className":15014,"style":194},[193],[35,15016,15017,15020],{"style":197},[35,15018],{"className":15019,"style":202},[201],[35,15021,15023],{"className":15022},[206,207,208,209],[35,15024,15026],{"className":15025},[134,213,209],[35,15027,109],{"className":15028},[134,209],[35,15030,221],{"className":15031},[220],[35,15033,15035],{"className":15034},[189],[35,15036,15038],{"className":15037,"style":228},[193],[35,15039],{},[35,15041],{"className":15042,"style":140},[139],[35,15044,65],{"className":15045},[144],[35,15047],{"className":15048,"style":140},[139],[35,15050,15052,15055,15222,15225,15228],{"className":15051},[125],[35,15053],{"className":15054,"style":10142},[129],[35,15056,15058,15061,15219],{"className":15057},[134],[35,15059],{"className":15060},[3348,3818],[35,15062,15064],{"className":15063},[3706],[35,15065,15067,15211],{"className":15066},[184,185],[35,15068,15070,15208],{"className":15069},[189],[35,15071,15073,15129,15137],{"className":15072,"style":4875},[193],[35,15074,15075,15078],{"style":3834},[35,15076],{"className":15077,"style":2801},[201],[35,15079,15081,15084,15087,15090,15093,15096,15102,15105,15108,15111,15114,15120,15123],{"className":15080},[134],[35,15082,8739],{"className":15083},[134],[35,15085],{"className":15086,"style":288},[139],[35,15088,1707],{"className":15089},[292],[35,15091],{"className":15092,"style":288},[139],[35,15094,9905],{"className":15095},[134],[35,15097,15099],{"className":15098},[134,213],[35,15100,9908],{"className":15101},[134],[35,15103],{"className":15104,"style":288},[139],[35,15106,8893],{"className":15107},[292],[35,15109],{"className":15110,"style":288},[139],[35,15112,14972],{"className":15113},[134],[35,15115,15117],{"className":15116},[134,213],[35,15118,3070],{"className":15119},[134],[35,15121,9995],{"className":15122},[134],[35,15124,15126],{"className":15125},[134,213],[35,15127,1393],{"className":15128},[134],[35,15130,15131,15134],{"style":3852},[35,15132],{"className":15133,"style":2801},[201],[35,15135],{"className":15136,"style":3860},[3859],[35,15138,15139,15142],{"style":3863},[35,15140],{"className":15141,"style":2801},[201],[35,15143,15145,15148,15151,15154,15157,15160,15166,15169,15172,15175,15178,15181,15187,15190,15193,15196,15199,15202],{"className":15144},[134],[35,15146,1344],{"className":15147},[134],[35,15149],{"className":15150,"style":288},[139],[35,15152,1707],{"className":15153},[292],[35,15155],{"className":15156,"style":288},[139],[35,15158,1314],{"className":15159},[134],[35,15161,15163],{"className":15162},[134],[35,15164,89],{"className":15165},[234],[35,15167,1519],{"className":15168},[134],[35,15170],{"className":15171,"style":288},[139],[35,15173,1707],{"className":15174},[292],[35,15176],{"className":15177,"style":288},[139],[35,15179,1509],{"className":15180},[134],[35,15182,15184],{"className":15183},[134],[35,15185,89],{"className":15186},[234],[35,15188,3179],{"className":15189},[134],[35,15191],{"className":15192,"style":288},[139],[35,15194,1707],{"className":15195},[292],[35,15197],{"className":15198,"style":288},[139],[35,15200,3049],{"className":15201},[134],[35,15203,15205],{"className":15204},[134,213],[35,15206,3052],{"className":15207},[134],[35,15209,221],{"className":15210},[220],[35,15212,15214],{"className":15213},[189],[35,15215,15217],{"className":15216,"style":9722},[193],[35,15218],{},[35,15220],{"className":15221},[3453,3818],[35,15223],{"className":15224,"style":140},[139],[35,15226,2494],{"className":15227},[144],[35,15229],{"className":15230,"style":140},[139],[35,15232,15234,15237],{"className":15233},[125],[35,15235],{"className":15236,"style":1627},[129],[35,15238,12462],{"className":15239},[134],[10,15241,15242,15243,15315,15316,15388],{},"The MoE buys cheap prefill, not a cheap decode budget. That pays off most for multi-turn chat, where each turn re-sends the full transcript: ",[35,15244,15246,15263],{"className":15245},[42],[35,15247,15249],{"className":15248},[46],[48,15250,15251],{"xmlns":50},[53,15252,15253,15261],{},[56,15254,15255],{},[79,15256,15257,15259],{},[59,15258,971],{},[84,15260,974],{},[114,15262,977],{"encoding":116},[35,15264,15266],{"className":15265,"ariaHidden":75},[121],[35,15267,15269,15272],{"className":15268},[125],[35,15270],{"className":15271,"style":409},[129],[35,15273,15275,15278],{"className":15274},[134],[35,15276,971],{"className":15277},[134,135],[35,15279,15281],{"className":15280},[180],[35,15282,15284,15307],{"className":15283},[184,185],[35,15285,15287,15304],{"className":15286},[189],[35,15288,15290],{"className":15289,"style":1005},[193],[35,15291,15292,15295],{"style":197},[35,15293],{"className":15294,"style":202},[201],[35,15296,15298],{"className":15297},[206,207,208,209],[35,15299,15301],{"className":15300},[134,213,209],[35,15302,974],{"className":15303},[134,209],[35,15305,221],{"className":15306},[220],[35,15308,15310],{"className":15309},[189],[35,15311,15313],{"className":15312,"style":228},[193],[35,15314],{}," climbs with the conversation while ",[35,15317,15319,15336],{"className":15318},[42],[35,15320,15322],{"className":15321},[46],[48,15323,15324],{"xmlns":50},[53,15325,15326,15334],{},[56,15327,15328],{},[79,15329,15330,15332],{},[59,15331,971],{},[84,15333,1050],{},[114,15335,1053],{"encoding":116},[35,15337,15339],{"className":15338,"ariaHidden":75},[121],[35,15340,15342,15345],{"className":15341},[125],[35,15343],{"className":15344,"style":409},[129],[35,15346,15348,15351],{"className":15347},[134],[35,15349,971],{"className":15350},[134,135],[35,15352,15354],{"className":15353},[180],[35,15355,15357,15380],{"className":15356},[184,185],[35,15358,15360,15377],{"className":15359},[189],[35,15361,15363],{"className":15362,"style":1081},[193],[35,15364,15365,15368],{"style":197},[35,15366],{"className":15367,"style":202},[201],[35,15369,15371],{"className":15370},[206,207,208,209],[35,15372,15374],{"className":15373},[134,213,209],[35,15375,1050],{"className":15376},[134,209],[35,15378,221],{"className":15379},[220],[35,15381,15383],{"className":15382},[189],[35,15384,15386],{"className":15385,"style":228},[193],[35,15387],{}," per turn stays short.",[2249,15390,15392,15395,15398],{"title":15391},"Why do labs still pick MoE?",[10,15393,15394],{},"Training is the first reason labs still pick MoE. A 30B-A3B costs roughly 3B-equivalent compute to train but can deliver quality closer to a dense 30B. The training costs usually dwarfs the serving bill.",[10,15396,15397],{},"The workload often fits MoE too. Frontier labs today focus on coding agents which use large code contexts and short tool responses. That shape is prefill-heavy, not decode-heavy, so MoE's cheap prefill is the right trade.",[10,15399,15400],{},"Prefix caching also shifts the decode picture. An agentic loop accumulates context each turn, but the serving engine caches the growing prefix and only prefills the incremental append. Re-reading all 60 GB of weights per decode step remains expensive, but the KV traffic that dominates the decode term is partly absorbed by the cache.",[10,15402,15403],{},"The cases line up like this; the bottleneck changes with traffic shape:",[352,15405,15406,15647],{},[355,15407,15408],{},[358,15409,15410,15413,15417,15420,15494,15568,15642,15645],{},[361,15411,15412],{"align":2209},"Model",[361,15414,15416],{"align":15415},"right","RPS",[361,15418,15419],{"align":15415},"In \u002F Out",[361,15421,15422],{"align":15415},[35,15423,15425,15442],{"className":15424},[42],[35,15426,15428],{"className":15427},[46],[48,15429,15430],{"xmlns":50},[53,15431,15432,15440],{},[56,15433,15434],{},[79,15435,15436,15438],{},[59,15437,61],{},[84,15439,86],{},[114,15441,399],{"encoding":116},[35,15443,15445],{"className":15444,"ariaHidden":75},[121],[35,15446,15448,15451],{"className":15447},[125],[35,15449],{"className":15450,"style":409},[129],[35,15452,15454,15457],{"className":15453},[134],[35,15455,61],{"className":15456},[134,135],[35,15458,15460],{"className":15459},[180],[35,15461,15463,15486],{"className":15462},[184,185],[35,15464,15466,15483],{"className":15465},[189],[35,15467,15469],{"className":15468,"style":194},[193],[35,15470,15471,15474],{"style":197},[35,15472],{"className":15473,"style":202},[201],[35,15475,15477],{"className":15476},[206,207,208,209],[35,15478,15480],{"className":15479},[134,213,209],[35,15481,86],{"className":15482},[134,209],[35,15484,221],{"className":15485},[220],[35,15487,15489],{"className":15488},[189],[35,15490,15492],{"className":15491,"style":228},[193],[35,15493],{},[361,15495,15496],{"align":15415},[35,15497,15499,15516],{"className":15498},[42],[35,15500,15502],{"className":15501},[46],[48,15503,15504],{"xmlns":50},[53,15505,15506,15514],{},[56,15507,15508],{},[79,15509,15510,15512],{},[59,15511,61],{},[84,15513,99],{},[114,15515,483],{"encoding":116},[35,15517,15519],{"className":15518,"ariaHidden":75},[121],[35,15520,15522,15525],{"className":15521},[125],[35,15523],{"className":15524,"style":493},[129],[35,15526,15528,15531],{"className":15527},[134],[35,15529,61],{"className":15530},[134,135],[35,15532,15534],{"className":15533},[180],[35,15535,15537,15560],{"className":15536},[184,185],[35,15538,15540,15557],{"className":15539},[189],[35,15541,15543],{"className":15542,"style":194},[193],[35,15544,15545,15548],{"style":197},[35,15546],{"className":15547,"style":202},[201],[35,15549,15551],{"className":15550},[206,207,208,209],[35,15552,15554],{"className":15553},[134,213,209],[35,15555,99],{"className":15556},[134,209],[35,15558,221],{"className":15559},[220],[35,15561,15563],{"className":15562},[189],[35,15564,15566],{"className":15565,"style":282},[193],[35,15567],{},[361,15569,15570],{"align":15415},[35,15571,15573,15590],{"className":15572},[42],[35,15574,15576],{"className":15575},[46],[48,15577,15578],{"xmlns":50},[53,15579,15580,15588],{},[56,15581,15582],{},[79,15583,15584,15586],{},[59,15585,61],{},[84,15587,109],{},[114,15589,567],{"encoding":116},[35,15591,15593],{"className":15592,"ariaHidden":75},[121],[35,15594,15596,15599],{"className":15595},[125],[35,15597],{"className":15598,"style":409},[129],[35,15600,15602,15605],{"className":15601},[134],[35,15603,61],{"className":15604},[134,135],[35,15606,15608],{"className":15607},[180],[35,15609,15611,15634],{"className":15610},[184,185],[35,15612,15614,15631],{"className":15613},[189],[35,15615,15617],{"className":15616,"style":194},[193],[35,15618,15619,15622],{"style":197},[35,15620],{"className":15621,"style":202},[201],[35,15623,15625],{"className":15624},[206,207,208,209],[35,15626,15628],{"className":15627},[134,213,209],[35,15629,109],{"className":15630},[134,209],[35,15632,221],{"className":15633},[220],[35,15635,15637],{"className":15636},[189],[35,15638,15640],{"className":15639,"style":228},[193],[35,15641],{},[361,15643,15644],{"align":15415},"Answer",[361,15646,370],{"align":2209},[372,15648,15649,15672,15696,15720,15742],{},[358,15650,15651,15653,15655,15658,15661,15664,15667,15669],{},[377,15652,2228],{"align":2209},[377,15654,1344],{"align":15415},[377,15656,15657],{"align":15415},"4K \u002F 1K",[377,15659,15660],{"align":15415},"~0.7",[377,15662,15663],{"align":15415},"~0.3",[377,15665,15666],{"align":15415},"~0.5",[377,15668,1314],{"align":15415},[377,15670,15671],{"align":2209},"rounds up to 1",[358,15673,15674,15676,15679,15681,15684,15687,15690,15693],{},[377,15675,2228],{"align":2209},[377,15677,15678],{"align":15415},"20",[377,15680,15657],{"align":15415},[377,15682,15683],{"align":15415},"~6",[377,15685,15686],{"align":15415},"~3",[377,15688,15689],{"align":15415},"~5",[377,15691,15692],{"align":15415},"~8",[377,15694,15695],{"align":2209},"decode throughput",[358,15697,15698,15700,15702,15705,15708,15711,15714,15717],{},[377,15699,2228],{"align":2209},[377,15701,1344],{"align":15415},[377,15703,15704],{"align":15415},"16K \u002F 1K",[377,15706,15707],{"align":15415},"~2",[377,15709,15710],{"align":15415},"~1.6",[377,15712,15713],{"align":15415},"~1.9",[377,15715,15716],{"align":15415},"~4",[377,15718,15719],{"align":2209},"prefill (long input)",[358,15721,15722,15724,15726,15729,15732,15734,15736,15739],{},[377,15723,2228],{"align":2209},[377,15725,1344],{"align":15415},[377,15727,15728],{"align":15415},"8K \u002F 8K",[377,15730,15731],{"align":15415},"~11",[377,15733,15660],{"align":15415},[377,15735,15731],{"align":15415},[377,15737,15738],{"align":15415},"~12",[377,15740,15741],{"align":2209},"KV + decode (long output)",[358,15743,15744,15747,15749,15751,15754,15757,15760,15762],{},[377,15745,15746],{"align":2209},"30B-A3B",[377,15748,1344],{"align":15415},[377,15750,15657],{"align":15415},[377,15752,15753],{"align":15415},"~1",[377,15755,15756],{"align":15415},"~0.1",[377,15758,15759],{"align":15415},"~0.8",[377,15761,15753],{"align":15415},[377,15763,15764],{"align":2209},"weights (MoE)",[3949,15766,3957,15768,15771],{"className":15767},[3952,3953,3954,3955,3956],[662,15769,3963],{"className":15770},[3961,3962],"\n  The same model can be memory-bound, prefill-bound, or decode-bound depending more on traffic shape than on parameter count alone.\n",[27,15773,15775],{"id":15774},"a-floor-not-a-target","A Floor, Not a Target",[10,15777,15778,15779,15851,15852,15924,15925,15997],{},"You now have an answer to both questions: the weights and KV cache fit on ",[35,15780,15782,15799],{"className":15781},[42],[35,15783,15785],{"className":15784},[46],[48,15786,15787],{"xmlns":50},[53,15788,15789,15797],{},[56,15790,15791],{},[79,15792,15793,15795],{},[59,15794,61],{},[84,15796,86],{},[114,15798,399],{"encoding":116},[35,15800,15802],{"className":15801,"ariaHidden":75},[121],[35,15803,15805,15808],{"className":15804},[125],[35,15806],{"className":15807,"style":409},[129],[35,15809,15811,15814],{"className":15810},[134],[35,15812,61],{"className":15813},[134,135],[35,15815,15817],{"className":15816},[180],[35,15818,15820,15843],{"className":15819},[184,185],[35,15821,15823,15840],{"className":15822},[189],[35,15824,15826],{"className":15825,"style":194},[193],[35,15827,15828,15831],{"style":197},[35,15829],{"className":15830,"style":202},[201],[35,15832,15834],{"className":15833},[206,207,208,209],[35,15835,15837],{"className":15836},[134,213,209],[35,15838,86],{"className":15839},[134,209],[35,15841,221],{"className":15842},[220],[35,15844,15846],{"className":15845},[189],[35,15847,15849],{"className":15848,"style":228},[193],[35,15850],{}," cards; throughput demands at least ",[35,15853,15855,15872],{"className":15854},[42],[35,15856,15858],{"className":15857},[46],[48,15859,15860],{"xmlns":50},[53,15861,15862,15870],{},[56,15863,15864],{},[79,15865,15866,15868],{},[59,15867,61],{},[84,15869,99],{},[114,15871,483],{"encoding":116},[35,15873,15875],{"className":15874,"ariaHidden":75},[121],[35,15876,15878,15881],{"className":15877},[125],[35,15879],{"className":15880,"style":493},[129],[35,15882,15884,15887],{"className":15883},[134],[35,15885,61],{"className":15886},[134,135],[35,15888,15890],{"className":15889},[180],[35,15891,15893,15916],{"className":15892},[184,185],[35,15894,15896,15913],{"className":15895},[189],[35,15897,15899],{"className":15898,"style":194},[193],[35,15900,15901,15904],{"style":197},[35,15902],{"className":15903,"style":202},[201],[35,15905,15907],{"className":15906},[206,207,208,209],[35,15908,15910],{"className":15909},[134,213,209],[35,15911,99],{"className":15912},[134,209],[35,15914,221],{"className":15915},[220],[35,15917,15919],{"className":15918},[189],[35,15920,15922],{"className":15921,"style":282},[193],[35,15923],{}," plus ",[35,15926,15928,15945],{"className":15927},[42],[35,15929,15931],{"className":15930},[46],[48,15932,15933],{"xmlns":50},[53,15934,15935,15943],{},[56,15936,15937],{},[79,15938,15939,15941],{},[59,15940,61],{},[84,15942,109],{},[114,15944,567],{"encoding":116},[35,15946,15948],{"className":15947,"ariaHidden":75},[121],[35,15949,15951,15954],{"className":15950},[125],[35,15952],{"className":15953,"style":409},[129],[35,15955,15957,15960],{"className":15956},[134],[35,15958,61],{"className":15959},[134,135],[35,15961,15963],{"className":15962},[180],[35,15964,15966,15989],{"className":15965},[184,185],[35,15967,15969,15986],{"className":15968},[189],[35,15970,15972],{"className":15971,"style":194},[193],[35,15973,15974,15977],{"style":197},[35,15975],{"className":15976,"style":202},[201],[35,15978,15980],{"className":15979},[206,207,208,209],[35,15981,15983],{"className":15982},[134,213,209],[35,15984,109],{"className":15985},[134,209],[35,15987,221],{"className":15988},[220],[35,15990,15992],{"className":15991},[189],[35,15993,15995],{"className":15994,"style":228},[193],[35,15996],{}," cards. Treat that result as a floor, not a target.",[10,15999,16000],{},"Real deployments usually land 20 to 30 percent higher. MFU sags under load, prefill and decode compete for bandwidth on shared cards, and tensor parallelism eats into both. The napkin gets the order of magnitude and names the bottleneck; a benchmark against your actual traffic tightens the number before you commit hardware.",[27,16002,16004],{"id":16003},"references","References",[16006,16007,16008],"content-references",{},[16009,16010,16011,16027,16039,16054,16066,16078,16090,16305,16428,16440],"ol",{},[659,16012,16013,16014,16022,16023,16026],{},"NVIDIA. ",[643,16015,16021],{"href":16016,"rel":16017,"target":16020},"https:\u002F\u002Fwww.nvidia.com\u002Fen-us\u002Fdata-center\u002Fh200\u002F",[16018,16019],"noopener","noreferrer","_blank","H200 Tensor Core GPU",". ",[16,16024,16025],{},"Datasheet",", 2024. 141 GB HBM3e, 4.8 TB\u002Fs. The datasheet quotes 1,979 bf16 TFLOPS, but that number assumes 2:4 sparsity; dense weights run at half, so we use ~990 TFLOPS throughout.",[659,16028,16029,16030,16022,16035,16038],{},"Ainslie et al. ",[643,16031,16034],{"href":16032,"rel":16033,"target":16020},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2305.13245",[16018,16019],"GQA: Training Generalized Multi-Query Transformer Models from Multi-Head Checkpoints",[16,16036,16037],{},"arXiv:2305.13245",", 2023. EMNLP 2023.",[659,16040,16041,16042,16047,16048,16053],{},"Gated DeltaNet (",[643,16043,16046],{"href":16044,"rel":16045,"target":16020},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2412.06464",[16018,16019],"Yang et al.",", ICLR 2025), the hybrid attention in Qwen3.5, and multi-head latent attention (",[643,16049,16052],{"href":16050,"rel":16051,"target":16020},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2405.04434",[16018,16019],"DeepSeek-V2","), the compressed-cache design. Both shrink the KV cache below standard attention, never above it.",[659,16055,16056,16057,16022,16062,16065],{},"Kwon et al. ",[643,16058,16061],{"href":16059,"rel":16060,"target":16020},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2309.06180",[16018,16019],"Efficient Memory Management for Large Language Model Serving with PagedAttention",[16,16063,16064],{},"arXiv:2309.06180",", 2023. SOSP 2023.",[659,16067,16068,16069,16022,16074,16077],{},"Williams, Waterman, and Patterson. ",[643,16070,16073],{"href":16071,"rel":16072,"target":16020},"https:\u002F\u002Fdl.acm.org\u002Fdoi\u002F10.1145\u002F1498765.1498785",[16018,16019],"Roofline: An Insightful Visual Performance Model for Floating-Point Programs and Multicore Architectures",[16,16075,16076],{},"Communications of the ACM",", 2009.",[659,16079,16080,16081,16022,16086,16089],{},"Qwen Team. ",[643,16082,16085],{"href":16083,"rel":16084,"target":16020},"https:\u002F\u002Fhuggingface.co\u002FQwen",[16018,16019],"Qwen3 and Qwen3.5 model cards",[16,16087,16088],{},"HuggingFace",", 2025–2026. Layer counts, KV-head, and head-dimension values for the 0.6B\u002F8B\u002F32B dense models and the Qwen3.5 hybrids.",[659,16091,16092,16093,16022,16098,16101,16102,16151,16152,16304],{},"Kaplan et al. ",[643,16094,16097],{"href":16095,"rel":16096,"target":16020},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2001.08361",[16018,16019],"Scaling Laws for Neural Language Models",[16,16099,16100],{},"arXiv:2001.08361",", 2020. The ",[35,16103,16105,16124],{"className":16104},[42],[35,16106,16108],{"className":16107},[46],[48,16109,16110],{"xmlns":50},[53,16111,16112,16121],{},[56,16113,16114,16116,16118],{},[63,16115,2494],{},[1342,16117,1344],{},[59,16119,16120],{},"N",[114,16122,16123],{"encoding":116},"\\approx 2N",[35,16125,16127,16139],{"className":16126,"ariaHidden":75},[121],[35,16128,16130,16133,16136],{"className":16129},[125],[35,16131],{"className":16132,"style":4033},[129],[35,16134,2494],{"className":16135},[144],[35,16137],{"className":16138,"style":140},[139],[35,16140,16142,16145,16148],{"className":16141},[125],[35,16143],{"className":16144,"style":130},[129],[35,16146,1344],{"className":16147},[134],[35,16149,16120],{"className":16150,"style":814},[134,135]," forward FLOPs per token (Eq. 2.2, Table 1), and the condition ",[35,16153,16155,16187],{"className":16154},[42],[35,16156,16158],{"className":16157},[46],[48,16159,16160],{"xmlns":50},[53,16161,16162,16184],{},[56,16163,16164,16171,16174,16176,16178],{},[79,16165,16166,16168],{},[59,16167,1885],{},[84,16169,16170],{},"ctx",[63,16172,16173],{},"\u003C",[1342,16175,8151],{},[84,16177,3503],{},[79,16179,16180,16182],{},[59,16181,1904],{},[84,16183,4238],{},[114,16185,16186],{"encoding":116},"n_\\text{ctx} \u003C 12\\,d_\\text{hidden}",[35,16188,16190,16249],{"className":16189,"ariaHidden":75},[121],[35,16191,16193,16197,16240,16243,16246],{"className":16192},[125],[35,16194],{"className":16195,"style":16196},[129],"height:0.6891em;vertical-align:-0.15em;",[35,16198,16200,16203],{"className":16199},[134],[35,16201,1885],{"className":16202},[134,135],[35,16204,16206],{"className":16205},[180],[35,16207,16209,16232],{"className":16208},[184,185],[35,16210,16212,16229],{"className":16211},[189],[35,16213,16215],{"className":16214,"style":1081},[193],[35,16216,16217,16220],{"style":197},[35,16218],{"className":16219,"style":202},[201],[35,16221,16223],{"className":16222},[206,207,208,209],[35,16224,16226],{"className":16225},[134,213,209],[35,16227,16170],{"className":16228},[134,209],[35,16230,221],{"className":16231},[220],[35,16233,16235],{"className":16234},[189],[35,16236,16238],{"className":16237,"style":228},[193],[35,16239],{},[35,16241],{"className":16242,"style":140},[139],[35,16244,16173],{"className":16245},[144],[35,16247],{"className":16248,"style":140},[139],[35,16250,16252,16255,16258,16261],{"className":16251},[125],[35,16253],{"className":16254,"style":2123},[129],[35,16256,8151],{"className":16257},[134],[35,16259],{"className":16260,"style":162},[139],[35,16262,16264,16267],{"className":16263},[134],[35,16265,1904],{"className":16266},[134,135],[35,16268,16270],{"className":16269},[180],[35,16271,16273,16296],{"className":16272},[184,185],[35,16274,16276,16293],{"className":16275},[189],[35,16277,16279],{"className":16278,"style":194},[193],[35,16280,16281,16284],{"style":197},[35,16282],{"className":16283,"style":202},[201],[35,16285,16287],{"className":16286},[206,207,208,209],[35,16288,16290],{"className":16289},[134,213,209],[35,16291,4238],{"className":16292},[134,209],[35,16294,221],{"className":16295},[220],[35,16297,16299],{"className":16298},[189],[35,16300,16302],{"className":16301,"style":228},[193],[35,16303],{}," for the attention term to stay negligible.",[659,16306,16307,16308,16022,16313,16316,16317,16427],{},"Pope et al. ",[643,16309,16312],{"href":16310,"rel":16311,"target":16020},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2211.05102",[16018,16019],"Efficiently Scaling Transformer Inference",[16,16314,16315],{},"arXiv:2211.05102",", MLSys 2023. Prefill compute-bound vs. decode memory-bound; the per-step cost ",[35,16318,16320,16354],{"className":16319},[42],[35,16321,16323],{"className":16322},[46],[48,16324,16325],{"xmlns":50},[53,16326,16327,16351],{},[56,16328,16329,16331,16334,16336,16339,16341,16344,16346,16348],{},[63,16330,77],{"stretchy":74},[84,16332,16333],{},"params",[63,16335,102],{},[84,16337,16338],{},"batch",[63,16340,1707],{},[84,16342,16343],{},"KV",[63,16345,112],{"stretchy":74},[59,16347,2749],{"mathvariant":1396},[84,16349,16350],{},"bandwidth",[114,16352,16353],{"encoding":116},"(\\text{params} + \\text{batch}\\times\\text{KV})\u002F\\text{bandwidth}",[35,16355,16357,16381,16403],{"className":16356,"ariaHidden":75},[121],[35,16358,16360,16363,16366,16372,16375,16378],{"className":16359},[125],[35,16361],{"className":16362,"style":1427},[129],[35,16364,77],{"className":16365},[3348],[35,16367,16369],{"className":16368},[134,213],[35,16370,16333],{"className":16371},[134],[35,16373],{"className":16374,"style":288},[139],[35,16376,102],{"className":16377},[292],[35,16379],{"className":16380,"style":288},[139],[35,16382,16384,16388,16394,16397,16400],{"className":16383},[125],[35,16385],{"className":16386,"style":16387},[129],"height:0.7778em;vertical-align:-0.0833em;",[35,16389,16391],{"className":16390},[134,213],[35,16392,16338],{"className":16393},[134],[35,16395],{"className":16396,"style":288},[139],[35,16398,1707],{"className":16399},[292],[35,16401],{"className":16402,"style":288},[139],[35,16404,16406,16409,16415,16418,16421],{"className":16405},[125],[35,16407],{"className":16408,"style":1427},[129],[35,16410,16412],{"className":16411},[134,213],[35,16413,16343],{"className":16414},[134],[35,16416,112],{"className":16417},[3453],[35,16419,2749],{"className":16420},[134],[35,16422,16424],{"className":16423},[134,213],[35,16425,16350],{"className":16426},[134],"; weights amortize across the batch, KV caches do not.",[659,16429,16430,16431,16022,16436,16439],{},"Databricks. ",[643,16432,16435],{"href":16433,"rel":16434,"target":16020},"https:\u002F\u002Fwww.databricks.com\u002Fblog\u002Fllm-inference-performance-engineering-best-practices",[16018,16019],"LLM Inference Performance Engineering: Best Practices",[16,16437,16438],{},"Blog",", 2023. Defines Model Bandwidth Utilization (MBU) and the memory-bandwidth limit on decode.",[659,16441,16442,16443,16448,16449,16510,16511,16540],{},"Kipply Chen. ",[643,16444,16447],{"href":16445,"rel":16446,"target":16020},"https:\u002F\u002Fkipp.ly\u002Ftransformer-inference-arithmetic\u002F",[16018,16019],"Transformer Inference Arithmetic",". 2022. Decode is memory-bandwidth-bound: the GPU re-reads all weights (",[35,16450,16452,16474],{"className":16451},[42],[35,16453,16455],{"className":16454},[46],[48,16456,16457],{"xmlns":50},[53,16458,16459,16471],{},[56,16460,16461,16463,16465,16467,16469],{},[63,16462,2494],{},[1342,16464,1344],{},[59,16466,680],{},[59,16468,2749],{"mathvariant":1396},[84,16470,868],{},[114,16472,16473],{"encoding":116},"\\approx 2P\u002F\\text{BW}",[35,16475,16477,16489],{"className":16476,"ariaHidden":75},[121],[35,16478,16480,16483,16486],{"className":16479},[125],[35,16481],{"className":16482,"style":4033},[129],[35,16484,2494],{"className":16485},[144],[35,16487],{"className":16488,"style":140},[139],[35,16490,16492,16495,16498,16501,16504],{"className":16491},[125],[35,16493],{"className":16494,"style":1427},[129],[35,16496,1344],{"className":16497},[134],[35,16499,680],{"className":16500,"style":695},[134,135],[35,16502,2749],{"className":16503},[134],[35,16505,16507],{"className":16506},[134,213],[35,16508,868],{"className":16509},[134]," per token) for each step, while prefill is compute-bound; the two meet at the flops-to-bandwidth ratio (batch ",[35,16512,16514,16528],{"className":16513},[42],[35,16515,16517],{"className":16516},[46],[48,16518,16519],{"xmlns":50},[53,16520,16521,16525],{},[56,16522,16523],{},[63,16524,2494],{},[114,16526,16527],{"encoding":116},"\\approx",[35,16529,16531],{"className":16530,"ariaHidden":75},[121],[35,16532,16534,16537],{"className":16533},[125],[35,16535],{"className":16536,"style":4033},[129],[35,16538,2494],{"className":16539},[144]," 208 on an A100).",[16542,16543,16544],"style",{},"html pre.shiki code .snl16, html code.shiki .snl16{--shiki-default:#F97583}html pre.shiki code .s95oV, html code.shiki .s95oV{--shiki-default:#E1E4E8}html pre.shiki code .sAwPA, html code.shiki .sAwPA{--shiki-default:#6A737D}html pre.shiki code .sDLfK, html code.shiki .sDLfK{--shiki-default:#79B8FF}html pre.shiki code .sU2Wk, html code.shiki .sU2Wk{--shiki-default:#9ECBFF}html pre.shiki code .svObZ, html code.shiki .svObZ{--shiki-default:#B392F0}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}",{"title":635,"searchDepth":12650,"depth":12650,"links":16546},[16547,16548,16549,16550,16554,16555,16556,16557,16558,16559,16560],{"id":29,"depth":12650,"text":30},{"id":650,"depth":12650,"text":651},{"id":1642,"depth":12650,"text":1643},{"id":1666,"depth":12650,"text":1667,"children":16551},[16552,16553],{"id":1674,"depth":12657,"text":1675},{"id":1842,"depth":12657,"text":1843},{"id":3967,"depth":12650,"text":3968},{"id":8621,"depth":12650,"text":8622},{"id":12158,"depth":12650,"text":12159},{"id":12619,"depth":12650,"text":12620},{"id":14299,"depth":12650,"text":14300},{"id":15774,"depth":12650,"text":15775},{"id":16003,"depth":12650,"text":16004},"2026-06-25","Napkin math for LLM serving. Two questions decide it: does it fit, and can it keep up. The formulas that answer both, from numbers you already remember.","md",{},"\u002Fwriting\u002Fhow-many-gpus-to-serve-an-llm",{"title":5,"description":16562},"writing\u002F06.how-many-gpus-to-serve-an-llm","W1FyL-_MPffZ0R4dM0IVOlOGQkUpwZyXdcWWNZkv3RQ",1782357799749]