[{"data":1,"prerenderedAt":1861},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fimpute-missing-values-by-group-with-pandas":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fimpute-missing-values-by-group-with-pandas":1829,"ancestors:\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fimpute-missing-values-by-group-with-pandas":1836,"children:\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fimpute-missing-values-by-group-with-pandas":1841,"related:\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fimpute-missing-values-by-group-with-pandas":1842},{"id":4,"title":5,"body":6,"dateModified":1795,"datePublished":1795,"description":1796,"extension":1797,"faq":1798,"keyword":1807,"kind":1808,"meta":1809,"navigation":103,"path":1818,"seo":1819,"slug":1822,"stem":1823,"topics":1824,"type":1827,"__hash__":1828},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fimpute-missing-values-by-group-with-pandas\u002Findex.md","Impute Missing Values by Group with pandas",{"type":7,"value":8,"toc":1783},"minimark",[9,29,34,62,68,72,75,284,298,302,312,404,418,655,659,662,962,969,1063,1067,1073,1395,1401,1405,1408,1581,1588,1662,1666,1701,1705,1718,1722,1728,1734,1740,1746,1750,1779],[10,11,12,13,17,18,22,23,28],"p",{},"A product list exported to Excel has a weight column with blanks scattered through it. Filling every blank with the overall average weight is quick, and wrong: the average weight of everything in the catalogue is a meaningless number for a missing bag of screws or a missing pallet of paving slabs. A much better guess is the typical weight ",[14,15,16],"em",{},"of similar items"," — the median of the same category, or of the same category and size class. pandas makes this group-wise imputation a one-liner with ",[19,20,21],"code",{},"groupby().transform()",", but doing it well takes a little more: a fallback for groups that are entirely blank, a minimum number of known values before trusting a group's median, a flag on every filled cell, and a check of how much the filling changed the figures you report. This page builds all of that. It belongs to ",[24,25,27],"a",{"href":26},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002F","Handling Missing Data in Excel Reports",".",[30,31,33],"h2",{"id":32},"prerequisites","Prerequisites",[35,36,41],"pre",{"className":37,"code":38,"language":39,"meta":40,"style":40},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl\n","bash","",[19,42,43],{"__ignoreMap":40},[44,45,48,52,56,59],"span",{"class":46,"line":47},"line",1,[44,49,51],{"class":50},"sMTad","pip",[44,53,55],{"class":54},"srMev"," install",[44,57,58],{"class":54}," pandas",[44,60,61],{"class":54}," openpyxl\n",[10,63,64,67],{},[19,65,66],{},"products.xlsx"," lists 19 products with a category, a size class and a weight in kilograms; 8 weights are blank. Some groups are fully known, some partly, and the two bulk gravel products have no known weight in their size class at all.",[30,69,71],{"id":70},"step-1-see-where-the-gaps-are","Step 1 — See where the gaps are",[10,73,74],{},"Before filling anything, count the gaps per group. That shows which groups have enough known values to estimate from:",[35,76,80],{"className":77,"code":78,"language":79,"meta":40,"style":40},"language-python shiki shiki-themes github-light github-dark-high-contrast","import pandas as pd\n\nproducts = pd.read_excel(\"products.xlsx\")\nsummary = products.groupby([\"Category\", \"Size\"]).agg(\n    rows=(\"Weight kg\", \"size\"),\n    known=(\"Weight kg\", \"count\"),\n    median=(\"Weight kg\", \"median\"),\n)\nsummary[\"missing\"] = summary[\"rows\"] - summary[\"known\"]\nprint(summary.to_string())\nprint(\"blank weights:\", products[\"Weight kg\"].isna().sum(), \"of\", len(products))\n","python",[19,81,82,98,105,123,146,169,188,207,212,245,255],{"__ignoreMap":40},[44,83,84,88,92,95],{"class":46,"line":47},[44,85,87],{"class":86},"s-kum","import",[44,89,91],{"class":90},"skGVy"," pandas ",[44,93,94],{"class":86},"as",[44,96,97],{"class":90}," pd\n",[44,99,101],{"class":46,"line":100},2,[44,102,104],{"emptyLinePlaceholder":103},true,"\n",[44,106,108,111,114,117,120],{"class":46,"line":107},3,[44,109,110],{"class":90},"products ",[44,112,113],{"class":86},"=",[44,115,116],{"class":90}," pd.read_excel(",[44,118,119],{"class":54},"\"products.xlsx\"",[44,121,122],{"class":90},")\n",[44,124,126,129,131,134,137,140,143],{"class":46,"line":125},4,[44,127,128],{"class":90},"summary ",[44,130,113],{"class":86},[44,132,133],{"class":90}," products.groupby([",[44,135,136],{"class":54},"\"Category\"",[44,138,139],{"class":90},", ",[44,141,142],{"class":54},"\"Size\"",[44,144,145],{"class":90},"]).agg(\n",[44,147,149,153,155,158,161,163,166],{"class":46,"line":148},5,[44,150,152],{"class":151},"sa561","    rows",[44,154,113],{"class":86},[44,156,157],{"class":90},"(",[44,159,160],{"class":54},"\"Weight kg\"",[44,162,139],{"class":90},[44,164,165],{"class":54},"\"size\"",[44,167,168],{"class":90},"),\n",[44,170,172,175,177,179,181,183,186],{"class":46,"line":171},6,[44,173,174],{"class":151},"    known",[44,176,113],{"class":86},[44,178,157],{"class":90},[44,180,160],{"class":54},[44,182,139],{"class":90},[44,184,185],{"class":54},"\"count\"",[44,187,168],{"class":90},[44,189,191,194,196,198,200,202,205],{"class":46,"line":190},7,[44,192,193],{"class":151},"    median",[44,195,113],{"class":86},[44,197,157],{"class":90},[44,199,160],{"class":54},[44,201,139],{"class":90},[44,203,204],{"class":54},"\"median\"",[44,206,168],{"class":90},[44,208,210],{"class":46,"line":209},8,[44,211,122],{"class":90},[44,213,215,218,221,224,226,229,232,234,237,239,242],{"class":46,"line":214},9,[44,216,217],{"class":90},"summary[",[44,219,220],{"class":54},"\"missing\"",[44,222,223],{"class":90},"] ",[44,225,113],{"class":86},[44,227,228],{"class":90}," summary[",[44,230,231],{"class":54},"\"rows\"",[44,233,223],{"class":90},[44,235,236],{"class":86},"-",[44,238,228],{"class":90},[44,240,241],{"class":54},"\"known\"",[44,243,244],{"class":90},"]\n",[44,246,248,252],{"class":46,"line":247},10,[44,249,251],{"class":250},"sP0c6","print",[44,253,254],{"class":90},"(summary.to_string())\n",[44,256,258,260,262,265,268,270,273,276,278,281],{"class":46,"line":257},11,[44,259,251],{"class":250},[44,261,157],{"class":90},[44,263,264],{"class":54},"\"blank weights:\"",[44,266,267],{"class":90},", products[",[44,269,160],{"class":54},[44,271,272],{"class":90},"].isna().sum(), ",[44,274,275],{"class":54},"\"of\"",[44,277,139],{"class":90},[44,279,280],{"class":250},"len",[44,282,283],{"class":90},"(products))\n",[10,285,286,289,290,293,294,28],{},[19,287,288],{},"size"," counts rows and ",[19,291,292],{},"count"," counts non-blank values, so their difference is the number of gaps. The table makes the problem concrete: Fixings\u002FSmall and Fixings\u002FMedium each have two known values, Tools\u002FLarge only one, Garden\u002FMedium one, and Landscaping\u002FBulk none. A group median from a single value is a weak estimate, and from none it does not exist. It also helps to look at the spread inside each group before deciding: a group whose known values range from 0.2 to 20 kg is not a good source for an estimate, however many values it has, and may mean the grouping columns are too coarse. A general overview of missing values is in ",[24,295,297],{"href":296},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Ffind-and-report-missing-values-in-an-excel-file\u002F","Find and Report Missing Values in an Excel File",[30,299,301],{"id":300},"step-2-fill-with-the-group-median","Step 2 — Fill with the group median",[10,303,304,307,308,311],{},[19,305,306],{},"groupby(...).transform(\"median\")"," returns a series aligned with the original rows, holding each row's group median. ",[19,309,310],{},"fillna"," with that series fills each blank from its own group:",[35,313,315],{"className":77,"code":314,"language":79,"meta":40,"style":40},"group_median = products.groupby([\"Category\", \"Size\"])[\"Weight kg\"].transform(\"median\")\nproducts[\"Weight simple\"] = products[\"Weight kg\"].fillna(group_median)\nprint(products[[\"SKU\", \"Category\", \"Size\", \"Weight kg\", \"Weight simple\"]].to_string(index=False))\n",[19,316,317,344,364],{"__ignoreMap":40},[44,318,319,322,324,326,328,330,332,335,337,340,342],{"class":46,"line":47},[44,320,321],{"class":90},"group_median ",[44,323,113],{"class":86},[44,325,133],{"class":90},[44,327,136],{"class":54},[44,329,139],{"class":90},[44,331,142],{"class":54},[44,333,334],{"class":90},"])[",[44,336,160],{"class":54},[44,338,339],{"class":90},"].transform(",[44,341,204],{"class":54},[44,343,122],{"class":90},[44,345,346,349,352,354,356,359,361],{"class":46,"line":100},[44,347,348],{"class":90},"products[",[44,350,351],{"class":54},"\"Weight simple\"",[44,353,223],{"class":90},[44,355,113],{"class":86},[44,357,358],{"class":90}," products[",[44,360,160],{"class":54},[44,362,363],{"class":90},"].fillna(group_median)\n",[44,365,366,368,371,374,376,378,380,382,384,386,388,390,393,396,398,401],{"class":46,"line":107},[44,367,251],{"class":250},[44,369,370],{"class":90},"(products[[",[44,372,373],{"class":54},"\"SKU\"",[44,375,139],{"class":90},[44,377,136],{"class":54},[44,379,139],{"class":90},[44,381,142],{"class":54},[44,383,139],{"class":90},[44,385,160],{"class":54},[44,387,139],{"class":90},[44,389,351],{"class":54},[44,391,392],{"class":90},"]].to_string(",[44,394,395],{"class":151},"index",[44,397,113],{"class":86},[44,399,400],{"class":250},"False",[44,402,403],{"class":90},"))\n",[10,405,406,409,410,413,414,417],{},[19,407,408],{},"transform"," is the key: unlike ",[19,411,412],{},"agg",", which returns one row per group, it returns one value per original row, so the result lines up with the column it fills. SCR-03 gets 0.225, the median of the two other small fixings; DRL-04 gets 2.95. The median is usually a better choice than the mean for weights, prices and durations, because one unusual item in a group pulls the mean much further than the median. Notice the bulk gravel rows are still blank — their group has no known values, so its median is ",[19,415,416],{},"NaN"," too.",[419,420,428,429,428,433,428,437,428,445,428,454,428,458,428,465,428,469,428,473,428,477,428,481,428,485,428,490,428,492,428,496,428,498,428,501,428,503,428,506,428,509,428,513,428,515,428,519,428,521,428,524,428,529,428,532,428,536,428,538,428,541,428,543,428,546,428,548,428,551,428,554,428,556,428,559,428,561,428,564,428,566,428,570,428,574,428,578,428,581,428,584,428,587,428,591,428,595,428,598,428,601,428,603,428,606,428,609,428,612,428,616,428,620,428,626,428,630,428,636,428,641,428,649],"svg",{"viewBox":421,"role":422,"ariaLabelledBy":423,"xmlns":426,"style":427},"4 0 752 192","img",[424,425],"imp-group-t","imp-group-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:752px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[430,431,432],"title",{"id":424},"Each blank takes its own group's median",[434,435,436],"desc",{"id":425},"Rows from three groups with some blank weights. Small fixings have known weights 0.20 and 0.25, so the blank gets 0.225. Medium tools have 2.80, 3.10 and 2.95, so the blank gets 2.95. Bulk landscaping has no known weight and stays blank after this step.",[438,439],"rect",{"x":440,"y":441,"width":442,"height":443,"fill":444},"4","0","752","192","#ffffff",[438,446],{"x":447,"y":448,"width":449,"height":450,"rx":441,"fill":451,"stroke":452,"style":453},"20.0","40.0","32.0","24.0","#e7ebef","var(--line,#cdd5e6)","stroke-width:1px",[438,455],{"x":456,"y":448,"width":457,"height":450,"rx":441,"fill":451,"stroke":452,"style":453},"52.0","64.8",[459,460,464],"text",{"x":461,"y":462,"style":463},"84.4","57.0","font-size:11px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","A",[438,466],{"x":467,"y":448,"width":468,"height":450,"rx":441,"fill":451,"stroke":452,"style":453},"116.8","99.6",[459,470,472],{"x":471,"y":462,"style":463},"166.6","B",[438,474],{"x":475,"y":448,"width":476,"height":450,"rx":441,"fill":451,"stroke":452,"style":453},"216.4","43.0",[459,478,480],{"x":479,"y":462,"style":463},"237.9","C",[438,482],{"x":447,"y":483,"width":449,"height":484,"rx":441,"fill":451,"stroke":452,"style":453},"64.0","28.0",[459,486,489],{"x":487,"y":488,"style":463},"36.0","83.0","1",[438,491],{"x":456,"y":483,"width":457,"height":484,"rx":441,"fill":444,"stroke":452,"style":453},[459,493,495],{"x":461,"y":488,"style":494},"font-size:10.5px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","SKU",[438,497],{"x":467,"y":483,"width":468,"height":484,"rx":441,"fill":444,"stroke":452,"style":453},[459,499,500],{"x":471,"y":488,"style":494},"Group",[438,502],{"x":475,"y":483,"width":476,"height":484,"rx":441,"fill":444,"stroke":452,"style":453},[459,504,505],{"x":479,"y":488,"style":494},"kg",[438,507],{"x":447,"y":508,"width":449,"height":484,"rx":441,"fill":451,"stroke":452,"style":453},"92.0",[459,510,512],{"x":487,"y":511,"style":463},"111.0","2",[438,514],{"x":456,"y":508,"width":457,"height":484,"rx":441,"fill":444,"stroke":452,"style":453},[459,516,518],{"x":461,"y":511,"style":517},"font-size:10.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","SCR-03",[438,520],{"x":467,"y":508,"width":468,"height":484,"rx":441,"fill":444,"stroke":452,"style":453},[459,522,523],{"x":471,"y":511,"style":517},"Fixings\u002FSmall",[438,525],{"x":475,"y":508,"width":476,"height":484,"rx":441,"fill":526,"stroke":527,"style":528},"#fce9e9","var(--danger,#dc2626)","stroke-width:1.5px",[438,530],{"x":447,"y":531,"width":449,"height":484,"rx":441,"fill":451,"stroke":452,"style":453},"120.0",[459,533,535],{"x":487,"y":534,"style":463},"139.0","3",[438,537],{"x":456,"y":531,"width":457,"height":484,"rx":441,"fill":444,"stroke":452,"style":453},[459,539,540],{"x":461,"y":534,"style":517},"DRL-04",[438,542],{"x":467,"y":531,"width":468,"height":484,"rx":441,"fill":444,"stroke":452,"style":453},[459,544,545],{"x":471,"y":534,"style":517},"Tools\u002FMedium",[438,547],{"x":475,"y":531,"width":476,"height":484,"rx":441,"fill":526,"stroke":527,"style":528},[438,549],{"x":447,"y":550,"width":449,"height":484,"rx":441,"fill":451,"stroke":452,"style":453},"148.0",[459,552,440],{"x":487,"y":553,"style":463},"167.0",[438,555],{"x":456,"y":550,"width":457,"height":484,"rx":441,"fill":444,"stroke":452,"style":453},[459,557,558],{"x":461,"y":553,"style":517},"GRV-01",[438,560],{"x":467,"y":550,"width":468,"height":484,"rx":441,"fill":444,"stroke":452,"style":453},[459,562,563],{"x":471,"y":553,"style":517},"Landsc.\u002FBulk",[438,565],{"x":475,"y":550,"width":476,"height":484,"rx":441,"fill":526,"stroke":527,"style":528},[438,567],{"x":568,"y":448,"width":569,"height":484,"rx":441,"fill":444,"stroke":452,"style":453},"587.8","65.0",[459,571,495],{"x":572,"y":573,"style":494},"620.3","59.0",[438,575],{"x":576,"y":448,"width":577,"height":484,"rx":441,"fill":444,"stroke":452,"style":453},"652.8","87.2",[459,579,505],{"x":580,"y":573,"style":494},"696.4",[438,582],{"x":568,"y":583,"width":569,"height":484,"rx":441,"fill":444,"stroke":452,"style":453},"68.0",[459,585,518],{"x":572,"y":586,"style":517},"87.0",[438,588],{"x":576,"y":583,"width":577,"height":484,"rx":441,"fill":589,"stroke":590,"style":528},"#d9f4f1","var(--teal,#0f9488)",[459,592,594],{"x":580,"y":586,"style":593},"font-size:10.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","0.225",[438,596],{"x":568,"y":597,"width":569,"height":484,"rx":441,"fill":444,"stroke":452,"style":453},"96.0",[459,599,540],{"x":572,"y":600,"style":517},"115.0",[438,602],{"x":576,"y":597,"width":577,"height":484,"rx":441,"fill":589,"stroke":590,"style":528},[459,604,605],{"x":580,"y":600,"style":593},"2.95",[438,607],{"x":568,"y":608,"width":569,"height":484,"rx":441,"fill":444,"stroke":452,"style":453},"124.0",[459,610,558],{"x":572,"y":611,"style":517},"143.0",[438,613],{"x":576,"y":608,"width":577,"height":484,"rx":441,"fill":614,"stroke":615,"style":528},"#fdefd8","var(--gold,#b4740a)",[459,617,619],{"x":580,"y":611,"style":618},"font-size:10.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","— still blank",[459,621,625],{"x":622,"y":623,"style":624},"139.7","29.0","font-size:12.5px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:middle","as exported",[459,627,629],{"x":628,"y":623,"style":624},"663.9","after fillna",[46,631],{"x1":632,"y1":597,"x2":633,"y2":597,"stroke":634,"style":635},"269.3","573.8","var(--brand,#5b5cf0)","stroke-width:2px",[637,638],"polygon",{"points":639,"fill":640},"577.8,96.0 568.8,101.0 568.8,91.0","#5b5cf0",[438,642],{"x":643,"y":644,"width":645,"height":646,"rx":647,"fill":444,"stroke":452,"style":648},"308.2","63.0","230.8","22.0","11","stroke-width:1.2px",[459,650,654],{"x":651,"y":652,"style":653},"423.6","78.0","font-size:11.5px;font-weight:400;fill:var(--brand-strong,#4338ca);text-anchor:middle;font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace","groupby().transform(\"median\")",[30,656,658],{"id":657},"step-3-a-fallback-chain-with-a-minimum-group-size","Step 3 — A fallback chain with a minimum group size",[10,660,661],{},"A sensible rule is: use the most specific group that has enough known values, otherwise step back to a broader one. Here, use the category-and-size median if that group has at least two known weights, otherwise the category median, otherwise leave the value blank and report it:",[35,663,665],{"className":77,"code":664,"language":79,"meta":40,"style":40},"MIN_KNOWN = 2\nlevels = [[\"Category\", \"Size\"], [\"Category\"]]\n\nproducts[\"Weight filled\"] = products[\"Weight kg\"]\nproducts[\"Filled from\"] = products[\"Weight kg\"].notna().map({True: \"known\", False: None})\n\nfor keys in levels:\n    g = products.groupby(keys)[\"Weight kg\"]\n    med = g.transform(\"median\")\n    enough = g.transform(\"count\") >= MIN_KNOWN\n    use = products[\"Weight filled\"].isna() & enough & med.notna()\n    products.loc[use, \"Weight filled\"] = med[use]\n    products.loc[use, \"Filled from\"] = \"median of \" + \"+\".join(keys).lower()\n\nprint(products[[\"SKU\", \"Category\", \"Size\", \"Weight kg\", \"Weight filled\", \"Filled from\"]].to_string(index=False))\nprint(\"still blank:\", list(products.loc[products[\"Weight filled\"].isna(), \"SKU\"]))\n",[19,666,667,678,702,706,723,761,765,779,793,807,827,852,867,890,895,934],{"__ignoreMap":40},[44,668,669,672,675],{"class":46,"line":47},[44,670,671],{"class":250},"MIN_KNOWN",[44,673,674],{"class":86}," =",[44,676,677],{"class":250}," 2\n",[44,679,680,683,685,688,690,692,694,697,699],{"class":46,"line":100},[44,681,682],{"class":90},"levels ",[44,684,113],{"class":86},[44,686,687],{"class":90}," [[",[44,689,136],{"class":54},[44,691,139],{"class":90},[44,693,142],{"class":54},[44,695,696],{"class":90},"], [",[44,698,136],{"class":54},[44,700,701],{"class":90},"]]\n",[44,703,704],{"class":46,"line":107},[44,705,104],{"emptyLinePlaceholder":103},[44,707,708,710,713,715,717,719,721],{"class":46,"line":125},[44,709,348],{"class":90},[44,711,712],{"class":54},"\"Weight filled\"",[44,714,223],{"class":90},[44,716,113],{"class":86},[44,718,358],{"class":90},[44,720,160],{"class":54},[44,722,244],{"class":90},[44,724,725,727,730,732,734,736,738,741,744,747,749,751,753,755,758],{"class":46,"line":148},[44,726,348],{"class":90},[44,728,729],{"class":54},"\"Filled from\"",[44,731,223],{"class":90},[44,733,113],{"class":86},[44,735,358],{"class":90},[44,737,160],{"class":54},[44,739,740],{"class":90},"].notna().map({",[44,742,743],{"class":250},"True",[44,745,746],{"class":90},": ",[44,748,241],{"class":54},[44,750,139],{"class":90},[44,752,400],{"class":250},[44,754,746],{"class":90},[44,756,757],{"class":250},"None",[44,759,760],{"class":90},"})\n",[44,762,763],{"class":46,"line":171},[44,764,104],{"emptyLinePlaceholder":103},[44,766,767,770,773,776],{"class":46,"line":190},[44,768,769],{"class":86},"for",[44,771,772],{"class":90}," keys ",[44,774,775],{"class":86},"in",[44,777,778],{"class":90}," levels:\n",[44,780,781,784,786,789,791],{"class":46,"line":209},[44,782,783],{"class":90},"    g ",[44,785,113],{"class":86},[44,787,788],{"class":90}," products.groupby(keys)[",[44,790,160],{"class":54},[44,792,244],{"class":90},[44,794,795,798,800,803,805],{"class":46,"line":214},[44,796,797],{"class":90},"    med ",[44,799,113],{"class":86},[44,801,802],{"class":90}," g.transform(",[44,804,204],{"class":54},[44,806,122],{"class":90},[44,808,809,812,814,816,818,821,824],{"class":46,"line":247},[44,810,811],{"class":90},"    enough ",[44,813,113],{"class":86},[44,815,802],{"class":90},[44,817,185],{"class":54},[44,819,820],{"class":90},") ",[44,822,823],{"class":86},">=",[44,825,826],{"class":250}," MIN_KNOWN\n",[44,828,829,832,834,836,838,841,844,847,849],{"class":46,"line":257},[44,830,831],{"class":90},"    use ",[44,833,113],{"class":86},[44,835,358],{"class":90},[44,837,712],{"class":54},[44,839,840],{"class":90},"].isna() ",[44,842,843],{"class":86},"&",[44,845,846],{"class":90}," enough ",[44,848,843],{"class":86},[44,850,851],{"class":90}," med.notna()\n",[44,853,855,858,860,862,864],{"class":46,"line":854},12,[44,856,857],{"class":90},"    products.loc[use, ",[44,859,712],{"class":54},[44,861,223],{"class":90},[44,863,113],{"class":86},[44,865,866],{"class":90}," med[use]\n",[44,868,870,872,874,876,878,881,884,887],{"class":46,"line":869},13,[44,871,857],{"class":90},[44,873,729],{"class":54},[44,875,223],{"class":90},[44,877,113],{"class":86},[44,879,880],{"class":54}," \"median of \"",[44,882,883],{"class":86}," +",[44,885,886],{"class":54}," \"+\"",[44,888,889],{"class":90},".join(keys).lower()\n",[44,891,893],{"class":46,"line":892},14,[44,894,104],{"emptyLinePlaceholder":103},[44,896,898,900,902,904,906,908,910,912,914,916,918,920,922,924,926,928,930,932],{"class":46,"line":897},15,[44,899,251],{"class":250},[44,901,370],{"class":90},[44,903,373],{"class":54},[44,905,139],{"class":90},[44,907,136],{"class":54},[44,909,139],{"class":90},[44,911,142],{"class":54},[44,913,139],{"class":90},[44,915,160],{"class":54},[44,917,139],{"class":90},[44,919,712],{"class":54},[44,921,139],{"class":90},[44,923,729],{"class":54},[44,925,392],{"class":90},[44,927,395],{"class":151},[44,929,113],{"class":86},[44,931,400],{"class":250},[44,933,403],{"class":90},[44,935,937,939,941,944,946,949,952,954,957,959],{"class":46,"line":936},16,[44,938,251],{"class":250},[44,940,157],{"class":90},[44,942,943],{"class":54},"\"still blank:\"",[44,945,139],{"class":90},[44,947,948],{"class":250},"list",[44,950,951],{"class":90},"(products.loc[products[",[44,953,712],{"class":54},[44,955,956],{"class":90},"].isna(), ",[44,958,373],{"class":54},[44,960,961],{"class":90},"]))\n",[10,963,964,965,968],{},"Each pass only touches rows that are still blank, so the more specific level wins whenever it qualifies. Tools\u002FLarge has only one known weight, so SAW-02 falls back to the Tools median of 3.025 kg — less than half the one saw we do know (6.2 kg), because the drills dominate the category. That is the fallback working as specified and still giving a poor estimate, and it is exactly why the ",[19,966,967],{},"Filled from"," column matters; Garden\u002FMedium has one known value and the whole Garden category has the same single value, so HOS-02 stays blank rather than being filled from one data point. The bulk gravel rows reach the Landscaping category median — but two slabs at 21–23.5 kg tell you nothing about a bulk bag of gravel, so the fallback chain is only as good as its levels. Whether a fallback is reasonable is a judgement about the data, and it is the part of this page worth discussing with whoever owns the figures.",[419,970,428,975,428,978,428,981,428,984,428,991,428,1002,428,1008,428,1013,428,1018,428,1021,428,1024,428,1031,428,1036,428,1038,428,1042,428,1045,428,1048,428,1055,428,1060],{"viewBox":971,"role":422,"ariaLabelledBy":972,"xmlns":426,"style":427},"4 0 752 125",[973,974],"imp-chain-t","imp-chain-d",[430,976,977],{"id":973},"The fallback chain for one blank value",[434,979,980],{"id":974},"For a blank weight, first try the median of the same category and size, if at least two weights are known there. If not, try the median of the whole category with the same rule. If neither qualifies, leave the value blank and report it.",[438,982],{"x":440,"y":441,"width":442,"height":983,"fill":444},"125",[438,985],{"x":447,"y":986,"width":987,"height":988,"rx":989,"fill":990,"stroke":634,"style":635},"16.0","216.0","93.0","12","#f0f4ff",[992,993,995],"g",{"transform":994},"translate(116.0,29.0) scale(1.000)",[996,997],"path",{"d":998,"fill":999,"stroke":1000,"style":1001},"M12 3l9 5-9 5-9-5z M3 13l9 5 9-5 M3 17l9 5 9-5","none","var(--brand-strong,#4338ca)","stroke-width:1.80px;stroke-linecap:round;stroke-linejoin:round",[459,1003,1007],{"x":1004,"y":1005,"style":1006},"128.0","75.5","font-size:13.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Category + size",[459,1009,1012],{"x":1004,"y":1010,"style":1011},"95.0","font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","median if ≥ 2 known",[46,1014],{"x1":1015,"y1":1016,"x2":1017,"y2":1016,"stroke":634,"style":635},"241.0","62.5","263.0",[637,1019],{"points":1020,"fill":640},"267.0,62.5 258.0,67.5 258.0,57.5",[438,1022],{"x":1023,"y":986,"width":987,"height":988,"rx":989,"fill":589,"stroke":590,"style":635},"272.0",[992,1025,1027],{"transform":1026},"translate(368.0,29.0) scale(1.000)",[996,1028],{"d":1029,"fill":999,"stroke":1030,"style":1001},"M3 5h18v14H3z M3 10h18 M3 15h18 M9 5v14","var(--teal-ink,#0b6157)",[459,1032,1035],{"x":1033,"y":1005,"style":1034},"380.0","font-size:13.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","Category only",[459,1037,1012],{"x":1033,"y":1010,"style":1011},[46,1039],{"x1":1040,"y1":1016,"x2":1041,"y2":1016,"stroke":634,"style":635},"493.0","515.0",[637,1043],{"points":1044,"fill":640},"519.0,62.5 510.0,67.5 510.0,57.5",[438,1046],{"x":1047,"y":986,"width":987,"height":988,"rx":989,"fill":614,"stroke":615,"style":635},"524.0",[992,1049,1051],{"transform":1050},"translate(620.0,29.0) scale(1.000)",[996,1052],{"d":1053,"fill":999,"stroke":1054,"style":1001},"M12 3.5l9.5 17h-19z M12 10v5 M12 17.8v.4","var(--gold-ink,#7a4e06)",[459,1056,1059],{"x":1057,"y":1005,"style":1058},"632.0","font-size:13.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","Leave blank",[459,1061,1062],{"x":1057,"y":1010,"style":1011},"report for review",[30,1064,1066],{"id":1065},"step-4-keep-a-flag-on-every-filled-cell","Step 4 — Keep a flag on every filled cell",[10,1068,1069,1070,1072],{},"An imputed number looks exactly like a measured one once it is in a spreadsheet. Keep the ",[19,1071,967],{}," column in the output and colour the imputed cells, so nobody mistakes an estimate for a fact:",[35,1074,1076],{"className":77,"code":1075,"language":79,"meta":40,"style":40},"from openpyxl.styles import Font, PatternFill\n\nout = products[[\"SKU\", \"Category\", \"Size\", \"Weight filled\", \"Filled from\"]].rename(columns={\"Weight filled\": \"Weight kg\"})\nwith pd.ExcelWriter(\"products_filled.xlsx\", engine=\"openpyxl\") as xw:\n    out.to_excel(xw, sheet_name=\"Products\", index=False)\n    ws = xw.sheets[\"Products\"]\n    grey = PatternFill(\"solid\", fgColor=\"EDEDED\")\n    for r, how in enumerate(out[\"Filled from\"], start=2):\n        if how and how != \"known\":\n            ws.cell(r, 4).fill = grey\n            ws.cell(r, 4).font = Font(italic=True)\n    for c in ws[1]:\n        c.font = Font(bold=True)\n    ws.column_dimensions[\"E\"].width = 28\nprint(out[\"Filled from\"].value_counts(dropna=False).to_dict())\n",[19,1077,1078,1091,1095,1142,1170,1193,1207,1232,1263,1285,1300,1323,1340,1358,1374],{"__ignoreMap":40},[44,1079,1080,1083,1086,1088],{"class":46,"line":47},[44,1081,1082],{"class":86},"from",[44,1084,1085],{"class":90}," openpyxl.styles ",[44,1087,87],{"class":86},[44,1089,1090],{"class":90}," Font, PatternFill\n",[44,1092,1093],{"class":46,"line":100},[44,1094,104],{"emptyLinePlaceholder":103},[44,1096,1097,1100,1102,1105,1107,1109,1111,1113,1115,1117,1119,1121,1123,1126,1129,1131,1134,1136,1138,1140],{"class":46,"line":107},[44,1098,1099],{"class":90},"out ",[44,1101,113],{"class":86},[44,1103,1104],{"class":90}," products[[",[44,1106,373],{"class":54},[44,1108,139],{"class":90},[44,1110,136],{"class":54},[44,1112,139],{"class":90},[44,1114,142],{"class":54},[44,1116,139],{"class":90},[44,1118,712],{"class":54},[44,1120,139],{"class":90},[44,1122,729],{"class":54},[44,1124,1125],{"class":90},"]].rename(",[44,1127,1128],{"class":151},"columns",[44,1130,113],{"class":86},[44,1132,1133],{"class":90},"{",[44,1135,712],{"class":54},[44,1137,746],{"class":90},[44,1139,160],{"class":54},[44,1141,760],{"class":90},[44,1143,1144,1147,1150,1153,1155,1158,1160,1163,1165,1167],{"class":46,"line":125},[44,1145,1146],{"class":86},"with",[44,1148,1149],{"class":90}," pd.ExcelWriter(",[44,1151,1152],{"class":54},"\"products_filled.xlsx\"",[44,1154,139],{"class":90},[44,1156,1157],{"class":151},"engine",[44,1159,113],{"class":86},[44,1161,1162],{"class":54},"\"openpyxl\"",[44,1164,820],{"class":90},[44,1166,94],{"class":86},[44,1168,1169],{"class":90}," xw:\n",[44,1171,1172,1175,1178,1180,1183,1185,1187,1189,1191],{"class":46,"line":148},[44,1173,1174],{"class":90},"    out.to_excel(xw, ",[44,1176,1177],{"class":151},"sheet_name",[44,1179,113],{"class":86},[44,1181,1182],{"class":54},"\"Products\"",[44,1184,139],{"class":90},[44,1186,395],{"class":151},[44,1188,113],{"class":86},[44,1190,400],{"class":250},[44,1192,122],{"class":90},[44,1194,1195,1198,1200,1203,1205],{"class":46,"line":171},[44,1196,1197],{"class":90},"    ws ",[44,1199,113],{"class":86},[44,1201,1202],{"class":90}," xw.sheets[",[44,1204,1182],{"class":54},[44,1206,244],{"class":90},[44,1208,1209,1212,1214,1217,1220,1222,1225,1227,1230],{"class":46,"line":190},[44,1210,1211],{"class":90},"    grey ",[44,1213,113],{"class":86},[44,1215,1216],{"class":90}," PatternFill(",[44,1218,1219],{"class":54},"\"solid\"",[44,1221,139],{"class":90},[44,1223,1224],{"class":151},"fgColor",[44,1226,113],{"class":86},[44,1228,1229],{"class":54},"\"EDEDED\"",[44,1231,122],{"class":90},[44,1233,1234,1237,1240,1242,1245,1248,1250,1253,1256,1258,1260],{"class":46,"line":209},[44,1235,1236],{"class":86},"    for",[44,1238,1239],{"class":90}," r, how ",[44,1241,775],{"class":86},[44,1243,1244],{"class":250}," enumerate",[44,1246,1247],{"class":90},"(out[",[44,1249,729],{"class":54},[44,1251,1252],{"class":90},"], ",[44,1254,1255],{"class":151},"start",[44,1257,113],{"class":86},[44,1259,512],{"class":250},[44,1261,1262],{"class":90},"):\n",[44,1264,1265,1268,1271,1274,1276,1279,1282],{"class":46,"line":214},[44,1266,1267],{"class":86},"        if",[44,1269,1270],{"class":90}," how ",[44,1272,1273],{"class":86},"and",[44,1275,1270],{"class":90},[44,1277,1278],{"class":86},"!=",[44,1280,1281],{"class":54}," \"known\"",[44,1283,1284],{"class":90},":\n",[44,1286,1287,1290,1292,1295,1297],{"class":46,"line":247},[44,1288,1289],{"class":90},"            ws.cell(r, ",[44,1291,440],{"class":250},[44,1293,1294],{"class":90},").fill ",[44,1296,113],{"class":86},[44,1298,1299],{"class":90}," grey\n",[44,1301,1302,1304,1306,1309,1311,1314,1317,1319,1321],{"class":46,"line":257},[44,1303,1289],{"class":90},[44,1305,440],{"class":250},[44,1307,1308],{"class":90},").font ",[44,1310,113],{"class":86},[44,1312,1313],{"class":90}," Font(",[44,1315,1316],{"class":151},"italic",[44,1318,113],{"class":86},[44,1320,743],{"class":250},[44,1322,122],{"class":90},[44,1324,1325,1327,1330,1332,1335,1337],{"class":46,"line":854},[44,1326,1236],{"class":86},[44,1328,1329],{"class":90}," c ",[44,1331,775],{"class":86},[44,1333,1334],{"class":90}," ws[",[44,1336,489],{"class":250},[44,1338,1339],{"class":90},"]:\n",[44,1341,1342,1345,1347,1349,1352,1354,1356],{"class":46,"line":869},[44,1343,1344],{"class":90},"        c.font ",[44,1346,113],{"class":86},[44,1348,1313],{"class":90},[44,1350,1351],{"class":151},"bold",[44,1353,113],{"class":86},[44,1355,743],{"class":250},[44,1357,122],{"class":90},[44,1359,1360,1363,1366,1369,1371],{"class":46,"line":892},[44,1361,1362],{"class":90},"    ws.column_dimensions[",[44,1364,1365],{"class":54},"\"E\"",[44,1367,1368],{"class":90},"].width ",[44,1370,113],{"class":86},[44,1372,1373],{"class":250}," 28\n",[44,1375,1376,1378,1380,1382,1385,1388,1390,1392],{"class":46,"line":897},[44,1377,251],{"class":250},[44,1379,1247],{"class":90},[44,1381,729],{"class":54},[44,1383,1384],{"class":90},"].value_counts(",[44,1386,1387],{"class":151},"dropna",[44,1389,113],{"class":86},[44,1391,400],{"class":250},[44,1393,1394],{"class":90},").to_dict())\n",[10,1396,1397,1398,1400],{},"Imputed weights are shaded grey and italic, and the ",[19,1399,967],{}," column says exactly which group's median was used. If the filled file feeds another process, the flag column also lets that process exclude estimated values where accuracy matters — a freight quote, for example, should probably not be based on an imputed weight.",[30,1402,1404],{"id":1403},"step-5-check-what-imputation-did-to-the-totals","Step 5 — Check what imputation did to the totals",[10,1406,1407],{},"Imputation changes any total or average built on the column. Compare the figures with and without filled values, per category, so the effect is visible before anyone reports them:",[35,1409,1411],{"className":77,"code":1410,"language":79,"meta":40,"style":40},"compare = products.groupby(\"Category\").agg(\n    known_rows=(\"Weight kg\", \"count\"),\n    sum_known=(\"Weight kg\", \"sum\"),\n    sum_filled=(\"Weight filled\", \"sum\"),\n    mean_known=(\"Weight kg\", \"mean\"),\n    mean_filled=(\"Weight filled\", \"mean\"),\n).round(3)\ncompare[\"sum change %\"] = ((compare[\"sum_filled\"] \u002F compare[\"sum_known\"] - 1) * 100).round(1)\nprint(compare.to_string())\n",[19,1412,1413,1428,1445,1463,1480,1498,1515,1524,1574],{"__ignoreMap":40},[44,1414,1415,1418,1420,1423,1425],{"class":46,"line":47},[44,1416,1417],{"class":90},"compare ",[44,1419,113],{"class":86},[44,1421,1422],{"class":90}," products.groupby(",[44,1424,136],{"class":54},[44,1426,1427],{"class":90},").agg(\n",[44,1429,1430,1433,1435,1437,1439,1441,1443],{"class":46,"line":100},[44,1431,1432],{"class":151},"    known_rows",[44,1434,113],{"class":86},[44,1436,157],{"class":90},[44,1438,160],{"class":54},[44,1440,139],{"class":90},[44,1442,185],{"class":54},[44,1444,168],{"class":90},[44,1446,1447,1450,1452,1454,1456,1458,1461],{"class":46,"line":107},[44,1448,1449],{"class":151},"    sum_known",[44,1451,113],{"class":86},[44,1453,157],{"class":90},[44,1455,160],{"class":54},[44,1457,139],{"class":90},[44,1459,1460],{"class":54},"\"sum\"",[44,1462,168],{"class":90},[44,1464,1465,1468,1470,1472,1474,1476,1478],{"class":46,"line":125},[44,1466,1467],{"class":151},"    sum_filled",[44,1469,113],{"class":86},[44,1471,157],{"class":90},[44,1473,712],{"class":54},[44,1475,139],{"class":90},[44,1477,1460],{"class":54},[44,1479,168],{"class":90},[44,1481,1482,1485,1487,1489,1491,1493,1496],{"class":46,"line":148},[44,1483,1484],{"class":151},"    mean_known",[44,1486,113],{"class":86},[44,1488,157],{"class":90},[44,1490,160],{"class":54},[44,1492,139],{"class":90},[44,1494,1495],{"class":54},"\"mean\"",[44,1497,168],{"class":90},[44,1499,1500,1503,1505,1507,1509,1511,1513],{"class":46,"line":171},[44,1501,1502],{"class":151},"    mean_filled",[44,1504,113],{"class":86},[44,1506,157],{"class":90},[44,1508,712],{"class":54},[44,1510,139],{"class":90},[44,1512,1495],{"class":54},[44,1514,168],{"class":90},[44,1516,1517,1520,1522],{"class":46,"line":190},[44,1518,1519],{"class":90},").round(",[44,1521,535],{"class":250},[44,1523,122],{"class":90},[44,1525,1526,1529,1532,1534,1536,1539,1542,1544,1547,1550,1553,1555,1557,1560,1562,1565,1568,1570,1572],{"class":46,"line":209},[44,1527,1528],{"class":90},"compare[",[44,1530,1531],{"class":54},"\"sum change %\"",[44,1533,223],{"class":90},[44,1535,113],{"class":86},[44,1537,1538],{"class":90}," ((compare[",[44,1540,1541],{"class":54},"\"sum_filled\"",[44,1543,223],{"class":90},[44,1545,1546],{"class":86},"\u002F",[44,1548,1549],{"class":90}," compare[",[44,1551,1552],{"class":54},"\"sum_known\"",[44,1554,223],{"class":90},[44,1556,236],{"class":86},[44,1558,1559],{"class":250}," 1",[44,1561,820],{"class":90},[44,1563,1564],{"class":86},"*",[44,1566,1567],{"class":250}," 100",[44,1569,1519],{"class":90},[44,1571,489],{"class":250},[44,1573,122],{"class":90},[44,1575,1576,1578],{"class":46,"line":214},[44,1577,251],{"class":250},[44,1579,1580],{"class":90},"(compare.to_string())\n",[10,1582,1583,1584,1587],{},"Group-median imputation leaves a group's mean almost unchanged when the fill comes from that same group — Fixings, Landscaping and Garden keep their means exactly — but it ",[14,1585,1586],{},"raises totals",", because rows that previously contributed nothing to a sum now contribute an estimate. Fixings' total weight rises by 50% and Landscaping's by 150%, almost all of it from the two gravel bags filled with the slab median. Tools shows the other risk: the saw filled from the drill-heavy category median pulls the Tools mean down from 3.76 to 3.50 kg. Whether such changes are acceptable depends on the use: for a typical weight per category they may not matter, for total shipping weight they matter a lot. Report the share of imputed values alongside any figure built on them.",[419,1589,428,1594,428,1597,428,1600,428,1603,428,1608,428,1615,428,1620,428,1624,428,1628,428,1632,428,1636,428,1639,428,1644,428,1649,428,1653,428,1656,428,1659],{"viewBox":1590,"role":422,"ariaLabelledBy":1591,"xmlns":426,"style":427},"4 0 752 207",[1592,1593],"imp-effect-t","imp-effect-d",[430,1595,1596],{"id":1592},"What group-median imputation does to summaries",[434,1598,1599],{"id":1593},"Group means do not move when blanks are filled from the same group's median, but fallbacks from a broader group can shift them, as for Tools. Group totals rise because blank rows now add an estimated value; where a fallback filled a whole subgroup the change is large, as for Landscaping at 150 percent.",[438,1601],{"x":440,"y":441,"width":442,"height":1602,"fill":444},"207",[438,1604],{"x":447,"y":986,"width":1605,"height":1606,"rx":1607,"fill":589,"stroke":590,"style":635},"351.0","175.0","14",[992,1609,1611],{"transform":1610},"translate(182.5,29.0) scale(1.083)",[996,1612],{"d":1613,"fill":999,"stroke":1030,"style":1614},"M5 12.5l4.5 4.5L19 7","stroke-width:1.66px;stroke-linecap:round;stroke-linejoin:round",[459,1616,1619],{"x":1617,"y":652,"style":1618},"195.5","font-size:14px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","means",[46,1621],{"x1":487,"y1":1622,"x2":1623,"y2":1622,"stroke":590,"style":453},"91.0","355.0",[459,1625,1627],{"x":1617,"y":511,"style":1626},"font-size:12px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","unchanged when filled from own group",[459,1629,1631],{"x":1617,"y":1630,"style":1626},"137.0","fallbacks can shift them (Tools 3.76 → 3.50)",[459,1633,1635],{"x":1617,"y":1634,"style":1626},"163.0","fine for typical-value reporting",[438,1637],{"x":1638,"y":986,"width":1605,"height":1606,"rx":1607,"fill":614,"stroke":615,"style":635},"389.0",[992,1640,1642],{"transform":1641},"translate(551.5,29.0) scale(1.083)",[996,1643],{"d":1053,"fill":999,"stroke":1054,"style":1614},[459,1645,1648],{"x":1646,"y":652,"style":1647},"564.5","font-size:14px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","totals",[46,1650],{"x1":1651,"y1":1622,"x2":1652,"y2":1622,"stroke":615,"style":453},"405.0","724.0",[459,1654,1655],{"x":1646,"y":511,"style":1626},"rise with every filled row",[459,1657,1658],{"x":1646,"y":1630,"style":1626},"Landscaping total up 150%",[459,1660,1661],{"x":1646,"y":1634,"style":1626},"report the imputed share",[30,1663,1665],{"id":1664},"common-pitfalls","Common pitfalls",[1667,1668,1669,1677,1683,1689,1695],"ul",{},[1670,1671,1672,1676],"li",{},[1673,1674,1675],"strong",{},"One column-wide average."," It ignores obvious structure; impute within groups.",[1670,1678,1679,1682],{},[1673,1680,1681],{},"Trusting a group of one."," Require a minimum number of known values.",[1670,1684,1685,1688],{},[1673,1686,1687],{},"Fallbacks across unlike items."," A broader group is only useful if its items are comparable.",[1670,1690,1691,1694],{},[1673,1692,1693],{},"Unflagged estimates."," Keep a column saying what was filled and how.",[1670,1696,1697,1700],{},[1673,1698,1699],{},"Reporting totals without saying they include estimates."," Show the imputed share.",[30,1702,1704],{"id":1703},"conclusion","Conclusion",[10,1706,1707,1708,1711,1712,1714,1715,1717],{},"Group-wise imputation fills each blank with a typical value from comparable rows: ",[19,1709,1710],{},"groupby(keys)[col].transform(\"median\")"," lines up a group median with every row, and ",[19,1713,310],{}," uses it. Make it robust with a fallback chain from specific to broader groups, a minimum count of known values before trusting a group, and a ",[19,1716,967],{}," flag on every estimate. Then compare sums and means with and without the filled values, because imputation barely moves averages but can noticeably raise totals.",[30,1719,1721],{"id":1720},"frequently-asked-questions","Frequently asked questions",[10,1723,1724,1727],{},[1673,1725,1726],{},"How do I fill missing values with the group mean in pandas?","\nGroup the column by the key columns, call transform(\"mean\"), and pass the result to fillna on the same column. transform returns a value per row aligned with the original data, so each blank gets its own group's mean.",[10,1729,1730,1733],{},[1673,1731,1732],{},"Should I use the mean or the median to impute?","\nThe median is usually safer for weights, prices and durations because it is not pulled by a few extreme values. Use the mean when the data is roughly symmetric.",[10,1735,1736,1739],{},[1673,1737,1738],{},"What happens when a whole group is missing?","\nIts median is NaN, so the values stay blank. Add a fallback to a broader group, or leave them blank and report them for review.",[10,1741,1742,1745],{},[1673,1743,1744],{},"Does imputation change my totals?","\nYes. Filled rows now contribute to sums, so totals rise. Group means change little. Report how many values were imputed alongside any figure that uses them.",[30,1747,1749],{"id":1748},"related","Related",[1667,1751,1752,1758,1765,1772],{},[1670,1753,1754,1755,1757],{},"Up one level: ",[24,1756,27],{"href":26}," — finding and treating gaps.",[1670,1759,1760,1764],{},[24,1761,1763],{"href":1762},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Ffill-missing-values-in-excel-with-pandas-fillna\u002F","Fill Missing Values in Excel with pandas fillna"," — the simpler fills.",[1670,1766,1767,1771],{},[24,1768,1770],{"href":1769},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Finterpolate-missing-numeric-values-in-excel-data\u002F","Interpolate Missing Numeric Values in Excel Data"," — filling along a sequence instead of a group.",[1670,1773,1774,1778],{},[24,1775,1777],{"href":1776},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fhow-missing-values-change-sums-averages-and-counts\u002F","How Missing Values Change Sums, Averages and Counts"," — the effect of blanks before you fill them.",[1780,1781,1782],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}",{"title":40,"searchDepth":100,"depth":100,"links":1784},[1785,1786,1787,1788,1789,1790,1791,1792,1793,1794],{"id":32,"depth":100,"text":33},{"id":70,"depth":100,"text":71},{"id":300,"depth":100,"text":301},{"id":657,"depth":100,"text":658},{"id":1065,"depth":100,"text":1066},{"id":1403,"depth":100,"text":1404},{"id":1664,"depth":100,"text":1665},{"id":1703,"depth":100,"text":1704},{"id":1720,"depth":100,"text":1721},{"id":1748,"depth":100,"text":1749},"2026-10-02","Fill gaps in Excel data with values that make sense for each row's group rather than one column-wide average: group means and medians with groupby transform, a fallback chain when a whole group is empty, minimum group sizes, flag columns that record what was filled, and a check that imputation did not distort the totals you report.","md",[1799,1801,1803,1805],{"q":1726,"a":1800},"Group the column by the key columns, call transform(\"mean\"), and pass the result to fillna on the same column. transform returns a value per row aligned with the original data, so each blank gets its own group's mean.",{"q":1732,"a":1802},"The median is usually safer for weights, prices and durations because it is not pulled by a few extreme values. Use the mean when the data is roughly symmetric.",{"q":1738,"a":1804},"Its median is NaN, so the values stay blank. Add a fallback to a broader group, or leave them blank and report them for review.",{"q":1744,"a":1806},"Yes. Filled rows now contribute to sums, so totals rise. Group means change little. Report how many values were imputed alongside any figure that uses them.","impute missing values by group with pandas",null,{"breadcrumb":1810},[1811,1813,1816],{"name":1812,"item":1546},"Home",{"name":1814,"item":1815},"Advanced Data Transformation and Cleaning for Excel Automation","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":1817,"item":26},"Handling Missing Data in Excel Reports with Pandas","\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fimpute-missing-values-by-group-with-pandas",{"title":1820,"description":1821},"Impute Missing Values by Group in Excel Data with pandas","Fill blanks with each group's median or mean using groupby transform, fall back to a wider group when a group is empty, require a minimum group size, flag imputed cells and check totals before and after.","impute-missing-values-by-group-with-pandas","advanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fimpute-missing-values-by-group-with-pandas\u002Findex",[1825,1826],"missing-data","pandas","how-to","gZOJGqgzHLcwiidAjEGW8o8HErsU53a3Lx4iz_vU5pE",[1830,1833],{"title":1777,"path":1831,"stem":1832,"children":-1},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fhow-missing-values-change-sums-averages-and-counts","advanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fhow-missing-values-change-sums-averages-and-counts\u002Findex",{"title":1770,"path":1834,"stem":1835,"children":-1},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Finterpolate-missing-numeric-values-in-excel-data","advanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Finterpolate-missing-numeric-values-in-excel-data\u002Findex",[1837,1839],{"path":1838,"title":1817},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports",{"path":1840,"title":1814},"\u002Fadvanced-data-transformation-and-cleaning",[],[1843,1846,1849,1852,1855,1858],{"path":1844,"title":1845,"kind":1827},"\u002Fadvanced-data-transformation-and-cleaning\u002Fapplying-conditional-formatting-with-openpyxl\u002Fhighlight-blank-cells-with-openpyxl","Highlight Blank Cells in Excel with openpyxl",{"path":1847,"title":1848,"kind":1827},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fcoalesce-several-excel-columns-into-one-with-pandas","Coalesce Several Excel Columns into One with pandas",{"path":1850,"title":1851,"kind":1827},"\u002Fadvanced-data-transformation-and-cleaning\u002Fexcel-formula-equivalents-in-pandas\u002Fiferror-and-isna-equivalents-in-pandas","IFERROR, IFNA, ISNA and ISNUMBER Equivalents in pandas",{"path":1853,"title":1854,"kind":1827},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fdrop-rows-with-missing-required-fields-in-pandas","Drop Rows with Missing Required Fields in pandas",{"path":1856,"title":1857,"kind":1827},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Ffill-gaps-in-per-item-time-series-with-a-staleness-limit","Fill Gaps in Per-Item Time Series with a Staleness Limit",{"path":1859,"title":1860,"kind":1827},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Ffill-missing-values-in-excel-with-pandas-fillna","Fill Missing Values in Excel with Pandas fillna",1790972096025]