[{"data":1,"prerenderedAt":2111},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fvisualise-missing-data-patterns-in-an-excel-file":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fvisualise-missing-data-patterns-in-an-excel-file":2077,"ancestors:\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fvisualise-missing-data-patterns-in-an-excel-file":2086,"children:\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fvisualise-missing-data-patterns-in-an-excel-file":2091,"related:\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fvisualise-missing-data-patterns-in-an-excel-file":2092},{"id":4,"title":5,"body":6,"dateModified":2043,"datePublished":2043,"description":2044,"extension":2045,"faq":2046,"keyword":2055,"kind":2056,"meta":2057,"navigation":99,"path":2066,"seo":2067,"slug":2070,"stem":2071,"topics":2072,"type":2075,"__hash__":2076},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fvisualise-missing-data-patterns-in-an-excel-file\u002Findex.md","Visualise Missing Data Patterns in an Excel File",{"type":7,"value":8,"toc":2031},"minimark",[9,24,29,58,64,68,71,241,260,264,274,868,871,1195,1199,1206,1335,1342,1346,1356,1468,1471,1581,1585,1588,1776,1788,1917,1921,1956,1960,1966,1970,1976,1982,1988,1994,1998,2027],[10,11,12,13,17,18,23],"p",{},"\"12% of the values are missing\" is the least useful thing you can say about missing data. What matters is the ",[14,15,16],"em",{},"pattern",". Blanks scattered at random through a column are a nuisance; blanks that all start in March point to a broken export; blanks that always occur in three columns together mean a whole form section was skipped; blanks concentrated in one branch's data mean one team is not filling something in. Each pattern has a different fix, and none of them is visible from a percentage. This page builds a missing-data report in Excel with pandas and openpyxl: a shaded map of every blank cell, the most frequent combinations of missing columns, missing rates over time and by source, and pairs of columns that go missing together. It belongs to ",[19,20,22],"a",{"href":21},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002F","Handling Missing Data in Excel Reports",".",[25,26,28],"h2",{"id":27},"prerequisites","Prerequisites",[30,31,36],"pre",{"className":32,"code":33,"language":34,"meta":35,"style":35},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl\n","bash","",[37,38,39],"code",{"__ignoreMap":35},[40,41,44,48,52,55],"span",{"class":42,"line":43},"line",1,[40,45,47],{"class":46},"sMTad","pip",[40,49,51],{"class":50},"srMev"," install",[40,53,54],{"class":50}," pandas",[40,56,57],{"class":50}," openpyxl\n",[10,59,60,63],{},[37,61,62],{},"orders.xlsx"," holds 240 orders from four branches between January and early October 2026, with ten columns. It has the kinds of gaps real exports have — and part of the point of this page is to find out what they are without being told.",[25,65,67],{"id":66},"step-1-the-headline-numbers-and-why-they-are-not-enough","Step 1 — The headline numbers, and why they are not enough",[10,69,70],{},"Start with the usual per-column count, so the more detailed views have something to explain:",[30,72,76],{"className":73,"code":74,"language":75,"meta":35,"style":35},"language-python shiki shiki-themes github-light github-dark-high-contrast","import pandas as pd\n\norders = pd.read_excel(\"orders.xlsx\")\nmissing = orders.isna()\nper_col = pd.DataFrame({\"missing\": missing.sum(), \"share %\": (missing.mean() * 100).round(1)})\nprint(per_col[per_col[\"missing\"] > 0].sort_values(\"missing\", ascending=False).to_string())\nprint(f\"overall: {missing.to_numpy().mean():.1%} of cells blank\")\n","python",[37,77,78,94,101,119,130,169,209],{"__ignoreMap":35},[40,79,80,84,88,91],{"class":42,"line":43},[40,81,83],{"class":82},"s-kum","import",[40,85,87],{"class":86},"skGVy"," pandas ",[40,89,90],{"class":82},"as",[40,92,93],{"class":86}," pd\n",[40,95,97],{"class":42,"line":96},2,[40,98,100],{"emptyLinePlaceholder":99},true,"\n",[40,102,104,107,110,113,116],{"class":42,"line":103},3,[40,105,106],{"class":86},"orders ",[40,108,109],{"class":82},"=",[40,111,112],{"class":86}," pd.read_excel(",[40,114,115],{"class":50},"\"orders.xlsx\"",[40,117,118],{"class":86},")\n",[40,120,122,125,127],{"class":42,"line":121},4,[40,123,124],{"class":86},"missing ",[40,126,109],{"class":82},[40,128,129],{"class":86}," orders.isna()\n",[40,131,133,136,138,141,144,147,150,153,156,160,163,166],{"class":42,"line":132},5,[40,134,135],{"class":86},"per_col ",[40,137,109],{"class":82},[40,139,140],{"class":86}," pd.DataFrame({",[40,142,143],{"class":50},"\"missing\"",[40,145,146],{"class":86},": missing.sum(), ",[40,148,149],{"class":50},"\"share %\"",[40,151,152],{"class":86},": (missing.mean() ",[40,154,155],{"class":82},"*",[40,157,159],{"class":158},"sP0c6"," 100",[40,161,162],{"class":86},").round(",[40,164,165],{"class":158},"1",[40,167,168],{"class":86},")})\n",[40,170,172,175,178,180,183,186,189,192,194,197,201,203,206],{"class":42,"line":171},6,[40,173,174],{"class":158},"print",[40,176,177],{"class":86},"(per_col[per_col[",[40,179,143],{"class":50},[40,181,182],{"class":86},"] ",[40,184,185],{"class":82},">",[40,187,188],{"class":158}," 0",[40,190,191],{"class":86},"].sort_values(",[40,193,143],{"class":50},[40,195,196],{"class":86},", ",[40,198,200],{"class":199},"sa561","ascending",[40,202,109],{"class":82},[40,204,205],{"class":158},"False",[40,207,208],{"class":86},").to_string())\n",[40,210,212,214,217,220,223,227,230,233,236,239],{"class":42,"line":211},7,[40,213,174],{"class":158},[40,215,216],{"class":86},"(",[40,218,219],{"class":82},"f",[40,221,222],{"class":50},"\"overall: ",[40,224,226],{"class":225},"sSjpA","{",[40,228,229],{"class":86},"missing.to_numpy().mean()",[40,231,232],{"class":82},":.1%",[40,234,235],{"class":225},"}",[40,237,238],{"class":50}," of cells blank\"",[40,240,118],{"class":86},[10,242,243,244,247,248,251,252,255,256,259],{},"Six columns have gaps, from 2% of Amount to 45% of Discount %, and about a tenth of all cells are blank. This tells you ",[14,245,246],{},"where"," to look, not ",[14,249,250],{},"why"," — Discount % being blank for almost half the rows could be people not entering it, a column that was retired, or a perfectly normal \"no discount\". The ",[37,253,254],{},"isna()"," frame built here is reused by every view below: one boolean per cell, ",[37,257,258],{},"True"," where the value is missing.",[25,261,263],{"id":262},"step-2-a-missingness-map-sheet","Step 2 — A missingness map sheet",[10,265,266,267,269,270,273],{},"The most direct picture is the data itself with every blank cell shaded. Writing ",[37,268,165],{}," for missing and ",[37,271,272],{},"0"," for present, with conditional formatting on top, gives a map Excel users can scroll, filter and sort:",[30,275,277],{"className":73,"code":276,"language":75,"meta":35,"style":35},"from openpyxl import Workbook\nfrom openpyxl.formatting.rule import CellIsRule\nfrom openpyxl.styles import Alignment, Font, PatternFill\n\nwb = Workbook()\nws = wb.active\nws.title = \"Missing map\"\ncols = list(orders.columns)\nws.append([\"Row\", \"Date\", \"Branch\"] + cols)\nfor i, (row_missing, date, branch) in enumerate(zip(missing.itertuples(index=False), orders[\"Date\"], orders[\"Branch\"]), start=2):\n    ws.append([i, date.date() if pd.notna(date) else None, branch] + [int(v) for v in row_missing])\nlast_col = ws.cell(1, ws.max_column).column_letter\nws.conditional_formatting.add(f\"D2:{last_col}{ws.max_row}\", CellIsRule(operator=\"equal\", formula=[\"1\"],\n                              fill=PatternFill(\"solid\", fgColor=\"D9534F\"), font=Font(color=\"D9534F\")))\nws.conditional_formatting.add(f\"D2:{last_col}{ws.max_row}\", CellIsRule(operator=\"equal\", formula=[\"0\"],\n                              fill=PatternFill(\"solid\", fgColor=\"E8EEF7\"), font=Font(color=\"E8EEF7\")))\nfor c in ws[1]:\n    c.font = Font(bold=True)\n    c.alignment = Alignment(text_rotation=90 if c.column > 3 else 0)\nfor col in range(4, ws.max_column + 1):\n    ws.column_dimensions[ws.cell(1, col).column_letter].width = 3.2\nws.freeze_panes = \"D2\"\nws.auto_filter.ref = ws.dimensions\nwb.save(\"missing_report.xlsx\")\nprint(\"map:\", ws.max_row - 1, \"rows ×\", len(cols), \"columns\")\n",[37,278,279,292,304,316,320,330,340,350,364,391,445,487,503,557,602,644,680,698,718,755,783,799,810,821,832],{"__ignoreMap":35},[40,280,281,284,287,289],{"class":42,"line":43},[40,282,283],{"class":82},"from",[40,285,286],{"class":86}," openpyxl ",[40,288,83],{"class":82},[40,290,291],{"class":86}," Workbook\n",[40,293,294,296,299,301],{"class":42,"line":96},[40,295,283],{"class":82},[40,297,298],{"class":86}," openpyxl.formatting.rule ",[40,300,83],{"class":82},[40,302,303],{"class":86}," CellIsRule\n",[40,305,306,308,311,313],{"class":42,"line":103},[40,307,283],{"class":82},[40,309,310],{"class":86}," openpyxl.styles ",[40,312,83],{"class":82},[40,314,315],{"class":86}," Alignment, Font, PatternFill\n",[40,317,318],{"class":42,"line":121},[40,319,100],{"emptyLinePlaceholder":99},[40,321,322,325,327],{"class":42,"line":132},[40,323,324],{"class":86},"wb ",[40,326,109],{"class":82},[40,328,329],{"class":86}," Workbook()\n",[40,331,332,335,337],{"class":42,"line":171},[40,333,334],{"class":86},"ws ",[40,336,109],{"class":82},[40,338,339],{"class":86}," wb.active\n",[40,341,342,345,347],{"class":42,"line":211},[40,343,344],{"class":86},"ws.title ",[40,346,109],{"class":82},[40,348,349],{"class":50}," \"Missing map\"\n",[40,351,353,356,358,361],{"class":42,"line":352},8,[40,354,355],{"class":86},"cols ",[40,357,109],{"class":82},[40,359,360],{"class":158}," list",[40,362,363],{"class":86},"(orders.columns)\n",[40,365,367,370,373,375,378,380,383,385,388],{"class":42,"line":366},9,[40,368,369],{"class":86},"ws.append([",[40,371,372],{"class":50},"\"Row\"",[40,374,196],{"class":86},[40,376,377],{"class":50},"\"Date\"",[40,379,196],{"class":86},[40,381,382],{"class":50},"\"Branch\"",[40,384,182],{"class":86},[40,386,387],{"class":82},"+",[40,389,390],{"class":86}," cols)\n",[40,392,394,397,400,403,406,408,411,414,417,419,421,424,426,429,431,434,437,439,442],{"class":42,"line":393},10,[40,395,396],{"class":82},"for",[40,398,399],{"class":86}," i, (row_missing, date, branch) ",[40,401,402],{"class":82},"in",[40,404,405],{"class":158}," enumerate",[40,407,216],{"class":86},[40,409,410],{"class":158},"zip",[40,412,413],{"class":86},"(missing.itertuples(",[40,415,416],{"class":199},"index",[40,418,109],{"class":82},[40,420,205],{"class":158},[40,422,423],{"class":86},"), orders[",[40,425,377],{"class":50},[40,427,428],{"class":86},"], orders[",[40,430,382],{"class":50},[40,432,433],{"class":86},"]), ",[40,435,436],{"class":199},"start",[40,438,109],{"class":82},[40,440,441],{"class":158},"2",[40,443,444],{"class":86},"):\n",[40,446,448,451,454,457,460,463,466,468,471,474,477,479,482,484],{"class":42,"line":447},11,[40,449,450],{"class":86},"    ws.append([i, date.date() ",[40,452,453],{"class":82},"if",[40,455,456],{"class":86}," pd.notna(date) ",[40,458,459],{"class":82},"else",[40,461,462],{"class":158}," None",[40,464,465],{"class":86},", branch] ",[40,467,387],{"class":82},[40,469,470],{"class":86}," [",[40,472,473],{"class":158},"int",[40,475,476],{"class":86},"(v) ",[40,478,396],{"class":82},[40,480,481],{"class":86}," v ",[40,483,402],{"class":82},[40,485,486],{"class":86}," row_missing])\n",[40,488,490,493,495,498,500],{"class":42,"line":489},12,[40,491,492],{"class":86},"last_col ",[40,494,109],{"class":82},[40,496,497],{"class":86}," ws.cell(",[40,499,165],{"class":158},[40,501,502],{"class":86},", ws.max_column).column_letter\n",[40,504,506,509,511,514,516,519,522,525,527,530,533,536,538,541,543,546,548,551,554],{"class":42,"line":505},13,[40,507,508],{"class":86},"ws.conditional_formatting.add(",[40,510,219],{"class":82},[40,512,513],{"class":50},"\"D2:",[40,515,226],{"class":225},[40,517,518],{"class":86},"last_col",[40,520,521],{"class":225},"}{",[40,523,524],{"class":86},"ws.max_row",[40,526,235],{"class":225},[40,528,529],{"class":50},"\"",[40,531,532],{"class":86},", CellIsRule(",[40,534,535],{"class":199},"operator",[40,537,109],{"class":82},[40,539,540],{"class":50},"\"equal\"",[40,542,196],{"class":86},[40,544,545],{"class":199},"formula",[40,547,109],{"class":82},[40,549,550],{"class":86},"[",[40,552,553],{"class":50},"\"1\"",[40,555,556],{"class":86},"],\n",[40,558,560,563,565,568,571,573,576,578,581,584,587,589,592,595,597,599],{"class":42,"line":559},14,[40,561,562],{"class":199},"                              fill",[40,564,109],{"class":82},[40,566,567],{"class":86},"PatternFill(",[40,569,570],{"class":50},"\"solid\"",[40,572,196],{"class":86},[40,574,575],{"class":199},"fgColor",[40,577,109],{"class":82},[40,579,580],{"class":50},"\"D9534F\"",[40,582,583],{"class":86},"), ",[40,585,586],{"class":199},"font",[40,588,109],{"class":82},[40,590,591],{"class":86},"Font(",[40,593,594],{"class":199},"color",[40,596,109],{"class":82},[40,598,580],{"class":50},[40,600,601],{"class":86},")))\n",[40,603,605,607,609,611,613,615,617,619,621,623,625,627,629,631,633,635,637,639,642],{"class":42,"line":604},15,[40,606,508],{"class":86},[40,608,219],{"class":82},[40,610,513],{"class":50},[40,612,226],{"class":225},[40,614,518],{"class":86},[40,616,521],{"class":225},[40,618,524],{"class":86},[40,620,235],{"class":225},[40,622,529],{"class":50},[40,624,532],{"class":86},[40,626,535],{"class":199},[40,628,109],{"class":82},[40,630,540],{"class":50},[40,632,196],{"class":86},[40,634,545],{"class":199},[40,636,109],{"class":82},[40,638,550],{"class":86},[40,640,641],{"class":50},"\"0\"",[40,643,556],{"class":86},[40,645,647,649,651,653,655,657,659,661,664,666,668,670,672,674,676,678],{"class":42,"line":646},16,[40,648,562],{"class":199},[40,650,109],{"class":82},[40,652,567],{"class":86},[40,654,570],{"class":50},[40,656,196],{"class":86},[40,658,575],{"class":199},[40,660,109],{"class":82},[40,662,663],{"class":50},"\"E8EEF7\"",[40,665,583],{"class":86},[40,667,586],{"class":199},[40,669,109],{"class":82},[40,671,591],{"class":86},[40,673,594],{"class":199},[40,675,109],{"class":82},[40,677,663],{"class":50},[40,679,601],{"class":86},[40,681,683,685,688,690,693,695],{"class":42,"line":682},17,[40,684,396],{"class":82},[40,686,687],{"class":86}," c ",[40,689,402],{"class":82},[40,691,692],{"class":86}," ws[",[40,694,165],{"class":158},[40,696,697],{"class":86},"]:\n",[40,699,701,704,706,709,712,714,716],{"class":42,"line":700},18,[40,702,703],{"class":86},"    c.font ",[40,705,109],{"class":82},[40,707,708],{"class":86}," Font(",[40,710,711],{"class":199},"bold",[40,713,109],{"class":82},[40,715,258],{"class":158},[40,717,118],{"class":86},[40,719,721,724,726,729,732,734,737,740,743,745,748,751,753],{"class":42,"line":720},19,[40,722,723],{"class":86},"    c.alignment ",[40,725,109],{"class":82},[40,727,728],{"class":86}," Alignment(",[40,730,731],{"class":199},"text_rotation",[40,733,109],{"class":82},[40,735,736],{"class":158},"90",[40,738,739],{"class":82}," if",[40,741,742],{"class":86}," c.column ",[40,744,185],{"class":82},[40,746,747],{"class":158}," 3",[40,749,750],{"class":82}," else",[40,752,188],{"class":158},[40,754,118],{"class":86},[40,756,758,760,763,765,768,770,773,776,778,781],{"class":42,"line":757},20,[40,759,396],{"class":82},[40,761,762],{"class":86}," col ",[40,764,402],{"class":82},[40,766,767],{"class":158}," range",[40,769,216],{"class":86},[40,771,772],{"class":158},"4",[40,774,775],{"class":86},", ws.max_column ",[40,777,387],{"class":82},[40,779,780],{"class":158}," 1",[40,782,444],{"class":86},[40,784,786,789,791,794,796],{"class":42,"line":785},21,[40,787,788],{"class":86},"    ws.column_dimensions[ws.cell(",[40,790,165],{"class":158},[40,792,793],{"class":86},", col).column_letter].width ",[40,795,109],{"class":82},[40,797,798],{"class":158}," 3.2\n",[40,800,802,805,807],{"class":42,"line":801},22,[40,803,804],{"class":86},"ws.freeze_panes ",[40,806,109],{"class":82},[40,808,809],{"class":50}," \"D2\"\n",[40,811,813,816,818],{"class":42,"line":812},23,[40,814,815],{"class":86},"ws.auto_filter.ref ",[40,817,109],{"class":82},[40,819,820],{"class":86}," ws.dimensions\n",[40,822,824,827,830],{"class":42,"line":823},24,[40,825,826],{"class":86},"wb.save(",[40,828,829],{"class":50},"\"missing_report.xlsx\"",[40,831,118],{"class":86},[40,833,835,837,839,842,845,848,850,852,855,857,860,863,866],{"class":42,"line":834},25,[40,836,174],{"class":158},[40,838,216],{"class":86},[40,840,841],{"class":50},"\"map:\"",[40,843,844],{"class":86},", ws.max_row ",[40,846,847],{"class":82},"-",[40,849,780],{"class":158},[40,851,196],{"class":86},[40,853,854],{"class":50},"\"rows ×\"",[40,856,196],{"class":86},[40,858,859],{"class":158},"len",[40,861,862],{"class":86},"(cols), ",[40,864,865],{"class":50},"\"columns\"",[40,867,118],{"class":86},[10,869,870],{},"Each data column becomes a narrow strip with a rotated header, red where a value is missing and pale blue where it is present; the font matches the fill so the 0s and 1s disappear into colour. The date and branch columns stay readable on the left, and the autofilter lets someone filter to one branch and watch its pattern. Scrolling down, the map shows a solid red band in Discount % from June onwards, three delivery columns that turn red on the same rows, and Sales rep gaps that appear only on some rows. These are patterns no percentage reveals.",[872,873,881,882,881,886,881,890,881,897,881,906,881,910,881,917,881,921,881,925,881,929,881,933,881,937,881,941,881,945,881,949,881,953,881,957,881,961,881,965,881,969,881,973,881,977,881,981,881,986,881,990,881,992,881,995,881,997,881,1000,881,1002,881,1005,881,1007,881,1010,881,1012,881,1015,881,1017,881,1020,881,1022,881,1025,881,1028,881,1031,881,1033,881,1037,881,1039,881,1042,881,1044,881,1046,881,1048,881,1050,881,1052,881,1054,881,1057,881,1061,881,1063,881,1066,881,1068,881,1071,881,1073,881,1075,881,1079,881,1083,881,1085,881,1087,881,1089,881,1091,881,1093,881,1095,881,1098,881,1101,881,1103,881,1106,881,1108,881,1110,881,1112,881,1114,881,1116,881,1118,881,1120,881,1122,881,1124,881,1127,881,1131,881,1133,881,1136,881,1138,881,1140,881,1142,881,1144,881,1146,881,1148,881,1150,881,1152,881,1154,881,1156,881,1159,881,1163,881,1165,881,1168,881,1170,881,1173,881,1175,881,1177,881,1179,881,1181,881,1183,881,1185,881,1187,881,1189,881,1191,881,1193],"svg",{"viewBox":874,"role":875,"ariaLabelledBy":876,"xmlns":879,"style":880},"87 0 586 224","img",[877,878],"mv-map-t","mv-map-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:586px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[883,884,885],"title",{"id":877},"A slice of the missingness map",[887,888,889],"desc",{"id":878},"Rows are orders and columns are fields. Red cells are blank values. Discount is blank for every row from June. Delivery postcode, delivery date and courier are blank together on the same rows. Sales rep is blank mainly on Cardiff rows.",[891,892],"rect",{"x":893,"y":272,"width":894,"height":895,"fill":896},"87","586","224","#ffffff",[891,898],{"x":899,"y":900,"width":901,"height":902,"rx":272,"fill":903,"stroke":904,"style":905},"103.4","16.0","32.0","24.0","#e7ebef","var(--line,#cdd5e6)","stroke-width:1px",[891,907],{"x":908,"y":900,"width":909,"height":902,"rx":272,"fill":903,"stroke":904,"style":905},"135.4","64.1",[911,912,916],"text",{"x":913,"y":914,"style":915},"167.5","33.0","font-size:11px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","A",[891,918],{"x":919,"y":900,"width":920,"height":902,"rx":272,"fill":903,"stroke":904,"style":905},"199.5","62.4",[911,922,924],{"x":923,"y":914,"style":915},"230.7","B",[891,926],{"x":927,"y":900,"width":928,"height":902,"rx":272,"fill":903,"stroke":904,"style":905},"261.9","66.8",[911,930,932],{"x":931,"y":914,"style":915},"295.3","C",[891,934],{"x":935,"y":900,"width":936,"height":902,"rx":272,"fill":903,"stroke":904,"style":905},"328.7","72.1",[911,938,940],{"x":939,"y":914,"style":915},"364.8","D",[891,942],{"x":943,"y":900,"width":944,"height":902,"rx":272,"fill":903,"stroke":904,"style":905},"400.9","74.9",[911,946,948],{"x":947,"y":914,"style":915},"438.3","E",[891,950],{"x":951,"y":900,"width":952,"height":902,"rx":272,"fill":903,"stroke":904,"style":905},"475.7","71.8",[911,954,956],{"x":955,"y":914,"style":915},"511.6","F",[891,958],{"x":959,"y":900,"width":960,"height":902,"rx":272,"fill":903,"stroke":904,"style":905},"547.5","64.3",[911,962,964],{"x":963,"y":914,"style":915},"579.7","G",[891,966],{"x":967,"y":900,"width":968,"height":902,"rx":272,"fill":903,"stroke":904,"style":905},"611.8","44.8",[911,970,972],{"x":971,"y":914,"style":915},"634.2","H",[891,974],{"x":899,"y":975,"width":901,"height":976,"rx":272,"fill":903,"stroke":904,"style":905},"40.0","28.0",[911,978,165],{"x":979,"y":980,"style":915},"119.4","59.0",[891,982],{"x":908,"y":975,"width":909,"height":976,"rx":272,"fill":983,"stroke":984,"style":985},"#f0f4ff","var(--brand,#5b5cf0)","stroke-width:1.5px",[911,987,989],{"x":913,"y":980,"style":988},"font-size:10.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Date",[891,991],{"x":919,"y":975,"width":920,"height":976,"rx":272,"fill":983,"stroke":984,"style":985},[911,993,994],{"x":923,"y":980,"style":988},"Branch",[891,996],{"x":927,"y":975,"width":928,"height":976,"rx":272,"fill":983,"stroke":984,"style":985},[911,998,999],{"x":931,"y":980,"style":988},"Amount",[891,1001],{"x":935,"y":975,"width":936,"height":976,"rx":272,"fill":983,"stroke":984,"style":985},[911,1003,1004],{"x":939,"y":980,"style":988},"Discount",[891,1006],{"x":943,"y":975,"width":944,"height":976,"rx":272,"fill":983,"stroke":984,"style":985},[911,1008,1009],{"x":947,"y":980,"style":988},"Postcode",[891,1011],{"x":951,"y":975,"width":952,"height":976,"rx":272,"fill":983,"stroke":984,"style":985},[911,1013,1014],{"x":955,"y":980,"style":988},"Del. date",[891,1016],{"x":959,"y":975,"width":960,"height":976,"rx":272,"fill":983,"stroke":984,"style":985},[911,1018,1019],{"x":963,"y":980,"style":988},"Courier",[891,1021],{"x":967,"y":975,"width":968,"height":976,"rx":272,"fill":983,"stroke":984,"style":985},[911,1023,1024],{"x":971,"y":980,"style":988},"Rep",[891,1026],{"x":899,"y":1027,"width":901,"height":976,"rx":272,"fill":903,"stroke":904,"style":905},"68.0",[911,1029,441],{"x":979,"y":1030,"style":915},"87.0",[891,1032],{"x":908,"y":1027,"width":909,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[911,1034,1036],{"x":913,"y":1030,"style":1035},"font-size:10.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","12 May",[891,1038],{"x":919,"y":1027,"width":920,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[911,1040,1041],{"x":923,"y":1030,"style":1035},"Leeds",[891,1043],{"x":927,"y":1027,"width":928,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[891,1045],{"x":935,"y":1027,"width":936,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[891,1047],{"x":943,"y":1027,"width":944,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[891,1049],{"x":951,"y":1027,"width":952,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[891,1051],{"x":959,"y":1027,"width":960,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[891,1053],{"x":967,"y":1027,"width":968,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[891,1055],{"x":899,"y":1056,"width":901,"height":976,"rx":272,"fill":903,"stroke":904,"style":905},"96.0",[911,1058,1060],{"x":979,"y":1059,"style":915},"115.0","3",[891,1062],{"x":908,"y":1056,"width":909,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[911,1064,1065],{"x":913,"y":1059,"style":1035},"28 May",[891,1067],{"x":919,"y":1056,"width":920,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[911,1069,1070],{"x":923,"y":1059,"style":1035},"Cardiff",[891,1072],{"x":927,"y":1056,"width":928,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[891,1074],{"x":935,"y":1056,"width":936,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[891,1076],{"x":943,"y":1056,"width":944,"height":976,"rx":272,"fill":1077,"stroke":1078,"style":985},"#fce9e9","var(--danger,#dc2626)",[911,1080,1082],{"x":947,"y":1059,"style":1081},"font-size:10.5px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","■",[891,1084],{"x":951,"y":1056,"width":952,"height":976,"rx":272,"fill":1077,"stroke":1078,"style":985},[911,1086,1082],{"x":955,"y":1059,"style":1081},[891,1088],{"x":959,"y":1056,"width":960,"height":976,"rx":272,"fill":1077,"stroke":1078,"style":985},[911,1090,1082],{"x":963,"y":1059,"style":1081},[891,1092],{"x":967,"y":1056,"width":968,"height":976,"rx":272,"fill":1077,"stroke":1078,"style":985},[911,1094,1082],{"x":971,"y":1059,"style":1081},[891,1096],{"x":899,"y":1097,"width":901,"height":976,"rx":272,"fill":903,"stroke":904,"style":905},"124.0",[911,1099,772],{"x":979,"y":1100,"style":915},"143.0",[891,1102],{"x":908,"y":1097,"width":909,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[911,1104,1105],{"x":913,"y":1100,"style":1035},"03 Jun",[891,1107],{"x":919,"y":1097,"width":920,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[911,1109,1041],{"x":923,"y":1100,"style":1035},[891,1111],{"x":927,"y":1097,"width":928,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[891,1113],{"x":935,"y":1097,"width":936,"height":976,"rx":272,"fill":1077,"stroke":1078,"style":985},[911,1115,1082],{"x":939,"y":1100,"style":1081},[891,1117],{"x":943,"y":1097,"width":944,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[891,1119],{"x":951,"y":1097,"width":952,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[891,1121],{"x":959,"y":1097,"width":960,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[891,1123],{"x":967,"y":1097,"width":968,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[891,1125],{"x":899,"y":1126,"width":901,"height":976,"rx":272,"fill":903,"stroke":904,"style":905},"152.0",[911,1128,1130],{"x":979,"y":1129,"style":915},"171.0","5",[891,1132],{"x":908,"y":1126,"width":909,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[911,1134,1135],{"x":913,"y":1129,"style":1035},"09 Jun",[891,1137],{"x":919,"y":1126,"width":920,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[911,1139,1070],{"x":923,"y":1129,"style":1035},[891,1141],{"x":927,"y":1126,"width":928,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[891,1143],{"x":935,"y":1126,"width":936,"height":976,"rx":272,"fill":1077,"stroke":1078,"style":985},[911,1145,1082],{"x":939,"y":1129,"style":1081},[891,1147],{"x":943,"y":1126,"width":944,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[891,1149],{"x":951,"y":1126,"width":952,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[891,1151],{"x":959,"y":1126,"width":960,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[891,1153],{"x":967,"y":1126,"width":968,"height":976,"rx":272,"fill":1077,"stroke":1078,"style":985},[911,1155,1082],{"x":971,"y":1129,"style":1081},[891,1157],{"x":899,"y":1158,"width":901,"height":976,"rx":272,"fill":903,"stroke":904,"style":905},"180.0",[911,1160,1162],{"x":979,"y":1161,"style":915},"199.0","6",[891,1164],{"x":908,"y":1158,"width":909,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[911,1166,1167],{"x":913,"y":1161,"style":1035},"17 Jun",[891,1169],{"x":919,"y":1158,"width":920,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[911,1171,1172],{"x":923,"y":1161,"style":1035},"Bristol",[891,1174],{"x":927,"y":1158,"width":928,"height":976,"rx":272,"fill":1077,"stroke":1078,"style":985},[911,1176,1082],{"x":931,"y":1161,"style":1081},[891,1178],{"x":935,"y":1158,"width":936,"height":976,"rx":272,"fill":1077,"stroke":1078,"style":985},[911,1180,1082],{"x":939,"y":1161,"style":1081},[891,1182],{"x":943,"y":1158,"width":944,"height":976,"rx":272,"fill":1077,"stroke":1078,"style":985},[911,1184,1082],{"x":947,"y":1161,"style":1081},[891,1186],{"x":951,"y":1158,"width":952,"height":976,"rx":272,"fill":1077,"stroke":1078,"style":985},[911,1188,1082],{"x":955,"y":1161,"style":1081},[891,1190],{"x":959,"y":1158,"width":960,"height":976,"rx":272,"fill":1077,"stroke":1078,"style":985},[911,1192,1082],{"x":963,"y":1161,"style":1081},[891,1194],{"x":967,"y":1158,"width":968,"height":976,"rx":272,"fill":896,"stroke":904,"style":905},[25,1196,1198],{"id":1197},"step-3-the-most-common-combinations-of-missing-columns","Step 3 — The most common combinations of missing columns",[10,1200,1201,1202,1205],{},"Turning each row's missingness into a pattern string and counting them shows which columns go missing ",[14,1203,1204],{},"together",":",[30,1207,1209],{"className":73,"code":1208,"language":75,"meta":35,"style":35},"patterns = missing.apply(lambda r: \" + \".join(c for c in cols if r[c]) or \"(complete)\", axis=1)\ntop = patterns.value_counts().head(8).rename_axis(\"Missing columns\").to_frame(\"Rows\")\ntop[\"share %\"] = (top[\"Rows\"] \u002F len(orders) * 100).round(1)\nprint(top.to_string())\n",[37,1210,1211,1264,1291,1328],{"__ignoreMap":35},[40,1212,1213,1216,1218,1221,1224,1227,1230,1233,1235,1237,1239,1242,1244,1247,1250,1253,1255,1258,1260,1262],{"class":42,"line":43},[40,1214,1215],{"class":86},"patterns ",[40,1217,109],{"class":82},[40,1219,1220],{"class":86}," missing.apply(",[40,1222,1223],{"class":82},"lambda",[40,1225,1226],{"class":86}," r: ",[40,1228,1229],{"class":50},"\" + \"",[40,1231,1232],{"class":86},".join(c ",[40,1234,396],{"class":82},[40,1236,687],{"class":86},[40,1238,402],{"class":82},[40,1240,1241],{"class":86}," cols ",[40,1243,453],{"class":82},[40,1245,1246],{"class":86}," r[c]) ",[40,1248,1249],{"class":82},"or",[40,1251,1252],{"class":50}," \"(complete)\"",[40,1254,196],{"class":86},[40,1256,1257],{"class":199},"axis",[40,1259,109],{"class":82},[40,1261,165],{"class":158},[40,1263,118],{"class":86},[40,1265,1266,1269,1271,1274,1277,1280,1283,1286,1289],{"class":42,"line":96},[40,1267,1268],{"class":86},"top ",[40,1270,109],{"class":82},[40,1272,1273],{"class":86}," patterns.value_counts().head(",[40,1275,1276],{"class":158},"8",[40,1278,1279],{"class":86},").rename_axis(",[40,1281,1282],{"class":50},"\"Missing columns\"",[40,1284,1285],{"class":86},").to_frame(",[40,1287,1288],{"class":50},"\"Rows\"",[40,1290,118],{"class":86},[40,1292,1293,1296,1298,1300,1302,1305,1307,1309,1312,1315,1318,1320,1322,1324,1326],{"class":42,"line":103},[40,1294,1295],{"class":86},"top[",[40,1297,149],{"class":50},[40,1299,182],{"class":86},[40,1301,109],{"class":82},[40,1303,1304],{"class":86}," (top[",[40,1306,1288],{"class":50},[40,1308,182],{"class":86},[40,1310,1311],{"class":82},"\u002F",[40,1313,1314],{"class":158}," len",[40,1316,1317],{"class":86},"(orders) ",[40,1319,155],{"class":82},[40,1321,159],{"class":158},[40,1323,162],{"class":86},[40,1325,165],{"class":158},[40,1327,118],{"class":86},[40,1329,1330,1332],{"class":42,"line":121},[40,1331,174],{"class":158},[40,1333,1334],{"class":86},"(top.to_string())\n",[10,1336,1337,1338,1341],{},"The output reads like a diagnosis. After the 95 complete rows, \"Discount %\" alone is the largest group — 86 orders from the June export change. \"Delivery postcode + Delivery date + Courier\" always appear together and never separately, which is the signature of a skipped ",[14,1339,1340],{},"section",": these are the store-collection orders, which genuinely have no delivery details. Those gaps are not errors at all; they should be documented and probably given an explicit \"Collected\" value instead of blanks. Mixed combinations such as \"Discount % + Delivery postcode + Delivery date + Courier\" are simply two independent causes landing on the same row — a June order that was also collected in store. A combination that appears only once or twice is often a genuine data-entry slip worth fixing by hand.",[25,1343,1345],{"id":1344},"step-4-missing-rate-over-time-and-by-source","Step 4 — Missing rate over time and by source",[10,1347,1348,1349,1352,1353,1355],{},"Many gaps are caused by ",[14,1350,1351],{},"when"," or ",[14,1354,246],{}," data was entered. A month-by-column table and a branch-by-column table of missing rates expose both:",[30,1357,1359],{"className":73,"code":1358,"language":75,"meta":35,"style":35},"by_month = (missing.groupby(orders[\"Date\"].dt.to_period(\"M\")).mean() * 100).round(0)\nby_branch = (missing.groupby(orders[\"Branch\"]).mean() * 100).round(0)\nshow = [\"Amount\", \"Discount %\", \"Delivery date\", \"Sales rep\"]\nprint(by_month[show].astype(int).to_string())\nprint(by_branch[show].astype(int).to_string())\n",[37,1360,1361,1392,1416,1446,1457],{"__ignoreMap":35},[40,1362,1363,1366,1368,1371,1373,1376,1379,1382,1384,1386,1388,1390],{"class":42,"line":43},[40,1364,1365],{"class":86},"by_month ",[40,1367,109],{"class":82},[40,1369,1370],{"class":86}," (missing.groupby(orders[",[40,1372,377],{"class":50},[40,1374,1375],{"class":86},"].dt.to_period(",[40,1377,1378],{"class":50},"\"M\"",[40,1380,1381],{"class":86},")).mean() ",[40,1383,155],{"class":82},[40,1385,159],{"class":158},[40,1387,162],{"class":86},[40,1389,272],{"class":158},[40,1391,118],{"class":86},[40,1393,1394,1397,1399,1401,1403,1406,1408,1410,1412,1414],{"class":42,"line":96},[40,1395,1396],{"class":86},"by_branch ",[40,1398,109],{"class":82},[40,1400,1370],{"class":86},[40,1402,382],{"class":50},[40,1404,1405],{"class":86},"]).mean() ",[40,1407,155],{"class":82},[40,1409,159],{"class":158},[40,1411,162],{"class":86},[40,1413,272],{"class":158},[40,1415,118],{"class":86},[40,1417,1418,1421,1423,1425,1428,1430,1433,1435,1438,1440,1443],{"class":42,"line":103},[40,1419,1420],{"class":86},"show ",[40,1422,109],{"class":82},[40,1424,470],{"class":86},[40,1426,1427],{"class":50},"\"Amount\"",[40,1429,196],{"class":86},[40,1431,1432],{"class":50},"\"Discount %\"",[40,1434,196],{"class":86},[40,1436,1437],{"class":50},"\"Delivery date\"",[40,1439,196],{"class":86},[40,1441,1442],{"class":50},"\"Sales rep\"",[40,1444,1445],{"class":86},"]\n",[40,1447,1448,1450,1453,1455],{"class":42,"line":121},[40,1449,174],{"class":158},[40,1451,1452],{"class":86},"(by_month[show].astype(",[40,1454,473],{"class":158},[40,1456,208],{"class":86},[40,1458,1459,1461,1464,1466],{"class":42,"line":132},[40,1460,174],{"class":158},[40,1462,1463],{"class":86},"(by_branch[show].astype(",[40,1465,473],{"class":158},[40,1467,208],{"class":86},[10,1469,1470],{},"By month, Discount % jumps from 0% to 100% in June and stays there — something changed in the export, not in customer behaviour, and the fix is to ask whoever owns the export. By branch, Sales rep is missing on 68% of Cardiff orders and never elsewhere — a process issue in one branch, fixed by a conversation rather than by code. Amount's small, even gap rate across months and branches looks like genuine random omissions. The delivery columns, by contrast, show a steady 10–20% in every month and branch — consistent with a normal share of store collections. These tables go in the report next to the map so a data owner can see the cause at a glance.",[872,1472,881,1478,881,1481,881,1484,881,1488,881,1492,881,1497,881,1502,881,1505,881,1509,881,1513,881,1518,881,1520,881,1525,881,1527,881,1530,881,1533,881,1537,881,1539,881,1543,881,1545,881,1548,881,1550,881,1554,881,1556,881,1560,881,1562,881,1565,881,1568,881,1570,881,1572,881,1576,881,1578],{"viewBox":1473,"role":875,"ariaLabelledBy":1474,"xmlns":879,"style":1477},"4 0 752 208",[1475,1476],"mv-causes-t","mv-causes-d","width:100%;max-width:752px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[883,1479,1480],{"id":1475},"Patterns and what they usually mean",[887,1482,1483],{"id":1476},"A column that becomes blank from a date onwards points to an export or system change. Columns that are blank together point to a skipped form section or a legitimately absent block. Blanks concentrated in one source point to a local process issue. Small, evenly spread gaps are usually random omissions.",[891,1485],{"x":772,"y":272,"width":1486,"height":1487,"fill":896},"752","208",[891,1489],{"x":1490,"y":900,"width":1491,"height":975,"rx":272,"fill":896,"stroke":904,"style":905},"20.0","101.9",[891,1493],{"x":1494,"y":900,"width":1495,"height":975,"rx":272,"fill":983,"stroke":984,"style":1496},"121.9","309.1","stroke-width:1.2px",[911,1498,16],{"x":1499,"y":1500,"style":1501},"276.4","41.0","font-size:13px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle",[891,1503],{"x":1504,"y":900,"width":1495,"height":975,"rx":272,"fill":983,"stroke":984,"style":1496},"430.9",[911,1506,1508],{"x":1507,"y":1500,"style":1501},"585.5","likely cause",[891,1510],{"x":1490,"y":1511,"width":1491,"height":1512,"rx":272,"fill":903,"stroke":904,"style":905},"56.0","34.0",[911,1514,1517],{"x":901,"y":1515,"style":1516},"78.0","font-size:12.5px;font-weight:700;fill:var(--text,#172033);text-anchor:start","Discount %",[891,1519],{"x":1494,"y":1511,"width":1495,"height":1512,"rx":272,"fill":896,"stroke":904,"style":905},[911,1521,1524],{"x":1499,"y":1522,"style":1523},"77.0","font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","100% blank from June",[891,1526],{"x":1504,"y":1511,"width":1495,"height":1512,"rx":272,"fill":896,"stroke":904,"style":905},[911,1528,1529],{"x":1507,"y":1522,"style":1523},"export changed",[891,1531],{"x":1490,"y":1532,"width":1491,"height":1512,"rx":272,"fill":903,"stroke":904,"style":905},"90.0",[911,1534,1536],{"x":901,"y":1535,"style":1516},"112.0","Delivery ×3",[891,1538],{"x":1494,"y":1532,"width":1495,"height":1512,"rx":272,"fill":896,"stroke":904,"style":905},[911,1540,1542],{"x":1499,"y":1541,"style":1523},"111.0","blank together",[891,1544],{"x":1504,"y":1532,"width":1495,"height":1512,"rx":272,"fill":896,"stroke":904,"style":905},[911,1546,1547],{"x":1507,"y":1541,"style":1523},"store collection, not an error",[891,1549],{"x":1490,"y":1097,"width":1491,"height":1512,"rx":272,"fill":903,"stroke":904,"style":905},[911,1551,1553],{"x":901,"y":1552,"style":1516},"146.0","Sales rep",[891,1555],{"x":1494,"y":1097,"width":1495,"height":1512,"rx":272,"fill":896,"stroke":904,"style":905},[911,1557,1559],{"x":1499,"y":1558,"style":1523},"145.0","mostly Cardiff",[891,1561],{"x":1504,"y":1097,"width":1495,"height":1512,"rx":272,"fill":896,"stroke":904,"style":905},[911,1563,1564],{"x":1507,"y":1558,"style":1523},"local process gap",[891,1566],{"x":1490,"y":1567,"width":1491,"height":1512,"rx":272,"fill":903,"stroke":904,"style":905},"158.0",[911,1569,999],{"x":901,"y":1158,"style":1516},[891,1571],{"x":1494,"y":1567,"width":1495,"height":1512,"rx":272,"fill":896,"stroke":904,"style":905},[911,1573,1575],{"x":1499,"y":1574,"style":1523},"179.0","~4%, spread evenly",[891,1577],{"x":1504,"y":1567,"width":1495,"height":1512,"rx":272,"fill":896,"stroke":904,"style":905},[911,1579,1580],{"x":1507,"y":1574,"style":1523},"random omissions",[25,1582,1584],{"id":1583},"step-5-columns-that-go-missing-together","Step 5 — Columns that go missing together",[10,1586,1587],{},"For wider sheets, a correlation matrix of the missingness indicators summarises co-missing columns in one table. A value near 1 means two columns are almost always blank on the same rows:",[30,1589,1591],{"className":73,"code":1590,"language":75,"meta":35,"style":35},"gaps = missing.loc[:, missing.any()].astype(int)\nco = gaps.corr().round(2)\nprint(co.to_string())\n\nwith pd.ExcelWriter(\"missing_report.xlsx\", engine=\"openpyxl\", mode=\"a\") as xw:\n    per_col.to_excel(xw, sheet_name=\"By column\")\n    top.to_excel(xw, sheet_name=\"Patterns\")\n    by_month.astype(str).to_excel(xw, sheet_name=\"By month\")\n    by_branch.to_excel(xw, sheet_name=\"By branch\")\n    co.to_excel(xw, sheet_name=\"Co-missing\")\nwith pd.ExcelFile(\"missing_report.xlsx\") as book:\n    print(\"sheets:\", book.sheet_names)\n",[37,1592,1593,1607,1621,1628,1632,1670,1685,1699,1719,1733,1747,1763],{"__ignoreMap":35},[40,1594,1595,1598,1600,1603,1605],{"class":42,"line":43},[40,1596,1597],{"class":86},"gaps ",[40,1599,109],{"class":82},[40,1601,1602],{"class":86}," missing.loc[:, missing.any()].astype(",[40,1604,473],{"class":158},[40,1606,118],{"class":86},[40,1608,1609,1612,1614,1617,1619],{"class":42,"line":96},[40,1610,1611],{"class":86},"co ",[40,1613,109],{"class":82},[40,1615,1616],{"class":86}," gaps.corr().round(",[40,1618,441],{"class":158},[40,1620,118],{"class":86},[40,1622,1623,1625],{"class":42,"line":103},[40,1624,174],{"class":158},[40,1626,1627],{"class":86},"(co.to_string())\n",[40,1629,1630],{"class":42,"line":121},[40,1631,100],{"emptyLinePlaceholder":99},[40,1633,1634,1637,1640,1642,1644,1647,1649,1652,1654,1657,1659,1662,1665,1667],{"class":42,"line":132},[40,1635,1636],{"class":82},"with",[40,1638,1639],{"class":86}," pd.ExcelWriter(",[40,1641,829],{"class":50},[40,1643,196],{"class":86},[40,1645,1646],{"class":199},"engine",[40,1648,109],{"class":82},[40,1650,1651],{"class":50},"\"openpyxl\"",[40,1653,196],{"class":86},[40,1655,1656],{"class":199},"mode",[40,1658,109],{"class":82},[40,1660,1661],{"class":50},"\"a\"",[40,1663,1664],{"class":86},") ",[40,1666,90],{"class":82},[40,1668,1669],{"class":86}," xw:\n",[40,1671,1672,1675,1678,1680,1683],{"class":42,"line":171},[40,1673,1674],{"class":86},"    per_col.to_excel(xw, ",[40,1676,1677],{"class":199},"sheet_name",[40,1679,109],{"class":82},[40,1681,1682],{"class":50},"\"By column\"",[40,1684,118],{"class":86},[40,1686,1687,1690,1692,1694,1697],{"class":42,"line":211},[40,1688,1689],{"class":86},"    top.to_excel(xw, ",[40,1691,1677],{"class":199},[40,1693,109],{"class":82},[40,1695,1696],{"class":50},"\"Patterns\"",[40,1698,118],{"class":86},[40,1700,1701,1704,1707,1710,1712,1714,1717],{"class":42,"line":352},[40,1702,1703],{"class":86},"    by_month.astype(",[40,1705,1706],{"class":158},"str",[40,1708,1709],{"class":86},").to_excel(xw, ",[40,1711,1677],{"class":199},[40,1713,109],{"class":82},[40,1715,1716],{"class":50},"\"By month\"",[40,1718,118],{"class":86},[40,1720,1721,1724,1726,1728,1731],{"class":42,"line":366},[40,1722,1723],{"class":86},"    by_branch.to_excel(xw, ",[40,1725,1677],{"class":199},[40,1727,109],{"class":82},[40,1729,1730],{"class":50},"\"By branch\"",[40,1732,118],{"class":86},[40,1734,1735,1738,1740,1742,1745],{"class":42,"line":393},[40,1736,1737],{"class":86},"    co.to_excel(xw, ",[40,1739,1677],{"class":199},[40,1741,109],{"class":82},[40,1743,1744],{"class":50},"\"Co-missing\"",[40,1746,118],{"class":86},[40,1748,1749,1751,1754,1756,1758,1760],{"class":42,"line":447},[40,1750,1636],{"class":82},[40,1752,1753],{"class":86}," pd.ExcelFile(",[40,1755,829],{"class":50},[40,1757,1664],{"class":86},[40,1759,90],{"class":82},[40,1761,1762],{"class":86}," book:\n",[40,1764,1765,1768,1770,1773],{"class":42,"line":489},[40,1766,1767],{"class":158},"    print",[40,1769,216],{"class":86},[40,1771,1772],{"class":50},"\"sheets:\"",[40,1774,1775],{"class":86},", book.sheet_names)\n",[10,1777,1778,1779,1782,1783,1787],{},"The three delivery columns correlate perfectly (1.0) with each other, confirming the section pattern from Step 3; Discount % and Sales rep are unrelated to everything else. Converting the month periods to strings before writing avoids Excel showing them oddly. ",[37,1780,1781],{},"mode=\"a\""," appends the summary sheets to the workbook that already holds the map, so the whole picture arrives in one file. A colour scale on the co-missing sheet, as in ",[19,1784,1786],{"href":1785},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcreating-pivot-tables-from-excel-data\u002Fturn-a-pivot-table-into-a-heatmap-in-excel\u002F","Turn a Pivot Table into a Heatmap in Excel",", makes strong pairs jump out.",[872,1789,881,1794,881,1797,881,1800,881,1803,881,1809,881,1820,881,1825,881,1829,881,1834,881,1839,881,1842,881,1849,881,1854,881,1857,881,1861,881,1864,881,1869,881,1876,881,1882,881,1886,881,1890,881,1894,881,1897,881,1902,881,1909,881,1914],{"viewBox":1790,"role":875,"ariaLabelledBy":1791,"xmlns":879,"style":1477},"4 0 752 142",[1792,1793],"mv-flow-t","mv-flow-d",[883,1795,1796],{"id":1792},"From blank cells to a diagnosis",[887,1798,1799],{"id":1793},"The isna frame drives every view. The map shows where blanks are, the pattern counts show which columns are blank together, the month and branch tables show when and where, and the correlation sheet summarises co-missing columns, leading to a fix for each cause.",[891,1801],{"x":772,"y":272,"width":1486,"height":1802,"fill":896},"142",[891,1804],{"x":1490,"y":900,"width":1805,"height":1806,"rx":1807,"fill":903,"stroke":904,"style":1808},"153.0","110.5","12","stroke-width:2px",[1810,1811,1813],"g",{"transform":1812},"translate(84.5,37.8) scale(1.000)",[1814,1815],"path",{"d":1816,"fill":1817,"stroke":1818,"style":1819},"M3 5h18v14H3z M3 10h18 M3 15h18 M9 5v14","none","var(--text,#172033)","stroke-width:1.80px;stroke-linecap:round;stroke-linejoin:round",[911,1821,254],{"x":1822,"y":1823,"style":1824},"96.5","84.2","font-size:13.5px;font-weight:700;fill:var(--text,#172033);text-anchor:middle",[911,1826,1828],{"x":1822,"y":1827,"style":1523},"103.8","one flag per cell",[42,1830],{"x1":1831,"y1":1832,"x2":1833,"y2":1832,"stroke":984,"style":1808},"178.0","71.2","200.0",[1835,1836],"polygon",{"points":1837,"fill":1838},"204.0,71.2 195.0,76.2 195.0,66.2","#5b5cf0",[891,1840],{"x":1841,"y":900,"width":1805,"height":1806,"rx":1807,"fill":983,"stroke":984,"style":1808},"209.0",[1810,1843,1845],{"transform":1844},"translate(273.5,37.8) scale(1.000)",[1814,1846],{"d":1847,"fill":1817,"stroke":1848,"style":1819},"M2 12s4-7 10-7 10 7 10 7-4 7-10 7S2 12 2 12z M12 9a3 3 0 1 0 .01 0z","var(--brand-strong,#4338ca)",[911,1850,1853],{"x":1851,"y":1823,"style":1852},"285.5","font-size:13.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Map",[911,1855,1856],{"x":1851,"y":1827,"style":1523},"shaded blanks in Excel",[42,1858],{"x1":1859,"y1":1832,"x2":1860,"y2":1832,"stroke":984,"style":1808},"367.0","389.0",[1835,1862],{"points":1863,"fill":1838},"393.0,71.2 384.0,76.2 384.0,66.2",[891,1865],{"x":1866,"y":900,"width":1805,"height":1806,"rx":1807,"fill":1867,"stroke":1868,"style":1808},"398.0","#d9f4f1","var(--teal,#0f9488)",[1810,1870,1872],{"transform":1871},"translate(462.5,29.0) scale(1.000)",[1814,1873],{"d":1874,"fill":1817,"stroke":1875,"style":1819},"M11 4a7 7 0 1 0 .01 0z M16 16l5 5","var(--teal-ink,#0b6157)",[911,1877,1881],{"x":1878,"y":1879,"style":1880},"474.5","75.5","font-size:13.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","Patterns",[911,1883,1885],{"x":1878,"y":1884,"style":1523},"95.0","combinations, month,",[911,1887,1889],{"x":1878,"y":1888,"style":1523},"112.5","branch",[42,1891],{"x1":1892,"y1":1832,"x2":1893,"y2":1832,"stroke":984,"style":1808},"556.0","578.0",[1835,1895],{"points":1896,"fill":1838},"582.0,71.2 573.0,76.2 573.0,66.2",[891,1898],{"x":1899,"y":900,"width":1805,"height":1806,"rx":1807,"fill":1900,"stroke":1901,"style":1808},"587.0","#fdefd8","var(--gold,#b4740a)",[1810,1903,1905],{"transform":1904},"translate(651.5,37.8) scale(1.000)",[1814,1906],{"d":1907,"fill":1817,"stroke":1908,"style":1819},"M5 12.5l4.5 4.5L19 7","var(--gold-ink,#7a4e06)",[911,1910,1913],{"x":1911,"y":1823,"style":1912},"663.5","font-size:13.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","Fix by cause",[911,1915,1916],{"x":1911,"y":1827,"style":1523},"export, form, process",[25,1918,1920],{"id":1919},"common-pitfalls","Common pitfalls",[1922,1923,1924,1932,1938,1944,1950],"ul",{},[1925,1926,1927,1931],"li",{},[1928,1929,1930],"strong",{},"Reporting only percentages."," They hide when, where and in which combinations data is missing.",[1925,1933,1934,1937],{},[1928,1935,1936],{},"Treating every blank as an error."," Some blanks are legitimate; give them an explicit value.",[1925,1939,1940,1943],{},[1928,1941,1942],{},"Imputing before diagnosing."," A column blank since an export change needs fixing at the source.",[1925,1945,1946,1949],{},[1928,1947,1948],{},"Huge maps."," For very large files, map a sample or aggregate by period first.",[1925,1951,1952,1955],{},[1928,1953,1954],{},"Writing periods straight to Excel."," Convert them to strings or dates first.",[25,1957,1959],{"id":1958},"conclusion","Conclusion",[10,1961,1962,1963,1965],{},"Missing data has patterns, and the patterns point to causes: an export that changed on a date, a form section that is skipped together, a branch that does not fill in a field, or genuine random omissions. Build the ",[37,1964,254],{}," frame once, then show it as a shaded map, as counted combinations of missing columns, as rates by month and by source, and as a co-missing correlation table — all in one Excel report a data owner can read. Fix each pattern at its cause before reaching for imputation.",[25,1967,1969],{"id":1968},"frequently-asked-questions","Frequently asked questions",[10,1971,1972,1975],{},[1928,1973,1974],{},"How do I see missing data patterns in Python?","\nBuild df.isna(), then count each row's combination of missing columns, group the flags by month or source to get missing rates, and correlate the flags to find columns that are blank together.",[10,1977,1978,1981],{},[1928,1979,1980],{},"How do I highlight blank cells across a whole sheet in Excel?","\nWrite a 1 for missing and 0 for present for every cell and add conditional formatting that fills 1s in a strong colour. Matching the font colour to the fill hides the numbers.",[10,1983,1984,1987],{},[1928,1985,1986],{},"How can I tell if missing values are random?","\nIf gaps are spread evenly across periods, sources and other columns they are likely random. Gaps that start on a date, cluster in one source or occur together in several columns have a specific cause.",[10,1989,1990,1993],{},[1928,1991,1992],{},"What does it mean when several columns are always missing together?","\nUsually a whole section was skipped or does not apply, such as delivery details for collected orders. Document it and use an explicit value rather than blanks.",[25,1995,1997],{"id":1996},"related","Related",[1922,1999,2000,2006,2013,2020],{},[1925,2001,2002,2003,2005],{},"Up one level: ",[19,2004,22],{"href":21}," — finding and treating gaps.",[1925,2007,2008,2012],{},[19,2009,2011],{"href":2010},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Ffind-and-report-missing-values-in-an-excel-file\u002F","Find and Report Missing Values in an Excel File"," — the counts this page goes beyond.",[1925,2014,2015,2019],{},[19,2016,2018],{"href":2017},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fprofile-an-excel-dataset-with-pandas\u002F","Profile an Excel Dataset with pandas"," — a wider look at a new file.",[1925,2021,2022,2026],{},[19,2023,2025],{"href":2024},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fimpute-missing-values-by-group-with-pandas\u002F","Impute Missing Values by Group with pandas"," — filling gaps once you understand them.",[2028,2029,2030],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}",{"title":35,"searchDepth":96,"depth":96,"links":2032},[2033,2034,2035,2036,2037,2038,2039,2040,2041,2042],{"id":27,"depth":96,"text":28},{"id":66,"depth":96,"text":67},{"id":262,"depth":96,"text":263},{"id":1197,"depth":96,"text":1198},{"id":1344,"depth":96,"text":1345},{"id":1583,"depth":96,"text":1584},{"id":1919,"depth":96,"text":1920},{"id":1958,"depth":96,"text":1959},{"id":1968,"depth":96,"text":1969},{"id":1996,"depth":96,"text":1997},"2026-10-02","See where and why data is missing, not just how much: a missingness map sheet that shades every blank cell, the most common combinations of missing columns, missing rates by month and by source, columns that tend to go missing together, and a one-sheet Excel report built with pandas and openpyxl that shows a data owner where to look.","md",[2047,2049,2051,2053],{"q":1974,"a":2048},"Build df.isna(), then count each row's combination of missing columns, group the flags by month or source to get missing rates, and correlate the flags to find columns that are blank together.",{"q":1980,"a":2050},"Write a 1 for missing and 0 for present for every cell and add conditional formatting that fills 1s in a strong colour. Matching the font colour to the fill hides the numbers.",{"q":1986,"a":2052},"If gaps are spread evenly across periods, sources and other columns they are likely random. Gaps that start on a date, cluster in one source or occur together in several columns have a specific cause.",{"q":1992,"a":2054},"Usually a whole section was skipped or does not apply, such as delivery details for collected orders. Document it and use an explicit value rather than blanks.","visualise missing data patterns in an excel file",null,{"breadcrumb":2058},[2059,2061,2064],{"name":2060,"item":1311},"Home",{"name":2062,"item":2063},"Advanced Data Transformation and Cleaning for Excel Automation","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":2065,"item":21},"Handling Missing Data in Excel Reports with Pandas","\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fvisualise-missing-data-patterns-in-an-excel-file",{"title":2068,"description":2069},"Visualise Missing Data Patterns in Excel with Python","Build a missingness map in Excel with pandas and openpyxl: shaded blank cells, common missing-column patterns, missing rate by month and source, co-missing columns and a report for data owners.","visualise-missing-data-patterns-in-an-excel-file","advanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fvisualise-missing-data-patterns-in-an-excel-file\u002Findex",[2073,2074],"missing-data","conditional-formatting","how-to","PVSdV1kckPDV9yyjOxJf8YlxvhVYzCn0E3TrYgV5yrQ",[2078,2082],{"title":2079,"path":2080,"stem":2081,"children":-1},"Tell Blank Cells, Empty Strings and Zeros Apart in Excel Data","\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Ftell-blank-cells-empty-strings-and-zeros-apart-in-excel-data","advanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Ftell-blank-cells-empty-strings-and-zeros-apart-in-excel-data\u002Findex",{"title":2083,"path":2084,"stem":2085,"children":-1},"Merging and Joining Excel DataFrames with Pandas","\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes","advanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Findex",[2087,2089],{"path":2088,"title":2065},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports",{"path":2090,"title":2062},"\u002Fadvanced-data-transformation-and-cleaning",[],[2093,2096,2099,2102,2105,2108],{"path":2094,"title":2095,"kind":2075},"\u002Fadvanced-data-transformation-and-cleaning\u002Fapplying-conditional-formatting-with-openpyxl\u002Fhighlight-blank-cells-with-openpyxl","Highlight Blank Cells in Excel with openpyxl",{"path":2097,"title":2098,"kind":2075},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fcoalesce-several-excel-columns-into-one-with-pandas","Coalesce Several Excel Columns into One with pandas",{"path":2100,"title":2101,"kind":2075},"\u002Fadvanced-data-transformation-and-cleaning\u002Fexcel-formula-equivalents-in-pandas\u002Fiferror-and-isna-equivalents-in-pandas","IFERROR, IFNA, ISNA and ISNUMBER Equivalents in pandas",{"path":2103,"title":2104,"kind":2075},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fdrop-rows-with-missing-required-fields-in-pandas","Drop Rows with Missing Required Fields in pandas",{"path":2106,"title":2107,"kind":2075},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Ffill-gaps-in-per-item-time-series-with-a-staleness-limit","Fill Gaps in Per-Item Time Series with a Staleness Limit",{"path":2109,"title":2110,"kind":2075},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Ffill-missing-values-in-excel-with-pandas-fillna","Fill Missing Values in Excel with Pandas fillna",1790972096121]