[{"data":1,"prerenderedAt":2047},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fflag-rows-with-too-many-missing-values":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fflag-rows-with-too-many-missing-values":2015,"ancestors:\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fflag-rows-with-too-many-missing-values":2023,"children:\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fflag-rows-with-too-many-missing-values":2028,"related:\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fflag-rows-with-too-many-missing-values":2029},{"id":4,"title":5,"body":6,"dateModified":1981,"datePublished":1981,"description":1982,"extension":1983,"faq":1984,"keyword":1993,"kind":1994,"meta":1995,"navigation":97,"path":2004,"seo":2005,"slug":2008,"stem":2009,"topics":2010,"type":2013,"__hash__":2014},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fflag-rows-with-too-many-missing-values\u002Findex.md","Flag Rows with Too Many Missing Values",{"type":7,"value":8,"toc":1969},"minimark",[9,28,33,62,65,69,309,320,324,327,705,719,955,959,962,1105,1112,1182,1186,1684,1691,1851,1855,1858,1862,1897,1901,1904,1908,1914,1920,1926,1932,1936,1965],[10,11,12,13,17,18,21,22,27],"p",{},"Column-level checks — \"the ",[14,15,16],"em",{},"Email"," column is 12% empty\" — tell you about a field. They do not tell you which ",[14,19,20],{},"records"," are unusable. A customer row missing an email is fine; a row missing the name, the account number and the country is a half-entered record that will end up in totals, mailing lists and deduplication as noise. Finding such rows is a row-wise question: how many of this record's fields are empty, and are the empty ones the fields that matter? This page answers it with pandas on a typical form-filled sheet: it turns placeholders such as \"n\u002Fa\", \"-\" and \"TBC\" into real missing values, counts missing fields per row, weights required fields more heavily than optional ones, applies a threshold, summarises completeness by the person who entered each record, and writes a workbook that highlights incomplete rows and lists them for follow-up. It belongs to ",[23,24,26],"a",{"href":25},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002F","Handling Missing Data in Excel Reports",".",[29,30,32],"h2",{"id":31},"prerequisites","Prerequisites",[34,35,40],"pre",{"className":36,"code":37,"language":38,"meta":39,"style":39},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl\n","bash","",[41,42,43],"code",{"__ignoreMap":39},[44,45,48,52,56,59],"span",{"class":46,"line":47},"line",1,[44,49,51],{"class":50},"sMTad","pip",[44,53,55],{"class":54},"srMev"," install",[44,57,58],{"class":54}," pandas",[44,60,61],{"class":54}," openpyxl\n",[10,63,64],{},"The sample is a 40-row account list entered by three people, with the usual gaps: empty cells, and cells holding \"n\u002Fa\", \"-\", \"TBC\" or a single space instead of a value. Three records were clearly abandoned half-way.",[29,66,68],{"id":67},"step-1-make-every-kind-of-missing-look-the-same","Step 1 — Make every kind of \"missing\" look the same",[34,70,74],{"className":71,"code":72,"language":73,"meta":39,"style":39},"language-python shiki shiki-themes github-light github-dark-high-contrast","import pandas as pd\n\nraw = pd.read_excel(\"accounts.xlsx\", dtype=str, keep_default_na=False)\nPLACEHOLDERS = {\"\", \"n\u002Fa\", \"na\", \"-\", \"--\", \"tbc\", \"tbd\", \"none\", \"unknown\", \"?\"}\n\ndef to_missing(s):\n    cleaned = s.str.strip()\n    return cleaned.mask(cleaned.str.lower().isin(PLACEHOLDERS))\n\ndf = raw.apply(to_missing)\nprint(\"empty cells as read:\", int((raw == \"\").sum().sum()), \"| missing after cleaning:\", int(df.isna().sum().sum()))\nprint(df.isna().sum().to_string())\n","python",[41,75,76,92,99,140,203,208,221,232,246,251,262,301],{"__ignoreMap":39},[44,77,78,82,86,89],{"class":46,"line":47},[44,79,81],{"class":80},"s-kum","import",[44,83,85],{"class":84},"skGVy"," pandas ",[44,87,88],{"class":80},"as",[44,90,91],{"class":84}," pd\n",[44,93,95],{"class":46,"line":94},2,[44,96,98],{"emptyLinePlaceholder":97},true,"\n",[44,100,102,105,108,111,114,117,121,123,127,129,132,134,137],{"class":46,"line":101},3,[44,103,104],{"class":84},"raw ",[44,106,107],{"class":80},"=",[44,109,110],{"class":84}," pd.read_excel(",[44,112,113],{"class":54},"\"accounts.xlsx\"",[44,115,116],{"class":84},", ",[44,118,120],{"class":119},"sa561","dtype",[44,122,107],{"class":80},[44,124,126],{"class":125},"sP0c6","str",[44,128,116],{"class":84},[44,130,131],{"class":119},"keep_default_na",[44,133,107],{"class":80},[44,135,136],{"class":125},"False",[44,138,139],{"class":84},")\n",[44,141,143,146,149,152,155,157,160,162,165,167,170,172,175,177,180,182,185,187,190,192,195,197,200],{"class":46,"line":142},4,[44,144,145],{"class":125},"PLACEHOLDERS",[44,147,148],{"class":80}," =",[44,150,151],{"class":84}," {",[44,153,154],{"class":54},"\"\"",[44,156,116],{"class":84},[44,158,159],{"class":54},"\"n\u002Fa\"",[44,161,116],{"class":84},[44,163,164],{"class":54},"\"na\"",[44,166,116],{"class":84},[44,168,169],{"class":54},"\"-\"",[44,171,116],{"class":84},[44,173,174],{"class":54},"\"--\"",[44,176,116],{"class":84},[44,178,179],{"class":54},"\"tbc\"",[44,181,116],{"class":84},[44,183,184],{"class":54},"\"tbd\"",[44,186,116],{"class":84},[44,188,189],{"class":54},"\"none\"",[44,191,116],{"class":84},[44,193,194],{"class":54},"\"unknown\"",[44,196,116],{"class":84},[44,198,199],{"class":54},"\"?\"",[44,201,202],{"class":84},"}\n",[44,204,206],{"class":46,"line":205},5,[44,207,98],{"emptyLinePlaceholder":97},[44,209,211,214,218],{"class":46,"line":210},6,[44,212,213],{"class":80},"def",[44,215,217],{"class":216},"s_Opv"," to_missing",[44,219,220],{"class":84},"(s):\n",[44,222,224,227,229],{"class":46,"line":223},7,[44,225,226],{"class":84},"    cleaned ",[44,228,107],{"class":80},[44,230,231],{"class":84}," s.str.strip()\n",[44,233,235,238,241,243],{"class":46,"line":234},8,[44,236,237],{"class":80},"    return",[44,239,240],{"class":84}," cleaned.mask(cleaned.str.lower().isin(",[44,242,145],{"class":125},[44,244,245],{"class":84},"))\n",[44,247,249],{"class":46,"line":248},9,[44,250,98],{"emptyLinePlaceholder":97},[44,252,254,257,259],{"class":46,"line":253},10,[44,255,256],{"class":84},"df ",[44,258,107],{"class":80},[44,260,261],{"class":84}," raw.apply(to_missing)\n",[44,263,265,268,271,274,276,279,282,285,288,291,294,296,298],{"class":46,"line":264},11,[44,266,267],{"class":125},"print",[44,269,270],{"class":84},"(",[44,272,273],{"class":54},"\"empty cells as read:\"",[44,275,116],{"class":84},[44,277,278],{"class":125},"int",[44,280,281],{"class":84},"((raw ",[44,283,284],{"class":80},"==",[44,286,287],{"class":54}," \"\"",[44,289,290],{"class":84},").sum().sum()), ",[44,292,293],{"class":54},"\"| missing after cleaning:\"",[44,295,116],{"class":84},[44,297,278],{"class":125},[44,299,300],{"class":84},"(df.isna().sum().sum()))\n",[44,302,304,306],{"class":46,"line":303},12,[44,305,267],{"class":125},[44,307,308],{"class":84},"(df.isna().sum().to_string())\n",[10,310,311,312,315,316,319],{},"Reading with ",[41,313,314],{},"dtype=str"," and ",[41,317,318],{},"keep_default_na=False"," gives the cells exactly as typed, so the code — not pandas' defaults — decides what counts as missing. Every value is trimmed, so a lone space counts as empty, and the placeholders people type when they have nothing to enter become real missing values. Counting before and after shows how much of the missingness was disguised as text: only 15 cells were literally empty, but 53 are missing once placeholders are recognised, so a column-level check on the raw data would have reported most of these fields as far more complete than they are. Keep the placeholder list short and explicit, and review it against the actual values in your data — a column of product codes may legitimately contain \"NA\" for North America, in which case that column needs its own rule.",[29,321,323],{"id":322},"step-2-score-each-row","Step 2 — Score each row",[10,325,326],{},"Not every field matters equally. The account number and company name are essential; industry and employee count are nice to have. Give each column a weight and score each row by the weighted share of its fields that are missing:",[34,328,330],{"className":71,"code":329,"language":73,"meta":39,"style":39},"WEIGHTS = {\"Account\": 3, \"Company\": 3, \"Contact\": 2, \"Email\": 2, \"Phone\": 1, \"Country\": 2, \"Industry\": 1, \"Employees\": 1}\ncols = list(WEIGHTS)\nmissing = df[cols].isna()\ndf[\"missing_fields\"] = missing.sum(axis=1)\ndf[\"missing_score\"] = (missing * pd.Series(WEIGHTS)).sum(axis=1) \u002F sum(WEIGHTS.values())\ndf[\"missing_list\"] = missing.apply(lambda r: \", \".join(c for c in cols if r[c]), axis=1)\nREQUIRED = [\"Account\", \"Company\"]\ndf[\"required_missing\"] = df[REQUIRED].isna().any(axis=1)\n\nTHRESHOLD = 0.25\ndf[\"flag\"] = df[\"required_missing\"] | (df[\"missing_score\"] >= THRESHOLD)\nprint(df.loc[df[\"flag\"], [\"Account\", \"missing_fields\", \"missing_score\", \"missing_list\"]].round(2).to_string(index=False))\n",[41,331,332,417,433,443,468,515,567,586,613,617,627,662],{"__ignoreMap":39},[44,333,334,337,339,341,344,347,350,352,355,357,359,361,364,366,369,371,374,376,378,380,383,385,388,390,393,395,397,399,402,404,406,408,411,413,415],{"class":46,"line":47},[44,335,336],{"class":125},"WEIGHTS",[44,338,148],{"class":80},[44,340,151],{"class":84},[44,342,343],{"class":54},"\"Account\"",[44,345,346],{"class":84},": ",[44,348,349],{"class":125},"3",[44,351,116],{"class":84},[44,353,354],{"class":54},"\"Company\"",[44,356,346],{"class":84},[44,358,349],{"class":125},[44,360,116],{"class":84},[44,362,363],{"class":54},"\"Contact\"",[44,365,346],{"class":84},[44,367,368],{"class":125},"2",[44,370,116],{"class":84},[44,372,373],{"class":54},"\"Email\"",[44,375,346],{"class":84},[44,377,368],{"class":125},[44,379,116],{"class":84},[44,381,382],{"class":54},"\"Phone\"",[44,384,346],{"class":84},[44,386,387],{"class":125},"1",[44,389,116],{"class":84},[44,391,392],{"class":54},"\"Country\"",[44,394,346],{"class":84},[44,396,368],{"class":125},[44,398,116],{"class":84},[44,400,401],{"class":54},"\"Industry\"",[44,403,346],{"class":84},[44,405,387],{"class":125},[44,407,116],{"class":84},[44,409,410],{"class":54},"\"Employees\"",[44,412,346],{"class":84},[44,414,387],{"class":125},[44,416,202],{"class":84},[44,418,419,422,424,427,429,431],{"class":46,"line":94},[44,420,421],{"class":84},"cols ",[44,423,107],{"class":80},[44,425,426],{"class":125}," list",[44,428,270],{"class":84},[44,430,336],{"class":125},[44,432,139],{"class":84},[44,434,435,438,440],{"class":46,"line":101},[44,436,437],{"class":84},"missing ",[44,439,107],{"class":80},[44,441,442],{"class":84}," df[cols].isna()\n",[44,444,445,448,451,454,456,459,462,464,466],{"class":46,"line":142},[44,446,447],{"class":84},"df[",[44,449,450],{"class":54},"\"missing_fields\"",[44,452,453],{"class":84},"] ",[44,455,107],{"class":80},[44,457,458],{"class":84}," missing.sum(",[44,460,461],{"class":119},"axis",[44,463,107],{"class":80},[44,465,387],{"class":125},[44,467,139],{"class":84},[44,469,470,472,475,477,479,482,485,488,490,493,495,497,499,502,505,508,510,512],{"class":46,"line":205},[44,471,447],{"class":84},[44,473,474],{"class":54},"\"missing_score\"",[44,476,453],{"class":84},[44,478,107],{"class":80},[44,480,481],{"class":84}," (missing ",[44,483,484],{"class":80},"*",[44,486,487],{"class":84}," pd.Series(",[44,489,336],{"class":125},[44,491,492],{"class":84},")).sum(",[44,494,461],{"class":119},[44,496,107],{"class":80},[44,498,387],{"class":125},[44,500,501],{"class":84},") ",[44,503,504],{"class":80},"\u002F",[44,506,507],{"class":125}," sum",[44,509,270],{"class":84},[44,511,336],{"class":125},[44,513,514],{"class":84},".values())\n",[44,516,517,519,522,524,526,529,532,535,538,541,544,547,550,553,556,559,561,563,565],{"class":46,"line":210},[44,518,447],{"class":84},[44,520,521],{"class":54},"\"missing_list\"",[44,523,453],{"class":84},[44,525,107],{"class":80},[44,527,528],{"class":84}," missing.apply(",[44,530,531],{"class":80},"lambda",[44,533,534],{"class":84}," r: ",[44,536,537],{"class":54},"\", \"",[44,539,540],{"class":84},".join(c ",[44,542,543],{"class":80},"for",[44,545,546],{"class":84}," c ",[44,548,549],{"class":80},"in",[44,551,552],{"class":84}," cols ",[44,554,555],{"class":80},"if",[44,557,558],{"class":84}," r[c]), ",[44,560,461],{"class":119},[44,562,107],{"class":80},[44,564,387],{"class":125},[44,566,139],{"class":84},[44,568,569,572,574,577,579,581,583],{"class":46,"line":223},[44,570,571],{"class":125},"REQUIRED",[44,573,148],{"class":80},[44,575,576],{"class":84}," [",[44,578,343],{"class":54},[44,580,116],{"class":84},[44,582,354],{"class":54},[44,584,585],{"class":84},"]\n",[44,587,588,590,593,595,597,600,602,605,607,609,611],{"class":46,"line":234},[44,589,447],{"class":84},[44,591,592],{"class":54},"\"required_missing\"",[44,594,453],{"class":84},[44,596,107],{"class":80},[44,598,599],{"class":84}," df[",[44,601,571],{"class":125},[44,603,604],{"class":84},"].isna().any(",[44,606,461],{"class":119},[44,608,107],{"class":80},[44,610,387],{"class":125},[44,612,139],{"class":84},[44,614,615],{"class":46,"line":248},[44,616,98],{"emptyLinePlaceholder":97},[44,618,619,622,624],{"class":46,"line":253},[44,620,621],{"class":125},"THRESHOLD",[44,623,148],{"class":80},[44,625,626],{"class":125}," 0.25\n",[44,628,629,631,634,636,638,640,642,644,647,650,652,654,657,660],{"class":46,"line":264},[44,630,447],{"class":84},[44,632,633],{"class":54},"\"flag\"",[44,635,453],{"class":84},[44,637,107],{"class":80},[44,639,599],{"class":84},[44,641,592],{"class":54},[44,643,453],{"class":84},[44,645,646],{"class":80},"|",[44,648,649],{"class":84}," (df[",[44,651,474],{"class":54},[44,653,453],{"class":84},[44,655,656],{"class":80},">=",[44,658,659],{"class":125}," THRESHOLD",[44,661,139],{"class":84},[44,663,664,666,669,671,674,676,678,680,682,684,686,688,691,693,696,699,701,703],{"class":46,"line":303},[44,665,267],{"class":125},[44,667,668],{"class":84},"(df.loc[df[",[44,670,633],{"class":54},[44,672,673],{"class":84},"], [",[44,675,343],{"class":54},[44,677,116],{"class":84},[44,679,450],{"class":54},[44,681,116],{"class":84},[44,683,474],{"class":54},[44,685,116],{"class":84},[44,687,521],{"class":54},[44,689,690],{"class":84},"]].round(",[44,692,368],{"class":125},[44,694,695],{"class":84},").to_string(",[44,697,698],{"class":119},"index",[44,700,107],{"class":80},[44,702,136],{"class":125},[44,704,245],{"class":84},[10,706,707,710,711,714,715,718],{},[41,708,709],{},"isna().sum(axis=1)"," counts missing fields per row; multiplying the missing mask by the weights and dividing by their total gives a score from 0 (complete) to 1 (empty). A row is flagged when a ",[14,712,713],{},"required"," field is missing, whatever its score, or when a quarter or more of the weighted information is absent. The ",[14,716,717],{},"missing_list"," column spells out which fields are empty, which is what the person following up needs. Five of the forty rows are flagged. The three abandoned records — AC-1007, AC-1023 and AC-1031 — score 0.6 or more and are missing their company name, a required field. Two others cross the threshold without a required gap: AC-1011 is missing its contact, phone and country, and AC-1015 its email, industry and employee count.",[720,721,729,730,729,734,729,738,729,746,729,755,729,759,729,766,729,770,729,774,729,778,729,782,729,784,729,788,729,790,729,793,729,796,729,799,729,801,729,805,729,807,729,810,729,813,729,816,729,818,729,821,729,823,729,826,729,829,729,832,729,834,729,837,729,839,729,842,729,845,729,849,729,852,729,856,729,859,729,861,729,864,729,866,729,869,729,871,729,874,729,876,729,878,729,880,729,883,729,888,729,892,729,894,729,896,729,898,729,901,729,903,729,906,729,908,729,910,729,912,729,915,729,917,729,919,729,925,729,929,729,936,729,941,729,949],"svg",{"viewBox":722,"role":723,"ariaLabelledBy":724,"xmlns":727,"style":728},"4 0 752 192","img",[725,726],"mr-score-t","mr-score-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:752px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[731,732,733],"title",{"id":725},"From missing cells to a row score",[735,736,737],"desc",{"id":726},"Each missing field adds its weight, required fields weighing more than optional ones, and the total is divided by the sum of all weights to give a score between 0 and 1. A row is flagged if a required field is missing or the score is a quarter or more.",[739,740],"rect",{"x":741,"y":742,"width":743,"height":744,"fill":745},"4","0","752","192","#ffffff",[739,747],{"x":748,"y":749,"width":750,"height":751,"rx":742,"fill":752,"stroke":753,"style":754},"20.0","40.0","32.0","24.0","#e7ebef","var(--line,#cdd5e6)","stroke-width:1px",[739,756],{"x":757,"y":749,"width":758,"height":751,"rx":742,"fill":752,"stroke":753,"style":754},"52.0","73.1",[760,761,765],"text",{"x":762,"y":763,"style":764},"88.5","57.0","font-size:11px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","A",[739,767],{"x":768,"y":749,"width":769,"height":751,"rx":742,"fill":752,"stroke":753,"style":754},"125.1","178.1",[760,771,773],{"x":772,"y":763,"style":764},"214.1","B",[739,775],{"x":748,"y":776,"width":750,"height":777,"rx":742,"fill":752,"stroke":753,"style":754},"64.0","28.0",[760,779,387],{"x":780,"y":781,"style":764},"36.0","83.0",[739,783],{"x":757,"y":776,"width":758,"height":777,"rx":742,"fill":745,"stroke":753,"style":754},[760,785,787],{"x":762,"y":781,"style":786},"font-size:12px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","Account",[739,789],{"x":768,"y":776,"width":769,"height":777,"rx":742,"fill":745,"stroke":753,"style":754},[760,791,792],{"x":772,"y":781,"style":786},"Missing fields",[739,794],{"x":748,"y":795,"width":750,"height":777,"rx":742,"fill":752,"stroke":753,"style":754},"92.0",[760,797,368],{"x":780,"y":798,"style":764},"111.0",[739,800],{"x":757,"y":795,"width":758,"height":777,"rx":742,"fill":745,"stroke":753,"style":754},[760,802,804],{"x":762,"y":798,"style":803},"font-size:12px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","AC-1007",[739,806],{"x":768,"y":795,"width":769,"height":777,"rx":742,"fill":745,"stroke":753,"style":754},[760,808,809],{"x":772,"y":798,"style":803},"6 fields incl. Company",[739,811],{"x":748,"y":812,"width":750,"height":777,"rx":742,"fill":752,"stroke":753,"style":754},"120.0",[760,814,349],{"x":780,"y":815,"style":764},"139.0",[739,817],{"x":757,"y":812,"width":758,"height":777,"rx":742,"fill":745,"stroke":753,"style":754},[760,819,820],{"x":762,"y":815,"style":803},"AC-1011",[739,822],{"x":768,"y":812,"width":769,"height":777,"rx":742,"fill":745,"stroke":753,"style":754},[760,824,825],{"x":772,"y":815,"style":803},"Contact, Phone, Country",[739,827],{"x":748,"y":828,"width":750,"height":777,"rx":742,"fill":752,"stroke":753,"style":754},"148.0",[760,830,741],{"x":780,"y":831,"style":764},"167.0",[739,833],{"x":757,"y":828,"width":758,"height":777,"rx":742,"fill":745,"stroke":753,"style":754},[760,835,836],{"x":762,"y":831,"style":803},"AC-1015",[739,838],{"x":768,"y":828,"width":769,"height":777,"rx":742,"fill":745,"stroke":753,"style":754},[760,840,841],{"x":772,"y":831,"style":803},"Email, Industry, Employees",[739,843],{"x":844,"y":749,"width":750,"height":751,"rx":742,"fill":752,"stroke":753,"style":754},"495.4",[739,846],{"x":847,"y":749,"width":848,"height":751,"rx":742,"fill":752,"stroke":753,"style":754},"527.4","55.9",[760,850,765],{"x":851,"y":763,"style":764},"555.3",[739,853],{"x":854,"y":749,"width":855,"height":751,"rx":742,"fill":752,"stroke":753,"style":754},"583.3","156.7",[760,857,773],{"x":858,"y":763,"style":764},"661.6",[739,860],{"x":844,"y":776,"width":750,"height":777,"rx":742,"fill":752,"stroke":753,"style":754},[760,862,387],{"x":863,"y":781,"style":764},"511.4",[739,865],{"x":847,"y":776,"width":848,"height":777,"rx":742,"fill":745,"stroke":753,"style":754},[760,867,868],{"x":851,"y":781,"style":786},"Score",[739,870],{"x":854,"y":776,"width":855,"height":777,"rx":742,"fill":745,"stroke":753,"style":754},[760,872,873],{"x":858,"y":781,"style":786},"Flag",[739,875],{"x":844,"y":795,"width":750,"height":777,"rx":742,"fill":752,"stroke":753,"style":754},[760,877,368],{"x":863,"y":798,"style":764},[739,879],{"x":847,"y":795,"width":848,"height":777,"rx":742,"fill":745,"stroke":753,"style":754},[760,881,882],{"x":851,"y":798,"style":803},"0.67",[739,884],{"x":854,"y":795,"width":855,"height":777,"rx":742,"fill":885,"stroke":886,"style":887},"#fdefd8","var(--gold,#b4740a)","stroke-width:1.5px",[760,889,891],{"x":858,"y":798,"style":890},"font-size:12px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","yes — required missing",[739,893],{"x":844,"y":812,"width":750,"height":777,"rx":742,"fill":752,"stroke":753,"style":754},[760,895,349],{"x":863,"y":815,"style":764},[739,897],{"x":847,"y":812,"width":848,"height":777,"rx":742,"fill":745,"stroke":753,"style":754},[760,899,900],{"x":851,"y":815,"style":803},"0.33",[739,902],{"x":854,"y":812,"width":855,"height":777,"rx":742,"fill":745,"stroke":753,"style":754},[760,904,905],{"x":858,"y":815,"style":803},"yes — score ≥ 0.25",[739,907],{"x":844,"y":828,"width":750,"height":777,"rx":742,"fill":752,"stroke":753,"style":754},[760,909,741],{"x":863,"y":831,"style":764},[739,911],{"x":847,"y":828,"width":848,"height":777,"rx":742,"fill":745,"stroke":753,"style":754},[760,913,914],{"x":851,"y":831,"style":803},"0.27",[739,916],{"x":854,"y":828,"width":855,"height":777,"rx":742,"fill":745,"stroke":753,"style":754},[760,918,905],{"x":858,"y":831,"style":803},[760,920,924],{"x":921,"y":922,"style":923},"161.6","29.0","font-size:12.5px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:middle","before",[760,926,928],{"x":927,"y":922,"style":923},"617.7","after",[46,930],{"x1":931,"y1":932,"x2":933,"y2":932,"stroke":934,"style":935},"313.2","108.0","481.4","var(--brand,#5b5cf0)","stroke-width:2px",[937,938],"polygon",{"points":939,"fill":940},"485.4,108.0 476.4,113.0 476.4,103.0","#5b5cf0",[739,942],{"x":943,"y":944,"width":945,"height":946,"rx":947,"fill":745,"stroke":753,"style":948},"338.4","75.0","121.8","22.0","11","stroke-width:1.2px",[760,950,954],{"x":951,"y":952,"style":953},"399.3","90.0","font-size:11.5px;font-weight:400;fill:var(--brand-strong,#4338ca);text-anchor:middle;font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace","weighted share",[29,956,958],{"id":957},"step-3-who-needs-to-follow-up","Step 3 — Who needs to follow up",[10,960,961],{},"Missing data usually has a source. Summarise completeness by the person or system that entered the records:",[34,963,965],{"className":71,"code":964,"language":73,"meta":39,"style":39},"by_person = df.groupby(\"Entered by\").agg(records=(\"Account\", \"count\"), flagged=(\"flag\", \"sum\"),\n                                         avg_score=(\"missing_score\", \"mean\"))\nby_person[\"flagged_share\"] = by_person[\"flagged\"] \u002F by_person[\"records\"]\nprint(by_person.round(2).to_string())\nfield_gaps = df[cols].isna().mean().sort_values(ascending=False)\nprint(field_gaps.round(2).to_string())\n",[41,966,967,1016,1034,1063,1075,1094],{"__ignoreMap":39},[44,968,969,972,974,977,980,983,985,987,989,991,993,996,999,1002,1004,1006,1008,1010,1013],{"class":46,"line":47},[44,970,971],{"class":84},"by_person ",[44,973,107],{"class":80},[44,975,976],{"class":84}," df.groupby(",[44,978,979],{"class":54},"\"Entered by\"",[44,981,982],{"class":84},").agg(",[44,984,20],{"class":119},[44,986,107],{"class":80},[44,988,270],{"class":84},[44,990,343],{"class":54},[44,992,116],{"class":84},[44,994,995],{"class":54},"\"count\"",[44,997,998],{"class":84},"), ",[44,1000,1001],{"class":119},"flagged",[44,1003,107],{"class":80},[44,1005,270],{"class":84},[44,1007,633],{"class":54},[44,1009,116],{"class":84},[44,1011,1012],{"class":54},"\"sum\"",[44,1014,1015],{"class":84},"),\n",[44,1017,1018,1021,1023,1025,1027,1029,1032],{"class":46,"line":94},[44,1019,1020],{"class":119},"                                         avg_score",[44,1022,107],{"class":80},[44,1024,270],{"class":84},[44,1026,474],{"class":54},[44,1028,116],{"class":84},[44,1030,1031],{"class":54},"\"mean\"",[44,1033,245],{"class":84},[44,1035,1036,1039,1042,1044,1046,1049,1052,1054,1056,1058,1061],{"class":46,"line":101},[44,1037,1038],{"class":84},"by_person[",[44,1040,1041],{"class":54},"\"flagged_share\"",[44,1043,453],{"class":84},[44,1045,107],{"class":80},[44,1047,1048],{"class":84}," by_person[",[44,1050,1051],{"class":54},"\"flagged\"",[44,1053,453],{"class":84},[44,1055,504],{"class":80},[44,1057,1048],{"class":84},[44,1059,1060],{"class":54},"\"records\"",[44,1062,585],{"class":84},[44,1064,1065,1067,1070,1072],{"class":46,"line":142},[44,1066,267],{"class":125},[44,1068,1069],{"class":84},"(by_person.round(",[44,1071,368],{"class":125},[44,1073,1074],{"class":84},").to_string())\n",[44,1076,1077,1080,1082,1085,1088,1090,1092],{"class":46,"line":205},[44,1078,1079],{"class":84},"field_gaps ",[44,1081,107],{"class":80},[44,1083,1084],{"class":84}," df[cols].isna().mean().sort_values(",[44,1086,1087],{"class":119},"ascending",[44,1089,107],{"class":80},[44,1091,136],{"class":125},[44,1093,139],{"class":84},[44,1095,1096,1098,1101,1103],{"class":46,"line":210},[44,1097,267],{"class":125},[44,1099,1100],{"class":84},"(field_gaps.round(",[44,1102,368],{"class":125},[44,1104,1074],{"class":84},[10,1106,1107,1108,27],{},"The per-person summary shows whether gaps are spread evenly or concentrated — here all three abandoned records were entered by ben, which suggests an interrupted session rather than a training issue, while amira's thirteen records are all complete enough. The per-field view puts industry and employee count at the top, each empty in 30% of records, with phone numbers close behind. The per-field view shows which fields are most often left empty across everyone, which points at the form rather than the people: a field that is empty for nearly a third of records is either unnecessary or badly explained. Missingness in general — which values are missing and in what patterns — is covered in ",[23,1109,1111],{"href":1110},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Ffind-and-report-missing-values-in-an-excel-file\u002F","Find and Report Missing Values in an Excel File",[720,1113,729,1119,729,1122,729,1125,729,1129,729,1134,729,1140,729,1146,729,1151,729,1154,729,1158,729,1161,729,1166,729,1169,729,1173,729,1177],{"viewBox":1114,"role":723,"ariaLabelledBy":1115,"xmlns":727,"style":1118},"4 0 744 140",[1116,1117],"mr-person-t","mr-person-d","width:100%;max-width:744px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[731,1120,1121],{"id":1116},"Flagged records by who entered them",[735,1123,1124],{"id":1117},"Amira entered 13 records with none flagged. Ben entered 17 with 3 flagged, the three abandoned records. Chloe entered 10 with 2 flagged.",[739,1126],{"x":741,"y":742,"width":1127,"height":1128,"fill":745},"744","140",[739,1130],{"x":748,"y":1131,"width":1132,"height":777,"rx":1133,"fill":752,"stroke":753,"style":754},"16.0","144.3","7",[760,1135,1139],{"x":1136,"y":1137,"style":1138},"92.1","34.5","font-size:12.5px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","amira (13 records)",[739,1141],{"x":1142,"y":1131,"width":1143,"height":777,"rx":1133,"fill":1144,"stroke":1145,"style":887},"174.3","6.0","#d9f4f1","var(--teal,#0f9488)",[760,1147,1150],{"x":1148,"y":1137,"style":1149},"188.3","font-size:12px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:start","0 flagged",[739,1152],{"x":748,"y":1153,"width":1132,"height":777,"rx":1133,"fill":752,"stroke":753,"style":754},"56.0",[760,1155,1157],{"x":1136,"y":1156,"style":1138},"74.5","ben (17 records)",[739,1159],{"x":1142,"y":1153,"width":1160,"height":777,"rx":1133,"fill":885,"stroke":886,"style":887},"491.1",[760,1162,1165],{"x":1163,"y":1156,"style":1164},"673.4","font-size:12px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:start","3 flagged",[739,1167],{"x":748,"y":1168,"width":1132,"height":777,"rx":1133,"fill":752,"stroke":753,"style":754},"96.0",[760,1170,1172],{"x":1136,"y":1171,"style":1138},"114.5","chloe (10 records)",[739,1174],{"x":1142,"y":1168,"width":1175,"height":777,"rx":1133,"fill":1176,"stroke":934,"style":887},"327.4","#f0f4ff",[760,1178,1181],{"x":1179,"y":1171,"style":1180},"509.7","font-size:12px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:start","2 flagged",[29,1183,1185],{"id":1184},"step-4-highlight-flagged-rows-and-write-a-follow-up-list","Step 4 — Highlight flagged rows and write a follow-up list",[34,1187,1189],{"className":71,"code":1188,"language":73,"meta":39,"style":39},"from openpyxl import load_workbook\nfrom openpyxl.styles import PatternFill\n\nout = df.drop(columns=[\"required_missing\"]).copy()\nwith pd.ExcelWriter(\"accounts_checked.xlsx\", engine=\"openpyxl\") as xw:\n    out.to_excel(xw, sheet_name=\"Accounts\", index=False)\n    out[out[\"flag\"]].sort_values(\"missing_score\", ascending=False)[\n        [\"Account\", \"Company\", \"Entered by\", \"missing_score\", \"missing_list\"]].to_excel(xw, sheet_name=\"Follow up\", index=False)\n    by_person.to_excel(xw, sheet_name=\"By person\")\n\nwb = load_workbook(\"accounts_checked.xlsx\")\nws = wb[\"Accounts\"]\nflag_col = [c.value for c in ws[1]].index(\"flag\") + 1\namber, grey = PatternFill(\"solid\", fgColor=\"FFEB9C\"), PatternFill(\"solid\", fgColor=\"F2F2F2\")\nfor row in ws.iter_rows(min_row=2):\n    if row[flag_col - 1].value:\n        for cell in row:\n            cell.fill = amber\n    for cell in row[:len(cols)]:\n        if cell.value is None:\n            cell.fill = grey if not row[flag_col - 1].value else PatternFill(\"solid\", fgColor=\"F4B183\")\nwb.save(\"accounts_checked.xlsx\")\nprint(f\"{int(out['flag'].sum())} of {len(out)} rows flagged; follow-up list written\")\n",[41,1190,1191,1204,1216,1220,1243,1271,1294,1317,1360,1374,1378,1392,1406,1441,1481,1504,1522,1536,1547,1566,1584,1626,1636],{"__ignoreMap":39},[44,1192,1193,1196,1199,1201],{"class":46,"line":47},[44,1194,1195],{"class":80},"from",[44,1197,1198],{"class":84}," openpyxl ",[44,1200,81],{"class":80},[44,1202,1203],{"class":84}," load_workbook\n",[44,1205,1206,1208,1211,1213],{"class":46,"line":94},[44,1207,1195],{"class":80},[44,1209,1210],{"class":84}," openpyxl.styles ",[44,1212,81],{"class":80},[44,1214,1215],{"class":84}," PatternFill\n",[44,1217,1218],{"class":46,"line":101},[44,1219,98],{"emptyLinePlaceholder":97},[44,1221,1222,1225,1227,1230,1233,1235,1238,1240],{"class":46,"line":142},[44,1223,1224],{"class":84},"out ",[44,1226,107],{"class":80},[44,1228,1229],{"class":84}," df.drop(",[44,1231,1232],{"class":119},"columns",[44,1234,107],{"class":80},[44,1236,1237],{"class":84},"[",[44,1239,592],{"class":54},[44,1241,1242],{"class":84},"]).copy()\n",[44,1244,1245,1248,1251,1254,1256,1259,1261,1264,1266,1268],{"class":46,"line":205},[44,1246,1247],{"class":80},"with",[44,1249,1250],{"class":84}," pd.ExcelWriter(",[44,1252,1253],{"class":54},"\"accounts_checked.xlsx\"",[44,1255,116],{"class":84},[44,1257,1258],{"class":119},"engine",[44,1260,107],{"class":80},[44,1262,1263],{"class":54},"\"openpyxl\"",[44,1265,501],{"class":84},[44,1267,88],{"class":80},[44,1269,1270],{"class":84}," xw:\n",[44,1272,1273,1276,1279,1281,1284,1286,1288,1290,1292],{"class":46,"line":210},[44,1274,1275],{"class":84},"    out.to_excel(xw, ",[44,1277,1278],{"class":119},"sheet_name",[44,1280,107],{"class":80},[44,1282,1283],{"class":54},"\"Accounts\"",[44,1285,116],{"class":84},[44,1287,698],{"class":119},[44,1289,107],{"class":80},[44,1291,136],{"class":125},[44,1293,139],{"class":84},[44,1295,1296,1299,1301,1304,1306,1308,1310,1312,1314],{"class":46,"line":223},[44,1297,1298],{"class":84},"    out[out[",[44,1300,633],{"class":54},[44,1302,1303],{"class":84},"]].sort_values(",[44,1305,474],{"class":54},[44,1307,116],{"class":84},[44,1309,1087],{"class":119},[44,1311,107],{"class":80},[44,1313,136],{"class":125},[44,1315,1316],{"class":84},")[\n",[44,1318,1319,1322,1324,1326,1328,1330,1332,1334,1336,1338,1340,1343,1345,1347,1350,1352,1354,1356,1358],{"class":46,"line":234},[44,1320,1321],{"class":84},"        [",[44,1323,343],{"class":54},[44,1325,116],{"class":84},[44,1327,354],{"class":54},[44,1329,116],{"class":84},[44,1331,979],{"class":54},[44,1333,116],{"class":84},[44,1335,474],{"class":54},[44,1337,116],{"class":84},[44,1339,521],{"class":54},[44,1341,1342],{"class":84},"]].to_excel(xw, ",[44,1344,1278],{"class":119},[44,1346,107],{"class":80},[44,1348,1349],{"class":54},"\"Follow up\"",[44,1351,116],{"class":84},[44,1353,698],{"class":119},[44,1355,107],{"class":80},[44,1357,136],{"class":125},[44,1359,139],{"class":84},[44,1361,1362,1365,1367,1369,1372],{"class":46,"line":248},[44,1363,1364],{"class":84},"    by_person.to_excel(xw, ",[44,1366,1278],{"class":119},[44,1368,107],{"class":80},[44,1370,1371],{"class":54},"\"By person\"",[44,1373,139],{"class":84},[44,1375,1376],{"class":46,"line":253},[44,1377,98],{"emptyLinePlaceholder":97},[44,1379,1380,1383,1385,1388,1390],{"class":46,"line":264},[44,1381,1382],{"class":84},"wb ",[44,1384,107],{"class":80},[44,1386,1387],{"class":84}," load_workbook(",[44,1389,1253],{"class":54},[44,1391,139],{"class":84},[44,1393,1394,1397,1399,1402,1404],{"class":46,"line":303},[44,1395,1396],{"class":84},"ws ",[44,1398,107],{"class":80},[44,1400,1401],{"class":84}," wb[",[44,1403,1283],{"class":54},[44,1405,585],{"class":84},[44,1407,1409,1412,1414,1417,1419,1421,1423,1426,1428,1431,1433,1435,1438],{"class":46,"line":1408},13,[44,1410,1411],{"class":84},"flag_col ",[44,1413,107],{"class":80},[44,1415,1416],{"class":84}," [c.value ",[44,1418,543],{"class":80},[44,1420,546],{"class":84},[44,1422,549],{"class":80},[44,1424,1425],{"class":84}," ws[",[44,1427,387],{"class":125},[44,1429,1430],{"class":84},"]].index(",[44,1432,633],{"class":54},[44,1434,501],{"class":84},[44,1436,1437],{"class":80},"+",[44,1439,1440],{"class":125}," 1\n",[44,1442,1444,1447,1449,1452,1455,1457,1460,1462,1465,1468,1470,1472,1474,1476,1479],{"class":46,"line":1443},14,[44,1445,1446],{"class":84},"amber, grey ",[44,1448,107],{"class":80},[44,1450,1451],{"class":84}," PatternFill(",[44,1453,1454],{"class":54},"\"solid\"",[44,1456,116],{"class":84},[44,1458,1459],{"class":119},"fgColor",[44,1461,107],{"class":80},[44,1463,1464],{"class":54},"\"FFEB9C\"",[44,1466,1467],{"class":84},"), PatternFill(",[44,1469,1454],{"class":54},[44,1471,116],{"class":84},[44,1473,1459],{"class":119},[44,1475,107],{"class":80},[44,1477,1478],{"class":54},"\"F2F2F2\"",[44,1480,139],{"class":84},[44,1482,1484,1486,1489,1491,1494,1497,1499,1501],{"class":46,"line":1483},15,[44,1485,543],{"class":80},[44,1487,1488],{"class":84}," row ",[44,1490,549],{"class":80},[44,1492,1493],{"class":84}," ws.iter_rows(",[44,1495,1496],{"class":119},"min_row",[44,1498,107],{"class":80},[44,1500,368],{"class":125},[44,1502,1503],{"class":84},"):\n",[44,1505,1507,1510,1513,1516,1519],{"class":46,"line":1506},16,[44,1508,1509],{"class":80},"    if",[44,1511,1512],{"class":84}," row[flag_col ",[44,1514,1515],{"class":80},"-",[44,1517,1518],{"class":125}," 1",[44,1520,1521],{"class":84},"].value:\n",[44,1523,1525,1528,1531,1533],{"class":46,"line":1524},17,[44,1526,1527],{"class":80},"        for",[44,1529,1530],{"class":84}," cell ",[44,1532,549],{"class":80},[44,1534,1535],{"class":84}," row:\n",[44,1537,1539,1542,1544],{"class":46,"line":1538},18,[44,1540,1541],{"class":84},"            cell.fill ",[44,1543,107],{"class":80},[44,1545,1546],{"class":84}," amber\n",[44,1548,1550,1553,1555,1557,1560,1563],{"class":46,"line":1549},19,[44,1551,1552],{"class":80},"    for",[44,1554,1530],{"class":84},[44,1556,549],{"class":80},[44,1558,1559],{"class":84}," row[:",[44,1561,1562],{"class":125},"len",[44,1564,1565],{"class":84},"(cols)]:\n",[44,1567,1569,1572,1575,1578,1581],{"class":46,"line":1568},20,[44,1570,1571],{"class":80},"        if",[44,1573,1574],{"class":84}," cell.value ",[44,1576,1577],{"class":80},"is",[44,1579,1580],{"class":125}," None",[44,1582,1583],{"class":84},":\n",[44,1585,1587,1589,1591,1594,1596,1599,1601,1603,1605,1608,1611,1613,1615,1617,1619,1621,1624],{"class":46,"line":1586},21,[44,1588,1541],{"class":84},[44,1590,107],{"class":80},[44,1592,1593],{"class":84}," grey ",[44,1595,555],{"class":80},[44,1597,1598],{"class":80}," not",[44,1600,1512],{"class":84},[44,1602,1515],{"class":80},[44,1604,1518],{"class":125},[44,1606,1607],{"class":84},"].value ",[44,1609,1610],{"class":80},"else",[44,1612,1451],{"class":84},[44,1614,1454],{"class":54},[44,1616,116],{"class":84},[44,1618,1459],{"class":119},[44,1620,107],{"class":80},[44,1622,1623],{"class":54},"\"F4B183\"",[44,1625,139],{"class":84},[44,1627,1629,1632,1634],{"class":46,"line":1628},22,[44,1630,1631],{"class":84},"wb.save(",[44,1633,1253],{"class":54},[44,1635,139],{"class":84},[44,1637,1639,1641,1643,1646,1649,1653,1655,1658,1661,1664,1667,1670,1672,1674,1677,1679,1682],{"class":46,"line":1638},23,[44,1640,267],{"class":125},[44,1642,270],{"class":84},[44,1644,1645],{"class":80},"f",[44,1647,1648],{"class":54},"\"",[44,1650,1652],{"class":1651},"sSjpA","{",[44,1654,278],{"class":125},[44,1656,1657],{"class":84},"(out[",[44,1659,1660],{"class":54},"'flag'",[44,1662,1663],{"class":84},"].sum())",[44,1665,1666],{"class":1651},"}",[44,1668,1669],{"class":54}," of ",[44,1671,1652],{"class":1651},[44,1673,1562],{"class":125},[44,1675,1676],{"class":84},"(out)",[44,1678,1666],{"class":1651},[44,1680,1681],{"class":54}," rows flagged; follow-up list written\"",[44,1683,139],{"class":84},[10,1685,1686,1687,1690],{},"Flagged rows are shaded amber across the whole record, and within them the empty cells are a darker orange, so the gaps are obvious at a glance; empty cells in unflagged rows get a light grey, visible but not alarming. The ",[14,1688,1689],{},"Follow up"," sheet lists flagged records worst first with exactly which fields are missing and who entered them, ready to send round. Placeholders were turned into true blanks along the way, so the cleaned sheet is also better input for whatever comes next. Run the same check every time the sheet is updated and compare the flagged list with the previous run: records that leave the list were fixed, and new entries show whether the problem is being solved or just moved.",[720,1692,729,1697,729,1700,729,1703,729,1706,729,1711,729,1722,729,1728,729,1733,729,1737,729,1742,729,1745,729,1748,729,1755,729,1761,729,1765,729,1769,729,1773,729,1777,729,1780,729,1783,729,1790,729,1795,729,1797,729,1801,729,1804,729,1807,729,1813,729,1816,729,1819,729,1822,729,1826,729,1829,729,1832,729,1839,729,1844,729,1848],{"viewBox":1693,"role":723,"ariaLabelledBy":1694,"xmlns":727,"style":728},"4 0 752 160",[1695,1696],"mr-flow-t","mr-flow-d",[731,1698,1699],{"id":1695},"Finding incomplete records",[735,1701,1702],{"id":1696},"Read every cell as text, turn placeholders and blanks into missing values, score each row by the weighted share of missing fields, flag rows with a missing required field or a high score, summarise by who entered them, and write highlighted rows plus a follow-up list.",[739,1704],{"x":741,"y":742,"width":743,"height":1705,"fill":745},"160",[739,1707],{"x":748,"y":1131,"width":1708,"height":1709,"rx":1710,"fill":752,"stroke":753,"style":935},"115.2","128.0","12",[1712,1713,1715],"g",{"transform":1714},"translate(65.6,37.8) scale(1.000)",[1716,1717],"path",{"d":1718,"fill":1719,"stroke":1720,"style":1721},"M3 5h18v14H3z M3 10h18 M3 15h18 M9 5v14","none","var(--text,#172033)","stroke-width:1.80px;stroke-linecap:round;stroke-linejoin:round",[760,1723,1727],{"x":1724,"y":1725,"style":1726},"77.6","84.2","font-size:13.5px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","Read as text",[760,1729,1732],{"x":1724,"y":1730,"style":1731},"103.8","font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","no default NA",[760,1734,1736],{"x":1724,"y":1735,"style":1731},"121.2","parsing",[46,1738],{"x1":1739,"y1":1740,"x2":1741,"y2":1740,"stroke":934,"style":935},"140.2","80.0","162.2",[937,1743],{"points":1744,"fill":940},"166.2,80.0 157.2,85.0 157.2,75.0",[739,1746],{"x":1747,"y":1131,"width":1708,"height":1709,"rx":1710,"fill":885,"stroke":886,"style":935},"171.2",[1712,1749,1751],{"transform":1750},"translate(216.8,29.0) scale(1.000)",[1716,1752],{"d":1753,"fill":1719,"stroke":1754,"style":1721},"M3 5h18l-7 8v6l-4 2v-8z","var(--gold-ink,#7a4e06)",[760,1756,1760],{"x":1757,"y":1758,"style":1759},"228.8","75.5","font-size:13.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","Placeholders",[760,1762,1764],{"x":1757,"y":1763,"style":1759},"93.0","→ missing",[760,1766,1768],{"x":1757,"y":1767,"style":1731},"112.5","n\u002Fa, -, TBC,",[760,1770,1772],{"x":1757,"y":1771,"style":1731},"130.0","blank",[46,1774],{"x1":1775,"y1":1740,"x2":1776,"y2":1740,"stroke":934,"style":935},"291.4","313.4",[937,1778],{"points":1779,"fill":940},"317.4,80.0 308.4,85.0 308.4,75.0",[739,1781],{"x":1782,"y":1131,"width":1708,"height":1709,"rx":1710,"fill":1176,"stroke":934,"style":935},"322.4",[1712,1784,1786],{"transform":1785},"translate(368.0,46.5) scale(1.000)",[1716,1787],{"d":1788,"fill":1719,"stroke":1789,"style":1721},"M5 9h14 M4 15h14 M10 4l-2 16 M16 4l-2 16","var(--brand-strong,#4338ca)",[760,1791,1794],{"x":1792,"y":1763,"style":1793},"380.0","font-size:13.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Score rows",[760,1796,954],{"x":1792,"y":1767,"style":1731},[46,1798],{"x1":1799,"y1":1740,"x2":1800,"y2":1740,"stroke":934,"style":935},"442.6","464.6",[937,1802],{"points":1803,"fill":940},"468.6,80.0 459.6,85.0 459.6,75.0",[739,1805],{"x":1806,"y":1131,"width":1708,"height":1709,"rx":1710,"fill":885,"stroke":886,"style":935},"473.6",[1712,1808,1810],{"transform":1809},"translate(519.2,37.8) scale(1.000)",[1716,1811],{"d":1812,"fill":1719,"stroke":1754,"style":1721},"M12 3.5l9.5 17h-19z M12 10v5 M12 17.8v.4",[760,1814,873],{"x":1815,"y":1725,"style":1759},"531.2",[760,1817,1818],{"x":1815,"y":1730,"style":1731},"required or ≥",[760,1820,1821],{"x":1815,"y":1735,"style":1731},"25%",[46,1823],{"x1":1824,"y1":1740,"x2":1825,"y2":1740,"stroke":934,"style":935},"593.8","615.8",[937,1827],{"points":1828,"fill":940},"619.8,80.0 610.8,85.0 610.8,75.0",[739,1830],{"x":1831,"y":1131,"width":1708,"height":1709,"rx":1710,"fill":1144,"stroke":1145,"style":935},"624.8",[1712,1833,1835],{"transform":1834},"translate(670.4,37.8) scale(1.000)",[1716,1836],{"d":1837,"fill":1719,"stroke":1838,"style":1721},"M5 12.5l4.5 4.5L19 7","var(--teal-ink,#0b6157)",[760,1840,1843],{"x":1841,"y":1725,"style":1842},"682.4","font-size:13.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","Highlight +",[760,1845,1847],{"x":1841,"y":1846,"style":1842},"101.8","list",[760,1849,1850],{"x":1841,"y":1735,"style":1731},"follow-up sheet",[29,1852,1854],{"id":1853},"choosing-the-weights-and-threshold","Choosing the weights and threshold",[10,1856,1857],{},"The weights and the threshold encode a judgement about what makes a record usable, so set them with the people who use the data rather than by guesswork. A sales team may consider a record without a phone number useless; a finance team may not care about phone numbers at all but insist on a VAT number. Start with two tiers — required fields that flag a row on their own, and everything else with weight 1 — and only introduce finer weights if the flagged list turns out too long or too short. Then look at the score distribution before fixing the threshold: if most incomplete rows score around 0.1 and the abandoned ones above 0.5, any threshold in between works; if scores are spread evenly, the threshold is a policy decision and worth writing down next to the list it produces.",[29,1859,1861],{"id":1860},"common-pitfalls","Common pitfalls",[1863,1864,1865,1873,1879,1885,1891],"ul",{},[1866,1867,1868,1872],"li",{},[1869,1870,1871],"strong",{},"Counting only empty cells."," \"n\u002Fa\", \"-\" and single spaces are missing too.",[1866,1874,1875,1878],{},[1869,1876,1877],{},"Treating all fields equally."," A missing account number matters more than a missing industry.",[1866,1880,1881,1884],{},[1869,1882,1883],{},"A threshold without required fields."," A row missing only its ID may still score low.",[1866,1886,1887,1890],{},[1869,1888,1889],{},"Flagging without saying why."," List the missing fields for each flagged row.",[1866,1892,1893,1896],{},[1869,1894,1895],{},"Blaming people for bad forms."," Check which fields are empty for everyone.",[29,1898,1900],{"id":1899},"conclusion","Conclusion",[10,1902,1903],{},"To find incomplete records, read the sheet as text, turn placeholders and blanks into missing values, and score each row by the weighted share of its fields that are missing, with required fields able to flag a row on their own. Summarise completeness by who entered the records and by field, then write the data with flagged rows highlighted and a follow-up list that names the missing fields for each record.",[29,1905,1907],{"id":1906},"frequently-asked-questions","Frequently asked questions",[10,1909,1910,1913],{},[1869,1911,1912],{},"How do I count missing values per row in pandas?","\nUse df.isna().sum(axis=1), after converting placeholders such as \"n\u002Fa\" and blank strings to missing values.",[10,1915,1916,1919],{},[1869,1917,1918],{},"How do I treat \"n\u002Fa\" and \"-\" as missing when reading Excel?","\nRead with dtype=str and keep_default_na=False, trim the values, and mask those that match a set of placeholder strings.",[10,1921,1922,1925],{},[1869,1923,1924],{},"How can some columns count more than others?","\nMultiply the missing mask by a Series of column weights, sum across the row and divide by the total weight to get a weighted missing share.",[10,1927,1928,1931],{},[1869,1929,1930],{},"How do I highlight incomplete rows in the Excel output?","\nWrite the data with pandas, then use openpyxl to fill every cell in rows whose flag column is true.",[29,1933,1935],{"id":1934},"related","Related",[1863,1937,1938,1944,1951,1958],{},[1866,1939,1940,1941,1943],{},"Up one level: ",[23,1942,26],{"href":25}," — gaps in Excel data.",[1866,1945,1946,1950],{},[23,1947,1949],{"href":1948},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fvisualise-missing-data-patterns-in-an-excel-file\u002F","Visualise Missing Data Patterns in an Excel File"," — seeing patterns of gaps.",[1866,1952,1953,1957],{},[23,1954,1956],{"href":1955},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fdrop-rows-with-missing-required-fields-in-pandas\u002F","Drop Rows with Missing Required Fields in pandas"," — removing instead of flagging.",[1866,1959,1960,1964],{},[23,1961,1963],{"href":1962},"\u002Fadvanced-data-transformation-and-cleaning\u002Fapplying-conditional-formatting-with-openpyxl\u002Fhighlight-blank-cells-with-openpyxl\u002F","Highlight Blank Cells with openpyxl"," — live highlighting of blanks.",[1966,1967,1968],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}",{"title":39,"searchDepth":94,"depth":94,"links":1970},[1971,1972,1973,1974,1975,1976,1977,1978,1979,1980],{"id":31,"depth":94,"text":32},{"id":67,"depth":94,"text":68},{"id":322,"depth":94,"text":323},{"id":957,"depth":94,"text":958},{"id":1184,"depth":94,"text":1185},{"id":1853,"depth":94,"text":1854},{"id":1860,"depth":94,"text":1861},{"id":1899,"depth":94,"text":1900},{"id":1906,"depth":94,"text":1907},{"id":1934,"depth":94,"text":1935},"2026-10-02","Find the incomplete records in an Excel sheet before they distort a report: count missing values per row with pandas, treat placeholders like \"n\u002Fa\", \"-\" and \"TBC\" as missing, weight required fields more than optional ones, choose a threshold, group the flagged rows by who entered them, and write a workbook where incomplete rows are highlighted and listed for follow-up.","md",[1985,1987,1989,1991],{"q":1912,"a":1986},"Use df.isna().sum(axis=1), after converting placeholders such as \"n\u002Fa\" and blank strings to missing values.",{"q":1918,"a":1988},"Read with dtype=str and keep_default_na=False, trim the values, and mask those that match a set of placeholder strings.",{"q":1924,"a":1990},"Multiply the missing mask by a Series of column weights, sum across the row and divide by the total weight to get a weighted missing share.",{"q":1930,"a":1992},"Write the data with pandas, then use openpyxl to fill every cell in rows whose flag column is true.","flag rows with too many missing values pandas",null,{"breadcrumb":1996},[1997,1999,2002],{"name":1998,"item":504},"Home",{"name":2000,"item":2001},"Advanced Data Transformation and Cleaning for Excel Automation","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":2003,"item":25},"Handling Missing Data in Excel Reports with Pandas","\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fflag-rows-with-too-many-missing-values",{"title":2006,"description":2007},"Flag Rows with Too Many Missing Values in Excel Data with pandas","Row-level completeness checks in pandas: placeholder text as missing, missing counts and shares per row, weighted required fields, thresholds, completeness by source, and highlighted rows plus a follow-up list in Excel.","flag-rows-with-too-many-missing-values","advanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fflag-rows-with-too-many-missing-values\u002Findex",[2011,2012],"missing-data","data-validation","how-to","ojEC7qeY5twUlVDkWtqI_5zZm3z9SjHNR6VV8Gs9kYk",[2016,2019],{"title":1111,"path":2017,"stem":2018,"children":-1},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Ffind-and-report-missing-values-in-an-excel-file","advanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Ffind-and-report-missing-values-in-an-excel-file\u002Findex",{"title":2020,"path":2021,"stem":2022,"children":-1},"Forward-Fill Blank Cells in Grouped Excel Reports with pandas","\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fforward-fill-blank-cells-in-grouped-excel-reports","advanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fforward-fill-blank-cells-in-grouped-excel-reports\u002Findex",[2024,2026],{"path":2025,"title":2003},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports",{"path":2027,"title":2000},"\u002Fadvanced-data-transformation-and-cleaning",[],[2030,2033,2036,2039,2041,2044],{"path":2031,"title":2032,"kind":2013},"\u002Fadvanced-data-transformation-and-cleaning\u002Fapplying-conditional-formatting-with-openpyxl\u002Fhighlight-blank-cells-with-openpyxl","Highlight Blank Cells in Excel with openpyxl",{"path":2034,"title":2035,"kind":2013},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fcoalesce-several-excel-columns-into-one-with-pandas","Coalesce Several Excel Columns into One with pandas",{"path":2037,"title":2038,"kind":2013},"\u002Fadvanced-data-transformation-and-cleaning\u002Fexcel-formula-equivalents-in-pandas\u002Fiferror-and-isna-equivalents-in-pandas","IFERROR, IFNA, ISNA and ISNUMBER Equivalents in pandas",{"path":2040,"title":1956,"kind":2013},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Fdrop-rows-with-missing-required-fields-in-pandas",{"path":2042,"title":2043,"kind":2013},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Ffill-gaps-in-per-item-time-series-with-a-staleness-limit","Fill Gaps in Per-Item Time Series with a Staleness Limit",{"path":2045,"title":2046,"kind":2013},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Ffill-missing-values-in-excel-with-pandas-fillna","Fill Missing Values in Excel with Pandas fillna",1790972095955]