[{"data":1,"prerenderedAt":4747},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python":4739},{"id":4,"title":5,"body":6,"dateModified":4711,"datePublished":4711,"description":4712,"extension":4713,"faq":4714,"meta":4725,"navigation":280,"path":4732,"seo":4733,"slug":4735,"stem":4736,"type":4737,"__hash__":4738},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Findex.md","Validating Excel Data with Python",{"type":7,"value":8,"toc":4692},"minimark",[9,45,213,218,245,249,252,656,659,663,666,1012,1022,1026,1041,1424,1431,1526,1538,1542,1545,1951,1969,1973,1976,2247,2254,2258,2261,2394,2411,2415,2426,2728,2740,2744,2853,2856,2860,2865,3133,3139,3146,3150,3153,3582,3585,3589,3592,4021,4044,4048,4051,4345,4351,4416,4419,4423,4529,4533,4564,4568,4577,4583,4589,4598,4604,4608,4611,4618,4621,4663,4666,4688],[10,11,12,13,17,18,21,22,25,26,29,30,34,35,38,39,44],"p",{},"Every reporting job eventually receives a file where a date is ",[14,15,16],"code",{},"31\u002F02\u002F2026",", a quantity is ",[14,19,20],{},"n\u002Fa",", and a region that used to be ",[14,23,24],{},"North"," is now ",[14,27,28],{},"north ",". Validation is the layer that turns those into a clear message instead of a wrong number in a board pack. There are two halves to it, and they solve different problems: ",[31,32,33],"strong",{},"pandas checks"," reject bad data that has already been entered, while ",[31,36,37],{},"openpyxl rules"," stop it being entered next time. This guide, part of ",[40,41,43],"a",{"href":42},"\u002Fadvanced-data-transformation-and-cleaning\u002F","Advanced Data Transformation and Cleaning",", builds both.",[46,47,55,56,55,60,55,64,55,71,55,80,55,87,55,92,55,99,55,104,55,112,55,117,55,121,55,124,55,128,55,135,55,139,55,144,55,148,55,154,55,160,55,165,55,169,55,174,55,178,55,185,55,191,55,195,55,199,55,201,55,205,55,209],"svg",{"viewBox":48,"role":49,"ariaLabelledBy":50,"xmlns":53,"style":54},"0 0 760 262","img",[51,52],"vd-two-t","vd-two-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:760px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[57,58,59],"title",{"id":51},"Two validation layers around an Excel workflow",[61,62,63],"desc",{"id":52},"On the way in, pandas checks the workbook a person submitted: structure, types and business rules, splitting rows into clean and quarantined. On the way out, openpyxl writes dropdowns and limits into the workbook that is sent back, so the next round of data entry is constrained.",[65,66],"rect",{"x":67,"y":67,"width":68,"height":69,"fill":70},"0","760","262","#ffffff",[65,72],{"x":73,"y":74,"width":75,"height":76,"rx":77,"fill":78,"stroke":79},"20","60","150","70","12","#ebebfd","var(--line,#cdd5e6)",[81,82,86],"text",{"x":83,"y":84,"style":85},"95","90","font-size:12.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","submitted.xlsx",[81,88,91],{"x":83,"y":89,"style":90},"112","font-size:11.5px;fill:var(--muted,#5b6780);text-anchor:middle","typed by a human",[93,94],"line",{"x1":95,"y1":83,"x2":96,"y2":83,"stroke":97,"style":98},"174","212","var(--muted,#5b6780)","stroke-width:2px",[100,101],"polygon",{"points":102,"fill":103},"216,95 204,89 204,101","#5b6780",[65,105],{"x":106,"y":107,"width":108,"height":109,"rx":77,"fill":110,"stroke":111,"style":98},"218","46","180","98","#d9f4f1","var(--teal,#0f9488)",[81,113,33],{"x":114,"y":115,"style":116},"308","72","font-size:12.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle",[81,118,120],{"x":114,"y":119,"style":90},"94","columns present?",[81,122,123],{"x":114,"y":89,"style":90},"types parse?",[81,125,127],{"x":114,"y":126,"style":90},"130","rules hold?",[93,129],{"x1":130,"y1":131,"x2":132,"y2":133,"stroke":111,"style":134},"402","76","446","52","stroke-width:1.5px",[100,136],{"points":137,"fill":138},"450,50 438,48 442,58","#0f766e",[93,140],{"x1":130,"y1":141,"x2":132,"y2":142,"stroke":143,"style":134},"118","142","var(--danger,#dc2626)",[100,145],{"points":146,"fill":147},"450,144 438,136 442,146","#d81b73",[65,149],{"x":150,"y":151,"width":75,"height":133,"rx":152,"fill":153,"stroke":79},"452","26","10","#f0f4ff",[81,155,159],{"x":156,"y":157,"style":158},"527","49","font-size:12px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","clean rows",[81,161,164],{"x":156,"y":162,"style":163},"68","font-size:11px;fill:var(--muted,#5b6780);text-anchor:middle","continue to the report",[65,166],{"x":150,"y":167,"width":75,"height":133,"rx":152,"fill":168,"stroke":143},"116","#fce9e9",[81,170,173],{"x":156,"y":171,"style":172},"139","font-size:12px;font-weight:700;fill:var(--danger,#dc2626);text-anchor:middle","quarantined",[81,175,177],{"x":156,"y":176,"style":163},"158","reported with reasons",[65,179],{"x":106,"y":180,"width":181,"height":182,"rx":77,"fill":183,"stroke":184,"style":98},"182","384","62","#fdefd8","var(--gold,#b4740a)",[81,186,190],{"x":187,"y":188,"style":189},"410","207","font-size:12.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","openpyxl DataValidation on the returned file",[81,192,194],{"x":187,"y":193,"style":90},"229","dropdowns, numeric limits, date ranges — fewer errors next round",[93,196],{"x1":83,"y1":197,"x2":83,"y2":96,"stroke":184,"style":198},"132","stroke-width:1.5px;stroke-dasharray:5 4",[93,200],{"x1":83,"y1":96,"x2":96,"y2":96,"stroke":184,"style":198},[100,202],{"points":203,"fill":204},"216,212 204,206 204,218","#8a5808",[81,206,208],{"x":207,"y":188,"style":90},"640","sent back to",[81,210,212],{"x":207,"y":211,"style":90},"225","the submitter",[214,215,217],"h2",{"id":216},"install-the-dependencies","Install the dependencies",[219,220,225],"pre",{"className":221,"code":222,"language":223,"meta":224,"style":224},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl\n","bash","",[14,226,227],{"__ignoreMap":224},[228,229,231,235,239,242],"span",{"class":93,"line":230},1,[228,232,234],{"class":233},"sMTad","pip",[228,236,238],{"class":237},"srMev"," install",[228,240,241],{"class":237}," pandas",[228,243,244],{"class":237}," openpyxl\n",[214,246,248],{"id":247},"create-a-workbook-with-realistic-problems","Create a workbook with realistic problems",[10,250,251],{},"Every example below runs against this file, which contains the failure modes real submissions have:",[219,253,257],{"className":254,"code":255,"language":256,"meta":224,"style":224},"language-python shiki shiki-themes github-light github-dark-high-contrast","import pandas as pd\n\nrows = [\n    {\"Order_ID\": 2001, \"Region\": \"North\", \"Order_Date\": \"2026-01-14\", \"Quantity\": 4, \"Unit_Price\": 19.99},\n    {\"Order_ID\": 2002, \"Region\": \"north \", \"Order_Date\": \"2026-01-15\", \"Quantity\": \"2\", \"Unit_Price\": 49.5},\n    {\"Order_ID\": 2003, \"Region\": \"Souht\", \"Order_Date\": \"15\u002F01\u002F2026\", \"Quantity\": -7, \"Unit_Price\": 12.25},\n    {\"Order_ID\": 2004, \"Region\": \"West\", \"Order_Date\": \"2026-02-30\", \"Quantity\": 5, \"Unit_Price\": None},\n    {\"Order_ID\": 2001, \"Region\": \"West\", \"Order_Date\": \"2026-01-19\", \"Quantity\": 3, \"Unit_Price\": 8.75},\n    {\"Order_ID\": None, \"Region\": \"\", \"Order_Date\": \"\", \"Quantity\": \"n\u002Fa\", \"Unit_Price\": 15.0},\n]\npd.DataFrame(rows).to_excel(\"submitted.xlsx\", index=False, sheet_name=\"Orders\")\nprint(\"wrote submitted.xlsx\")\n","python",[14,258,259,275,282,294,354,404,457,507,555,603,609,642],{"__ignoreMap":224},[228,260,261,265,269,272],{"class":93,"line":230},[228,262,264],{"class":263},"s-kum","import",[228,266,268],{"class":267},"skGVy"," pandas ",[228,270,271],{"class":263},"as",[228,273,274],{"class":267}," pd\n",[228,276,278],{"class":93,"line":277},2,[228,279,281],{"emptyLinePlaceholder":280},true,"\n",[228,283,285,288,291],{"class":93,"line":284},3,[228,286,287],{"class":267},"rows ",[228,289,290],{"class":263},"=",[228,292,293],{"class":267}," [\n",[228,295,297,300,303,306,310,313,316,318,321,323,326,328,331,333,336,338,341,343,346,348,351],{"class":93,"line":296},4,[228,298,299],{"class":267},"    {",[228,301,302],{"class":237},"\"Order_ID\"",[228,304,305],{"class":267},": ",[228,307,309],{"class":308},"sP0c6","2001",[228,311,312],{"class":267},", ",[228,314,315],{"class":237},"\"Region\"",[228,317,305],{"class":267},[228,319,320],{"class":237},"\"North\"",[228,322,312],{"class":267},[228,324,325],{"class":237},"\"Order_Date\"",[228,327,305],{"class":267},[228,329,330],{"class":237},"\"2026-01-14\"",[228,332,312],{"class":267},[228,334,335],{"class":237},"\"Quantity\"",[228,337,305],{"class":267},[228,339,340],{"class":308},"4",[228,342,312],{"class":267},[228,344,345],{"class":237},"\"Unit_Price\"",[228,347,305],{"class":267},[228,349,350],{"class":308},"19.99",[228,352,353],{"class":267},"},\n",[228,355,357,359,361,363,366,368,370,372,375,377,379,381,384,386,388,390,393,395,397,399,402],{"class":93,"line":356},5,[228,358,299],{"class":267},[228,360,302],{"class":237},[228,362,305],{"class":267},[228,364,365],{"class":308},"2002",[228,367,312],{"class":267},[228,369,315],{"class":237},[228,371,305],{"class":267},[228,373,374],{"class":237},"\"north \"",[228,376,312],{"class":267},[228,378,325],{"class":237},[228,380,305],{"class":267},[228,382,383],{"class":237},"\"2026-01-15\"",[228,385,312],{"class":267},[228,387,335],{"class":237},[228,389,305],{"class":267},[228,391,392],{"class":237},"\"2\"",[228,394,312],{"class":267},[228,396,345],{"class":237},[228,398,305],{"class":267},[228,400,401],{"class":308},"49.5",[228,403,353],{"class":267},[228,405,407,409,411,413,416,418,420,422,425,427,429,431,434,436,438,440,443,446,448,450,452,455],{"class":93,"line":406},6,[228,408,299],{"class":267},[228,410,302],{"class":237},[228,412,305],{"class":267},[228,414,415],{"class":308},"2003",[228,417,312],{"class":267},[228,419,315],{"class":237},[228,421,305],{"class":267},[228,423,424],{"class":237},"\"Souht\"",[228,426,312],{"class":267},[228,428,325],{"class":237},[228,430,305],{"class":267},[228,432,433],{"class":237},"\"15\u002F01\u002F2026\"",[228,435,312],{"class":267},[228,437,335],{"class":237},[228,439,305],{"class":267},[228,441,442],{"class":263},"-",[228,444,445],{"class":308},"7",[228,447,312],{"class":267},[228,449,345],{"class":237},[228,451,305],{"class":267},[228,453,454],{"class":308},"12.25",[228,456,353],{"class":267},[228,458,460,462,464,466,469,471,473,475,478,480,482,484,487,489,491,493,496,498,500,502,505],{"class":93,"line":459},7,[228,461,299],{"class":267},[228,463,302],{"class":237},[228,465,305],{"class":267},[228,467,468],{"class":308},"2004",[228,470,312],{"class":267},[228,472,315],{"class":237},[228,474,305],{"class":267},[228,476,477],{"class":237},"\"West\"",[228,479,312],{"class":267},[228,481,325],{"class":237},[228,483,305],{"class":267},[228,485,486],{"class":237},"\"2026-02-30\"",[228,488,312],{"class":267},[228,490,335],{"class":237},[228,492,305],{"class":267},[228,494,495],{"class":308},"5",[228,497,312],{"class":267},[228,499,345],{"class":237},[228,501,305],{"class":267},[228,503,504],{"class":308},"None",[228,506,353],{"class":267},[228,508,510,512,514,516,518,520,522,524,526,528,530,532,535,537,539,541,544,546,548,550,553],{"class":93,"line":509},8,[228,511,299],{"class":267},[228,513,302],{"class":237},[228,515,305],{"class":267},[228,517,309],{"class":308},[228,519,312],{"class":267},[228,521,315],{"class":237},[228,523,305],{"class":267},[228,525,477],{"class":237},[228,527,312],{"class":267},[228,529,325],{"class":237},[228,531,305],{"class":267},[228,533,534],{"class":237},"\"2026-01-19\"",[228,536,312],{"class":267},[228,538,335],{"class":237},[228,540,305],{"class":267},[228,542,543],{"class":308},"3",[228,545,312],{"class":267},[228,547,345],{"class":237},[228,549,305],{"class":267},[228,551,552],{"class":308},"8.75",[228,554,353],{"class":267},[228,556,558,560,562,564,566,568,570,572,575,577,579,581,583,585,587,589,592,594,596,598,601],{"class":93,"line":557},9,[228,559,299],{"class":267},[228,561,302],{"class":237},[228,563,305],{"class":267},[228,565,504],{"class":308},[228,567,312],{"class":267},[228,569,315],{"class":237},[228,571,305],{"class":267},[228,573,574],{"class":237},"\"\"",[228,576,312],{"class":267},[228,578,325],{"class":237},[228,580,305],{"class":267},[228,582,574],{"class":237},[228,584,312],{"class":267},[228,586,335],{"class":237},[228,588,305],{"class":267},[228,590,591],{"class":237},"\"n\u002Fa\"",[228,593,312],{"class":267},[228,595,345],{"class":237},[228,597,305],{"class":267},[228,599,600],{"class":308},"15.0",[228,602,353],{"class":267},[228,604,606],{"class":93,"line":605},10,[228,607,608],{"class":267},"]\n",[228,610,612,615,618,620,624,626,629,631,634,636,639],{"class":93,"line":611},11,[228,613,614],{"class":267},"pd.DataFrame(rows).to_excel(",[228,616,617],{"class":237},"\"submitted.xlsx\"",[228,619,312],{"class":267},[228,621,623],{"class":622},"sa561","index",[228,625,290],{"class":263},[228,627,628],{"class":308},"False",[228,630,312],{"class":267},[228,632,633],{"class":622},"sheet_name",[228,635,290],{"class":263},[228,637,638],{"class":237},"\"Orders\"",[228,640,641],{"class":267},")\n",[228,643,645,648,651,654],{"class":93,"line":644},12,[228,646,647],{"class":308},"print",[228,649,650],{"class":267},"(",[228,652,653],{"class":237},"\"wrote submitted.xlsx\"",[228,655,641],{"class":267},[10,657,658],{},"Six rows, seven distinct problems: a trailing space, a misspelt region, a negative quantity, an impossible date, a duplicate order number, a missing price and a text value in a numeric column. A validation layer should describe all seven without stopping at the first.",[214,660,662],{"id":661},"layer-1-structural-checks","Layer 1: structural checks",[10,664,665],{},"Before looking at any value, confirm the file has the shape your code expects. Structural failures are worth stopping the job for, because nothing downstream can be trusted:",[219,667,669],{"className":254,"code":668,"language":256,"meta":224,"style":224},"import pandas as pd\n\nREQUIRED = [\"Order_ID\", \"Region\", \"Order_Date\", \"Quantity\", \"Unit_Price\"]\n\ndef load_and_check_structure(path, sheet=\"Orders\"):\n    sheets = pd.ExcelFile(path).sheet_names\n    if sheet not in sheets:\n        raise ValueError(f\"sheet {sheet!r} not found — file has {sheets}\")\n\n    df = pd.read_excel(path, sheet_name=sheet, dtype=object)\n    df.columns = [str(c).strip() for c in df.columns]\n\n    missing = [c for c in REQUIRED if c not in df.columns]\n    if missing:\n        raise ValueError(f\"missing column(s): {missing}; found {list(df.columns)}\")\n    if df.empty:\n        raise ValueError(\"the sheet has headers but no data rows\")\n    return df[REQUIRED]\n\ndf = load_and_check_structure(\"submitted.xlsx\")\nprint(df.shape)\n",[14,670,671,681,685,716,720,739,749,766,810,814,841,868,872,903,911,949,957,971,984,989,1004],{"__ignoreMap":224},[228,672,673,675,677,679],{"class":93,"line":230},[228,674,264],{"class":263},[228,676,268],{"class":267},[228,678,271],{"class":263},[228,680,274],{"class":267},[228,682,683],{"class":93,"line":277},[228,684,281],{"emptyLinePlaceholder":280},[228,686,687,690,693,696,698,700,702,704,706,708,710,712,714],{"class":93,"line":284},[228,688,689],{"class":308},"REQUIRED",[228,691,692],{"class":263}," =",[228,694,695],{"class":267}," [",[228,697,302],{"class":237},[228,699,312],{"class":267},[228,701,315],{"class":237},[228,703,312],{"class":267},[228,705,325],{"class":237},[228,707,312],{"class":267},[228,709,335],{"class":237},[228,711,312],{"class":267},[228,713,345],{"class":237},[228,715,608],{"class":267},[228,717,718],{"class":93,"line":296},[228,719,281],{"emptyLinePlaceholder":280},[228,721,722,725,729,732,734,736],{"class":93,"line":356},[228,723,724],{"class":263},"def",[228,726,728],{"class":727},"s_Opv"," load_and_check_structure",[228,730,731],{"class":267},"(path, sheet",[228,733,290],{"class":263},[228,735,638],{"class":237},[228,737,738],{"class":267},"):\n",[228,740,741,744,746],{"class":93,"line":406},[228,742,743],{"class":267},"    sheets ",[228,745,290],{"class":263},[228,747,748],{"class":267}," pd.ExcelFile(path).sheet_names\n",[228,750,751,754,757,760,763],{"class":93,"line":459},[228,752,753],{"class":263},"    if",[228,755,756],{"class":267}," sheet ",[228,758,759],{"class":263},"not",[228,761,762],{"class":263}," in",[228,764,765],{"class":267}," sheets:\n",[228,767,768,771,774,776,779,782,786,789,792,795,798,800,803,805,808],{"class":93,"line":509},[228,769,770],{"class":263},"        raise",[228,772,773],{"class":308}," ValueError",[228,775,650],{"class":267},[228,777,778],{"class":263},"f",[228,780,781],{"class":237},"\"sheet ",[228,783,785],{"class":784},"sSjpA","{",[228,787,788],{"class":267},"sheet",[228,790,791],{"class":263},"!r",[228,793,794],{"class":784},"}",[228,796,797],{"class":237}," not found — file has ",[228,799,785],{"class":784},[228,801,802],{"class":267},"sheets",[228,804,794],{"class":784},[228,806,807],{"class":237},"\"",[228,809,641],{"class":267},[228,811,812],{"class":93,"line":557},[228,813,281],{"emptyLinePlaceholder":280},[228,815,816,819,821,824,826,828,831,834,836,839],{"class":93,"line":605},[228,817,818],{"class":267},"    df ",[228,820,290],{"class":263},[228,822,823],{"class":267}," pd.read_excel(path, ",[228,825,633],{"class":622},[228,827,290],{"class":263},[228,829,830],{"class":267},"sheet, ",[228,832,833],{"class":622},"dtype",[228,835,290],{"class":263},[228,837,838],{"class":308},"object",[228,840,641],{"class":267},[228,842,843,846,848,850,853,856,859,862,865],{"class":93,"line":611},[228,844,845],{"class":267},"    df.columns ",[228,847,290],{"class":263},[228,849,695],{"class":267},[228,851,852],{"class":308},"str",[228,854,855],{"class":267},"(c).strip() ",[228,857,858],{"class":263},"for",[228,860,861],{"class":267}," c ",[228,863,864],{"class":263},"in",[228,866,867],{"class":267}," df.columns]\n",[228,869,870],{"class":93,"line":644},[228,871,281],{"emptyLinePlaceholder":280},[228,873,875,878,880,883,885,887,889,892,895,897,899,901],{"class":93,"line":874},13,[228,876,877],{"class":267},"    missing ",[228,879,290],{"class":263},[228,881,882],{"class":267}," [c ",[228,884,858],{"class":263},[228,886,861],{"class":267},[228,888,864],{"class":263},[228,890,891],{"class":308}," REQUIRED",[228,893,894],{"class":263}," if",[228,896,861],{"class":267},[228,898,759],{"class":263},[228,900,762],{"class":263},[228,902,867],{"class":267},[228,904,906,908],{"class":93,"line":905},14,[228,907,753],{"class":263},[228,909,910],{"class":267}," missing:\n",[228,912,914,916,918,920,922,925,927,930,932,935,937,940,943,945,947],{"class":93,"line":913},15,[228,915,770],{"class":263},[228,917,773],{"class":308},[228,919,650],{"class":267},[228,921,778],{"class":263},[228,923,924],{"class":237},"\"missing column(s): ",[228,926,785],{"class":784},[228,928,929],{"class":267},"missing",[228,931,794],{"class":784},[228,933,934],{"class":237},"; found ",[228,936,785],{"class":784},[228,938,939],{"class":308},"list",[228,941,942],{"class":267},"(df.columns)",[228,944,794],{"class":784},[228,946,807],{"class":237},[228,948,641],{"class":267},[228,950,952,954],{"class":93,"line":951},16,[228,953,753],{"class":263},[228,955,956],{"class":267}," df.empty:\n",[228,958,960,962,964,966,969],{"class":93,"line":959},17,[228,961,770],{"class":263},[228,963,773],{"class":308},[228,965,650],{"class":267},[228,967,968],{"class":237},"\"the sheet has headers but no data rows\"",[228,970,641],{"class":267},[228,972,974,977,980,982],{"class":93,"line":973},18,[228,975,976],{"class":263},"    return",[228,978,979],{"class":267}," df[",[228,981,689],{"class":308},[228,983,608],{"class":267},[228,985,987],{"class":93,"line":986},19,[228,988,281],{"emptyLinePlaceholder":280},[228,990,992,995,997,1000,1002],{"class":93,"line":991},20,[228,993,994],{"class":267},"df ",[228,996,290],{"class":263},[228,998,999],{"class":267}," load_and_check_structure(",[228,1001,617],{"class":237},[228,1003,641],{"class":267},[228,1005,1007,1009],{"class":93,"line":1006},21,[228,1008,647],{"class":308},[228,1010,1011],{"class":267},"(df.shape)\n",[10,1013,1014,1015,1018,1019,1021],{},"Reading with ",[14,1016,1017],{},"dtype=object"," is deliberate. Letting pandas guess types hides exactly the problems you are looking for: a column with one ",[14,1020,591],{}," silently becomes text, and a date column with one European date becomes a mix of timestamps and strings. Read everything raw, then coerce with your eyes open.",[214,1023,1025],{"id":1024},"layer-2-type-coercion-that-records-its-failures","Layer 2: type coercion that records its failures",[10,1027,1028,1029,1032,1033,1036,1037,1040],{},"Coercion and validation are the same operation if you keep the failures. ",[14,1030,1031],{},"errors=\"coerce\""," turns anything unparsable into ",[14,1034,1035],{},"NaT","\u002F",[14,1038,1039],{},"NaN",", and comparing before and after tells you which rows broke:",[219,1042,1044],{"className":254,"code":1043,"language":256,"meta":224,"style":224},"import pandas as pd\n\ndef coerce_types(df):\n    issues = []\n    out = df.copy()\n\n    out[\"Order_ID\"] = pd.to_numeric(out[\"Order_ID\"], errors=\"coerce\")\n    out[\"Quantity\"] = pd.to_numeric(out[\"Quantity\"], errors=\"coerce\")\n    out[\"Unit_Price\"] = pd.to_numeric(out[\"Unit_Price\"], errors=\"coerce\")\n    out[\"Order_Date\"] = pd.to_datetime(out[\"Order_Date\"], errors=\"coerce\", format=\"mixed\")\n\n    for column in (\"Order_ID\", \"Quantity\", \"Unit_Price\", \"Order_Date\"):\n        broke = out[column].isna() & df[column].notna() & (df[column].astype(str).str.strip() != \"\")\n        for idx in out.index[broke]:\n            issues.append({\n                \"row\": int(idx) + 2,                    # +2: header row and zero-based index\n                \"column\": column,\n                \"value\": df.loc[idx, column],\n                \"problem\": f\"could not be read as {'a date' if column == 'Order_Date' else 'a number'}\",\n            })\n    return out, issues\n\ntyped, issues = coerce_types(df)\nfor issue in issues:\n    print(issue)\n",[14,1045,1046,1056,1060,1070,1080,1090,1094,1124,1148,1172,1207,1211,1240,1274,1287,1292,1318,1326,1334,1374,1379,1386,1391,1402,1415],{"__ignoreMap":224},[228,1047,1048,1050,1052,1054],{"class":93,"line":230},[228,1049,264],{"class":263},[228,1051,268],{"class":267},[228,1053,271],{"class":263},[228,1055,274],{"class":267},[228,1057,1058],{"class":93,"line":277},[228,1059,281],{"emptyLinePlaceholder":280},[228,1061,1062,1064,1067],{"class":93,"line":284},[228,1063,724],{"class":263},[228,1065,1066],{"class":727}," coerce_types",[228,1068,1069],{"class":267},"(df):\n",[228,1071,1072,1075,1077],{"class":93,"line":296},[228,1073,1074],{"class":267},"    issues ",[228,1076,290],{"class":263},[228,1078,1079],{"class":267}," []\n",[228,1081,1082,1085,1087],{"class":93,"line":356},[228,1083,1084],{"class":267},"    out ",[228,1086,290],{"class":263},[228,1088,1089],{"class":267}," df.copy()\n",[228,1091,1092],{"class":93,"line":406},[228,1093,281],{"emptyLinePlaceholder":280},[228,1095,1096,1099,1101,1104,1106,1109,1111,1114,1117,1119,1122],{"class":93,"line":459},[228,1097,1098],{"class":267},"    out[",[228,1100,302],{"class":237},[228,1102,1103],{"class":267},"] ",[228,1105,290],{"class":263},[228,1107,1108],{"class":267}," pd.to_numeric(out[",[228,1110,302],{"class":237},[228,1112,1113],{"class":267},"], ",[228,1115,1116],{"class":622},"errors",[228,1118,290],{"class":263},[228,1120,1121],{"class":237},"\"coerce\"",[228,1123,641],{"class":267},[228,1125,1126,1128,1130,1132,1134,1136,1138,1140,1142,1144,1146],{"class":93,"line":509},[228,1127,1098],{"class":267},[228,1129,335],{"class":237},[228,1131,1103],{"class":267},[228,1133,290],{"class":263},[228,1135,1108],{"class":267},[228,1137,335],{"class":237},[228,1139,1113],{"class":267},[228,1141,1116],{"class":622},[228,1143,290],{"class":263},[228,1145,1121],{"class":237},[228,1147,641],{"class":267},[228,1149,1150,1152,1154,1156,1158,1160,1162,1164,1166,1168,1170],{"class":93,"line":557},[228,1151,1098],{"class":267},[228,1153,345],{"class":237},[228,1155,1103],{"class":267},[228,1157,290],{"class":263},[228,1159,1108],{"class":267},[228,1161,345],{"class":237},[228,1163,1113],{"class":267},[228,1165,1116],{"class":622},[228,1167,290],{"class":263},[228,1169,1121],{"class":237},[228,1171,641],{"class":267},[228,1173,1174,1176,1178,1180,1182,1185,1187,1189,1191,1193,1195,1197,1200,1202,1205],{"class":93,"line":605},[228,1175,1098],{"class":267},[228,1177,325],{"class":237},[228,1179,1103],{"class":267},[228,1181,290],{"class":263},[228,1183,1184],{"class":267}," pd.to_datetime(out[",[228,1186,325],{"class":237},[228,1188,1113],{"class":267},[228,1190,1116],{"class":622},[228,1192,290],{"class":263},[228,1194,1121],{"class":237},[228,1196,312],{"class":267},[228,1198,1199],{"class":622},"format",[228,1201,290],{"class":263},[228,1203,1204],{"class":237},"\"mixed\"",[228,1206,641],{"class":267},[228,1208,1209],{"class":93,"line":611},[228,1210,281],{"emptyLinePlaceholder":280},[228,1212,1213,1216,1219,1221,1224,1226,1228,1230,1232,1234,1236,1238],{"class":93,"line":644},[228,1214,1215],{"class":263},"    for",[228,1217,1218],{"class":267}," column ",[228,1220,864],{"class":263},[228,1222,1223],{"class":267}," (",[228,1225,302],{"class":237},[228,1227,312],{"class":267},[228,1229,335],{"class":237},[228,1231,312],{"class":267},[228,1233,345],{"class":237},[228,1235,312],{"class":267},[228,1237,325],{"class":237},[228,1239,738],{"class":267},[228,1241,1242,1245,1247,1250,1253,1256,1258,1261,1263,1266,1269,1272],{"class":93,"line":874},[228,1243,1244],{"class":267},"        broke ",[228,1246,290],{"class":263},[228,1248,1249],{"class":267}," out[column].isna() ",[228,1251,1252],{"class":263},"&",[228,1254,1255],{"class":267}," df[column].notna() ",[228,1257,1252],{"class":263},[228,1259,1260],{"class":267}," (df[column].astype(",[228,1262,852],{"class":308},[228,1264,1265],{"class":267},").str.strip() ",[228,1267,1268],{"class":263},"!=",[228,1270,1271],{"class":237}," \"\"",[228,1273,641],{"class":267},[228,1275,1276,1279,1282,1284],{"class":93,"line":905},[228,1277,1278],{"class":263},"        for",[228,1280,1281],{"class":267}," idx ",[228,1283,864],{"class":263},[228,1285,1286],{"class":267}," out.index[broke]:\n",[228,1288,1289],{"class":93,"line":913},[228,1290,1291],{"class":267},"            issues.append({\n",[228,1293,1294,1297,1299,1302,1305,1308,1311,1314],{"class":93,"line":951},[228,1295,1296],{"class":237},"                \"row\"",[228,1298,305],{"class":267},[228,1300,1301],{"class":308},"int",[228,1303,1304],{"class":267},"(idx) ",[228,1306,1307],{"class":263},"+",[228,1309,1310],{"class":308}," 2",[228,1312,1313],{"class":267},",                    ",[228,1315,1317],{"class":1316},"s-wDw","# +2: header row and zero-based index\n",[228,1319,1320,1323],{"class":93,"line":959},[228,1321,1322],{"class":237},"                \"column\"",[228,1324,1325],{"class":267},": column,\n",[228,1327,1328,1331],{"class":93,"line":973},[228,1329,1330],{"class":237},"                \"value\"",[228,1332,1333],{"class":267},": df.loc[idx, column],\n",[228,1335,1336,1339,1341,1343,1346,1348,1351,1353,1355,1358,1361,1364,1367,1369,1371],{"class":93,"line":986},[228,1337,1338],{"class":237},"                \"problem\"",[228,1340,305],{"class":267},[228,1342,778],{"class":263},[228,1344,1345],{"class":237},"\"could not be read as ",[228,1347,785],{"class":784},[228,1349,1350],{"class":237},"'a date'",[228,1352,894],{"class":263},[228,1354,1218],{"class":267},[228,1356,1357],{"class":263},"==",[228,1359,1360],{"class":237}," 'Order_Date'",[228,1362,1363],{"class":263}," else",[228,1365,1366],{"class":237}," 'a number'",[228,1368,794],{"class":784},[228,1370,807],{"class":237},[228,1372,1373],{"class":267},",\n",[228,1375,1376],{"class":93,"line":991},[228,1377,1378],{"class":267},"            })\n",[228,1380,1381,1383],{"class":93,"line":1006},[228,1382,976],{"class":263},[228,1384,1385],{"class":267}," out, issues\n",[228,1387,1389],{"class":93,"line":1388},22,[228,1390,281],{"emptyLinePlaceholder":280},[228,1392,1394,1397,1399],{"class":93,"line":1393},23,[228,1395,1396],{"class":267},"typed, issues ",[228,1398,290],{"class":263},[228,1400,1401],{"class":267}," coerce_types(df)\n",[228,1403,1405,1407,1410,1412],{"class":93,"line":1404},24,[228,1406,858],{"class":263},[228,1408,1409],{"class":267}," issue ",[228,1411,864],{"class":263},[228,1413,1414],{"class":267}," issues:\n",[228,1416,1418,1421],{"class":93,"line":1417},25,[228,1419,1420],{"class":308},"    print",[228,1422,1423],{"class":267},"(issue)\n",[10,1425,1426,1427,1430],{},"The ",[14,1428,1429],{},"+2"," on the row number is the small courtesy that makes a validation report usable: it converts a pandas index into the row number the reader will see in Excel, so they can go straight to the cell.",[46,1432,55,1437,55,1440,55,1443,55,1446,55,1452,55,1456,55,1459,55,1466,55,1470,55,1473,55,1477,55,1481,55,1484,55,1487,55,1489,55,1492,55,1495,55,1498,55,1501,55,1503,55,1505,55,1508,55,1513,55,1518,55,1520,55,1523],{"viewBox":1433,"role":49,"ariaLabelledBy":1434,"xmlns":53,"style":54},"0 0 760 244",[1435,1436],"vd-coerce-t","vd-coerce-d",[57,1438,1439],{"id":1435},"How coercion turns bad values into recorded issues",[61,1441,1442],{"id":1436},"Three raw values enter coercion: the text 2 becomes the number 2 and is fine, the text n\u002Fa becomes NaN and is recorded as unreadable, and the impossible date 2026-02-30 becomes NaT and is recorded. A value that was already blank produces no issue.",[65,1444],{"x":67,"y":67,"width":68,"height":1445,"fill":70},"244",[81,1447,1451],{"x":1448,"y":1449,"style":1450},"120","30","font-size:12.5px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:middle","raw value",[81,1453,1455],{"x":1454,"y":1449,"style":1450},"380","after coercion",[81,1457,1458],{"x":207,"y":1449,"style":1450},"recorded?",[65,1460],{"x":1461,"y":1462,"width":1463,"height":1464,"rx":1465,"fill":153,"stroke":79},"24","44","192","40","8",[81,1467,392],{"x":1448,"y":1468,"style":1469},"69","font-size:12.5px;fill:var(--text,#172033);text-anchor:middle;font-family:ui-monospace,Menlo,monospace",[65,1471],{"x":1472,"y":1462,"width":1463,"height":1464,"rx":1465,"fill":110,"stroke":111},"284",[81,1474,1476],{"x":1454,"y":1468,"style":1475},"font-size:12.5px;fill:var(--teal,#0f9488);text-anchor:middle;font-family:ui-monospace,Menlo,monospace","2",[81,1478,1480],{"x":207,"y":1468,"style":1479},"font-size:12px;fill:var(--muted,#5b6780);text-anchor:middle","no — clean",[65,1482],{"x":1461,"y":1483,"width":1463,"height":1464,"rx":1465,"fill":153,"stroke":79},"96",[81,1485,591],{"x":1448,"y":1486,"style":1469},"121",[65,1488],{"x":1472,"y":1483,"width":1463,"height":1464,"rx":1465,"fill":168,"stroke":143},[81,1490,1039],{"x":1454,"y":1486,"style":1491},"font-size:12.5px;fill:var(--danger,#dc2626);text-anchor:middle;font-family:ui-monospace,Menlo,monospace",[81,1493,1494],{"x":207,"y":1486,"style":172},"yes — unreadable number",[65,1496],{"x":1461,"y":1497,"width":1463,"height":1464,"rx":1465,"fill":153,"stroke":79},"148",[81,1499,486],{"x":1448,"y":1500,"style":1469},"173",[65,1502],{"x":1472,"y":1497,"width":1463,"height":1464,"rx":1465,"fill":168,"stroke":143},[81,1504,1035],{"x":1454,"y":1500,"style":1491},[81,1506,1507],{"x":207,"y":1500,"style":172},"yes — impossible date",[65,1509],{"x":1461,"y":1510,"width":1463,"height":1511,"rx":1465,"fill":1512,"stroke":79},"200","36","#f0f2f5",[81,1514,1517],{"x":1448,"y":1515,"style":1516},"223","font-size:12.5px;fill:var(--muted,#5b6780);text-anchor:middle","(blank)",[65,1519],{"x":1472,"y":1510,"width":1463,"height":1511,"rx":1465,"fill":1512,"stroke":79},[81,1521,1039],{"x":1454,"y":1515,"style":1522},"font-size:12.5px;fill:var(--muted,#5b6780);text-anchor:middle;font-family:ui-monospace,Menlo,monospace",[81,1524,1525],{"x":207,"y":1515,"style":1479},"no — was already empty",[10,1527,1528,1529,1533,1534,1537],{},"The bottom row is the subtlety worth building in from the start: a value that was blank to begin with is a ",[1530,1531,1532],"em",{},"missing data"," question, not a ",[1530,1535,1536],{},"bad data"," question. Conflating them produces validation reports full of noise, and readers stop opening them.",[214,1539,1541],{"id":1540},"layer-3-business-rules","Layer 3: business rules",[10,1543,1544],{},"Type checks say a value is a number; business rules say it is a plausible one. Express each rule as a boolean mask with a message, so adding a rule is one entry rather than a new branch:",[219,1546,1548],{"className":254,"code":1547,"language":256,"meta":224,"style":224},"import pandas as pd\n\nVALID_REGIONS = {\"North\", \"South\", \"West\", \"Central\"}\n\ndef apply_rules(df):\n    normalised = df.copy()\n    normalised[\"Region\"] = normalised[\"Region\"].astype(str).str.strip().str.title()\n\n    rules = [\n        (\"Order_ID\", normalised[\"Order_ID\"].isna(), \"order id is missing\"),\n        (\"Order_ID\", normalised[\"Order_ID\"].duplicated(keep=False) & normalised[\"Order_ID\"].notna(),\n         \"order id appears more than once\"),\n        (\"Region\", ~normalised[\"Region\"].isin(VALID_REGIONS), \"region is not on the approved list\"),\n        (\"Quantity\", normalised[\"Quantity\"] \u003C= 0, \"quantity must be greater than zero\"),\n        (\"Unit_Price\", normalised[\"Unit_Price\"].isna(), \"unit price is missing\"),\n        (\"Order_Date\", normalised[\"Order_Date\"] > pd.Timestamp(\"today\"), \"order date is in the future\"),\n    ]\n\n    issues = []\n    for column, mask, message in rules:\n        for idx in normalised.index[mask.fillna(False)]:\n            issues.append({\n                \"row\": int(idx) + 2,\n                \"column\": column,\n                \"value\": df.loc[idx, column],\n                \"problem\": message,\n            })\n    return normalised, issues\n\nchecked, rule_issues = apply_rules(typed)\nprint(f\"{len(rule_issues)} rule violation(s)\")\n",[14,1549,1550,1560,1564,1593,1597,1606,1615,1639,1643,1652,1673,1705,1712,1741,1766,1783,1811,1816,1820,1828,1840,1856,1860,1876,1882,1888,1896,1901,1909,1914,1925],{"__ignoreMap":224},[228,1551,1552,1554,1556,1558],{"class":93,"line":230},[228,1553,264],{"class":263},[228,1555,268],{"class":267},[228,1557,271],{"class":263},[228,1559,274],{"class":267},[228,1561,1562],{"class":93,"line":277},[228,1563,281],{"emptyLinePlaceholder":280},[228,1565,1566,1569,1571,1574,1576,1578,1581,1583,1585,1587,1590],{"class":93,"line":284},[228,1567,1568],{"class":308},"VALID_REGIONS",[228,1570,692],{"class":263},[228,1572,1573],{"class":267}," {",[228,1575,320],{"class":237},[228,1577,312],{"class":267},[228,1579,1580],{"class":237},"\"South\"",[228,1582,312],{"class":267},[228,1584,477],{"class":237},[228,1586,312],{"class":267},[228,1588,1589],{"class":237},"\"Central\"",[228,1591,1592],{"class":267},"}\n",[228,1594,1595],{"class":93,"line":296},[228,1596,281],{"emptyLinePlaceholder":280},[228,1598,1599,1601,1604],{"class":93,"line":356},[228,1600,724],{"class":263},[228,1602,1603],{"class":727}," apply_rules",[228,1605,1069],{"class":267},[228,1607,1608,1611,1613],{"class":93,"line":406},[228,1609,1610],{"class":267},"    normalised ",[228,1612,290],{"class":263},[228,1614,1089],{"class":267},[228,1616,1617,1620,1622,1624,1626,1629,1631,1634,1636],{"class":93,"line":459},[228,1618,1619],{"class":267},"    normalised[",[228,1621,315],{"class":237},[228,1623,1103],{"class":267},[228,1625,290],{"class":263},[228,1627,1628],{"class":267}," normalised[",[228,1630,315],{"class":237},[228,1632,1633],{"class":267},"].astype(",[228,1635,852],{"class":308},[228,1637,1638],{"class":267},").str.strip().str.title()\n",[228,1640,1641],{"class":93,"line":509},[228,1642,281],{"emptyLinePlaceholder":280},[228,1644,1645,1648,1650],{"class":93,"line":557},[228,1646,1647],{"class":267},"    rules ",[228,1649,290],{"class":263},[228,1651,293],{"class":267},[228,1653,1654,1657,1659,1662,1664,1667,1670],{"class":93,"line":605},[228,1655,1656],{"class":267},"        (",[228,1658,302],{"class":237},[228,1660,1661],{"class":267},", normalised[",[228,1663,302],{"class":237},[228,1665,1666],{"class":267},"].isna(), ",[228,1668,1669],{"class":237},"\"order id is missing\"",[228,1671,1672],{"class":267},"),\n",[228,1674,1675,1677,1679,1681,1683,1686,1689,1691,1693,1696,1698,1700,1702],{"class":93,"line":611},[228,1676,1656],{"class":267},[228,1678,302],{"class":237},[228,1680,1661],{"class":267},[228,1682,302],{"class":237},[228,1684,1685],{"class":267},"].duplicated(",[228,1687,1688],{"class":622},"keep",[228,1690,290],{"class":263},[228,1692,628],{"class":308},[228,1694,1695],{"class":267},") ",[228,1697,1252],{"class":263},[228,1699,1628],{"class":267},[228,1701,302],{"class":237},[228,1703,1704],{"class":267},"].notna(),\n",[228,1706,1707,1710],{"class":93,"line":644},[228,1708,1709],{"class":237},"         \"order id appears more than once\"",[228,1711,1672],{"class":267},[228,1713,1714,1716,1718,1720,1723,1726,1728,1731,1733,1736,1739],{"class":93,"line":874},[228,1715,1656],{"class":267},[228,1717,315],{"class":237},[228,1719,312],{"class":267},[228,1721,1722],{"class":263},"~",[228,1724,1725],{"class":267},"normalised[",[228,1727,315],{"class":237},[228,1729,1730],{"class":267},"].isin(",[228,1732,1568],{"class":308},[228,1734,1735],{"class":267},"), ",[228,1737,1738],{"class":237},"\"region is not on the approved list\"",[228,1740,1672],{"class":267},[228,1742,1743,1745,1747,1749,1751,1753,1756,1759,1761,1764],{"class":93,"line":905},[228,1744,1656],{"class":267},[228,1746,335],{"class":237},[228,1748,1661],{"class":267},[228,1750,335],{"class":237},[228,1752,1103],{"class":267},[228,1754,1755],{"class":263},"\u003C=",[228,1757,1758],{"class":308}," 0",[228,1760,312],{"class":267},[228,1762,1763],{"class":237},"\"quantity must be greater than zero\"",[228,1765,1672],{"class":267},[228,1767,1768,1770,1772,1774,1776,1778,1781],{"class":93,"line":913},[228,1769,1656],{"class":267},[228,1771,345],{"class":237},[228,1773,1661],{"class":267},[228,1775,345],{"class":237},[228,1777,1666],{"class":267},[228,1779,1780],{"class":237},"\"unit price is missing\"",[228,1782,1672],{"class":267},[228,1784,1785,1787,1789,1791,1793,1795,1798,1801,1804,1806,1809],{"class":93,"line":951},[228,1786,1656],{"class":267},[228,1788,325],{"class":237},[228,1790,1661],{"class":267},[228,1792,325],{"class":237},[228,1794,1103],{"class":267},[228,1796,1797],{"class":263},">",[228,1799,1800],{"class":267}," pd.Timestamp(",[228,1802,1803],{"class":237},"\"today\"",[228,1805,1735],{"class":267},[228,1807,1808],{"class":237},"\"order date is in the future\"",[228,1810,1672],{"class":267},[228,1812,1813],{"class":93,"line":959},[228,1814,1815],{"class":267},"    ]\n",[228,1817,1818],{"class":93,"line":973},[228,1819,281],{"emptyLinePlaceholder":280},[228,1821,1822,1824,1826],{"class":93,"line":986},[228,1823,1074],{"class":267},[228,1825,290],{"class":263},[228,1827,1079],{"class":267},[228,1829,1830,1832,1835,1837],{"class":93,"line":991},[228,1831,1215],{"class":263},[228,1833,1834],{"class":267}," column, mask, message ",[228,1836,864],{"class":263},[228,1838,1839],{"class":267}," rules:\n",[228,1841,1842,1844,1846,1848,1851,1853],{"class":93,"line":1006},[228,1843,1278],{"class":263},[228,1845,1281],{"class":267},[228,1847,864],{"class":263},[228,1849,1850],{"class":267}," normalised.index[mask.fillna(",[228,1852,628],{"class":308},[228,1854,1855],{"class":267},")]:\n",[228,1857,1858],{"class":93,"line":1388},[228,1859,1291],{"class":267},[228,1861,1862,1864,1866,1868,1870,1872,1874],{"class":93,"line":1393},[228,1863,1296],{"class":237},[228,1865,305],{"class":267},[228,1867,1301],{"class":308},[228,1869,1304],{"class":267},[228,1871,1307],{"class":263},[228,1873,1310],{"class":308},[228,1875,1373],{"class":267},[228,1877,1878,1880],{"class":93,"line":1404},[228,1879,1322],{"class":237},[228,1881,1325],{"class":267},[228,1883,1884,1886],{"class":93,"line":1417},[228,1885,1330],{"class":237},[228,1887,1333],{"class":267},[228,1889,1891,1893],{"class":93,"line":1890},26,[228,1892,1338],{"class":237},[228,1894,1895],{"class":267},": message,\n",[228,1897,1899],{"class":93,"line":1898},27,[228,1900,1378],{"class":267},[228,1902,1904,1906],{"class":93,"line":1903},28,[228,1905,976],{"class":263},[228,1907,1908],{"class":267}," normalised, issues\n",[228,1910,1912],{"class":93,"line":1911},29,[228,1913,281],{"emptyLinePlaceholder":280},[228,1915,1917,1920,1922],{"class":93,"line":1916},30,[228,1918,1919],{"class":267},"checked, rule_issues ",[228,1921,290],{"class":263},[228,1923,1924],{"class":267}," apply_rules(typed)\n",[228,1926,1928,1930,1932,1934,1936,1938,1941,1944,1946,1949],{"class":93,"line":1927},31,[228,1929,647],{"class":308},[228,1931,650],{"class":267},[228,1933,778],{"class":263},[228,1935,807],{"class":237},[228,1937,785],{"class":784},[228,1939,1940],{"class":308},"len",[228,1942,1943],{"class":267},"(rule_issues)",[228,1945,794],{"class":784},[228,1947,1948],{"class":237}," rule violation(s)\"",[228,1950,641],{"class":267},[10,1952,1953,1954,1957,1958,1960,1961,1963,1964,1968],{},"Normalising ",[14,1955,1956],{},"Region"," before testing it is what makes the rule fair: ",[14,1959,374],{}," is a formatting problem, not a wrong region, and correcting it silently while rejecting ",[14,1962,424],{}," is exactly the behaviour a submitter expects. Where cleaning ends and validation begins is a judgement call — the rule of thumb is to auto-fix anything unambiguous and report anything that requires a decision. The ",[40,1965,1967],{"href":1966},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002F","cleaning Excel data with pandas"," guide covers the fixing side in depth.",[214,1970,1972],{"id":1971},"layer-4-quarantine-instead-of-crashing","Layer 4: quarantine instead of crashing",[10,1974,1975],{},"With the issues collected, split the frame. Clean rows go on to the report; problem rows are held back with their reasons attached:",[219,1977,1979],{"className":254,"code":1978,"language":256,"meta":224,"style":224},"import pandas as pd\n\ndef split_clean_and_quarantine(df, issues):\n    bad_rows = {issue[\"row\"] - 2 for issue in issues}\n    clean = df.drop(index=bad_rows, errors=\"ignore\")\n    quarantined = df.loc[sorted(bad_rows)].copy()\n\n    reasons = {}\n    for issue in issues:\n        reasons.setdefault(issue[\"row\"] - 2, []).append(f\"{issue['column']}: {issue['problem']}\")\n    quarantined[\"Why_rejected\"] = [\n        \"; \".join(reasons.get(idx, [])) for idx in quarantined.index\n    ]\n    return clean, quarantined\n\nclean, quarantined = split_clean_and_quarantine(checked, issues + rule_issues)\nprint(f\"{len(clean)} clean row(s), {len(quarantined)} quarantined\")\n",[14,1980,1981,1991,1995,2005,2034,2060,2076,2080,2090,2100,2150,2164,2181,2185,2192,2196,2211],{"__ignoreMap":224},[228,1982,1983,1985,1987,1989],{"class":93,"line":230},[228,1984,264],{"class":263},[228,1986,268],{"class":267},[228,1988,271],{"class":263},[228,1990,274],{"class":267},[228,1992,1993],{"class":93,"line":277},[228,1994,281],{"emptyLinePlaceholder":280},[228,1996,1997,1999,2002],{"class":93,"line":284},[228,1998,724],{"class":263},[228,2000,2001],{"class":727}," split_clean_and_quarantine",[228,2003,2004],{"class":267},"(df, issues):\n",[228,2006,2007,2010,2012,2015,2018,2020,2022,2024,2027,2029,2031],{"class":93,"line":296},[228,2008,2009],{"class":267},"    bad_rows ",[228,2011,290],{"class":263},[228,2013,2014],{"class":267}," {issue[",[228,2016,2017],{"class":237},"\"row\"",[228,2019,1103],{"class":267},[228,2021,442],{"class":263},[228,2023,1310],{"class":308},[228,2025,2026],{"class":263}," for",[228,2028,1409],{"class":267},[228,2030,864],{"class":263},[228,2032,2033],{"class":267}," issues}\n",[228,2035,2036,2039,2041,2044,2046,2048,2051,2053,2055,2058],{"class":93,"line":356},[228,2037,2038],{"class":267},"    clean ",[228,2040,290],{"class":263},[228,2042,2043],{"class":267}," df.drop(",[228,2045,623],{"class":622},[228,2047,290],{"class":263},[228,2049,2050],{"class":267},"bad_rows, ",[228,2052,1116],{"class":622},[228,2054,290],{"class":263},[228,2056,2057],{"class":237},"\"ignore\"",[228,2059,641],{"class":267},[228,2061,2062,2065,2067,2070,2073],{"class":93,"line":406},[228,2063,2064],{"class":267},"    quarantined ",[228,2066,290],{"class":263},[228,2068,2069],{"class":267}," df.loc[",[228,2071,2072],{"class":308},"sorted",[228,2074,2075],{"class":267},"(bad_rows)].copy()\n",[228,2077,2078],{"class":93,"line":459},[228,2079,281],{"emptyLinePlaceholder":280},[228,2081,2082,2085,2087],{"class":93,"line":509},[228,2083,2084],{"class":267},"    reasons ",[228,2086,290],{"class":263},[228,2088,2089],{"class":267}," {}\n",[228,2091,2092,2094,2096,2098],{"class":93,"line":557},[228,2093,1215],{"class":263},[228,2095,1409],{"class":267},[228,2097,864],{"class":263},[228,2099,1414],{"class":267},[228,2101,2102,2105,2107,2109,2111,2113,2116,2118,2120,2122,2125,2128,2131,2133,2135,2137,2139,2142,2144,2146,2148],{"class":93,"line":605},[228,2103,2104],{"class":267},"        reasons.setdefault(issue[",[228,2106,2017],{"class":237},[228,2108,1103],{"class":267},[228,2110,442],{"class":263},[228,2112,1310],{"class":308},[228,2114,2115],{"class":267},", []).append(",[228,2117,778],{"class":263},[228,2119,807],{"class":237},[228,2121,785],{"class":784},[228,2123,2124],{"class":267},"issue[",[228,2126,2127],{"class":237},"'column'",[228,2129,2130],{"class":267},"]",[228,2132,794],{"class":784},[228,2134,305],{"class":237},[228,2136,785],{"class":784},[228,2138,2124],{"class":267},[228,2140,2141],{"class":237},"'problem'",[228,2143,2130],{"class":267},[228,2145,794],{"class":784},[228,2147,807],{"class":237},[228,2149,641],{"class":267},[228,2151,2152,2155,2158,2160,2162],{"class":93,"line":611},[228,2153,2154],{"class":267},"    quarantined[",[228,2156,2157],{"class":237},"\"Why_rejected\"",[228,2159,1103],{"class":267},[228,2161,290],{"class":263},[228,2163,293],{"class":267},[228,2165,2166,2169,2172,2174,2176,2178],{"class":93,"line":644},[228,2167,2168],{"class":237},"        \"; \"",[228,2170,2171],{"class":267},".join(reasons.get(idx, [])) ",[228,2173,858],{"class":263},[228,2175,1281],{"class":267},[228,2177,864],{"class":263},[228,2179,2180],{"class":267}," quarantined.index\n",[228,2182,2183],{"class":93,"line":874},[228,2184,1815],{"class":267},[228,2186,2187,2189],{"class":93,"line":905},[228,2188,976],{"class":263},[228,2190,2191],{"class":267}," clean, quarantined\n",[228,2193,2194],{"class":93,"line":913},[228,2195,281],{"emptyLinePlaceholder":280},[228,2197,2198,2201,2203,2206,2208],{"class":93,"line":951},[228,2199,2200],{"class":267},"clean, quarantined ",[228,2202,290],{"class":263},[228,2204,2205],{"class":267}," split_clean_and_quarantine(checked, issues ",[228,2207,1307],{"class":263},[228,2209,2210],{"class":267}," rule_issues)\n",[228,2212,2213,2215,2217,2219,2221,2223,2225,2228,2230,2233,2235,2237,2240,2242,2245],{"class":93,"line":959},[228,2214,647],{"class":308},[228,2216,650],{"class":267},[228,2218,778],{"class":263},[228,2220,807],{"class":237},[228,2222,785],{"class":784},[228,2224,1940],{"class":308},[228,2226,2227],{"class":267},"(clean)",[228,2229,794],{"class":784},[228,2231,2232],{"class":237}," clean row(s), ",[228,2234,785],{"class":784},[228,2236,1940],{"class":308},[228,2238,2239],{"class":267},"(quarantined)",[228,2241,794],{"class":784},[228,2243,2244],{"class":237}," quarantined\"",[228,2246,641],{"class":267},[10,2248,2249,2250,2253],{},"Stopping the whole job on one bad row is rarely the right behaviour for a recurring report: the business still needs Monday's numbers, and one malformed line should not hold them up. Stopping on a ",[1530,2251,2252],{},"structural"," problem is different — if a column is missing, every row is suspect.",[214,2255,2257],{"id":2256},"layer-5-report-the-failures-back-in-excel","Layer 5: report the failures back in Excel",[10,2259,2260],{},"Submitters live in Excel, so the error report belongs in the workbook they receive:",[219,2262,2264],{"className":254,"code":2263,"language":256,"meta":224,"style":224},"import pandas as pd\n\nwith pd.ExcelWriter(\"validated.xlsx\", engine=\"openpyxl\") as writer:\n    clean.to_excel(writer, sheet_name=\"Clean\", index=False)\n    quarantined.to_excel(writer, sheet_name=\"Rejected\", index=False)\n    pd.DataFrame(issues + rule_issues).to_excel(writer, sheet_name=\"Issues\", index=False)\n\nprint(\"wrote validated.xlsx with Clean, Rejected and Issues sheets\")\n",[14,2265,2266,2276,2280,2308,2330,2352,2379,2383],{"__ignoreMap":224},[228,2267,2268,2270,2272,2274],{"class":93,"line":230},[228,2269,264],{"class":263},[228,2271,268],{"class":267},[228,2273,271],{"class":263},[228,2275,274],{"class":267},[228,2277,2278],{"class":93,"line":277},[228,2279,281],{"emptyLinePlaceholder":280},[228,2281,2282,2285,2288,2291,2293,2296,2298,2301,2303,2305],{"class":93,"line":284},[228,2283,2284],{"class":263},"with",[228,2286,2287],{"class":267}," pd.ExcelWriter(",[228,2289,2290],{"class":237},"\"validated.xlsx\"",[228,2292,312],{"class":267},[228,2294,2295],{"class":622},"engine",[228,2297,290],{"class":263},[228,2299,2300],{"class":237},"\"openpyxl\"",[228,2302,1695],{"class":267},[228,2304,271],{"class":263},[228,2306,2307],{"class":267}," writer:\n",[228,2309,2310,2313,2315,2317,2320,2322,2324,2326,2328],{"class":93,"line":296},[228,2311,2312],{"class":267},"    clean.to_excel(writer, ",[228,2314,633],{"class":622},[228,2316,290],{"class":263},[228,2318,2319],{"class":237},"\"Clean\"",[228,2321,312],{"class":267},[228,2323,623],{"class":622},[228,2325,290],{"class":263},[228,2327,628],{"class":308},[228,2329,641],{"class":267},[228,2331,2332,2335,2337,2339,2342,2344,2346,2348,2350],{"class":93,"line":356},[228,2333,2334],{"class":267},"    quarantined.to_excel(writer, ",[228,2336,633],{"class":622},[228,2338,290],{"class":263},[228,2340,2341],{"class":237},"\"Rejected\"",[228,2343,312],{"class":267},[228,2345,623],{"class":622},[228,2347,290],{"class":263},[228,2349,628],{"class":308},[228,2351,641],{"class":267},[228,2353,2354,2357,2359,2362,2364,2366,2369,2371,2373,2375,2377],{"class":93,"line":406},[228,2355,2356],{"class":267},"    pd.DataFrame(issues ",[228,2358,1307],{"class":263},[228,2360,2361],{"class":267}," rule_issues).to_excel(writer, ",[228,2363,633],{"class":622},[228,2365,290],{"class":263},[228,2367,2368],{"class":237},"\"Issues\"",[228,2370,312],{"class":267},[228,2372,623],{"class":622},[228,2374,290],{"class":263},[228,2376,628],{"class":308},[228,2378,641],{"class":267},[228,2380,2381],{"class":93,"line":459},[228,2382,281],{"emptyLinePlaceholder":280},[228,2384,2385,2387,2389,2392],{"class":93,"line":509},[228,2386,647],{"class":308},[228,2388,650],{"class":267},[228,2390,2391],{"class":237},"\"wrote validated.xlsx with Clean, Rejected and Issues sheets\"",[228,2393,641],{"class":267},[10,2395,2396,2397,2400,2401,2405,2406,2410],{},"Three sheets, one file, no email thread. Making the ",[14,2398,2399],{},"Issues"," sheet the ",[40,2402,2404],{"href":2403},"\u002Fautomating-reporting-workflows\u002Fbuilding-multi-sheet-excel-dashboards\u002F","active sheet"," so it opens first is a small touch that gets problems fixed faster, and ",[40,2407,2409],{"href":2408},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fhighlight-invalid-cells-in-excel-with-python\u002F","highlighting the offending cells"," on the original data turns the report into something a person can act on without cross-referencing row numbers.",[214,2412,2414],{"id":2413},"layer-6-stop-the-next-round-of-bad-data","Layer 6: stop the next round of bad data",[10,2416,2417,2418,2421,2422,2425],{},"Everything so far reacts to problems. ",[14,2419,2420],{},"openpyxl","'s ",[14,2423,2424],{},"DataValidation"," prevents them, by writing Excel's own validation rules into the file you send back:",[219,2427,2429],{"className":254,"code":2428,"language":256,"meta":224,"style":224},"from openpyxl import load_workbook\nfrom openpyxl.worksheet.datavalidation import DataValidation\n\nwb = load_workbook(\"validated.xlsx\")\nws = wb[\"Clean\"]\nlast = ws.max_row + 200        # leave room for new rows\n\nregion = DataValidation(\n    type=\"list\",\n    formula1='\"North,South,West,Central\"',\n    allow_blank=False,\n    showErrorMessage=True,\n)\nregion.errorTitle = \"Invalid region\"\nregion.error = \"Pick a region from the dropdown.\"\nws.add_data_validation(region)\nregion.add(f\"B2:B{last}\")\n\nquantity = DataValidation(type=\"whole\", operator=\"greaterThan\", formula1=0, showErrorMessage=True)\nquantity.errorTitle = \"Invalid quantity\"\nquantity.error = \"Quantity must be a whole number greater than zero.\"\nws.add_data_validation(quantity)\nquantity.add(f\"D2:D{last}\")\n\nwb.save(\"validated_with_rules.xlsx\")\n",[14,2430,2431,2444,2456,2460,2474,2488,2506,2510,2520,2532,2544,2555,2567,2571,2581,2591,2596,2617,2621,2669,2679,2689,2694,2714,2718],{"__ignoreMap":224},[228,2432,2433,2436,2439,2441],{"class":93,"line":230},[228,2434,2435],{"class":263},"from",[228,2437,2438],{"class":267}," openpyxl ",[228,2440,264],{"class":263},[228,2442,2443],{"class":267}," load_workbook\n",[228,2445,2446,2448,2451,2453],{"class":93,"line":277},[228,2447,2435],{"class":263},[228,2449,2450],{"class":267}," openpyxl.worksheet.datavalidation ",[228,2452,264],{"class":263},[228,2454,2455],{"class":267}," DataValidation\n",[228,2457,2458],{"class":93,"line":284},[228,2459,281],{"emptyLinePlaceholder":280},[228,2461,2462,2465,2467,2470,2472],{"class":93,"line":296},[228,2463,2464],{"class":267},"wb ",[228,2466,290],{"class":263},[228,2468,2469],{"class":267}," load_workbook(",[228,2471,2290],{"class":237},[228,2473,641],{"class":267},[228,2475,2476,2479,2481,2484,2486],{"class":93,"line":356},[228,2477,2478],{"class":267},"ws ",[228,2480,290],{"class":263},[228,2482,2483],{"class":267}," wb[",[228,2485,2319],{"class":237},[228,2487,608],{"class":267},[228,2489,2490,2493,2495,2498,2500,2503],{"class":93,"line":406},[228,2491,2492],{"class":267},"last ",[228,2494,290],{"class":263},[228,2496,2497],{"class":267}," ws.max_row ",[228,2499,1307],{"class":263},[228,2501,2502],{"class":308}," 200",[228,2504,2505],{"class":1316},"        # leave room for new rows\n",[228,2507,2508],{"class":93,"line":459},[228,2509,281],{"emptyLinePlaceholder":280},[228,2511,2512,2515,2517],{"class":93,"line":509},[228,2513,2514],{"class":267},"region ",[228,2516,290],{"class":263},[228,2518,2519],{"class":267}," DataValidation(\n",[228,2521,2522,2525,2527,2530],{"class":93,"line":557},[228,2523,2524],{"class":622},"    type",[228,2526,290],{"class":263},[228,2528,2529],{"class":237},"\"list\"",[228,2531,1373],{"class":267},[228,2533,2534,2537,2539,2542],{"class":93,"line":605},[228,2535,2536],{"class":622},"    formula1",[228,2538,290],{"class":263},[228,2540,2541],{"class":237},"'\"North,South,West,Central\"'",[228,2543,1373],{"class":267},[228,2545,2546,2549,2551,2553],{"class":93,"line":611},[228,2547,2548],{"class":622},"    allow_blank",[228,2550,290],{"class":263},[228,2552,628],{"class":308},[228,2554,1373],{"class":267},[228,2556,2557,2560,2562,2565],{"class":93,"line":644},[228,2558,2559],{"class":622},"    showErrorMessage",[228,2561,290],{"class":263},[228,2563,2564],{"class":308},"True",[228,2566,1373],{"class":267},[228,2568,2569],{"class":93,"line":874},[228,2570,641],{"class":267},[228,2572,2573,2576,2578],{"class":93,"line":905},[228,2574,2575],{"class":267},"region.errorTitle ",[228,2577,290],{"class":263},[228,2579,2580],{"class":237}," \"Invalid region\"\n",[228,2582,2583,2586,2588],{"class":93,"line":913},[228,2584,2585],{"class":267},"region.error ",[228,2587,290],{"class":263},[228,2589,2590],{"class":237}," \"Pick a region from the dropdown.\"\n",[228,2592,2593],{"class":93,"line":951},[228,2594,2595],{"class":267},"ws.add_data_validation(region)\n",[228,2597,2598,2601,2603,2606,2608,2611,2613,2615],{"class":93,"line":959},[228,2599,2600],{"class":267},"region.add(",[228,2602,778],{"class":263},[228,2604,2605],{"class":237},"\"B2:B",[228,2607,785],{"class":784},[228,2609,2610],{"class":267},"last",[228,2612,794],{"class":784},[228,2614,807],{"class":237},[228,2616,641],{"class":267},[228,2618,2619],{"class":93,"line":973},[228,2620,281],{"emptyLinePlaceholder":280},[228,2622,2623,2626,2628,2631,2634,2636,2639,2641,2644,2646,2649,2651,2654,2656,2658,2660,2663,2665,2667],{"class":93,"line":986},[228,2624,2625],{"class":267},"quantity ",[228,2627,290],{"class":263},[228,2629,2630],{"class":267}," DataValidation(",[228,2632,2633],{"class":622},"type",[228,2635,290],{"class":263},[228,2637,2638],{"class":237},"\"whole\"",[228,2640,312],{"class":267},[228,2642,2643],{"class":622},"operator",[228,2645,290],{"class":263},[228,2647,2648],{"class":237},"\"greaterThan\"",[228,2650,312],{"class":267},[228,2652,2653],{"class":622},"formula1",[228,2655,290],{"class":263},[228,2657,67],{"class":308},[228,2659,312],{"class":267},[228,2661,2662],{"class":622},"showErrorMessage",[228,2664,290],{"class":263},[228,2666,2564],{"class":308},[228,2668,641],{"class":267},[228,2670,2671,2674,2676],{"class":93,"line":991},[228,2672,2673],{"class":267},"quantity.errorTitle ",[228,2675,290],{"class":263},[228,2677,2678],{"class":237}," \"Invalid quantity\"\n",[228,2680,2681,2684,2686],{"class":93,"line":1006},[228,2682,2683],{"class":267},"quantity.error ",[228,2685,290],{"class":263},[228,2687,2688],{"class":237}," \"Quantity must be a whole number greater than zero.\"\n",[228,2690,2691],{"class":93,"line":1388},[228,2692,2693],{"class":267},"ws.add_data_validation(quantity)\n",[228,2695,2696,2699,2701,2704,2706,2708,2710,2712],{"class":93,"line":1393},[228,2697,2698],{"class":267},"quantity.add(",[228,2700,778],{"class":263},[228,2702,2703],{"class":237},"\"D2:D",[228,2705,785],{"class":784},[228,2707,2610],{"class":267},[228,2709,794],{"class":784},[228,2711,807],{"class":237},[228,2713,641],{"class":267},[228,2715,2716],{"class":93,"line":1404},[228,2717,281],{"emptyLinePlaceholder":280},[228,2719,2720,2723,2726],{"class":93,"line":1417},[228,2721,2722],{"class":267},"wb.save(",[228,2724,2725],{"class":237},"\"validated_with_rules.xlsx\"",[228,2727,641],{"class":267},[10,2729,2730,2731,2734,2735,2739],{},"Excel now refuses the values your pandas layer would have rejected, at the moment someone types them. Two limits are worth knowing: these rules constrain ",[1530,2732,2733],{},"typing",", not pasting, in many Excel versions, and openpyxl itself does not enforce them when writing — so the Python checks stay necessary. ",[40,2736,2738],{"href":2737},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fadd-dropdown-data-validation-to-excel-with-openpyxl\u002F","Adding dropdown validation"," covers the list-based rules in more detail, including lists that read from a named range.",[214,2741,2743],{"id":2742},"choosing-where-each-check-belongs","Choosing where each check belongs",[2745,2746,2747,2766],"table",{},[2748,2749,2750],"thead",{},[2751,2752,2753,2757,2760,2763],"tr",{},[2754,2755,2756],"th",{},"Check",[2754,2758,2759],{},"pandas, on read",[2754,2761,2762],{},"openpyxl, on write",[2754,2764,2765],{},"Both",[2767,2768,2769,2783,2794,2806,2818,2829,2842],"tbody",{},[2751,2770,2771,2775,2778,2781],{},[2772,2773,2774],"td",{},"Required column present",[2772,2776,2777],{},"Yes",[2772,2779,2780],{},"—",[2772,2782,2780],{},[2751,2784,2785,2788,2790,2792],{},[2772,2786,2787],{},"Value parses as a number or date",[2772,2789,2777],{},[2772,2791,2780],{},[2772,2793,2780],{},[2751,2795,2796,2799,2801,2804],{},[2772,2797,2798],{},"Value is on an approved list",[2772,2800,2777],{},[2772,2802,2803],{},"Dropdown",[2772,2805,2777],{},[2751,2807,2808,2811,2813,2816],{},[2772,2809,2810],{},"Number within a range",[2772,2812,2777],{},[2772,2814,2815],{},"Numeric rule",[2772,2817,2777],{},[2751,2819,2820,2823,2825,2827],{},[2772,2821,2822],{},"Cross-row rules (duplicates, totals)",[2772,2824,2777],{},[2772,2826,2780],{},[2772,2828,2780],{},[2751,2830,2831,2834,2836,2839],{},[2772,2832,2833],{},"Cross-column rules (end date after start)",[2772,2835,2777],{},[2772,2837,2838],{},"Formula rule",[2772,2840,2841],{},"Sometimes",[2751,2843,2844,2847,2849,2851],{},[2772,2845,2846],{},"Cosmetic normalisation (trimming, casing)",[2772,2848,2777],{},[2772,2850,2780],{},[2772,2852,2780],{},[10,2854,2855],{},"Anything involving more than one row can only be done in pandas — Excel's validation looks at one cell at a time. Anything meant to guide a human typing belongs in the workbook. The middle rows of that table are the ones worth implementing twice: the dropdown makes the right value easy to pick, and the pandas check still catches the value that arrived by paste, by import, or from a copy of the file that predates the rules. Duplicating a rule in two places is only a maintenance problem if the two definitions can drift, so keep the approved list in one place — a lookup sheet or a constants module — and generate both the Excel rule and the Python check from it.",[214,2857,2859],{"id":2858},"validate-against-reference-data-not-a-hardcoded-list","Validate against reference data, not a hardcoded list",[10,2861,2862,2864],{},[14,2863,1568],{}," as a Python set works until someone opens a new region and nobody tells you. Reference lists belong in data, and the most convenient place for them is usually a lookup sheet in the same workbook or a small master file that the business owns:",[219,2866,2868],{"className":254,"code":2867,"language":256,"meta":224,"style":224},"import pandas as pd\n\ndef load_reference(path, sheet=\"Lookups\", column=\"Region\"):\n    ref = pd.read_excel(path, sheet_name=sheet, usecols=[column])\n    values = (\n        ref[column].dropna().astype(str).str.strip().str.title().unique().tolist()\n    )\n    if not values:\n        raise ValueError(f\"reference list {sheet}!{column} is empty — refusing to validate\")\n    return set(values)\n\n# A missing lookup sheet should fail loudly rather than silently approving everything\ntry:\n    valid_regions = load_reference(\"master_data.xlsx\")\nexcept (FileNotFoundError, ValueError) as exc:\n    raise SystemExit(f\"cannot validate regions: {exc}\")\n\nunknown = sorted(set(checked[\"Region\"].dropna()) - valid_regions)\nprint(\"regions not in the master list:\", unknown)\n",[14,2869,2870,2880,2884,2907,2930,2940,2950,2955,2965,2999,3009,3013,3018,3026,3041,3063,3089,3093,3121],{"__ignoreMap":224},[228,2871,2872,2874,2876,2878],{"class":93,"line":230},[228,2873,264],{"class":263},[228,2875,268],{"class":267},[228,2877,271],{"class":263},[228,2879,274],{"class":267},[228,2881,2882],{"class":93,"line":277},[228,2883,281],{"emptyLinePlaceholder":280},[228,2885,2886,2888,2891,2893,2895,2898,2901,2903,2905],{"class":93,"line":284},[228,2887,724],{"class":263},[228,2889,2890],{"class":727}," load_reference",[228,2892,731],{"class":267},[228,2894,290],{"class":263},[228,2896,2897],{"class":237},"\"Lookups\"",[228,2899,2900],{"class":267},", column",[228,2902,290],{"class":263},[228,2904,315],{"class":237},[228,2906,738],{"class":267},[228,2908,2909,2912,2914,2916,2918,2920,2922,2925,2927],{"class":93,"line":296},[228,2910,2911],{"class":267},"    ref ",[228,2913,290],{"class":263},[228,2915,823],{"class":267},[228,2917,633],{"class":622},[228,2919,290],{"class":263},[228,2921,830],{"class":267},[228,2923,2924],{"class":622},"usecols",[228,2926,290],{"class":263},[228,2928,2929],{"class":267},"[column])\n",[228,2931,2932,2935,2937],{"class":93,"line":356},[228,2933,2934],{"class":267},"    values ",[228,2936,290],{"class":263},[228,2938,2939],{"class":267}," (\n",[228,2941,2942,2945,2947],{"class":93,"line":406},[228,2943,2944],{"class":267},"        ref[column].dropna().astype(",[228,2946,852],{"class":308},[228,2948,2949],{"class":267},").str.strip().str.title().unique().tolist()\n",[228,2951,2952],{"class":93,"line":459},[228,2953,2954],{"class":267},"    )\n",[228,2956,2957,2959,2962],{"class":93,"line":509},[228,2958,753],{"class":263},[228,2960,2961],{"class":263}," not",[228,2963,2964],{"class":267}," values:\n",[228,2966,2967,2969,2971,2973,2975,2978,2980,2982,2984,2987,2989,2992,2994,2997],{"class":93,"line":557},[228,2968,770],{"class":263},[228,2970,773],{"class":308},[228,2972,650],{"class":267},[228,2974,778],{"class":263},[228,2976,2977],{"class":237},"\"reference list ",[228,2979,785],{"class":784},[228,2981,788],{"class":267},[228,2983,794],{"class":784},[228,2985,2986],{"class":237},"!",[228,2988,785],{"class":784},[228,2990,2991],{"class":267},"column",[228,2993,794],{"class":784},[228,2995,2996],{"class":237}," is empty — refusing to validate\"",[228,2998,641],{"class":267},[228,3000,3001,3003,3006],{"class":93,"line":605},[228,3002,976],{"class":263},[228,3004,3005],{"class":308}," set",[228,3007,3008],{"class":267},"(values)\n",[228,3010,3011],{"class":93,"line":611},[228,3012,281],{"emptyLinePlaceholder":280},[228,3014,3015],{"class":93,"line":644},[228,3016,3017],{"class":1316},"# A missing lookup sheet should fail loudly rather than silently approving everything\n",[228,3019,3020,3023],{"class":93,"line":874},[228,3021,3022],{"class":263},"try",[228,3024,3025],{"class":267},":\n",[228,3027,3028,3031,3033,3036,3039],{"class":93,"line":905},[228,3029,3030],{"class":267},"    valid_regions ",[228,3032,290],{"class":263},[228,3034,3035],{"class":267}," load_reference(",[228,3037,3038],{"class":237},"\"master_data.xlsx\"",[228,3040,641],{"class":267},[228,3042,3043,3046,3048,3051,3053,3056,3058,3060],{"class":93,"line":913},[228,3044,3045],{"class":263},"except",[228,3047,1223],{"class":267},[228,3049,3050],{"class":308},"FileNotFoundError",[228,3052,312],{"class":267},[228,3054,3055],{"class":308},"ValueError",[228,3057,1695],{"class":267},[228,3059,271],{"class":263},[228,3061,3062],{"class":267}," exc:\n",[228,3064,3065,3068,3071,3073,3075,3078,3080,3083,3085,3087],{"class":93,"line":951},[228,3066,3067],{"class":263},"    raise",[228,3069,3070],{"class":308}," SystemExit",[228,3072,650],{"class":267},[228,3074,778],{"class":263},[228,3076,3077],{"class":237},"\"cannot validate regions: ",[228,3079,785],{"class":784},[228,3081,3082],{"class":267},"exc",[228,3084,794],{"class":784},[228,3086,807],{"class":237},[228,3088,641],{"class":267},[228,3090,3091],{"class":93,"line":959},[228,3092,281],{"emptyLinePlaceholder":280},[228,3094,3095,3098,3100,3103,3105,3108,3111,3113,3116,3118],{"class":93,"line":973},[228,3096,3097],{"class":267},"unknown ",[228,3099,290],{"class":263},[228,3101,3102],{"class":308}," sorted",[228,3104,650],{"class":267},[228,3106,3107],{"class":308},"set",[228,3109,3110],{"class":267},"(checked[",[228,3112,315],{"class":237},[228,3114,3115],{"class":267},"].dropna()) ",[228,3117,442],{"class":263},[228,3119,3120],{"class":267}," valid_regions)\n",[228,3122,3123,3125,3127,3130],{"class":93,"line":986},[228,3124,647],{"class":308},[228,3126,650],{"class":267},[228,3128,3129],{"class":237},"\"regions not in the master list:\"",[228,3131,3132],{"class":267},", unknown)\n",[10,3134,3135,3136,3138],{},"The empty-list guard matters more than it looks. A reference list that fails to load and quietly becomes an empty set turns every row into a violation; one that becomes ",[14,3137,504],{}," and is skipped turns validation off entirely. Both have shipped in production somewhere, and the second is much harder to notice.",[10,3140,3141,3142,3145],{},"Where the reference data is genuinely large — a product catalogue with 40,000 SKUs — checking membership with a Python set is still the fastest approach, and ",[14,3143,3144],{},"df[\"SKU\"].isin(catalogue)"," is a single vectorised pass. Reserve database round trips for the cases where the list changes during the run.",[214,3147,3149],{"id":3148},"rules-that-span-columns-rows-or-files","Rules that span columns, rows or files",[10,3151,3152],{},"The checks that catch real errors are usually relational. A quantity is fine and a price is fine, but their product does not match the total the submitter typed. Express these the same way as any other rule:",[219,3154,3156],{"className":254,"code":3155,"language":256,"meta":224,"style":224},"import pandas as pd\n\ndef relational_checks(df, previous_month=None):\n    issues = []\n\n    if {\"Start_Date\", \"End_Date\"} \u003C= set(df.columns):\n        bad = df[\"End_Date\"] \u003C df[\"Start_Date\"]\n        for idx in df.index[bad.fillna(False)]:\n            issues.append({\"row\": int(idx) + 2, \"column\": \"End_Date\",\n                           \"value\": df.loc[idx, \"End_Date\"], \"problem\": \"ends before it starts\"})\n\n    if {\"Quantity\", \"Unit_Price\", \"Line_Total\"} \u003C= set(df.columns):\n        expected = df[\"Quantity\"] * df[\"Unit_Price\"]\n        mismatch = (df[\"Line_Total\"] - expected).abs() > 0.005\n        for idx in df.index[mismatch.fillna(False)]:\n            issues.append({\"row\": int(idx) + 2, \"column\": \"Line_Total\",\n                           \"value\": df.loc[idx, \"Line_Total\"],\n                           \"problem\": f\"does not equal quantity × price ({expected[idx]:.2f})\"})\n\n    if previous_month is not None:\n        shrunk = len(df) \u003C len(previous_month) * 0.5\n        if shrunk:\n            issues.append({\"row\": 0, \"column\": \"(file)\", \"value\": len(df),\n                           \"problem\": f\"row count halved versus last month ({len(previous_month)})\"})\n    return issues\n",[14,3157,3158,3168,3172,3188,3196,3200,3224,3246,3261,3289,3312,3316,3341,3363,3387,3402,3428,3439,3466,3470,3487,3512,3520,3551,3575],{"__ignoreMap":224},[228,3159,3160,3162,3164,3166],{"class":93,"line":230},[228,3161,264],{"class":263},[228,3163,268],{"class":267},[228,3165,271],{"class":263},[228,3167,274],{"class":267},[228,3169,3170],{"class":93,"line":277},[228,3171,281],{"emptyLinePlaceholder":280},[228,3173,3174,3176,3179,3182,3184,3186],{"class":93,"line":284},[228,3175,724],{"class":263},[228,3177,3178],{"class":727}," relational_checks",[228,3180,3181],{"class":267},"(df, previous_month",[228,3183,290],{"class":263},[228,3185,504],{"class":308},[228,3187,738],{"class":267},[228,3189,3190,3192,3194],{"class":93,"line":296},[228,3191,1074],{"class":267},[228,3193,290],{"class":263},[228,3195,1079],{"class":267},[228,3197,3198],{"class":93,"line":356},[228,3199,281],{"emptyLinePlaceholder":280},[228,3201,3202,3204,3206,3209,3211,3214,3217,3219,3221],{"class":93,"line":406},[228,3203,753],{"class":263},[228,3205,1573],{"class":267},[228,3207,3208],{"class":237},"\"Start_Date\"",[228,3210,312],{"class":267},[228,3212,3213],{"class":237},"\"End_Date\"",[228,3215,3216],{"class":267},"} ",[228,3218,1755],{"class":263},[228,3220,3005],{"class":308},[228,3222,3223],{"class":267},"(df.columns):\n",[228,3225,3226,3229,3231,3233,3235,3237,3240,3242,3244],{"class":93,"line":459},[228,3227,3228],{"class":267},"        bad ",[228,3230,290],{"class":263},[228,3232,979],{"class":267},[228,3234,3213],{"class":237},[228,3236,1103],{"class":267},[228,3238,3239],{"class":263},"\u003C",[228,3241,979],{"class":267},[228,3243,3208],{"class":237},[228,3245,608],{"class":267},[228,3247,3248,3250,3252,3254,3257,3259],{"class":93,"line":509},[228,3249,1278],{"class":263},[228,3251,1281],{"class":267},[228,3253,864],{"class":263},[228,3255,3256],{"class":267}," df.index[bad.fillna(",[228,3258,628],{"class":308},[228,3260,1855],{"class":267},[228,3262,3263,3266,3268,3270,3272,3274,3276,3278,3280,3283,3285,3287],{"class":93,"line":557},[228,3264,3265],{"class":267},"            issues.append({",[228,3267,2017],{"class":237},[228,3269,305],{"class":267},[228,3271,1301],{"class":308},[228,3273,1304],{"class":267},[228,3275,1307],{"class":263},[228,3277,1310],{"class":308},[228,3279,312],{"class":267},[228,3281,3282],{"class":237},"\"column\"",[228,3284,305],{"class":267},[228,3286,3213],{"class":237},[228,3288,1373],{"class":267},[228,3290,3291,3294,3297,3299,3301,3304,3306,3309],{"class":93,"line":605},[228,3292,3293],{"class":237},"                           \"value\"",[228,3295,3296],{"class":267},": df.loc[idx, ",[228,3298,3213],{"class":237},[228,3300,1113],{"class":267},[228,3302,3303],{"class":237},"\"problem\"",[228,3305,305],{"class":267},[228,3307,3308],{"class":237},"\"ends before it starts\"",[228,3310,3311],{"class":267},"})\n",[228,3313,3314],{"class":93,"line":611},[228,3315,281],{"emptyLinePlaceholder":280},[228,3317,3318,3320,3322,3324,3326,3328,3330,3333,3335,3337,3339],{"class":93,"line":644},[228,3319,753],{"class":263},[228,3321,1573],{"class":267},[228,3323,335],{"class":237},[228,3325,312],{"class":267},[228,3327,345],{"class":237},[228,3329,312],{"class":267},[228,3331,3332],{"class":237},"\"Line_Total\"",[228,3334,3216],{"class":267},[228,3336,1755],{"class":263},[228,3338,3005],{"class":308},[228,3340,3223],{"class":267},[228,3342,3343,3346,3348,3350,3352,3354,3357,3359,3361],{"class":93,"line":874},[228,3344,3345],{"class":267},"        expected ",[228,3347,290],{"class":263},[228,3349,979],{"class":267},[228,3351,335],{"class":237},[228,3353,1103],{"class":267},[228,3355,3356],{"class":263},"*",[228,3358,979],{"class":267},[228,3360,345],{"class":237},[228,3362,608],{"class":267},[228,3364,3365,3368,3370,3373,3375,3377,3379,3382,3384],{"class":93,"line":905},[228,3366,3367],{"class":267},"        mismatch ",[228,3369,290],{"class":263},[228,3371,3372],{"class":267}," (df[",[228,3374,3332],{"class":237},[228,3376,1103],{"class":267},[228,3378,442],{"class":263},[228,3380,3381],{"class":267}," expected).abs() ",[228,3383,1797],{"class":263},[228,3385,3386],{"class":308}," 0.005\n",[228,3388,3389,3391,3393,3395,3398,3400],{"class":93,"line":913},[228,3390,1278],{"class":263},[228,3392,1281],{"class":267},[228,3394,864],{"class":263},[228,3396,3397],{"class":267}," df.index[mismatch.fillna(",[228,3399,628],{"class":308},[228,3401,1855],{"class":267},[228,3403,3404,3406,3408,3410,3412,3414,3416,3418,3420,3422,3424,3426],{"class":93,"line":951},[228,3405,3265],{"class":267},[228,3407,2017],{"class":237},[228,3409,305],{"class":267},[228,3411,1301],{"class":308},[228,3413,1304],{"class":267},[228,3415,1307],{"class":263},[228,3417,1310],{"class":308},[228,3419,312],{"class":267},[228,3421,3282],{"class":237},[228,3423,305],{"class":267},[228,3425,3332],{"class":237},[228,3427,1373],{"class":267},[228,3429,3430,3432,3434,3436],{"class":93,"line":959},[228,3431,3293],{"class":237},[228,3433,3296],{"class":267},[228,3435,3332],{"class":237},[228,3437,3438],{"class":267},"],\n",[228,3440,3441,3444,3446,3448,3451,3453,3456,3459,3461,3464],{"class":93,"line":973},[228,3442,3443],{"class":237},"                           \"problem\"",[228,3445,305],{"class":267},[228,3447,778],{"class":263},[228,3449,3450],{"class":237},"\"does not equal quantity × price (",[228,3452,785],{"class":784},[228,3454,3455],{"class":267},"expected[idx]",[228,3457,3458],{"class":263},":.2f",[228,3460,794],{"class":784},[228,3462,3463],{"class":237},")\"",[228,3465,3311],{"class":267},[228,3467,3468],{"class":93,"line":986},[228,3469,281],{"emptyLinePlaceholder":280},[228,3471,3472,3474,3477,3480,3482,3485],{"class":93,"line":991},[228,3473,753],{"class":263},[228,3475,3476],{"class":267}," previous_month ",[228,3478,3479],{"class":263},"is",[228,3481,2961],{"class":263},[228,3483,3484],{"class":308}," None",[228,3486,3025],{"class":267},[228,3488,3489,3492,3494,3497,3500,3502,3504,3507,3509],{"class":93,"line":1006},[228,3490,3491],{"class":267},"        shrunk ",[228,3493,290],{"class":263},[228,3495,3496],{"class":308}," len",[228,3498,3499],{"class":267},"(df) ",[228,3501,3239],{"class":263},[228,3503,3496],{"class":308},[228,3505,3506],{"class":267},"(previous_month) ",[228,3508,3356],{"class":263},[228,3510,3511],{"class":308}," 0.5\n",[228,3513,3514,3517],{"class":93,"line":1388},[228,3515,3516],{"class":263},"        if",[228,3518,3519],{"class":267}," shrunk:\n",[228,3521,3522,3524,3526,3528,3530,3532,3534,3536,3539,3541,3544,3546,3548],{"class":93,"line":1393},[228,3523,3265],{"class":267},[228,3525,2017],{"class":237},[228,3527,305],{"class":267},[228,3529,67],{"class":308},[228,3531,312],{"class":267},[228,3533,3282],{"class":237},[228,3535,305],{"class":267},[228,3537,3538],{"class":237},"\"(file)\"",[228,3540,312],{"class":267},[228,3542,3543],{"class":237},"\"value\"",[228,3545,305],{"class":267},[228,3547,1940],{"class":308},[228,3549,3550],{"class":267},"(df),\n",[228,3552,3553,3555,3557,3559,3562,3564,3566,3569,3571,3573],{"class":93,"line":1404},[228,3554,3443],{"class":237},[228,3556,305],{"class":267},[228,3558,778],{"class":263},[228,3560,3561],{"class":237},"\"row count halved versus last month (",[228,3563,785],{"class":784},[228,3565,1940],{"class":308},[228,3567,3568],{"class":267},"(previous_month)",[228,3570,794],{"class":784},[228,3572,3463],{"class":237},[228,3574,3311],{"class":267},[228,3576,3577,3579],{"class":93,"line":1417},[228,3578,976],{"class":263},[228,3580,3581],{"class":267}," issues\n",[10,3583,3584],{},"The tolerance of half a cent in the arithmetic check is not laziness — floating-point money almost never matches exactly, and an exact comparison produces a validation report where every row is wrong. The row-count check is a different species again: it says nothing about any single row, and it is the one that catches a truncated export or a filter left on before the file was saved.",[214,3586,3588],{"id":3587},"wire-validation-into-a-scheduled-job","Wire validation into a scheduled job",[10,3590,3591],{},"A validation layer earns its keep when it runs unattended. Give it an exit code, a log line per failure class and a way to alert a human:",[219,3593,3595],{"className":254,"code":3594,"language":256,"meta":224,"style":224},"import logging\nimport sys\n\nlogging.basicConfig(\n    level=logging.INFO,\n    format=\"%(asctime)s %(levelname)s %(message)s\",\n    handlers=[logging.FileHandler(\"validation.log\"), logging.StreamHandler()],\n)\nlog = logging.getLogger(\"validate\")\n\ndef run(path):\n    try:\n        raw = load_and_check_structure(path)\n    except ValueError as exc:\n        log.error(\"structural failure in %s: %s\", path, exc)\n        return 2                                   # nothing usable — page someone\n\n    typed, type_issues = coerce_types(raw)\n    checked, rule_issues = apply_rules(typed)\n    all_issues = type_issues + rule_issues\n    clean, quarantined = split_clean_and_quarantine(checked, all_issues)\n\n    for issue in all_issues[:50]:\n        log.warning(\"row %s %s: %s (value=%r)\",\n                    issue[\"row\"], issue[\"column\"], issue[\"problem\"], issue[\"value\"])\n    log.info(\"%s: %d clean, %d quarantined, %d issue(s)\",\n             path, len(clean), len(quarantined), len(all_issues))\n\n    if len(quarantined) > len(checked) * 0.2:      # more than a fifth rejected\n        log.error(\"rejection rate above threshold — treating as a failed submission\")\n        return 1\n    return 0\n\nif __name__ == \"__main__\":\n    sys.exit(run(\"submitted.xlsx\"))\n",[14,3596,3597,3604,3611,3615,3620,3635,3657,3673,3677,3692,3696,3706,3713,3723,3735,3755,3765,3769,3779,3788,3803,3813,3817,3834,3861,3884,3913,3933,3937,3964,3973,3980,3988,3993,4010],{"__ignoreMap":224},[228,3598,3599,3601],{"class":93,"line":230},[228,3600,264],{"class":263},[228,3602,3603],{"class":267}," logging\n",[228,3605,3606,3608],{"class":93,"line":277},[228,3607,264],{"class":263},[228,3609,3610],{"class":267}," sys\n",[228,3612,3613],{"class":93,"line":284},[228,3614,281],{"emptyLinePlaceholder":280},[228,3616,3617],{"class":93,"line":296},[228,3618,3619],{"class":267},"logging.basicConfig(\n",[228,3621,3622,3625,3627,3630,3633],{"class":93,"line":356},[228,3623,3624],{"class":622},"    level",[228,3626,290],{"class":263},[228,3628,3629],{"class":267},"logging.",[228,3631,3632],{"class":308},"INFO",[228,3634,1373],{"class":267},[228,3636,3637,3640,3642,3644,3647,3650,3653,3655],{"class":93,"line":406},[228,3638,3639],{"class":622},"    format",[228,3641,290],{"class":263},[228,3643,807],{"class":237},[228,3645,3646],{"class":784},"%(asctime)s",[228,3648,3649],{"class":784}," %(levelname)s",[228,3651,3652],{"class":784}," %(message)s",[228,3654,807],{"class":237},[228,3656,1373],{"class":267},[228,3658,3659,3662,3664,3667,3670],{"class":93,"line":459},[228,3660,3661],{"class":622},"    handlers",[228,3663,290],{"class":263},[228,3665,3666],{"class":267},"[logging.FileHandler(",[228,3668,3669],{"class":237},"\"validation.log\"",[228,3671,3672],{"class":267},"), logging.StreamHandler()],\n",[228,3674,3675],{"class":93,"line":509},[228,3676,641],{"class":267},[228,3678,3679,3682,3684,3687,3690],{"class":93,"line":557},[228,3680,3681],{"class":267},"log ",[228,3683,290],{"class":263},[228,3685,3686],{"class":267}," logging.getLogger(",[228,3688,3689],{"class":237},"\"validate\"",[228,3691,641],{"class":267},[228,3693,3694],{"class":93,"line":605},[228,3695,281],{"emptyLinePlaceholder":280},[228,3697,3698,3700,3703],{"class":93,"line":611},[228,3699,724],{"class":263},[228,3701,3702],{"class":727}," run",[228,3704,3705],{"class":267},"(path):\n",[228,3707,3708,3711],{"class":93,"line":644},[228,3709,3710],{"class":263},"    try",[228,3712,3025],{"class":267},[228,3714,3715,3718,3720],{"class":93,"line":874},[228,3716,3717],{"class":267},"        raw ",[228,3719,290],{"class":263},[228,3721,3722],{"class":267}," load_and_check_structure(path)\n",[228,3724,3725,3728,3730,3733],{"class":93,"line":905},[228,3726,3727],{"class":263},"    except",[228,3729,773],{"class":308},[228,3731,3732],{"class":263}," as",[228,3734,3062],{"class":267},[228,3736,3737,3740,3743,3746,3748,3750,3752],{"class":93,"line":913},[228,3738,3739],{"class":267},"        log.error(",[228,3741,3742],{"class":237},"\"structural failure in ",[228,3744,3745],{"class":784},"%s",[228,3747,305],{"class":237},[228,3749,3745],{"class":784},[228,3751,807],{"class":237},[228,3753,3754],{"class":267},", path, exc)\n",[228,3756,3757,3760,3762],{"class":93,"line":951},[228,3758,3759],{"class":263},"        return",[228,3761,1310],{"class":308},[228,3763,3764],{"class":1316},"                                   # nothing usable — page someone\n",[228,3766,3767],{"class":93,"line":959},[228,3768,281],{"emptyLinePlaceholder":280},[228,3770,3771,3774,3776],{"class":93,"line":973},[228,3772,3773],{"class":267},"    typed, type_issues ",[228,3775,290],{"class":263},[228,3777,3778],{"class":267}," coerce_types(raw)\n",[228,3780,3781,3784,3786],{"class":93,"line":986},[228,3782,3783],{"class":267},"    checked, rule_issues ",[228,3785,290],{"class":263},[228,3787,1924],{"class":267},[228,3789,3790,3793,3795,3798,3800],{"class":93,"line":991},[228,3791,3792],{"class":267},"    all_issues ",[228,3794,290],{"class":263},[228,3796,3797],{"class":267}," type_issues ",[228,3799,1307],{"class":263},[228,3801,3802],{"class":267}," rule_issues\n",[228,3804,3805,3808,3810],{"class":93,"line":1006},[228,3806,3807],{"class":267},"    clean, quarantined ",[228,3809,290],{"class":263},[228,3811,3812],{"class":267}," split_clean_and_quarantine(checked, all_issues)\n",[228,3814,3815],{"class":93,"line":1388},[228,3816,281],{"emptyLinePlaceholder":280},[228,3818,3819,3821,3823,3825,3828,3831],{"class":93,"line":1393},[228,3820,1215],{"class":263},[228,3822,1409],{"class":267},[228,3824,864],{"class":263},[228,3826,3827],{"class":267}," all_issues[:",[228,3829,3830],{"class":308},"50",[228,3832,3833],{"class":267},"]:\n",[228,3835,3836,3839,3842,3844,3847,3849,3851,3854,3857,3859],{"class":93,"line":1404},[228,3837,3838],{"class":267},"        log.warning(",[228,3840,3841],{"class":237},"\"row ",[228,3843,3745],{"class":784},[228,3845,3846],{"class":784}," %s",[228,3848,305],{"class":237},[228,3850,3745],{"class":784},[228,3852,3853],{"class":237}," (value=",[228,3855,3856],{"class":784},"%r",[228,3858,3463],{"class":237},[228,3860,1373],{"class":267},[228,3862,3863,3866,3868,3871,3873,3875,3877,3879,3881],{"class":93,"line":1417},[228,3864,3865],{"class":267},"                    issue[",[228,3867,2017],{"class":237},[228,3869,3870],{"class":267},"], issue[",[228,3872,3282],{"class":237},[228,3874,3870],{"class":267},[228,3876,3303],{"class":237},[228,3878,3870],{"class":267},[228,3880,3543],{"class":237},[228,3882,3883],{"class":267},"])\n",[228,3885,3886,3889,3891,3893,3895,3898,3901,3903,3906,3908,3911],{"class":93,"line":1890},[228,3887,3888],{"class":267},"    log.info(",[228,3890,807],{"class":237},[228,3892,3745],{"class":784},[228,3894,305],{"class":237},[228,3896,3897],{"class":784},"%d",[228,3899,3900],{"class":237}," clean, ",[228,3902,3897],{"class":784},[228,3904,3905],{"class":237}," quarantined, ",[228,3907,3897],{"class":784},[228,3909,3910],{"class":237}," issue(s)\"",[228,3912,1373],{"class":267},[228,3914,3915,3918,3920,3923,3925,3928,3930],{"class":93,"line":1898},[228,3916,3917],{"class":267},"             path, ",[228,3919,1940],{"class":308},[228,3921,3922],{"class":267},"(clean), ",[228,3924,1940],{"class":308},[228,3926,3927],{"class":267},"(quarantined), ",[228,3929,1940],{"class":308},[228,3931,3932],{"class":267},"(all_issues))\n",[228,3934,3935],{"class":93,"line":1903},[228,3936,281],{"emptyLinePlaceholder":280},[228,3938,3939,3941,3943,3946,3948,3950,3953,3955,3958,3961],{"class":93,"line":1911},[228,3940,753],{"class":263},[228,3942,3496],{"class":308},[228,3944,3945],{"class":267},"(quarantined) ",[228,3947,1797],{"class":263},[228,3949,3496],{"class":308},[228,3951,3952],{"class":267},"(checked) ",[228,3954,3356],{"class":263},[228,3956,3957],{"class":308}," 0.2",[228,3959,3960],{"class":267},":      ",[228,3962,3963],{"class":1316},"# more than a fifth rejected\n",[228,3965,3966,3968,3971],{"class":93,"line":1916},[228,3967,3739],{"class":267},[228,3969,3970],{"class":237},"\"rejection rate above threshold — treating as a failed submission\"",[228,3972,641],{"class":267},[228,3974,3975,3977],{"class":93,"line":1927},[228,3976,3759],{"class":263},[228,3978,3979],{"class":308}," 1\n",[228,3981,3983,3985],{"class":93,"line":3982},32,[228,3984,976],{"class":263},[228,3986,3987],{"class":308}," 0\n",[228,3989,3991],{"class":93,"line":3990},33,[228,3992,281],{"emptyLinePlaceholder":280},[228,3994,3996,3999,4002,4005,4008],{"class":93,"line":3995},34,[228,3997,3998],{"class":263},"if",[228,4000,4001],{"class":308}," __name__",[228,4003,4004],{"class":263}," ==",[228,4006,4007],{"class":237}," \"__main__\"",[228,4009,3025],{"class":267},[228,4011,4013,4016,4018],{"class":93,"line":4012},35,[228,4014,4015],{"class":267},"    sys.exit(run(",[228,4017,617],{"class":237},[228,4019,4020],{"class":267},"))\n",[10,4022,4023,4024,4026,4027,4030,4031,4033,4034,4038,4039,4043],{},"Three exit codes carry the whole story: ",[14,4025,67],{}," for a good run, ",[14,4028,4029],{},"1"," for a submission bad enough to need a human, and ",[14,4032,1476],{}," for a file that could not be read at all. A scheduler can act on each differently, and the ",[40,4035,4037],{"href":4036},"\u002Fautomating-reporting-workflows\u002Fscheduling-python-excel-scripts-with-cron\u002F","scheduled reporting"," and ",[40,4040,4042],{"href":4041},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002F","error alerting"," guides show how to turn those codes into retries and emails.",[214,4045,4047],{"id":4046},"test-the-rules-not-just-the-data","Test the rules, not just the data",[10,4049,4050],{},"Validation code is the part of a pipeline nobody exercises until something goes wrong, which is precisely why it deserves tests. Each rule gets a row that should pass and a row that should fail:",[219,4052,4054],{"className":254,"code":4053,"language":256,"meta":224,"style":224},"import pandas as pd\nimport pytest\n\ndef issues_for(row):\n    df = pd.DataFrame([row])\n    typed, type_issues = coerce_types(df)\n    _, rule_issues = apply_rules(typed)\n    return [i[\"problem\"] for i in type_issues + rule_issues]\n\nGOOD = {\"Order_ID\": 1, \"Region\": \"North\", \"Order_Date\": \"2026-01-14\",\n        \"Quantity\": 3, \"Unit_Price\": 10.0}\n\ndef test_clean_row_has_no_issues():\n    assert issues_for(GOOD) == []\n\n@pytest.mark.parametrize(\"field,value,expected\", [\n    (\"Quantity\", -1, \"quantity must be greater than zero\"),\n    (\"Region\", \"Atlantis\", \"region is not on the approved list\"),\n    (\"Unit_Price\", None, \"unit price is missing\"),\n    (\"Order_Date\", \"2026-02-30\", \"could not be read as a date\"),\n])\ndef test_each_rule_fires(field, value, expected):\n    assert expected in issues_for({**GOOD, field: value})\n",[14,4055,4056,4066,4073,4077,4087,4096,4104,4113,4138,4142,4175,4195,4199,4209,4225,4229,4242,4261,4278,4294,4311,4315,4325],{"__ignoreMap":224},[228,4057,4058,4060,4062,4064],{"class":93,"line":230},[228,4059,264],{"class":263},[228,4061,268],{"class":267},[228,4063,271],{"class":263},[228,4065,274],{"class":267},[228,4067,4068,4070],{"class":93,"line":277},[228,4069,264],{"class":263},[228,4071,4072],{"class":267}," pytest\n",[228,4074,4075],{"class":93,"line":284},[228,4076,281],{"emptyLinePlaceholder":280},[228,4078,4079,4081,4084],{"class":93,"line":296},[228,4080,724],{"class":263},[228,4082,4083],{"class":727}," issues_for",[228,4085,4086],{"class":267},"(row):\n",[228,4088,4089,4091,4093],{"class":93,"line":356},[228,4090,818],{"class":267},[228,4092,290],{"class":263},[228,4094,4095],{"class":267}," pd.DataFrame([row])\n",[228,4097,4098,4100,4102],{"class":93,"line":406},[228,4099,3773],{"class":267},[228,4101,290],{"class":263},[228,4103,1401],{"class":267},[228,4105,4106,4109,4111],{"class":93,"line":459},[228,4107,4108],{"class":267},"    _, rule_issues ",[228,4110,290],{"class":263},[228,4112,1924],{"class":267},[228,4114,4115,4117,4120,4122,4124,4126,4129,4131,4133,4135],{"class":93,"line":509},[228,4116,976],{"class":263},[228,4118,4119],{"class":267}," [i[",[228,4121,3303],{"class":237},[228,4123,1103],{"class":267},[228,4125,858],{"class":263},[228,4127,4128],{"class":267}," i ",[228,4130,864],{"class":263},[228,4132,3797],{"class":267},[228,4134,1307],{"class":263},[228,4136,4137],{"class":267}," rule_issues]\n",[228,4139,4140],{"class":93,"line":557},[228,4141,281],{"emptyLinePlaceholder":280},[228,4143,4144,4147,4149,4151,4153,4155,4157,4159,4161,4163,4165,4167,4169,4171,4173],{"class":93,"line":605},[228,4145,4146],{"class":308},"GOOD",[228,4148,692],{"class":263},[228,4150,1573],{"class":267},[228,4152,302],{"class":237},[228,4154,305],{"class":267},[228,4156,4029],{"class":308},[228,4158,312],{"class":267},[228,4160,315],{"class":237},[228,4162,305],{"class":267},[228,4164,320],{"class":237},[228,4166,312],{"class":267},[228,4168,325],{"class":237},[228,4170,305],{"class":267},[228,4172,330],{"class":237},[228,4174,1373],{"class":267},[228,4176,4177,4180,4182,4184,4186,4188,4190,4193],{"class":93,"line":611},[228,4178,4179],{"class":237},"        \"Quantity\"",[228,4181,305],{"class":267},[228,4183,543],{"class":308},[228,4185,312],{"class":267},[228,4187,345],{"class":237},[228,4189,305],{"class":267},[228,4191,4192],{"class":308},"10.0",[228,4194,1592],{"class":267},[228,4196,4197],{"class":93,"line":644},[228,4198,281],{"emptyLinePlaceholder":280},[228,4200,4201,4203,4206],{"class":93,"line":874},[228,4202,724],{"class":263},[228,4204,4205],{"class":727}," test_clean_row_has_no_issues",[228,4207,4208],{"class":267},"():\n",[228,4210,4211,4214,4217,4219,4221,4223],{"class":93,"line":905},[228,4212,4213],{"class":263},"    assert",[228,4215,4216],{"class":267}," issues_for(",[228,4218,4146],{"class":308},[228,4220,1695],{"class":267},[228,4222,1357],{"class":263},[228,4224,1079],{"class":267},[228,4226,4227],{"class":93,"line":913},[228,4228,281],{"emptyLinePlaceholder":280},[228,4230,4231,4234,4236,4239],{"class":93,"line":951},[228,4232,4233],{"class":727},"@pytest.mark.parametrize",[228,4235,650],{"class":267},[228,4237,4238],{"class":237},"\"field,value,expected\"",[228,4240,4241],{"class":267},", [\n",[228,4243,4244,4247,4249,4251,4253,4255,4257,4259],{"class":93,"line":959},[228,4245,4246],{"class":267},"    (",[228,4248,335],{"class":237},[228,4250,312],{"class":267},[228,4252,442],{"class":263},[228,4254,4029],{"class":308},[228,4256,312],{"class":267},[228,4258,1763],{"class":237},[228,4260,1672],{"class":267},[228,4262,4263,4265,4267,4269,4272,4274,4276],{"class":93,"line":973},[228,4264,4246],{"class":267},[228,4266,315],{"class":237},[228,4268,312],{"class":267},[228,4270,4271],{"class":237},"\"Atlantis\"",[228,4273,312],{"class":267},[228,4275,1738],{"class":237},[228,4277,1672],{"class":267},[228,4279,4280,4282,4284,4286,4288,4290,4292],{"class":93,"line":986},[228,4281,4246],{"class":267},[228,4283,345],{"class":237},[228,4285,312],{"class":267},[228,4287,504],{"class":308},[228,4289,312],{"class":267},[228,4291,1780],{"class":237},[228,4293,1672],{"class":267},[228,4295,4296,4298,4300,4302,4304,4306,4309],{"class":93,"line":991},[228,4297,4246],{"class":267},[228,4299,325],{"class":237},[228,4301,312],{"class":267},[228,4303,486],{"class":237},[228,4305,312],{"class":267},[228,4307,4308],{"class":237},"\"could not be read as a date\"",[228,4310,1672],{"class":267},[228,4312,4313],{"class":93,"line":1006},[228,4314,3883],{"class":267},[228,4316,4317,4319,4322],{"class":93,"line":1388},[228,4318,724],{"class":263},[228,4320,4321],{"class":727}," test_each_rule_fires",[228,4323,4324],{"class":267},"(field, value, expected):\n",[228,4326,4327,4329,4332,4334,4337,4340,4342],{"class":93,"line":1393},[228,4328,4213],{"class":263},[228,4330,4331],{"class":267}," expected ",[228,4333,864],{"class":263},[228,4335,4336],{"class":267}," issues_for({",[228,4338,4339],{"class":263},"**",[228,4341,4146],{"class":308},[228,4343,4344],{"class":267},", field: value})\n",[10,4346,1426,4347,4350],{},[14,4348,4349],{},"test_clean_row_has_no_issues"," case is the one that pays for itself. Rules tend to accumulate, and a new rule that is slightly too strict shows up here immediately rather than in a Monday morning report where 80% of rows have been quarantined.",[46,4352,55,4357,55,4360,55,4363,55,4365,55,4368,55,4370,55,4374,55,4379,55,4383,55,4386,55,4388,55,4393,55,4397,55,4400,55,4402,55,4404,55,4409,55,4413],{"viewBox":4353,"role":49,"ariaLabelledBy":4354,"xmlns":53,"style":54},"0 0 760 218",[4355,4356],"vd-sev-t","vd-sev-d",[57,4358,4359],{"id":4355},"Three severities and what each one should do",[61,4361,4362],{"id":4356},"A ladder of three responses. A structural failure such as a missing column stops the job with exit code 2. A high rejection rate returns exit code 1 for a human to look at. Individual bad rows are quarantined and reported while the job continues with exit code 0.",[65,4364],{"x":67,"y":67,"width":68,"height":106,"fill":70},[65,4366],{"x":1461,"y":1461,"width":4367,"height":133,"rx":77,"fill":168,"stroke":143,"style":98},"700",[65,4369],{"x":1461,"y":1461,"width":1465,"height":133,"rx":340,"fill":147},[81,4371,4373],{"x":133,"y":107,"style":4372},"font-size:13px;font-weight:700;fill:var(--danger,#dc2626)","structural — missing sheet or column",[81,4375,4378],{"x":133,"y":4376,"style":4377},"66","font-size:11.5px;fill:var(--muted,#5b6780)","nothing downstream can be trusted · stop the job · exit 2",[81,4380,1476],{"x":4367,"y":4381,"style":4382},"58","font-size:20px;font-weight:700;fill:var(--danger,#dc2626);text-anchor:end",[65,4384],{"x":1461,"y":4385,"width":4367,"height":133,"rx":77,"fill":183,"stroke":184,"style":98},"86",[65,4387],{"x":1461,"y":4385,"width":1465,"height":133,"rx":340,"fill":204},[81,4389,4392],{"x":133,"y":4390,"style":4391},"108","font-size:13px;font-weight:700;fill:var(--gold,#b4740a)","systemic — a fifth of rows rejected",[81,4394,4396],{"x":133,"y":4395,"style":4377},"128","the submission itself is wrong · alert a person · exit 1",[81,4398,4029],{"x":4367,"y":1448,"style":4399},"font-size:20px;font-weight:700;fill:var(--gold,#b4740a);text-anchor:end",[65,4401],{"x":1461,"y":1497,"width":4367,"height":133,"rx":77,"fill":110,"stroke":111,"style":98},[65,4403],{"x":1461,"y":1497,"width":1465,"height":133,"rx":340,"fill":138},[81,4405,4408],{"x":133,"y":4406,"style":4407},"170","font-size:13px;font-weight:700;fill:var(--teal,#0f9488)","row-level — a handful of bad values",[81,4410,4412],{"x":133,"y":4411,"style":4377},"190","quarantine, report, deliver the rest · exit 0",[81,4414,67],{"x":4367,"y":180,"style":4415},"font-size:20px;font-weight:700;fill:var(--teal,#0f9488);text-anchor:end",[10,4417,4418],{},"Deciding the severity of each rule up front is what stops a validation layer from being either an alarm nobody trusts or a filter that silently drops half the data.",[214,4420,4422],{"id":4421},"common-errors-and-fixes","Common errors and fixes",[2745,4424,4425,4438],{},[2748,4426,4427],{},[2751,4428,4429,4432,4435],{},[2754,4430,4431],{},"Symptom",[2754,4433,4434],{},"Cause",[2754,4436,4437],{},"Fix",[2767,4439,4440,4451,4466,4477,4493,4504,4515],{},[2751,4441,4442,4445,4448],{},[2772,4443,4444],{},"Every row flagged as an invalid region",[2772,4446,4447],{},"Reference list failed to load and became empty",[2772,4449,4450],{},"Raise when the reference list is empty",[2751,4452,4453,4456,4459],{},[2772,4454,4455],{},"Dates silently wrong by month\u002Fday",[2772,4457,4458],{},"Mixed European and ISO dates in one column",[2772,4460,4461,4462,4465],{},"Read raw, then ",[14,4463,4464],{},"pd.to_datetime(..., format=\"mixed\")"," and check the failures",[2751,4467,4468,4471,4474],{},[2772,4469,4470],{},"Valid rows rejected for whitespace",[2772,4472,4473],{},"Compared before normalising",[2772,4475,4476],{},"Strip and case-normalise, then compare",[2751,4478,4479,4482,4487],{},[2772,4480,4481],{},"Numbers read as text",[2772,4483,4484,4485],{},"The column contained one ",[14,4486,591],{},[2772,4488,4489,4490,4492],{},"Read with ",[14,4491,1017],{},", coerce explicitly",[2751,4494,4495,4498,4501],{},[2772,4496,4497],{},"Arithmetic check fails on every row",[2772,4499,4500],{},"Exact float comparison",[2772,4502,4503],{},"Compare with a tolerance",[2751,4505,4506,4509,4512],{},[2772,4507,4508],{},"Report full of blank-cell issues",[2772,4510,4511],{},"Missing data conflated with bad data",[2772,4513,4514],{},"Only flag values that were present and unparsable",[2751,4516,4517,4520,4523],{},[2772,4518,4519],{},"Duplicate check misses cases",[2772,4521,4522],{},"Key column has trailing spaces or mixed case",[2772,4524,4525,4526],{},"Normalise the key before ",[14,4527,4528],{},"duplicated()",[214,4530,4532],{"id":4531},"key-takeaways","Key takeaways",[4534,4535,4536,4543,4549,4552,4555,4558],"ul",{},[4537,4538,4539,4540,4542],"li",{},"Read submissions with ",[14,4541,1017],{}," so type problems stay visible instead of being guessed away.",[4537,4544,4545,4546,4548],{},"Coerce with ",[14,4547,1031],{}," and record which values failed — coercion and validation are one step.",[4537,4550,4551],{},"Separate \"was blank\" from \"was unreadable\", or your reports fill with noise.",[4537,4553,4554],{},"Express business rules as mask-and-message pairs so the list is easy to extend and easy to audit.",[4537,4556,4557],{},"Quarantine bad rows and deliver the good ones; reserve hard failures for structural problems.",[4537,4559,4560,4561,4563],{},"Return the workbook with ",[14,4562,2424],{}," rules so the next submission has fewer problems to find.",[214,4565,4567],{"id":4566},"frequently-asked-questions","Frequently asked questions",[10,4569,4570,4573,4574,4576],{},[31,4571,4572],{},"What is the difference between validating in pandas and in openpyxl?","\npandas validates data that already exists, so your job can reject a bad file. openpyxl ",[14,4575,2424],{}," writes rules into the workbook so the next person typing into it is constrained at entry time. Production workflows usually need both.",[10,4578,4579,4582],{},[31,4580,4581],{},"Where should validation run in a reporting pipeline?","\nImmediately after reading the source and before any transformation. Failing at the door keeps bad values out of joins and aggregations, where they are far harder to trace.",[10,4584,4585,4588],{},[31,4586,4587],{},"Should a validation failure stop the whole job?","\nStop on structural failures such as a missing column or an unparsable date column. For row-level problems it is usually better to quarantine the bad rows, process the rest, and report what was dropped.",[10,4590,4591,4594,4595,4597],{},[31,4592,4593],{},"Does openpyxl enforce its own validation rules when writing?","\nNo. ",[14,4596,2424],{}," is metadata for Excel's user interface. openpyxl will happily write a value that breaks the rule, so check values in Python as well.",[10,4599,4600,4603],{},[31,4601,4602],{},"How do I tell users which rows failed?","\nWrite the failures to their own sheet in the delivered workbook, with the source row number, the column, the offending value and a plain-language reason.",[214,4605,4607],{"id":4606},"related","Related",[10,4609,4610],{},"Up to the parent guide:",[4534,4612,4613],{},[4537,4614,4615,4617],{},[40,4616,43],{"href":42}," — the wider ingest, clean, transform and export pipeline.",[10,4619,4620],{},"Go deeper here:",[4534,4622,4623,4629,4636,4643,4649,4656],{},[4537,4624,4625,4628],{},[40,4626,4627],{"href":2737},"Add Dropdown Data Validation to Excel with openpyxl"," — list rules, named ranges and error messages.",[4537,4630,4631,4635],{},[40,4632,4634],{"href":4633},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fvalidate-excel-columns-before-import-with-pandas\u002F","Validate Excel Columns Before Import with pandas"," — the structural gate in detail.",[4537,4637,4638,4642],{},[40,4639,4641],{"href":4640},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fcheck-excel-data-types-with-pandas\u002F","Check Excel Data Types with pandas"," — dtypes, coercion and mixed columns.",[4537,4644,4645,4648],{},[40,4646,4647],{"href":2408},"Highlight Invalid Cells in Excel with Python"," — mark the exact cells that failed.",[4537,4650,4651,4655],{},[40,4652,4654],{"href":4653},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Ffind-duplicate-rows-in-excel-with-python\u002F","Find Duplicate Rows in Excel with Python"," — the cross-row check Excel cannot do.",[4537,4657,4658,4662],{},[40,4659,4661],{"href":4660},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fcompare-two-excel-files-for-differences-with-python\u002F","Compare Two Excel Files for Differences with Python"," — validate one version against another.",[10,4664,4665],{},"Related areas:",[4534,4667,4668,4674,4681],{},[4537,4669,4670,4673],{},[40,4671,4672],{"href":1966},"Cleaning Excel Data with pandas"," — the fixing counterpart to these checks.",[4537,4675,4676,4680],{},[40,4677,4679],{"href":4678},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002F","Handling Missing Data in Excel Reports"," — deciding what a blank means.",[4537,4682,4683,4687],{},[40,4684,4686],{"href":4685},"\u002Fadvanced-data-transformation-and-cleaning\u002Fapplying-conditional-formatting-with-openpyxl\u002F","Applying Conditional Formatting with openpyxl"," — making problems visible in the delivered file.",[4689,4690,4691],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}",{"title":224,"searchDepth":277,"depth":277,"links":4693},[4694,4695,4696,4697,4698,4699,4700,4701,4702,4703,4704,4705,4706,4707,4708,4709,4710],{"id":216,"depth":277,"text":217},{"id":247,"depth":277,"text":248},{"id":661,"depth":277,"text":662},{"id":1024,"depth":277,"text":1025},{"id":1540,"depth":277,"text":1541},{"id":1971,"depth":277,"text":1972},{"id":2256,"depth":277,"text":2257},{"id":2413,"depth":277,"text":2414},{"id":2742,"depth":277,"text":2743},{"id":2858,"depth":277,"text":2859},{"id":3148,"depth":277,"text":3149},{"id":3587,"depth":277,"text":3588},{"id":4046,"depth":277,"text":4047},{"id":4421,"depth":277,"text":4422},{"id":4531,"depth":277,"text":4532},{"id":4566,"depth":277,"text":4567},{"id":4606,"depth":277,"text":4607},"2026-08-01","Two kinds of spreadsheet validation with Python — pandas checks that reject bad data before it reaches a report, and openpyxl rules that stop bad data being typed in the first place.","md",[4715,4717,4719,4721,4723],{"q":4572,"a":4716},"pandas validates data that already exists, so your job can reject a bad file. openpyxl DataValidation writes rules into the workbook so the next person typing into it is constrained at entry time. Production workflows usually need both.",{"q":4581,"a":4718},"Immediately after reading the source and before any transformation. Failing at the door keeps bad values out of joins and aggregations, where they are far harder to trace.",{"q":4587,"a":4720},"Stop on structural failures such as a missing column or an unparsable date column. For row-level problems it is usually better to quarantine the bad rows, process the rest, and report what was dropped.",{"q":4593,"a":4722},"No. DataValidation is metadata for Excel's user interface. openpyxl will happily write a value that breaks the rule, so check values in Python as well.",{"q":4602,"a":4724},"Write the failures to their own sheet in the delivered workbook, with the source row number, the column, the offending value and a plain-language reason.",{"breadcrumb":4726},[4727,4729,4730],{"name":4728,"item":1036},"Home",{"name":43,"item":42},{"name":5,"item":4731},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002F","\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python",{"title":5,"description":4734},"Build a validation layer for Excel workflows: schema and type checks in pandas, business rules, an error report sheet, and openpyxl DataValidation dropdowns and limits.","validating-excel-data-with-python","advanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Findex","guide","7KDFw4fZtwGxyOki8ZeRJbGMS8o5Mf02X24KLDNcDGI",[4740,4744],{"title":4741,"path":4742,"stem":4743,"children":-1},"Merge Two Excel Files on a Common Column in Python","\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fmerge-two-excel-files-on-common-column-python","advanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fmerge-two-excel-files-on-common-column-python\u002Findex",{"title":4627,"path":4745,"stem":4746,"children":-1},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fadd-dropdown-data-validation-to-excel-with-openpyxl","advanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fadd-dropdown-data-validation-to-excel-with-openpyxl\u002Findex",1785584462252]