[{"data":1,"prerenderedAt":2268},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fremove-subtotal-and-total-rows-from-an-excel-export":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fremove-subtotal-and-total-rows-from-an-excel-export":2234,"ancestors:\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fremove-subtotal-and-total-rows-from-an-excel-export":2242,"children:\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fremove-subtotal-and-total-rows-from-an-excel-export":2248,"related:\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fremove-subtotal-and-total-rows-from-an-excel-export":2249},{"id":4,"title":5,"body":6,"dateModified":2200,"datePublished":2200,"description":2201,"extension":2202,"faq":2203,"keyword":2212,"kind":2213,"meta":2214,"navigation":366,"path":2222,"seo":2223,"slug":2226,"stem":2227,"topics":2228,"type":2232,"__hash__":2233},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fremove-subtotal-and-total-rows-from-an-excel-export\u002Findex.md","Remove Subtotal and Total Rows from an Excel Export",{"type":7,"value":8,"toc":2186},"minimark",[9,37,302,307,335,338,573,577,580,688,691,695,714,850,873,966,970,973,1041,1049,1053,1063,1431,1437,1441,1444,1688,1691,1783,1787,1790,1969,1975,1979,2087,2091,2102,2106,2112,2116,2122,2128,2134,2140,2144,2182],[10,11,12,13,17,18,22,23,26,27,30,31,36],"p",{},"Reports exported from accounting packages, BI tools and Excel's own ",[14,15,16],"strong",{},"Data → Subtotal"," command interleave\ndetail rows with rollups: two rows for the North region, then a ",[19,20,21],"code",{},"North Total",", two rows for South, a\n",[19,24,25],{},"South Total",", and a ",[19,28,29],{},"Grand Total"," at the bottom. Read that sheet with pandas and sum the revenue column,\nand the answer is three times the truth — every euro is counted once in its detail row, once more in its\nregion's subtotal, and a third time in the grand total. This guide, part of\n",[32,33,35],"a",{"href":34},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002F","Extracting Data from Messy Excel Sheets",",\nremoves those rows reliably and then puts them to work as a free correctness check.",[38,39,47,48,47,52,47,56,47,63,47,70,47,79,47,83,47,89,47,92,47,96,47,100,47,104,47,106,47,110,47,112,47,116,47,118,47,121,47,123,47,126,47,129,47,133,47,135,47,139,47,141,47,144,47,146,47,149,47,152,47,156,47,158,47,160,47,162,47,165,47,167,47,170,47,173,47,177,47,181,47,184,47,186,47,188,47,191,47,198,47,203,47,206,47,210,47,212,47,215,47,217,47,220,47,222,47,225,47,228,47,232,47,234,47,236,47,238,47,241,47,243,47,246,47,249,47,253,47,255,47,257,47,259,47,261,47,264,47,267,47,270,47,273,47,276,47,280,47,283,47,285,47,287,47,290,47,293,47,297],"svg",{"viewBox":40,"role":41,"ariaLabelledBy":42,"xmlns":45,"style":46},"0 0 466 374","img",[43,44],"sub-sheet-t","sub-sheet-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:466px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[49,50,51],"title",{"id":43},"Detail rows interleaved with subtotal and grand total rows",[53,54,55],"desc",{"id":44},"Rows 2 and 3 are North detail rows, row 4 is the North subtotal, rows 5 and 6 are South detail rows, row 7 is the South subtotal, and row 8 is the grand total.",[57,58],"rect",{"x":59,"y":59,"width":60,"height":61,"fill":62},"0","466","374","#ffffff",[64,65,69],"text",{"x":66,"y":67,"style":68},"233","32","font-size:13px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","summing column C counts every value three times over",[57,71],{"x":72,"y":73,"width":74,"height":75,"rx":59,"fill":76,"stroke":77,"style":78},"20","50","36","30","#e7ebef","var(--line,#cdd5e6)","stroke-width:1px",[57,80],{"x":81,"y":73,"width":82,"height":75,"rx":59,"fill":76,"stroke":77,"style":78},"56","100",[64,84,88],{"x":85,"y":86,"style":87},"106","70","font-size:11.5px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","A",[57,90],{"x":91,"y":73,"width":86,"height":75,"rx":59,"fill":76,"stroke":77,"style":78},"156",[64,93,95],{"x":94,"y":86,"style":87},"191","B",[57,97],{"x":98,"y":73,"width":99,"height":75,"rx":59,"fill":76,"stroke":77,"style":78},"226","80",[64,101,103],{"x":102,"y":86,"style":87},"266","C",[57,105],{"x":72,"y":99,"width":74,"height":75,"rx":59,"fill":76,"stroke":77,"style":78},[64,107,109],{"x":108,"y":82,"style":87},"38","1",[57,111],{"x":81,"y":99,"width":82,"height":75,"rx":59,"fill":76,"stroke":77,"style":78},[64,113,115],{"x":85,"y":82,"style":114},"font-size:11.5px;font-weight:400;fill:var(--muted,#5b6780);text-anchor:middle","Region",[57,117],{"x":91,"y":99,"width":86,"height":75,"rx":59,"fill":76,"stroke":77,"style":78},[64,119,120],{"x":94,"y":82,"style":114},"Rep",[57,122],{"x":98,"y":99,"width":99,"height":75,"rx":59,"fill":76,"stroke":77,"style":78},[64,124,125],{"x":102,"y":82,"style":114},"Revenue",[57,127],{"x":72,"y":128,"width":74,"height":75,"rx":59,"fill":76,"stroke":77,"style":78},"110",[64,130,132],{"x":108,"y":131,"style":87},"130","2",[57,134],{"x":81,"y":128,"width":82,"height":75,"rx":59,"fill":62,"stroke":77,"style":78},[64,136,138],{"x":85,"y":131,"style":137},"font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","North",[57,140],{"x":91,"y":128,"width":86,"height":75,"rx":59,"fill":62,"stroke":77,"style":78},[64,142,143],{"x":94,"y":131,"style":137},"Ana",[57,145],{"x":98,"y":128,"width":99,"height":75,"rx":59,"fill":62,"stroke":77,"style":78},[64,147,148],{"x":102,"y":131,"style":137},"1200",[57,150],{"x":72,"y":151,"width":74,"height":75,"rx":59,"fill":76,"stroke":77,"style":78},"140",[64,153,155],{"x":108,"y":154,"style":87},"160","3",[57,157],{"x":81,"y":151,"width":82,"height":75,"rx":59,"fill":62,"stroke":77,"style":78},[64,159,138],{"x":85,"y":154,"style":137},[57,161],{"x":91,"y":151,"width":86,"height":75,"rx":59,"fill":62,"stroke":77,"style":78},[64,163,164],{"x":94,"y":154,"style":137},"Ben",[57,166],{"x":98,"y":151,"width":99,"height":75,"rx":59,"fill":62,"stroke":77,"style":78},[64,168,169],{"x":102,"y":154,"style":137},"800",[57,171],{"x":72,"y":172,"width":74,"height":75,"rx":59,"fill":76,"stroke":77,"style":78},"170",[64,174,176],{"x":108,"y":175,"style":87},"190","4",[57,178],{"x":81,"y":172,"width":82,"height":75,"rx":59,"fill":179,"stroke":180,"style":78},"#fdefd8","var(--gold,#b4740a)",[64,182,21],{"x":85,"y":175,"style":183},"font-size:11.5px;font-weight:600;fill:var(--gold-ink,#7a4e06);text-anchor:middle",[57,185],{"x":91,"y":172,"width":86,"height":75,"rx":59,"fill":179,"stroke":180,"style":78},[57,187],{"x":98,"y":172,"width":99,"height":75,"rx":59,"fill":179,"stroke":180,"style":78},[64,189,190],{"x":102,"y":175,"style":183},"2000",[57,192],{"x":193,"y":194,"width":195,"height":196,"rx":197,"fill":179,"stroke":180,"style":78},"326","172","120","26","8",[64,199,202],{"x":200,"y":201,"style":183},"386","189","subtotal",[57,204],{"x":72,"y":205,"width":74,"height":75,"rx":59,"fill":76,"stroke":77,"style":78},"200",[64,207,209],{"x":108,"y":208,"style":87},"220","5",[57,211],{"x":81,"y":205,"width":82,"height":75,"rx":59,"fill":62,"stroke":77,"style":78},[64,213,214],{"x":85,"y":208,"style":137},"South",[57,216],{"x":91,"y":205,"width":86,"height":75,"rx":59,"fill":62,"stroke":77,"style":78},[64,218,219],{"x":94,"y":208,"style":137},"Cara",[57,221],{"x":98,"y":205,"width":99,"height":75,"rx":59,"fill":62,"stroke":77,"style":78},[64,223,224],{"x":102,"y":208,"style":137},"950",[57,226],{"x":72,"y":227,"width":74,"height":75,"rx":59,"fill":76,"stroke":77,"style":78},"230",[64,229,231],{"x":108,"y":230,"style":87},"250","6",[57,233],{"x":81,"y":227,"width":82,"height":75,"rx":59,"fill":62,"stroke":77,"style":78},[64,235,214],{"x":85,"y":230,"style":137},[57,237],{"x":91,"y":227,"width":86,"height":75,"rx":59,"fill":62,"stroke":77,"style":78},[64,239,240],{"x":94,"y":230,"style":137},"Dev",[57,242],{"x":98,"y":227,"width":99,"height":75,"rx":59,"fill":62,"stroke":77,"style":78},[64,244,245],{"x":102,"y":230,"style":137},"650",[57,247],{"x":72,"y":248,"width":74,"height":75,"rx":59,"fill":76,"stroke":77,"style":78},"260",[64,250,252],{"x":108,"y":251,"style":87},"280","7",[57,254],{"x":81,"y":248,"width":82,"height":75,"rx":59,"fill":179,"stroke":180,"style":78},[64,256,25],{"x":85,"y":251,"style":183},[57,258],{"x":91,"y":248,"width":86,"height":75,"rx":59,"fill":179,"stroke":180,"style":78},[57,260],{"x":98,"y":248,"width":99,"height":75,"rx":59,"fill":179,"stroke":180,"style":78},[64,262,263],{"x":102,"y":251,"style":183},"1600",[57,265],{"x":193,"y":266,"width":195,"height":196,"rx":197,"fill":179,"stroke":180,"style":78},"262",[64,268,202],{"x":200,"y":269,"style":183},"279",[57,271],{"x":72,"y":272,"width":74,"height":75,"rx":59,"fill":76,"stroke":77,"style":78},"290",[64,274,197],{"x":108,"y":275,"style":87},"310",[57,277],{"x":81,"y":272,"width":82,"height":75,"rx":59,"fill":278,"stroke":279,"style":78},"#fee8f2","var(--accent,#f43f8f)",[64,281,29],{"x":85,"y":275,"style":282},"font-size:11.5px;font-weight:600;fill:var(--accent-ink,#be185d);text-anchor:middle",[57,284],{"x":91,"y":272,"width":86,"height":75,"rx":59,"fill":278,"stroke":279,"style":78},[57,286],{"x":98,"y":272,"width":99,"height":75,"rx":59,"fill":278,"stroke":279,"style":78},[64,288,289],{"x":102,"y":275,"style":282},"3600",[57,291],{"x":193,"y":292,"width":195,"height":196,"rx":197,"fill":278,"stroke":279,"style":78},"292",[64,294,296],{"x":200,"y":295,"style":282},"309","grand total",[64,298,301],{"x":66,"y":299,"style":300},"352","font-size:12.5px;font-weight:400;fill:var(--muted,#5b6780);text-anchor:middle","naive sum = 10,800; the real total is 3,600",[303,304,306],"h2",{"id":305},"prerequisites","Prerequisites",[308,309,314],"pre",{"className":310,"code":311,"language":312,"meta":313,"style":313},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl\n","bash","",[19,315,316],{"__ignoreMap":313},[317,318,321,325,329,332],"span",{"class":319,"line":320},"line",1,[317,322,324],{"class":323},"sMTad","pip",[317,326,328],{"class":327},"srMev"," install",[317,330,331],{"class":327}," pandas",[317,333,334],{"class":327}," openpyxl\n",[10,336,337],{},"The first file mirrors the diagram, with totals typed as values the way most exports write them:",[308,339,343],{"className":340,"code":341,"language":342,"meta":313,"style":313},"language-python shiki shiki-themes github-light github-dark-high-contrast","from openpyxl import Workbook\n\nwb = Workbook()\nws = wb.active\nws.title = \"Sales\"\nws.append([\"Region\", \"Rep\", \"Revenue\"])\nfor row in [(\"North\", \"Ana\", 1200), (\"North\", \"Ben\", 800), (\"North Total\", None, 2000),\n            (\"South\", \"Cara\", 950), (\"South\", \"Dev\", 650), (\"South Total\", None, 1600),\n            (\"Grand Total\", None, 3600)]:\n    ws.append(list(row))\nwb.save(\"sales_subtotals.xlsx\")\n","python",[19,344,345,361,368,380,391,402,425,484,530,549,561],{"__ignoreMap":313},[317,346,347,351,355,358],{"class":319,"line":320},[317,348,350],{"class":349},"s-kum","from",[317,352,354],{"class":353},"skGVy"," openpyxl ",[317,356,357],{"class":349},"import",[317,359,360],{"class":353}," Workbook\n",[317,362,364],{"class":319,"line":363},2,[317,365,367],{"emptyLinePlaceholder":366},true,"\n",[317,369,371,374,377],{"class":319,"line":370},3,[317,372,373],{"class":353},"wb ",[317,375,376],{"class":349},"=",[317,378,379],{"class":353}," Workbook()\n",[317,381,383,386,388],{"class":319,"line":382},4,[317,384,385],{"class":353},"ws ",[317,387,376],{"class":349},[317,389,390],{"class":353}," wb.active\n",[317,392,394,397,399],{"class":319,"line":393},5,[317,395,396],{"class":353},"ws.title ",[317,398,376],{"class":349},[317,400,401],{"class":327}," \"Sales\"\n",[317,403,405,408,411,414,417,419,422],{"class":319,"line":404},6,[317,406,407],{"class":353},"ws.append([",[317,409,410],{"class":327},"\"Region\"",[317,412,413],{"class":353},", ",[317,415,416],{"class":327},"\"Rep\"",[317,418,413],{"class":353},[317,420,421],{"class":327},"\"Revenue\"",[317,423,424],{"class":353},"])\n",[317,426,428,431,434,437,440,443,445,448,450,453,456,458,460,463,465,467,469,472,474,477,479,481],{"class":319,"line":427},7,[317,429,430],{"class":349},"for",[317,432,433],{"class":353}," row ",[317,435,436],{"class":349},"in",[317,438,439],{"class":353}," [(",[317,441,442],{"class":327},"\"North\"",[317,444,413],{"class":353},[317,446,447],{"class":327},"\"Ana\"",[317,449,413],{"class":353},[317,451,148],{"class":452},"sP0c6",[317,454,455],{"class":353},"), (",[317,457,442],{"class":327},[317,459,413],{"class":353},[317,461,462],{"class":327},"\"Ben\"",[317,464,413],{"class":353},[317,466,169],{"class":452},[317,468,455],{"class":353},[317,470,471],{"class":327},"\"North Total\"",[317,473,413],{"class":353},[317,475,476],{"class":452},"None",[317,478,413],{"class":353},[317,480,190],{"class":452},[317,482,483],{"class":353},"),\n",[317,485,487,490,493,495,498,500,502,504,506,508,511,513,515,517,520,522,524,526,528],{"class":319,"line":486},8,[317,488,489],{"class":353},"            (",[317,491,492],{"class":327},"\"South\"",[317,494,413],{"class":353},[317,496,497],{"class":327},"\"Cara\"",[317,499,413],{"class":353},[317,501,224],{"class":452},[317,503,455],{"class":353},[317,505,492],{"class":327},[317,507,413],{"class":353},[317,509,510],{"class":327},"\"Dev\"",[317,512,413],{"class":353},[317,514,245],{"class":452},[317,516,455],{"class":353},[317,518,519],{"class":327},"\"South Total\"",[317,521,413],{"class":353},[317,523,476],{"class":452},[317,525,413],{"class":353},[317,527,263],{"class":452},[317,529,483],{"class":353},[317,531,533,535,538,540,542,544,546],{"class":319,"line":532},9,[317,534,489],{"class":353},[317,536,537],{"class":327},"\"Grand Total\"",[317,539,413],{"class":353},[317,541,476],{"class":452},[317,543,413],{"class":353},[317,545,289],{"class":452},[317,547,548],{"class":353},")]:\n",[317,550,552,555,558],{"class":319,"line":551},10,[317,553,554],{"class":353},"    ws.append(",[317,556,557],{"class":452},"list",[317,559,560],{"class":353},"(row))\n",[317,562,564,567,570],{"class":319,"line":563},11,[317,565,566],{"class":353},"wb.save(",[317,568,569],{"class":327},"\"sales_subtotals.xlsx\"",[317,571,572],{"class":353},")\n",[303,574,576],{"id":575},"step-1-confirm-the-problem-with-a-quick-sum","Step 1 — Confirm the problem with a quick sum",[10,578,579],{},"Before removing anything, measure it. Comparing the column sum with the value in the grand-total row tells\nyou immediately whether rollups are present and roughly how many layers there are: a sum of three times the\ngrand total means one layer of subtotals plus the grand total, and each extra layer of grouping — region, then\ncountry, then continent — adds another multiple.",[308,581,583],{"className":340,"code":582,"language":342,"meta":313,"style":313},"import pandas as pd\n\ndf = pd.read_excel(\"sales_subtotals.xlsx\", sheet_name=\"Sales\")\ncolumn_sum = df[\"Revenue\"].sum()\ngrand = df.loc[df[\"Region\"] == \"Grand Total\", \"Revenue\"].item()\nprint(column_sum, grand, column_sum \u002F grand)                     # 10800 3600 3.0\n",[19,584,585,598,602,627,642,670],{"__ignoreMap":313},[317,586,587,589,592,595],{"class":319,"line":320},[317,588,357],{"class":349},[317,590,591],{"class":353}," pandas ",[317,593,594],{"class":349},"as",[317,596,597],{"class":353}," pd\n",[317,599,600],{"class":319,"line":363},[317,601,367],{"emptyLinePlaceholder":366},[317,603,604,607,609,612,614,616,620,622,625],{"class":319,"line":370},[317,605,606],{"class":353},"df ",[317,608,376],{"class":349},[317,610,611],{"class":353}," pd.read_excel(",[317,613,569],{"class":327},[317,615,413],{"class":353},[317,617,619],{"class":618},"sa561","sheet_name",[317,621,376],{"class":349},[317,623,624],{"class":327},"\"Sales\"",[317,626,572],{"class":353},[317,628,629,632,634,637,639],{"class":319,"line":382},[317,630,631],{"class":353},"column_sum ",[317,633,376],{"class":349},[317,635,636],{"class":353}," df[",[317,638,421],{"class":327},[317,640,641],{"class":353},"].sum()\n",[317,643,644,647,649,652,654,657,660,663,665,667],{"class":319,"line":393},[317,645,646],{"class":353},"grand ",[317,648,376],{"class":349},[317,650,651],{"class":353}," df.loc[df[",[317,653,410],{"class":327},[317,655,656],{"class":353},"] ",[317,658,659],{"class":349},"==",[317,661,662],{"class":327}," \"Grand Total\"",[317,664,413],{"class":353},[317,666,421],{"class":327},[317,668,669],{"class":353},"].item()\n",[317,671,672,675,678,681,684],{"class":319,"line":404},[317,673,674],{"class":452},"print",[317,676,677],{"class":353},"(column_sum, grand, column_sum ",[317,679,680],{"class":349},"\u002F",[317,682,683],{"class":353}," grand)                     ",[317,685,687],{"class":686},"s-wDw","# 10800 3600 3.0\n",[10,689,690],{},"A ratio that is not a whole number is informative too. It usually means some groups have subtotals and\nothers do not, or that a filter was applied to the detail after the totals were calculated — both reasons to\nlook at the file before trusting any automatic cleanup.",[303,692,694],{"id":693},"step-2-remove-rows-whose-label-says-total","Step 2 — Remove rows whose label says \"total\"",[10,696,697,698,701,702,705,706,709,710,713],{},"Most exports label their rollups, so the simplest filter matches the word in the label column. Use a word\nboundary so that a customer called ",[19,699,700],{},"Totally Tiles Ltd"," survives, and allow ",[19,703,704],{},"Subtotal"," and ",[19,707,708],{},"Sub-total"," as\nwell as ",[19,711,712],{},"Total",".",[308,715,717],{"className":340,"code":716,"language":342,"meta":313,"style":313},"TOTAL_LABEL = r\"\\b(?:sub-?)?total\\b\"\n\nis_total = df[\"Region\"].astype(str).str.contains(TOTAL_LABEL, case=False, regex=True)\ndetail = df[~is_total].reset_index(drop=True)\nprint(detail)\nprint(detail[\"Revenue\"].sum())          # 3600\n",[19,718,719,756,760,804,828,835],{"__ignoreMap":313},[317,720,721,724,727,730,733,736,739,742,745,747,750,753],{"class":319,"line":320},[317,722,723],{"class":452},"TOTAL_LABEL",[317,725,726],{"class":349}," =",[317,728,729],{"class":349}," r",[317,731,732],{"class":327},"\"",[317,734,735],{"class":452},"\\b(?:",[317,737,738],{"class":327},"sub-",[317,740,741],{"class":349},"?",[317,743,744],{"class":452},")",[317,746,741],{"class":349},[317,748,749],{"class":327},"total",[317,751,752],{"class":452},"\\b",[317,754,755],{"class":327},"\"\n",[317,757,758],{"class":319,"line":363},[317,759,367],{"emptyLinePlaceholder":366},[317,761,762,765,767,769,771,774,777,780,782,784,787,789,792,794,797,799,802],{"class":319,"line":370},[317,763,764],{"class":353},"is_total ",[317,766,376],{"class":349},[317,768,636],{"class":353},[317,770,410],{"class":327},[317,772,773],{"class":353},"].astype(",[317,775,776],{"class":452},"str",[317,778,779],{"class":353},").str.contains(",[317,781,723],{"class":452},[317,783,413],{"class":353},[317,785,786],{"class":618},"case",[317,788,376],{"class":349},[317,790,791],{"class":452},"False",[317,793,413],{"class":353},[317,795,796],{"class":618},"regex",[317,798,376],{"class":349},[317,800,801],{"class":452},"True",[317,803,572],{"class":353},[317,805,806,809,811,813,816,819,822,824,826],{"class":319,"line":382},[317,807,808],{"class":353},"detail ",[317,810,376],{"class":349},[317,812,636],{"class":353},[317,814,815],{"class":349},"~",[317,817,818],{"class":353},"is_total].reset_index(",[317,820,821],{"class":618},"drop",[317,823,376],{"class":349},[317,825,801],{"class":452},[317,827,572],{"class":353},[317,829,830,832],{"class":319,"line":393},[317,831,674],{"class":452},[317,833,834],{"class":353},"(detail)\n",[317,836,837,839,842,844,847],{"class":319,"line":404},[317,838,674],{"class":452},[317,840,841],{"class":353},"(detail[",[317,843,421],{"class":327},[317,845,846],{"class":353},"].sum())          ",[317,848,849],{"class":686},"# 3600\n",[10,851,852,855,856,859,860,863,864,866,867,869,870,872],{},[19,853,854],{},"astype(str)"," matters more than it looks. A label column that is otherwise empty on detail rows — common\nwhen a report prints the region once per group — holds ",[19,857,858],{},"NaN",", and ",[19,861,862],{},".str.contains"," on a float ",[19,865,858],{}," returns\n",[19,868,858],{}," rather than ",[19,871,791],{},", which breaks the boolean mask.",[38,874,47,880,47,883,47,886,47,889,47,895,47,901,47,904,47,908,47,912,47,916,47,921,47,926,47,930,47,933,47,936,47,939,47,944,47,949,47,953,47,956,47,959,47,962],{"viewBox":875,"role":41,"ariaLabelledBy":876,"xmlns":45,"style":879},"0 0 760 191",[877,878],"sub-signals-t","sub-signals-d","width:100%;max-width:760px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[49,881,882],{"id":877},"Three signals that identify a rollup row",[53,884,885],{"id":878},"A label containing total, a row where the detail columns such as Rep are empty, and a SUBTOTAL formula in the value column each identify rollup rows, with different strengths.",[57,887],{"x":59,"y":59,"width":888,"height":94,"fill":62},"760",[57,890],{"x":72,"y":72,"width":891,"height":892,"rx":893,"fill":179,"stroke":180,"style":894},"226.7","127","14","stroke-width:2px",[64,896,900],{"x":897,"y":898,"style":899},"133.3","46","font-size:13px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","label says total",[319,902],{"x1":74,"y1":81,"x2":903,"y2":81,"stroke":180,"style":78},"230.7",[64,905,907],{"x":897,"y":906,"style":137},"78","North Total, Grand Total",[64,909,911],{"x":897,"y":910,"style":137},"101","simplest filter",[64,913,915],{"x":897,"y":914,"style":137},"124","misses unlabelled rows",[57,917],{"x":918,"y":72,"width":891,"height":892,"rx":893,"fill":919,"stroke":920,"style":894},"266.7","#f0f4ff","var(--brand,#5b5cf0)",[64,922,925],{"x":923,"y":898,"style":924},"380","font-size:13px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","detail fields empty",[319,927],{"x1":928,"y1":81,"x2":929,"y2":81,"stroke":920,"style":78},"282.7","477.3",[64,931,932],{"x":923,"y":906,"style":137},"Rep is blank",[64,934,935],{"x":923,"y":910,"style":137},"works without labels",[64,937,938],{"x":923,"y":914,"style":137},"needs a mandatory field",[57,940],{"x":941,"y":72,"width":891,"height":892,"rx":893,"fill":942,"stroke":943,"style":894},"513.3","#d9f4f1","var(--teal,#0f9488)",[64,945,948],{"x":946,"y":898,"style":947},"626.7","font-size:13px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","SUBTOTAL formula",[319,950],{"x1":951,"y1":81,"x2":952,"y2":81,"stroke":943,"style":78},"529.3","724",[64,954,955],{"x":946,"y":906,"style":137},"=SUBTOTAL(9, C2:C3)",[64,957,958],{"x":946,"y":910,"style":137},"exact, never guesses",[64,960,961],{"x":946,"y":914,"style":137},"needs openpyxl",[64,963,965],{"x":923,"y":964,"style":300},"177","combine two signals when the export is inconsistent",[303,967,969],{"id":968},"step-3-detect-rollups-by-their-empty-detail-fields","Step 3 — Detect rollups by their empty detail fields",[10,971,972],{},"Some tools do not label totals at all; they print a bold number with nothing next to it. A rollup still has a\ntell: it has no value in the fields that identify a single record. Every real sale has a rep, so a row with\na revenue figure and no rep is not a sale.",[308,974,976],{"className":340,"code":975,"language":342,"meta":313,"style":313},"is_rollup = df[\"Rep\"].isna() & df[\"Revenue\"].notna()\nprint(df[is_rollup])\ndetail_by_shape = df[~is_rollup].reset_index(drop=True)\nprint(detail_by_shape.equals(detail))   # True for this file\n",[19,977,978,1002,1009,1031],{"__ignoreMap":313},[317,979,980,983,985,987,989,992,995,997,999],{"class":319,"line":320},[317,981,982],{"class":353},"is_rollup ",[317,984,376],{"class":349},[317,986,636],{"class":353},[317,988,416],{"class":327},[317,990,991],{"class":353},"].isna() ",[317,993,994],{"class":349},"&",[317,996,636],{"class":353},[317,998,421],{"class":327},[317,1000,1001],{"class":353},"].notna()\n",[317,1003,1004,1006],{"class":319,"line":363},[317,1005,674],{"class":452},[317,1007,1008],{"class":353},"(df[is_rollup])\n",[317,1010,1011,1014,1016,1018,1020,1023,1025,1027,1029],{"class":319,"line":370},[317,1012,1013],{"class":353},"detail_by_shape ",[317,1015,376],{"class":349},[317,1017,636],{"class":353},[317,1019,815],{"class":349},[317,1021,1022],{"class":353},"is_rollup].reset_index(",[317,1024,821],{"class":618},[317,1026,376],{"class":349},[317,1028,801],{"class":452},[317,1030,572],{"class":353},[317,1032,1033,1035,1038],{"class":319,"line":382},[317,1034,674],{"class":452},[317,1036,1037],{"class":353},"(detail_by_shape.equals(detail))   ",[317,1039,1040],{"class":686},"# True for this file\n",[10,1042,1043,1044,1048],{},"Pick the identifying column carefully. It must be mandatory for every genuine detail row — an order ID,\nan invoice number, an employee code. A column that is ",[1045,1046,1047],"em",{},"usually"," filled, such as a comment or a secondary\ncontact, will silently drop real records the day one of them is blank. When no single column is reliable,\nrequire several to be empty at once.",[303,1050,1052],{"id":1051},"step-4-find-subtotal-formulas-with-openpyxl","Step 4 — Find SUBTOTAL formulas with openpyxl",[10,1054,1055,1056,1058,1059,1062],{},"Excel's ",[14,1057,16],{}," command and most hand-built reports write the rollups as ",[19,1060,1061],{},"SUBTOTAL"," formulas.\nopenpyxl can read the formula text, which identifies rollup rows exactly, without any assumption about\nlabels or empty cells. This file writes the totals as formulas and leaves the group label on the first\ndetail row only, as many reports do:",[308,1064,1066],{"className":340,"code":1065,"language":342,"meta":313,"style":313},"from openpyxl import Workbook, load_workbook\n\nwb = Workbook()\nws = wb.active\nws.append([\"Region\", \"Rep\", \"Revenue\"])\nws.append([\"North\", \"Ana\", 1200])\nws.append([None, \"Ben\", 800])\nws.append([\"North Total\", None, \"=SUBTOTAL(9,C2:C3)\"])\nws.append([\"South\", \"Cara\", 950])\nws.append([None, \"Dev\", 650])\nws.append([\"South Total\", None, \"=SUBTOTAL(9,C5:C6)\"])\nwb.save(\"formula_subtotals.xlsx\")\n\nbook = load_workbook(\"formula_subtotals.xlsx\")          # formulas, not cached values\nsheet = book.active\nsubtotal_rows = {\n    cell.row for cell in sheet[\"C\"]\n    if isinstance(cell.value, str) and cell.value.upper().startswith(\"=SUBTOTAL(\")\n}\nprint(sorted(subtotal_rows))    # [4, 7]\n\nraw = pd.read_excel(\"formula_subtotals.xlsx\")\nexcel_row = pd.RangeIndex(len(raw)) + 2\nclean = raw[~excel_row.isin(subtotal_rows)].copy()\nclean[\"Region\"] = clean[\"Region\"].ffill()                # label printed once per group\nprint(clean)\n",[19,1067,1068,1079,1083,1091,1099,1115,1131,1147,1164,1180,1196,1213,1223,1228,1247,1258,1269,1291,1319,1325,1342,1347,1361,1384,1400,1423],{"__ignoreMap":313},[317,1069,1070,1072,1074,1076],{"class":319,"line":320},[317,1071,350],{"class":349},[317,1073,354],{"class":353},[317,1075,357],{"class":349},[317,1077,1078],{"class":353}," Workbook, load_workbook\n",[317,1080,1081],{"class":319,"line":363},[317,1082,367],{"emptyLinePlaceholder":366},[317,1084,1085,1087,1089],{"class":319,"line":370},[317,1086,373],{"class":353},[317,1088,376],{"class":349},[317,1090,379],{"class":353},[317,1092,1093,1095,1097],{"class":319,"line":382},[317,1094,385],{"class":353},[317,1096,376],{"class":349},[317,1098,390],{"class":353},[317,1100,1101,1103,1105,1107,1109,1111,1113],{"class":319,"line":393},[317,1102,407],{"class":353},[317,1104,410],{"class":327},[317,1106,413],{"class":353},[317,1108,416],{"class":327},[317,1110,413],{"class":353},[317,1112,421],{"class":327},[317,1114,424],{"class":353},[317,1116,1117,1119,1121,1123,1125,1127,1129],{"class":319,"line":404},[317,1118,407],{"class":353},[317,1120,442],{"class":327},[317,1122,413],{"class":353},[317,1124,447],{"class":327},[317,1126,413],{"class":353},[317,1128,148],{"class":452},[317,1130,424],{"class":353},[317,1132,1133,1135,1137,1139,1141,1143,1145],{"class":319,"line":427},[317,1134,407],{"class":353},[317,1136,476],{"class":452},[317,1138,413],{"class":353},[317,1140,462],{"class":327},[317,1142,413],{"class":353},[317,1144,169],{"class":452},[317,1146,424],{"class":353},[317,1148,1149,1151,1153,1155,1157,1159,1162],{"class":319,"line":486},[317,1150,407],{"class":353},[317,1152,471],{"class":327},[317,1154,413],{"class":353},[317,1156,476],{"class":452},[317,1158,413],{"class":353},[317,1160,1161],{"class":327},"\"=SUBTOTAL(9,C2:C3)\"",[317,1163,424],{"class":353},[317,1165,1166,1168,1170,1172,1174,1176,1178],{"class":319,"line":532},[317,1167,407],{"class":353},[317,1169,492],{"class":327},[317,1171,413],{"class":353},[317,1173,497],{"class":327},[317,1175,413],{"class":353},[317,1177,224],{"class":452},[317,1179,424],{"class":353},[317,1181,1182,1184,1186,1188,1190,1192,1194],{"class":319,"line":551},[317,1183,407],{"class":353},[317,1185,476],{"class":452},[317,1187,413],{"class":353},[317,1189,510],{"class":327},[317,1191,413],{"class":353},[317,1193,245],{"class":452},[317,1195,424],{"class":353},[317,1197,1198,1200,1202,1204,1206,1208,1211],{"class":319,"line":563},[317,1199,407],{"class":353},[317,1201,519],{"class":327},[317,1203,413],{"class":353},[317,1205,476],{"class":452},[317,1207,413],{"class":353},[317,1209,1210],{"class":327},"\"=SUBTOTAL(9,C5:C6)\"",[317,1212,424],{"class":353},[317,1214,1216,1218,1221],{"class":319,"line":1215},12,[317,1217,566],{"class":353},[317,1219,1220],{"class":327},"\"formula_subtotals.xlsx\"",[317,1222,572],{"class":353},[317,1224,1226],{"class":319,"line":1225},13,[317,1227,367],{"emptyLinePlaceholder":366},[317,1229,1231,1234,1236,1239,1241,1244],{"class":319,"line":1230},14,[317,1232,1233],{"class":353},"book ",[317,1235,376],{"class":349},[317,1237,1238],{"class":353}," load_workbook(",[317,1240,1220],{"class":327},[317,1242,1243],{"class":353},")          ",[317,1245,1246],{"class":686},"# formulas, not cached values\n",[317,1248,1250,1253,1255],{"class":319,"line":1249},15,[317,1251,1252],{"class":353},"sheet ",[317,1254,376],{"class":349},[317,1256,1257],{"class":353}," book.active\n",[317,1259,1261,1264,1266],{"class":319,"line":1260},16,[317,1262,1263],{"class":353},"subtotal_rows ",[317,1265,376],{"class":349},[317,1267,1268],{"class":353}," {\n",[317,1270,1272,1275,1277,1280,1282,1285,1288],{"class":319,"line":1271},17,[317,1273,1274],{"class":353},"    cell.row ",[317,1276,430],{"class":349},[317,1278,1279],{"class":353}," cell ",[317,1281,436],{"class":349},[317,1283,1284],{"class":353}," sheet[",[317,1286,1287],{"class":327},"\"C\"",[317,1289,1290],{"class":353},"]\n",[317,1292,1294,1297,1300,1303,1305,1308,1311,1314,1317],{"class":319,"line":1293},18,[317,1295,1296],{"class":349},"    if",[317,1298,1299],{"class":452}," isinstance",[317,1301,1302],{"class":353},"(cell.value, ",[317,1304,776],{"class":452},[317,1306,1307],{"class":353},") ",[317,1309,1310],{"class":349},"and",[317,1312,1313],{"class":353}," cell.value.upper().startswith(",[317,1315,1316],{"class":327},"\"=SUBTOTAL(\"",[317,1318,572],{"class":353},[317,1320,1322],{"class":319,"line":1321},19,[317,1323,1324],{"class":353},"}\n",[317,1326,1328,1330,1333,1336,1339],{"class":319,"line":1327},20,[317,1329,674],{"class":452},[317,1331,1332],{"class":353},"(",[317,1334,1335],{"class":452},"sorted",[317,1337,1338],{"class":353},"(subtotal_rows))    ",[317,1340,1341],{"class":686},"# [4, 7]\n",[317,1343,1345],{"class":319,"line":1344},21,[317,1346,367],{"emptyLinePlaceholder":366},[317,1348,1350,1353,1355,1357,1359],{"class":319,"line":1349},22,[317,1351,1352],{"class":353},"raw ",[317,1354,376],{"class":349},[317,1356,611],{"class":353},[317,1358,1220],{"class":327},[317,1360,572],{"class":353},[317,1362,1364,1367,1369,1372,1375,1378,1381],{"class":319,"line":1363},23,[317,1365,1366],{"class":353},"excel_row ",[317,1368,376],{"class":349},[317,1370,1371],{"class":353}," pd.RangeIndex(",[317,1373,1374],{"class":452},"len",[317,1376,1377],{"class":353},"(raw)) ",[317,1379,1380],{"class":349},"+",[317,1382,1383],{"class":452}," 2\n",[317,1385,1387,1390,1392,1395,1397],{"class":319,"line":1386},24,[317,1388,1389],{"class":353},"clean ",[317,1391,376],{"class":349},[317,1393,1394],{"class":353}," raw[",[317,1396,815],{"class":349},[317,1398,1399],{"class":353},"excel_row.isin(subtotal_rows)].copy()\n",[317,1401,1403,1406,1408,1410,1412,1415,1417,1420],{"class":319,"line":1402},25,[317,1404,1405],{"class":353},"clean[",[317,1407,410],{"class":327},[317,1409,656],{"class":353},[317,1411,376],{"class":349},[317,1413,1414],{"class":353}," clean[",[317,1416,410],{"class":327},[317,1418,1419],{"class":353},"].ffill()                ",[317,1421,1422],{"class":686},"# label printed once per group\n",[317,1424,1426,1428],{"class":319,"line":1425},26,[317,1427,674],{"class":452},[317,1429,1430],{"class":353},"(clean)\n",[10,1432,1433,1434,1436],{},"The forward fill at the end repairs the other half of this layout: the region is only written on the first\nrow of each group, so without it half of the detail rows would have no region. A file written by openpyxl\nhas no cached results for its formulas, which is why pandas shows ",[19,1435,858],{}," in those rows here; a file saved by\nExcel carries the computed totals.",[303,1438,1440],{"id":1439},"step-5-reconcile-your-sums-against-the-removed-totals","Step 5 — Reconcile your sums against the removed totals",[10,1442,1443],{},"Deleting the totals throws away information that the export's author computed for you. Keep them in a\nseparate frame and compare them with your own group sums. If they match, you know both that the right rows\nwere removed and that nothing was lost in the read.",[308,1445,1447],{"className":340,"code":1446,"language":342,"meta":313,"style":313},"totals = df[is_total].copy()\ntotals[\"Region\"] = totals[\"Region\"].str.replace(r\"\\s*total$\", \"\", case=False, regex=True)\nreported = totals.set_index(\"Region\")[\"Revenue\"]\n\ngrand_total = reported.pop(\"Grand\")\ncomputed = detail.groupby(\"Region\")[\"Revenue\"].sum()\n\nmismatch = (computed - reported.reindex(computed.index)).abs() > 0.005\nif mismatch.any() or abs(computed.sum() - grand_total) > 0.005:\n    raise ValueError(f\"totals do not reconcile:\\n{pd.concat([computed, reported], axis=1)}\")\nprint(\"reconciled:\", computed.to_dict(), \"grand total\", grand_total)\n",[19,1448,1449,1459,1519,1538,1542,1557,1575,1579,1601,1631,1670],{"__ignoreMap":313},[317,1450,1451,1454,1456],{"class":319,"line":320},[317,1452,1453],{"class":353},"totals ",[317,1455,376],{"class":349},[317,1457,1458],{"class":353}," df[is_total].copy()\n",[317,1460,1461,1464,1466,1468,1470,1473,1475,1478,1481,1483,1486,1489,1491,1494,1496,1498,1501,1503,1505,1507,1509,1511,1513,1515,1517],{"class":319,"line":363},[317,1462,1463],{"class":353},"totals[",[317,1465,410],{"class":327},[317,1467,656],{"class":353},[317,1469,376],{"class":349},[317,1471,1472],{"class":353}," totals[",[317,1474,410],{"class":327},[317,1476,1477],{"class":353},"].str.replace(",[317,1479,1480],{"class":349},"r",[317,1482,732],{"class":327},[317,1484,1485],{"class":452},"\\s",[317,1487,1488],{"class":349},"*",[317,1490,749],{"class":327},[317,1492,1493],{"class":452},"$",[317,1495,732],{"class":327},[317,1497,413],{"class":353},[317,1499,1500],{"class":327},"\"\"",[317,1502,413],{"class":353},[317,1504,786],{"class":618},[317,1506,376],{"class":349},[317,1508,791],{"class":452},[317,1510,413],{"class":353},[317,1512,796],{"class":618},[317,1514,376],{"class":349},[317,1516,801],{"class":452},[317,1518,572],{"class":353},[317,1520,1521,1524,1526,1529,1531,1534,1536],{"class":319,"line":370},[317,1522,1523],{"class":353},"reported ",[317,1525,376],{"class":349},[317,1527,1528],{"class":353}," totals.set_index(",[317,1530,410],{"class":327},[317,1532,1533],{"class":353},")[",[317,1535,421],{"class":327},[317,1537,1290],{"class":353},[317,1539,1540],{"class":319,"line":382},[317,1541,367],{"emptyLinePlaceholder":366},[317,1543,1544,1547,1549,1552,1555],{"class":319,"line":393},[317,1545,1546],{"class":353},"grand_total ",[317,1548,376],{"class":349},[317,1550,1551],{"class":353}," reported.pop(",[317,1553,1554],{"class":327},"\"Grand\"",[317,1556,572],{"class":353},[317,1558,1559,1562,1564,1567,1569,1571,1573],{"class":319,"line":404},[317,1560,1561],{"class":353},"computed ",[317,1563,376],{"class":349},[317,1565,1566],{"class":353}," detail.groupby(",[317,1568,410],{"class":327},[317,1570,1533],{"class":353},[317,1572,421],{"class":327},[317,1574,641],{"class":353},[317,1576,1577],{"class":319,"line":427},[317,1578,367],{"emptyLinePlaceholder":366},[317,1580,1581,1584,1586,1589,1592,1595,1598],{"class":319,"line":486},[317,1582,1583],{"class":353},"mismatch ",[317,1585,376],{"class":349},[317,1587,1588],{"class":353}," (computed ",[317,1590,1591],{"class":349},"-",[317,1593,1594],{"class":353}," reported.reindex(computed.index)).abs() ",[317,1596,1597],{"class":349},">",[317,1599,1600],{"class":452}," 0.005\n",[317,1602,1603,1606,1609,1612,1615,1618,1620,1623,1625,1628],{"class":319,"line":532},[317,1604,1605],{"class":349},"if",[317,1607,1608],{"class":353}," mismatch.any() ",[317,1610,1611],{"class":349},"or",[317,1613,1614],{"class":452}," abs",[317,1616,1617],{"class":353},"(computed.sum() ",[317,1619,1591],{"class":349},[317,1621,1622],{"class":353}," grand_total) ",[317,1624,1597],{"class":349},[317,1626,1627],{"class":452}," 0.005",[317,1629,1630],{"class":353},":\n",[317,1632,1633,1636,1639,1641,1644,1647,1651,1654,1657,1659,1661,1663,1666,1668],{"class":319,"line":551},[317,1634,1635],{"class":349},"    raise",[317,1637,1638],{"class":452}," ValueError",[317,1640,1332],{"class":353},[317,1642,1643],{"class":349},"f",[317,1645,1646],{"class":327},"\"totals do not reconcile:",[317,1648,1650],{"class":1649},"sSjpA","\\n{",[317,1652,1653],{"class":353},"pd.concat([computed, reported], ",[317,1655,1656],{"class":618},"axis",[317,1658,376],{"class":349},[317,1660,109],{"class":452},[317,1662,744],{"class":353},[317,1664,1665],{"class":1649},"}",[317,1667,732],{"class":327},[317,1669,572],{"class":353},[317,1671,1672,1674,1676,1679,1682,1685],{"class":319,"line":563},[317,1673,674],{"class":452},[317,1675,1332],{"class":353},[317,1677,1678],{"class":327},"\"reconciled:\"",[317,1680,1681],{"class":353},", computed.to_dict(), ",[317,1683,1684],{"class":327},"\"grand total\"",[317,1686,1687],{"class":353},", grand_total)\n",[10,1689,1690],{},"The small tolerance absorbs floating-point rounding in currency columns. A mismatch larger than that usually\nmeans one of three things: a detail row was filtered out by mistake, the export applied a filter before it\ncalculated its totals, or the totals are formatted as rounded numbers while the detail keeps decimals.",[38,1692,47,1697,47,1700,47,1703,47,1705,47,1710,47,1715,47,1719,47,1724,47,1729,47,1732,47,1737,47,1740,47,1744,47,1747,47,1750,47,1755,47,1758,47,1762,47,1766,47,1769,47,1772,47,1777,47,1780],{"viewBox":1693,"role":41,"ariaLabelledBy":1694,"xmlns":45,"style":879},"0 0 760 156",[1695,1696],"sub-reconcile-t","sub-reconcile-d",[49,1698,1699],{"id":1695},"Removing rollups and using them as a check",[53,1701,1702],{"id":1696},"Split the export into detail rows and total rows, sum the detail by group, compare with the reported subtotals and grand total, and only continue when they reconcile.",[57,1704],{"x":59,"y":59,"width":888,"height":91,"fill":62},[57,1706],{"x":72,"y":72,"width":1707,"height":1708,"rx":1709,"fill":76,"stroke":77,"style":894},"150","86","12",[64,1711,1714],{"x":1712,"y":73,"style":1713},"95","font-size:14px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:middle","split",[64,1716,1718],{"x":1712,"y":1717,"style":114},"72","detail vs totals",[319,1720],{"x1":1721,"y1":1722,"x2":1723,"y2":1722,"stroke":920,"style":894},"175","63","204",[1725,1726],"polygon",{"points":1727,"fill":1728},"207,63 198,58 198,68","#5b5cf0",[57,1730],{"x":1731,"y":72,"width":1707,"height":1708,"rx":1709,"fill":919,"stroke":920,"style":894},"210",[64,1733,1736],{"x":1734,"y":73,"style":1735},"285","font-size:14px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","group sums",[64,1738,1739],{"x":1734,"y":1717,"style":137},"from detail rows",[319,1741],{"x1":1742,"y1":1722,"x2":1743,"y2":1722,"stroke":920,"style":894},"365","394",[1725,1745],{"points":1746,"fill":1728},"397,63 388,58 388,68",[57,1748],{"x":1749,"y":72,"width":1707,"height":1708,"rx":1709,"fill":179,"stroke":180,"style":894},"400",[64,1751,1754],{"x":1752,"y":73,"style":1753},"475","font-size:14px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","compare",[64,1756,1757],{"x":1752,"y":1717,"style":137},"with reported",[64,1759,1761],{"x":1752,"y":1760,"style":137},"92","subtotals",[319,1763],{"x1":1764,"y1":1722,"x2":1765,"y2":1722,"stroke":920,"style":894},"555","584",[1725,1767],{"points":1768,"fill":1728},"587,63 578,58 578,68",[57,1770],{"x":1771,"y":72,"width":1707,"height":1708,"rx":1709,"fill":942,"stroke":943,"style":894},"590",[64,1773,1776],{"x":1774,"y":73,"style":1775},"665","font-size:14px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","continue",[64,1778,1779],{"x":1774,"y":1717,"style":137},"only when equal",[64,1781,1782],{"x":923,"y":151,"style":300},"the totals you delete are the cheapest test you will ever get",[303,1784,1786],{"id":1785},"step-6-wrap-it-as-a-reusable-cleaner","Step 6 — Wrap it as a reusable cleaner",[10,1788,1789],{},"Put the rules in one function so every export passes through the same logic, and return both halves so the\ncaller can reconcile or log them.",[308,1791,1793],{"className":340,"code":1792,"language":342,"meta":313,"style":313},"def split_rollups(frame: pd.DataFrame, label_col: str, id_col: str, value_col: str):\n    label_hit = frame[label_col].astype(str).str.contains(TOTAL_LABEL, case=False, regex=True)\n    shape_hit = frame[id_col].isna() & frame[value_col].notna()\n    rollup = label_hit | shape_hit\n    return frame[~rollup].reset_index(drop=True), frame[rollup].reset_index(drop=True)\n\ndetail_rows, rollup_rows = split_rollups(df, \"Region\", \"Rep\", \"Revenue\")\nprint(len(detail_rows), \"detail rows,\", len(rollup_rows), \"rollup rows\")\n",[19,1794,1795,1822,1856,1871,1887,1917,1921,1943],{"__ignoreMap":313},[317,1796,1797,1800,1804,1807,1809,1812,1814,1817,1819],{"class":319,"line":320},[317,1798,1799],{"class":349},"def",[317,1801,1803],{"class":1802},"s_Opv"," split_rollups",[317,1805,1806],{"class":353},"(frame: pd.DataFrame, label_col: ",[317,1808,776],{"class":452},[317,1810,1811],{"class":353},", id_col: ",[317,1813,776],{"class":452},[317,1815,1816],{"class":353},", value_col: ",[317,1818,776],{"class":452},[317,1820,1821],{"class":353},"):\n",[317,1823,1824,1827,1829,1832,1834,1836,1838,1840,1842,1844,1846,1848,1850,1852,1854],{"class":319,"line":363},[317,1825,1826],{"class":353},"    label_hit ",[317,1828,376],{"class":349},[317,1830,1831],{"class":353}," frame[label_col].astype(",[317,1833,776],{"class":452},[317,1835,779],{"class":353},[317,1837,723],{"class":452},[317,1839,413],{"class":353},[317,1841,786],{"class":618},[317,1843,376],{"class":349},[317,1845,791],{"class":452},[317,1847,413],{"class":353},[317,1849,796],{"class":618},[317,1851,376],{"class":349},[317,1853,801],{"class":452},[317,1855,572],{"class":353},[317,1857,1858,1861,1863,1866,1868],{"class":319,"line":370},[317,1859,1860],{"class":353},"    shape_hit ",[317,1862,376],{"class":349},[317,1864,1865],{"class":353}," frame[id_col].isna() ",[317,1867,994],{"class":349},[317,1869,1870],{"class":353}," frame[value_col].notna()\n",[317,1872,1873,1876,1878,1881,1884],{"class":319,"line":382},[317,1874,1875],{"class":353},"    rollup ",[317,1877,376],{"class":349},[317,1879,1880],{"class":353}," label_hit ",[317,1882,1883],{"class":349},"|",[317,1885,1886],{"class":353}," shape_hit\n",[317,1888,1889,1892,1895,1897,1900,1902,1904,1906,1909,1911,1913,1915],{"class":319,"line":393},[317,1890,1891],{"class":349},"    return",[317,1893,1894],{"class":353}," frame[",[317,1896,815],{"class":349},[317,1898,1899],{"class":353},"rollup].reset_index(",[317,1901,821],{"class":618},[317,1903,376],{"class":349},[317,1905,801],{"class":452},[317,1907,1908],{"class":353},"), frame[rollup].reset_index(",[317,1910,821],{"class":618},[317,1912,376],{"class":349},[317,1914,801],{"class":452},[317,1916,572],{"class":353},[317,1918,1919],{"class":319,"line":404},[317,1920,367],{"emptyLinePlaceholder":366},[317,1922,1923,1926,1928,1931,1933,1935,1937,1939,1941],{"class":319,"line":427},[317,1924,1925],{"class":353},"detail_rows, rollup_rows ",[317,1927,376],{"class":349},[317,1929,1930],{"class":353}," split_rollups(df, ",[317,1932,410],{"class":327},[317,1934,413],{"class":353},[317,1936,416],{"class":327},[317,1938,413],{"class":353},[317,1940,421],{"class":327},[317,1942,572],{"class":353},[317,1944,1945,1947,1949,1951,1954,1957,1959,1961,1964,1967],{"class":319,"line":486},[317,1946,674],{"class":452},[317,1948,1332],{"class":353},[317,1950,1374],{"class":452},[317,1952,1953],{"class":353},"(detail_rows), ",[317,1955,1956],{"class":327},"\"detail rows,\"",[317,1958,413],{"class":353},[317,1960,1374],{"class":452},[317,1962,1963],{"class":353},"(rollup_rows), ",[317,1965,1966],{"class":327},"\"rollup rows\"",[317,1968,572],{"class":353},[10,1970,1971,1972,1974],{},"Combining the label rule with the shape rule with ",[19,1973,1883],{}," catches both labelled and unlabelled rollups. Log the\nnumber of rollup rows per file; a sudden change in that count is an early signal that the export's grouping\nhas changed.",[303,1976,1978],{"id":1977},"common-pitfalls","Common pitfalls",[1980,1981,1982,1998],"table",{},[1983,1984,1985],"thead",{},[1986,1987,1988,1992,1995],"tr",{},[1989,1990,1991],"th",{},"Symptom",[1989,1993,1994],{},"Cause",[1989,1996,1997],{},"Fix",[1999,2000,2001,2016,2034,2045,2056,2073],"tbody",{},[1986,2002,2003,2007,2010],{},[2004,2005,2006],"td",{},"A real customer disappears",[2004,2008,2009],{},"Label regex matched inside a name",[2004,2011,2012,2013],{},"Use word boundaries: ",[19,2014,2015],{},"\\btotal\\b",[1986,2017,2018,2023,2028],{},[2004,2019,2020,2021],{},"Boolean mask error on ",[19,2022,862],{},[2004,2024,2025,2026],{},"Label column has ",[19,2027,858],{},[2004,2029,2030,2031,2033],{},"Call ",[19,2032,854],{}," first",[1986,2035,2036,2039,2042],{},[2004,2037,2038],{},"Detail rows removed by the shape rule",[2004,2040,2041],{},"The identifying column is optional",[2004,2043,2044],{},"Choose a mandatory field, or require several empty",[1986,2046,2047,2050,2053],{},[2004,2048,2049],{},"Totals do not reconcile by a small amount",[2004,2051,2052],{},"Rounded totals versus unrounded detail",[2004,2054,2055],{},"Compare with a tolerance, or round both",[1986,2057,2058,2063,2066],{},[2004,2059,2060,2061],{},"Subtotal rows read as ",[19,2062,858],{},[2004,2064,2065],{},"Formulas written by a script, never calculated",[2004,2067,2068,2069],{},"Detect by formula, or recalculate as in ",[32,2070,2072],{"href":2071},"\u002Fgetting-started-with-python-excel-automation\u002Fworking-with-excel-formulas-in-python\u002Frecalculate-excel-formulas-without-excel-in-python\u002F","Recalculate Excel Formulas Without Excel in Python",[1986,2074,2075,2078,2081],{},[2004,2076,2077],{},"Half the rows have no group",[2004,2079,2080],{},"Group label printed once per block",[2004,2082,2083,2086],{},[19,2084,2085],{},"ffill()"," the label column after removing rollups",[303,2088,2090],{"id":2089},"performance-and-scale-notes","Performance and scale notes",[10,2092,2093,2094,2097,2098,2101],{},"Both filters are vectorised string and null checks, so they are effectively free next to the read itself. The\nopenpyxl formula scan costs a full workbook load; on a large file, restrict it to the one value column with\n",[19,2095,2096],{},"sheet[\"C\"]"," as shown, or use ",[19,2099,2100],{},"read_only=True",", where cell values still expose formula strings and the scan\nstreams through the sheet instead of building it in memory. If the same\nreport format arrives every day, run the reconciliation every day too — it is a few milliseconds, and it is\nthe check that notices when an upstream filter changed.",[303,2103,2105],{"id":2104},"conclusion","Conclusion",[10,2107,2108,2109,2111],{},"Rollup rows are data about the data, and they have to come out before any aggregation. Find them by their\nlabel, by the absence of a record identifier, or — most precisely — by their ",[19,2110,1061],{}," formulas, and use a\ncombination when the export is inconsistent. Then keep the removed rows long enough to reconcile your own\ngroup sums against them. It turns a cleaning step into a correctness test that costs nothing.",[303,2113,2115],{"id":2114},"frequently-asked-questions","Frequently asked questions",[10,2117,2118,2121],{},[14,2119,2120],{},"Why is my pandas sum a whole multiple of the real total?","\nBecause the sheet contains rollups that repeat the detail values: one layer of subtotals plus a grand total triples the sum. Removing the rollup rows brings it back to the true figure.",[10,2123,2124,2127],{},[14,2125,2126],{},"Can I just drop the last row to remove the grand total?","\nOnly if you are certain it is always the last row and there are no subtotals. Matching the label or the formula is safer and costs no more.",[10,2129,2130,2133],{},[14,2131,2132],{},"How do I keep the subtotals as a separate summary table?","\nSelect the rollup rows instead of discarding them, strip the word Total from the label, and write them to their own sheet. They make a convenient check against your own groupby.",[10,2135,2136,2139],{},[14,2137,2138],{},"Does pandas pivot_table produce the same kind of total rows?","\nOnly when margins=True, which adds an All row and column. Those are easy to remove by label, but it is simpler not to request margins when the data is going on to further processing.",[303,2141,2143],{"id":2142},"related","Related",[2145,2146,2147,2154,2161,2168,2175],"ul",{},[2148,2149,2150,2151,2153],"li",{},"Up one level: ",[32,2152,35],{"href":34}," — the full set of techniques for report-shaped sheets.",[2148,2155,2156,2160],{},[32,2157,2159],{"href":2158},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Ffind-the-header-row-in-an-excel-sheet-with-a-shifting-layout\u002F","Find the Header Row in an Excel Sheet with a Shifting Layout"," — cleaning the top of the same kind of export.",[2148,2162,2163,2167],{},[32,2164,2166],{"href":2165},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fread-multiple-tables-from-one-excel-sheet\u002F","Read Multiple Tables from One Excel Sheet"," — when blocks rather than totals share the sheet.",[2148,2169,2170,2174],{},[32,2171,2173],{"href":2172},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcreating-pivot-tables-from-excel-data\u002Fcreate-pivot-table-from-excel-with-pandas\u002F","Create a Pivot Table from Excel with Pandas"," — rebuilding the rollups properly from clean detail.",[2148,2176,2177,2181],{},[32,2178,2180],{"href":2179},"\u002Fadvanced-data-transformation-and-cleaning\u002Fexcel-formula-equivalents-in-pandas\u002Fsumif-and-sumifs-equivalent-in-pandas\u002F","SUMIF and SUMIFS Equivalent in pandas"," — conditional totals once the detail is clean.",[2183,2184,2185],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}",{"title":313,"searchDepth":363,"depth":363,"links":2187},[2188,2189,2190,2191,2192,2193,2194,2195,2196,2197,2198,2199],{"id":305,"depth":363,"text":306},{"id":575,"depth":363,"text":576},{"id":693,"depth":363,"text":694},{"id":968,"depth":363,"text":969},{"id":1051,"depth":363,"text":1052},{"id":1439,"depth":363,"text":1440},{"id":1785,"depth":363,"text":1786},{"id":1977,"depth":363,"text":1978},{"id":2089,"depth":363,"text":2090},{"id":2104,"depth":363,"text":2105},{"id":2114,"depth":363,"text":2115},{"id":2142,"depth":363,"text":2143},"2026-09-17","Subtotal and grand-total rows double every sum you compute. Find them by label, by missing detail fields or by SUBTOTAL formulas, remove them, and use the totals to check your own numbers.","md",[2204,2206,2208,2210],{"q":2120,"a":2205},"Because the sheet contains rollups that repeat the detail values: one layer of subtotals plus a grand total triples the sum. Removing the rollup rows brings it back to the true figure.",{"q":2126,"a":2207},"Only if you are certain it is always the last row and there are no subtotals. Matching the label or the formula is safer and costs no more.",{"q":2132,"a":2209},"Select the rollup rows instead of discarding them, strip the word Total from the label, and write them to their own sheet. They make a convenient check against your own groupby.",{"q":2138,"a":2211},"Only when margins=True, which adds an All row and column. Those are easy to remove by label, but it is simpler not to request margins when the data is going on to further processing.","remove subtotal and total rows from an excel export",null,{"breadcrumb":2215},[2216,2218,2221],{"name":2217,"item":680},"Home",{"name":2219,"item":2220},"Advanced Data Transformation and Cleaning","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":35,"item":34},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fremove-subtotal-and-total-rows-from-an-excel-export",{"title":2224,"description":2225},"Remove Subtotal and Total Rows from an Excel Export in pandas","Strip subtotal and grand total rows from Excel data in Python: match total labels, detect rows with empty detail columns, find SUBTOTAL formulas with openpyxl, and reconcile the sums.","remove-subtotal-and-total-rows-from-an-excel-export","advanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fremove-subtotal-and-total-rows-from-an-excel-export\u002Findex",[2229,2230,2231],"messy-sheets","data-cleaning","pandas","how-to","HoiVVGIKFyf67xXbXLoz7lXIvmspwy5octdicyHQpD4",[2235,2238],{"title":2166,"path":2236,"stem":2237,"children":-1},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fread-multiple-tables-from-one-excel-sheet","advanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fread-multiple-tables-from-one-excel-sheet\u002Findex",{"title":2239,"path":2240,"stem":2241,"children":-1},"Skip Hidden and Filtered Rows When Reading Excel","\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fskip-hidden-and-filtered-rows-when-reading-excel","advanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fskip-hidden-and-filtered-rows-when-reading-excel\u002Findex",[2243,2245],{"path":2244,"title":35},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets",{"path":2246,"title":2247},"\u002Fadvanced-data-transformation-and-cleaning","Advanced Data Transformation and Cleaning for Excel Automation",[],[2250,2253,2256,2259,2262,2265],{"path":2251,"title":2252,"kind":2232},"\u002Fformatting-and-charting-excel-reports-with-python\u002Fstyling-excel-cells-with-openpyxl\u002Ffreeze-header-row-in-excel-with-openpyxl","Freeze the Header Row in Excel with openpyxl",{"path":2254,"title":2255,"kind":2232},"\u002Fgetting-started-with-python-excel-automation\u002Freading-excel-files-with-pandas\u002Fhandle-merged-cells-when-reading-excel-with-pandas","Handle Merged Cells When Reading Excel with pandas",{"path":2257,"title":2258,"kind":2232},"\u002Fgetting-started-with-python-excel-automation\u002Fusing-openpyxl-for-excel-file-manipulation\u002Funmerge-cells-and-fill-values-with-openpyxl","Unmerge Cells and Fill the Values with openpyxl",{"path":2260,"title":2261,"kind":2232},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fcombine-sheets-with-different-layouts-into-one-table","Combine Sheets with Different Layouts into One Table",{"path":2263,"title":2264,"kind":2232},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fextract-hyperlink-urls-from-excel-cells","Extract Hyperlink URLs from Excel Cells with openpyxl",{"path":2266,"title":2267,"kind":2232},"\u002Fadvanced-data-transformation-and-cleaning\u002Fextracting-data-from-messy-excel-sheets\u002Fextract-report-header-fields-from-an-excel-sheet","Extract Report Header Fields from an Excel Sheet with Python",1790972092922]