[{"data":1,"prerenderedAt":2509},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Faggregate-a-huge-excel-file-without-loading-it":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Faggregate-a-huge-excel-file-without-loading-it":2477,"ancestors:\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Faggregate-a-huge-excel-file-without-loading-it":2485,"children:\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Faggregate-a-huge-excel-file-without-loading-it":2489,"related:\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Faggregate-a-huge-excel-file-without-loading-it":2490},{"id":4,"title":5,"body":6,"dateModified":2444,"datePublished":2444,"description":2445,"extension":2446,"faq":2447,"keyword":2456,"kind":2457,"meta":2458,"navigation":115,"path":2466,"seo":2467,"slug":2470,"stem":2471,"topics":2472,"type":2475,"__hash__":2476},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Faggregate-a-huge-excel-file-without-loading-it\u002Findex.md","Aggregate a Huge Excel File Without Loading It",{"type":7,"value":8,"toc":2432},"minimark",[9,24,29,61,67,71,74,566,577,720,724,735,1046,1068,1072,1075,1480,1483,1538,1542,1545,1963,1974,1978,1981,2185,2192,2318,2322,2357,2361,2367,2371,2377,2383,2389,2395,2399,2428],[10,11,12,13,17,18,23],"p",{},"The question is small — total revenue and order count per region and month — but the workbook is not: half a million rows that pandas reads into several hundred megabytes before a single sum is computed. On a laptop that is slow; on a small server, in a container or in a scheduled job sharing a machine, it can fail outright. Most summary statistics do not need the whole table at once. A sum, a count, a minimum or a maximum can be updated one row at a time, so a ",[14,15,16],"em",{},"streaming group-by"," reads each row, updates a small dictionary of running totals and forgets the row. Memory then depends on the number of groups, not the number of rows. This page builds that aggregation over openpyxl and the faster calamine reader, deals with the statistics that cannot be streamed so simply, and checks the result against pandas. It belongs to ",[19,20,22],"a",{"href":21},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002F","Working with Large Excel Files in Python",".",[25,26,28],"h2",{"id":27},"prerequisites","Prerequisites",[30,31,36],"pre",{"className":32,"code":33,"language":34,"meta":35,"style":35},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install openpyxl python-calamine pandas\n","bash","",[37,38,39],"code",{"__ignoreMap":35},[40,41,44,48,52,55,58],"span",{"class":42,"line":43},"line",1,[40,45,47],{"class":46},"sMTad","pip",[40,49,51],{"class":50},"srMev"," install",[40,53,54],{"class":50}," openpyxl",[40,56,57],{"class":50}," python-calamine",[40,59,60],{"class":50}," pandas\n",[10,62,63,66],{},[37,64,65],{},"sales_big.xlsx"," has one sheet of 300,000 sales lines: date, region, product, quantity and unit price. Large enough to show the memory difference clearly, small enough to run in a minute; the same code works unchanged on files several times larger.",[25,68,70],{"id":69},"step-1-a-streaming-group-by","Step 1 — A streaming group-by",[10,72,73],{},"The core is a dictionary keyed by the group, holding running totals. Each row updates one entry and is then discarded:",[30,75,79],{"className":76,"code":77,"language":78,"meta":35,"style":35},"language-python shiki shiki-themes github-light github-dark-high-contrast","from collections import defaultdict\nfrom openpyxl import load_workbook\n\ndef new_acc():\n    return {\"orders\": 0, \"units\": 0, \"revenue\": 0.0, \"max_line\": float(\"-inf\"), \"min_line\": float(\"inf\")}\n\ndef aggregate(rows, header):\n    i = {name: header.index(name) for name in header}\n    groups = defaultdict(new_acc)\n    for r in rows:\n        key = (r[i[\"Region\"]], r[i[\"Date\"]].strftime(\"%Y-%m\"))\n        line = r[i[\"Qty\"]] * r[i[\"Unit price\"]]\n        a = groups[key]\n        a[\"orders\"] += 1\n        a[\"units\"] += r[i[\"Qty\"]]\n        a[\"revenue\"] += line\n        a[\"max_line\"] = max(a[\"max_line\"], line)\n        a[\"min_line\"] = min(a[\"min_line\"], line)\n    return groups\n\nwb = load_workbook(\"sales_big.xlsx\", read_only=True)\nrows = wb[\"Sales\"].iter_rows(values_only=True)\nheader = list(next(rows))\ngroups = aggregate(rows, header)\nwb.close()\nprint(len(groups), \"groups\")\nprint(sorted(groups.items())[0])\n","python",[37,80,81,97,110,117,130,203,208,219,243,254,268,297,325,336,353,370,384,406,426,434,439,467,493,512,523,529,548],{"__ignoreMap":35},[40,82,83,87,91,94],{"class":42,"line":43},[40,84,86],{"class":85},"s-kum","from",[40,88,90],{"class":89},"skGVy"," collections ",[40,92,93],{"class":85},"import",[40,95,96],{"class":89}," defaultdict\n",[40,98,100,102,105,107],{"class":42,"line":99},2,[40,101,86],{"class":85},[40,103,104],{"class":89}," openpyxl ",[40,106,93],{"class":85},[40,108,109],{"class":89}," load_workbook\n",[40,111,113],{"class":42,"line":112},3,[40,114,116],{"emptyLinePlaceholder":115},true,"\n",[40,118,120,123,127],{"class":42,"line":119},4,[40,121,122],{"class":85},"def",[40,124,126],{"class":125},"s_Opv"," new_acc",[40,128,129],{"class":89},"():\n",[40,131,133,136,139,142,145,149,152,155,157,159,161,164,166,169,171,174,176,179,182,185,188,191,193,195,197,200],{"class":42,"line":132},5,[40,134,135],{"class":85},"    return",[40,137,138],{"class":89}," {",[40,140,141],{"class":50},"\"orders\"",[40,143,144],{"class":89},": ",[40,146,148],{"class":147},"sP0c6","0",[40,150,151],{"class":89},", ",[40,153,154],{"class":50},"\"units\"",[40,156,144],{"class":89},[40,158,148],{"class":147},[40,160,151],{"class":89},[40,162,163],{"class":50},"\"revenue\"",[40,165,144],{"class":89},[40,167,168],{"class":147},"0.0",[40,170,151],{"class":89},[40,172,173],{"class":50},"\"max_line\"",[40,175,144],{"class":89},[40,177,178],{"class":147},"float",[40,180,181],{"class":89},"(",[40,183,184],{"class":50},"\"-inf\"",[40,186,187],{"class":89},"), ",[40,189,190],{"class":50},"\"min_line\"",[40,192,144],{"class":89},[40,194,178],{"class":147},[40,196,181],{"class":89},[40,198,199],{"class":50},"\"inf\"",[40,201,202],{"class":89},")}\n",[40,204,206],{"class":42,"line":205},6,[40,207,116],{"emptyLinePlaceholder":115},[40,209,211,213,216],{"class":42,"line":210},7,[40,212,122],{"class":85},[40,214,215],{"class":125}," aggregate",[40,217,218],{"class":89},"(rows, header):\n",[40,220,222,225,228,231,234,237,240],{"class":42,"line":221},8,[40,223,224],{"class":89},"    i ",[40,226,227],{"class":85},"=",[40,229,230],{"class":89}," {name: header.index(name) ",[40,232,233],{"class":85},"for",[40,235,236],{"class":89}," name ",[40,238,239],{"class":85},"in",[40,241,242],{"class":89}," header}\n",[40,244,246,249,251],{"class":42,"line":245},9,[40,247,248],{"class":89},"    groups ",[40,250,227],{"class":85},[40,252,253],{"class":89}," defaultdict(new_acc)\n",[40,255,257,260,263,265],{"class":42,"line":256},10,[40,258,259],{"class":85},"    for",[40,261,262],{"class":89}," r ",[40,264,239],{"class":85},[40,266,267],{"class":89}," rows:\n",[40,269,271,274,276,279,282,285,288,291,294],{"class":42,"line":270},11,[40,272,273],{"class":89},"        key ",[40,275,227],{"class":85},[40,277,278],{"class":89}," (r[i[",[40,280,281],{"class":50},"\"Region\"",[40,283,284],{"class":89},"]], r[i[",[40,286,287],{"class":50},"\"Date\"",[40,289,290],{"class":89},"]].strftime(",[40,292,293],{"class":50},"\"%Y-%m\"",[40,295,296],{"class":89},"))\n",[40,298,300,303,305,308,311,314,317,319,322],{"class":42,"line":299},12,[40,301,302],{"class":89},"        line ",[40,304,227],{"class":85},[40,306,307],{"class":89}," r[i[",[40,309,310],{"class":50},"\"Qty\"",[40,312,313],{"class":89},"]] ",[40,315,316],{"class":85},"*",[40,318,307],{"class":89},[40,320,321],{"class":50},"\"Unit price\"",[40,323,324],{"class":89},"]]\n",[40,326,328,331,333],{"class":42,"line":327},13,[40,329,330],{"class":89},"        a ",[40,332,227],{"class":85},[40,334,335],{"class":89}," groups[key]\n",[40,337,339,342,344,347,350],{"class":42,"line":338},14,[40,340,341],{"class":89},"        a[",[40,343,141],{"class":50},[40,345,346],{"class":89},"] ",[40,348,349],{"class":85},"+=",[40,351,352],{"class":147}," 1\n",[40,354,356,358,360,362,364,366,368],{"class":42,"line":355},15,[40,357,341],{"class":89},[40,359,154],{"class":50},[40,361,346],{"class":89},[40,363,349],{"class":85},[40,365,307],{"class":89},[40,367,310],{"class":50},[40,369,324],{"class":89},[40,371,373,375,377,379,381],{"class":42,"line":372},16,[40,374,341],{"class":89},[40,376,163],{"class":50},[40,378,346],{"class":89},[40,380,349],{"class":85},[40,382,383],{"class":89}," line\n",[40,385,387,389,391,393,395,398,401,403],{"class":42,"line":386},17,[40,388,341],{"class":89},[40,390,173],{"class":50},[40,392,346],{"class":89},[40,394,227],{"class":85},[40,396,397],{"class":147}," max",[40,399,400],{"class":89},"(a[",[40,402,173],{"class":50},[40,404,405],{"class":89},"], line)\n",[40,407,409,411,413,415,417,420,422,424],{"class":42,"line":408},18,[40,410,341],{"class":89},[40,412,190],{"class":50},[40,414,346],{"class":89},[40,416,227],{"class":85},[40,418,419],{"class":147}," min",[40,421,400],{"class":89},[40,423,190],{"class":50},[40,425,405],{"class":89},[40,427,429,431],{"class":42,"line":428},19,[40,430,135],{"class":85},[40,432,433],{"class":89}," groups\n",[40,435,437],{"class":42,"line":436},20,[40,438,116],{"emptyLinePlaceholder":115},[40,440,442,445,447,450,453,455,459,461,464],{"class":42,"line":441},21,[40,443,444],{"class":89},"wb ",[40,446,227],{"class":85},[40,448,449],{"class":89}," load_workbook(",[40,451,452],{"class":50},"\"sales_big.xlsx\"",[40,454,151],{"class":89},[40,456,458],{"class":457},"sa561","read_only",[40,460,227],{"class":85},[40,462,463],{"class":147},"True",[40,465,466],{"class":89},")\n",[40,468,470,473,475,478,481,484,487,489,491],{"class":42,"line":469},22,[40,471,472],{"class":89},"rows ",[40,474,227],{"class":85},[40,476,477],{"class":89}," wb[",[40,479,480],{"class":50},"\"Sales\"",[40,482,483],{"class":89},"].iter_rows(",[40,485,486],{"class":457},"values_only",[40,488,227],{"class":85},[40,490,463],{"class":147},[40,492,466],{"class":89},[40,494,496,499,501,504,506,509],{"class":42,"line":495},23,[40,497,498],{"class":89},"header ",[40,500,227],{"class":85},[40,502,503],{"class":147}," list",[40,505,181],{"class":89},[40,507,508],{"class":147},"next",[40,510,511],{"class":89},"(rows))\n",[40,513,515,518,520],{"class":42,"line":514},24,[40,516,517],{"class":89},"groups ",[40,519,227],{"class":85},[40,521,522],{"class":89}," aggregate(rows, header)\n",[40,524,526],{"class":42,"line":525},25,[40,527,528],{"class":89},"wb.close()\n",[40,530,532,535,537,540,543,546],{"class":42,"line":531},26,[40,533,534],{"class":147},"print",[40,536,181],{"class":89},[40,538,539],{"class":147},"len",[40,541,542],{"class":89},"(groups), ",[40,544,545],{"class":50},"\"groups\"",[40,547,466],{"class":89},[40,549,551,553,555,558,561,563],{"class":42,"line":550},27,[40,552,534],{"class":147},[40,554,181],{"class":89},[40,556,557],{"class":147},"sorted",[40,559,560],{"class":89},"(groups.items())[",[40,562,148],{"class":147},[40,564,565],{"class":89},"])\n",[10,567,568,569,572,573,576],{},"openpyxl's read-only mode yields one row at a time without building the sheet in memory. The accumulator for each region and month holds five numbers, so the whole result is 36 small dictionaries, whatever the length of the file. ",[37,570,571],{},"defaultdict"," creates an accumulator the first time a group is seen. Averages come at the end: average revenue per order is ",[37,574,575],{},"revenue \u002F orders"," — that is why the count is kept alongside the sum. Any statistic that can be built from such running values works this way: sums, counts, means, minima, maxima, and even variance (by also accumulating the sum of squares, or better, with Welford's method).",[578,579,587,588,587,592,587,596,587,603,587,613,587,624,587,631,587,636,587,642,587,647,587,651,587,658,587,663,587,666,587,670,587,673,587,678,587,685,587,690,587,693,587,697,587,700,587,705,587,712,587,717],"svg",{"viewBox":580,"role":581,"ariaLabelledBy":582,"xmlns":585,"style":586},"4 0 752 125","img",[583,584],"agg-stream-t","agg-stream-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:752px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[589,590,591],"title",{"id":583},"A streaming group-by",[593,594,595],"desc",{"id":584},"Each row is read on its own, its group key and line value are computed, the group's running totals for count, units, revenue, minimum and maximum are updated, and the row is forgotten. Memory holds only one accumulator per group.",[597,598],"rect",{"x":599,"y":148,"width":600,"height":601,"fill":602},"4","752","125","#ffffff",[597,604],{"x":605,"y":606,"width":607,"height":608,"rx":609,"fill":610,"stroke":611,"style":612},"20.0","16.0","153.0","93.0","12","#e7ebef","var(--line,#cdd5e6)","stroke-width:2px",[614,615,617],"g",{"transform":616},"translate(84.5,29.0) scale(1.000)",[618,619],"path",{"d":620,"fill":621,"stroke":622,"style":623},"M4 4h16v16H4z M4 9h16 M4 14h16 M10 4v16","none","var(--text,#172033)","stroke-width:1.80px;stroke-linecap:round;stroke-linejoin:round",[625,626,630],"text",{"x":627,"y":628,"style":629},"96.5","75.5","font-size:13.5px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","Read one row",[625,632,635],{"x":627,"y":633,"style":634},"95.0","font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","read-only \u002F calamine",[42,637],{"x1":638,"y1":639,"x2":640,"y2":639,"stroke":641,"style":612},"178.0","62.5","200.0","var(--brand,#5b5cf0)",[643,644],"polygon",{"points":645,"fill":646},"204.0,62.5 195.0,67.5 195.0,57.5","#5b5cf0",[597,648],{"x":649,"y":606,"width":607,"height":608,"rx":609,"fill":650,"stroke":641,"style":612},"209.0","#f0f4ff",[614,652,654],{"transform":653},"translate(273.5,29.0) scale(1.000)",[618,655],{"d":656,"fill":621,"stroke":657,"style":623},"M5 9h14 M4 15h14 M10 4l-2 16 M16 4l-2 16","var(--brand-strong,#4338ca)",[625,659,662],{"x":660,"y":628,"style":661},"285.5","font-size:13.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Group key",[625,664,665],{"x":660,"y":633,"style":634},"region + month",[42,667],{"x1":668,"y1":639,"x2":669,"y2":639,"stroke":641,"style":612},"367.0","389.0",[643,671],{"points":672,"fill":646},"393.0,62.5 384.0,67.5 384.0,57.5",[597,674],{"x":675,"y":606,"width":607,"height":608,"rx":609,"fill":676,"stroke":677,"style":612},"398.0","#d9f4f1","var(--teal,#0f9488)",[614,679,681],{"transform":680},"translate(462.5,29.0) scale(1.000)",[618,682],{"d":683,"fill":621,"stroke":684,"style":623},"M12 3l9 5-9 5-9-5z M3 13l9 5 9-5 M3 17l9 5 9-5","var(--teal-ink,#0b6157)",[625,686,689],{"x":687,"y":628,"style":688},"474.5","font-size:13.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","Update totals",[625,691,692],{"x":687,"y":633,"style":634},"count, sum, min, max",[42,694],{"x1":695,"y1":639,"x2":696,"y2":639,"stroke":641,"style":612},"556.0","578.0",[643,698],{"points":699,"fill":646},"582.0,62.5 573.0,67.5 573.0,57.5",[597,701],{"x":702,"y":606,"width":607,"height":608,"rx":609,"fill":703,"stroke":704,"style":612},"587.0","#fdefd8","var(--gold,#b4740a)",[614,706,708],{"transform":707},"translate(651.5,29.0) scale(1.000)",[618,709],{"d":710,"fill":621,"stroke":711,"style":623},"M5 12.5l4.5 4.5L19 7","var(--gold-ink,#7a4e06)",[625,713,716],{"x":714,"y":628,"style":715},"663.5","font-size:13.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","Forget the row",[625,718,719],{"x":714,"y":633,"style":634},"memory ∝ groups",[25,721,723],{"id":722},"step-2-the-same-loop-over-calamine-faster","Step 2 — The same loop over calamine, faster",[10,725,726,727,730,731,734],{},"openpyxl parses XML in Python; python-calamine parses it in Rust and is much faster. Its ",[37,728,729],{},"iter_rows"," also yields one row at a time, so the same ",[37,732,733],{},"aggregate"," function works:",[30,736,738],{"className":76,"code":737,"language":78,"meta":35,"style":35},"import time\nfrom python_calamine import CalamineWorkbook\n\nt0 = time.perf_counter()\nwb = load_workbook(\"sales_big.xlsx\", read_only=True)\nrows = wb[\"Sales\"].iter_rows(values_only=True)\ng1 = aggregate(rows, list(next(rows)))\nwb.close()\nt_openpyxl = time.perf_counter() - t0\n\nt0 = time.perf_counter()\nsheet = CalamineWorkbook.from_path(\"sales_big.xlsx\").get_sheet_by_name(\"Sales\")\nrows = sheet.iter_rows()\ng2 = aggregate(rows, list(next(rows)))\nt_calamine = time.perf_counter() - t0\n\nsame = all(abs(g1[k][\"revenue\"] - g2[k][\"revenue\"]) \u003C 1e-6 and g1[k][\"orders\"] == g2[k][\"orders\"] for k in g1)\nprint(f\"openpyxl read-only: {t_openpyxl:.1f}s | calamine: {t_calamine:.1f}s | same totals: {same}\")\n",[37,739,740,747,759,763,773,793,813,833,837,853,857,865,884,893,910,923,927,994],{"__ignoreMap":35},[40,741,742,744],{"class":42,"line":43},[40,743,93],{"class":85},[40,745,746],{"class":89}," time\n",[40,748,749,751,754,756],{"class":42,"line":99},[40,750,86],{"class":85},[40,752,753],{"class":89}," python_calamine ",[40,755,93],{"class":85},[40,757,758],{"class":89}," CalamineWorkbook\n",[40,760,761],{"class":42,"line":112},[40,762,116],{"emptyLinePlaceholder":115},[40,764,765,768,770],{"class":42,"line":119},[40,766,767],{"class":89},"t0 ",[40,769,227],{"class":85},[40,771,772],{"class":89}," time.perf_counter()\n",[40,774,775,777,779,781,783,785,787,789,791],{"class":42,"line":132},[40,776,444],{"class":89},[40,778,227],{"class":85},[40,780,449],{"class":89},[40,782,452],{"class":50},[40,784,151],{"class":89},[40,786,458],{"class":457},[40,788,227],{"class":85},[40,790,463],{"class":147},[40,792,466],{"class":89},[40,794,795,797,799,801,803,805,807,809,811],{"class":42,"line":205},[40,796,472],{"class":89},[40,798,227],{"class":85},[40,800,477],{"class":89},[40,802,480],{"class":50},[40,804,483],{"class":89},[40,806,486],{"class":457},[40,808,227],{"class":85},[40,810,463],{"class":147},[40,812,466],{"class":89},[40,814,815,818,820,823,826,828,830],{"class":42,"line":210},[40,816,817],{"class":89},"g1 ",[40,819,227],{"class":85},[40,821,822],{"class":89}," aggregate(rows, ",[40,824,825],{"class":147},"list",[40,827,181],{"class":89},[40,829,508],{"class":147},[40,831,832],{"class":89},"(rows)))\n",[40,834,835],{"class":42,"line":221},[40,836,528],{"class":89},[40,838,839,842,844,847,850],{"class":42,"line":245},[40,840,841],{"class":89},"t_openpyxl ",[40,843,227],{"class":85},[40,845,846],{"class":89}," time.perf_counter() ",[40,848,849],{"class":85},"-",[40,851,852],{"class":89}," t0\n",[40,854,855],{"class":42,"line":256},[40,856,116],{"emptyLinePlaceholder":115},[40,858,859,861,863],{"class":42,"line":270},[40,860,767],{"class":89},[40,862,227],{"class":85},[40,864,772],{"class":89},[40,866,867,870,872,875,877,880,882],{"class":42,"line":299},[40,868,869],{"class":89},"sheet ",[40,871,227],{"class":85},[40,873,874],{"class":89}," CalamineWorkbook.from_path(",[40,876,452],{"class":50},[40,878,879],{"class":89},").get_sheet_by_name(",[40,881,480],{"class":50},[40,883,466],{"class":89},[40,885,886,888,890],{"class":42,"line":327},[40,887,472],{"class":89},[40,889,227],{"class":85},[40,891,892],{"class":89}," sheet.iter_rows()\n",[40,894,895,898,900,902,904,906,908],{"class":42,"line":338},[40,896,897],{"class":89},"g2 ",[40,899,227],{"class":85},[40,901,822],{"class":89},[40,903,825],{"class":147},[40,905,181],{"class":89},[40,907,508],{"class":147},[40,909,832],{"class":89},[40,911,912,915,917,919,921],{"class":42,"line":355},[40,913,914],{"class":89},"t_calamine ",[40,916,227],{"class":85},[40,918,846],{"class":89},[40,920,849],{"class":85},[40,922,852],{"class":89},[40,924,925],{"class":42,"line":372},[40,926,116],{"emptyLinePlaceholder":115},[40,928,929,932,934,937,939,942,945,947,949,951,954,956,959,962,965,968,971,973,975,978,980,982,984,986,989,991],{"class":42,"line":386},[40,930,931],{"class":89},"same ",[40,933,227],{"class":85},[40,935,936],{"class":147}," all",[40,938,181],{"class":89},[40,940,941],{"class":147},"abs",[40,943,944],{"class":89},"(g1[k][",[40,946,163],{"class":50},[40,948,346],{"class":89},[40,950,849],{"class":85},[40,952,953],{"class":89}," g2[k][",[40,955,163],{"class":50},[40,957,958],{"class":89},"]) ",[40,960,961],{"class":85},"\u003C",[40,963,964],{"class":147}," 1e-6",[40,966,967],{"class":85}," and",[40,969,970],{"class":89}," g1[k][",[40,972,141],{"class":50},[40,974,346],{"class":89},[40,976,977],{"class":85},"==",[40,979,953],{"class":89},[40,981,141],{"class":50},[40,983,346],{"class":89},[40,985,233],{"class":85},[40,987,988],{"class":89}," k ",[40,990,239],{"class":85},[40,992,993],{"class":89}," g1)\n",[40,995,996,998,1000,1003,1006,1010,1013,1016,1019,1022,1024,1027,1029,1031,1034,1036,1039,1041,1044],{"class":42,"line":408},[40,997,534],{"class":147},[40,999,181],{"class":89},[40,1001,1002],{"class":85},"f",[40,1004,1005],{"class":50},"\"openpyxl read-only: ",[40,1007,1009],{"class":1008},"sSjpA","{",[40,1011,1012],{"class":89},"t_openpyxl",[40,1014,1015],{"class":85},":.1f",[40,1017,1018],{"class":1008},"}",[40,1020,1021],{"class":50},"s | calamine: ",[40,1023,1009],{"class":1008},[40,1025,1026],{"class":89},"t_calamine",[40,1028,1015],{"class":85},[40,1030,1018],{"class":1008},[40,1032,1033],{"class":50},"s | same totals: ",[40,1035,1009],{"class":1008},[40,1037,1038],{"class":89},"same",[40,1040,1018],{"class":1008},[40,1042,1043],{"class":50},"\"",[40,1045,466],{"class":89},[10,1047,1048,1049,1052,1053,1056,1057,1060,1061,1064,1065,1067],{},"Both readers give identical totals; calamine took about 1 second against openpyxl's 8 on this file. Two details differ: calamine returns all numbers as floats (so quantities arrive as ",[37,1050,1051],{},"3.0","), and dates as ",[37,1054,1055],{},"date"," or ",[37,1058,1059],{},"datetime"," objects depending on the cell, which ",[37,1062,1063],{},"strftime"," handles either way. Note that calamine's ",[37,1066,729],{}," still parses the sheet up front in Rust — it is faster, but its memory use is closer to the size of the sheet's values than openpyxl's strictly row-by-row reading, so for the very largest files on the most constrained machines, openpyxl read-only remains the lowest-memory option.",[25,1069,1071],{"id":1070},"step-3-compare-with-pandas-on-time-and-memory","Step 3 — Compare with pandas, on time and memory",[10,1073,1074],{},"To see what streaming buys, measure peak Python memory for the pandas approach and the streaming one:",[30,1076,1078],{"className":76,"code":1077,"language":78,"meta":35,"style":35},"import tracemalloc\nimport pandas as pd\n\ndef peak(fn):\n    tracemalloc.start()\n    out = fn()\n    p = tracemalloc.get_traced_memory()[1]\n    tracemalloc.stop()\n    return out, round(p \u002F 2**20, 1)\n\ndef with_pandas():\n    df = pd.read_excel(\"sales_big.xlsx\")\n    df[\"Revenue\"] = df[\"Qty\"] * df[\"Unit price\"]\n    return df.groupby([df[\"Region\"], df[\"Date\"].dt.strftime(\"%Y-%m\")]).agg(orders=(\"Qty\", \"size\"), revenue=(\"Revenue\", \"sum\"))\n\ndef streamed():\n    wb = load_workbook(\"sales_big.xlsx\", read_only=True)\n    rows = wb[\"Sales\"].iter_rows(values_only=True)\n    out = aggregate(rows, list(next(rows)))\n    wb.close()\n    return out\n\npd_result, pd_mb = peak(with_pandas)\nst_result, st_mb = peak(streamed)\ndiff = max(abs(pd_result.loc[k, \"revenue\"] - v[\"revenue\"]) for k, v in st_result.items())\nprint(f\"pandas peak {pd_mb} MB | streamed peak {st_mb} MB | largest revenue difference {diff:.6f}\")\n",[37,1079,1080,1087,1100,1104,1114,1119,1129,1145,1150,1181,1185,1194,1208,1235,1289,1293,1302,1323,1344,1360,1365,1372,1376,1386,1396,1435],{"__ignoreMap":35},[40,1081,1082,1084],{"class":42,"line":43},[40,1083,93],{"class":85},[40,1085,1086],{"class":89}," tracemalloc\n",[40,1088,1089,1091,1094,1097],{"class":42,"line":99},[40,1090,93],{"class":85},[40,1092,1093],{"class":89}," pandas ",[40,1095,1096],{"class":85},"as",[40,1098,1099],{"class":89}," pd\n",[40,1101,1102],{"class":42,"line":112},[40,1103,116],{"emptyLinePlaceholder":115},[40,1105,1106,1108,1111],{"class":42,"line":119},[40,1107,122],{"class":85},[40,1109,1110],{"class":125}," peak",[40,1112,1113],{"class":89},"(fn):\n",[40,1115,1116],{"class":42,"line":132},[40,1117,1118],{"class":89},"    tracemalloc.start()\n",[40,1120,1121,1124,1126],{"class":42,"line":205},[40,1122,1123],{"class":89},"    out ",[40,1125,227],{"class":85},[40,1127,1128],{"class":89}," fn()\n",[40,1130,1131,1134,1136,1139,1142],{"class":42,"line":210},[40,1132,1133],{"class":89},"    p ",[40,1135,227],{"class":85},[40,1137,1138],{"class":89}," tracemalloc.get_traced_memory()[",[40,1140,1141],{"class":147},"1",[40,1143,1144],{"class":89},"]\n",[40,1146,1147],{"class":42,"line":221},[40,1148,1149],{"class":89},"    tracemalloc.stop()\n",[40,1151,1152,1154,1157,1160,1163,1166,1169,1172,1175,1177,1179],{"class":42,"line":245},[40,1153,135],{"class":85},[40,1155,1156],{"class":89}," out, ",[40,1158,1159],{"class":147},"round",[40,1161,1162],{"class":89},"(p ",[40,1164,1165],{"class":85},"\u002F",[40,1167,1168],{"class":147}," 2",[40,1170,1171],{"class":85},"**",[40,1173,1174],{"class":147},"20",[40,1176,151],{"class":89},[40,1178,1141],{"class":147},[40,1180,466],{"class":89},[40,1182,1183],{"class":42,"line":256},[40,1184,116],{"emptyLinePlaceholder":115},[40,1186,1187,1189,1192],{"class":42,"line":270},[40,1188,122],{"class":85},[40,1190,1191],{"class":125}," with_pandas",[40,1193,129],{"class":89},[40,1195,1196,1199,1201,1204,1206],{"class":42,"line":299},[40,1197,1198],{"class":89},"    df ",[40,1200,227],{"class":85},[40,1202,1203],{"class":89}," pd.read_excel(",[40,1205,452],{"class":50},[40,1207,466],{"class":89},[40,1209,1210,1213,1216,1218,1220,1223,1225,1227,1229,1231,1233],{"class":42,"line":327},[40,1211,1212],{"class":89},"    df[",[40,1214,1215],{"class":50},"\"Revenue\"",[40,1217,346],{"class":89},[40,1219,227],{"class":85},[40,1221,1222],{"class":89}," df[",[40,1224,310],{"class":50},[40,1226,346],{"class":89},[40,1228,316],{"class":85},[40,1230,1222],{"class":89},[40,1232,321],{"class":50},[40,1234,1144],{"class":89},[40,1236,1237,1239,1242,1244,1247,1249,1252,1254,1257,1260,1262,1264,1266,1268,1271,1273,1276,1278,1280,1282,1284,1287],{"class":42,"line":338},[40,1238,135],{"class":85},[40,1240,1241],{"class":89}," df.groupby([df[",[40,1243,281],{"class":50},[40,1245,1246],{"class":89},"], df[",[40,1248,287],{"class":50},[40,1250,1251],{"class":89},"].dt.strftime(",[40,1253,293],{"class":50},[40,1255,1256],{"class":89},")]).agg(",[40,1258,1259],{"class":457},"orders",[40,1261,227],{"class":85},[40,1263,181],{"class":89},[40,1265,310],{"class":50},[40,1267,151],{"class":89},[40,1269,1270],{"class":50},"\"size\"",[40,1272,187],{"class":89},[40,1274,1275],{"class":457},"revenue",[40,1277,227],{"class":85},[40,1279,181],{"class":89},[40,1281,1215],{"class":50},[40,1283,151],{"class":89},[40,1285,1286],{"class":50},"\"sum\"",[40,1288,296],{"class":89},[40,1290,1291],{"class":42,"line":355},[40,1292,116],{"emptyLinePlaceholder":115},[40,1294,1295,1297,1300],{"class":42,"line":372},[40,1296,122],{"class":85},[40,1298,1299],{"class":125}," streamed",[40,1301,129],{"class":89},[40,1303,1304,1307,1309,1311,1313,1315,1317,1319,1321],{"class":42,"line":386},[40,1305,1306],{"class":89},"    wb ",[40,1308,227],{"class":85},[40,1310,449],{"class":89},[40,1312,452],{"class":50},[40,1314,151],{"class":89},[40,1316,458],{"class":457},[40,1318,227],{"class":85},[40,1320,463],{"class":147},[40,1322,466],{"class":89},[40,1324,1325,1328,1330,1332,1334,1336,1338,1340,1342],{"class":42,"line":408},[40,1326,1327],{"class":89},"    rows ",[40,1329,227],{"class":85},[40,1331,477],{"class":89},[40,1333,480],{"class":50},[40,1335,483],{"class":89},[40,1337,486],{"class":457},[40,1339,227],{"class":85},[40,1341,463],{"class":147},[40,1343,466],{"class":89},[40,1345,1346,1348,1350,1352,1354,1356,1358],{"class":42,"line":428},[40,1347,1123],{"class":89},[40,1349,227],{"class":85},[40,1351,822],{"class":89},[40,1353,825],{"class":147},[40,1355,181],{"class":89},[40,1357,508],{"class":147},[40,1359,832],{"class":89},[40,1361,1362],{"class":42,"line":436},[40,1363,1364],{"class":89},"    wb.close()\n",[40,1366,1367,1369],{"class":42,"line":441},[40,1368,135],{"class":85},[40,1370,1371],{"class":89}," out\n",[40,1373,1374],{"class":42,"line":469},[40,1375,116],{"emptyLinePlaceholder":115},[40,1377,1378,1381,1383],{"class":42,"line":495},[40,1379,1380],{"class":89},"pd_result, pd_mb ",[40,1382,227],{"class":85},[40,1384,1385],{"class":89}," peak(with_pandas)\n",[40,1387,1388,1391,1393],{"class":42,"line":514},[40,1389,1390],{"class":89},"st_result, st_mb ",[40,1392,227],{"class":85},[40,1394,1395],{"class":89}," peak(streamed)\n",[40,1397,1398,1401,1403,1405,1407,1409,1412,1414,1416,1418,1421,1423,1425,1427,1430,1432],{"class":42,"line":525},[40,1399,1400],{"class":89},"diff ",[40,1402,227],{"class":85},[40,1404,397],{"class":147},[40,1406,181],{"class":89},[40,1408,941],{"class":147},[40,1410,1411],{"class":89},"(pd_result.loc[k, ",[40,1413,163],{"class":50},[40,1415,346],{"class":89},[40,1417,849],{"class":85},[40,1419,1420],{"class":89}," v[",[40,1422,163],{"class":50},[40,1424,958],{"class":89},[40,1426,233],{"class":85},[40,1428,1429],{"class":89}," k, v ",[40,1431,239],{"class":85},[40,1433,1434],{"class":89}," st_result.items())\n",[40,1436,1437,1439,1441,1443,1446,1448,1451,1453,1456,1458,1461,1463,1466,1468,1471,1474,1476,1478],{"class":42,"line":531},[40,1438,534],{"class":147},[40,1440,181],{"class":89},[40,1442,1002],{"class":85},[40,1444,1445],{"class":50},"\"pandas peak ",[40,1447,1009],{"class":1008},[40,1449,1450],{"class":89},"pd_mb",[40,1452,1018],{"class":1008},[40,1454,1455],{"class":50}," MB | streamed peak ",[40,1457,1009],{"class":1008},[40,1459,1460],{"class":89},"st_mb",[40,1462,1018],{"class":1008},[40,1464,1465],{"class":50}," MB | largest revenue difference ",[40,1467,1009],{"class":1008},[40,1469,1470],{"class":89},"diff",[40,1472,1473],{"class":85},":.6f",[40,1475,1018],{"class":1008},[40,1477,1043],{"class":50},[40,1479,466],{"class":89},[10,1481,1482],{},"The pandas version peaked at about 111 MB for this file — the whole DataFrame plus openpyxl's working memory while reading — and grows in proportion to the rows. The streaming version peaked at about 24 MB, which is mostly openpyxl's own parsing buffers plus the 36 accumulators, and it does not grow with the file. The totals agree to floating-point rounding. Streaming is not faster than pandas with the same reader — pandas also has to parse every row — so the reason to stream is memory, not speed.",[578,1484,587,1490,587,1493,587,1496,587,1500,587,1506,587,1512,587,1519,587,1524,587,1527,587,1530,587,1533],{"viewBox":1485,"role":581,"ariaLabelledBy":1486,"xmlns":585,"style":1489},"4 0 744 100",[1487,1488],"agg-mem-t","agg-mem-d","width:100%;max-width:744px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[589,1491,1492],{"id":1487},"Peak Python memory aggregating 300,000 rows",[593,1494,1495],{"id":1488},"Reading the whole sheet with pandas and grouping peaked at about 111 megabytes in this run. The streaming group-by over openpyxl read-only rows peaked at about 24 megabytes, mostly the reader's own buffers, because only one row and one accumulator per group are held.",[597,1497],{"x":599,"y":148,"width":1498,"height":1499,"fill":602},"744","100",[597,1501],{"x":605,"y":606,"width":1502,"height":1503,"rx":1504,"fill":610,"stroke":611,"style":1505},"217.7","28.0","7","stroke-width:1px",[625,1507,1511],{"x":1508,"y":1509,"style":1510},"128.8","34.5","font-size:12.5px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","pandas read_excel + groupby",[597,1513],{"x":1514,"y":606,"width":1515,"height":1503,"rx":1504,"fill":1516,"stroke":1517,"style":1518},"247.7","317.5","#fce9e9","var(--danger,#dc2626)","stroke-width:1.5px",[625,1520,1523],{"x":1521,"y":1509,"style":1522},"573.1","font-size:12px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:start","≈ 111 MB, grows with rows",[597,1525],{"x":605,"y":1526,"width":1502,"height":1503,"rx":1504,"fill":610,"stroke":611,"style":1505},"56.0",[625,1528,16],{"x":1508,"y":1529,"style":1510},"74.5",[597,1531],{"x":1514,"y":1526,"width":1532,"height":1503,"rx":1504,"fill":676,"stroke":677,"style":1518},"68.0",[625,1534,1537],{"x":1535,"y":1529,"style":1536},"323.7","font-size:12px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:start","≈ 24 MB, flat",[25,1539,1541],{"id":1540},"step-4-statistics-that-need-more-than-running-totals","Step 4 — Statistics that need more than running totals",[10,1543,1544],{},"Medians, percentiles and distinct counts cannot be updated from a few numbers per group. Two practical options keep memory bounded:",[30,1546,1548],{"className":76,"code":1547,"language":78,"meta":35,"style":35},"from collections import Counter\n\ndef median_from_counts(counter):\n    total = sum(counter.values())\n    values = sorted(counter)\n    def nth(k):                                   # k-th smallest value, 0-based\n        running = 0\n        for v in values:\n            running += counter[v]\n            if running > k:\n                return v\n    if total % 2:\n        return float(nth(total \u002F\u002F 2))\n    return (nth(total \u002F\u002F 2 - 1) + nth(total \u002F\u002F 2)) \u002F 2\n\nqty_counts = defaultdict(Counter)\nproducts_seen = defaultdict(set)\nwb = load_workbook(\"sales_big.xlsx\", read_only=True)\nrows = wb[\"Sales\"].iter_rows(values_only=True)\nh = list(next(rows))\nfor r in rows:\n    qty_counts[r[h.index(\"Region\")]][r[h.index(\"Qty\")]] += 1\n    products_seen[r[h.index(\"Region\")]].add(r[h.index(\"Product\")])\nwb.close()\ncheck = pd.read_excel(\"sales_big.xlsx\", usecols=[\"Region\", \"Qty\"]).groupby(\"Region\")[\"Qty\"].median()\nfor region in sorted(qty_counts):\n    print(region, \"median qty\", median_from_counts(qty_counts[region]), \"| pandas\", check[region],\n          \"| distinct products\", len(products_seen[region]))\n",[37,1549,1550,1561,1565,1575,1588,1601,1616,1626,1639,1649,1663,1671,1687,1705,1743,1747,1757,1772,1792,1812,1827,1837,1856,1872,1876,1916,1930,1950],{"__ignoreMap":35},[40,1551,1552,1554,1556,1558],{"class":42,"line":43},[40,1553,86],{"class":85},[40,1555,90],{"class":89},[40,1557,93],{"class":85},[40,1559,1560],{"class":89}," Counter\n",[40,1562,1563],{"class":42,"line":99},[40,1564,116],{"emptyLinePlaceholder":115},[40,1566,1567,1569,1572],{"class":42,"line":112},[40,1568,122],{"class":85},[40,1570,1571],{"class":125}," median_from_counts",[40,1573,1574],{"class":89},"(counter):\n",[40,1576,1577,1580,1582,1585],{"class":42,"line":119},[40,1578,1579],{"class":89},"    total ",[40,1581,227],{"class":85},[40,1583,1584],{"class":147}," sum",[40,1586,1587],{"class":89},"(counter.values())\n",[40,1589,1590,1593,1595,1598],{"class":42,"line":132},[40,1591,1592],{"class":89},"    values ",[40,1594,227],{"class":85},[40,1596,1597],{"class":147}," sorted",[40,1599,1600],{"class":89},"(counter)\n",[40,1602,1603,1606,1609,1612],{"class":42,"line":205},[40,1604,1605],{"class":85},"    def",[40,1607,1608],{"class":125}," nth",[40,1610,1611],{"class":89},"(k):                                   ",[40,1613,1615],{"class":1614},"s-wDw","# k-th smallest value, 0-based\n",[40,1617,1618,1621,1623],{"class":42,"line":210},[40,1619,1620],{"class":89},"        running ",[40,1622,227],{"class":85},[40,1624,1625],{"class":147}," 0\n",[40,1627,1628,1631,1634,1636],{"class":42,"line":221},[40,1629,1630],{"class":85},"        for",[40,1632,1633],{"class":89}," v ",[40,1635,239],{"class":85},[40,1637,1638],{"class":89}," values:\n",[40,1640,1641,1644,1646],{"class":42,"line":245},[40,1642,1643],{"class":89},"            running ",[40,1645,349],{"class":85},[40,1647,1648],{"class":89}," counter[v]\n",[40,1650,1651,1654,1657,1660],{"class":42,"line":256},[40,1652,1653],{"class":85},"            if",[40,1655,1656],{"class":89}," running ",[40,1658,1659],{"class":85},">",[40,1661,1662],{"class":89}," k:\n",[40,1664,1665,1668],{"class":42,"line":270},[40,1666,1667],{"class":85},"                return",[40,1669,1670],{"class":89}," v\n",[40,1672,1673,1676,1679,1682,1684],{"class":42,"line":299},[40,1674,1675],{"class":85},"    if",[40,1677,1678],{"class":89}," total ",[40,1680,1681],{"class":85},"%",[40,1683,1168],{"class":147},[40,1685,1686],{"class":89},":\n",[40,1688,1689,1692,1695,1698,1701,1703],{"class":42,"line":327},[40,1690,1691],{"class":85},"        return",[40,1693,1694],{"class":147}," float",[40,1696,1697],{"class":89},"(nth(total ",[40,1699,1700],{"class":85},"\u002F\u002F",[40,1702,1168],{"class":147},[40,1704,296],{"class":89},[40,1706,1707,1709,1712,1714,1716,1719,1722,1725,1728,1731,1733,1735,1738,1740],{"class":42,"line":338},[40,1708,135],{"class":85},[40,1710,1711],{"class":89}," (nth(total ",[40,1713,1700],{"class":85},[40,1715,1168],{"class":147},[40,1717,1718],{"class":85}," -",[40,1720,1721],{"class":147}," 1",[40,1723,1724],{"class":89},") ",[40,1726,1727],{"class":85},"+",[40,1729,1730],{"class":89}," nth(total ",[40,1732,1700],{"class":85},[40,1734,1168],{"class":147},[40,1736,1737],{"class":89},")) ",[40,1739,1165],{"class":85},[40,1741,1742],{"class":147}," 2\n",[40,1744,1745],{"class":42,"line":355},[40,1746,116],{"emptyLinePlaceholder":115},[40,1748,1749,1752,1754],{"class":42,"line":372},[40,1750,1751],{"class":89},"qty_counts ",[40,1753,227],{"class":85},[40,1755,1756],{"class":89}," defaultdict(Counter)\n",[40,1758,1759,1762,1764,1767,1770],{"class":42,"line":386},[40,1760,1761],{"class":89},"products_seen ",[40,1763,227],{"class":85},[40,1765,1766],{"class":89}," defaultdict(",[40,1768,1769],{"class":147},"set",[40,1771,466],{"class":89},[40,1773,1774,1776,1778,1780,1782,1784,1786,1788,1790],{"class":42,"line":408},[40,1775,444],{"class":89},[40,1777,227],{"class":85},[40,1779,449],{"class":89},[40,1781,452],{"class":50},[40,1783,151],{"class":89},[40,1785,458],{"class":457},[40,1787,227],{"class":85},[40,1789,463],{"class":147},[40,1791,466],{"class":89},[40,1793,1794,1796,1798,1800,1802,1804,1806,1808,1810],{"class":42,"line":428},[40,1795,472],{"class":89},[40,1797,227],{"class":85},[40,1799,477],{"class":89},[40,1801,480],{"class":50},[40,1803,483],{"class":89},[40,1805,486],{"class":457},[40,1807,227],{"class":85},[40,1809,463],{"class":147},[40,1811,466],{"class":89},[40,1813,1814,1817,1819,1821,1823,1825],{"class":42,"line":436},[40,1815,1816],{"class":89},"h ",[40,1818,227],{"class":85},[40,1820,503],{"class":147},[40,1822,181],{"class":89},[40,1824,508],{"class":147},[40,1826,511],{"class":89},[40,1828,1829,1831,1833,1835],{"class":42,"line":441},[40,1830,233],{"class":85},[40,1832,262],{"class":89},[40,1834,239],{"class":85},[40,1836,267],{"class":89},[40,1838,1839,1842,1844,1847,1849,1852,1854],{"class":42,"line":469},[40,1840,1841],{"class":89},"    qty_counts[r[h.index(",[40,1843,281],{"class":50},[40,1845,1846],{"class":89},")]][r[h.index(",[40,1848,310],{"class":50},[40,1850,1851],{"class":89},")]] ",[40,1853,349],{"class":85},[40,1855,352],{"class":147},[40,1857,1858,1861,1863,1866,1869],{"class":42,"line":495},[40,1859,1860],{"class":89},"    products_seen[r[h.index(",[40,1862,281],{"class":50},[40,1864,1865],{"class":89},")]].add(r[h.index(",[40,1867,1868],{"class":50},"\"Product\"",[40,1870,1871],{"class":89},")])\n",[40,1873,1874],{"class":42,"line":514},[40,1875,528],{"class":89},[40,1877,1878,1881,1883,1885,1887,1889,1892,1894,1897,1899,1901,1903,1906,1908,1911,1913],{"class":42,"line":525},[40,1879,1880],{"class":89},"check ",[40,1882,227],{"class":85},[40,1884,1203],{"class":89},[40,1886,452],{"class":50},[40,1888,151],{"class":89},[40,1890,1891],{"class":457},"usecols",[40,1893,227],{"class":85},[40,1895,1896],{"class":89},"[",[40,1898,281],{"class":50},[40,1900,151],{"class":89},[40,1902,310],{"class":50},[40,1904,1905],{"class":89},"]).groupby(",[40,1907,281],{"class":50},[40,1909,1910],{"class":89},")[",[40,1912,310],{"class":50},[40,1914,1915],{"class":89},"].median()\n",[40,1917,1918,1920,1923,1925,1927],{"class":42,"line":531},[40,1919,233],{"class":85},[40,1921,1922],{"class":89}," region ",[40,1924,239],{"class":85},[40,1926,1597],{"class":147},[40,1928,1929],{"class":89},"(qty_counts):\n",[40,1931,1932,1935,1938,1941,1944,1947],{"class":42,"line":550},[40,1933,1934],{"class":147},"    print",[40,1936,1937],{"class":89},"(region, ",[40,1939,1940],{"class":50},"\"median qty\"",[40,1942,1943],{"class":89},", median_from_counts(qty_counts[region]), ",[40,1945,1946],{"class":50},"\"| pandas\"",[40,1948,1949],{"class":89},", check[region],\n",[40,1951,1953,1956,1958,1960],{"class":42,"line":1952},28,[40,1954,1955],{"class":50},"          \"| distinct products\"",[40,1957,151],{"class":89},[40,1959,539],{"class":147},[40,1961,1962],{"class":89},"(products_seen[region]))\n",[10,1964,1965,1966,1969,1970,1973],{},"When the values are discrete — quantities, ratings, whole-euro amounts — a ",[37,1967,1968],{},"Counter"," of value frequencies per group is small (here eleven possible quantities) and gives an ",[14,1971,1972],{},"exact"," median or any percentile. For distinct counts, a set of seen values per group works while the number of distinct values is modest, as for products here; for millions of distinct customer IDs, store compact hashes, or accept an approximate algorithm such as HyperLogLog. For continuous values with no natural rounding, an exact streaming median needs either a second pass or an approximate sketch — or simply a sample, since a median from a random 10% of a large file is usually accurate enough for a report. The medians here match pandas exactly.",[25,1975,1977],{"id":1976},"step-5-write-the-summary","Step 5 — Write the summary",[10,1979,1980],{},"The aggregated result is tiny, so pandas is the right tool from here on:",[30,1982,1984],{"className":76,"code":1983,"language":78,"meta":35,"style":35},"summary = pd.DataFrame([\n    {\"Region\": region, \"Month\": month, \"Orders\": a[\"orders\"], \"Units\": a[\"units\"],\n     \"Revenue\": round(a[\"revenue\"], 2), \"Avg per order\": round(a[\"revenue\"] \u002F a[\"orders\"], 2),\n     \"Largest line\": a[\"max_line\"]}\n    for (region, month), a in sorted(groups.items())\n])\nsummary.to_excel(\"sales_summary.xlsx\", index=False)\nprint(summary.head(4).to_string(index=False))\nprint(\"summary rows:\", len(summary), \"| grand total revenue:\", round(summary[\"Revenue\"].sum(), 2))\n",[37,1985,1986,1996,2033,2080,2092,2106,2110,2130,2150],{"__ignoreMap":35},[40,1987,1988,1991,1993],{"class":42,"line":43},[40,1989,1990],{"class":89},"summary ",[40,1992,227],{"class":85},[40,1994,1995],{"class":89}," pd.DataFrame([\n",[40,1997,1998,2001,2003,2006,2009,2012,2015,2018,2020,2023,2026,2028,2030],{"class":42,"line":99},[40,1999,2000],{"class":89},"    {",[40,2002,281],{"class":50},[40,2004,2005],{"class":89},": region, ",[40,2007,2008],{"class":50},"\"Month\"",[40,2010,2011],{"class":89},": month, ",[40,2013,2014],{"class":50},"\"Orders\"",[40,2016,2017],{"class":89},": a[",[40,2019,141],{"class":50},[40,2021,2022],{"class":89},"], ",[40,2024,2025],{"class":50},"\"Units\"",[40,2027,2017],{"class":89},[40,2029,154],{"class":50},[40,2031,2032],{"class":89},"],\n",[40,2034,2035,2038,2040,2042,2044,2046,2048,2051,2053,2056,2058,2060,2062,2064,2066,2068,2071,2073,2075,2077],{"class":42,"line":112},[40,2036,2037],{"class":50},"     \"Revenue\"",[40,2039,144],{"class":89},[40,2041,1159],{"class":147},[40,2043,400],{"class":89},[40,2045,163],{"class":50},[40,2047,2022],{"class":89},[40,2049,2050],{"class":147},"2",[40,2052,187],{"class":89},[40,2054,2055],{"class":50},"\"Avg per order\"",[40,2057,144],{"class":89},[40,2059,1159],{"class":147},[40,2061,400],{"class":89},[40,2063,163],{"class":50},[40,2065,346],{"class":89},[40,2067,1165],{"class":85},[40,2069,2070],{"class":89}," a[",[40,2072,141],{"class":50},[40,2074,2022],{"class":89},[40,2076,2050],{"class":147},[40,2078,2079],{"class":89},"),\n",[40,2081,2082,2085,2087,2089],{"class":42,"line":119},[40,2083,2084],{"class":50},"     \"Largest line\"",[40,2086,2017],{"class":89},[40,2088,173],{"class":50},[40,2090,2091],{"class":89},"]}\n",[40,2093,2094,2096,2099,2101,2103],{"class":42,"line":132},[40,2095,259],{"class":85},[40,2097,2098],{"class":89}," (region, month), a ",[40,2100,239],{"class":85},[40,2102,1597],{"class":147},[40,2104,2105],{"class":89},"(groups.items())\n",[40,2107,2108],{"class":42,"line":205},[40,2109,565],{"class":89},[40,2111,2112,2115,2118,2120,2123,2125,2128],{"class":42,"line":210},[40,2113,2114],{"class":89},"summary.to_excel(",[40,2116,2117],{"class":50},"\"sales_summary.xlsx\"",[40,2119,151],{"class":89},[40,2121,2122],{"class":457},"index",[40,2124,227],{"class":85},[40,2126,2127],{"class":147},"False",[40,2129,466],{"class":89},[40,2131,2132,2134,2137,2139,2142,2144,2146,2148],{"class":42,"line":221},[40,2133,534],{"class":147},[40,2135,2136],{"class":89},"(summary.head(",[40,2138,599],{"class":147},[40,2140,2141],{"class":89},").to_string(",[40,2143,2122],{"class":457},[40,2145,227],{"class":85},[40,2147,2127],{"class":147},[40,2149,296],{"class":89},[40,2151,2152,2154,2156,2159,2161,2163,2166,2169,2171,2173,2176,2178,2181,2183],{"class":42,"line":245},[40,2153,534],{"class":147},[40,2155,181],{"class":89},[40,2157,2158],{"class":50},"\"summary rows:\"",[40,2160,151],{"class":89},[40,2162,539],{"class":147},[40,2164,2165],{"class":89},"(summary), ",[40,2167,2168],{"class":50},"\"| grand total revenue:\"",[40,2170,151],{"class":89},[40,2172,1159],{"class":147},[40,2174,2175],{"class":89},"(summary[",[40,2177,1215],{"class":50},[40,2179,2180],{"class":89},"].sum(), ",[40,2182,2050],{"class":147},[40,2184,296],{"class":89},[10,2186,2187,2188,23],{},"Thirty-six rows summarise 300,000; the average per order is computed from the accumulated revenue and count, which is why both were kept. From here, formatting, a pivot by month or a chart work exactly as for any small table. Streaming reads in general are covered in ",[19,2189,2191],{"href":2190},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fspeed-up-openpyxl-with-read-only-mode\u002F","Speed Up openpyxl with Read-Only Mode",[578,2193,587,2198,587,2201,587,2204,587,2207,587,2211,587,2216,587,2222,587,2225,587,2229,587,2232,587,2237,587,2239,587,2243,587,2245,587,2248,587,2251,587,2255,587,2257,587,2261,587,2263,587,2265,587,2268,587,2272,587,2274,587,2278,587,2280,587,2282,587,2285,587,2289,587,2291,587,2295,587,2297,587,2300,587,2303,587,2307,587,2309,587,2313,587,2315],{"viewBox":2194,"role":581,"ariaLabelledBy":2195,"xmlns":585,"style":586},"4 0 752 242",[2196,2197],"agg-stats-t","agg-stats-d",[589,2199,2200],{"id":2196},"Which statistics can be streamed, and how",[593,2202,2203],{"id":2197},"Count, sum, minimum, maximum and mean can be updated row by row with a few numbers per group. Variance needs the running sum of squares or Welford's method. Median and percentiles need value counts for discrete data, a second pass or a sketch. Distinct counts need a set or an approximate sketch.",[597,2205],{"x":599,"y":148,"width":600,"height":2206,"fill":602},"242",[597,2208],{"x":605,"y":606,"width":2209,"height":2210,"rx":148,"fill":602,"stroke":611,"style":1505},"165.9","40.0",[597,2212],{"x":2213,"y":606,"width":2214,"height":2210,"rx":148,"fill":650,"stroke":641,"style":2215},"185.9","277.1","stroke-width:1.2px",[625,2217,2221],{"x":2218,"y":2219,"style":2220},"324.4","41.0","font-size:13px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","kept per group",[597,2223],{"x":2224,"y":606,"width":2214,"height":2210,"rx":148,"fill":650,"stroke":641,"style":2215},"462.9",[625,2226,2228],{"x":2227,"y":2219,"style":2220},"601.5","memory",[597,2230],{"x":605,"y":1526,"width":2209,"height":2231,"rx":148,"fill":610,"stroke":611,"style":1505},"34.0",[625,2233,692],{"x":2234,"y":2235,"style":2236},"32.0","78.0","font-size:12.5px;font-weight:700;fill:var(--text,#172033);text-anchor:start",[597,2238],{"x":2213,"y":1526,"width":2214,"height":2231,"rx":148,"fill":602,"stroke":611,"style":1505},[625,2240,2242],{"x":2218,"y":2241,"style":634},"77.0","running numbers",[597,2244],{"x":2224,"y":1526,"width":2214,"height":2231,"rx":148,"fill":602,"stroke":611,"style":1505},[625,2246,2247],{"x":2227,"y":2241,"style":634},"tiny",[597,2249],{"x":605,"y":2250,"width":2209,"height":2231,"rx":148,"fill":610,"stroke":611,"style":1505},"90.0",[625,2252,2254],{"x":2234,"y":2253,"style":2236},"112.0","mean",[597,2256],{"x":2213,"y":2250,"width":2214,"height":2231,"rx":148,"fill":602,"stroke":611,"style":1505},[625,2258,2260],{"x":2218,"y":2259,"style":634},"111.0","sum + count",[597,2262],{"x":2224,"y":2250,"width":2214,"height":2231,"rx":148,"fill":602,"stroke":611,"style":1505},[625,2264,2247],{"x":2227,"y":2259,"style":634},[597,2266],{"x":605,"y":2267,"width":2209,"height":2231,"rx":148,"fill":610,"stroke":611,"style":1505},"124.0",[625,2269,2271],{"x":2234,"y":2270,"style":2236},"146.0","variance",[597,2273],{"x":2213,"y":2267,"width":2214,"height":2231,"rx":148,"fill":602,"stroke":611,"style":1505},[625,2275,2277],{"x":2218,"y":2276,"style":634},"145.0","Welford (n, mean, M2)",[597,2279],{"x":2224,"y":2267,"width":2214,"height":2231,"rx":148,"fill":602,"stroke":611,"style":1505},[625,2281,2247],{"x":2227,"y":2276,"style":634},[597,2283],{"x":605,"y":2284,"width":2209,"height":2231,"rx":148,"fill":610,"stroke":611,"style":1505},"158.0",[625,2286,2288],{"x":2234,"y":2287,"style":2236},"180.0","median, percentiles",[597,2290],{"x":2213,"y":2284,"width":2214,"height":2231,"rx":148,"fill":602,"stroke":611,"style":1505},[625,2292,2294],{"x":2218,"y":2293,"style":634},"179.0","Counter of values",[597,2296],{"x":2224,"y":2284,"width":2214,"height":2231,"rx":148,"fill":602,"stroke":611,"style":1505},[625,2298,2299],{"x":2227,"y":2293,"style":634},"small if discrete",[597,2301],{"x":605,"y":2302,"width":2209,"height":2231,"rx":148,"fill":610,"stroke":611,"style":1505},"192.0",[625,2304,2306],{"x":2234,"y":2305,"style":2236},"214.0","distinct count",[597,2308],{"x":2213,"y":2302,"width":2214,"height":2231,"rx":148,"fill":602,"stroke":611,"style":1505},[625,2310,2312],{"x":2218,"y":2311,"style":634},"213.0","set or HyperLogLog",[597,2314],{"x":2224,"y":2302,"width":2214,"height":2231,"rx":148,"fill":602,"stroke":611,"style":1505},[625,2316,2317],{"x":2227,"y":2311,"style":634},"grows with distinct values",[25,2319,2321],{"id":2320},"common-pitfalls","Common pitfalls",[2323,2324,2325,2333,2339,2345,2351],"ul",{},[2326,2327,2328,2332],"li",{},[2329,2330,2331],"strong",{},"Loading the whole sheet to compute a few totals."," Stream and accumulate instead.",[2326,2334,2335,2338],{},[2329,2336,2337],{},"Keeping only the mean."," Keep sum and count so averages can be combined and recomputed.",[2326,2340,2341,2344],{},[2329,2342,2343],{},"Assuming medians stream like sums."," Use value counts, a second pass, a sketch or a sample.",[2326,2346,2347,2350],{},[2329,2348,2349],{},"Different number types from different readers."," calamine returns floats for integers.",[2326,2352,2353,2356],{},[2329,2354,2355],{},"Expecting streaming to be faster than pandas."," It saves memory; speed comes from the reader.",[25,2358,2360],{"id":2359},"conclusion","Conclusion",[10,2362,2363,2364,2366],{},"To aggregate an Excel sheet too large to load comfortably, stream it: read one row at a time with openpyxl's read-only mode or python-calamine's ",[37,2365,729],{},", update a dictionary of running totals per group — count, sum, minimum, maximum — and compute averages from sums and counts at the end. Memory then depends on the number of groups, not rows. Medians and distinct counts need a bit more: value counts for discrete data, sets or sketches for distinct values, or a sample. Hand the small result to pandas for formatting and output.",[25,2368,2370],{"id":2369},"frequently-asked-questions","Frequently asked questions",[10,2372,2373,2376],{},[2329,2374,2375],{},"How do I sum an Excel file that is too big for pandas?","\nRead it row by row with openpyxl load_workbook(read_only=True) and iter_rows(values_only=True), adding each row's values to running totals in a dictionary keyed by group.",[10,2378,2379,2382],{},[2329,2380,2381],{},"Can I calculate an average without loading all rows?","\nYes. Keep a running sum and count for each group and divide at the end.",[10,2384,2385,2388],{},[2329,2386,2387],{},"How do I get a median from a streamed Excel file?","\nFor discrete values, count how often each value occurs per group and read the median from the counts. For continuous values, use a second pass, an approximate sketch or a random sample.",[10,2390,2391,2394],{},[2329,2392,2393],{},"Is a streaming aggregation faster than pandas?","\nNot usually, because both must parse every row. Its advantage is memory, which stays small regardless of file size.",[25,2396,2398],{"id":2397},"related","Related",[2323,2400,2401,2407,2414,2421],{},[2326,2402,2403,2404,2406],{},"Up one level: ",[19,2405,22],{"href":21}," — big files, fast code.",[2326,2408,2409,2413],{},[19,2410,2412],{"href":2411},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fread-large-excel-file-in-chunks-with-pandas\u002F","Read a Large Excel File in Chunks with pandas"," — chunks instead of single rows.",[2326,2415,2416,2420],{},[19,2417,2419],{"href":2418},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Freduce-memory-when-reading-excel-with-pandas\u002F","Reduce Memory When Reading Excel with pandas"," — when pandas is still the right tool.",[2326,2422,2423,2427],{},[19,2424,2426],{"href":2425},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcreating-pivot-tables-from-excel-data\u002Fcreate-pivot-table-from-excel-with-pandas\u002F","Create a Pivot Table from Excel with pandas"," — the in-memory equivalent.",[2429,2430,2431],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}",{"title":35,"searchDepth":99,"depth":99,"links":2433},[2434,2435,2436,2437,2438,2439,2440,2441,2442,2443],{"id":27,"depth":99,"text":28},{"id":69,"depth":99,"text":70},{"id":722,"depth":99,"text":723},{"id":1070,"depth":99,"text":1071},{"id":1540,"depth":99,"text":1541},{"id":1976,"depth":99,"text":1977},{"id":2320,"depth":99,"text":2321},{"id":2359,"depth":99,"text":2360},{"id":2369,"depth":99,"text":2370},{"id":2397,"depth":99,"text":2398},"2026-10-02","Sum, count, average and find the minimum and maximum per group in an Excel sheet with hundreds of thousands of rows while holding only one row at a time: a streaming group-by over openpyxl read-only or python-calamine rows, statistics that can and cannot be computed in one pass, an exact median with bounded memory where values are discrete, and a check against pandas on a sample.","md",[2448,2450,2452,2454],{"q":2375,"a":2449},"Read it row by row with openpyxl load_workbook(read_only=True) and iter_rows(values_only=True), adding each row's values to running totals in a dictionary keyed by group.",{"q":2381,"a":2451},"Yes. Keep a running sum and count for each group and divide at the end.",{"q":2387,"a":2453},"For discrete values, count how often each value occurs per group and read the median from the counts. For continuous values, use a second pass, an approximate sketch or a random sample.",{"q":2393,"a":2455},"Not usually, because both must parse every row. Its advantage is memory, which stays small regardless of file size.","aggregate a huge excel file without loading it",null,{"breadcrumb":2459},[2460,2462,2465],{"name":2461,"item":1165},"Home",{"name":2463,"item":2464},"Advanced Data Transformation and Cleaning for Excel Automation","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":22,"item":21},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Faggregate-a-huge-excel-file-without-loading-it",{"title":2468,"description":2469},"Aggregate a Huge Excel File Without Loading It in Python","Streaming group-by over a large xlsx: accumulate sums, counts, min, max and means per group from openpyxl read-only or calamine rows, handle medians and distinct counts, and compare memory with pandas.","aggregate-a-huge-excel-file-without-loading-it","advanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Faggregate-a-huge-excel-file-without-loading-it\u002Findex",[2473,2474],"large-files","pivots-and-summaries","how-to","U1vyhS7GSybQhZ6ulA7G2ScV2RRJ0DeklRDTm5Fv0pc",[2478,2481],{"title":22,"path":2479,"stem":2480,"children":-1},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python","advanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Findex",{"title":2482,"path":2483,"stem":2484,"children":-1},"Convert Excel to CSV with Python","\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fconvert-excel-to-csv-with-python","advanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fconvert-excel-to-csv-with-python\u002Findex",[2486,2488],{"path":2487,"title":2463},"\u002Fadvanced-data-transformation-and-cleaning",{"path":2479,"title":22},[],[2491,2494,2497,2500,2503,2506],{"path":2492,"title":2493,"kind":2475},"\u002Fadvanced-data-transformation-and-cleaning\u002Freading-excel-with-polars-and-arrow\u002Fspeed-up-pandas-excel-reads-with-the-calamine-engine","Speed Up pandas Excel Reads with the calamine Engine",{"path":2495,"title":2496,"kind":2475},"\u002Fautomating-reporting-workflows\u002Ftesting-and-packaging-excel-automation-scripts\u002Fprofile-a-slow-excel-report-script","Profile a Slow Excel Report Script in Python",{"path":2498,"title":2499,"kind":2475},"\u002Fformatting-and-charting-excel-reports-with-python\u002Fbuilding-excel-reports-with-xlsxwriter\u002Fwrite-a-million-rows-to-excel-with-xlsxwriter-constant-memory","Write a Million Rows to Excel with xlsxwriter",{"path":2501,"title":2502,"kind":2475},"\u002Fgetting-started-with-python-excel-automation\u002Fchoosing-a-python-excel-library\u002Fbenchmark-python-excel-read-and-write-speed","Benchmark Python Excel Read and Write Speed",{"path":2504,"title":2505,"kind":2475},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fexport-a-large-database-table-to-excel-in-chunks","Export a Large Database Table to Excel in Chunks",{"path":2507,"title":2508,"kind":2475},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fquery-excel-files-with-sql-using-duckdb","Query Excel Files with SQL Using DuckDB",1790972106593]