[{"data":1,"prerenderedAt":2322},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fread-large-excel-file-in-chunks-with-pandas":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fread-large-excel-file-in-chunks-with-pandas":2314},{"id":4,"title":5,"body":6,"dateModified":2283,"datePublished":2283,"description":2284,"extension":2285,"faq":2286,"meta":2297,"navigation":237,"path":2307,"seo":2308,"slug":2310,"stem":2311,"type":2312,"__hash__":2313},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fread-large-excel-file-in-chunks-with-pandas\u002Findex.md","Read a Large Excel File in Chunks with pandas",{"type":7,"value":8,"toc":2268},"minimark",[9,35,170,175,202,206,209,344,351,355,358,861,880,884,887,1295,1308,1312,1315,1387,1610,1617,1621,1624,1836,1848,1852,1907,1910,1914,1917,2008,2011,2015,2133,2137,2143,2147,2157,2161,2184,2196,2206,2212,2216,2219,2228,2231,2264],[10,11,12,16,17,20,21,24,25,28,29,34],"p",{},[13,14,15],"code",{},"pd.read_csv"," takes a ",[13,18,19],{},"chunksize"," and hands you an iterator; ",[13,22,23],{},"pd.read_excel"," does not, and the reason is structural rather than an oversight. A CSV is a stream of lines, while an ",[13,26,27],{},".xlsx"," sheet is zipped XML that has to be parsed before any row is addressable. Chunking Excel therefore means building the iterator yourself. This guide, part of ",[30,31,33],"a",{"href":32},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002F","Working with Large Excel Files in Python",", shows the pattern and the aggregation rules that go with it.",[36,37,45,46,45,50,45,54,45,61,45,70,45,77,45,82,45,90,45,95,45,100,45,104,45,107,45,111,45,114,45,119,45,124,45,128,45,133,45,137,45,142,45,147,45,150,45,156,45,159,45,164],"svg",{"viewBox":38,"role":39,"ariaLabelledBy":40,"xmlns":43,"style":44},"0 0 740 236","img",[41,42],"ch-flow-t","ch-flow-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:740px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[47,48,49],"title",{"id":41},"Streaming rows into fixed-size DataFrame chunks",[51,52,53],"desc",{"id":42},"openpyxl yields rows one at a time from the sheet. A buffer collects fifty thousand of them, converts the batch into a DataFrame, aggregates it, adds the result to a running total, and discards the chunk before filling the buffer again.",[55,56],"rect",{"x":57,"y":57,"width":58,"height":59,"fill":60},"0","740","236","#ffffff",[55,62],{"x":63,"y":64,"width":65,"height":66,"rx":67,"fill":68,"stroke":69},"20","80","140","76","12","#ebebfd","var(--line,#cdd5e6)",[71,72,76],"text",{"x":73,"y":74,"style":75},"90","110","font-size:12.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","openpyxl",[71,78,81],{"x":73,"y":79,"style":80},"132","font-size:11.5px;fill:var(--muted,#5b6780);text-anchor:middle","one row at a time",[83,84],"line",{"x1":85,"y1":86,"x2":87,"y2":86,"stroke":88,"style":89},"164","118","196","var(--muted,#5b6780)","stroke-width:2px",[91,92],"polygon",{"points":93,"fill":94},"200,118 188,112 188,124","#5b6780",[55,96],{"x":97,"y":64,"width":65,"height":66,"rx":67,"fill":98,"stroke":99,"style":89},"202","#f0f4ff","var(--brand,#5b5cf0)",[71,101,103],{"x":102,"y":74,"style":75},"272","buffer",[71,105,106],{"x":102,"y":79,"style":80},"50,000 rows",[83,108],{"x1":109,"y1":86,"x2":110,"y2":86,"stroke":88,"style":89},"346","378",[91,112],{"points":113,"fill":94},"382,118 370,112 370,124",[55,115],{"x":116,"y":64,"width":117,"height":66,"rx":67,"fill":118},"384","150","#5b5cf0",[71,120,123],{"x":121,"y":74,"style":122},"459","font-size:12.5px;font-weight:700;fill:#ffffff;text-anchor:middle","DataFrame chunk",[71,125,127],{"x":121,"y":79,"style":126},"font-size:11.5px;fill:rgba(255,255,255,0.88);text-anchor:middle","transform, group",[83,129],{"x1":130,"y1":86,"x2":131,"y2":86,"stroke":132,"style":89},"538","570","var(--teal,#0f9488)",[91,134],{"points":135,"fill":136},"574,118 562,112 562,124","#0f766e",[55,138],{"x":139,"y":64,"width":140,"height":66,"rx":67,"fill":141,"stroke":132,"style":89},"576","146","#d9f4f1",[71,143,146],{"x":144,"y":74,"style":145},"649","font-size:12.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","running total",[71,148,149],{"x":144,"y":79,"style":80},"small and constant",[151,152],"path",{"d":153,"fill":154,"stroke":88,"style":155},"M459 160 C 459 190, 272 190, 272 162","none","stroke-width:1.5px;stroke-dasharray:5 4",[91,157],{"points":158,"fill":94},"272,158 266,170 278,170",[71,160,163],{"x":161,"y":162,"style":80},"366","204","chunk discarded, buffer refills — peak memory stays at one chunk",[71,165,169],{"x":166,"y":167,"style":168},"370","42","font-size:12.5px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","Read once, in order, and never hold more than one batch.",[171,172,174],"h2",{"id":173},"prerequisites","Prerequisites",[176,177,182],"pre",{"className":178,"code":179,"language":180,"meta":181,"style":181},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl\n","bash","",[13,183,184],{"__ignoreMap":181},[185,186,188,192,196,199],"span",{"class":83,"line":187},1,[185,189,191],{"class":190},"sMTad","pip",[185,193,195],{"class":194},"srMev"," install",[185,197,198],{"class":194}," pandas",[185,200,201],{"class":194}," openpyxl\n",[171,203,205],{"id":204},"why-not-skiprows-and-nrows","Why not skiprows and nrows?",[10,207,208],{},"The obvious approach looks like paging, and it is a trap:",[176,210,214],{"className":211,"code":212,"language":213,"meta":181,"style":181},"language-python shiki shiki-themes github-light github-dark-high-contrast","import pandas as pd\n\n# Works, but each call re-parses the whole sheet from the beginning\npage1 = pd.read_excel(\"large.xlsx\", sheet_name=\"Orders\", skiprows=0, nrows=50_000)\npage2 = pd.read_excel(\"large.xlsx\", sheet_name=\"Orders\", skiprows=50_000, nrows=50_000, header=None)\n","python",[13,215,216,232,239,246,296],{"__ignoreMap":181},[185,217,218,222,226,229],{"class":83,"line":187},[185,219,221],{"class":220},"s-kum","import",[185,223,225],{"class":224},"skGVy"," pandas ",[185,227,228],{"class":220},"as",[185,230,231],{"class":224}," pd\n",[185,233,235],{"class":83,"line":234},2,[185,236,238],{"emptyLinePlaceholder":237},true,"\n",[185,240,242],{"class":83,"line":241},3,[185,243,245],{"class":244},"s-wDw","# Works, but each call re-parses the whole sheet from the beginning\n",[185,247,249,252,255,258,261,264,268,270,273,275,278,280,283,285,288,290,293],{"class":83,"line":248},4,[185,250,251],{"class":224},"page1 ",[185,253,254],{"class":220},"=",[185,256,257],{"class":224}," pd.read_excel(",[185,259,260],{"class":194},"\"large.xlsx\"",[185,262,263],{"class":224},", ",[185,265,267],{"class":266},"sa561","sheet_name",[185,269,254],{"class":220},[185,271,272],{"class":194},"\"Orders\"",[185,274,263],{"class":224},[185,276,277],{"class":266},"skiprows",[185,279,254],{"class":220},[185,281,57],{"class":282},"sP0c6",[185,284,263],{"class":224},[185,286,287],{"class":266},"nrows",[185,289,254],{"class":220},[185,291,292],{"class":282},"50_000",[185,294,295],{"class":224},")\n",[185,297,299,302,304,306,308,310,312,314,316,318,320,322,324,326,328,330,332,334,337,339,342],{"class":83,"line":298},5,[185,300,301],{"class":224},"page2 ",[185,303,254],{"class":220},[185,305,257],{"class":224},[185,307,260],{"class":194},[185,309,263],{"class":224},[185,311,267],{"class":266},[185,313,254],{"class":220},[185,315,272],{"class":194},[185,317,263],{"class":224},[185,319,277],{"class":266},[185,321,254],{"class":220},[185,323,292],{"class":282},[185,325,263],{"class":224},[185,327,287],{"class":266},[185,329,254],{"class":220},[185,331,292],{"class":282},[185,333,263],{"class":224},[185,335,336],{"class":266},"header",[185,338,254],{"class":220},[185,340,341],{"class":282},"None",[185,343,295],{"class":224},[10,345,346,347,350],{},"Each ",[13,348,349],{},"read_excel"," opens the archive, parses the sheet XML and then throws away the rows it was told to skip. Eight pages means eight full parses, so the \"chunked\" version is slower than reading the file once. It is fine for grabbing a sample from the top; it is not a way to process a large file.",[171,352,354],{"id":353},"the-chunk-generator","The chunk generator",[10,356,357],{},"Build the iterator on openpyxl's streaming reader instead. One parse, one row at a time, batched into DataFrames:",[176,359,361],{"className":211,"code":360,"language":213,"meta":181,"style":181},"import pandas as pd\nfrom openpyxl import load_workbook\n\ndef excel_chunks(path, sheet, size=50_000, columns=None):\n    \"\"\"Yield DataFrames of at most `size` rows from a worksheet.\"\"\"\n    wb = load_workbook(path, read_only=True, data_only=True)\n    try:\n        ws = wb[sheet]\n        rows = ws.iter_rows(values_only=True)\n        header = [str(h).strip() if h is not None else \"\" for h in next(rows)]\n        wanted = columns or header\n        picks = [header.index(c) for c in wanted]\n\n        buffer = []\n        for row in rows:\n            if all(v is None for v in row):\n                continue                          # skip fully blank rows\n            buffer.append([row[i] for i in picks])\n            if len(buffer) >= size:\n                yield pd.DataFrame(buffer, columns=wanted)\n                buffer.clear()\n        if buffer:\n            yield pd.DataFrame(buffer, columns=wanted)\n    finally:\n        wb.close()\n\nfor i, chunk in enumerate(excel_chunks(\"large.xlsx\", \"Orders\", size=50_000), start=1):\n    print(f\"chunk {i}: {len(chunk)} rows, {chunk.memory_usage(deep=True).sum() \u002F 1e6:.1f} MB\")\n",[13,362,363,373,386,390,416,421,451,460,471,491,543,560,582,587,598,612,638,647,663,680,697,703,712,726,734,740,745,789],{"__ignoreMap":181},[185,364,365,367,369,371],{"class":83,"line":187},[185,366,221],{"class":220},[185,368,225],{"class":224},[185,370,228],{"class":220},[185,372,231],{"class":224},[185,374,375,378,381,383],{"class":83,"line":234},[185,376,377],{"class":220},"from",[185,379,380],{"class":224}," openpyxl ",[185,382,221],{"class":220},[185,384,385],{"class":224}," load_workbook\n",[185,387,388],{"class":83,"line":241},[185,389,238],{"emptyLinePlaceholder":237},[185,391,392,395,399,402,404,406,409,411,413],{"class":83,"line":248},[185,393,394],{"class":220},"def",[185,396,398],{"class":397},"s_Opv"," excel_chunks",[185,400,401],{"class":224},"(path, sheet, size",[185,403,254],{"class":220},[185,405,292],{"class":282},[185,407,408],{"class":224},", columns",[185,410,254],{"class":220},[185,412,341],{"class":282},[185,414,415],{"class":224},"):\n",[185,417,418],{"class":83,"line":298},[185,419,420],{"class":194},"    \"\"\"Yield DataFrames of at most `size` rows from a worksheet.\"\"\"\n",[185,422,424,427,429,432,435,437,440,442,445,447,449],{"class":83,"line":423},6,[185,425,426],{"class":224},"    wb ",[185,428,254],{"class":220},[185,430,431],{"class":224}," load_workbook(path, ",[185,433,434],{"class":266},"read_only",[185,436,254],{"class":220},[185,438,439],{"class":282},"True",[185,441,263],{"class":224},[185,443,444],{"class":266},"data_only",[185,446,254],{"class":220},[185,448,439],{"class":282},[185,450,295],{"class":224},[185,452,454,457],{"class":83,"line":453},7,[185,455,456],{"class":220},"    try",[185,458,459],{"class":224},":\n",[185,461,463,466,468],{"class":83,"line":462},8,[185,464,465],{"class":224},"        ws ",[185,467,254],{"class":220},[185,469,470],{"class":224}," wb[sheet]\n",[185,472,474,477,479,482,485,487,489],{"class":83,"line":473},9,[185,475,476],{"class":224},"        rows ",[185,478,254],{"class":220},[185,480,481],{"class":224}," ws.iter_rows(",[185,483,484],{"class":266},"values_only",[185,486,254],{"class":220},[185,488,439],{"class":282},[185,490,295],{"class":224},[185,492,494,497,499,502,505,508,511,514,517,520,523,526,529,532,534,537,540],{"class":83,"line":493},10,[185,495,496],{"class":224},"        header ",[185,498,254],{"class":220},[185,500,501],{"class":224}," [",[185,503,504],{"class":282},"str",[185,506,507],{"class":224},"(h).strip() ",[185,509,510],{"class":220},"if",[185,512,513],{"class":224}," h ",[185,515,516],{"class":220},"is",[185,518,519],{"class":220}," not",[185,521,522],{"class":282}," None",[185,524,525],{"class":220}," else",[185,527,528],{"class":194}," \"\"",[185,530,531],{"class":220}," for",[185,533,513],{"class":224},[185,535,536],{"class":220},"in",[185,538,539],{"class":282}," next",[185,541,542],{"class":224},"(rows)]\n",[185,544,546,549,551,554,557],{"class":83,"line":545},11,[185,547,548],{"class":224},"        wanted ",[185,550,254],{"class":220},[185,552,553],{"class":224}," columns ",[185,555,556],{"class":220},"or",[185,558,559],{"class":224}," header\n",[185,561,563,566,568,571,574,577,579],{"class":83,"line":562},12,[185,564,565],{"class":224},"        picks ",[185,567,254],{"class":220},[185,569,570],{"class":224}," [header.index(c) ",[185,572,573],{"class":220},"for",[185,575,576],{"class":224}," c ",[185,578,536],{"class":220},[185,580,581],{"class":224}," wanted]\n",[185,583,585],{"class":83,"line":584},13,[185,586,238],{"emptyLinePlaceholder":237},[185,588,590,593,595],{"class":83,"line":589},14,[185,591,592],{"class":224},"        buffer ",[185,594,254],{"class":220},[185,596,597],{"class":224}," []\n",[185,599,601,604,607,609],{"class":83,"line":600},15,[185,602,603],{"class":220},"        for",[185,605,606],{"class":224}," row ",[185,608,536],{"class":220},[185,610,611],{"class":224}," rows:\n",[185,613,615,618,621,624,626,628,630,633,635],{"class":83,"line":614},16,[185,616,617],{"class":220},"            if",[185,619,620],{"class":282}," all",[185,622,623],{"class":224},"(v ",[185,625,516],{"class":220},[185,627,522],{"class":282},[185,629,531],{"class":220},[185,631,632],{"class":224}," v ",[185,634,536],{"class":220},[185,636,637],{"class":224}," row):\n",[185,639,641,644],{"class":83,"line":640},17,[185,642,643],{"class":220},"                continue",[185,645,646],{"class":244},"                          # skip fully blank rows\n",[185,648,650,653,655,658,660],{"class":83,"line":649},18,[185,651,652],{"class":224},"            buffer.append([row[i] ",[185,654,573],{"class":220},[185,656,657],{"class":224}," i ",[185,659,536],{"class":220},[185,661,662],{"class":224}," picks])\n",[185,664,666,668,671,674,677],{"class":83,"line":665},19,[185,667,617],{"class":220},[185,669,670],{"class":282}," len",[185,672,673],{"class":224},"(buffer) ",[185,675,676],{"class":220},">=",[185,678,679],{"class":224}," size:\n",[185,681,683,686,689,692,694],{"class":83,"line":682},20,[185,684,685],{"class":220},"                yield",[185,687,688],{"class":224}," pd.DataFrame(buffer, ",[185,690,691],{"class":266},"columns",[185,693,254],{"class":220},[185,695,696],{"class":224},"wanted)\n",[185,698,700],{"class":83,"line":699},21,[185,701,702],{"class":224},"                buffer.clear()\n",[185,704,706,709],{"class":83,"line":705},22,[185,707,708],{"class":220},"        if",[185,710,711],{"class":224}," buffer:\n",[185,713,715,718,720,722,724],{"class":83,"line":714},23,[185,716,717],{"class":220},"            yield",[185,719,688],{"class":224},[185,721,691],{"class":266},[185,723,254],{"class":220},[185,725,696],{"class":224},[185,727,729,732],{"class":83,"line":728},24,[185,730,731],{"class":220},"    finally",[185,733,459],{"class":224},[185,735,737],{"class":83,"line":736},25,[185,738,739],{"class":224},"        wb.close()\n",[185,741,743],{"class":83,"line":742},26,[185,744,238],{"emptyLinePlaceholder":237},[185,746,748,750,753,755,758,761,763,765,767,769,772,774,776,779,782,784,787],{"class":83,"line":747},27,[185,749,573],{"class":220},[185,751,752],{"class":224}," i, chunk ",[185,754,536],{"class":220},[185,756,757],{"class":282}," enumerate",[185,759,760],{"class":224},"(excel_chunks(",[185,762,260],{"class":194},[185,764,263],{"class":224},[185,766,272],{"class":194},[185,768,263],{"class":224},[185,770,771],{"class":266},"size",[185,773,254],{"class":220},[185,775,292],{"class":282},[185,777,778],{"class":224},"), ",[185,780,781],{"class":266},"start",[185,783,254],{"class":220},[185,785,786],{"class":282},"1",[185,788,415],{"class":224},[185,790,792,795,798,801,804,808,811,814,817,819,822,825,827,830,832,835,838,840,842,845,848,851,854,856,859],{"class":83,"line":791},28,[185,793,794],{"class":282},"    print",[185,796,797],{"class":224},"(",[185,799,800],{"class":220},"f",[185,802,803],{"class":194},"\"chunk ",[185,805,807],{"class":806},"sSjpA","{",[185,809,810],{"class":224},"i",[185,812,813],{"class":806},"}",[185,815,816],{"class":194},": ",[185,818,807],{"class":806},[185,820,821],{"class":282},"len",[185,823,824],{"class":224},"(chunk)",[185,826,813],{"class":806},[185,828,829],{"class":194}," rows, ",[185,831,807],{"class":806},[185,833,834],{"class":224},"chunk.memory_usage(",[185,836,837],{"class":266},"deep",[185,839,254],{"class":220},[185,841,439],{"class":282},[185,843,844],{"class":224},").sum() ",[185,846,847],{"class":220},"\u002F",[185,849,850],{"class":282}," 1e6",[185,852,853],{"class":220},":.1f",[185,855,813],{"class":806},[185,857,858],{"class":194}," MB\"",[185,860,295],{"class":224},[10,862,863,864,867,868,871,872,875,876,879],{},"Three details make this production-worthy. ",[13,865,866],{},"values_only=True"," yields plain tuples rather than ",[13,869,870],{},"Cell"," objects, which is the difference between fast and pointless. The ",[13,873,874],{},"finally"," block closes the workbook even if the consumer stops early, so a ",[13,877,878],{},"break"," in the caller cannot leak a file handle. And picking only the columns you asked for keeps each chunk narrow, which matters more than the row count.",[171,881,883],{"id":882},"aggregating-across-chunks","Aggregating across chunks",[10,885,886],{},"The whole point is to produce something small. Any aggregation that can be combined works — the accumulator is a tiny frame that lives across iterations:",[176,888,890],{"className":211,"code":889,"language":213,"meta":181,"style":181},"import pandas as pd\n\nrunning = None\nrows_seen = 0\n\nfor chunk in excel_chunks(\"large.xlsx\", \"Orders\",\n                          columns=[\"Order_Date\", \"Region\", \"Quantity\", \"Unit_Price\"]):\n    chunk[\"Revenue\"] = chunk[\"Quantity\"] * chunk[\"Unit_Price\"]\n    chunk[\"Month\"] = pd.to_datetime(chunk[\"Order_Date\"]).dt.to_period(\"M\").astype(str)\n\n    part = chunk.groupby([\"Month\", \"Region\"], observed=True).agg(\n        Revenue=(\"Revenue\", \"sum\"),\n        Orders=(\"Revenue\", \"size\"),\n        Largest=(\"Revenue\", \"max\"),\n    )\n    if running is None:\n        running = part\n    else:\n        running = running.add(part, fill_value=0)\n        running[\"Largest\"] = running[\"Largest\"].combine(part[\"Largest\"], max, fill_value=0)\n    rows_seen += len(chunk)\n\nsummary = running.reset_index().sort_values([\"Month\", \"Region\"])\nprint(f\"{rows_seen:,} rows -> {len(summary)} summary rows\")\n",[13,891,892,902,906,916,926,930,951,982,1012,1041,1045,1074,1093,1111,1129,1134,1148,1158,1165,1183,1220,1233,1237,1256],{"__ignoreMap":181},[185,893,894,896,898,900],{"class":83,"line":187},[185,895,221],{"class":220},[185,897,225],{"class":224},[185,899,228],{"class":220},[185,901,231],{"class":224},[185,903,904],{"class":83,"line":234},[185,905,238],{"emptyLinePlaceholder":237},[185,907,908,911,913],{"class":83,"line":241},[185,909,910],{"class":224},"running ",[185,912,254],{"class":220},[185,914,915],{"class":282}," None\n",[185,917,918,921,923],{"class":83,"line":248},[185,919,920],{"class":224},"rows_seen ",[185,922,254],{"class":220},[185,924,925],{"class":282}," 0\n",[185,927,928],{"class":83,"line":298},[185,929,238],{"emptyLinePlaceholder":237},[185,931,932,934,937,939,942,944,946,948],{"class":83,"line":423},[185,933,573],{"class":220},[185,935,936],{"class":224}," chunk ",[185,938,536],{"class":220},[185,940,941],{"class":224}," excel_chunks(",[185,943,260],{"class":194},[185,945,263],{"class":224},[185,947,272],{"class":194},[185,949,950],{"class":224},",\n",[185,952,953,956,958,961,964,966,969,971,974,976,979],{"class":83,"line":453},[185,954,955],{"class":266},"                          columns",[185,957,254],{"class":220},[185,959,960],{"class":224},"[",[185,962,963],{"class":194},"\"Order_Date\"",[185,965,263],{"class":224},[185,967,968],{"class":194},"\"Region\"",[185,970,263],{"class":224},[185,972,973],{"class":194},"\"Quantity\"",[185,975,263],{"class":224},[185,977,978],{"class":194},"\"Unit_Price\"",[185,980,981],{"class":224},"]):\n",[185,983,984,987,990,993,995,998,1000,1002,1005,1007,1009],{"class":83,"line":462},[185,985,986],{"class":224},"    chunk[",[185,988,989],{"class":194},"\"Revenue\"",[185,991,992],{"class":224},"] ",[185,994,254],{"class":220},[185,996,997],{"class":224}," chunk[",[185,999,973],{"class":194},[185,1001,992],{"class":224},[185,1003,1004],{"class":220},"*",[185,1006,997],{"class":224},[185,1008,978],{"class":194},[185,1010,1011],{"class":224},"]\n",[185,1013,1014,1016,1019,1021,1023,1026,1028,1031,1034,1037,1039],{"class":83,"line":473},[185,1015,986],{"class":224},[185,1017,1018],{"class":194},"\"Month\"",[185,1020,992],{"class":224},[185,1022,254],{"class":220},[185,1024,1025],{"class":224}," pd.to_datetime(chunk[",[185,1027,963],{"class":194},[185,1029,1030],{"class":224},"]).dt.to_period(",[185,1032,1033],{"class":194},"\"M\"",[185,1035,1036],{"class":224},").astype(",[185,1038,504],{"class":282},[185,1040,295],{"class":224},[185,1042,1043],{"class":83,"line":493},[185,1044,238],{"emptyLinePlaceholder":237},[185,1046,1047,1050,1052,1055,1057,1059,1061,1064,1067,1069,1071],{"class":83,"line":545},[185,1048,1049],{"class":224},"    part ",[185,1051,254],{"class":220},[185,1053,1054],{"class":224}," chunk.groupby([",[185,1056,1018],{"class":194},[185,1058,263],{"class":224},[185,1060,968],{"class":194},[185,1062,1063],{"class":224},"], ",[185,1065,1066],{"class":266},"observed",[185,1068,254],{"class":220},[185,1070,439],{"class":282},[185,1072,1073],{"class":224},").agg(\n",[185,1075,1076,1079,1081,1083,1085,1087,1090],{"class":83,"line":562},[185,1077,1078],{"class":266},"        Revenue",[185,1080,254],{"class":220},[185,1082,797],{"class":224},[185,1084,989],{"class":194},[185,1086,263],{"class":224},[185,1088,1089],{"class":194},"\"sum\"",[185,1091,1092],{"class":224},"),\n",[185,1094,1095,1098,1100,1102,1104,1106,1109],{"class":83,"line":584},[185,1096,1097],{"class":266},"        Orders",[185,1099,254],{"class":220},[185,1101,797],{"class":224},[185,1103,989],{"class":194},[185,1105,263],{"class":224},[185,1107,1108],{"class":194},"\"size\"",[185,1110,1092],{"class":224},[185,1112,1113,1116,1118,1120,1122,1124,1127],{"class":83,"line":589},[185,1114,1115],{"class":266},"        Largest",[185,1117,254],{"class":220},[185,1119,797],{"class":224},[185,1121,989],{"class":194},[185,1123,263],{"class":224},[185,1125,1126],{"class":194},"\"max\"",[185,1128,1092],{"class":224},[185,1130,1131],{"class":83,"line":600},[185,1132,1133],{"class":224},"    )\n",[185,1135,1136,1139,1142,1144,1146],{"class":83,"line":614},[185,1137,1138],{"class":220},"    if",[185,1140,1141],{"class":224}," running ",[185,1143,516],{"class":220},[185,1145,522],{"class":282},[185,1147,459],{"class":224},[185,1149,1150,1153,1155],{"class":83,"line":640},[185,1151,1152],{"class":224},"        running ",[185,1154,254],{"class":220},[185,1156,1157],{"class":224}," part\n",[185,1159,1160,1163],{"class":83,"line":649},[185,1161,1162],{"class":220},"    else",[185,1164,459],{"class":224},[185,1166,1167,1169,1171,1174,1177,1179,1181],{"class":83,"line":665},[185,1168,1152],{"class":224},[185,1170,254],{"class":220},[185,1172,1173],{"class":224}," running.add(part, ",[185,1175,1176],{"class":266},"fill_value",[185,1178,254],{"class":220},[185,1180,57],{"class":282},[185,1182,295],{"class":224},[185,1184,1185,1188,1191,1193,1195,1198,1200,1203,1205,1207,1210,1212,1214,1216,1218],{"class":83,"line":682},[185,1186,1187],{"class":224},"        running[",[185,1189,1190],{"class":194},"\"Largest\"",[185,1192,992],{"class":224},[185,1194,254],{"class":220},[185,1196,1197],{"class":224}," running[",[185,1199,1190],{"class":194},[185,1201,1202],{"class":224},"].combine(part[",[185,1204,1190],{"class":194},[185,1206,1063],{"class":224},[185,1208,1209],{"class":282},"max",[185,1211,263],{"class":224},[185,1213,1176],{"class":266},[185,1215,254],{"class":220},[185,1217,57],{"class":282},[185,1219,295],{"class":224},[185,1221,1222,1225,1228,1230],{"class":83,"line":699},[185,1223,1224],{"class":224},"    rows_seen ",[185,1226,1227],{"class":220},"+=",[185,1229,670],{"class":282},[185,1231,1232],{"class":224},"(chunk)\n",[185,1234,1235],{"class":83,"line":705},[185,1236,238],{"emptyLinePlaceholder":237},[185,1238,1239,1242,1244,1247,1249,1251,1253],{"class":83,"line":714},[185,1240,1241],{"class":224},"summary ",[185,1243,254],{"class":220},[185,1245,1246],{"class":224}," running.reset_index().sort_values([",[185,1248,1018],{"class":194},[185,1250,263],{"class":224},[185,1252,968],{"class":194},[185,1254,1255],{"class":224},"])\n",[185,1257,1258,1261,1263,1265,1268,1270,1273,1276,1278,1281,1283,1285,1288,1290,1293],{"class":83,"line":728},[185,1259,1260],{"class":282},"print",[185,1262,797],{"class":224},[185,1264,800],{"class":220},[185,1266,1267],{"class":194},"\"",[185,1269,807],{"class":806},[185,1271,1272],{"class":224},"rows_seen",[185,1274,1275],{"class":220},":,",[185,1277,813],{"class":806},[185,1279,1280],{"class":194}," rows -> ",[185,1282,807],{"class":806},[185,1284,821],{"class":282},[185,1286,1287],{"class":224},"(summary)",[185,1289,813],{"class":806},[185,1291,1292],{"class":194}," summary rows\"",[185,1294,295],{"class":224},[10,1296,1297,1300,1301,1304,1305,1307],{},[13,1298,1299],{},"add(..., fill_value=0)"," handles the case where a group appears in one chunk and not another, which is guaranteed to happen as soon as the data is sorted by anything. Note that ",[13,1302,1303],{},"Largest"," needs ",[13,1306,1209],{}," rather than addition — combining aggregates correctly means knowing which operator each column takes.",[171,1309,1311],{"id":1310},"the-averages-trap","The averages trap",[10,1313,1314],{},"Means do not combine. The average of chunk averages is only correct when every chunk has the same number of rows, and the last chunk never does:",[36,1316,45,1321,45,1324,45,1327,45,1330,45,1337,45,1343,45,1347,45,1350,45,1354,45,1358,45,1362,45,1367,45,1371,45,1375,45,1379,45,1383],{"viewBox":1317,"role":39,"ariaLabelledBy":1318,"xmlns":43,"style":44},"0 0 740 224",[1319,1320],"ch-mean-t","ch-mean-d",[47,1322,1323],{"id":1319},"Why averaging chunk averages is wrong",[51,1325,1326],{"id":1320},"A chunk of fifty thousand rows averaging ten and a final chunk of two rows averaging one hundred give a true overall average of ten point zero zero four. Averaging the two chunk means instead gives fifty five, which is wildly wrong.",[55,1328],{"x":57,"y":57,"width":58,"height":1329,"fill":60},"224",[55,1331],{"x":1332,"y":1333,"width":1334,"height":1335,"rx":1336,"fill":98,"stroke":69},"24","34","300","66","10",[71,1338,1342],{"x":1339,"y":1340,"style":1341},"174","60","font-size:12px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","chunk 1",[71,1344,1346],{"x":1339,"y":1345,"style":80},"84","50,000 rows · mean 10",[55,1348],{"x":1332,"y":1349,"width":1334,"height":1335,"rx":1336,"fill":98,"stroke":69},"112",[71,1351,1353],{"x":1339,"y":1352,"style":1341},"138","chunk 2 (the remainder)",[71,1355,1357],{"x":1339,"y":1356,"style":80},"162","2 rows · mean 100",[55,1359],{"x":1360,"y":1333,"width":1361,"height":1335,"rx":1336,"fill":141,"stroke":132,"style":89},"380","336",[71,1363,1366],{"x":1364,"y":1340,"style":1365},"548","font-size:12px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","sum ÷ count",[71,1368,1370],{"x":1364,"y":1345,"style":1369},"font-size:12px;fill:var(--text,#172033);text-anchor:middle","500,200 ÷ 50,002 = 10.004 ✓",[55,1372],{"x":1360,"y":1349,"width":1361,"height":1335,"rx":1336,"fill":1373,"stroke":1374,"style":89},"#fce9e9","var(--danger,#dc2626)",[71,1376,1378],{"x":1364,"y":1352,"style":1377},"font-size:12px;font-weight:700;fill:var(--danger,#dc2626);text-anchor:middle","mean of means",[71,1380,1382],{"x":1364,"y":1356,"style":1381},"font-size:12px;fill:var(--danger,#dc2626);text-anchor:middle","(10 + 100) ÷ 2 = 55 ✗",[71,1384,1386],{"x":166,"y":1385,"style":80},"206","Accumulate the numerator and the denominator; divide once, at the end.",[176,1388,1390],{"className":211,"code":1389,"language":213,"meta":181,"style":181},"import pandas as pd\n\ntotals = None\nfor chunk in excel_chunks(\"large.xlsx\", \"Orders\", columns=[\"Region\", \"Quantity\", \"Unit_Price\"]):\n    chunk[\"Revenue\"] = chunk[\"Quantity\"] * chunk[\"Unit_Price\"]\n    part = chunk.groupby(\"Region\", observed=True).agg(\n        Revenue_Sum=(\"Revenue\", \"sum\"), Row_Count=(\"Revenue\", \"size\")\n    )\n    totals = part if totals is None else totals.add(part, fill_value=0)\n\ntotals[\"Average_Order\"] = totals[\"Revenue_Sum\"] \u002F totals[\"Row_Count\"]\nprint(totals.round(2))\n",[13,1391,1392,1402,1406,1415,1451,1475,1496,1528,1532,1564,1568,1597],{"__ignoreMap":181},[185,1393,1394,1396,1398,1400],{"class":83,"line":187},[185,1395,221],{"class":220},[185,1397,225],{"class":224},[185,1399,228],{"class":220},[185,1401,231],{"class":224},[185,1403,1404],{"class":83,"line":234},[185,1405,238],{"emptyLinePlaceholder":237},[185,1407,1408,1411,1413],{"class":83,"line":241},[185,1409,1410],{"class":224},"totals ",[185,1412,254],{"class":220},[185,1414,915],{"class":282},[185,1416,1417,1419,1421,1423,1425,1427,1429,1431,1433,1435,1437,1439,1441,1443,1445,1447,1449],{"class":83,"line":248},[185,1418,573],{"class":220},[185,1420,936],{"class":224},[185,1422,536],{"class":220},[185,1424,941],{"class":224},[185,1426,260],{"class":194},[185,1428,263],{"class":224},[185,1430,272],{"class":194},[185,1432,263],{"class":224},[185,1434,691],{"class":266},[185,1436,254],{"class":220},[185,1438,960],{"class":224},[185,1440,968],{"class":194},[185,1442,263],{"class":224},[185,1444,973],{"class":194},[185,1446,263],{"class":224},[185,1448,978],{"class":194},[185,1450,981],{"class":224},[185,1452,1453,1455,1457,1459,1461,1463,1465,1467,1469,1471,1473],{"class":83,"line":298},[185,1454,986],{"class":224},[185,1456,989],{"class":194},[185,1458,992],{"class":224},[185,1460,254],{"class":220},[185,1462,997],{"class":224},[185,1464,973],{"class":194},[185,1466,992],{"class":224},[185,1468,1004],{"class":220},[185,1470,997],{"class":224},[185,1472,978],{"class":194},[185,1474,1011],{"class":224},[185,1476,1477,1479,1481,1484,1486,1488,1490,1492,1494],{"class":83,"line":423},[185,1478,1049],{"class":224},[185,1480,254],{"class":220},[185,1482,1483],{"class":224}," chunk.groupby(",[185,1485,968],{"class":194},[185,1487,263],{"class":224},[185,1489,1066],{"class":266},[185,1491,254],{"class":220},[185,1493,439],{"class":282},[185,1495,1073],{"class":224},[185,1497,1498,1501,1503,1505,1507,1509,1511,1513,1516,1518,1520,1522,1524,1526],{"class":83,"line":453},[185,1499,1500],{"class":266},"        Revenue_Sum",[185,1502,254],{"class":220},[185,1504,797],{"class":224},[185,1506,989],{"class":194},[185,1508,263],{"class":224},[185,1510,1089],{"class":194},[185,1512,778],{"class":224},[185,1514,1515],{"class":266},"Row_Count",[185,1517,254],{"class":220},[185,1519,797],{"class":224},[185,1521,989],{"class":194},[185,1523,263],{"class":224},[185,1525,1108],{"class":194},[185,1527,295],{"class":224},[185,1529,1530],{"class":83,"line":462},[185,1531,1133],{"class":224},[185,1533,1534,1537,1539,1542,1544,1547,1549,1551,1553,1556,1558,1560,1562],{"class":83,"line":473},[185,1535,1536],{"class":224},"    totals ",[185,1538,254],{"class":220},[185,1540,1541],{"class":224}," part ",[185,1543,510],{"class":220},[185,1545,1546],{"class":224}," totals ",[185,1548,516],{"class":220},[185,1550,522],{"class":282},[185,1552,525],{"class":220},[185,1554,1555],{"class":224}," totals.add(part, ",[185,1557,1176],{"class":266},[185,1559,254],{"class":220},[185,1561,57],{"class":282},[185,1563,295],{"class":224},[185,1565,1566],{"class":83,"line":493},[185,1567,238],{"emptyLinePlaceholder":237},[185,1569,1570,1573,1576,1578,1580,1583,1586,1588,1590,1592,1595],{"class":83,"line":545},[185,1571,1572],{"class":224},"totals[",[185,1574,1575],{"class":194},"\"Average_Order\"",[185,1577,992],{"class":224},[185,1579,254],{"class":220},[185,1581,1582],{"class":224}," totals[",[185,1584,1585],{"class":194},"\"Revenue_Sum\"",[185,1587,992],{"class":224},[185,1589,847],{"class":220},[185,1591,1582],{"class":224},[185,1593,1594],{"class":194},"\"Row_Count\"",[185,1596,1011],{"class":224},[185,1598,1599,1601,1604,1607],{"class":83,"line":562},[185,1600,1260],{"class":282},[185,1602,1603],{"class":224},"(totals.round(",[185,1605,1606],{"class":282},"2",[185,1608,1609],{"class":224},"))\n",[10,1611,1612,1613,1616],{},"The same reasoning rules out a few other aggregates entirely. Medians, percentiles and exact distinct counts need the whole distribution, so either collect just that one column across the file, or accept an approximation — ",[13,1614,1615],{},"nunique"," per chunk summed is an upper bound, not an answer.",[171,1618,1620],{"id":1619},"filter-early-inside-the-loop","Filter early, inside the loop",[10,1622,1623],{},"If the job only needs part of the file, discard rows as soon as they arrive. The filtered rows never join an accumulator and never cost anything again:",[176,1625,1627],{"className":211,"code":1626,"language":213,"meta":181,"style":181},"import pandas as pd\n\nWINDOW_START = pd.Timestamp(\"2024-06-01\")\nkept = []\n\nfor chunk in excel_chunks(\"large.xlsx\", \"Orders\",\n                          columns=[\"Order_Date\", \"Region\", \"Quantity\", \"Unit_Price\"]):\n    chunk[\"Order_Date\"] = pd.to_datetime(chunk[\"Order_Date\"], errors=\"coerce\")\n    recent = chunk[chunk[\"Order_Date\"] >= WINDOW_START]\n    if not recent.empty:\n        kept.append(recent)\n\nresult = pd.concat(kept, ignore_index=True) if kept else pd.DataFrame()\nprint(f\"{len(result):,} row(s) in the window\")\n",[13,1628,1629,1639,1643,1659,1668,1672,1690,1714,1740,1761,1770,1775,1779,1810],{"__ignoreMap":181},[185,1630,1631,1633,1635,1637],{"class":83,"line":187},[185,1632,221],{"class":220},[185,1634,225],{"class":224},[185,1636,228],{"class":220},[185,1638,231],{"class":224},[185,1640,1641],{"class":83,"line":234},[185,1642,238],{"emptyLinePlaceholder":237},[185,1644,1645,1648,1651,1654,1657],{"class":83,"line":241},[185,1646,1647],{"class":282},"WINDOW_START",[185,1649,1650],{"class":220}," =",[185,1652,1653],{"class":224}," pd.Timestamp(",[185,1655,1656],{"class":194},"\"2024-06-01\"",[185,1658,295],{"class":224},[185,1660,1661,1664,1666],{"class":83,"line":248},[185,1662,1663],{"class":224},"kept ",[185,1665,254],{"class":220},[185,1667,597],{"class":224},[185,1669,1670],{"class":83,"line":298},[185,1671,238],{"emptyLinePlaceholder":237},[185,1673,1674,1676,1678,1680,1682,1684,1686,1688],{"class":83,"line":423},[185,1675,573],{"class":220},[185,1677,936],{"class":224},[185,1679,536],{"class":220},[185,1681,941],{"class":224},[185,1683,260],{"class":194},[185,1685,263],{"class":224},[185,1687,272],{"class":194},[185,1689,950],{"class":224},[185,1691,1692,1694,1696,1698,1700,1702,1704,1706,1708,1710,1712],{"class":83,"line":453},[185,1693,955],{"class":266},[185,1695,254],{"class":220},[185,1697,960],{"class":224},[185,1699,963],{"class":194},[185,1701,263],{"class":224},[185,1703,968],{"class":194},[185,1705,263],{"class":224},[185,1707,973],{"class":194},[185,1709,263],{"class":224},[185,1711,978],{"class":194},[185,1713,981],{"class":224},[185,1715,1716,1718,1720,1722,1724,1726,1728,1730,1733,1735,1738],{"class":83,"line":462},[185,1717,986],{"class":224},[185,1719,963],{"class":194},[185,1721,992],{"class":224},[185,1723,254],{"class":220},[185,1725,1025],{"class":224},[185,1727,963],{"class":194},[185,1729,1063],{"class":224},[185,1731,1732],{"class":266},"errors",[185,1734,254],{"class":220},[185,1736,1737],{"class":194},"\"coerce\"",[185,1739,295],{"class":224},[185,1741,1742,1745,1747,1750,1752,1754,1756,1759],{"class":83,"line":473},[185,1743,1744],{"class":224},"    recent ",[185,1746,254],{"class":220},[185,1748,1749],{"class":224}," chunk[chunk[",[185,1751,963],{"class":194},[185,1753,992],{"class":224},[185,1755,676],{"class":220},[185,1757,1758],{"class":282}," WINDOW_START",[185,1760,1011],{"class":224},[185,1762,1763,1765,1767],{"class":83,"line":493},[185,1764,1138],{"class":220},[185,1766,519],{"class":220},[185,1768,1769],{"class":224}," recent.empty:\n",[185,1771,1772],{"class":83,"line":545},[185,1773,1774],{"class":224},"        kept.append(recent)\n",[185,1776,1777],{"class":83,"line":562},[185,1778,238],{"emptyLinePlaceholder":237},[185,1780,1781,1784,1786,1789,1792,1794,1796,1799,1801,1804,1807],{"class":83,"line":584},[185,1782,1783],{"class":224},"result ",[185,1785,254],{"class":220},[185,1787,1788],{"class":224}," pd.concat(kept, ",[185,1790,1791],{"class":266},"ignore_index",[185,1793,254],{"class":220},[185,1795,439],{"class":282},[185,1797,1798],{"class":224},") ",[185,1800,510],{"class":220},[185,1802,1803],{"class":224}," kept ",[185,1805,1806],{"class":220},"else",[185,1808,1809],{"class":224}," pd.DataFrame()\n",[185,1811,1812,1814,1816,1818,1820,1822,1824,1827,1829,1831,1834],{"class":83,"line":589},[185,1813,1260],{"class":282},[185,1815,797],{"class":224},[185,1817,800],{"class":220},[185,1819,1267],{"class":194},[185,1821,807],{"class":806},[185,1823,821],{"class":282},[185,1825,1826],{"class":224},"(result)",[185,1828,1275],{"class":220},[185,1830,813],{"class":806},[185,1832,1833],{"class":194}," row(s) in the window\"",[185,1835,295],{"class":224},[10,1837,1838,1839,1842,1843,1847],{},"Collecting a filtered subset with ",[13,1840,1841],{},"concat"," at the end is a legitimate middle ground: peak memory is the size of the ",[1844,1845,1846],"em",{},"result",", not of the file. It only works while the filter is selective — if half the rows survive, you are back to loading half the workbook and should reconsider the approach.",[171,1849,1851],{"id":1850},"choosing-a-chunk-size","Choosing a chunk size",[1853,1854,1855,1868],"table",{},[1856,1857,1858],"thead",{},[1859,1860,1861,1865],"tr",{},[1862,1863,1864],"th",{},"Chunk size",[1862,1866,1867],{},"Behaviour",[1869,1870,1871,1880,1888,1896],"tbody",{},[1859,1872,1873,1877],{},[1874,1875,1876],"td",{},"1,000",[1874,1878,1879],{},"pandas overhead per chunk dominates; slow",[1859,1881,1882,1885],{},[1874,1883,1884],{},"20,000 – 100,000",[1874,1886,1887],{},"The useful range for most spreadsheet data",[1859,1889,1890,1893],{},[1874,1891,1892],{},"500,000",[1874,1894,1895],{},"Approaching a full load; memory advantage mostly gone",[1859,1897,1898,1901],{},[1874,1899,1900],{},"Whole file",[1874,1902,1903,1904,1906],{},"Use ",[13,1905,349],{}," and stop reading this page",[10,1908,1909],{},"Tune it by watching peak memory rather than by intuition: a chunk of 50,000 rows with six numeric columns is a few megabytes, while the same chunk with a wide free-text column can be a hundred. The chunk you can afford depends on the columns you kept, which is one more argument for pruning them first.",[171,1911,1913],{"id":1912},"which-accumulator-does-each-aggregate-need","Which accumulator does each aggregate need?",[10,1915,1916],{},"Before writing the loop, decide what each output column accumulates. Getting this wrong produces numbers that look reasonable and are not:",[36,1918,45,1923,45,1926,45,1929,45,1932,45,1937,45,1943,45,1946,45,1950,45,1955,45,1959,45,1962,45,1966,45,1969,45,1973,45,1976,45,1978,45,1983,45,1987,45,1990,45,1994,45,1997,45,2001,45,2004],{"viewBox":1919,"role":39,"ariaLabelledBy":1920,"xmlns":43,"style":44},"0 0 740 220",[1921,1922],"ch-agg-t","ch-agg-d",[47,1924,1925],{"id":1921},"How each aggregate combines across chunks",[51,1927,1928],{"id":1922},"Sums and counts add. Minima and maxima take the smaller or larger of the two. Means need a sum and a count kept separately. Medians, percentiles and exact distinct counts cannot be combined and need the whole column.",[55,1930],{"x":57,"y":57,"width":58,"height":1931,"fill":60},"220",[55,1933],{"x":1332,"y":1934,"width":1935,"height":1333,"rx":1936,"fill":68},"26","692","8",[71,1938,1942],{"x":1939,"y":1940,"style":1941},"46","48","font-size:12px;font-weight:700;fill:var(--brand-strong,#4338ca)","aggregate",[71,1944,1945],{"x":1334,"y":1940,"style":1941},"combine with",[71,1947,1949],{"x":1948,"y":1940,"style":1941},"560","safe to chunk?",[55,1951],{"x":1332,"y":1952,"width":1935,"height":1953,"rx":1954,"fill":141,"stroke":132},"64","30","6",[71,1956,1958],{"x":1939,"y":1345,"style":1957},"font-size:11.5px;fill:var(--text,#172033)","sum, count, size",[71,1960,1961],{"x":1334,"y":1345,"style":1957},"addition",[71,1963,1965],{"x":1948,"y":1345,"style":1964},"font-size:11.5px;font-weight:700;fill:var(--teal,#0f9488)","yes",[55,1967],{"x":1332,"y":1968,"width":1935,"height":1953,"rx":1954,"fill":141,"stroke":132},"96",[71,1970,1972],{"x":1939,"y":1971,"style":1957},"116","min, max",[71,1974,1975],{"x":1334,"y":1971,"style":1957},"element-wise min \u002F max",[71,1977,1965],{"x":1948,"y":1971,"style":1964},[55,1979],{"x":1332,"y":1980,"width":1935,"height":1953,"rx":1954,"fill":1981,"stroke":1982},"128","#fdefd8","var(--gold,#b4740a)",[71,1984,1986],{"x":1939,"y":1985,"style":1957},"148","mean, weighted average",[71,1988,1989],{"x":1334,"y":1985,"style":1957},"sum and count separately",[71,1991,1993],{"x":1948,"y":1985,"style":1992},"font-size:11.5px;font-weight:700;fill:var(--gold,#b4740a)","with care",[55,1995],{"x":1332,"y":1996,"width":1935,"height":1953,"rx":1954,"fill":1373,"stroke":1374},"160",[71,1998,2000],{"x":1939,"y":1999,"style":1957},"180","median, percentile, exact nunique",[71,2002,2003],{"x":1334,"y":1999,"style":1957},"cannot be combined",[71,2005,2007],{"x":1948,"y":1999,"style":2006},"font-size:11.5px;font-weight:700;fill:var(--danger,#dc2626)","no — collect the column",[10,2009,2010],{},"For the bottom row there is still a way through: stream just that one column into a list or a NumPy array. A single float column of 400,000 values is about three megabytes, so an exact median is affordable even when the full frame is not.",[171,2012,2014],{"id":2013},"common-pitfalls-and-gotchas","Common pitfalls and gotchas",[1853,2016,2017,2030],{},[1856,2018,2019],{},[1859,2020,2021,2024,2027],{},[1862,2022,2023],{},"Symptom",[1862,2025,2026],{},"Cause",[1862,2028,2029],{},"Fix",[1869,2031,2032,2045,2056,2067,2085,2102,2116],{},[1859,2033,2034,2037,2042],{},[1874,2035,2036],{},"Chunked read slower than a full read",[1874,2038,2039,2041],{},[13,2040,277],{}," paging, re-parsing each call",[1874,2043,2044],{},"Use an openpyxl row iterator",[1859,2046,2047,2050,2053],{},[1874,2048,2049],{},"Memory still climbs",[1874,2051,2052],{},"Every chunk appended to a list",[1874,2054,2055],{},"Aggregate per chunk, keep only the accumulator",[1859,2057,2058,2061,2064],{},[1874,2059,2060],{},"Averages slightly wrong",[1874,2062,2063],{},"Chunk means averaged",[1874,2065,2066],{},"Accumulate sum and count; divide at the end",[1859,2068,2069,2072,2080],{},[1874,2070,2071],{},"Groups missing from the result",[1874,2073,2074,2077,2078],{},[13,2075,2076],{},"add"," without ",[13,2079,1176],{},[1874,2081,2082],{},[13,2083,2084],{},"running.add(part, fill_value=0)",[1859,2086,2087,2093,2096],{},[1874,2088,2089,2092],{},[13,2090,2091],{},"Too many open files"," after many runs",[1874,2094,2095],{},"Workbook never closed",[1874,2097,2098,2099,2101],{},"Close it in a ",[13,2100,874],{}," block",[1859,2103,2104,2107,2110],{},[1874,2105,2106],{},"Header row appears in the data",[1874,2108,2109],{},"Iterator not advanced past row 1",[1874,2111,2112,2113],{},"Consume the header with ",[13,2114,2115],{},"next(rows)",[1859,2117,2118,2125,2128],{},[1874,2119,2120,2121,2124],{},"Blank rows create ",[13,2122,2123],{},"NaN","-only records",[1874,2126,2127],{},"Trailing formatted rows in the sheet",[1874,2129,2130,2131],{},"Skip rows where every value is ",[13,2132,341],{},[171,2134,2136],{"id":2135},"performance-and-scale-notes","Performance and scale notes",[10,2138,2139,2140,2142],{},"One parse of the sheet is unavoidable, so the floor on time is however long openpyxl takes to stream the file — typically a few seconds per hundred thousand rows with ",[13,2141,866],{},". Everything you save beyond that comes from doing less per row: fewer columns picked, no per-row Python function calls, and vectorised work on the chunk rather than on individual values. If the same large file is read repeatedly, convert it once to Parquet and read that instead; a chunked Excel read is the right tool for a file you receive once, not for one you query daily.",[171,2144,2146],{"id":2145},"conclusion","Conclusion",[10,2148,2149,2150,2152,2153,2156],{},"Excel has no ",[13,2151,19],{},", so build one: stream rows with openpyxl in read-only mode, batch them into DataFrames of twenty to a hundred thousand rows, and aggregate each batch into a small accumulator. Keep the columns narrow, combine with ",[13,2154,2155],{},"add(fill_value=0)",", handle means as sum-and-count, and never hold more than one chunk. Peak memory then depends on your chunk size rather than on the size of the file.",[171,2158,2160],{"id":2159},"frequently-asked-questions","Frequently asked questions",[10,2162,2163,2173,2174,2176,2177,2180,2181,2183],{},[2164,2165,2166,2167,2169,2170,2172],"strong",{},"Does ",[13,2168,349],{}," support ",[13,2171,19],{},"?","\nNo. ",[13,2175,19],{}," exists for ",[13,2178,2179],{},"read_csv"," only, because CSV can be read as a stream. An ",[13,2182,27],{}," sheet is zipped XML that has to be parsed, so chunking has to be built on a row iterator.",[10,2185,2186,2195],{},[2164,2187,2188,2189,2191,2192,2194],{},"Can I use ",[13,2190,277],{}," and ",[13,2193,287],{}," to page through the file?","\nYou can, but each call re-parses the sheet from the start, so a ten-page read costs roughly ten full reads. Use it only for sampling.",[10,2197,2198,2201,2202,2205],{},[2164,2199,2200],{},"Which aggregations work across chunks?","\nSums, counts, minima, maxima and any ",[13,2203,2204],{},"groupby"," of those. Means need the sum and the count accumulated separately. Medians and exact distinct counts need all the data at once.",[10,2207,2208,2211],{},[2164,2209,2210],{},"What chunk size should I use?","\nLarge enough that pandas overhead is amortised, small enough to fit comfortably in your memory budget — 20,000 to 100,000 rows suits most spreadsheet data.",[171,2213,2215],{"id":2214},"related","Related",[10,2217,2218],{},"Up to the parent guide:",[2220,2221,2222],"ul",{},[2223,2224,2225,2227],"li",{},[30,2226,33],{"href":32}," — the full set of large-file techniques.",[10,2229,2230],{},"Related guides:",[2220,2232,2233,2240,2250,2257],{},[2223,2234,2235,2239],{},[30,2236,2238],{"href":2237},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fspeed-up-openpyxl-with-read-only-mode\u002F","Speed Up openpyxl with read_only Mode"," — the streaming reader this pattern is built on.",[2223,2241,2242,2246,2247,2249],{},[30,2243,2245],{"href":2244},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fconvert-excel-to-csv-with-python\u002F","Convert Excel to CSV with Python"," — when a real ",[13,2248,19],{}," becomes available.",[2223,2251,2252,2256],{},[30,2253,2255],{"href":2254},"\u002Fgetting-started-with-python-excel-automation\u002Freading-excel-files-with-pandas\u002Fread-specific-columns-from-excel-with-pandas\u002F","Read Specific Columns from Excel with pandas"," — narrowing the read before anything else.",[2223,2258,2259,2263],{},[30,2260,2262],{"href":2261},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcreating-pivot-tables-from-excel-data\u002F","Creating Pivot Tables from Excel Data"," — the aggregation these chunks feed.",[2265,2266,2267],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}",{"title":181,"searchDepth":234,"depth":234,"links":2269},[2270,2271,2272,2273,2274,2275,2276,2277,2278,2279,2280,2281,2282],{"id":173,"depth":234,"text":174},{"id":204,"depth":234,"text":205},{"id":353,"depth":234,"text":354},{"id":882,"depth":234,"text":883},{"id":1310,"depth":234,"text":1311},{"id":1619,"depth":234,"text":1620},{"id":1850,"depth":234,"text":1851},{"id":1912,"depth":234,"text":1913},{"id":2013,"depth":234,"text":2014},{"id":2135,"depth":234,"text":2136},{"id":2145,"depth":234,"text":2146},{"id":2159,"depth":234,"text":2160},{"id":2214,"depth":234,"text":2215},"2026-08-01","pandas has no chunksize for Excel — build one with an openpyxl row generator, aggregate chunk by chunk, and keep peak memory flat on files with hundreds of thousands of rows.","md",[2287,2290,2293,2295],{"q":2288,"a":2289},"Does read_excel support chunksize?","No. chunksize exists for read_csv only, because CSV can be read as a stream. An xlsx sheet is zipped XML that has to be parsed, so chunking has to be built on a row iterator.",{"q":2291,"a":2292},"Can I use skiprows and nrows to page through the file?","You can, but each call re-parses the sheet from the start, so a ten-page read costs roughly ten full reads. Use it only for sampling.",{"q":2200,"a":2294},"Sums, counts, minima, maxima and any groupby of those. Means need the sum and the count accumulated separately. Medians and exact distinct counts need all the data at once.",{"q":2210,"a":2296},"Large enough that pandas overhead is amortised, small enough to fit comfortably in your memory budget — 20,000 to 100,000 rows suits most spreadsheet data.",{"breadcrumb":2298},[2299,2301,2304,2305],{"name":2300,"item":847},"Home",{"name":2302,"item":2303},"Advanced Data Transformation and Cleaning","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":33,"item":32},{"name":5,"item":2306},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fread-large-excel-file-in-chunks-with-pandas\u002F","\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fread-large-excel-file-in-chunks-with-pandas",{"title":5,"description":2309},"Why read_excel has no chunksize, how to stream rows with openpyxl and batch them into DataFrames, which aggregations combine across chunks, and how to keep means correct.","read-large-excel-file-in-chunks-with-pandas","advanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fread-large-excel-file-in-chunks-with-pandas\u002Findex","how-to","NLQi54Uy4l6DWLv_hZjrctKj8rmAbilkH4L94H2JJgk",[2315,2319],{"title":2316,"path":2317,"stem":2318,"children":-1},"Process Multiple Excel Files in Parallel with Python","\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fprocess-multiple-excel-files-in-parallel-with-python","advanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fprocess-multiple-excel-files-in-parallel-with-python\u002Findex",{"title":2238,"path":2320,"stem":2321,"children":-1},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fspeed-up-openpyxl-with-read-only-mode","advanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fspeed-up-openpyxl-with-read-only-mode\u002Findex",1785584462728]