[{"data":1,"prerenderedAt":4187},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python":4179},{"id":4,"title":5,"body":6,"dateModified":4150,"datePublished":4150,"description":4151,"extension":4152,"faq":4153,"meta":4165,"navigation":252,"path":4172,"seo":4173,"slug":4175,"stem":4176,"type":4177,"__hash__":4178},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Findex.md","Working with Large Excel Files in Python",{"type":7,"value":8,"toc":4130},"minimark",[9,19,162,167,198,204,208,211,572,575,579,586,828,839,843,846,1116,1122,1245,1249,1252,1534,1548,1555,1559,1566,2007,2010,2014,2021,2148,2164,2168,2175,2315,2318,2322,2333,2436,2443,2447,2542,2546,2552,2748,2757,2767,2771,2774,3077,3083,3086,3090,3093,3324,3336,3419,3422,3426,3429,3824,3832,3836,3958,3962,4001,4005,4014,4023,4032,4038,4044,4048,4051,4058,4061,4098,4101,4126],[10,11,12,13,18],"p",{},"Spreadsheets grow until something breaks: a script that took four seconds takes four minutes, a laptop starts swapping, or a scheduled job is killed for using too much memory. Almost always the fix is not a faster machine but a different access pattern — streaming instead of loading, reading four columns instead of forty, and handing bulk data to a format built for it. This guide, part of ",[14,15,17],"a",{"href":16},"\u002Fadvanced-data-transformation-and-cleaning\u002F","Advanced Data Transformation and Cleaning",", works through those patterns in the order you should try them.",[20,21,29,30,29,34,29,38,29,45,29,54,29,61,29,66,29,74,29,79,29,87,29,92,29,96,29,101,29,105,29,108,29,116,29,122,29,126,29,130,29,134,29,141,29,147,29,151,29,156,29,158],"svg",{"viewBox":22,"role":23,"ariaLabelledBy":24,"xmlns":27,"style":28},"0 0 760 258","img",[25,26],"lg-cost-t","lg-cost-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:760px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[31,32,33],"title",{"id":25},"Why a small file becomes a large amount of memory",[35,36,37],"desc",{"id":26},"A 40 megabyte xlsx file is zipped XML. Unzipping expands it to several hundred megabytes of XML, and turning every cell into a Python object expands it again to over a gigabyte. Streaming modes skip the final step by handling one row at a time.",[39,40],"rect",{"x":41,"y":41,"width":42,"height":43,"fill":44},"0","760","258","#ffffff",[39,46],{"x":47,"y":48,"width":49,"height":50,"rx":51,"fill":52,"stroke":53},"30","60","150","70","12","#ebebfd","var(--line,#cdd5e6)",[55,56,60],"text",{"x":57,"y":58,"style":59},"105","88","font-size:13px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","report.xlsx",[55,62,65],{"x":57,"y":63,"style":64},"112","font-size:12px;fill:var(--muted,#5b6780);text-anchor:middle","40 MB on disk",[67,68],"line",{"x1":69,"y1":70,"x2":71,"y2":70,"stroke":72,"style":73},"184","95","220","var(--muted,#5b6780)","stroke-width:2px",[75,76],"polygon",{"points":77,"fill":78},"224,95 212,89 212,101","#5b6780",[39,80],{"x":81,"y":82,"width":83,"height":84,"rx":51,"fill":85,"stroke":86},"226","52","180","86","#f0f4ff","var(--brand,#5b5cf0)",[55,88,91],{"x":89,"y":90,"style":59},"316","80","unzipped XML",[55,93,95],{"x":89,"y":94,"style":64},"104","~400 MB of text",[55,97,100],{"x":89,"y":98,"style":99},"124","font-size:11.5px;fill:var(--muted,#5b6780);text-anchor:middle","one element per cell",[67,102],{"x1":103,"y1":70,"x2":104,"y2":70,"stroke":72,"style":73},"410","446",[75,106],{"points":107,"fill":78},"450,95 438,89 438,101",[39,109],{"x":110,"y":111,"width":112,"height":113,"rx":51,"fill":114,"stroke":115,"style":73},"452","44","278","102","#fce9e9","var(--danger,#dc2626)",[55,117,121],{"x":118,"y":119,"style":120},"591","72","font-size:13px;font-weight:700;fill:var(--danger,#dc2626);text-anchor:middle","Python objects",[55,123,125],{"x":118,"y":124,"style":64},"96","1 GB or more resident",[55,127,129],{"x":118,"y":128,"style":99},"118","value + style ref + coordinate,",[55,131,133],{"x":118,"y":132,"style":99},"136","for every single cell",[39,135],{"x":81,"y":136,"width":137,"height":138,"rx":51,"fill":139,"stroke":140,"style":73},"176","504","58","#d9f4f1","var(--teal,#0f9488)",[55,142,146],{"x":143,"y":144,"style":145},"478","200","font-size:12.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","streaming modes stop at the middle box",[55,148,150],{"x":143,"y":149,"style":99},"222","read_only and write_only hold one row at a time, not the whole sheet",[67,152],{"x1":57,"y1":153,"x2":57,"y2":154,"stroke":140,"style":155},"134","205","stroke-width:1.5px;stroke-dasharray:5 4",[67,157],{"x1":57,"y1":154,"x2":71,"y2":154,"stroke":140,"style":155},[75,159],{"points":160,"fill":161},"224,205 212,199 212,211","#0f766e",[163,164,166],"h2",{"id":165},"install-the-dependencies","Install the dependencies",[168,169,174],"pre",{"className":170,"code":171,"language":172,"meta":173,"style":173},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl pyarrow\n","bash","",[175,176,177],"code",{"__ignoreMap":173},[178,179,181,185,189,192,195],"span",{"class":67,"line":180},1,[178,182,184],{"class":183},"sMTad","pip",[178,186,188],{"class":187},"srMev"," install",[178,190,191],{"class":187}," pandas",[178,193,194],{"class":187}," openpyxl",[178,196,197],{"class":187}," pyarrow\n",[10,199,200,203],{},[175,201,202],{},"pyarrow"," is only needed for the Parquet handoff at the end, but it is the single most useful addition to a large-data toolbox.",[163,205,207],{"id":206},"make-a-big-file-to-experiment-on","Make a big file to experiment on",[10,209,210],{},"Every measurement below is more convincing on your own hardware, so start by generating something substantial:",[168,212,216],{"className":213,"code":214,"language":215,"meta":173,"style":173},"language-python shiki shiki-themes github-light github-dark-high-contrast","import numpy as np\nimport pandas as pd\n\nrows = 400_000\nrng = np.random.default_rng(seed=7)\n\ndf = pd.DataFrame({\n    \"Order_ID\": np.arange(1, rows + 1),\n    \"Order_Date\": pd.date_range(\"2024-01-01\", periods=rows, freq=\"min\"),\n    \"Region\": rng.choice([\"North\", \"South\", \"West\", \"Central\"], rows),\n    \"SKU\": rng.choice([f\"SKU-{n:04d}\" for n in range(500)], rows),\n    \"Quantity\": rng.integers(1, 20, rows),\n    \"Unit_Price\": rng.uniform(5, 200, rows).round(2),\n    \"Notes\": \"\",\n})\ndf.to_excel(\"large.xlsx\", index=False, sheet_name=\"Orders\")\nprint(df.memory_usage(deep=True).sum() \u002F 1e6, \"MB in pandas\")\n","python",[175,217,218,234,247,254,267,290,295,306,330,363,393,444,463,487,502,508,539],{"__ignoreMap":173},[178,219,220,224,228,231],{"class":67,"line":180},[178,221,223],{"class":222},"s-kum","import",[178,225,227],{"class":226},"skGVy"," numpy ",[178,229,230],{"class":222},"as",[178,232,233],{"class":226}," np\n",[178,235,237,239,242,244],{"class":67,"line":236},2,[178,238,223],{"class":222},[178,240,241],{"class":226}," pandas ",[178,243,230],{"class":222},[178,245,246],{"class":226}," pd\n",[178,248,250],{"class":67,"line":249},3,[178,251,253],{"emptyLinePlaceholder":252},true,"\n",[178,255,257,260,263],{"class":67,"line":256},4,[178,258,259],{"class":226},"rows ",[178,261,262],{"class":222},"=",[178,264,266],{"class":265},"sP0c6"," 400_000\n",[178,268,270,273,275,278,282,284,287],{"class":67,"line":269},5,[178,271,272],{"class":226},"rng ",[178,274,262],{"class":222},[178,276,277],{"class":226}," np.random.default_rng(",[178,279,281],{"class":280},"sa561","seed",[178,283,262],{"class":222},[178,285,286],{"class":265},"7",[178,288,289],{"class":226},")\n",[178,291,293],{"class":67,"line":292},6,[178,294,253],{"emptyLinePlaceholder":252},[178,296,298,301,303],{"class":67,"line":297},7,[178,299,300],{"class":226},"df ",[178,302,262],{"class":222},[178,304,305],{"class":226}," pd.DataFrame({\n",[178,307,309,312,315,318,321,324,327],{"class":67,"line":308},8,[178,310,311],{"class":187},"    \"Order_ID\"",[178,313,314],{"class":226},": np.arange(",[178,316,317],{"class":265},"1",[178,319,320],{"class":226},", rows ",[178,322,323],{"class":222},"+",[178,325,326],{"class":265}," 1",[178,328,329],{"class":226},"),\n",[178,331,333,336,339,342,345,348,350,353,356,358,361],{"class":67,"line":332},9,[178,334,335],{"class":187},"    \"Order_Date\"",[178,337,338],{"class":226},": pd.date_range(",[178,340,341],{"class":187},"\"2024-01-01\"",[178,343,344],{"class":226},", ",[178,346,347],{"class":280},"periods",[178,349,262],{"class":222},[178,351,352],{"class":226},"rows, ",[178,354,355],{"class":280},"freq",[178,357,262],{"class":222},[178,359,360],{"class":187},"\"min\"",[178,362,329],{"class":226},[178,364,366,369,372,375,377,380,382,385,387,390],{"class":67,"line":365},10,[178,367,368],{"class":187},"    \"Region\"",[178,370,371],{"class":226},": rng.choice([",[178,373,374],{"class":187},"\"North\"",[178,376,344],{"class":226},[178,378,379],{"class":187},"\"South\"",[178,381,344],{"class":226},[178,383,384],{"class":187},"\"West\"",[178,386,344],{"class":226},[178,388,389],{"class":187},"\"Central\"",[178,391,392],{"class":226},"], rows),\n",[178,394,396,399,401,404,407,411,414,417,420,423,426,429,432,435,438,441],{"class":67,"line":395},11,[178,397,398],{"class":187},"    \"SKU\"",[178,400,371],{"class":226},[178,402,403],{"class":222},"f",[178,405,406],{"class":187},"\"SKU-",[178,408,410],{"class":409},"sSjpA","{",[178,412,413],{"class":226},"n",[178,415,416],{"class":222},":04d",[178,418,419],{"class":409},"}",[178,421,422],{"class":187},"\"",[178,424,425],{"class":222}," for",[178,427,428],{"class":226}," n ",[178,430,431],{"class":222},"in",[178,433,434],{"class":265}," range",[178,436,437],{"class":226},"(",[178,439,440],{"class":265},"500",[178,442,443],{"class":226},")], rows),\n",[178,445,447,450,453,455,457,460],{"class":67,"line":446},12,[178,448,449],{"class":187},"    \"Quantity\"",[178,451,452],{"class":226},": rng.integers(",[178,454,317],{"class":265},[178,456,344],{"class":226},[178,458,459],{"class":265},"20",[178,461,462],{"class":226},", rows),\n",[178,464,466,469,472,475,477,479,482,485],{"class":67,"line":465},13,[178,467,468],{"class":187},"    \"Unit_Price\"",[178,470,471],{"class":226},": rng.uniform(",[178,473,474],{"class":265},"5",[178,476,344],{"class":226},[178,478,144],{"class":265},[178,480,481],{"class":226},", rows).round(",[178,483,484],{"class":265},"2",[178,486,329],{"class":226},[178,488,490,493,496,499],{"class":67,"line":489},14,[178,491,492],{"class":187},"    \"Notes\"",[178,494,495],{"class":226},": ",[178,497,498],{"class":187},"\"\"",[178,500,501],{"class":226},",\n",[178,503,505],{"class":67,"line":504},15,[178,506,507],{"class":226},"})\n",[178,509,511,514,517,519,522,524,527,529,532,534,537],{"class":67,"line":510},16,[178,512,513],{"class":226},"df.to_excel(",[178,515,516],{"class":187},"\"large.xlsx\"",[178,518,344],{"class":226},[178,520,521],{"class":280},"index",[178,523,262],{"class":222},[178,525,526],{"class":265},"False",[178,528,344],{"class":226},[178,530,531],{"class":280},"sheet_name",[178,533,262],{"class":222},[178,535,536],{"class":187},"\"Orders\"",[178,538,289],{"class":226},[178,540,542,545,548,551,553,556,559,562,565,567,570],{"class":67,"line":541},17,[178,543,544],{"class":265},"print",[178,546,547],{"class":226},"(df.memory_usage(",[178,549,550],{"class":280},"deep",[178,552,262],{"class":222},[178,554,555],{"class":265},"True",[178,557,558],{"class":226},").sum() ",[178,560,561],{"class":222},"\u002F",[178,563,564],{"class":265}," 1e6",[178,566,344],{"class":226},[178,568,569],{"class":187},"\"MB in pandas\"",[178,571,289],{"class":226},[10,573,574],{},"That write alone takes a while — which is itself the lesson. Writing 400,000 rows through the default engine builds every cell as an object before serialising.",[163,576,578],{"id":577},"step-1-read-fewer-columns","Step 1: read fewer columns",[10,580,581,582,585],{},"Before any clever streaming, read less. ",[175,583,584],{},"usecols"," is applied during parsing, so skipped columns are never converted into Python objects at all:",[168,587,589],{"className":213,"code":588,"language":215,"meta":173,"style":173},"import time\nimport pandas as pd\n\nstart = time.perf_counter()\nfull = pd.read_excel(\"large.xlsx\", sheet_name=\"Orders\")\nprint(f\"all columns: {time.perf_counter() - start:.1f}s, \"\n      f\"{full.memory_usage(deep=True).sum() \u002F 1e6:.0f} MB\")\n\nstart = time.perf_counter()\nslim = pd.read_excel(\"large.xlsx\", sheet_name=\"Orders\",\n                     usecols=[\"Order_Date\", \"Region\", \"Quantity\", \"Unit_Price\"])\nprint(f\"four columns: {time.perf_counter() - start:.1f}s, \"\n      f\"{slim.memory_usage(deep=True).sum() \u002F 1e6:.0f} MB\")\n",[175,590,591,598,608,612,622,644,674,708,712,720,741,772,797],{"__ignoreMap":173},[178,592,593,595],{"class":67,"line":180},[178,594,223],{"class":222},[178,596,597],{"class":226}," time\n",[178,599,600,602,604,606],{"class":67,"line":236},[178,601,223],{"class":222},[178,603,241],{"class":226},[178,605,230],{"class":222},[178,607,246],{"class":226},[178,609,610],{"class":67,"line":249},[178,611,253],{"emptyLinePlaceholder":252},[178,613,614,617,619],{"class":67,"line":256},[178,615,616],{"class":226},"start ",[178,618,262],{"class":222},[178,620,621],{"class":226}," time.perf_counter()\n",[178,623,624,627,629,632,634,636,638,640,642],{"class":67,"line":269},[178,625,626],{"class":226},"full ",[178,628,262],{"class":222},[178,630,631],{"class":226}," pd.read_excel(",[178,633,516],{"class":187},[178,635,344],{"class":226},[178,637,531],{"class":280},[178,639,262],{"class":222},[178,641,536],{"class":187},[178,643,289],{"class":226},[178,645,646,648,650,652,655,657,660,663,666,669,671],{"class":67,"line":292},[178,647,544],{"class":265},[178,649,437],{"class":226},[178,651,403],{"class":222},[178,653,654],{"class":187},"\"all columns: ",[178,656,410],{"class":409},[178,658,659],{"class":226},"time.perf_counter() ",[178,661,662],{"class":222},"-",[178,664,665],{"class":226}," start",[178,667,668],{"class":222},":.1f",[178,670,419],{"class":409},[178,672,673],{"class":187},"s, \"\n",[178,675,676,679,681,683,686,688,690,692,694,696,698,701,703,706],{"class":67,"line":297},[178,677,678],{"class":222},"      f",[178,680,422],{"class":187},[178,682,410],{"class":409},[178,684,685],{"class":226},"full.memory_usage(",[178,687,550],{"class":280},[178,689,262],{"class":222},[178,691,555],{"class":265},[178,693,558],{"class":226},[178,695,561],{"class":222},[178,697,564],{"class":265},[178,699,700],{"class":222},":.0f",[178,702,419],{"class":409},[178,704,705],{"class":187}," MB\"",[178,707,289],{"class":226},[178,709,710],{"class":67,"line":308},[178,711,253],{"emptyLinePlaceholder":252},[178,713,714,716,718],{"class":67,"line":332},[178,715,616],{"class":226},[178,717,262],{"class":222},[178,719,621],{"class":226},[178,721,722,725,727,729,731,733,735,737,739],{"class":67,"line":365},[178,723,724],{"class":226},"slim ",[178,726,262],{"class":222},[178,728,631],{"class":226},[178,730,516],{"class":187},[178,732,344],{"class":226},[178,734,531],{"class":280},[178,736,262],{"class":222},[178,738,536],{"class":187},[178,740,501],{"class":226},[178,742,743,746,748,751,754,756,759,761,764,766,769],{"class":67,"line":395},[178,744,745],{"class":280},"                     usecols",[178,747,262],{"class":222},[178,749,750],{"class":226},"[",[178,752,753],{"class":187},"\"Order_Date\"",[178,755,344],{"class":226},[178,757,758],{"class":187},"\"Region\"",[178,760,344],{"class":226},[178,762,763],{"class":187},"\"Quantity\"",[178,765,344],{"class":226},[178,767,768],{"class":187},"\"Unit_Price\"",[178,770,771],{"class":226},"])\n",[178,773,774,776,778,780,783,785,787,789,791,793,795],{"class":67,"line":446},[178,775,544],{"class":265},[178,777,437],{"class":226},[178,779,403],{"class":222},[178,781,782],{"class":187},"\"four columns: ",[178,784,410],{"class":409},[178,786,659],{"class":226},[178,788,662],{"class":222},[178,790,665],{"class":226},[178,792,668],{"class":222},[178,794,419],{"class":409},[178,796,673],{"class":187},[178,798,799,801,803,805,808,810,812,814,816,818,820,822,824,826],{"class":67,"line":465},[178,800,678],{"class":222},[178,802,422],{"class":187},[178,804,410],{"class":409},[178,806,807],{"class":226},"slim.memory_usage(",[178,809,550],{"class":280},[178,811,262],{"class":222},[178,813,555],{"class":265},[178,815,558],{"class":226},[178,817,561],{"class":222},[178,819,564],{"class":265},[178,821,700],{"class":222},[178,823,419],{"class":409},[178,825,705],{"class":187},[178,827,289],{"class":226},[10,829,830,831,835,836,838],{},"On a typical export the saving is large, because the columns you do not need are usually the wide text ones. This is the cheapest optimisation available and the one most often skipped — see ",[14,832,834],{"href":833},"\u002Fgetting-started-with-python-excel-automation\u002Freading-excel-files-with-pandas\u002Fread-specific-columns-from-excel-with-pandas\u002F","reading specific columns from Excel with pandas"," for the full set of ",[175,837,584],{}," forms.",[163,840,842],{"id":841},"step-2-right-size-the-dtypes","Step 2: right-size the dtypes",[10,844,845],{},"pandas defaults to 64-bit numbers and Python objects for text. Neither is necessary for most spreadsheet data:",[168,847,849],{"className":213,"code":848,"language":215,"meta":173,"style":173},"import pandas as pd\n\nslim = pd.read_excel(\"large.xlsx\", sheet_name=\"Orders\",\n                     usecols=[\"Order_Date\", \"Region\", \"SKU\", \"Quantity\", \"Unit_Price\"])\nbefore = slim.memory_usage(deep=True).sum() \u002F 1e6\n\nslim[\"Quantity\"] = pd.to_numeric(slim[\"Quantity\"], downcast=\"integer\")     # int64 -> int8\nslim[\"Unit_Price\"] = pd.to_numeric(slim[\"Unit_Price\"], downcast=\"float\")   # float64 -> float32\nslim[\"Region\"] = slim[\"Region\"].astype(\"category\")                         # 4 distinct values\nslim[\"SKU\"] = slim[\"SKU\"].astype(\"category\")                               # 500 distinct values\n\nafter = slim.memory_usage(deep=True).sum() \u002F 1e6\nprint(f\"{before:.0f} MB -> {after:.0f} MB\")\n",[175,850,851,861,865,885,914,937,941,976,1005,1032,1056,1060,1081],{"__ignoreMap":173},[178,852,853,855,857,859],{"class":67,"line":180},[178,854,223],{"class":222},[178,856,241],{"class":226},[178,858,230],{"class":222},[178,860,246],{"class":226},[178,862,863],{"class":67,"line":236},[178,864,253],{"emptyLinePlaceholder":252},[178,866,867,869,871,873,875,877,879,881,883],{"class":67,"line":249},[178,868,724],{"class":226},[178,870,262],{"class":222},[178,872,631],{"class":226},[178,874,516],{"class":187},[178,876,344],{"class":226},[178,878,531],{"class":280},[178,880,262],{"class":222},[178,882,536],{"class":187},[178,884,501],{"class":226},[178,886,887,889,891,893,895,897,899,901,904,906,908,910,912],{"class":67,"line":256},[178,888,745],{"class":280},[178,890,262],{"class":222},[178,892,750],{"class":226},[178,894,753],{"class":187},[178,896,344],{"class":226},[178,898,758],{"class":187},[178,900,344],{"class":226},[178,902,903],{"class":187},"\"SKU\"",[178,905,344],{"class":226},[178,907,763],{"class":187},[178,909,344],{"class":226},[178,911,768],{"class":187},[178,913,771],{"class":226},[178,915,916,919,921,924,926,928,930,932,934],{"class":67,"line":269},[178,917,918],{"class":226},"before ",[178,920,262],{"class":222},[178,922,923],{"class":226}," slim.memory_usage(",[178,925,550],{"class":280},[178,927,262],{"class":222},[178,929,555],{"class":265},[178,931,558],{"class":226},[178,933,561],{"class":222},[178,935,936],{"class":265}," 1e6\n",[178,938,939],{"class":67,"line":292},[178,940,253],{"emptyLinePlaceholder":252},[178,942,943,946,948,951,953,956,958,961,964,966,969,972],{"class":67,"line":297},[178,944,945],{"class":226},"slim[",[178,947,763],{"class":187},[178,949,950],{"class":226},"] ",[178,952,262],{"class":222},[178,954,955],{"class":226}," pd.to_numeric(slim[",[178,957,763],{"class":187},[178,959,960],{"class":226},"], ",[178,962,963],{"class":280},"downcast",[178,965,262],{"class":222},[178,967,968],{"class":187},"\"integer\"",[178,970,971],{"class":226},")     ",[178,973,975],{"class":974},"s-wDw","# int64 -> int8\n",[178,977,978,980,982,984,986,988,990,992,994,996,999,1002],{"class":67,"line":308},[178,979,945],{"class":226},[178,981,768],{"class":187},[178,983,950],{"class":226},[178,985,262],{"class":222},[178,987,955],{"class":226},[178,989,768],{"class":187},[178,991,960],{"class":226},[178,993,963],{"class":280},[178,995,262],{"class":222},[178,997,998],{"class":187},"\"float\"",[178,1000,1001],{"class":226},")   ",[178,1003,1004],{"class":974},"# float64 -> float32\n",[178,1006,1007,1009,1011,1013,1015,1018,1020,1023,1026,1029],{"class":67,"line":332},[178,1008,945],{"class":226},[178,1010,758],{"class":187},[178,1012,950],{"class":226},[178,1014,262],{"class":222},[178,1016,1017],{"class":226}," slim[",[178,1019,758],{"class":187},[178,1021,1022],{"class":226},"].astype(",[178,1024,1025],{"class":187},"\"category\"",[178,1027,1028],{"class":226},")                         ",[178,1030,1031],{"class":974},"# 4 distinct values\n",[178,1033,1034,1036,1038,1040,1042,1044,1046,1048,1050,1053],{"class":67,"line":365},[178,1035,945],{"class":226},[178,1037,903],{"class":187},[178,1039,950],{"class":226},[178,1041,262],{"class":222},[178,1043,1017],{"class":226},[178,1045,903],{"class":187},[178,1047,1022],{"class":226},[178,1049,1025],{"class":187},[178,1051,1052],{"class":226},")                               ",[178,1054,1055],{"class":974},"# 500 distinct values\n",[178,1057,1058],{"class":67,"line":395},[178,1059,253],{"emptyLinePlaceholder":252},[178,1061,1062,1065,1067,1069,1071,1073,1075,1077,1079],{"class":67,"line":446},[178,1063,1064],{"class":226},"after ",[178,1066,262],{"class":222},[178,1068,923],{"class":226},[178,1070,550],{"class":280},[178,1072,262],{"class":222},[178,1074,555],{"class":265},[178,1076,558],{"class":226},[178,1078,561],{"class":222},[178,1080,936],{"class":265},[178,1082,1083,1085,1087,1089,1091,1093,1096,1098,1100,1103,1105,1108,1110,1112,1114],{"class":67,"line":465},[178,1084,544],{"class":265},[178,1086,437],{"class":226},[178,1088,403],{"class":222},[178,1090,422],{"class":187},[178,1092,410],{"class":409},[178,1094,1095],{"class":226},"before",[178,1097,700],{"class":222},[178,1099,419],{"class":409},[178,1101,1102],{"class":187}," MB -> ",[178,1104,410],{"class":409},[178,1106,1107],{"class":226},"after",[178,1109,700],{"class":222},[178,1111,419],{"class":409},[178,1113,705],{"class":187},[178,1115,289],{"class":226},[10,1117,1118,1121],{},[175,1119,1120],{},"category"," is the big win on spreadsheet data, because columns like region, status and product code repeat the same handful of strings for hundreds of thousands of rows. Downcasting numbers helps less but is free. Do it after reading, once, rather than in every downstream step.",[20,1123,29,1128,29,1131,29,1134,29,1137,29,1141,29,1146,29,1149,29,1153,29,1157,29,1160,29,1162,29,1166,29,1169,29,1172,29,1175,29,1178,29,1181,29,1185,29,1188,29,1191,29,1195,29,1198,29,1203,29,1208,29,1211,29,1214,29,1218,29,1221,29,1224,29,1229,29,1232,29,1235,29,1241],{"viewBox":1124,"role":23,"ariaLabelledBy":1125,"xmlns":27,"style":28},"0 0 760 236",[1126,1127],"lg-order-t","lg-order-d",[31,1129,1130],{"id":1126},"The order to try optimisations in",[35,1132,1133],{"id":1127},"Five steps in increasing order of effort: read fewer columns, right-size the dtypes, stream with read-only mode, process in chunks, and finally change format to CSV or Parquet. Each step is only worth taking if the previous one was not enough.",[39,1135],{"x":41,"y":41,"width":42,"height":1136,"fill":44},"236",[39,1138],{"x":1139,"y":50,"width":1140,"height":124,"rx":51,"fill":139,"stroke":140,"style":73},"16","132",[55,1142,1145],{"x":1143,"y":1144,"style":145},"82","100","1 · columns",[55,1147,584],{"x":1143,"y":98,"style":1148},"font-size:11px;fill:var(--muted,#5b6780);text-anchor:middle",[55,1150,1152],{"x":1143,"y":1151,"style":1148},"144","one argument",[67,1154],{"x1":1155,"y1":128,"x2":1156,"y2":128,"stroke":72,"style":73},"152","174",[75,1158],{"points":1159,"fill":78},"178,118 166,112 166,124",[39,1161],{"x":83,"y":50,"width":1140,"height":124,"rx":51,"fill":139,"stroke":140,"style":73},[55,1163,1165],{"x":1164,"y":1144,"style":145},"246","2 · dtypes",[55,1167,1168],{"x":1164,"y":98,"style":1148},"category, downcast",[55,1170,1171],{"x":1164,"y":1151,"style":1148},"three lines",[67,1173],{"x1":89,"y1":128,"x2":1174,"y2":128,"stroke":72,"style":73},"338",[75,1176],{"points":1177,"fill":78},"342,118 330,112 330,124",[39,1179],{"x":1180,"y":50,"width":1140,"height":124,"rx":51,"fill":52,"stroke":53},"344",[55,1182,1184],{"x":103,"y":1144,"style":1183},"font-size:12.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","3 · stream",[55,1186,1187],{"x":103,"y":98,"style":1148},"read_only rows",[55,1189,1190],{"x":103,"y":1151,"style":1148},"rewrite the loop",[67,1192],{"x1":1193,"y1":128,"x2":1194,"y2":128,"stroke":72,"style":73},"480","502",[75,1196],{"points":1197,"fill":78},"506,118 494,112 494,124",[39,1199],{"x":1200,"y":50,"width":63,"height":124,"rx":51,"fill":1201,"stroke":1202},"508","#fdefd8","var(--gold,#b4740a)",[55,1204,1207],{"x":1205,"y":1144,"style":1206},"564","font-size:12.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","4 · chunk",[55,1209,1210],{"x":1205,"y":98,"style":1148},"aggregate per",[55,1212,1213],{"x":1205,"y":1151,"style":1148},"block",[67,1215],{"x1":1216,"y1":128,"x2":1217,"y2":128,"stroke":72,"style":73},"624","646",[75,1219],{"points":1220,"fill":78},"650,118 638,112 638,124",[39,1222],{"x":1223,"y":50,"width":124,"height":124,"rx":51,"fill":114,"stroke":115},"652",[55,1225,1228],{"x":1226,"y":1144,"style":1227},"700","font-size:12.5px;font-weight:700;fill:var(--danger,#dc2626);text-anchor:middle","5 · format",[55,1230,1231],{"x":1226,"y":98,"style":1148},"CSV, Parquet",[55,1233,1234],{"x":1226,"y":1151,"style":1148},"change the deal",[55,1236,1240],{"x":1237,"y":1238,"style":1239},"380","38","font-size:12.5px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","Stop at the first step that makes the job fast enough.",[55,1242,1244],{"x":1237,"y":1243,"style":99},"204","effort increases left to right — so does the amount of code you have to maintain",[163,1246,1248],{"id":1247},"step-3-stream-rows-with-read_only-mode","Step 3: stream rows with read_only mode",[10,1250,1251],{},"When you need every row but only a few values from each, openpyxl's read-only mode parses the sheet lazily and never builds the whole grid:",[168,1253,1255],{"className":213,"code":1254,"language":215,"meta":173,"style":173},"from openpyxl import load_workbook\n\nwb = load_workbook(\"large.xlsx\", read_only=True, data_only=True)\nws = wb[\"Orders\"]\n\nheader = [c.value for c in next(ws.iter_rows(min_row=1, max_row=1))]\nqty_col = header.index(\"Quantity\")\nprice_col = header.index(\"Unit_Price\")\nregion_col = header.index(\"Region\")\n\ntotals = {}\nfor row in ws.iter_rows(min_row=2, values_only=True):\n    region = row[region_col]\n    totals[region] = totals.get(region, 0) + (row[qty_col] or 0) * (row[price_col] or 0)\n\nwb.close()\nprint({k: round(v) for k, v in totals.items()})\n",[175,1256,1257,1270,1274,1306,1321,1325,1368,1382,1395,1408,1412,1422,1452,1462,1502,1506,1511],{"__ignoreMap":173},[178,1258,1259,1262,1265,1267],{"class":67,"line":180},[178,1260,1261],{"class":222},"from",[178,1263,1264],{"class":226}," openpyxl ",[178,1266,223],{"class":222},[178,1268,1269],{"class":226}," load_workbook\n",[178,1271,1272],{"class":67,"line":236},[178,1273,253],{"emptyLinePlaceholder":252},[178,1275,1276,1279,1281,1284,1286,1288,1291,1293,1295,1297,1300,1302,1304],{"class":67,"line":249},[178,1277,1278],{"class":226},"wb ",[178,1280,262],{"class":222},[178,1282,1283],{"class":226}," load_workbook(",[178,1285,516],{"class":187},[178,1287,344],{"class":226},[178,1289,1290],{"class":280},"read_only",[178,1292,262],{"class":222},[178,1294,555],{"class":265},[178,1296,344],{"class":226},[178,1298,1299],{"class":280},"data_only",[178,1301,262],{"class":222},[178,1303,555],{"class":265},[178,1305,289],{"class":226},[178,1307,1308,1311,1313,1316,1318],{"class":67,"line":256},[178,1309,1310],{"class":226},"ws ",[178,1312,262],{"class":222},[178,1314,1315],{"class":226}," wb[",[178,1317,536],{"class":187},[178,1319,1320],{"class":226},"]\n",[178,1322,1323],{"class":67,"line":269},[178,1324,253],{"emptyLinePlaceholder":252},[178,1326,1327,1330,1332,1335,1338,1341,1343,1346,1349,1352,1354,1356,1358,1361,1363,1365],{"class":67,"line":292},[178,1328,1329],{"class":226},"header ",[178,1331,262],{"class":222},[178,1333,1334],{"class":226}," [c.value ",[178,1336,1337],{"class":222},"for",[178,1339,1340],{"class":226}," c ",[178,1342,431],{"class":222},[178,1344,1345],{"class":265}," next",[178,1347,1348],{"class":226},"(ws.iter_rows(",[178,1350,1351],{"class":280},"min_row",[178,1353,262],{"class":222},[178,1355,317],{"class":265},[178,1357,344],{"class":226},[178,1359,1360],{"class":280},"max_row",[178,1362,262],{"class":222},[178,1364,317],{"class":265},[178,1366,1367],{"class":226},"))]\n",[178,1369,1370,1373,1375,1378,1380],{"class":67,"line":297},[178,1371,1372],{"class":226},"qty_col ",[178,1374,262],{"class":222},[178,1376,1377],{"class":226}," header.index(",[178,1379,763],{"class":187},[178,1381,289],{"class":226},[178,1383,1384,1387,1389,1391,1393],{"class":67,"line":308},[178,1385,1386],{"class":226},"price_col ",[178,1388,262],{"class":222},[178,1390,1377],{"class":226},[178,1392,768],{"class":187},[178,1394,289],{"class":226},[178,1396,1397,1400,1402,1404,1406],{"class":67,"line":332},[178,1398,1399],{"class":226},"region_col ",[178,1401,262],{"class":222},[178,1403,1377],{"class":226},[178,1405,758],{"class":187},[178,1407,289],{"class":226},[178,1409,1410],{"class":67,"line":365},[178,1411,253],{"emptyLinePlaceholder":252},[178,1413,1414,1417,1419],{"class":67,"line":395},[178,1415,1416],{"class":226},"totals ",[178,1418,262],{"class":222},[178,1420,1421],{"class":226}," {}\n",[178,1423,1424,1426,1429,1431,1434,1436,1438,1440,1442,1445,1447,1449],{"class":67,"line":446},[178,1425,1337],{"class":222},[178,1427,1428],{"class":226}," row ",[178,1430,431],{"class":222},[178,1432,1433],{"class":226}," ws.iter_rows(",[178,1435,1351],{"class":280},[178,1437,262],{"class":222},[178,1439,484],{"class":265},[178,1441,344],{"class":226},[178,1443,1444],{"class":280},"values_only",[178,1446,262],{"class":222},[178,1448,555],{"class":265},[178,1450,1451],{"class":226},"):\n",[178,1453,1454,1457,1459],{"class":67,"line":465},[178,1455,1456],{"class":226},"    region ",[178,1458,262],{"class":222},[178,1460,1461],{"class":226}," row[region_col]\n",[178,1463,1464,1467,1469,1472,1474,1477,1479,1482,1485,1488,1490,1493,1496,1498,1500],{"class":67,"line":489},[178,1465,1466],{"class":226},"    totals[region] ",[178,1468,262],{"class":222},[178,1470,1471],{"class":226}," totals.get(region, ",[178,1473,41],{"class":265},[178,1475,1476],{"class":226},") ",[178,1478,323],{"class":222},[178,1480,1481],{"class":226}," (row[qty_col] ",[178,1483,1484],{"class":222},"or",[178,1486,1487],{"class":265}," 0",[178,1489,1476],{"class":226},[178,1491,1492],{"class":222},"*",[178,1494,1495],{"class":226}," (row[price_col] ",[178,1497,1484],{"class":222},[178,1499,1487],{"class":265},[178,1501,289],{"class":226},[178,1503,1504],{"class":67,"line":504},[178,1505,253],{"emptyLinePlaceholder":252},[178,1507,1508],{"class":67,"line":510},[178,1509,1510],{"class":226},"wb.close()\n",[178,1512,1513,1515,1518,1521,1524,1526,1529,1531],{"class":67,"line":541},[178,1514,544],{"class":265},[178,1516,1517],{"class":226},"({k: ",[178,1519,1520],{"class":265},"round",[178,1522,1523],{"class":226},"(v) ",[178,1525,1337],{"class":222},[178,1527,1528],{"class":226}," k, v ",[178,1530,431],{"class":222},[178,1532,1533],{"class":226}," totals.items()})\n",[10,1535,1536,1539,1540,1543,1544,1547],{},[175,1537,1538],{},"values_only=True"," is what makes this fast: openpyxl yields plain tuples instead of ",[175,1541,1542],{},"Cell"," objects, skipping the most expensive allocation in the whole library. ",[175,1545,1546],{},"wb.close()"," matters too — a read-only workbook holds the underlying zip open until you close it, and a long-running job that forgets will run out of file handles.",[10,1549,1550,1551,1554],{},"The trade is real: a read-only sheet cannot be written to, ",[175,1552,1553],{},"ws.max_row"," may be unreliable until the sheet has been walked, and cell styling is not available. For aggregation and extraction that is a fair price.",[163,1556,1558],{"id":1557},"step-4-process-in-chunks","Step 4: process in chunks",[10,1560,1561,1562,1565],{},"Sometimes you genuinely need DataFrame operations but not the whole frame at once. ",[175,1563,1564],{},"openpyxl"," streaming plus a chunk buffer gives you both:",[168,1567,1569],{"className":213,"code":1568,"language":215,"meta":173,"style":173},"import pandas as pd\nfrom openpyxl import load_workbook\n\ndef excel_chunks(path, sheet, size=50_000, columns=None):\n    wb = load_workbook(path, read_only=True, data_only=True)\n    ws = wb[sheet]\n    rows = ws.iter_rows(values_only=True)\n    header = list(next(rows))\n    keep = [header.index(c) for c in (columns or header)]\n\n    buffer = []\n    for row in rows:\n        buffer.append([row[i] for i in keep])\n        if len(buffer) >= size:\n            yield pd.DataFrame(buffer, columns=columns or header)\n            buffer = []\n    if buffer:\n        yield pd.DataFrame(buffer, columns=columns or header)\n    wb.close()\n\nrunning = None\nfor chunk in excel_chunks(\"large.xlsx\", \"Orders\", columns=[\"Region\", \"Quantity\", \"Unit_Price\"]):\n    chunk[\"Revenue\"] = chunk[\"Quantity\"] * chunk[\"Unit_Price\"]\n    part = chunk.groupby(\"Region\", observed=True)[\"Revenue\"].sum()\n    running = part if running is None else running.add(part, fill_value=0)\n\nprint(running.round(2))\n",[175,1570,1571,1581,1591,1595,1622,1648,1658,1675,1693,1717,1721,1731,1743,1758,1775,1796,1805,1813,1831,1837,1842,1853,1893,1921,1951,1989,1994],{"__ignoreMap":173},[178,1572,1573,1575,1577,1579],{"class":67,"line":180},[178,1574,223],{"class":222},[178,1576,241],{"class":226},[178,1578,230],{"class":222},[178,1580,246],{"class":226},[178,1582,1583,1585,1587,1589],{"class":67,"line":236},[178,1584,1261],{"class":222},[178,1586,1264],{"class":226},[178,1588,223],{"class":222},[178,1590,1269],{"class":226},[178,1592,1593],{"class":67,"line":249},[178,1594,253],{"emptyLinePlaceholder":252},[178,1596,1597,1600,1604,1607,1609,1612,1615,1617,1620],{"class":67,"line":256},[178,1598,1599],{"class":222},"def",[178,1601,1603],{"class":1602},"s_Opv"," excel_chunks",[178,1605,1606],{"class":226},"(path, sheet, size",[178,1608,262],{"class":222},[178,1610,1611],{"class":265},"50_000",[178,1613,1614],{"class":226},", columns",[178,1616,262],{"class":222},[178,1618,1619],{"class":265},"None",[178,1621,1451],{"class":226},[178,1623,1624,1627,1629,1632,1634,1636,1638,1640,1642,1644,1646],{"class":67,"line":269},[178,1625,1626],{"class":226},"    wb ",[178,1628,262],{"class":222},[178,1630,1631],{"class":226}," load_workbook(path, ",[178,1633,1290],{"class":280},[178,1635,262],{"class":222},[178,1637,555],{"class":265},[178,1639,344],{"class":226},[178,1641,1299],{"class":280},[178,1643,262],{"class":222},[178,1645,555],{"class":265},[178,1647,289],{"class":226},[178,1649,1650,1653,1655],{"class":67,"line":292},[178,1651,1652],{"class":226},"    ws ",[178,1654,262],{"class":222},[178,1656,1657],{"class":226}," wb[sheet]\n",[178,1659,1660,1663,1665,1667,1669,1671,1673],{"class":67,"line":297},[178,1661,1662],{"class":226},"    rows ",[178,1664,262],{"class":222},[178,1666,1433],{"class":226},[178,1668,1444],{"class":280},[178,1670,262],{"class":222},[178,1672,555],{"class":265},[178,1674,289],{"class":226},[178,1676,1677,1680,1682,1685,1687,1690],{"class":67,"line":308},[178,1678,1679],{"class":226},"    header ",[178,1681,262],{"class":222},[178,1683,1684],{"class":265}," list",[178,1686,437],{"class":226},[178,1688,1689],{"class":265},"next",[178,1691,1692],{"class":226},"(rows))\n",[178,1694,1695,1698,1700,1703,1705,1707,1709,1712,1714],{"class":67,"line":332},[178,1696,1697],{"class":226},"    keep ",[178,1699,262],{"class":222},[178,1701,1702],{"class":226}," [header.index(c) ",[178,1704,1337],{"class":222},[178,1706,1340],{"class":226},[178,1708,431],{"class":222},[178,1710,1711],{"class":226}," (columns ",[178,1713,1484],{"class":222},[178,1715,1716],{"class":226}," header)]\n",[178,1718,1719],{"class":67,"line":365},[178,1720,253],{"emptyLinePlaceholder":252},[178,1722,1723,1726,1728],{"class":67,"line":395},[178,1724,1725],{"class":226},"    buffer ",[178,1727,262],{"class":222},[178,1729,1730],{"class":226}," []\n",[178,1732,1733,1736,1738,1740],{"class":67,"line":446},[178,1734,1735],{"class":222},"    for",[178,1737,1428],{"class":226},[178,1739,431],{"class":222},[178,1741,1742],{"class":226}," rows:\n",[178,1744,1745,1748,1750,1753,1755],{"class":67,"line":465},[178,1746,1747],{"class":226},"        buffer.append([row[i] ",[178,1749,1337],{"class":222},[178,1751,1752],{"class":226}," i ",[178,1754,431],{"class":222},[178,1756,1757],{"class":226}," keep])\n",[178,1759,1760,1763,1766,1769,1772],{"class":67,"line":489},[178,1761,1762],{"class":222},"        if",[178,1764,1765],{"class":265}," len",[178,1767,1768],{"class":226},"(buffer) ",[178,1770,1771],{"class":222},">=",[178,1773,1774],{"class":226}," size:\n",[178,1776,1777,1780,1783,1786,1788,1791,1793],{"class":67,"line":504},[178,1778,1779],{"class":222},"            yield",[178,1781,1782],{"class":226}," pd.DataFrame(buffer, ",[178,1784,1785],{"class":280},"columns",[178,1787,262],{"class":222},[178,1789,1790],{"class":226},"columns ",[178,1792,1484],{"class":222},[178,1794,1795],{"class":226}," header)\n",[178,1797,1798,1801,1803],{"class":67,"line":510},[178,1799,1800],{"class":226},"            buffer ",[178,1802,262],{"class":222},[178,1804,1730],{"class":226},[178,1806,1807,1810],{"class":67,"line":541},[178,1808,1809],{"class":222},"    if",[178,1811,1812],{"class":226}," buffer:\n",[178,1814,1816,1819,1821,1823,1825,1827,1829],{"class":67,"line":1815},18,[178,1817,1818],{"class":222},"        yield",[178,1820,1782],{"class":226},[178,1822,1785],{"class":280},[178,1824,262],{"class":222},[178,1826,1790],{"class":226},[178,1828,1484],{"class":222},[178,1830,1795],{"class":226},[178,1832,1834],{"class":67,"line":1833},19,[178,1835,1836],{"class":226},"    wb.close()\n",[178,1838,1840],{"class":67,"line":1839},20,[178,1841,253],{"emptyLinePlaceholder":252},[178,1843,1845,1848,1850],{"class":67,"line":1844},21,[178,1846,1847],{"class":226},"running ",[178,1849,262],{"class":222},[178,1851,1852],{"class":265}," None\n",[178,1854,1856,1858,1861,1863,1866,1868,1870,1872,1874,1876,1878,1880,1882,1884,1886,1888,1890],{"class":67,"line":1855},22,[178,1857,1337],{"class":222},[178,1859,1860],{"class":226}," chunk ",[178,1862,431],{"class":222},[178,1864,1865],{"class":226}," excel_chunks(",[178,1867,516],{"class":187},[178,1869,344],{"class":226},[178,1871,536],{"class":187},[178,1873,344],{"class":226},[178,1875,1785],{"class":280},[178,1877,262],{"class":222},[178,1879,750],{"class":226},[178,1881,758],{"class":187},[178,1883,344],{"class":226},[178,1885,763],{"class":187},[178,1887,344],{"class":226},[178,1889,768],{"class":187},[178,1891,1892],{"class":226},"]):\n",[178,1894,1896,1899,1902,1904,1906,1909,1911,1913,1915,1917,1919],{"class":67,"line":1895},23,[178,1897,1898],{"class":226},"    chunk[",[178,1900,1901],{"class":187},"\"Revenue\"",[178,1903,950],{"class":226},[178,1905,262],{"class":222},[178,1907,1908],{"class":226}," chunk[",[178,1910,763],{"class":187},[178,1912,950],{"class":226},[178,1914,1492],{"class":222},[178,1916,1908],{"class":226},[178,1918,768],{"class":187},[178,1920,1320],{"class":226},[178,1922,1924,1927,1929,1932,1934,1936,1939,1941,1943,1946,1948],{"class":67,"line":1923},24,[178,1925,1926],{"class":226},"    part ",[178,1928,262],{"class":222},[178,1930,1931],{"class":226}," chunk.groupby(",[178,1933,758],{"class":187},[178,1935,344],{"class":226},[178,1937,1938],{"class":280},"observed",[178,1940,262],{"class":222},[178,1942,555],{"class":265},[178,1944,1945],{"class":226},")[",[178,1947,1901],{"class":187},[178,1949,1950],{"class":226},"].sum()\n",[178,1952,1954,1957,1959,1962,1965,1968,1971,1974,1977,1980,1983,1985,1987],{"class":67,"line":1953},25,[178,1955,1956],{"class":226},"    running ",[178,1958,262],{"class":222},[178,1960,1961],{"class":226}," part ",[178,1963,1964],{"class":222},"if",[178,1966,1967],{"class":226}," running ",[178,1969,1970],{"class":222},"is",[178,1972,1973],{"class":265}," None",[178,1975,1976],{"class":222}," else",[178,1978,1979],{"class":226}," running.add(part, ",[178,1981,1982],{"class":280},"fill_value",[178,1984,262],{"class":222},[178,1986,41],{"class":265},[178,1988,289],{"class":226},[178,1990,1992],{"class":67,"line":1991},26,[178,1993,253],{"emptyLinePlaceholder":252},[178,1995,1997,1999,2002,2004],{"class":67,"line":1996},27,[178,1998,544],{"class":265},[178,2000,2001],{"class":226},"(running.round(",[178,2003,484],{"class":265},[178,2005,2006],{"class":226},"))\n",[10,2008,2009],{},"The pattern generalises to any aggregation that can be combined across chunks — sums, counts, minima and maxima all work. Averages need care: accumulate the sum and the count separately and divide at the end, because the mean of chunk means is only correct when the chunks are the same size.",[163,2011,2013],{"id":2012},"step-5-writing-large-output","Step 5: writing large output",[10,2015,2016,2017,2020],{},"Writing has its own streaming mode. ",[175,2018,2019],{},"write_only=True"," builds the file row by row and never holds the sheet:",[168,2022,2024],{"className":213,"code":2023,"language":215,"meta":173,"style":173},"from openpyxl import Workbook\n\nwb = Workbook(write_only=True)\nws = wb.create_sheet(\"Export\")\nws.append([\"Order_ID\", \"Region\", \"Revenue\"])\n\nfor order_id in range(1, 300_001):\n    ws.append([order_id, \"North\", order_id * 1.5])\n\nwb.save(\"streamed.xlsx\")\n",[175,2025,2026,2037,2041,2059,2073,2091,2095,2117,2134,2138],{"__ignoreMap":173},[178,2027,2028,2030,2032,2034],{"class":67,"line":180},[178,2029,1261],{"class":222},[178,2031,1264],{"class":226},[178,2033,223],{"class":222},[178,2035,2036],{"class":226}," Workbook\n",[178,2038,2039],{"class":67,"line":236},[178,2040,253],{"emptyLinePlaceholder":252},[178,2042,2043,2045,2047,2050,2053,2055,2057],{"class":67,"line":249},[178,2044,1278],{"class":226},[178,2046,262],{"class":222},[178,2048,2049],{"class":226}," Workbook(",[178,2051,2052],{"class":280},"write_only",[178,2054,262],{"class":222},[178,2056,555],{"class":265},[178,2058,289],{"class":226},[178,2060,2061,2063,2065,2068,2071],{"class":67,"line":256},[178,2062,1310],{"class":226},[178,2064,262],{"class":222},[178,2066,2067],{"class":226}," wb.create_sheet(",[178,2069,2070],{"class":187},"\"Export\"",[178,2072,289],{"class":226},[178,2074,2075,2078,2081,2083,2085,2087,2089],{"class":67,"line":269},[178,2076,2077],{"class":226},"ws.append([",[178,2079,2080],{"class":187},"\"Order_ID\"",[178,2082,344],{"class":226},[178,2084,758],{"class":187},[178,2086,344],{"class":226},[178,2088,1901],{"class":187},[178,2090,771],{"class":226},[178,2092,2093],{"class":67,"line":292},[178,2094,253],{"emptyLinePlaceholder":252},[178,2096,2097,2099,2102,2104,2106,2108,2110,2112,2115],{"class":67,"line":297},[178,2098,1337],{"class":222},[178,2100,2101],{"class":226}," order_id ",[178,2103,431],{"class":222},[178,2105,434],{"class":265},[178,2107,437],{"class":226},[178,2109,317],{"class":265},[178,2111,344],{"class":226},[178,2113,2114],{"class":265},"300_001",[178,2116,1451],{"class":226},[178,2118,2119,2122,2124,2127,2129,2132],{"class":67,"line":308},[178,2120,2121],{"class":226},"    ws.append([order_id, ",[178,2123,374],{"class":187},[178,2125,2126],{"class":226},", order_id ",[178,2128,1492],{"class":222},[178,2130,2131],{"class":265}," 1.5",[178,2133,771],{"class":226},[178,2135,2136],{"class":67,"line":332},[178,2137,253],{"emptyLinePlaceholder":252},[178,2139,2140,2143,2146],{"class":67,"line":365},[178,2141,2142],{"class":226},"wb.save(",[178,2144,2145],{"class":187},"\"streamed.xlsx\"",[178,2147,289],{"class":226},[10,2149,2150,2151,2155,2156,2159,2160,2163],{},"Rows must be appended in order and cannot be revisited, so column widths, ",[14,2152,2154],{"href":2153},"\u002Fadvanced-data-transformation-and-cleaning\u002Fapplying-conditional-formatting-with-openpyxl\u002F","conditional formatting"," and totals have to be planned before the loop rather than added afterwards. ",[175,2157,2158],{},"xlsxwriter"," offers an equivalent through its ",[175,2161,2162],{},"constant_memory"," option, and is generally the faster of the two for pure generation.",[163,2165,2167],{"id":2166},"step-6-know-when-to-leave-xlsx-behind","Step 6: know when to leave xlsx behind",[10,2169,2170,2171,2174],{},"The honest answer for genuinely large data is that ",[175,2172,2173],{},".xlsx"," is the wrong container. It is a presentation format that happens to hold data, and every technique above is a way of paying less for that mismatch:",[168,2176,2178],{"className":213,"code":2177,"language":215,"meta":173,"style":173},"import pandas as pd\n\ndf = pd.read_excel(\"large.xlsx\", sheet_name=\"Orders\")\n\ndf.to_csv(\"orders.csv\", index=False)              # universal, streamable, no types\ndf.to_parquet(\"orders.parquet\", index=False)      # typed, compressed, columnar\n\n# Later runs read the fast format instead\nfast = pd.read_parquet(\"orders.parquet\", columns=[\"Region\", \"Quantity\", \"Unit_Price\"])\nprint(len(fast))\n",[175,2179,2180,2190,2194,2214,2218,2240,2262,2266,2271,2303],{"__ignoreMap":173},[178,2181,2182,2184,2186,2188],{"class":67,"line":180},[178,2183,223],{"class":222},[178,2185,241],{"class":226},[178,2187,230],{"class":222},[178,2189,246],{"class":226},[178,2191,2192],{"class":67,"line":236},[178,2193,253],{"emptyLinePlaceholder":252},[178,2195,2196,2198,2200,2202,2204,2206,2208,2210,2212],{"class":67,"line":249},[178,2197,300],{"class":226},[178,2199,262],{"class":222},[178,2201,631],{"class":226},[178,2203,516],{"class":187},[178,2205,344],{"class":226},[178,2207,531],{"class":280},[178,2209,262],{"class":222},[178,2211,536],{"class":187},[178,2213,289],{"class":226},[178,2215,2216],{"class":67,"line":256},[178,2217,253],{"emptyLinePlaceholder":252},[178,2219,2220,2223,2226,2228,2230,2232,2234,2237],{"class":67,"line":269},[178,2221,2222],{"class":226},"df.to_csv(",[178,2224,2225],{"class":187},"\"orders.csv\"",[178,2227,344],{"class":226},[178,2229,521],{"class":280},[178,2231,262],{"class":222},[178,2233,526],{"class":265},[178,2235,2236],{"class":226},")              ",[178,2238,2239],{"class":974},"# universal, streamable, no types\n",[178,2241,2242,2245,2248,2250,2252,2254,2256,2259],{"class":67,"line":292},[178,2243,2244],{"class":226},"df.to_parquet(",[178,2246,2247],{"class":187},"\"orders.parquet\"",[178,2249,344],{"class":226},[178,2251,521],{"class":280},[178,2253,262],{"class":222},[178,2255,526],{"class":265},[178,2257,2258],{"class":226},")      ",[178,2260,2261],{"class":974},"# typed, compressed, columnar\n",[178,2263,2264],{"class":67,"line":297},[178,2265,253],{"emptyLinePlaceholder":252},[178,2267,2268],{"class":67,"line":308},[178,2269,2270],{"class":974},"# Later runs read the fast format instead\n",[178,2272,2273,2276,2278,2281,2283,2285,2287,2289,2291,2293,2295,2297,2299,2301],{"class":67,"line":332},[178,2274,2275],{"class":226},"fast ",[178,2277,262],{"class":222},[178,2279,2280],{"class":226}," pd.read_parquet(",[178,2282,2247],{"class":187},[178,2284,344],{"class":226},[178,2286,1785],{"class":280},[178,2288,262],{"class":222},[178,2290,750],{"class":226},[178,2292,758],{"class":187},[178,2294,344],{"class":226},[178,2296,763],{"class":187},[178,2298,344],{"class":226},[178,2300,768],{"class":187},[178,2302,771],{"class":226},[178,2304,2305,2307,2309,2312],{"class":67,"line":365},[178,2306,544],{"class":265},[178,2308,437],{"class":226},[178,2310,2311],{"class":265},"len",[178,2313,2314],{"class":226},"(fast))\n",[10,2316,2317],{},"Parquet keeps dtypes, compresses well, and lets you read three columns of a hundred without touching the rest. The pattern that works in practice is a split: bulk data moves through CSV or Parquet, and Excel is produced only for the summary a person actually opens — usually a few hundred rows, which is exactly the size spreadsheets are good at.",[163,2319,2321],{"id":2320},"one-more-lever-skip-the-styles","One more lever: skip the styles",[10,2323,2324,2325,2328,2329,2332],{},"A surprising share of the time spent opening a workbook goes on its style table rather than its values. openpyxl parses every font, fill and border definition before it hands you a single cell, and a file that has been edited by many people over several years accumulates thousands of them. ",[175,2326,2327],{},"rich_text=False"," (the default) and ",[175,2330,2331],{},"read_only=True"," already avoid most of the cost, and where you control the source file, reducing the number of distinct styles helps every consumer:",[168,2334,2336],{"className":213,"code":2335,"language":215,"meta":173,"style":173},"from openpyxl import load_workbook\n\nwb = load_workbook(\"large.xlsx\")\nprint(f\"{len(wb._cell_styles)} distinct cell style(s)\")\nprint(f\"{len(wb._fonts)} font(s), {len(wb._fills)} fill(s), {len(wb._borders)} border(s)\")\n",[175,2337,2338,2348,2352,2364,2388],{"__ignoreMap":173},[178,2339,2340,2342,2344,2346],{"class":67,"line":180},[178,2341,1261],{"class":222},[178,2343,1264],{"class":226},[178,2345,223],{"class":222},[178,2347,1269],{"class":226},[178,2349,2350],{"class":67,"line":236},[178,2351,253],{"emptyLinePlaceholder":252},[178,2353,2354,2356,2358,2360,2362],{"class":67,"line":249},[178,2355,1278],{"class":226},[178,2357,262],{"class":222},[178,2359,1283],{"class":226},[178,2361,516],{"class":187},[178,2363,289],{"class":226},[178,2365,2366,2368,2370,2372,2374,2376,2378,2381,2383,2386],{"class":67,"line":256},[178,2367,544],{"class":265},[178,2369,437],{"class":226},[178,2371,403],{"class":222},[178,2373,422],{"class":187},[178,2375,410],{"class":409},[178,2377,2311],{"class":265},[178,2379,2380],{"class":226},"(wb._cell_styles)",[178,2382,419],{"class":409},[178,2384,2385],{"class":187}," distinct cell style(s)\"",[178,2387,289],{"class":226},[178,2389,2390,2392,2394,2396,2398,2400,2402,2405,2407,2410,2412,2414,2417,2419,2422,2424,2426,2429,2431,2434],{"class":67,"line":269},[178,2391,544],{"class":265},[178,2393,437],{"class":226},[178,2395,403],{"class":222},[178,2397,422],{"class":187},[178,2399,410],{"class":409},[178,2401,2311],{"class":265},[178,2403,2404],{"class":226},"(wb._fonts)",[178,2406,419],{"class":409},[178,2408,2409],{"class":187}," font(s), ",[178,2411,410],{"class":409},[178,2413,2311],{"class":265},[178,2415,2416],{"class":226},"(wb._fills)",[178,2418,419],{"class":409},[178,2420,2421],{"class":187}," fill(s), ",[178,2423,410],{"class":409},[178,2425,2311],{"class":265},[178,2427,2428],{"class":226},"(wb._borders)",[178,2430,419],{"class":409},[178,2432,2433],{"class":187}," border(s)\"",[178,2435,289],{"class":226},[10,2437,2438,2439,2442],{},"Those counts are diagnostic rather than actionable on a file you receive, but they explain the otherwise baffling case of two files with identical row counts where one takes five times longer to open. When your own job generates the file, reuse a small number of named style objects rather than constructing a new ",[175,2440,2441],{},"Font"," per cell — the visual result is the same and the style table stays tiny.",[163,2444,2446],{"id":2445},"what-to-reach-for-when","What to reach for when",[2448,2449,2450,2466],"table",{},[2451,2452,2453],"thead",{},[2454,2455,2456,2460,2463],"tr",{},[2457,2458,2459],"th",{},"Symptom",[2457,2461,2462],{},"First thing to try",[2457,2464,2465],{},"If that is not enough",[2467,2468,2469,2485,2496,2507,2520,2531],"tbody",{},[2454,2470,2471,2475,2480],{},[2472,2473,2474],"td",{},"Read is slow, memory fine",[2472,2476,2477,2479],{},[175,2478,584],{}," and a named sheet",[2472,2481,2482,2484],{},[175,2483,1290],{}," streaming",[2454,2486,2487,2490,2493],{},[2472,2488,2489],{},"Memory spikes on read",[2472,2491,2492],{},"Category dtypes and downcasting",[2472,2494,2495],{},"Chunked processing",[2454,2497,2498,2501,2504],{},[2472,2499,2500],{},"Job killed by the scheduler",[2472,2502,2503],{},"Streaming aggregation",[2472,2505,2506],{},"Convert to Parquet once, read that",[2454,2508,2509,2512,2517],{},[2472,2510,2511],{},"Write takes minutes",[2472,2513,2514,2516],{},[175,2515,2052],{}," or xlsxwriter",[2472,2518,2519],{},"Write CSV, produce a small summary workbook",[2454,2521,2522,2525,2528],{},[2472,2523,2524],{},"File is slow for readers too",[2472,2526,2527],{},"Fewer formulas, fewer styled cells",[2472,2529,2530],{},"Split into a data file plus a summary",[2454,2532,2533,2536,2539],{},[2472,2534,2535],{},"Everything is slow",[2472,2537,2538],{},"Column pruning first",[2472,2540,2541],{},"Change format — the file is the problem",[163,2543,2545],{"id":2544},"reading-only-part-of-a-very-wide-sheet","Reading only part of a very wide sheet",[10,2547,2548,2549,2551],{},"Some exports are not tall but wide: two hundred columns of which you need six. ",[175,2550,584],{}," handles the pandas side, and openpyxl can restrict the cell range directly, which avoids parsing the rest of each row:",[168,2553,2555],{"className":213,"code":2554,"language":215,"meta":173,"style":173},"from openpyxl import load_workbook\nfrom openpyxl.utils import column_index_from_string\n\nwb = load_workbook(\"wide.xlsx\", read_only=True, data_only=True)\nws = wb[\"Data\"]\n\nfirst = column_index_from_string(\"B\")\nlast = column_index_from_string(\"G\")\n\nfor row in ws.iter_rows(min_row=2, min_col=first, max_col=last, values_only=True):\n    if row[0] is None:\n        continue                      # skip rows where the key column is blank\n    # row now holds exactly columns B..G for this record\n    pass\n\nwb.close()\n",[175,2556,2557,2567,2579,2583,2612,2625,2629,2644,2658,2662,2704,2722,2730,2735,2740,2744],{"__ignoreMap":173},[178,2558,2559,2561,2563,2565],{"class":67,"line":180},[178,2560,1261],{"class":222},[178,2562,1264],{"class":226},[178,2564,223],{"class":222},[178,2566,1269],{"class":226},[178,2568,2569,2571,2574,2576],{"class":67,"line":236},[178,2570,1261],{"class":222},[178,2572,2573],{"class":226}," openpyxl.utils ",[178,2575,223],{"class":222},[178,2577,2578],{"class":226}," column_index_from_string\n",[178,2580,2581],{"class":67,"line":249},[178,2582,253],{"emptyLinePlaceholder":252},[178,2584,2585,2587,2589,2591,2594,2596,2598,2600,2602,2604,2606,2608,2610],{"class":67,"line":256},[178,2586,1278],{"class":226},[178,2588,262],{"class":222},[178,2590,1283],{"class":226},[178,2592,2593],{"class":187},"\"wide.xlsx\"",[178,2595,344],{"class":226},[178,2597,1290],{"class":280},[178,2599,262],{"class":222},[178,2601,555],{"class":265},[178,2603,344],{"class":226},[178,2605,1299],{"class":280},[178,2607,262],{"class":222},[178,2609,555],{"class":265},[178,2611,289],{"class":226},[178,2613,2614,2616,2618,2620,2623],{"class":67,"line":269},[178,2615,1310],{"class":226},[178,2617,262],{"class":222},[178,2619,1315],{"class":226},[178,2621,2622],{"class":187},"\"Data\"",[178,2624,1320],{"class":226},[178,2626,2627],{"class":67,"line":292},[178,2628,253],{"emptyLinePlaceholder":252},[178,2630,2631,2634,2636,2639,2642],{"class":67,"line":297},[178,2632,2633],{"class":226},"first ",[178,2635,262],{"class":222},[178,2637,2638],{"class":226}," column_index_from_string(",[178,2640,2641],{"class":187},"\"B\"",[178,2643,289],{"class":226},[178,2645,2646,2649,2651,2653,2656],{"class":67,"line":308},[178,2647,2648],{"class":226},"last ",[178,2650,262],{"class":222},[178,2652,2638],{"class":226},[178,2654,2655],{"class":187},"\"G\"",[178,2657,289],{"class":226},[178,2659,2660],{"class":67,"line":332},[178,2661,253],{"emptyLinePlaceholder":252},[178,2663,2664,2666,2668,2670,2672,2674,2676,2678,2680,2683,2685,2688,2691,2693,2696,2698,2700,2702],{"class":67,"line":365},[178,2665,1337],{"class":222},[178,2667,1428],{"class":226},[178,2669,431],{"class":222},[178,2671,1433],{"class":226},[178,2673,1351],{"class":280},[178,2675,262],{"class":222},[178,2677,484],{"class":265},[178,2679,344],{"class":226},[178,2681,2682],{"class":280},"min_col",[178,2684,262],{"class":222},[178,2686,2687],{"class":226},"first, ",[178,2689,2690],{"class":280},"max_col",[178,2692,262],{"class":222},[178,2694,2695],{"class":226},"last, ",[178,2697,1444],{"class":280},[178,2699,262],{"class":222},[178,2701,555],{"class":265},[178,2703,1451],{"class":226},[178,2705,2706,2708,2711,2713,2715,2717,2719],{"class":67,"line":395},[178,2707,1809],{"class":222},[178,2709,2710],{"class":226}," row[",[178,2712,41],{"class":265},[178,2714,950],{"class":226},[178,2716,1970],{"class":222},[178,2718,1973],{"class":265},[178,2720,2721],{"class":226},":\n",[178,2723,2724,2727],{"class":67,"line":446},[178,2725,2726],{"class":222},"        continue",[178,2728,2729],{"class":974},"                      # skip rows where the key column is blank\n",[178,2731,2732],{"class":67,"line":465},[178,2733,2734],{"class":974},"    # row now holds exactly columns B..G for this record\n",[178,2736,2737],{"class":67,"line":489},[178,2738,2739],{"class":222},"    pass\n",[178,2741,2742],{"class":67,"line":504},[178,2743,253],{"emptyLinePlaceholder":252},[178,2745,2746],{"class":67,"line":510},[178,2747,1510],{"class":226},[10,2749,2750,2751,2753,2754,2756],{},"Bounding ",[175,2752,2682],{}," and ",[175,2755,2690],{}," is worth doing even in streaming mode, because a row of two hundred values is two hundred tuple entries whether you look at them or not. On a sheet where the interesting columns sit at the left, this alone can halve the parse time.",[10,2758,2759,2760,2762,2763,2766],{},"The same reasoning applies to rows. If a monthly job only needs the current month and the sheet is sorted by date, ",[175,2761,1351],{}," plus an early ",[175,2764,2765],{},"break"," once the dates run past your window turns a full-file scan into a partial one — provided you can trust the sort, which is worth asserting rather than assuming.",[163,2768,2770],{"id":2769},"measure-before-you-optimise","Measure before you optimise",[10,2772,2773],{},"Every recommendation above has exceptions, and the only way to know which applies to your file is to measure it. Three numbers are enough: wall-clock time, peak memory, and the size of the frame you ended up with:",[168,2775,2777],{"className":213,"code":2776,"language":215,"meta":173,"style":173},"import time\nimport tracemalloc\n\nimport pandas as pd\n\ndef measure(label, fn):\n    tracemalloc.start()\n    start = time.perf_counter()\n    result = fn()\n    elapsed = time.perf_counter() - start\n    _, peak = tracemalloc.get_traced_memory()\n    tracemalloc.stop()\n    size = result.memory_usage(deep=True).sum() \u002F 1e6 if hasattr(result, \"memory_usage\") else 0\n    print(f\"{label:28} {elapsed:6.1f}s  peak {peak \u002F 1e6:7.0f} MB  frame {size:6.0f} MB\")\n    return result\n\nmeasure(\"everything\", lambda: pd.read_excel(\"large.xlsx\", sheet_name=\"Orders\"))\nmeasure(\"four columns\", lambda: pd.read_excel(\n    \"large.xlsx\", sheet_name=\"Orders\",\n    usecols=[\"Order_Date\", \"Region\", \"Quantity\", \"Unit_Price\"]))\n",[175,2778,2779,2785,2792,2796,2806,2810,2820,2825,2834,2844,2859,2869,2874,2916,2982,2990,2994,3022,3036,3051],{"__ignoreMap":173},[178,2780,2781,2783],{"class":67,"line":180},[178,2782,223],{"class":222},[178,2784,597],{"class":226},[178,2786,2787,2789],{"class":67,"line":236},[178,2788,223],{"class":222},[178,2790,2791],{"class":226}," tracemalloc\n",[178,2793,2794],{"class":67,"line":249},[178,2795,253],{"emptyLinePlaceholder":252},[178,2797,2798,2800,2802,2804],{"class":67,"line":256},[178,2799,223],{"class":222},[178,2801,241],{"class":226},[178,2803,230],{"class":222},[178,2805,246],{"class":226},[178,2807,2808],{"class":67,"line":269},[178,2809,253],{"emptyLinePlaceholder":252},[178,2811,2812,2814,2817],{"class":67,"line":292},[178,2813,1599],{"class":222},[178,2815,2816],{"class":1602}," measure",[178,2818,2819],{"class":226},"(label, fn):\n",[178,2821,2822],{"class":67,"line":297},[178,2823,2824],{"class":226},"    tracemalloc.start()\n",[178,2826,2827,2830,2832],{"class":67,"line":308},[178,2828,2829],{"class":226},"    start ",[178,2831,262],{"class":222},[178,2833,621],{"class":226},[178,2835,2836,2839,2841],{"class":67,"line":332},[178,2837,2838],{"class":226},"    result ",[178,2840,262],{"class":222},[178,2842,2843],{"class":226}," fn()\n",[178,2845,2846,2849,2851,2854,2856],{"class":67,"line":365},[178,2847,2848],{"class":226},"    elapsed ",[178,2850,262],{"class":222},[178,2852,2853],{"class":226}," time.perf_counter() ",[178,2855,662],{"class":222},[178,2857,2858],{"class":226}," start\n",[178,2860,2861,2864,2866],{"class":67,"line":395},[178,2862,2863],{"class":226},"    _, peak ",[178,2865,262],{"class":222},[178,2867,2868],{"class":226}," tracemalloc.get_traced_memory()\n",[178,2870,2871],{"class":67,"line":446},[178,2872,2873],{"class":226},"    tracemalloc.stop()\n",[178,2875,2876,2879,2881,2884,2886,2888,2890,2892,2894,2896,2899,2902,2905,2908,2910,2913],{"class":67,"line":465},[178,2877,2878],{"class":226},"    size ",[178,2880,262],{"class":222},[178,2882,2883],{"class":226}," result.memory_usage(",[178,2885,550],{"class":280},[178,2887,262],{"class":222},[178,2889,555],{"class":265},[178,2891,558],{"class":226},[178,2893,561],{"class":222},[178,2895,564],{"class":265},[178,2897,2898],{"class":222}," if",[178,2900,2901],{"class":265}," hasattr",[178,2903,2904],{"class":226},"(result, ",[178,2906,2907],{"class":187},"\"memory_usage\"",[178,2909,1476],{"class":226},[178,2911,2912],{"class":222},"else",[178,2914,2915],{"class":265}," 0\n",[178,2917,2918,2921,2923,2925,2927,2929,2932,2935,2937,2940,2943,2946,2948,2951,2953,2956,2958,2960,2963,2965,2968,2970,2973,2976,2978,2980],{"class":67,"line":489},[178,2919,2920],{"class":265},"    print",[178,2922,437],{"class":226},[178,2924,403],{"class":222},[178,2926,422],{"class":187},[178,2928,410],{"class":409},[178,2930,2931],{"class":226},"label",[178,2933,2934],{"class":222},":28",[178,2936,419],{"class":409},[178,2938,2939],{"class":409}," {",[178,2941,2942],{"class":226},"elapsed",[178,2944,2945],{"class":222},":6.1f",[178,2947,419],{"class":409},[178,2949,2950],{"class":187},"s  peak ",[178,2952,410],{"class":409},[178,2954,2955],{"class":226},"peak ",[178,2957,561],{"class":222},[178,2959,564],{"class":265},[178,2961,2962],{"class":222},":7.0f",[178,2964,419],{"class":409},[178,2966,2967],{"class":187}," MB  frame ",[178,2969,410],{"class":409},[178,2971,2972],{"class":226},"size",[178,2974,2975],{"class":222},":6.0f",[178,2977,419],{"class":409},[178,2979,705],{"class":187},[178,2981,289],{"class":226},[178,2983,2984,2987],{"class":67,"line":504},[178,2985,2986],{"class":222},"    return",[178,2988,2989],{"class":226}," result\n",[178,2991,2992],{"class":67,"line":510},[178,2993,253],{"emptyLinePlaceholder":252},[178,2995,2996,2999,3002,3004,3007,3010,3012,3014,3016,3018,3020],{"class":67,"line":541},[178,2997,2998],{"class":226},"measure(",[178,3000,3001],{"class":187},"\"everything\"",[178,3003,344],{"class":226},[178,3005,3006],{"class":222},"lambda",[178,3008,3009],{"class":226},": pd.read_excel(",[178,3011,516],{"class":187},[178,3013,344],{"class":226},[178,3015,531],{"class":280},[178,3017,262],{"class":222},[178,3019,536],{"class":187},[178,3021,2006],{"class":226},[178,3023,3024,3026,3029,3031,3033],{"class":67,"line":1815},[178,3025,2998],{"class":226},[178,3027,3028],{"class":187},"\"four columns\"",[178,3030,344],{"class":226},[178,3032,3006],{"class":222},[178,3034,3035],{"class":226},": pd.read_excel(\n",[178,3037,3038,3041,3043,3045,3047,3049],{"class":67,"line":1833},[178,3039,3040],{"class":187},"    \"large.xlsx\"",[178,3042,344],{"class":226},[178,3044,531],{"class":280},[178,3046,262],{"class":222},[178,3048,536],{"class":187},[178,3050,501],{"class":226},[178,3052,3053,3056,3058,3060,3062,3064,3066,3068,3070,3072,3074],{"class":67,"line":1839},[178,3054,3055],{"class":280},"    usecols",[178,3057,262],{"class":222},[178,3059,750],{"class":226},[178,3061,753],{"class":187},[178,3063,344],{"class":226},[178,3065,758],{"class":187},[178,3067,344],{"class":226},[178,3069,763],{"class":187},[178,3071,344],{"class":226},[178,3073,768],{"class":187},[178,3075,3076],{"class":226},"]))\n",[10,3078,3079,3082],{},[175,3080,3081],{},"tracemalloc"," reports Python allocations only, which is exactly what you want here — the growth is in objects, not in buffers. Run the comparison on your real file rather than on a generated one: real exports have wide free-text columns, merged cells and per-cell styling, and those are precisely the things that make a read expensive.",[10,3084,3085],{},"The number that decides your approach is peak memory against the limit your job runs under. A container capped at 512 MB cannot load a frame that peaks at 900 MB no matter how briefly, and streaming is not an optimisation there — it is the only option.",[163,3087,3089],{"id":3088},"do-not-make-the-readers-file-large-either","Do not make the reader's file large either",[10,3091,3092],{},"A file that is slow for your script is usually slow for the person who opens it, and the causes are different. Python cares about cell count; Excel cares about formulas, styles and conditional formats:",[168,3094,3096],{"className":213,"code":3095,"language":215,"meta":173,"style":173},"from openpyxl import load_workbook\n\nwb = load_workbook(\"large.xlsx\")\nws = wb[\"Orders\"]\n\ncells = ws.max_row * ws.max_column\nformulas = sum(\n    1 for row in ws.iter_rows() for c in row\n    if isinstance(c.value, str) and c.value.startswith(\"=\")\n)\nrules = sum(len(rng.rules) for rng in ws.conditional_formatting)\n\nprint(f\"{cells:,} cells · {formulas:,} formulas · {rules} conditional-format rule(s)\")\nif formulas > 50_000:\n    print(\"consider writing values instead of formulas for the bulk rows\")\n",[175,3097,3098,3108,3112,3124,3136,3140,3155,3168,3191,3217,3221,3247,3251,3298,3313],{"__ignoreMap":173},[178,3099,3100,3102,3104,3106],{"class":67,"line":180},[178,3101,1261],{"class":222},[178,3103,1264],{"class":226},[178,3105,223],{"class":222},[178,3107,1269],{"class":226},[178,3109,3110],{"class":67,"line":236},[178,3111,253],{"emptyLinePlaceholder":252},[178,3113,3114,3116,3118,3120,3122],{"class":67,"line":249},[178,3115,1278],{"class":226},[178,3117,262],{"class":222},[178,3119,1283],{"class":226},[178,3121,516],{"class":187},[178,3123,289],{"class":226},[178,3125,3126,3128,3130,3132,3134],{"class":67,"line":256},[178,3127,1310],{"class":226},[178,3129,262],{"class":222},[178,3131,1315],{"class":226},[178,3133,536],{"class":187},[178,3135,1320],{"class":226},[178,3137,3138],{"class":67,"line":269},[178,3139,253],{"emptyLinePlaceholder":252},[178,3141,3142,3145,3147,3150,3152],{"class":67,"line":292},[178,3143,3144],{"class":226},"cells ",[178,3146,262],{"class":222},[178,3148,3149],{"class":226}," ws.max_row ",[178,3151,1492],{"class":222},[178,3153,3154],{"class":226}," ws.max_column\n",[178,3156,3157,3160,3162,3165],{"class":67,"line":297},[178,3158,3159],{"class":226},"formulas ",[178,3161,262],{"class":222},[178,3163,3164],{"class":265}," sum",[178,3166,3167],{"class":226},"(\n",[178,3169,3170,3173,3175,3177,3179,3182,3184,3186,3188],{"class":67,"line":308},[178,3171,3172],{"class":265},"    1",[178,3174,425],{"class":222},[178,3176,1428],{"class":226},[178,3178,431],{"class":222},[178,3180,3181],{"class":226}," ws.iter_rows() ",[178,3183,1337],{"class":222},[178,3185,1340],{"class":226},[178,3187,431],{"class":222},[178,3189,3190],{"class":226}," row\n",[178,3192,3193,3195,3198,3201,3204,3206,3209,3212,3215],{"class":67,"line":332},[178,3194,1809],{"class":222},[178,3196,3197],{"class":265}," isinstance",[178,3199,3200],{"class":226},"(c.value, ",[178,3202,3203],{"class":265},"str",[178,3205,1476],{"class":226},[178,3207,3208],{"class":222},"and",[178,3210,3211],{"class":226}," c.value.startswith(",[178,3213,3214],{"class":187},"\"=\"",[178,3216,289],{"class":226},[178,3218,3219],{"class":67,"line":365},[178,3220,289],{"class":226},[178,3222,3223,3226,3228,3230,3232,3234,3237,3239,3242,3244],{"class":67,"line":395},[178,3224,3225],{"class":226},"rules ",[178,3227,262],{"class":222},[178,3229,3164],{"class":265},[178,3231,437],{"class":226},[178,3233,2311],{"class":265},[178,3235,3236],{"class":226},"(rng.rules) ",[178,3238,1337],{"class":222},[178,3240,3241],{"class":226}," rng ",[178,3243,431],{"class":222},[178,3245,3246],{"class":226}," ws.conditional_formatting)\n",[178,3248,3249],{"class":67,"line":446},[178,3250,253],{"emptyLinePlaceholder":252},[178,3252,3253,3255,3257,3259,3261,3263,3266,3269,3271,3274,3276,3279,3281,3283,3286,3288,3291,3293,3296],{"class":67,"line":465},[178,3254,544],{"class":265},[178,3256,437],{"class":226},[178,3258,403],{"class":222},[178,3260,422],{"class":187},[178,3262,410],{"class":409},[178,3264,3265],{"class":226},"cells",[178,3267,3268],{"class":222},":,",[178,3270,419],{"class":409},[178,3272,3273],{"class":187}," cells · ",[178,3275,410],{"class":409},[178,3277,3278],{"class":226},"formulas",[178,3280,3268],{"class":222},[178,3282,419],{"class":409},[178,3284,3285],{"class":187}," formulas · ",[178,3287,410],{"class":409},[178,3289,3290],{"class":226},"rules",[178,3292,419],{"class":409},[178,3294,3295],{"class":187}," conditional-format rule(s)\"",[178,3297,289],{"class":226},[178,3299,3300,3302,3305,3308,3311],{"class":67,"line":489},[178,3301,1964],{"class":222},[178,3303,3304],{"class":226}," formulas ",[178,3306,3307],{"class":222},">",[178,3309,3310],{"class":265}," 50_000",[178,3312,2721],{"class":226},[178,3314,3315,3317,3319,3322],{"class":67,"line":504},[178,3316,2920],{"class":265},[178,3318,437],{"class":226},[178,3320,3321],{"class":187},"\"consider writing values instead of formulas for the bulk rows\"",[178,3323,289],{"class":226},[10,3325,3326,3327,3331,3332,3335],{},"Two habits keep delivered workbooks quick. Write values rather than formulas for bulk rows, keeping ",[14,3328,3330],{"href":3329},"\u002Fgetting-started-with-python-excel-automation\u002Fworking-with-excel-formulas-in-python\u002F","live formulas"," for the handful of summary cells a reader might change. And apply conditional formatting to a range rather than to individual cells — one rule over ",[175,3333,3334],{},"A2:D100000"," costs a fraction of ten thousand rules over single cells, and looks identical.",[20,3337,29,3342,29,3345,29,3348,29,3350,29,3353,29,3357,29,3360,29,3365,29,3368,29,3373,29,3378,29,3382,29,3387,29,3390,29,3393,29,3397,29,3402,29,3407,29,3411,29,3413,29,3416],{"viewBox":3338,"role":23,"ariaLabelledBy":3339,"xmlns":27,"style":28},"0 0 760 220",[3340,3341],"lg-split-t","lg-split-d",[31,3343,3344],{"id":3340},"Splitting one heavy workbook into a data file and a summary",[35,3346,3347],{"id":3341},"Four hundred thousand raw rows go to a Parquet or CSV file that only machines read. The aggregation produces a two hundred row summary workbook with formatting and charts, which is the file people open.",[39,3349],{"x":41,"y":41,"width":42,"height":71,"fill":44},[39,3351],{"x":3352,"y":50,"width":83,"height":90,"rx":51,"fill":52,"stroke":53},"24",[55,3354,3356],{"x":3355,"y":1144,"style":1183},"114","400,000 raw rows",[55,3358,3359],{"x":3355,"y":98,"style":99},"from the source system",[67,3361],{"x1":3362,"y1":3363,"x2":3364,"y2":3363,"stroke":72,"style":73},"208","110","244",[75,3366],{"points":3367,"fill":78},"248,110 236,104 236,116",[39,3369],{"x":3370,"y":50,"width":3371,"height":90,"rx":51,"fill":3372},"250","170","#5b5cf0",[55,3374,3377],{"x":3375,"y":1144,"style":3376},"335","font-size:12.5px;font-weight:700;fill:#ffffff;text-anchor:middle","aggregate",[55,3379,3381],{"x":3375,"y":98,"style":3380},"font-size:11.5px;fill:rgba(255,255,255,0.88);text-anchor:middle","streamed or chunked",[67,3383],{"x1":3384,"y1":124,"x2":3385,"y2":3386,"stroke":140,"style":73},"424","470","62",[75,3388],{"points":3389,"fill":161},"474,60 462,58 464,68",[67,3391],{"x1":3384,"y1":98,"x2":3385,"y2":3392,"stroke":1202,"style":73},"158",[75,3394],{"points":3395,"fill":3396},"474,160 462,152 464,162","#8a5808",[39,3398],{"x":3399,"y":3400,"width":3401,"height":50,"rx":51,"fill":139,"stroke":140,"style":73},"476","26","260",[55,3403,3406],{"x":3404,"y":3405,"style":145},"606","54","summary.xlsx — 200 rows",[55,3408,3410],{"x":3404,"y":3409,"style":99},"78","styled, charted, opens instantly",[39,3412],{"x":3399,"y":98,"width":3401,"height":50,"rx":51,"fill":1201,"stroke":1202,"style":73},[55,3414,3415],{"x":3404,"y":1155,"style":1206},"orders.parquet — all rows",[55,3417,3418],{"x":3404,"y":136,"style":99},"machines only, typed and compressed",[10,3420,3421],{},"That split is the single structural decision that makes large reporting workflows pleasant. The detail is still available to anyone who needs it, and the artefact people actually open stays a few hundred kilobytes.",[163,3423,3425],{"id":3424},"a-worked-pipeline","A worked pipeline",[10,3427,3428],{},"Putting the pieces together, a monthly job over a very large export looks like this:",[168,3430,3432],{"className":213,"code":3431,"language":215,"meta":173,"style":173},"import pandas as pd\n\nSOURCE = \"large.xlsx\"\nCOLUMNS = [\"Order_Date\", \"Region\", \"SKU\", \"Quantity\", \"Unit_Price\"]\n\ndef build_summary():\n    running = None\n    for chunk in excel_chunks(SOURCE, \"Orders\", size=50_000, columns=COLUMNS):\n        chunk[\"Revenue\"] = chunk[\"Quantity\"] * chunk[\"Unit_Price\"]\n        chunk[\"Month\"] = pd.to_datetime(chunk[\"Order_Date\"]).dt.to_period(\"M\").astype(str)\n        part = chunk.groupby([\"Month\", \"Region\"], observed=True).agg(\n            Revenue=(\"Revenue\", \"sum\"), Orders=(\"Revenue\", \"size\")\n        )\n        running = part if running is None else running.add(part, fill_value=0)\n    return running.reset_index().sort_values([\"Month\", \"Region\"])\n\nsummary = build_summary()\nsummary.to_parquet(\"orders_summary.parquet\", index=False)\n\nwith pd.ExcelWriter(\"summary.xlsx\", engine=\"openpyxl\") as writer:\n    summary.to_excel(writer, sheet_name=\"Monthly\", index=False)\n\nprint(f\"{len(summary)} summary row(s) from a 400,000-row source\")\n",[175,3433,3434,3444,3448,3459,3489,3493,3503,3511,3545,3570,3599,3626,3661,3666,3695,3710,3714,3724,3742,3746,3774,3796,3800],{"__ignoreMap":173},[178,3435,3436,3438,3440,3442],{"class":67,"line":180},[178,3437,223],{"class":222},[178,3439,241],{"class":226},[178,3441,230],{"class":222},[178,3443,246],{"class":226},[178,3445,3446],{"class":67,"line":236},[178,3447,253],{"emptyLinePlaceholder":252},[178,3449,3450,3453,3456],{"class":67,"line":249},[178,3451,3452],{"class":265},"SOURCE",[178,3454,3455],{"class":222}," =",[178,3457,3458],{"class":187}," \"large.xlsx\"\n",[178,3460,3461,3464,3466,3469,3471,3473,3475,3477,3479,3481,3483,3485,3487],{"class":67,"line":256},[178,3462,3463],{"class":265},"COLUMNS",[178,3465,3455],{"class":222},[178,3467,3468],{"class":226}," [",[178,3470,753],{"class":187},[178,3472,344],{"class":226},[178,3474,758],{"class":187},[178,3476,344],{"class":226},[178,3478,903],{"class":187},[178,3480,344],{"class":226},[178,3482,763],{"class":187},[178,3484,344],{"class":226},[178,3486,768],{"class":187},[178,3488,1320],{"class":226},[178,3490,3491],{"class":67,"line":269},[178,3492,253],{"emptyLinePlaceholder":252},[178,3494,3495,3497,3500],{"class":67,"line":292},[178,3496,1599],{"class":222},[178,3498,3499],{"class":1602}," build_summary",[178,3501,3502],{"class":226},"():\n",[178,3504,3505,3507,3509],{"class":67,"line":297},[178,3506,1956],{"class":226},[178,3508,262],{"class":222},[178,3510,1852],{"class":265},[178,3512,3513,3515,3517,3519,3521,3523,3525,3527,3529,3531,3533,3535,3537,3539,3541,3543],{"class":67,"line":308},[178,3514,1735],{"class":222},[178,3516,1860],{"class":226},[178,3518,431],{"class":222},[178,3520,1865],{"class":226},[178,3522,3452],{"class":265},[178,3524,344],{"class":226},[178,3526,536],{"class":187},[178,3528,344],{"class":226},[178,3530,2972],{"class":280},[178,3532,262],{"class":222},[178,3534,1611],{"class":265},[178,3536,344],{"class":226},[178,3538,1785],{"class":280},[178,3540,262],{"class":222},[178,3542,3463],{"class":265},[178,3544,1451],{"class":226},[178,3546,3547,3550,3552,3554,3556,3558,3560,3562,3564,3566,3568],{"class":67,"line":332},[178,3548,3549],{"class":226},"        chunk[",[178,3551,1901],{"class":187},[178,3553,950],{"class":226},[178,3555,262],{"class":222},[178,3557,1908],{"class":226},[178,3559,763],{"class":187},[178,3561,950],{"class":226},[178,3563,1492],{"class":222},[178,3565,1908],{"class":226},[178,3567,768],{"class":187},[178,3569,1320],{"class":226},[178,3571,3572,3574,3577,3579,3581,3584,3586,3589,3592,3595,3597],{"class":67,"line":365},[178,3573,3549],{"class":226},[178,3575,3576],{"class":187},"\"Month\"",[178,3578,950],{"class":226},[178,3580,262],{"class":222},[178,3582,3583],{"class":226}," pd.to_datetime(chunk[",[178,3585,753],{"class":187},[178,3587,3588],{"class":226},"]).dt.to_period(",[178,3590,3591],{"class":187},"\"M\"",[178,3593,3594],{"class":226},").astype(",[178,3596,3203],{"class":265},[178,3598,289],{"class":226},[178,3600,3601,3604,3606,3609,3611,3613,3615,3617,3619,3621,3623],{"class":67,"line":395},[178,3602,3603],{"class":226},"        part ",[178,3605,262],{"class":222},[178,3607,3608],{"class":226}," chunk.groupby([",[178,3610,3576],{"class":187},[178,3612,344],{"class":226},[178,3614,758],{"class":187},[178,3616,960],{"class":226},[178,3618,1938],{"class":280},[178,3620,262],{"class":222},[178,3622,555],{"class":265},[178,3624,3625],{"class":226},").agg(\n",[178,3627,3628,3631,3633,3635,3637,3639,3642,3645,3648,3650,3652,3654,3656,3659],{"class":67,"line":446},[178,3629,3630],{"class":280},"            Revenue",[178,3632,262],{"class":222},[178,3634,437],{"class":226},[178,3636,1901],{"class":187},[178,3638,344],{"class":226},[178,3640,3641],{"class":187},"\"sum\"",[178,3643,3644],{"class":226},"), ",[178,3646,3647],{"class":280},"Orders",[178,3649,262],{"class":222},[178,3651,437],{"class":226},[178,3653,1901],{"class":187},[178,3655,344],{"class":226},[178,3657,3658],{"class":187},"\"size\"",[178,3660,289],{"class":226},[178,3662,3663],{"class":67,"line":465},[178,3664,3665],{"class":226},"        )\n",[178,3667,3668,3671,3673,3675,3677,3679,3681,3683,3685,3687,3689,3691,3693],{"class":67,"line":489},[178,3669,3670],{"class":226},"        running ",[178,3672,262],{"class":222},[178,3674,1961],{"class":226},[178,3676,1964],{"class":222},[178,3678,1967],{"class":226},[178,3680,1970],{"class":222},[178,3682,1973],{"class":265},[178,3684,1976],{"class":222},[178,3686,1979],{"class":226},[178,3688,1982],{"class":280},[178,3690,262],{"class":222},[178,3692,41],{"class":265},[178,3694,289],{"class":226},[178,3696,3697,3699,3702,3704,3706,3708],{"class":67,"line":504},[178,3698,2986],{"class":222},[178,3700,3701],{"class":226}," running.reset_index().sort_values([",[178,3703,3576],{"class":187},[178,3705,344],{"class":226},[178,3707,758],{"class":187},[178,3709,771],{"class":226},[178,3711,3712],{"class":67,"line":510},[178,3713,253],{"emptyLinePlaceholder":252},[178,3715,3716,3719,3721],{"class":67,"line":541},[178,3717,3718],{"class":226},"summary ",[178,3720,262],{"class":222},[178,3722,3723],{"class":226}," build_summary()\n",[178,3725,3726,3729,3732,3734,3736,3738,3740],{"class":67,"line":1815},[178,3727,3728],{"class":226},"summary.to_parquet(",[178,3730,3731],{"class":187},"\"orders_summary.parquet\"",[178,3733,344],{"class":226},[178,3735,521],{"class":280},[178,3737,262],{"class":222},[178,3739,526],{"class":265},[178,3741,289],{"class":226},[178,3743,3744],{"class":67,"line":1833},[178,3745,253],{"emptyLinePlaceholder":252},[178,3747,3748,3751,3754,3757,3759,3762,3764,3767,3769,3771],{"class":67,"line":1839},[178,3749,3750],{"class":222},"with",[178,3752,3753],{"class":226}," pd.ExcelWriter(",[178,3755,3756],{"class":187},"\"summary.xlsx\"",[178,3758,344],{"class":226},[178,3760,3761],{"class":280},"engine",[178,3763,262],{"class":222},[178,3765,3766],{"class":187},"\"openpyxl\"",[178,3768,1476],{"class":226},[178,3770,230],{"class":222},[178,3772,3773],{"class":226}," writer:\n",[178,3775,3776,3779,3781,3783,3786,3788,3790,3792,3794],{"class":67,"line":1844},[178,3777,3778],{"class":226},"    summary.to_excel(writer, ",[178,3780,531],{"class":280},[178,3782,262],{"class":222},[178,3784,3785],{"class":187},"\"Monthly\"",[178,3787,344],{"class":226},[178,3789,521],{"class":280},[178,3791,262],{"class":222},[178,3793,526],{"class":265},[178,3795,289],{"class":226},[178,3797,3798],{"class":67,"line":1855},[178,3799,253],{"emptyLinePlaceholder":252},[178,3801,3802,3804,3806,3808,3810,3812,3814,3817,3819,3822],{"class":67,"line":1895},[178,3803,544],{"class":265},[178,3805,437],{"class":226},[178,3807,403],{"class":222},[178,3809,422],{"class":187},[178,3811,410],{"class":409},[178,3813,2311],{"class":265},[178,3815,3816],{"class":226},"(summary)",[178,3818,419],{"class":409},[178,3820,3821],{"class":187}," summary row(s) from a 400,000-row source\"",[178,3823,289],{"class":226},[10,3825,3826,3827,3831],{},"Peak memory stays at roughly one chunk, the workbook a person receives has a few hundred rows, and the full detail is one Parquet read away. Wrap it in the ",[14,3828,3830],{"href":3829},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002F","error handling and logging"," patterns and it is ready to be scheduled.",[163,3833,3835],{"id":3834},"common-errors-and-fixes","Common errors and fixes",[2448,3837,3838,3850],{},[2451,3839,3840],{},[2454,3841,3842,3844,3847],{},[2457,3843,2459],{},[2457,3845,3846],{},"Cause",[2457,3848,3849],{},"Fix",[2467,3851,3852,3871,3884,3900,3916,3927,3940],{},[2454,3853,3854,3863,3866],{},[2472,3855,3856,3859,3860],{},[175,3857,3858],{},"MemoryError"," on ",[175,3861,3862],{},"read_excel",[2472,3864,3865],{},"Whole sheet materialised as objects",[2472,3867,3868,3870],{},[175,3869,584],{},", category dtypes, or stream instead",[2454,3872,3873,3876,3879],{},[2472,3874,3875],{},"Job killed with no traceback",[2472,3877,3878],{},"The scheduler's memory limit was hit",[2472,3880,3881,3882],{},"Chunk the processing; measure peak with ",[175,3883,3081],{},[2454,3885,3886,3891,3894],{},[2472,3887,3888,3890],{},[175,3889,1553],{}," wrong in read-only mode",[2472,3892,3893],{},"Dimensions are lazy until the sheet is walked",[2472,3895,3896,3897],{},"Count rows as you iterate, or call ",[175,3898,3899],{},"ws.calculate_dimension()",[2454,3901,3902,3907,3910],{},[2472,3903,3904],{},[175,3905,3906],{},"Too many open files",[2472,3908,3909],{},"Read-only workbooks never closed",[2472,3911,3912,3913,3915],{},"Call ",[175,3914,1546],{},", or use a context manager",[2454,3917,3918,3921,3924],{},[2472,3919,3920],{},"Written file is enormous",[2472,3922,3923],{},"Per-cell styles or formulas on every row",[2472,3925,3926],{},"Style ranges, write values for bulk rows",[2454,3928,3929,3934,3937],{},[2472,3930,3931,3933],{},[175,3932,2052],{}," sheet raises on cell access",[2472,3935,3936],{},"Rows cannot be revisited in that mode",[2472,3938,3939],{},"Compute everything before appending the row",[2454,3941,3942,3945,3951],{},[2472,3943,3944],{},"Rounding differs after downcasting",[2472,3946,3947,3950],{},[175,3948,3949],{},"float32"," has fewer significant digits",[2472,3952,3953,3954,3957],{},"Keep money as ",[175,3955,3956],{},"float64"," or as integer cents",[163,3959,3961],{"id":3960},"key-takeaways","Key takeaways",[3963,3964,3965,3971,3974,3979,3987,3990,3998],"ul",{},[3966,3967,3968,3970],"li",{},[175,3969,2173],{}," in memory is routinely twenty to fifty times its size on disk, because every cell becomes an object.",[3966,3972,3973],{},"Reading fewer columns is the cheapest and most effective optimisation; try it before anything clever.",[3966,3975,3976,3978],{},[175,3977,1120],{}," dtypes shrink the repeated-string columns that dominate spreadsheet exports.",[3966,3980,3981,3983,3984,3986],{},[175,3982,1290],{}," with ",[175,3985,1538],{}," streams rows as tuples and skips the expensive allocations — close the workbook afterwards.",[3966,3988,3989],{},"Chunked processing works for any aggregation that combines across blocks; means need sums and counts kept separately.",[3966,3991,3992,3994,3995,3997],{},[175,3993,2052],{}," (or xlsxwriter's ",[175,3996,2162],{},") streams output, at the cost of not being able to revisit a row.",[3966,3999,4000],{},"Past a certain size the right move is a format change: bulk data in Parquet or CSV, a small workbook for the reader.",[163,4002,4004],{"id":4003},"frequently-asked-questions","Frequently asked questions",[10,4006,4007,4011,4013],{},[4008,4009,4010],"strong",{},"Why does a 40 MB xlsx use over a gigabyte of memory?",[175,4012,2173],{}," is zipped XML. Every cell becomes a Python object carrying its value, style reference and coordinate, so the in-memory form is routinely twenty to fifty times the file size.",[10,4015,4016,4019,4020,4022],{},[4008,4017,4018],{},"What is the single biggest win on a slow read?","\nReading fewer columns. ",[175,4021,584],{}," is applied while parsing, so the cells you skip are never turned into objects at all.",[10,4024,4025,4031],{},[4008,4026,4027,4028,4030],{},"Does ",[175,4029,1290],{}," mode change what I can do?","\nYes. A read-only worksheet streams rows and cannot be written to, and cells are lightweight read-only objects without full style information.",[10,4033,4034,4037],{},[4008,4035,4036],{},"When should I stop using xlsx altogether?","\nWhen the data is machine-to-machine. Convert to CSV or Parquet for the pipeline and produce a spreadsheet only for the summary a person actually reads.",[10,4039,4040,4043],{},[4008,4041,4042],{},"Is there a row limit?","\nA worksheet holds at most 1,048,576 rows and 16,384 columns. Long before that, opening the file becomes painful for the reader, which is a better limit to design against.",[163,4045,4047],{"id":4046},"related","Related",[10,4049,4050],{},"Up to the parent guide:",[3963,4052,4053],{},[3966,4054,4055,4057],{},[14,4056,17],{"href":16}," — the pipeline these techniques sit inside.",[10,4059,4060],{},"Go deeper here:",[3963,4062,4063,4070,4077,4084,4091],{},[3966,4064,4065,4069],{},[14,4066,4068],{"href":4067},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fread-large-excel-file-in-chunks-with-pandas\u002F","Read a Large Excel File in Chunks with pandas"," — the chunking pattern in full.",[3966,4071,4072,4076],{},[14,4073,4075],{"href":4074},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fspeed-up-openpyxl-with-read-only-mode\u002F","Speed Up openpyxl with read_only Mode"," — streaming reads and their limits.",[3966,4078,4079,4083],{},[14,4080,4082],{"href":4081},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fwrite-large-dataframes-to-excel-with-write-only-mode\u002F","Write Large DataFrames to Excel with write_only Mode"," — generating big workbooks without the memory.",[3966,4085,4086,4090],{},[14,4087,4089],{"href":4088},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fconvert-excel-to-csv-with-python\u002F","Convert Excel to CSV with Python"," — the handoff to a streamable format.",[3966,4092,4093,4097],{},[14,4094,4096],{"href":4095},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fprocess-multiple-excel-files-in-parallel-with-python\u002F","Process Multiple Excel Files in Parallel with Python"," — using more than one core on a folder of files.",[10,4099,4100],{},"Related areas:",[3963,4102,4103,4112,4119],{},[3966,4104,4105,4108,4109,4111],{},[14,4106,4107],{"href":833},"Read Specific Columns from Excel with pandas"," — the ",[175,4110,584],{}," forms in detail.",[3966,4113,4114,4118],{},[14,4115,4117],{"href":4116},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002F","Cleaning Excel Data with pandas"," — the transformations you will be streaming.",[3966,4120,4121,4125],{},[14,4122,4124],{"href":4123},"\u002Fgetting-started-with-python-excel-automation\u002Fworking-with-multiple-excel-sheets-in-python\u002Fcombine-multiple-excel-files-into-one-python\u002F","Combine Multiple Excel Files into One with Python"," — where large inputs usually come from.",[4127,4128,4129],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}",{"title":173,"searchDepth":236,"depth":236,"links":4131},[4132,4133,4134,4135,4136,4137,4138,4139,4140,4141,4142,4143,4144,4145,4146,4147,4148,4149],{"id":165,"depth":236,"text":166},{"id":206,"depth":236,"text":207},{"id":577,"depth":236,"text":578},{"id":841,"depth":236,"text":842},{"id":1247,"depth":236,"text":1248},{"id":1557,"depth":236,"text":1558},{"id":2012,"depth":236,"text":2013},{"id":2166,"depth":236,"text":2167},{"id":2320,"depth":236,"text":2321},{"id":2445,"depth":236,"text":2446},{"id":2544,"depth":236,"text":2545},{"id":2769,"depth":236,"text":2770},{"id":3088,"depth":236,"text":3089},{"id":3424,"depth":236,"text":3425},{"id":3834,"depth":236,"text":3835},{"id":3960,"depth":236,"text":3961},{"id":4003,"depth":236,"text":4004},{"id":4046,"depth":236,"text":4047},"2026-08-01","Read, transform and write spreadsheets that no longer fit comfortably in memory — read-only and write-only openpyxl modes, chunked pandas reads, column pruning, and when to leave .xlsx behind.","md",[4154,4156,4158,4161,4163],{"q":4010,"a":4155},".xlsx is zipped XML. Every cell becomes a Python object carrying its value, style reference and coordinate, so the in-memory form is routinely twenty to fifty times the file size.",{"q":4018,"a":4157},"Reading fewer columns. usecols is applied while parsing, so the cells you skip are never turned into objects at all.",{"q":4159,"a":4160},"Does read_only mode change what I can do?","Yes. A read-only worksheet streams rows and cannot be written to, and cells are lightweight read-only objects without full style information.",{"q":4036,"a":4162},"When the data is machine-to-machine. Convert to CSV or Parquet for the pipeline and produce a spreadsheet only for the summary a person actually reads.",{"q":4042,"a":4164},"A worksheet holds at most 1,048,576 rows and 16,384 columns. Long before that, opening the file becomes painful for the reader, which is a better limit to design against.",{"breadcrumb":4166},[4167,4169,4170],{"name":4168,"item":561},"Home",{"name":17,"item":16},{"name":5,"item":4171},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002F","\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python",{"title":5,"description":4174},"Practical techniques for large .xlsx files: openpyxl read_only and write_only streaming, chunked and column-limited pandas reads, dtype downcasting, CSV and Parquet handoffs.","working-with-large-excel-files-in-python","advanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Findex","guide","pq0aY11oFXFNUJ1k-uAzujQKSqq7x7c4sAY0Q6yD6U8",[4180,4184],{"title":4181,"path":4182,"stem":4183,"children":-1},"Validate Excel Columns Before Import with pandas","\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fvalidate-excel-columns-before-import-with-pandas","advanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fvalidate-excel-columns-before-import-with-pandas\u002Findex",{"title":4089,"path":4185,"stem":4186,"children":-1},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fconvert-excel-to-csv-with-python","advanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fconvert-excel-to-csv-with-python\u002Findex",1785584462252]