[{"data":1,"prerenderedAt":2202},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fstream-excel-rows-into-a-database-with-openpyxl":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fstream-excel-rows-into-a-database-with-openpyxl":2168,"ancestors:\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fstream-excel-rows-into-a-database-with-openpyxl":2177,"children:\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fstream-excel-rows-into-a-database-with-openpyxl":2182,"related:\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fstream-excel-rows-into-a-database-with-openpyxl":2183},{"id":4,"title":5,"body":6,"dateModified":2134,"datePublished":2134,"description":2135,"extension":2136,"faq":2137,"keyword":2146,"kind":2147,"meta":2148,"navigation":292,"path":2156,"seo":2157,"slug":2160,"stem":2161,"topics":2162,"type":2166,"__hash__":2167},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fstream-excel-rows-into-a-database-with-openpyxl\u002Findex.md","Stream Excel Rows into a Database with openpyxl",{"type":7,"value":8,"toc":2122},"minimark",[9,24,200,205,232,242,648,655,659,662,922,937,941,944,1205,1212,1216,1229,1580,1583,1644,1648,1651,1728,1743,1747,1754,1865,1876,2007,2010,2014,2047,2051,2057,2061,2067,2073,2079,2085,2089,2118],[10,11,12,13,17,18,23],"p",{},"pandas is the natural way to read an Excel file, but it builds the whole sheet in memory before anything is written. For a large export going straight into a database — a year of transactions, a full product catalogue — that is unnecessary: each row only needs to be read, checked and inserted. openpyxl's read-only mode yields rows one at a time, and the database's ",[14,15,16],"code",{},"executemany"," inserts them in batches. Memory stays flat however long the sheet is. This guide, part of ",[19,20,22],"a",{"href":21},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002F","Working with Large Excel Files in Python",", builds a streaming loader with validation, a rejects table and a safe rerun, using SQLite so every example runs as written.",[25,26,34,35,34,39,34,43,34,51,34,61,34,72,34,79,34,83,34,88,34,94,34,99,34,102,34,108,34,112,34,116,34,119,34,123,34,126,34,131,34,138,34,143,34,146,34,149,34,153,34,156,34,161,34,168,34,173,34,177,34,181,34,184,34,187,34,193,34,197],"svg",{"viewBox":27,"role":28,"ariaLabelledBy":29,"xmlns":32,"style":33},"4 0 752 142","img",[30,31],"st-flow-t","st-flow-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:752px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[36,37,38],"title",{"id":30},"A streaming load, one batch at a time",[40,41,42],"desc",{"id":31},"openpyxl read-only mode yields rows as tuples. Each batch of five thousand rows is converted and validated, good rows are inserted with executemany and bad rows go to a rejects table, and the whole load commits in one transaction before the workbook is closed.",[44,45],"rect",{"x":46,"y":47,"width":48,"height":49,"fill":50},"4","0","752","142","#ffffff",[44,52],{"x":53,"y":54,"width":55,"height":56,"rx":57,"fill":58,"stroke":59,"style":60},"20.0","16.0","115.2","110.5","12","#f0f4ff","var(--brand,#5b5cf0)","stroke-width:2px",[62,63,65],"g",{"transform":64},"translate(65.6,29.0) scale(1.000)",[66,67],"path",{"d":68,"fill":69,"stroke":70,"style":71},"M4 4h16v16H4z M4 9h16 M4 14h16 M10 4v16","none","var(--brand-strong,#4338ca)","stroke-width:1.80px;stroke-linecap:round;stroke-linejoin:round",[73,74,78],"text",{"x":75,"y":76,"style":77},"77.6","75.5","font-size:13.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Open",[73,80,82],{"x":75,"y":81,"style":77},"93.0","read-only",[73,84,87],{"x":75,"y":85,"style":86},"112.5","font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","load_workbook",[89,90],"line",{"x1":91,"y1":92,"x2":93,"y2":92,"stroke":59,"style":60},"140.2","71.2","162.2",[95,96],"polygon",{"points":97,"fill":98},"166.2,71.2 157.2,76.2 157.2,66.2","#5b5cf0",[44,100],{"x":101,"y":54,"width":55,"height":56,"rx":57,"fill":58,"stroke":59,"style":60},"171.2",[62,103,105],{"transform":104},"translate(216.8,29.0) scale(1.000)",[66,106],{"d":107,"fill":69,"stroke":70,"style":71},"M12 3l9 5-9 5-9-5z M3 13l9 5 9-5 M3 17l9 5 9-5",[73,109,111],{"x":110,"y":76,"style":77},"228.8","Take a batch",[73,113,115],{"x":110,"y":114,"style":86},"95.0","islice(rows,",[73,117,118],{"x":110,"y":85,"style":86},"5000)",[89,120],{"x1":121,"y1":92,"x2":122,"y2":92,"stroke":59,"style":60},"291.4","313.4",[95,124],{"points":125,"fill":98},"317.4,71.2 308.4,76.2 308.4,66.2",[44,127],{"x":128,"y":54,"width":55,"height":56,"rx":57,"fill":129,"stroke":130,"style":60},"322.4","#fdefd8","var(--gold,#b4740a)",[62,132,134],{"transform":133},"translate(368.0,29.0) scale(1.000)",[66,135],{"d":136,"fill":69,"stroke":137,"style":71},"M3 5h18l-7 8v6l-4 2v-8z","var(--gold-ink,#7a4e06)",[73,139,142],{"x":140,"y":76,"style":141},"380.0","font-size:13.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","Convert +",[73,144,145],{"x":140,"y":81,"style":141},"check",[73,147,148],{"x":140,"y":85,"style":86},"per row",[89,150],{"x1":151,"y1":92,"x2":152,"y2":92,"stroke":59,"style":60},"442.6","464.6",[95,154],{"points":155,"fill":98},"468.6,71.2 459.6,76.2 459.6,66.2",[44,157],{"x":158,"y":54,"width":55,"height":56,"rx":57,"fill":159,"stroke":160,"style":60},"473.6","#d9f4f1","var(--teal,#0f9488)",[62,162,164],{"transform":163},"translate(519.2,37.8) scale(1.000)",[66,165],{"d":166,"fill":69,"stroke":167,"style":71},"M5 6c0-1.7 3.1-3 7-3s7 1.3 7 3-3.1 3-7 3-7-1.3-7-3z M5 6v12c0 1.7 3.1 3 7 3s7-1.3 7-3V6 M5 12c0 1.7 3.1 3 7 3s7-1.3 7-3","var(--teal-ink,#0b6157)",[73,169,16],{"x":170,"y":171,"style":172},"531.2","84.2","font-size:13.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle",[73,174,176],{"x":170,"y":175,"style":86},"103.8","good → table",[89,178],{"x1":179,"y1":92,"x2":180,"y2":92,"stroke":59,"style":60},"593.8","615.8",[95,182],{"points":183,"fill":98},"619.8,71.2 610.8,76.2 610.8,66.2",[44,185],{"x":186,"y":54,"width":55,"height":56,"rx":57,"fill":58,"stroke":59,"style":60},"624.8",[62,188,190],{"transform":189},"translate(670.4,37.8) scale(1.000)",[66,191],{"d":192,"fill":69,"stroke":70,"style":71},"M6 11h12v9H6z M8.5 11V8a3.5 3.5 0 0 1 7 0v3",[73,194,196],{"x":195,"y":171,"style":77},"682.4","Commit",[73,198,199],{"x":195,"y":175,"style":86},"one transaction",[201,202,204],"h2",{"id":203},"prerequisites","Prerequisites",[206,207,212],"pre",{"className":208,"code":209,"language":210,"meta":211,"style":211},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install openpyxl pandas\n","bash","",[14,213,214],{"__ignoreMap":211},[215,216,218,222,226,229],"span",{"class":89,"line":217},1,[215,219,221],{"class":220},"sMTad","pip",[215,223,225],{"class":224},"srMev"," install",[215,227,228],{"class":224}," openpyxl",[215,230,231],{"class":224}," pandas\n",[10,233,234,237,238,241],{},[14,235,236],{},"sqlite3"," ships with Python; the same pattern works with any DB-API driver, such as ",[14,239,240],{},"psycopg"," for PostgreSQL. A sample export — small here so it runs quickly, with two bad rows mixed in:",[206,243,247],{"className":244,"code":245,"language":246,"meta":211,"style":211},"language-python shiki shiki-themes github-light github-dark-high-contrast","import datetime as dt\nimport sqlite3\nfrom itertools import islice\n\nfrom openpyxl import Workbook, load_workbook\n\nwb = Workbook(write_only=True)\nws = wb.create_sheet(\"Orders\")\nws.append([\"Order ID\", \"Date\", \"Region\", \"Product\", \"Qty\", \"Price\"])\nfor i in range(1, 12001):\n    ws.append([i, dt.datetime(2026, 1, 1) + dt.timedelta(minutes=i), \"North\" if i % 2 else \"South\",\n               f\"SKU-{i % 50:03d}\", i % 20 + 1, round(1 + (i % 97) \u002F 10, 2)])\nws.append([12001, \"not a date\", \"East\", \"SKU-001\", 3, 4.5])\nws.append([12002, dt.datetime(2026, 2, 1), \"West\", \"SKU-002\", -5, 2.0])\nwb.save(\"orders_export.xlsx\")\n","python",[14,248,249,265,273,287,294,307,312,337,353,391,420,477,556,590,637],{"__ignoreMap":211},[215,250,251,255,259,262],{"class":89,"line":217},[215,252,254],{"class":253},"s-kum","import",[215,256,258],{"class":257},"skGVy"," datetime ",[215,260,261],{"class":253},"as",[215,263,264],{"class":257}," dt\n",[215,266,268,270],{"class":89,"line":267},2,[215,269,254],{"class":253},[215,271,272],{"class":257}," sqlite3\n",[215,274,276,279,282,284],{"class":89,"line":275},3,[215,277,278],{"class":253},"from",[215,280,281],{"class":257}," itertools ",[215,283,254],{"class":253},[215,285,286],{"class":257}," islice\n",[215,288,290],{"class":89,"line":289},4,[215,291,293],{"emptyLinePlaceholder":292},true,"\n",[215,295,297,299,302,304],{"class":89,"line":296},5,[215,298,278],{"class":253},[215,300,301],{"class":257}," openpyxl ",[215,303,254],{"class":253},[215,305,306],{"class":257}," Workbook, load_workbook\n",[215,308,310],{"class":89,"line":309},6,[215,311,293],{"emptyLinePlaceholder":292},[215,313,315,318,321,324,328,330,334],{"class":89,"line":314},7,[215,316,317],{"class":257},"wb ",[215,319,320],{"class":253},"=",[215,322,323],{"class":257}," Workbook(",[215,325,327],{"class":326},"sa561","write_only",[215,329,320],{"class":253},[215,331,333],{"class":332},"sP0c6","True",[215,335,336],{"class":257},")\n",[215,338,340,343,345,348,351],{"class":89,"line":339},8,[215,341,342],{"class":257},"ws ",[215,344,320],{"class":253},[215,346,347],{"class":257}," wb.create_sheet(",[215,349,350],{"class":224},"\"Orders\"",[215,352,336],{"class":257},[215,354,356,359,362,365,368,370,373,375,378,380,383,385,388],{"class":89,"line":355},9,[215,357,358],{"class":257},"ws.append([",[215,360,361],{"class":224},"\"Order ID\"",[215,363,364],{"class":257},", ",[215,366,367],{"class":224},"\"Date\"",[215,369,364],{"class":257},[215,371,372],{"class":224},"\"Region\"",[215,374,364],{"class":257},[215,376,377],{"class":224},"\"Product\"",[215,379,364],{"class":257},[215,381,382],{"class":224},"\"Qty\"",[215,384,364],{"class":257},[215,386,387],{"class":224},"\"Price\"",[215,389,390],{"class":257},"])\n",[215,392,394,397,400,403,406,409,412,414,417],{"class":89,"line":393},10,[215,395,396],{"class":253},"for",[215,398,399],{"class":257}," i ",[215,401,402],{"class":253},"in",[215,404,405],{"class":332}," range",[215,407,408],{"class":257},"(",[215,410,411],{"class":332},"1",[215,413,364],{"class":257},[215,415,416],{"class":332},"12001",[215,418,419],{"class":257},"):\n",[215,421,423,426,429,431,433,435,437,440,443,446,449,451,454,457,460,462,465,468,471,474],{"class":89,"line":422},11,[215,424,425],{"class":257},"    ws.append([i, dt.datetime(",[215,427,428],{"class":332},"2026",[215,430,364],{"class":257},[215,432,411],{"class":332},[215,434,364],{"class":257},[215,436,411],{"class":332},[215,438,439],{"class":257},") ",[215,441,442],{"class":253},"+",[215,444,445],{"class":257}," dt.timedelta(",[215,447,448],{"class":326},"minutes",[215,450,320],{"class":253},[215,452,453],{"class":257},"i), ",[215,455,456],{"class":224},"\"North\"",[215,458,459],{"class":253}," if",[215,461,399],{"class":257},[215,463,464],{"class":253},"%",[215,466,467],{"class":332}," 2",[215,469,470],{"class":253}," else",[215,472,473],{"class":224}," \"South\"",[215,475,476],{"class":257},",\n",[215,478,480,483,486,490,493,495,498,501,504,507,510,512,515,518,521,523,526,528,530,532,535,537,540,542,545,548,550,553],{"class":89,"line":479},12,[215,481,482],{"class":253},"               f",[215,484,485],{"class":224},"\"SKU-",[215,487,489],{"class":488},"sSjpA","{",[215,491,492],{"class":257},"i ",[215,494,464],{"class":253},[215,496,497],{"class":332}," 50",[215,499,500],{"class":253},":03d",[215,502,503],{"class":488},"}",[215,505,506],{"class":224},"\"",[215,508,509],{"class":257},", i ",[215,511,464],{"class":253},[215,513,514],{"class":332}," 20",[215,516,517],{"class":253}," +",[215,519,520],{"class":332}," 1",[215,522,364],{"class":257},[215,524,525],{"class":332},"round",[215,527,408],{"class":257},[215,529,411],{"class":332},[215,531,517],{"class":253},[215,533,534],{"class":257}," (i ",[215,536,464],{"class":253},[215,538,539],{"class":332}," 97",[215,541,439],{"class":257},[215,543,544],{"class":253},"\u002F",[215,546,547],{"class":332}," 10",[215,549,364],{"class":257},[215,551,552],{"class":332},"2",[215,554,555],{"class":257},")])\n",[215,557,559,561,563,565,568,570,573,575,578,580,583,585,588],{"class":89,"line":558},13,[215,560,358],{"class":257},[215,562,416],{"class":332},[215,564,364],{"class":257},[215,566,567],{"class":224},"\"not a date\"",[215,569,364],{"class":257},[215,571,572],{"class":224},"\"East\"",[215,574,364],{"class":257},[215,576,577],{"class":224},"\"SKU-001\"",[215,579,364],{"class":257},[215,581,582],{"class":332},"3",[215,584,364],{"class":257},[215,586,587],{"class":332},"4.5",[215,589,390],{"class":257},[215,591,593,595,598,601,603,605,607,609,611,614,617,619,622,624,627,630,632,635],{"class":89,"line":592},14,[215,594,358],{"class":257},[215,596,597],{"class":332},"12002",[215,599,600],{"class":257},", dt.datetime(",[215,602,428],{"class":332},[215,604,364],{"class":257},[215,606,552],{"class":332},[215,608,364],{"class":257},[215,610,411],{"class":332},[215,612,613],{"class":257},"), ",[215,615,616],{"class":224},"\"West\"",[215,618,364],{"class":257},[215,620,621],{"class":224},"\"SKU-002\"",[215,623,364],{"class":257},[215,625,626],{"class":253},"-",[215,628,629],{"class":332},"5",[215,631,364],{"class":257},[215,633,634],{"class":332},"2.0",[215,636,390],{"class":257},[215,638,640,643,646],{"class":89,"line":639},15,[215,641,642],{"class":257},"wb.save(",[215,644,645],{"class":224},"\"orders_export.xlsx\"",[215,647,336],{"class":257},[10,649,650,651,654],{},"Writing the sample with ",[14,652,653],{},"write_only=True"," is the same idea in reverse: rows go to disk as they are appended.",[201,656,658],{"id":657},"step-1-open-read-only-and-map-the-header","Step 1 — Open read-only and map the header",[10,660,661],{},"Read-only mode parses the sheet lazily as you iterate. Take the header row first and look columns up by name, so a reordered export does not load prices into the quantity column:",[206,663,665],{"className":244,"code":664,"language":246,"meta":211,"style":211},"wb = load_workbook(\"orders_export.xlsx\", read_only=True, data_only=True)\nws = wb[\"Orders\"]\nrows = ws.iter_rows(values_only=True)\nheader = [str(h).strip() for h in next(rows)]\nCOLUMNS = {\"Order ID\": \"order_id\", \"Date\": \"order_date\", \"Region\": \"region\",\n           \"Product\": \"product\", \"Qty\": \"qty\", \"Price\": \"price\"}\nmissing = set(COLUMNS) - set(header)\nif missing:\n    raise ValueError(f\"missing columns: {sorted(missing)}\")\nindex = {target: header.index(source) for source, target in COLUMNS.items()}\nprint(index)\n",[14,666,667,698,712,731,760,799,830,853,861,891,914],{"__ignoreMap":211},[215,668,669,671,673,676,678,680,683,685,687,689,692,694,696],{"class":89,"line":217},[215,670,317],{"class":257},[215,672,320],{"class":253},[215,674,675],{"class":257}," load_workbook(",[215,677,645],{"class":224},[215,679,364],{"class":257},[215,681,682],{"class":326},"read_only",[215,684,320],{"class":253},[215,686,333],{"class":332},[215,688,364],{"class":257},[215,690,691],{"class":326},"data_only",[215,693,320],{"class":253},[215,695,333],{"class":332},[215,697,336],{"class":257},[215,699,700,702,704,707,709],{"class":89,"line":267},[215,701,342],{"class":257},[215,703,320],{"class":253},[215,705,706],{"class":257}," wb[",[215,708,350],{"class":224},[215,710,711],{"class":257},"]\n",[215,713,714,717,719,722,725,727,729],{"class":89,"line":275},[215,715,716],{"class":257},"rows ",[215,718,320],{"class":253},[215,720,721],{"class":257}," ws.iter_rows(",[215,723,724],{"class":326},"values_only",[215,726,320],{"class":253},[215,728,333],{"class":332},[215,730,336],{"class":257},[215,732,733,736,738,741,744,747,749,752,754,757],{"class":89,"line":289},[215,734,735],{"class":257},"header ",[215,737,320],{"class":253},[215,739,740],{"class":257}," [",[215,742,743],{"class":332},"str",[215,745,746],{"class":257},"(h).strip() ",[215,748,396],{"class":253},[215,750,751],{"class":257}," h ",[215,753,402],{"class":253},[215,755,756],{"class":332}," next",[215,758,759],{"class":257},"(rows)]\n",[215,761,762,765,768,771,773,776,779,781,783,785,788,790,792,794,797],{"class":89,"line":296},[215,763,764],{"class":332},"COLUMNS",[215,766,767],{"class":253}," =",[215,769,770],{"class":257}," {",[215,772,361],{"class":224},[215,774,775],{"class":257},": ",[215,777,778],{"class":224},"\"order_id\"",[215,780,364],{"class":257},[215,782,367],{"class":224},[215,784,775],{"class":257},[215,786,787],{"class":224},"\"order_date\"",[215,789,364],{"class":257},[215,791,372],{"class":224},[215,793,775],{"class":257},[215,795,796],{"class":224},"\"region\"",[215,798,476],{"class":257},[215,800,801,804,806,809,811,813,815,818,820,822,824,827],{"class":89,"line":309},[215,802,803],{"class":224},"           \"Product\"",[215,805,775],{"class":257},[215,807,808],{"class":224},"\"product\"",[215,810,364],{"class":257},[215,812,382],{"class":224},[215,814,775],{"class":257},[215,816,817],{"class":224},"\"qty\"",[215,819,364],{"class":257},[215,821,387],{"class":224},[215,823,775],{"class":257},[215,825,826],{"class":224},"\"price\"",[215,828,829],{"class":257},"}\n",[215,831,832,835,837,840,842,844,846,848,850],{"class":89,"line":314},[215,833,834],{"class":257},"missing ",[215,836,320],{"class":253},[215,838,839],{"class":332}," set",[215,841,408],{"class":257},[215,843,764],{"class":332},[215,845,439],{"class":257},[215,847,626],{"class":253},[215,849,839],{"class":332},[215,851,852],{"class":257},"(header)\n",[215,854,855,858],{"class":89,"line":339},[215,856,857],{"class":253},"if",[215,859,860],{"class":257}," missing:\n",[215,862,863,866,869,871,874,877,879,882,885,887,889],{"class":89,"line":355},[215,864,865],{"class":253},"    raise",[215,867,868],{"class":332}," ValueError",[215,870,408],{"class":257},[215,872,873],{"class":253},"f",[215,875,876],{"class":224},"\"missing columns: ",[215,878,489],{"class":488},[215,880,881],{"class":332},"sorted",[215,883,884],{"class":257},"(missing)",[215,886,503],{"class":488},[215,888,506],{"class":224},[215,890,336],{"class":257},[215,892,893,896,898,901,903,906,908,911],{"class":89,"line":393},[215,894,895],{"class":257},"index ",[215,897,320],{"class":253},[215,899,900],{"class":257}," {target: header.index(source) ",[215,902,396],{"class":253},[215,904,905],{"class":257}," source, target ",[215,907,402],{"class":253},[215,909,910],{"class":332}," COLUMNS",[215,912,913],{"class":257},".items()}\n",[215,915,916,919],{"class":89,"line":422},[215,917,918],{"class":332},"print",[215,920,921],{"class":257},"(index)\n",[10,923,924,927,928,931,932,936],{},[14,925,926],{},"data_only=True"," returns the cached results of any formulas instead of the formula text. ",[14,929,930],{},"values_only=True"," yields plain tuples rather than cell objects, which is both faster and lighter. Read-only mode itself is covered in ",[19,933,935],{"href":934},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fspeed-up-openpyxl-with-read-only-mode\u002F","Speed Up openpyxl with Read-Only Mode",".",[201,938,940],{"id":939},"step-2-convert-and-validate-each-row","Step 2 — Convert and validate each row",[10,942,943],{},"Each row is converted to the database's types and checked. A row that fails is not an exception to stop on; it goes to a rejects list with the reason:",[206,945,947],{"className":244,"code":946,"language":246,"meta":211,"style":211},"def convert(row, line):\n    rec = {name: row[i] for name, i in index.items()}\n    problems = []\n    if not isinstance(rec[\"order_date\"], dt.datetime):\n        problems.append(\"date is not a date\")\n    if not isinstance(rec[\"qty\"], (int, float)) or rec[\"qty\"] \u003C= 0:\n        problems.append(\"qty must be positive\")\n    if problems:\n        return None, (line, \"; \".join(problems), repr(row)[:200])\n    rec[\"order_date\"] = rec[\"order_date\"].isoformat(sep=\" \")\n    rec[\"qty\"] = int(rec[\"qty\"])\n    return tuple(rec[c] for c in [\"order_id\", \"order_date\", \"region\", \"product\", \"qty\", \"price\"]), None\n",[14,948,949,961,981,991,1010,1020,1066,1075,1082,1110,1138,1157],{"__ignoreMap":211},[215,950,951,954,958],{"class":89,"line":217},[215,952,953],{"class":253},"def",[215,955,957],{"class":956},"s_Opv"," convert",[215,959,960],{"class":257},"(row, line):\n",[215,962,963,966,968,971,973,976,978],{"class":89,"line":267},[215,964,965],{"class":257},"    rec ",[215,967,320],{"class":253},[215,969,970],{"class":257}," {name: row[i] ",[215,972,396],{"class":253},[215,974,975],{"class":257}," name, i ",[215,977,402],{"class":253},[215,979,980],{"class":257}," index.items()}\n",[215,982,983,986,988],{"class":89,"line":275},[215,984,985],{"class":257},"    problems ",[215,987,320],{"class":253},[215,989,990],{"class":257}," []\n",[215,992,993,996,999,1002,1005,1007],{"class":89,"line":289},[215,994,995],{"class":253},"    if",[215,997,998],{"class":253}," not",[215,1000,1001],{"class":332}," isinstance",[215,1003,1004],{"class":257},"(rec[",[215,1006,787],{"class":224},[215,1008,1009],{"class":257},"], dt.datetime):\n",[215,1011,1012,1015,1018],{"class":89,"line":296},[215,1013,1014],{"class":257},"        problems.append(",[215,1016,1017],{"class":224},"\"date is not a date\"",[215,1019,336],{"class":257},[215,1021,1022,1024,1026,1028,1030,1032,1035,1038,1040,1043,1046,1049,1052,1054,1057,1060,1063],{"class":89,"line":309},[215,1023,995],{"class":253},[215,1025,998],{"class":253},[215,1027,1001],{"class":332},[215,1029,1004],{"class":257},[215,1031,817],{"class":224},[215,1033,1034],{"class":257},"], (",[215,1036,1037],{"class":332},"int",[215,1039,364],{"class":257},[215,1041,1042],{"class":332},"float",[215,1044,1045],{"class":257},")) ",[215,1047,1048],{"class":253},"or",[215,1050,1051],{"class":257}," rec[",[215,1053,817],{"class":224},[215,1055,1056],{"class":257},"] ",[215,1058,1059],{"class":253},"\u003C=",[215,1061,1062],{"class":332}," 0",[215,1064,1065],{"class":257},":\n",[215,1067,1068,1070,1073],{"class":89,"line":314},[215,1069,1014],{"class":257},[215,1071,1072],{"class":224},"\"qty must be positive\"",[215,1074,336],{"class":257},[215,1076,1077,1079],{"class":89,"line":339},[215,1078,995],{"class":253},[215,1080,1081],{"class":257}," problems:\n",[215,1083,1084,1087,1090,1093,1096,1099,1102,1105,1108],{"class":89,"line":355},[215,1085,1086],{"class":253},"        return",[215,1088,1089],{"class":332}," None",[215,1091,1092],{"class":257},", (line, ",[215,1094,1095],{"class":224},"\"; \"",[215,1097,1098],{"class":257},".join(problems), ",[215,1100,1101],{"class":332},"repr",[215,1103,1104],{"class":257},"(row)[:",[215,1106,1107],{"class":332},"200",[215,1109,390],{"class":257},[215,1111,1112,1115,1117,1119,1121,1123,1125,1128,1131,1133,1136],{"class":89,"line":393},[215,1113,1114],{"class":257},"    rec[",[215,1116,787],{"class":224},[215,1118,1056],{"class":257},[215,1120,320],{"class":253},[215,1122,1051],{"class":257},[215,1124,787],{"class":224},[215,1126,1127],{"class":257},"].isoformat(",[215,1129,1130],{"class":326},"sep",[215,1132,320],{"class":253},[215,1134,1135],{"class":224},"\" \"",[215,1137,336],{"class":257},[215,1139,1140,1142,1144,1146,1148,1151,1153,1155],{"class":89,"line":422},[215,1141,1114],{"class":257},[215,1143,817],{"class":224},[215,1145,1056],{"class":257},[215,1147,320],{"class":253},[215,1149,1150],{"class":332}," int",[215,1152,1004],{"class":257},[215,1154,817],{"class":224},[215,1156,390],{"class":257},[215,1158,1159,1162,1165,1168,1170,1173,1175,1177,1179,1181,1183,1185,1187,1189,1191,1193,1195,1197,1199,1202],{"class":89,"line":479},[215,1160,1161],{"class":253},"    return",[215,1163,1164],{"class":332}," tuple",[215,1166,1167],{"class":257},"(rec[c] ",[215,1169,396],{"class":253},[215,1171,1172],{"class":257}," c ",[215,1174,402],{"class":253},[215,1176,740],{"class":257},[215,1178,778],{"class":224},[215,1180,364],{"class":257},[215,1182,787],{"class":224},[215,1184,364],{"class":257},[215,1186,796],{"class":224},[215,1188,364],{"class":257},[215,1190,808],{"class":224},[215,1192,364],{"class":257},[215,1194,817],{"class":224},[215,1196,364],{"class":257},[215,1198,826],{"class":224},[215,1200,1201],{"class":257},"]), ",[215,1203,1204],{"class":332},"None\n",[10,1206,1207,1208,1211],{},"Keeping the sheet line number — the Excel row the record came from — makes every reject traceable to a cell someone can open. Dates are stored as ISO text, which SQLite compares and sorts correctly; a server database would take the ",[14,1209,1210],{},"datetime"," directly.",[201,1213,1215],{"id":1214},"step-3-insert-in-batches-inside-one-transaction","Step 3 — Insert in batches inside one transaction",[10,1217,1218,1221,1222,1224,1225,1228],{},[14,1219,1220],{},"islice"," takes the next batch from the row iterator without materialising the rest. Each batch is converted, the good rows are inserted with one ",[14,1223,16],{}," call, and the rejects are collected. The ",[14,1226,1227],{},"with con:"," block wraps the whole load in a transaction:",[206,1230,1232],{"className":244,"code":1231,"language":246,"meta":211,"style":211},"con = sqlite3.connect(\"orders.db\")\ncon.executescript(\"\"\"\n    DROP TABLE IF EXISTS orders; DROP TABLE IF EXISTS rejects;\n    CREATE TABLE orders (order_id INTEGER PRIMARY KEY, order_date TEXT NOT NULL,\n                         region TEXT, product TEXT, qty INTEGER, price REAL);\n    CREATE TABLE rejects (sheet_line INTEGER, problem TEXT, raw TEXT);\n\"\"\")\n\nBATCH = 5000\nline = 1                                  # the header was line 1\nloaded = rejected = 0\nwith con:\n    while batch := list(islice(rows, BATCH)):\n        good, bad = [], []\n        for row in batch:\n            line += 1\n            if all(v is None for v in row):\n                continue                  # skip blank rows\n            rec, rej = convert(row, line)\n            (good if rec else bad).append(rec or rej)\n        con.executemany(\"INSERT INTO orders VALUES (?, ?, ?, ?, ?, ?)\", good)\n        con.executemany(\"INSERT INTO rejects VALUES (?, ?, ?)\", bad)\n        loaded += len(good)\n        rejected += len(bad)\n        print(f\"line {line}: {loaded} loaded, {rejected} rejected\")\nwb.close()\nprint(con.execute(\"SELECT sheet_line, problem FROM rejects\").fetchall())\n# [(12002, 'date is not a date'), (12003, 'qty must be positive')]\n",[14,1233,1234,1249,1257,1262,1267,1272,1277,1284,1288,1298,1311,1326,1334,1356,1366,1379,1391,1419,1428,1439,1461,1473,1484,1498,1511,1554,1560,1574],{"__ignoreMap":211},[215,1235,1236,1239,1241,1244,1247],{"class":89,"line":217},[215,1237,1238],{"class":257},"con ",[215,1240,320],{"class":253},[215,1242,1243],{"class":257}," sqlite3.connect(",[215,1245,1246],{"class":224},"\"orders.db\"",[215,1248,336],{"class":257},[215,1250,1251,1254],{"class":89,"line":267},[215,1252,1253],{"class":257},"con.executescript(",[215,1255,1256],{"class":224},"\"\"\"\n",[215,1258,1259],{"class":89,"line":275},[215,1260,1261],{"class":224},"    DROP TABLE IF EXISTS orders; DROP TABLE IF EXISTS rejects;\n",[215,1263,1264],{"class":89,"line":289},[215,1265,1266],{"class":224},"    CREATE TABLE orders (order_id INTEGER PRIMARY KEY, order_date TEXT NOT NULL,\n",[215,1268,1269],{"class":89,"line":296},[215,1270,1271],{"class":224},"                         region TEXT, product TEXT, qty INTEGER, price REAL);\n",[215,1273,1274],{"class":89,"line":309},[215,1275,1276],{"class":224},"    CREATE TABLE rejects (sheet_line INTEGER, problem TEXT, raw TEXT);\n",[215,1278,1279,1282],{"class":89,"line":314},[215,1280,1281],{"class":224},"\"\"\"",[215,1283,336],{"class":257},[215,1285,1286],{"class":89,"line":339},[215,1287,293],{"emptyLinePlaceholder":292},[215,1289,1290,1293,1295],{"class":89,"line":355},[215,1291,1292],{"class":332},"BATCH",[215,1294,767],{"class":253},[215,1296,1297],{"class":332}," 5000\n",[215,1299,1300,1303,1305,1307],{"class":89,"line":393},[215,1301,1302],{"class":257},"line ",[215,1304,320],{"class":253},[215,1306,520],{"class":332},[215,1308,1310],{"class":1309},"s-wDw","                                  # the header was line 1\n",[215,1312,1313,1316,1318,1321,1323],{"class":89,"line":422},[215,1314,1315],{"class":257},"loaded ",[215,1317,320],{"class":253},[215,1319,1320],{"class":257}," rejected ",[215,1322,320],{"class":253},[215,1324,1325],{"class":332}," 0\n",[215,1327,1328,1331],{"class":89,"line":479},[215,1329,1330],{"class":253},"with",[215,1332,1333],{"class":257}," con:\n",[215,1335,1336,1339,1342,1345,1348,1351,1353],{"class":89,"line":558},[215,1337,1338],{"class":253},"    while",[215,1340,1341],{"class":257}," batch ",[215,1343,1344],{"class":253},":=",[215,1346,1347],{"class":332}," list",[215,1349,1350],{"class":257},"(islice(rows, ",[215,1352,1292],{"class":332},[215,1354,1355],{"class":257},")):\n",[215,1357,1358,1361,1363],{"class":89,"line":592},[215,1359,1360],{"class":257},"        good, bad ",[215,1362,320],{"class":253},[215,1364,1365],{"class":257}," [], []\n",[215,1367,1368,1371,1374,1376],{"class":89,"line":639},[215,1369,1370],{"class":253},"        for",[215,1372,1373],{"class":257}," row ",[215,1375,402],{"class":253},[215,1377,1378],{"class":257}," batch:\n",[215,1380,1382,1385,1388],{"class":89,"line":1381},16,[215,1383,1384],{"class":257},"            line ",[215,1386,1387],{"class":253},"+=",[215,1389,1390],{"class":332}," 1\n",[215,1392,1394,1397,1400,1403,1406,1408,1411,1414,1416],{"class":89,"line":1393},17,[215,1395,1396],{"class":253},"            if",[215,1398,1399],{"class":332}," all",[215,1401,1402],{"class":257},"(v ",[215,1404,1405],{"class":253},"is",[215,1407,1089],{"class":332},[215,1409,1410],{"class":253}," for",[215,1412,1413],{"class":257}," v ",[215,1415,402],{"class":253},[215,1417,1418],{"class":257}," row):\n",[215,1420,1422,1425],{"class":89,"line":1421},18,[215,1423,1424],{"class":253},"                continue",[215,1426,1427],{"class":1309},"                  # skip blank rows\n",[215,1429,1431,1434,1436],{"class":89,"line":1430},19,[215,1432,1433],{"class":257},"            rec, rej ",[215,1435,320],{"class":253},[215,1437,1438],{"class":257}," convert(row, line)\n",[215,1440,1442,1445,1447,1450,1453,1456,1458],{"class":89,"line":1441},20,[215,1443,1444],{"class":257},"            (good ",[215,1446,857],{"class":253},[215,1448,1449],{"class":257}," rec ",[215,1451,1452],{"class":253},"else",[215,1454,1455],{"class":257}," bad).append(rec ",[215,1457,1048],{"class":253},[215,1459,1460],{"class":257}," rej)\n",[215,1462,1464,1467,1470],{"class":89,"line":1463},21,[215,1465,1466],{"class":257},"        con.executemany(",[215,1468,1469],{"class":224},"\"INSERT INTO orders VALUES (?, ?, ?, ?, ?, ?)\"",[215,1471,1472],{"class":257},", good)\n",[215,1474,1476,1478,1481],{"class":89,"line":1475},22,[215,1477,1466],{"class":257},[215,1479,1480],{"class":224},"\"INSERT INTO rejects VALUES (?, ?, ?)\"",[215,1482,1483],{"class":257},", bad)\n",[215,1485,1487,1490,1492,1495],{"class":89,"line":1486},23,[215,1488,1489],{"class":257},"        loaded ",[215,1491,1387],{"class":253},[215,1493,1494],{"class":332}," len",[215,1496,1497],{"class":257},"(good)\n",[215,1499,1501,1504,1506,1508],{"class":89,"line":1500},24,[215,1502,1503],{"class":257},"        rejected ",[215,1505,1387],{"class":253},[215,1507,1494],{"class":332},[215,1509,1510],{"class":257},"(bad)\n",[215,1512,1514,1517,1519,1521,1524,1526,1528,1530,1532,1534,1537,1539,1542,1544,1547,1549,1552],{"class":89,"line":1513},25,[215,1515,1516],{"class":332},"        print",[215,1518,408],{"class":257},[215,1520,873],{"class":253},[215,1522,1523],{"class":224},"\"line ",[215,1525,489],{"class":488},[215,1527,89],{"class":257},[215,1529,503],{"class":488},[215,1531,775],{"class":224},[215,1533,489],{"class":488},[215,1535,1536],{"class":257},"loaded",[215,1538,503],{"class":488},[215,1540,1541],{"class":224}," loaded, ",[215,1543,489],{"class":488},[215,1545,1546],{"class":257},"rejected",[215,1548,503],{"class":488},[215,1550,1551],{"class":224}," rejected\"",[215,1553,336],{"class":257},[215,1555,1557],{"class":89,"line":1556},26,[215,1558,1559],{"class":257},"wb.close()\n",[215,1561,1563,1565,1568,1571],{"class":89,"line":1562},27,[215,1564,918],{"class":332},[215,1566,1567],{"class":257},"(con.execute(",[215,1569,1570],{"class":224},"\"SELECT sheet_line, problem FROM rejects\"",[215,1572,1573],{"class":257},").fetchall())\n",[215,1575,1577],{"class":89,"line":1576},28,[215,1578,1579],{"class":1309},"# [(12002, 'date is not a date'), (12003, 'qty must be positive')]\n",[10,1581,1582],{},"If anything raises mid-load — a disk full, a constraint violation — the transaction rolls back and the table is left as it was, not half-filled. Closing the workbook releases the file handle that read-only mode keeps open while iterating.",[25,1584,34,1590,34,1593,34,1596,34,1600,34,1608,34,1614,34,1619,34,1624,34,1627,34,1631,34,1634,34,1639],{"viewBox":1585,"role":28,"ariaLabelledBy":1586,"xmlns":32,"style":1589},"4 0 744 126",[1587,1588],"st-bench-t","st-bench-d","width:100%;max-width:744px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[36,1591,1592],{"id":1587},"Loading 200,000 rows into SQLite",[40,1594,1595],{"id":1588},"Measured on an 8.6 MB workbook with 200,000 rows. Streaming with openpyxl read-only mode and executemany peaked at 73 MB and took 5.4 seconds. Reading the sheet with pandas and writing it with to_sql peaked at 245 MB and took 6.9 seconds. A bare interpreter with sqlite3 and openpyxl imported used 48 MB.",[44,1597],{"x":46,"y":47,"width":1598,"height":1599,"fill":50},"744","126",[44,1601],{"x":53,"y":54,"width":1602,"height":1603,"rx":1604,"fill":1605,"stroke":1606,"style":1607},"175.5","28.0","7","#e7ebef","var(--line,#cdd5e6)","stroke-width:1px",[73,1609,1613],{"x":1610,"y":1611,"style":1612},"107.8","34.5","font-size:12.5px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","stream + executemany",[44,1615],{"x":1616,"y":54,"width":1617,"height":1603,"rx":1604,"fill":159,"stroke":160,"style":1618},"205.5","140.4","stroke-width:1.5px",[73,1620,1623],{"x":1621,"y":1611,"style":1622},"353.9","font-size:12px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:start","73 MB",[44,1625],{"x":53,"y":1626,"width":1602,"height":1603,"rx":1604,"fill":1605,"stroke":1606,"style":1607},"56.0",[73,1628,1630],{"x":1610,"y":1629,"style":1612},"74.5","read_excel + to_sql",[44,1632],{"x":1616,"y":1626,"width":1633,"height":1603,"rx":1604,"fill":129,"stroke":130,"style":1618},"471.1",[73,1635,1638],{"x":1636,"y":1629,"style":1637},"684.6","font-size:12px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:start","245 MB",[73,1640,1643],{"x":140,"y":1641,"style":1642},"106.0","font-size:12.5px;font-weight:400;fill:var(--muted,#5b6780);text-anchor:middle","peak process memory; 5.4 s versus 6.9 s",[201,1645,1647],{"id":1646},"step-4-check-the-load-against-the-sheet","Step 4 — Check the load against the sheet",[10,1649,1650],{},"Streaming has no DataFrame to inspect afterwards, so reconcile with counts: every non-blank data line is either loaded or rejected, and the table holds exactly the loaded rows:",[206,1652,1654],{"className":244,"code":1653,"language":246,"meta":211,"style":211},"data_lines = line - 1\ncount = con.execute(\"SELECT COUNT(*) FROM orders\").fetchone()[0]\nprint(data_lines, loaded, rejected, count)       # 12002 12000 2 12000\nassert count == loaded and loaded + rejected == data_lines\n",[14,1655,1656,1670,1690,1700],{"__ignoreMap":211},[215,1657,1658,1661,1663,1666,1668],{"class":89,"line":217},[215,1659,1660],{"class":257},"data_lines ",[215,1662,320],{"class":253},[215,1664,1665],{"class":257}," line ",[215,1667,626],{"class":253},[215,1669,1390],{"class":332},[215,1671,1672,1675,1677,1680,1683,1686,1688],{"class":89,"line":267},[215,1673,1674],{"class":257},"count ",[215,1676,320],{"class":253},[215,1678,1679],{"class":257}," con.execute(",[215,1681,1682],{"class":224},"\"SELECT COUNT(*) FROM orders\"",[215,1684,1685],{"class":257},").fetchone()[",[215,1687,47],{"class":332},[215,1689,711],{"class":257},[215,1691,1692,1694,1697],{"class":89,"line":275},[215,1693,918],{"class":332},[215,1695,1696],{"class":257},"(data_lines, loaded, rejected, count)       ",[215,1698,1699],{"class":1309},"# 12002 12000 2 12000\n",[215,1701,1702,1705,1708,1711,1714,1717,1719,1721,1723,1725],{"class":89,"line":289},[215,1703,1704],{"class":253},"assert",[215,1706,1707],{"class":257}," count ",[215,1709,1710],{"class":253},"==",[215,1712,1713],{"class":257}," loaded ",[215,1715,1716],{"class":253},"and",[215,1718,1713],{"class":257},[215,1720,442],{"class":253},[215,1722,1320],{"class":257},[215,1724,1710],{"class":253},[215,1726,1727],{"class":257}," data_lines\n",[10,1729,1730,1731,1734,1735,1738,1739,936],{},"On the full 200,000-row test file the same loader finished in 5.4 seconds with a peak of 73 MB, against 245 MB for ",[14,1732,1733],{},"read_excel"," followed by ",[14,1736,1737],{},"to_sql",". The speed is similar; the memory is not, and it stays flat as the sheet grows. Reconciling against totals supplied by the source system is covered in ",[19,1740,1742],{"href":1741},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Freconcile-excel-totals-with-control-totals\u002F","Reconcile Excel Totals Against Control Totals with Python",[201,1744,1746],{"id":1745},"step-5-make-reruns-safe","Step 5 — Make reruns safe",[10,1748,1749,1750,1753],{},"A scheduled load will eventually run twice on the same file. With ",[14,1751,1752],{},"order_id"," as the primary key, a second run fails on the first duplicate and rolls back — safe, but noisy. Two patterns make reruns clean. Replace the target atomically by loading into a staging table and swapping at the end; or record each loaded file and skip it next time:",[206,1755,1757],{"className":244,"code":1756,"language":246,"meta":211,"style":211},"import os\n\ncon.execute(\"CREATE TABLE IF NOT EXISTS loads (file TEXT, mtime REAL, rows INTEGER, PRIMARY KEY (file, mtime))\")\nkey = (\"orders_export.xlsx\", os.path.getmtime(\"orders_export.xlsx\"))\nif con.execute(\"SELECT 1 FROM loads WHERE file = ? AND mtime = ?\", key).fetchone():\n    print(\"already loaded, skipping\")\nelse:\n    with con:\n        con.execute(\"INSERT INTO loads VALUES (?, ?, ?)\", (*key, loaded))\n    print(\"recorded load\")\n",[14,1758,1759,1766,1770,1780,1800,1812,1824,1830,1837,1854],{"__ignoreMap":211},[215,1760,1761,1763],{"class":89,"line":217},[215,1762,254],{"class":253},[215,1764,1765],{"class":257}," os\n",[215,1767,1768],{"class":89,"line":267},[215,1769,293],{"emptyLinePlaceholder":292},[215,1771,1772,1775,1778],{"class":89,"line":275},[215,1773,1774],{"class":257},"con.execute(",[215,1776,1777],{"class":224},"\"CREATE TABLE IF NOT EXISTS loads (file TEXT, mtime REAL, rows INTEGER, PRIMARY KEY (file, mtime))\"",[215,1779,336],{"class":257},[215,1781,1782,1785,1787,1790,1792,1795,1797],{"class":89,"line":289},[215,1783,1784],{"class":257},"key ",[215,1786,320],{"class":253},[215,1788,1789],{"class":257}," (",[215,1791,645],{"class":224},[215,1793,1794],{"class":257},", os.path.getmtime(",[215,1796,645],{"class":224},[215,1798,1799],{"class":257},"))\n",[215,1801,1802,1804,1806,1809],{"class":89,"line":296},[215,1803,857],{"class":253},[215,1805,1679],{"class":257},[215,1807,1808],{"class":224},"\"SELECT 1 FROM loads WHERE file = ? AND mtime = ?\"",[215,1810,1811],{"class":257},", key).fetchone():\n",[215,1813,1814,1817,1819,1822],{"class":89,"line":309},[215,1815,1816],{"class":332},"    print",[215,1818,408],{"class":257},[215,1820,1821],{"class":224},"\"already loaded, skipping\"",[215,1823,336],{"class":257},[215,1825,1826,1828],{"class":89,"line":314},[215,1827,1452],{"class":253},[215,1829,1065],{"class":257},[215,1831,1832,1835],{"class":89,"line":339},[215,1833,1834],{"class":253},"    with",[215,1836,1333],{"class":257},[215,1838,1839,1842,1845,1848,1851],{"class":89,"line":355},[215,1840,1841],{"class":257},"        con.execute(",[215,1843,1844],{"class":224},"\"INSERT INTO loads VALUES (?, ?, ?)\"",[215,1846,1847],{"class":257},", (",[215,1849,1850],{"class":253},"*",[215,1852,1853],{"class":257},"key, loaded))\n",[215,1855,1856,1858,1860,1863],{"class":89,"line":393},[215,1857,1816],{"class":332},[215,1859,408],{"class":257},[215,1861,1862],{"class":224},"\"recorded load\"",[215,1864,336],{"class":257},[10,1866,1867,1868,1871,1872,936],{},"In a real job the ",[14,1869,1870],{},"loads"," insert goes inside the same transaction as the data, so a file is marked as loaded only if its rows committed. For sheets that are edited and reloaded, an upsert keyed on the order ID updates changed rows instead; see ",[19,1873,1875],{"href":1874},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fupsert-excel-rows-into-a-database-table\u002F","Upsert Excel Rows into a Database Table with Python",[25,1877,34,1882,34,1885,34,1888,34,1891,34,1896,34,1903,34,1909,34,1914,34,1920,34,1927,34,1931,34,1936,34,1940,34,1946,34,1950,34,1955,34,1958,34,1963,34,1968,34,1972,34,1976,34,1981,34,1984,34,1989,34,1992,34,1999,34,2002],{"viewBox":1878,"role":28,"ariaLabelledBy":1879,"xmlns":32,"style":33},"4 0 752 233",[1880,1881],"st-rerun-t","st-rerun-d",[36,1883,1884],{"id":1880},"Two ways to make a load safe to rerun",[40,1886,1887],{"id":1881},"Loading into a staging table and swapping it for the target replaces the data completely on every run and suits full extracts. Recording each loaded file with its modification time skips files already loaded and suits folders that receive new files, but needs an upsert if a file can change.",[44,1889],{"x":46,"y":47,"width":48,"height":1890,"fill":50},"233",[44,1892],{"x":53,"y":54,"width":1893,"height":1894,"rx":1895,"fill":58,"stroke":59,"style":60},"351.0","201.0","14",[62,1897,1899],{"transform":1898},"translate(182.5,29.0) scale(1.083)",[66,1900],{"d":1901,"fill":69,"stroke":70,"style":1902},"M20 11a8 8 0 1 0-2.3 5.7 M20 4v7h-7","stroke-width:1.66px;stroke-linecap:round;stroke-linejoin:round",[73,1904,1908],{"x":1905,"y":1906,"style":1907},"195.5","78.0","font-size:14px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Staging table + swap",[89,1910],{"x1":1911,"y1":1912,"x2":1913,"y2":1912,"stroke":59,"style":1607},"36.0","91.0","355.0",[73,1915,1919],{"x":1916,"y":1917,"style":1918},"60.0","111.0","font-size:12px;font-weight:400;fill:var(--text,#172033);text-anchor:start","target always matches the file",[62,1921,1923],{"transform":1922},"translate(38.5,99.5) scale(0.625)",[66,1924],{"d":1925,"fill":69,"stroke":160,"style":1926},"M5 12.5l4.5 4.5L19 7","stroke-width:2.88px;stroke-linecap:round;stroke-linejoin:round",[73,1928,1930],{"x":1916,"y":1929,"style":1918},"137.0","readers never see a half load",[62,1932,1934],{"transform":1933},"translate(38.5,125.5) scale(0.625)",[66,1935],{"d":1925,"fill":69,"stroke":160,"style":1926},[73,1937,1939],{"x":1916,"y":1938,"style":1918},"163.0","reloads everything each run",[62,1941,1943],{"transform":1942},"translate(38.5,151.5) scale(0.625)",[66,1944],{"d":1945,"fill":69,"stroke":130,"style":1926},"M12 3.5l9.5 17h-19z M12 10v5 M12 17.8v.4",[73,1947,1949],{"x":1916,"y":1948,"style":1918},"189.0","suits full extracts",[62,1951,1953],{"transform":1952},"translate(38.5,177.5) scale(0.625)",[66,1954],{"d":1925,"fill":69,"stroke":160,"style":1926},[44,1956],{"x":1957,"y":54,"width":1893,"height":1894,"rx":1895,"fill":159,"stroke":160,"style":60},"389.0",[62,1959,1961],{"transform":1960},"translate(551.5,29.0) scale(1.083)",[66,1962],{"d":1925,"fill":69,"stroke":167,"style":1902},[73,1964,1967],{"x":1965,"y":1906,"style":1966},"564.5","font-size:14px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","Load ledger",[89,1969],{"x1":1970,"y1":1912,"x2":1971,"y2":1912,"stroke":160,"style":1607},"405.0","724.0",[73,1973,1975],{"x":1974,"y":1917,"style":1918},"429.0","each file loaded once",[62,1977,1979],{"transform":1978},"translate(407.5,99.5) scale(0.625)",[66,1980],{"d":1925,"fill":69,"stroke":160,"style":1926},[73,1982,1983],{"x":1974,"y":1929,"style":1918},"cheap for many small files",[62,1985,1987],{"transform":1986},"translate(407.5,125.5) scale(0.625)",[66,1988],{"d":1925,"fill":69,"stroke":160,"style":1926},[73,1990,1991],{"x":1974,"y":1938,"style":1918},"edited files need an upsert",[62,1993,1995],{"transform":1994},"translate(407.5,151.5) scale(0.625)",[66,1996],{"d":1997,"fill":69,"stroke":1998,"style":1926},"M6 6l12 12 M18 6L6 18","var(--accent,#f43f8f)",[73,2000,2001],{"x":1974,"y":1948,"style":1918},"suits incoming folders",[62,2003,2005],{"transform":2004},"translate(407.5,177.5) scale(0.625)",[66,2006],{"d":1925,"fill":69,"stroke":160,"style":1926},[10,2008,2009],{},"Which pattern fits depends on what the file is. A full extract that replaces yesterday's is best loaded into staging and swapped; a folder that receives a new file every day is best tracked with a ledger. Either way, the load becomes something you can run again after a failure without thinking about what the previous attempt left behind.",[201,2011,2013],{"id":2012},"common-pitfalls","Common pitfalls",[2015,2016,2017,2029,2041],"ul",{},[2018,2019,2020,2024,2025,2028],"li",{},[2021,2022,2023],"strong",{},"Trusting the sheet's dimensions."," Some exporters write a wrong used range; read-only mode may then yield thousands of empty rows. Skip rows whose values are all ",[14,2026,2027],{},"None",", as the loader does.",[2018,2030,2031,2034,2035,2038,2039,936],{},[2021,2032,2033],{},"Converting the whole sheet first."," ",[14,2036,2037],{},"list(ws.iter_rows())"," defeats streaming by materialising every row. Iterate, or take batches with ",[14,2040,1220],{},[2018,2042,2043,2046],{},[2021,2044,2045],{},"Committing per row."," One transaction per row is dramatically slower. Commit once, or once per large batch if the load must be resumable.",[201,2048,2050],{"id":2049},"conclusion","Conclusion",[10,2052,2053,2054,2056],{},"To load a big sheet into a database without loading it into memory, open it with openpyxl's read-only mode, map the header by name, convert and validate each row, and insert batches with ",[14,2055,16],{}," inside one transaction, sending bad rows to a rejects table with their sheet line numbers. Reconcile loaded plus rejected against the lines read, and record each load so reruns are safe.",[201,2058,2060],{"id":2059},"frequently-asked-questions","Frequently asked questions",[10,2062,2063,2066],{},[2021,2064,2065],{},"How do I load a large Excel file into a database without running out of memory?","\nOpen it with openpyxl's load_workbook(read_only=True), iterate the rows with iter_rows(values_only=True), and insert them in batches of a few thousand with executemany. Only one batch is in memory at a time.",[10,2068,2069,2072],{},[2021,2070,2071],{},"Is streaming slower than pandas read_excel plus to_sql?","\nNot necessarily. In our test on 200,000 rows, streaming into SQLite took 5.4 seconds with a 73 MB peak, while reading with pandas and calling to_sql took 6.9 seconds with a 245 MB peak.",[10,2074,2075,2078],{},[2021,2076,2077],{},"What batch size should I use?","\nA few thousand rows per executemany call is a good default. Much smaller batches pay per-call overhead; much larger ones hold more rows in memory without making inserts noticeably faster.",[10,2080,2081,2084],{},[2021,2082,2083],{},"How do I make the load safe to run twice?","\nLoad into a staging table and replace the target in one transaction at the end, or use the file's name and modification time as a load ID and skip files that are already recorded.",[201,2086,2088],{"id":2087},"related","Related",[2015,2090,2091,2097,2104,2111],{},[2018,2092,2093,2094,2096],{},"Up one level: ",[19,2095,22],{"href":21}," — techniques for big workbooks.",[2018,2098,2099,2103],{},[19,2100,2102],{"href":2101},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Freduce-memory-when-reading-excel-with-pandas\u002F","Reduce Memory When Reading Excel with pandas"," — when a DataFrame is still the right tool.",[2018,2105,2106,2110],{},[19,2107,2109],{"href":2108},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fload-an-excel-file-into-a-sql-database-with-pandas\u002F","Load an Excel File into a SQL Database with pandas"," — the simpler route for smaller files.",[2018,2112,2113,2117],{},[19,2114,2116],{"href":2115},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fwrite-large-dataframes-to-excel-with-write-only-mode\u002F","Write Large DataFrames to Excel with Write-Only Mode"," — streaming in the other direction.",[2119,2120,2121],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}",{"title":211,"searchDepth":267,"depth":267,"links":2123},[2124,2125,2126,2127,2128,2129,2130,2131,2132,2133],{"id":203,"depth":267,"text":204},{"id":657,"depth":267,"text":658},{"id":939,"depth":267,"text":940},{"id":1214,"depth":267,"text":1215},{"id":1646,"depth":267,"text":1647},{"id":1745,"depth":267,"text":1746},{"id":2012,"depth":267,"text":2013},{"id":2049,"depth":267,"text":2050},{"id":2059,"depth":267,"text":2060},{"id":2087,"depth":267,"text":2088},"2026-09-23","Load a large Excel sheet into a database without ever holding it in memory — read rows with openpyxl's read-only mode, convert and validate each row, insert in batches inside a transaction, divert bad rows to a rejects table and make the load safe to rerun.","md",[2138,2140,2142,2144],{"q":2065,"a":2139},"Open it with openpyxl's load_workbook(read_only=True), iterate the rows with iter_rows(values_only=True), and insert them in batches of a few thousand with executemany. Only one batch is in memory at a time.",{"q":2071,"a":2141},"Not necessarily. In our test on 200,000 rows, streaming into SQLite took 5.4 seconds with a 73 MB peak, while reading with pandas and calling to_sql took 6.9 seconds with a 245 MB peak.",{"q":2077,"a":2143},"A few thousand rows per executemany call is a good default. Much smaller batches pay per-call overhead; much larger ones hold more rows in memory without making inserts noticeably faster.",{"q":2083,"a":2145},"Load into a staging table and replace the target in one transaction at the end, or use the file's name and modification time as a load ID and skip files that are already recorded.","stream excel rows into a database with openpyxl",null,{"breadcrumb":2149},[2150,2152,2155],{"name":2151,"item":544},"Home",{"name":2153,"item":2154},"Advanced Data Transformation and Cleaning for Excel Automation","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":22,"item":21},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fstream-excel-rows-into-a-database-with-openpyxl",{"title":2158,"description":2159},"Stream Excel Rows into a Database with openpyxl (low memory)","Stream a large Excel sheet into SQLite or another database with openpyxl read_only and executemany — batching with islice, per-row conversion and validation, a rejects table, transactions, progress logging and a measured comparison with pandas.","stream-excel-rows-into-a-database-with-openpyxl","advanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fstream-excel-rows-into-a-database-with-openpyxl\u002Findex",[2163,2164,2165],"large-files","data-sources","openpyxl","how-to","GxkpS1S0OxylTt0CLEw6-MGNxCvfbKkB3krcX-QlgfY",[2169,2173],{"title":2170,"path":2171,"stem":2172,"children":-1},"Split a Large Excel File into Files of N Rows","\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fsplit-a-large-excel-file-into-files-of-n-rows","advanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fsplit-a-large-excel-file-into-files-of-n-rows\u002Findex",{"title":2174,"path":2175,"stem":2176,"children":-1},"Take a Random Sample of Rows from a Large Excel File","\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Ftake-a-random-sample-of-rows-from-a-large-excel-file","advanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Ftake-a-random-sample-of-rows-from-a-large-excel-file\u002Findex",[2178,2180],{"path":2179,"title":2153},"\u002Fadvanced-data-transformation-and-cleaning",{"path":2181,"title":22},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python",[],[2184,2187,2190,2193,2196,2199],{"path":2185,"title":2186,"kind":2166},"\u002Fadvanced-data-transformation-and-cleaning\u002Freading-excel-with-polars-and-arrow\u002Fspeed-up-pandas-excel-reads-with-the-calamine-engine","Speed Up pandas Excel Reads with the calamine Engine",{"path":2188,"title":2189,"kind":2166},"\u002Fautomating-reporting-workflows\u002Ftesting-and-packaging-excel-automation-scripts\u002Fprofile-a-slow-excel-report-script","Profile a Slow Excel Report Script in Python",{"path":2191,"title":2192,"kind":2166},"\u002Fformatting-and-charting-excel-reports-with-python\u002Fbuilding-excel-reports-with-xlsxwriter\u002Fwrite-a-million-rows-to-excel-with-xlsxwriter-constant-memory","Write a Million Rows to Excel with xlsxwriter",{"path":2194,"title":2195,"kind":2166},"\u002Fgetting-started-with-python-excel-automation\u002Fchoosing-a-python-excel-library\u002Fbenchmark-python-excel-read-and-write-speed","Benchmark Python Excel Read and Write Speed",{"path":2197,"title":2198,"kind":2166},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fexport-a-large-database-table-to-excel-in-chunks","Export a Large Database Table to Excel in Chunks",{"path":2200,"title":2201,"kind":2166},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fquery-excel-files-with-sql-using-duckdb","Query Excel Files with SQL Using DuckDB",1790972106861]