[{"data":1,"prerenderedAt":1910},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fread-excel-attachments-into-a-database-nightly":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fread-excel-attachments-into-a-database-nightly":1876,"ancestors:\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fread-excel-attachments-into-a-database-nightly":1885,"children:\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fread-excel-attachments-into-a-database-nightly":1890,"related:\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fread-excel-attachments-into-a-database-nightly":1891},{"id":4,"title":5,"body":6,"dateModified":1843,"datePublished":1843,"description":1844,"extension":1845,"faq":1846,"keyword":1855,"kind":1856,"meta":1857,"navigation":101,"path":1865,"seo":1866,"slug":1869,"stem":1870,"topics":1871,"type":1874,"__hash__":1875},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fread-excel-attachments-into-a-database-nightly\u002Findex.md","Read Excel Attachments into a Database Nightly",{"type":7,"value":8,"toc":1832},"minimark",[9,24,29,58,61,65,210,222,308,312,707,729,865,869,1410,1413,1647,1651,1711,1719,1723,1758,1762,1765,1769,1775,1785,1791,1797,1801,1828],[10,11,12,13,18,19,23],"p",{},"Branches, suppliers and partners still send data the simplest way they can: an Excel file attached to an email. Once a job saves those attachments into a folder — the subject of ",[14,15,17],"a",{"href":16},"\u002Fautomating-reporting-workflows\u002Femailing-excel-reports-with-smtplib\u002Fdownload-excel-attachments-from-an-email-inbox\u002F","Download Excel Attachments from an Email Inbox"," — the next job has to get them into a database, every night, unattended. That second job is where the trouble usually is. The same file gets sent twice and loaded twice. A file with a renamed column loads half its rows before failing. A corrected file arrives for a day already loaded, and the database ends up with both versions. This page builds a nightly loader that handles all of that: files identified by a hash of their content so a resend is skipped, each workbook validated before anything is written, each file loaded in its own transaction with an upsert on the business key so a corrected file replaces rather than duplicates, every attempt recorded in a load log, rejected files moved to a quarantine folder, and a short summary at the end. It belongs to ",[14,20,22],{"href":21},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002F","Moving Data Between Excel and Databases",".",[25,26,28],"h2",{"id":27},"prerequisites","Prerequisites",[30,31,36],"pre",{"className":32,"code":33,"language":34,"meta":35,"style":35},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl\n","bash","",[37,38,39],"code",{"__ignoreMap":35},[40,41,44,48,52,55],"span",{"class":42,"line":43},"line",1,[40,45,47],{"class":46},"sMTad","pip",[40,49,51],{"class":50},"srMev"," install",[40,53,54],{"class":50}," pandas",[40,56,57],{"class":50}," openpyxl\n",[10,59,60],{},"The examples use SQLite, which is part of Python; the same pattern works with PostgreSQL or SQL Server through SQLAlchemy, using their own upsert syntax. Tonight's inbox folder holds six files: two branch returns, an identical resend of one of them, a file with renamed columns, a corrected version of the York return, and an attachment that is not a workbook at all.",[25,62,64],{"id":63},"step-1-the-tables-data-and-a-load-log","Step 1 — The tables: data and a load log",[30,66,70],{"className":67,"code":68,"language":69,"meta":35,"style":35},"language-python shiki shiki-themes github-light github-dark-high-contrast","import sqlite3\nfrom contextlib import closing\n\nDB = \"sales.db\"\nwith closing(sqlite3.connect(DB)) as con, con:\n    con.executescript(\"\"\"\n    CREATE TABLE IF NOT EXISTS daily_sales (\n        branch TEXT NOT NULL, date TEXT NOT NULL, product TEXT NOT NULL,\n        units INTEGER NOT NULL CHECK (units >= 0), revenue REAL NOT NULL,\n        source_file TEXT NOT NULL, loaded_at TEXT NOT NULL,\n        PRIMARY KEY (branch, date, product));\n    CREATE TABLE IF NOT EXISTS load_log (\n        file_hash TEXT PRIMARY KEY, file_name TEXT, status TEXT, rows INTEGER, message TEXT, loaded_at TEXT);\n    \"\"\")\nprint(\"tables ready\")\n","python",[37,71,72,82,96,103,116,136,145,151,157,163,169,175,181,187,196],{"__ignoreMap":35},[40,73,74,78],{"class":42,"line":43},[40,75,77],{"class":76},"s-kum","import",[40,79,81],{"class":80},"skGVy"," sqlite3\n",[40,83,85,88,91,93],{"class":42,"line":84},2,[40,86,87],{"class":76},"from",[40,89,90],{"class":80}," contextlib ",[40,92,77],{"class":76},[40,94,95],{"class":80}," closing\n",[40,97,99],{"class":42,"line":98},3,[40,100,102],{"emptyLinePlaceholder":101},true,"\n",[40,104,106,110,113],{"class":42,"line":105},4,[40,107,109],{"class":108},"sP0c6","DB",[40,111,112],{"class":76}," =",[40,114,115],{"class":50}," \"sales.db\"\n",[40,117,119,122,125,127,130,133],{"class":42,"line":118},5,[40,120,121],{"class":76},"with",[40,123,124],{"class":80}," closing(sqlite3.connect(",[40,126,109],{"class":108},[40,128,129],{"class":80},")) ",[40,131,132],{"class":76},"as",[40,134,135],{"class":80}," con, con:\n",[40,137,139,142],{"class":42,"line":138},6,[40,140,141],{"class":80},"    con.executescript(",[40,143,144],{"class":50},"\"\"\"\n",[40,146,148],{"class":42,"line":147},7,[40,149,150],{"class":50},"    CREATE TABLE IF NOT EXISTS daily_sales (\n",[40,152,154],{"class":42,"line":153},8,[40,155,156],{"class":50},"        branch TEXT NOT NULL, date TEXT NOT NULL, product TEXT NOT NULL,\n",[40,158,160],{"class":42,"line":159},9,[40,161,162],{"class":50},"        units INTEGER NOT NULL CHECK (units >= 0), revenue REAL NOT NULL,\n",[40,164,166],{"class":42,"line":165},10,[40,167,168],{"class":50},"        source_file TEXT NOT NULL, loaded_at TEXT NOT NULL,\n",[40,170,172],{"class":42,"line":171},11,[40,173,174],{"class":50},"        PRIMARY KEY (branch, date, product));\n",[40,176,178],{"class":42,"line":177},12,[40,179,180],{"class":50},"    CREATE TABLE IF NOT EXISTS load_log (\n",[40,182,184],{"class":42,"line":183},13,[40,185,186],{"class":50},"        file_hash TEXT PRIMARY KEY, file_name TEXT, status TEXT, rows INTEGER, message TEXT, loaded_at TEXT);\n",[40,188,190,193],{"class":42,"line":189},14,[40,191,192],{"class":50},"    \"\"\"",[40,194,195],{"class":80},")\n",[40,197,199,202,205,208],{"class":42,"line":198},15,[40,200,201],{"class":108},"print",[40,203,204],{"class":80},"(",[40,206,207],{"class":50},"\"tables ready\"",[40,209,195],{"class":80},[10,211,212,213,217,218,221],{},"The data table's primary key is the ",[214,215,216],"em",{},"business key"," — branch, date and product — the combination that identifies one fact. Making it the key is what allows a corrected file to replace the earlier figures instead of adding a second set: there can only ever be one row for York, 1 October, product A. Each row also records the file it came from and when, which is how a figure in a report can be traced back to an email. The load log has one row per file ",[214,219,220],{},"content",", keyed by a hash of the bytes, with the outcome and a message. It is both the job's memory — which files have been handled — and its audit trail.",[223,224,232,233,232,237,232,241,232,249,232,259,232,266,232,272,232,277,232,281,232,285,232,290,232,295,232,299,232,302,232,305],"svg",{"viewBox":225,"role":226,"ariaLabelledBy":227,"xmlns":230,"style":231},"4 0 752 173","img",[228,229],"nightly-upsert-t","nightly-upsert-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:752px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[234,235,236],"title",{"id":228},"Insert or upsert for daily returns",[238,239,240],"desc",{"id":229},"With a plain insert, a corrected file adds a second set of rows for the same branch, date and product, and totals double. With an upsert on the business key, the corrected figures replace the earlier ones and there is always one row per fact.",[242,243],"rect",{"x":244,"y":245,"width":246,"height":247,"fill":248},"4","0","752","173","#ffffff",[242,250],{"x":251,"y":252,"width":253,"height":254,"rx":255,"fill":256,"stroke":257,"style":258},"20.0","16.0","351.0","141.0","14","#fce9e9","var(--danger,#dc2626)","stroke-width:2px",[260,261,265],"text",{"x":262,"y":263,"style":264},"195.5","44.0","font-size:14px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","Plain INSERT",[42,267],{"x1":268,"y1":269,"x2":270,"y2":269,"stroke":257,"style":271},"36.0","57.0","355.0","stroke-width:1px",[260,273,276],{"x":262,"y":274,"style":275},"77.0","font-size:12px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","correction adds rows",[260,278,280],{"x":262,"y":279,"style":275},"103.0","York, 1 Oct, A twice",[260,282,284],{"x":262,"y":283,"style":275},"129.0","totals double count",[242,286],{"x":287,"y":252,"width":253,"height":254,"rx":255,"fill":288,"stroke":289,"style":258},"389.0","#d9f4f1","var(--teal,#0f9488)",[260,291,294],{"x":292,"y":263,"style":293},"564.5","font-size:14px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","Upsert on the business key",[42,296],{"x1":297,"y1":269,"x2":298,"y2":269,"stroke":289,"style":271},"405.0","724.0",[260,300,301],{"x":292,"y":274,"style":275},"correction replaces rows",[260,303,304],{"x":292,"y":279,"style":275},"one row per branch, date, product",[260,306,307],{"x":292,"y":283,"style":275},"source_file shows the latest",[25,309,311],{"id":310},"step-2-validate-a-workbook-before-writing-anything","Step 2 — Validate a workbook before writing anything",[30,313,315],{"className":67,"code":314,"language":69,"meta":35,"style":35},"import pandas as pd\n\nEXPECTED = [\"Branch\", \"Date\", \"Product\", \"Units\", \"Revenue\"]\n\ndef read_and_validate(path):\n    df = pd.read_excel(path)\n    missing = [c for c in EXPECTED if c not in df.columns]\n    if missing:\n        raise ValueError(f\"missing columns {missing}; found {list(df.columns)}\")\n    df = df[EXPECTED].copy()\n    df[\"Date\"] = pd.to_datetime(df[\"Date\"], errors=\"coerce\").dt.strftime(\"%Y-%m-%d\")\n    bad = df[df[\"Date\"].isna() | (pd.to_numeric(df[\"Units\"], errors=\"coerce\") \u003C 0) | df[[\"Branch\", \"Product\"]].isna().any(axis=1)]\n    if len(bad):\n        raise ValueError(f\"{len(bad)} invalid row(s), first at row {bad.index[0] + 2}\")\n    if df.duplicated([\"Branch\", \"Date\", \"Product\"]).any():\n        raise ValueError(\"the same branch, date and product appears twice in the file\")\n    return df\n",[37,316,317,329,333,370,374,386,397,433,441,485,499,541,608,618,664,684,698],{"__ignoreMap":35},[40,318,319,321,324,326],{"class":42,"line":43},[40,320,77],{"class":76},[40,322,323],{"class":80}," pandas ",[40,325,132],{"class":76},[40,327,328],{"class":80}," pd\n",[40,330,331],{"class":42,"line":84},[40,332,102],{"emptyLinePlaceholder":101},[40,334,335,338,340,343,346,349,352,354,357,359,362,364,367],{"class":42,"line":98},[40,336,337],{"class":108},"EXPECTED",[40,339,112],{"class":76},[40,341,342],{"class":80}," [",[40,344,345],{"class":50},"\"Branch\"",[40,347,348],{"class":80},", ",[40,350,351],{"class":50},"\"Date\"",[40,353,348],{"class":80},[40,355,356],{"class":50},"\"Product\"",[40,358,348],{"class":80},[40,360,361],{"class":50},"\"Units\"",[40,363,348],{"class":80},[40,365,366],{"class":50},"\"Revenue\"",[40,368,369],{"class":80},"]\n",[40,371,372],{"class":42,"line":105},[40,373,102],{"emptyLinePlaceholder":101},[40,375,376,379,383],{"class":42,"line":118},[40,377,378],{"class":76},"def",[40,380,382],{"class":381},"s_Opv"," read_and_validate",[40,384,385],{"class":80},"(path):\n",[40,387,388,391,394],{"class":42,"line":138},[40,389,390],{"class":80},"    df ",[40,392,393],{"class":76},"=",[40,395,396],{"class":80}," pd.read_excel(path)\n",[40,398,399,402,404,407,410,413,416,419,422,424,427,430],{"class":42,"line":147},[40,400,401],{"class":80},"    missing ",[40,403,393],{"class":76},[40,405,406],{"class":80}," [c ",[40,408,409],{"class":76},"for",[40,411,412],{"class":80}," c ",[40,414,415],{"class":76},"in",[40,417,418],{"class":108}," EXPECTED",[40,420,421],{"class":76}," if",[40,423,412],{"class":80},[40,425,426],{"class":76},"not",[40,428,429],{"class":76}," in",[40,431,432],{"class":80}," df.columns]\n",[40,434,435,438],{"class":42,"line":153},[40,436,437],{"class":76},"    if",[40,439,440],{"class":80}," missing:\n",[40,442,443,446,449,451,454,457,461,464,467,470,472,475,478,480,483],{"class":42,"line":159},[40,444,445],{"class":76},"        raise",[40,447,448],{"class":108}," ValueError",[40,450,204],{"class":80},[40,452,453],{"class":76},"f",[40,455,456],{"class":50},"\"missing columns ",[40,458,460],{"class":459},"sSjpA","{",[40,462,463],{"class":80},"missing",[40,465,466],{"class":459},"}",[40,468,469],{"class":50},"; found ",[40,471,460],{"class":459},[40,473,474],{"class":108},"list",[40,476,477],{"class":80},"(df.columns)",[40,479,466],{"class":459},[40,481,482],{"class":50},"\"",[40,484,195],{"class":80},[40,486,487,489,491,494,496],{"class":42,"line":165},[40,488,390],{"class":80},[40,490,393],{"class":76},[40,492,493],{"class":80}," df[",[40,495,337],{"class":108},[40,497,498],{"class":80},"].copy()\n",[40,500,501,504,506,509,511,514,516,519,523,525,528,531,534,537,539],{"class":42,"line":171},[40,502,503],{"class":80},"    df[",[40,505,351],{"class":50},[40,507,508],{"class":80},"] ",[40,510,393],{"class":76},[40,512,513],{"class":80}," pd.to_datetime(df[",[40,515,351],{"class":50},[40,517,518],{"class":80},"], ",[40,520,522],{"class":521},"sa561","errors",[40,524,393],{"class":76},[40,526,527],{"class":50},"\"coerce\"",[40,529,530],{"class":80},").dt.strftime(",[40,532,533],{"class":50},"\"%Y-%m-",[40,535,536],{"class":459},"%d",[40,538,482],{"class":50},[40,540,195],{"class":80},[40,542,543,546,548,551,553,556,559,562,564,566,568,570,572,575,578,581,583,585,588,590,592,594,597,600,602,605],{"class":42,"line":177},[40,544,545],{"class":80},"    bad ",[40,547,393],{"class":76},[40,549,550],{"class":80}," df[df[",[40,552,351],{"class":50},[40,554,555],{"class":80},"].isna() ",[40,557,558],{"class":76},"|",[40,560,561],{"class":80}," (pd.to_numeric(df[",[40,563,361],{"class":50},[40,565,518],{"class":80},[40,567,522],{"class":521},[40,569,393],{"class":76},[40,571,527],{"class":50},[40,573,574],{"class":80},") ",[40,576,577],{"class":76},"\u003C",[40,579,580],{"class":108}," 0",[40,582,574],{"class":80},[40,584,558],{"class":76},[40,586,587],{"class":80}," df[[",[40,589,345],{"class":50},[40,591,348],{"class":80},[40,593,356],{"class":50},[40,595,596],{"class":80},"]].isna().any(",[40,598,599],{"class":521},"axis",[40,601,393],{"class":76},[40,603,604],{"class":108},"1",[40,606,607],{"class":80},")]\n",[40,609,610,612,615],{"class":42,"line":183},[40,611,437],{"class":76},[40,613,614],{"class":108}," len",[40,616,617],{"class":80},"(bad):\n",[40,619,620,622,624,626,628,630,632,635,638,640,643,645,648,650,652,655,658,660,662],{"class":42,"line":189},[40,621,445],{"class":76},[40,623,448],{"class":108},[40,625,204],{"class":80},[40,627,453],{"class":76},[40,629,482],{"class":50},[40,631,460],{"class":459},[40,633,634],{"class":108},"len",[40,636,637],{"class":80},"(bad)",[40,639,466],{"class":459},[40,641,642],{"class":50}," invalid row(s), first at row ",[40,644,460],{"class":459},[40,646,647],{"class":80},"bad.index[",[40,649,245],{"class":108},[40,651,508],{"class":80},[40,653,654],{"class":76},"+",[40,656,657],{"class":108}," 2",[40,659,466],{"class":459},[40,661,482],{"class":50},[40,663,195],{"class":80},[40,665,666,668,671,673,675,677,679,681],{"class":42,"line":198},[40,667,437],{"class":76},[40,669,670],{"class":80}," df.duplicated([",[40,672,345],{"class":50},[40,674,348],{"class":80},[40,676,351],{"class":50},[40,678,348],{"class":80},[40,680,356],{"class":50},[40,682,683],{"class":80},"]).any():\n",[40,685,687,689,691,693,696],{"class":42,"line":686},16,[40,688,445],{"class":76},[40,690,448],{"class":108},[40,692,204],{"class":80},[40,694,695],{"class":50},"\"the same branch, date and product appears twice in the file\"",[40,697,195],{"class":80},[40,699,701,704],{"class":42,"line":700},17,[40,702,703],{"class":76},"    return",[40,705,706],{"class":80}," df\n",[10,708,709,710,713,714,717,718,713,721,724,725,23],{},"Every check runs before the database is touched. The columns must be the expected ones — the Hull file, with ",[214,711,712],{},"Day"," and ",[214,715,716],{},"Qty"," instead of ",[214,719,720],{},"Date",[214,722,723],{},"Units",", fails here with a message that lists what it found. Dates must parse, units must not be negative, keys must be present, and the file must not contain the same branch, date and product twice. A file is either entirely valid or rejected entirely; loading the good rows of a bad file leaves a day half-reported, which is worse than not loading it at all. More thorough row-level checks are covered in ",[14,726,728],{"href":727},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fvalidate-excel-columns-before-import-with-pandas\u002F","Validate Excel Columns Before Import with pandas",[223,730,232,735,232,738,232,741,232,744,232,751,232,757,232,761,232,766,232,770,232,776,232,781,232,786,232,792,232,796,232,800,232,804,232,807,232,810,232,815,232,818,232,822,232,825,232,829,232,834,232,837,232,840,232,843,232,847,232,850,232,853,232,858,232,862],{"viewBox":731,"role":226,"ariaLabelledBy":732,"xmlns":230,"style":231},"4 0 752 126",[733,734],"nightly-file-t","nightly-file-d",[234,736,737],{"id":733},"What happens to each file",[238,739,740],{"id":734},"Each file's content is hashed and skipped if the hash is already in the load log. Otherwise it is read and validated; a file that fails is moved to quarantine and logged. A valid file is written in one transaction with an upsert on branch, date and product, logged as loaded and moved to the processed folder.",[242,742],{"x":244,"y":245,"width":246,"height":743,"fill":248},"126",[242,745],{"x":251,"y":252,"width":746,"height":747,"rx":748,"fill":749,"stroke":750,"style":258},"115.2","94.0","12","#e7ebef","var(--line,#cdd5e6)",[260,752,756],{"x":753,"y":754,"style":755},"77.6","41.5","font-size:13.5px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","Hash the",[260,758,760],{"x":753,"y":759,"style":755},"59.0","bytes",[260,762,765],{"x":753,"y":763,"style":764},"78.5","font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","seen before →",[260,767,769],{"x":753,"y":768,"style":764},"96.0","skip",[42,771],{"x1":772,"y1":773,"x2":774,"y2":773,"stroke":775,"style":258},"140.2","63.0","162.2","var(--brand,#5b5cf0)",[777,778],"polygon",{"points":779,"fill":780},"166.2,63.0 157.2,68.0 157.2,58.0","#5b5cf0",[242,782],{"x":783,"y":252,"width":746,"height":747,"rx":748,"fill":784,"stroke":785,"style":258},"171.2","#fdefd8","var(--gold,#b4740a)",[260,787,791],{"x":788,"y":789,"style":790},"228.8","50.2","font-size:13.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","Read +",[260,793,795],{"x":788,"y":794,"style":790},"67.8","validate",[260,797,799],{"x":788,"y":798,"style":764},"87.2","all or nothing",[42,801],{"x1":802,"y1":773,"x2":803,"y2":773,"stroke":775,"style":258},"291.4","313.4",[777,805],{"points":806,"fill":780},"317.4,63.0 308.4,68.0 308.4,58.0",[242,808],{"x":809,"y":252,"width":746,"height":747,"rx":748,"fill":256,"stroke":257,"style":258},"322.4",[260,811,814],{"x":812,"y":759,"style":813},"380.0","font-size:13.5px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","Invalid",[260,816,817],{"x":812,"y":763,"style":764},"quarantine + log",[42,819],{"x1":820,"y1":773,"x2":821,"y2":773,"stroke":775,"style":258},"442.6","464.6",[777,823],{"points":824,"fill":780},"468.6,63.0 459.6,68.0 459.6,58.0",[242,826],{"x":827,"y":252,"width":746,"height":747,"rx":748,"fill":828,"stroke":775,"style":258},"473.6","#f0f4ff",[260,830,833],{"x":831,"y":754,"style":832},"531.2","font-size:13.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Upsert in one",[260,835,836],{"x":831,"y":759,"style":832},"transaction",[260,838,839],{"x":831,"y":763,"style":764},"branch + date +",[260,841,842],{"x":831,"y":768,"style":764},"product",[42,844],{"x1":845,"y1":773,"x2":846,"y2":773,"stroke":775,"style":258},"593.8","615.8",[777,848],{"points":849,"fill":780},"619.8,63.0 610.8,68.0 610.8,58.0",[242,851],{"x":852,"y":252,"width":746,"height":747,"rx":748,"fill":288,"stroke":289,"style":258},"624.8",[260,854,857],{"x":855,"y":789,"style":856},"682.4","font-size:13.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","Log + move",[260,859,861],{"x":855,"y":860,"style":764},"69.8","processed",[260,863,864],{"x":855,"y":798,"style":764},"folder",[25,866,868],{"id":867},"step-3-load-each-file-once-in-its-own-transaction","Step 3 — Load each file once, in its own transaction",[30,870,872],{"className":67,"code":871,"language":69,"meta":35,"style":35},"import hashlib, shutil\nfrom datetime import datetime\nfrom pathlib import Path\n\nUPSERT = \"\"\"INSERT INTO daily_sales (branch, date, product, units, revenue, source_file, loaded_at)\n            VALUES (?, ?, ?, ?, ?, ?, ?)\n            ON CONFLICT (branch, date, product) DO UPDATE SET\n              units = excluded.units, revenue = excluded.revenue,\n              source_file = excluded.source_file, loaded_at = excluded.loaded_at\"\"\"\n\ndef load_folder(inbox, processed, quarantine):\n    results = []\n    for path in sorted(Path(inbox).glob(\"*.xlsx\"), key=lambda p: p.stat().st_mtime):\n        digest = hashlib.sha256(path.read_bytes()).hexdigest()\n        now = datetime.now().isoformat(timespec=\"seconds\")\n        with closing(sqlite3.connect(DB)) as con:\n            if con.execute(\"SELECT 1 FROM load_log WHERE file_hash = ? AND status = 'loaded'\", (digest,)).fetchone():\n                results.append((path.name, \"skipped\", 0, \"identical file already loaded\"))\n                shutil.move(path, Path(processed) \u002F path.name)\n                continue\n            try:\n                df = read_and_validate(path)\n                with con:                                            # one transaction per file\n                    con.executemany(UPSERT, [(*r, path.name, now) for r in df.itertuples(index=False)])\n                    con.execute(\"INSERT OR REPLACE INTO load_log VALUES (?, ?, 'loaded', ?, '', ?)\", (digest, path.name, len(df), now))\n                results.append((path.name, \"loaded\", len(df), \"\"))\n                shutil.move(path, Path(processed) \u002F path.name)\n            except Exception as e:\n                with con:\n                    con.execute(\"INSERT OR REPLACE INTO load_log VALUES (?, ?, 'rejected', 0, ?, ?)\", (digest, path.name, str(e)[:200], now))\n                results.append((path.name, \"rejected\", 0, str(e)[:80]))\n                shutil.move(path, Path(quarantine) \u002F path.name)\n    return pd.DataFrame(results, columns=[\"File\", \"Status\", \"Rows\", \"Message\"])\n\nfor d in (\"processed\", \"quarantine\"):\n    Path(d).mkdir(exist_ok=True)\nsummary = load_folder(\"inbox\", \"processed\", \"quarantine\")\nprint(summary.to_string(index=False))\n",[37,873,874,881,893,905,909,919,924,929,934,939,943,953,963,994,1004,1024,1040,1054,1075,1087,1093,1102,1113,1126,1164,1181,1201,1210,1225,1232,1254,1278,1288,1325,1330,1354,1370,1394],{"__ignoreMap":35},[40,875,876,878],{"class":42,"line":43},[40,877,77],{"class":76},[40,879,880],{"class":80}," hashlib, shutil\n",[40,882,883,885,888,890],{"class":42,"line":84},[40,884,87],{"class":76},[40,886,887],{"class":80}," datetime ",[40,889,77],{"class":76},[40,891,892],{"class":80}," datetime\n",[40,894,895,897,900,902],{"class":42,"line":98},[40,896,87],{"class":76},[40,898,899],{"class":80}," pathlib ",[40,901,77],{"class":76},[40,903,904],{"class":80}," Path\n",[40,906,907],{"class":42,"line":105},[40,908,102],{"emptyLinePlaceholder":101},[40,910,911,914,916],{"class":42,"line":118},[40,912,913],{"class":108},"UPSERT",[40,915,112],{"class":76},[40,917,918],{"class":50}," \"\"\"INSERT INTO daily_sales (branch, date, product, units, revenue, source_file, loaded_at)\n",[40,920,921],{"class":42,"line":138},[40,922,923],{"class":50},"            VALUES (?, ?, ?, ?, ?, ?, ?)\n",[40,925,926],{"class":42,"line":147},[40,927,928],{"class":50},"            ON CONFLICT (branch, date, product) DO UPDATE SET\n",[40,930,931],{"class":42,"line":153},[40,932,933],{"class":50},"              units = excluded.units, revenue = excluded.revenue,\n",[40,935,936],{"class":42,"line":159},[40,937,938],{"class":50},"              source_file = excluded.source_file, loaded_at = excluded.loaded_at\"\"\"\n",[40,940,941],{"class":42,"line":165},[40,942,102],{"emptyLinePlaceholder":101},[40,944,945,947,950],{"class":42,"line":171},[40,946,378],{"class":76},[40,948,949],{"class":381}," load_folder",[40,951,952],{"class":80},"(inbox, processed, quarantine):\n",[40,954,955,958,960],{"class":42,"line":177},[40,956,957],{"class":80},"    results ",[40,959,393],{"class":76},[40,961,962],{"class":80}," []\n",[40,964,965,968,971,973,976,979,982,985,988,991],{"class":42,"line":183},[40,966,967],{"class":76},"    for",[40,969,970],{"class":80}," path ",[40,972,415],{"class":76},[40,974,975],{"class":108}," sorted",[40,977,978],{"class":80},"(Path(inbox).glob(",[40,980,981],{"class":50},"\"*.xlsx\"",[40,983,984],{"class":80},"), ",[40,986,987],{"class":521},"key",[40,989,990],{"class":76},"=lambda",[40,992,993],{"class":80}," p: p.stat().st_mtime):\n",[40,995,996,999,1001],{"class":42,"line":189},[40,997,998],{"class":80},"        digest ",[40,1000,393],{"class":76},[40,1002,1003],{"class":80}," hashlib.sha256(path.read_bytes()).hexdigest()\n",[40,1005,1006,1009,1011,1014,1017,1019,1022],{"class":42,"line":198},[40,1007,1008],{"class":80},"        now ",[40,1010,393],{"class":76},[40,1012,1013],{"class":80}," datetime.now().isoformat(",[40,1015,1016],{"class":521},"timespec",[40,1018,393],{"class":76},[40,1020,1021],{"class":50},"\"seconds\"",[40,1023,195],{"class":80},[40,1025,1026,1029,1031,1033,1035,1037],{"class":42,"line":686},[40,1027,1028],{"class":76},"        with",[40,1030,124],{"class":80},[40,1032,109],{"class":108},[40,1034,129],{"class":80},[40,1036,132],{"class":76},[40,1038,1039],{"class":80}," con:\n",[40,1041,1042,1045,1048,1051],{"class":42,"line":700},[40,1043,1044],{"class":76},"            if",[40,1046,1047],{"class":80}," con.execute(",[40,1049,1050],{"class":50},"\"SELECT 1 FROM load_log WHERE file_hash = ? AND status = 'loaded'\"",[40,1052,1053],{"class":80},", (digest,)).fetchone():\n",[40,1055,1057,1060,1063,1065,1067,1069,1072],{"class":42,"line":1056},18,[40,1058,1059],{"class":80},"                results.append((path.name, ",[40,1061,1062],{"class":50},"\"skipped\"",[40,1064,348],{"class":80},[40,1066,245],{"class":108},[40,1068,348],{"class":80},[40,1070,1071],{"class":50},"\"identical file already loaded\"",[40,1073,1074],{"class":80},"))\n",[40,1076,1078,1081,1084],{"class":42,"line":1077},19,[40,1079,1080],{"class":80},"                shutil.move(path, Path(processed) ",[40,1082,1083],{"class":76},"\u002F",[40,1085,1086],{"class":80}," path.name)\n",[40,1088,1090],{"class":42,"line":1089},20,[40,1091,1092],{"class":76},"                continue\n",[40,1094,1096,1099],{"class":42,"line":1095},21,[40,1097,1098],{"class":76},"            try",[40,1100,1101],{"class":80},":\n",[40,1103,1105,1108,1110],{"class":42,"line":1104},22,[40,1106,1107],{"class":80},"                df ",[40,1109,393],{"class":76},[40,1111,1112],{"class":80}," read_and_validate(path)\n",[40,1114,1116,1119,1122],{"class":42,"line":1115},23,[40,1117,1118],{"class":76},"                with",[40,1120,1121],{"class":80}," con:                                            ",[40,1123,1125],{"class":1124},"s-wDw","# one transaction per file\n",[40,1127,1129,1132,1134,1137,1140,1143,1145,1148,1150,1153,1156,1158,1161],{"class":42,"line":1128},24,[40,1130,1131],{"class":80},"                    con.executemany(",[40,1133,913],{"class":108},[40,1135,1136],{"class":80},", [(",[40,1138,1139],{"class":76},"*",[40,1141,1142],{"class":80},"r, path.name, now) ",[40,1144,409],{"class":76},[40,1146,1147],{"class":80}," r ",[40,1149,415],{"class":76},[40,1151,1152],{"class":80}," df.itertuples(",[40,1154,1155],{"class":521},"index",[40,1157,393],{"class":76},[40,1159,1160],{"class":108},"False",[40,1162,1163],{"class":80},")])\n",[40,1165,1167,1170,1173,1176,1178],{"class":42,"line":1166},25,[40,1168,1169],{"class":80},"                    con.execute(",[40,1171,1172],{"class":50},"\"INSERT OR REPLACE INTO load_log VALUES (?, ?, 'loaded', ?, '', ?)\"",[40,1174,1175],{"class":80},", (digest, path.name, ",[40,1177,634],{"class":108},[40,1179,1180],{"class":80},"(df), now))\n",[40,1182,1184,1186,1189,1191,1193,1196,1199],{"class":42,"line":1183},26,[40,1185,1059],{"class":80},[40,1187,1188],{"class":50},"\"loaded\"",[40,1190,348],{"class":80},[40,1192,634],{"class":108},[40,1194,1195],{"class":80},"(df), ",[40,1197,1198],{"class":50},"\"\"",[40,1200,1074],{"class":80},[40,1202,1204,1206,1208],{"class":42,"line":1203},27,[40,1205,1080],{"class":80},[40,1207,1083],{"class":76},[40,1209,1086],{"class":80},[40,1211,1213,1216,1219,1222],{"class":42,"line":1212},28,[40,1214,1215],{"class":76},"            except",[40,1217,1218],{"class":108}," Exception",[40,1220,1221],{"class":76}," as",[40,1223,1224],{"class":80}," e:\n",[40,1226,1228,1230],{"class":42,"line":1227},29,[40,1229,1118],{"class":76},[40,1231,1039],{"class":80},[40,1233,1235,1237,1240,1242,1245,1248,1251],{"class":42,"line":1234},30,[40,1236,1169],{"class":80},[40,1238,1239],{"class":50},"\"INSERT OR REPLACE INTO load_log VALUES (?, ?, 'rejected', 0, ?, ?)\"",[40,1241,1175],{"class":80},[40,1243,1244],{"class":108},"str",[40,1246,1247],{"class":80},"(e)[:",[40,1249,1250],{"class":108},"200",[40,1252,1253],{"class":80},"], now))\n",[40,1255,1257,1259,1262,1264,1266,1268,1270,1272,1275],{"class":42,"line":1256},31,[40,1258,1059],{"class":80},[40,1260,1261],{"class":50},"\"rejected\"",[40,1263,348],{"class":80},[40,1265,245],{"class":108},[40,1267,348],{"class":80},[40,1269,1244],{"class":108},[40,1271,1247],{"class":80},[40,1273,1274],{"class":108},"80",[40,1276,1277],{"class":80},"]))\n",[40,1279,1281,1284,1286],{"class":42,"line":1280},32,[40,1282,1283],{"class":80},"                shutil.move(path, Path(quarantine) ",[40,1285,1083],{"class":76},[40,1287,1086],{"class":80},[40,1289,1291,1293,1296,1299,1301,1304,1307,1309,1312,1314,1317,1319,1322],{"class":42,"line":1290},33,[40,1292,703],{"class":76},[40,1294,1295],{"class":80}," pd.DataFrame(results, ",[40,1297,1298],{"class":521},"columns",[40,1300,393],{"class":76},[40,1302,1303],{"class":80},"[",[40,1305,1306],{"class":50},"\"File\"",[40,1308,348],{"class":80},[40,1310,1311],{"class":50},"\"Status\"",[40,1313,348],{"class":80},[40,1315,1316],{"class":50},"\"Rows\"",[40,1318,348],{"class":80},[40,1320,1321],{"class":50},"\"Message\"",[40,1323,1324],{"class":80},"])\n",[40,1326,1328],{"class":42,"line":1327},34,[40,1329,102],{"emptyLinePlaceholder":101},[40,1331,1333,1335,1338,1340,1343,1346,1348,1351],{"class":42,"line":1332},35,[40,1334,409],{"class":76},[40,1336,1337],{"class":80}," d ",[40,1339,415],{"class":76},[40,1341,1342],{"class":80}," (",[40,1344,1345],{"class":50},"\"processed\"",[40,1347,348],{"class":80},[40,1349,1350],{"class":50},"\"quarantine\"",[40,1352,1353],{"class":80},"):\n",[40,1355,1357,1360,1363,1365,1368],{"class":42,"line":1356},36,[40,1358,1359],{"class":80},"    Path(d).mkdir(",[40,1361,1362],{"class":521},"exist_ok",[40,1364,393],{"class":76},[40,1366,1367],{"class":108},"True",[40,1369,195],{"class":80},[40,1371,1373,1376,1378,1381,1384,1386,1388,1390,1392],{"class":42,"line":1372},37,[40,1374,1375],{"class":80},"summary ",[40,1377,393],{"class":76},[40,1379,1380],{"class":80}," load_folder(",[40,1382,1383],{"class":50},"\"inbox\"",[40,1385,348],{"class":80},[40,1387,1345],{"class":50},[40,1389,348],{"class":80},[40,1391,1350],{"class":50},[40,1393,195],{"class":80},[40,1395,1397,1399,1402,1404,1406,1408],{"class":42,"line":1396},38,[40,1398,201],{"class":108},[40,1400,1401],{"class":80},"(summary.to_string(",[40,1403,1155],{"class":521},[40,1405,393],{"class":76},[40,1407,1160],{"class":108},[40,1409,1074],{"class":80},[10,1411,1412],{},"Files are processed oldest first, so when a corrected file arrives after the original, the correction is applied last. Each file's content is hashed with SHA-256 and looked up in the load log: a file whose exact bytes were already loaded is skipped and filed away, which is how the resent Leeds return is handled — its name differs, its content does not. A valid file is written inside a single transaction together with its log entry: either every row and the log line are committed, or, if anything fails, none of them are. The upsert inserts new rows and, for a branch, date and product that already exist, overwrites the figures — so the corrected York file updates product A from 7 units to 8 without creating a duplicate. Rejected files are logged with the reason and moved to a quarantine folder, out of the way of the next run but kept for whoever has to follow up.",[223,1414,232,1420,232,1423,232,1426,232,1431,232,1436,232,1440,232,1446,232,1450,232,1454,232,1458,232,1462,232,1466,232,1470,232,1474,232,1477,232,1480,232,1484,232,1486,232,1489,232,1491,232,1494,232,1496,232,1499,232,1502,232,1506,232,1508,232,1511,232,1513,232,1516,232,1518,232,1520,232,1522,232,1524,232,1528,232,1530,232,1533,232,1535,232,1537,232,1539,232,1541,232,1543,232,1546,232,1549,232,1551,232,1554,232,1556,232,1560,232,1562,232,1564,232,1566,232,1569,232,1572,232,1576,232,1578,232,1581,232,1583,232,1587,232,1589,232,1591,232,1593,232,1596,232,1599,232,1603,232,1605,232,1608,232,1610,232,1613,232,1615,232,1617,232,1619,232,1622,232,1625,232,1629,232,1631,232,1634,232,1636,232,1638,232,1640,232,1642,232,1644],{"viewBox":1415,"role":226,"ariaLabelledBy":1416,"xmlns":230,"style":1419},"69 0 622 252",[1417,1418],"nightly-summary-t","nightly-summary-d","width:100%;max-width:622px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[234,1421,1422],{"id":1417},"Tonight's results",[238,1424,1425],{"id":1418},"The Leeds and York returns load two rows each. The resent Leeds file is skipped as identical. The Hull file is rejected for missing columns. The corrected York file loads two rows, updating product A. The notes file is rejected because it is not a valid workbook.",[242,1427],{"x":1428,"y":245,"width":1429,"height":1430,"fill":248},"69","622","252",[242,1432],{"x":1433,"y":252,"width":1434,"height":1435,"rx":245,"fill":749,"stroke":750,"style":271},"85.3","32.0","24.0",[242,1437],{"x":1438,"y":252,"width":1439,"height":1435,"rx":245,"fill":749,"stroke":750,"style":271},"117.3","225.6",[260,1441,1445],{"x":1442,"y":1443,"style":1444},"230.1","33.0","font-size:11px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","A",[242,1447],{"x":1448,"y":252,"width":1449,"height":1435,"rx":245,"fill":749,"stroke":750,"style":271},"342.9","75.0",[260,1451,1453],{"x":1452,"y":1443,"style":1444},"380.4","B",[242,1455],{"x":1456,"y":252,"width":1457,"height":1435,"rx":245,"fill":749,"stroke":750,"style":271},"418.0","57.8",[260,1459,1461],{"x":1460,"y":1443,"style":1444},"446.8","C",[242,1463],{"x":1464,"y":252,"width":1465,"height":1435,"rx":245,"fill":749,"stroke":750,"style":271},"475.7","198.9",[260,1467,1469],{"x":1468,"y":1443,"style":1444},"575.2","D",[242,1471],{"x":1433,"y":1472,"width":1434,"height":1473,"rx":245,"fill":749,"stroke":750,"style":271},"40.0","28.0",[260,1475,604],{"x":1476,"y":759,"style":1444},"101.3",[242,1478],{"x":1438,"y":1472,"width":1439,"height":1473,"rx":245,"fill":828,"stroke":775,"style":1479},"stroke-width:1.5px",[260,1481,1483],{"x":1442,"y":759,"style":1482},"font-size:12px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","File",[242,1485],{"x":1448,"y":1472,"width":1449,"height":1473,"rx":245,"fill":828,"stroke":775,"style":1479},[260,1487,1488],{"x":1452,"y":759,"style":1482},"Status",[242,1490],{"x":1456,"y":1472,"width":1457,"height":1473,"rx":245,"fill":828,"stroke":775,"style":1479},[260,1492,1493],{"x":1460,"y":759,"style":1482},"Rows",[242,1495],{"x":1464,"y":1472,"width":1465,"height":1473,"rx":245,"fill":828,"stroke":775,"style":1479},[260,1497,1498],{"x":1468,"y":759,"style":1482},"Reason",[242,1500],{"x":1433,"y":1501,"width":1434,"height":1473,"rx":245,"fill":749,"stroke":750,"style":271},"68.0",[260,1503,1505],{"x":1476,"y":1504,"style":1444},"87.0","2",[242,1507],{"x":1438,"y":1501,"width":1439,"height":1473,"rx":245,"fill":248,"stroke":750,"style":271},[260,1509,1510],{"x":1442,"y":1504,"style":275},"leeds_2026-10-01.xlsx",[242,1512],{"x":1448,"y":1501,"width":1449,"height":1473,"rx":245,"fill":248,"stroke":750,"style":271},[260,1514,1515],{"x":1452,"y":1504,"style":275},"loaded",[242,1517],{"x":1456,"y":1501,"width":1457,"height":1473,"rx":245,"fill":248,"stroke":750,"style":271},[260,1519,1505],{"x":1460,"y":1504,"style":275},[242,1521],{"x":1464,"y":1501,"width":1465,"height":1473,"rx":245,"fill":248,"stroke":750,"style":271},[242,1523],{"x":1433,"y":768,"width":1434,"height":1473,"rx":245,"fill":749,"stroke":750,"style":271},[260,1525,1527],{"x":1476,"y":1526,"style":1444},"115.0","3",[242,1529],{"x":1438,"y":768,"width":1439,"height":1473,"rx":245,"fill":248,"stroke":750,"style":271},[260,1531,1532],{"x":1442,"y":1526,"style":275},"york_2026-10-01.xlsx",[242,1534],{"x":1448,"y":768,"width":1449,"height":1473,"rx":245,"fill":248,"stroke":750,"style":271},[260,1536,1515],{"x":1452,"y":1526,"style":275},[242,1538],{"x":1456,"y":768,"width":1457,"height":1473,"rx":245,"fill":248,"stroke":750,"style":271},[260,1540,1505],{"x":1460,"y":1526,"style":275},[242,1542],{"x":1464,"y":768,"width":1465,"height":1473,"rx":245,"fill":248,"stroke":750,"style":271},[242,1544],{"x":1433,"y":1545,"width":1434,"height":1473,"rx":245,"fill":749,"stroke":750,"style":271},"124.0",[260,1547,244],{"x":1476,"y":1548,"style":1444},"143.0",[242,1550],{"x":1438,"y":1545,"width":1439,"height":1473,"rx":245,"fill":248,"stroke":750,"style":271},[260,1552,1553],{"x":1442,"y":1548,"style":275},"leeds_2026-10-01 (resent).xlsx",[242,1555],{"x":1448,"y":1545,"width":1449,"height":1473,"rx":245,"fill":784,"stroke":785,"style":1479},[260,1557,1559],{"x":1452,"y":1548,"style":1558},"font-size:12px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","skipped",[242,1561],{"x":1456,"y":1545,"width":1457,"height":1473,"rx":245,"fill":248,"stroke":750,"style":271},[260,1563,245],{"x":1460,"y":1548,"style":275},[242,1565],{"x":1464,"y":1545,"width":1465,"height":1473,"rx":245,"fill":248,"stroke":750,"style":271},[260,1567,1568],{"x":1468,"y":1548,"style":275},"identical file already loaded",[242,1570],{"x":1433,"y":1571,"width":1434,"height":1473,"rx":245,"fill":749,"stroke":750,"style":271},"152.0",[260,1573,1575],{"x":1476,"y":1574,"style":1444},"171.0","5",[242,1577],{"x":1438,"y":1571,"width":1439,"height":1473,"rx":245,"fill":248,"stroke":750,"style":271},[260,1579,1580],{"x":1442,"y":1574,"style":275},"hull_2026-10-01.xlsx",[242,1582],{"x":1448,"y":1571,"width":1449,"height":1473,"rx":245,"fill":256,"stroke":257,"style":1479},[260,1584,1586],{"x":1452,"y":1574,"style":1585},"font-size:12px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","rejected",[242,1588],{"x":1456,"y":1571,"width":1457,"height":1473,"rx":245,"fill":248,"stroke":750,"style":271},[260,1590,245],{"x":1460,"y":1574,"style":275},[242,1592],{"x":1464,"y":1571,"width":1465,"height":1473,"rx":245,"fill":248,"stroke":750,"style":271},[260,1594,1595],{"x":1468,"y":1574,"style":275},"missing columns Date, Units",[242,1597],{"x":1433,"y":1598,"width":1434,"height":1473,"rx":245,"fill":749,"stroke":750,"style":271},"180.0",[260,1600,1602],{"x":1476,"y":1601,"style":1444},"199.0","6",[242,1604],{"x":1438,"y":1598,"width":1439,"height":1473,"rx":245,"fill":248,"stroke":750,"style":271},[260,1606,1607],{"x":1442,"y":1601,"style":275},"york_2026-10-01_corrected.xlsx",[242,1609],{"x":1448,"y":1598,"width":1449,"height":1473,"rx":245,"fill":288,"stroke":289,"style":1479},[260,1611,1515],{"x":1452,"y":1601,"style":1612},"font-size:12px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle",[242,1614],{"x":1456,"y":1598,"width":1457,"height":1473,"rx":245,"fill":248,"stroke":750,"style":271},[260,1616,1505],{"x":1460,"y":1601,"style":275},[242,1618],{"x":1464,"y":1598,"width":1465,"height":1473,"rx":245,"fill":248,"stroke":750,"style":271},[260,1620,1621],{"x":1468,"y":1601,"style":275},"updates product A",[242,1623],{"x":1433,"y":1624,"width":1434,"height":1473,"rx":245,"fill":749,"stroke":750,"style":271},"208.0",[260,1626,1628],{"x":1476,"y":1627,"style":1444},"227.0","7",[242,1630],{"x":1438,"y":1624,"width":1439,"height":1473,"rx":245,"fill":248,"stroke":750,"style":271},[260,1632,1633],{"x":1442,"y":1627,"style":275},"notes.xlsx",[242,1635],{"x":1448,"y":1624,"width":1449,"height":1473,"rx":245,"fill":256,"stroke":257,"style":1479},[260,1637,1586],{"x":1452,"y":1627,"style":1585},[242,1639],{"x":1456,"y":1624,"width":1457,"height":1473,"rx":245,"fill":248,"stroke":750,"style":271},[260,1641,245],{"x":1460,"y":1627,"style":275},[242,1643],{"x":1464,"y":1624,"width":1465,"height":1473,"rx":245,"fill":248,"stroke":750,"style":271},[260,1645,1646],{"x":1468,"y":1627,"style":275},"not a valid workbook",[25,1648,1650],{"id":1649},"step-4-check-the-database-and-report","Step 4 — Check the database and report",[30,1652,1654],{"className":67,"code":1653,"language":69,"meta":35,"style":35},"with closing(sqlite3.connect(DB)) as con:\n    print(pd.read_sql(\"SELECT branch, date, product, units, revenue, source_file FROM daily_sales ORDER BY branch, product\", con).to_string(index=False))\n    print(pd.read_sql(\"SELECT status, COUNT(*) AS files FROM load_log GROUP BY status\", con).to_string(index=False))\n",[37,1655,1656,1670,1692],{"__ignoreMap":35},[40,1657,1658,1660,1662,1664,1666,1668],{"class":42,"line":43},[40,1659,121],{"class":76},[40,1661,124],{"class":80},[40,1663,109],{"class":108},[40,1665,129],{"class":80},[40,1667,132],{"class":76},[40,1669,1039],{"class":80},[40,1671,1672,1675,1678,1681,1684,1686,1688,1690],{"class":42,"line":84},[40,1673,1674],{"class":108},"    print",[40,1676,1677],{"class":80},"(pd.read_sql(",[40,1679,1680],{"class":50},"\"SELECT branch, date, product, units, revenue, source_file FROM daily_sales ORDER BY branch, product\"",[40,1682,1683],{"class":80},", con).to_string(",[40,1685,1155],{"class":521},[40,1687,393],{"class":76},[40,1689,1160],{"class":108},[40,1691,1074],{"class":80},[40,1693,1694,1696,1698,1701,1703,1705,1707,1709],{"class":42,"line":98},[40,1695,1674],{"class":108},[40,1697,1677],{"class":80},[40,1699,1700],{"class":50},"\"SELECT status, COUNT(*) AS files FROM load_log GROUP BY status\"",[40,1702,1683],{"class":80},[40,1704,1155],{"class":521},[40,1706,393],{"class":76},[40,1708,1160],{"class":108},[40,1710,1074],{"class":80},[10,1712,1713,1714,1718],{},"The table now holds one row per branch, date and product: Leeds' two products from the original file and York's two from the corrected one, with product A showing 8 units and the corrected file as its source. Nothing from the Hull file or the non-workbook is there, and the resend added nothing. The load log records every file the job has seen, so tomorrow's run will not reprocess any of them. The night's summary table is what goes to whoever looks after the data — a short email listing loaded, skipped and rejected files, as in ",[14,1715,1717],{"href":1716},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Fsend-an-alert-when-an-excel-report-job-fails\u002F","Send an Alert When an Excel Report Job Fails",", with the rejected ones needing a reply to the sender.",[25,1720,1722],{"id":1721},"common-pitfalls","Common pitfalls",[1724,1725,1726,1734,1740,1746,1752],"ul",{},[1727,1728,1729,1733],"li",{},[1730,1731,1732],"strong",{},"Deduplicating by file name."," Resends get new names; hash the content.",[1727,1735,1736,1739],{},[1730,1737,1738],{},"Loading row by row with autocommit."," One failure leaves a file half-loaded; use one transaction per file.",[1727,1741,1742,1745],{},[1730,1743,1744],{},"Plain INSERT for daily returns."," Corrections then duplicate; upsert on the business key.",[1727,1747,1748,1751],{},[1730,1749,1750],{},"Deleting bad files."," Move them to quarantine with the reason in the log.",[1727,1753,1754,1757],{},[1730,1755,1756],{},"No load log."," Without it there is no audit trail and no protection against reprocessing.",[25,1759,1761],{"id":1760},"conclusion","Conclusion",[10,1763,1764],{},"A nightly loader for emailed workbooks should identify files by a hash of their content and skip ones already loaded, validate each workbook completely before writing, write each file in a single transaction with an upsert on the business key so corrections replace earlier figures, log every file's outcome in a table, and move rejected files to quarantine with the reason. The result is a database that matches the latest version of every return, and a log that explains how it got there.",[25,1766,1768],{"id":1767},"frequently-asked-questions","Frequently asked questions",[10,1770,1771,1774],{},[1730,1772,1773],{},"How do I stop the same Excel file being loaded twice?","\nHash the file's bytes with SHA-256 and record the hash in a load log; skip any file whose hash is already logged as loaded.",[10,1776,1777,1780,1781,1784],{},[1730,1778,1779],{},"How do I load a corrected file without creating duplicates?","\nMake the business key the primary key and use an upsert — ",[37,1782,1783],{},"INSERT … ON CONFLICT … DO UPDATE"," in SQLite and PostgreSQL.",[10,1786,1787,1790],{},[1730,1788,1789],{},"Why load each file in its own transaction?","\nSo a failure part-way through leaves the database unchanged for that file instead of half-loaded.",[10,1792,1793,1796],{},[1730,1794,1795],{},"What should happen to files that fail validation?","\nMove them to a quarantine folder, record the reason in the load log and tell the sender.",[25,1798,1800],{"id":1799},"related","Related",[1724,1802,1803,1809,1814,1821],{},[1727,1804,1805,1806,1808],{},"Up one level: ",[14,1807,22],{"href":21}," — Excel and databases.",[1727,1810,1811,1813],{},[14,1812,17],{"href":16}," — getting the files.",[1727,1815,1816,1820],{},[14,1817,1819],{"href":1818},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fload-excel-into-sqlite-with-a-defined-schema\u002F","Load Excel into SQLite with a Defined Schema"," — table design.",[1727,1822,1823,1827],{},[14,1824,1826],{"href":1825},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Fquarantine-bad-excel-files-in-a-batch-job\u002F","Quarantine Bad Excel Files in a Batch Job"," — handling rejects.",[1829,1830,1831],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}",{"title":35,"searchDepth":84,"depth":84,"links":1833},[1834,1835,1836,1837,1838,1839,1840,1841,1842],{"id":27,"depth":84,"text":28},{"id":63,"depth":84,"text":64},{"id":310,"depth":84,"text":311},{"id":867,"depth":84,"text":868},{"id":1649,"depth":84,"text":1650},{"id":1721,"depth":84,"text":1722},{"id":1760,"depth":84,"text":1761},{"id":1767,"depth":84,"text":1768},{"id":1799,"depth":84,"text":1800},"2026-10-02","Load the spreadsheets that arrive by email into a database every night without duplicates or half-loaded files: take the saved attachments from an inbox folder, skip files already loaded by their content hash, validate each workbook's columns and values, write each file in its own transaction with an upsert on the business key, record every load in a log table, move rejected files to a quarantine folder, and report the night's results.","md",[1847,1849,1851,1853],{"q":1773,"a":1848},"Hash the file's bytes with SHA-256 and record the hash in a load log; skip any file whose hash is already logged as loaded.",{"q":1779,"a":1850},"Make the business key the primary key and use an upsert — INSERT … ON CONFLICT … DO UPDATE in SQLite and PostgreSQL.",{"q":1789,"a":1852},"So a failure part-way through leaves the database unchanged for that file instead of half-loaded.",{"q":1795,"a":1854},"Move them to a quarantine folder, record the reason in the load log and tell the sender.","load excel attachments into database nightly",null,{"breadcrumb":1858},[1859,1861,1864],{"name":1860,"item":1083},"Home",{"name":1862,"item":1863},"Advanced Data Transformation and Cleaning for Excel Automation","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":22,"item":21},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fread-excel-attachments-into-a-database-nightly",{"title":1867,"description":1868},"Load Emailed Excel Attachments into a Database Nightly","A nightly loader for Excel attachments: content-hash deduplication, column and value validation, one transaction per file, upserts on a business key, a load log table, quarantine for bad files and a summary.","read-excel-attachments-into-a-database-nightly","advanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fread-excel-attachments-into-a-database-nightly\u002Findex",[1872,1873],"data-sources","scheduling","how-to","s_abtih7fbGW4EzcFIDYM-gxsfE1JM83eG1RBY_akaM",[1877,1881],{"title":1878,"path":1879,"stem":1880,"children":-1},"Query Excel Files with SQL Using DuckDB","\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fquery-excel-files-with-sql-using-duckdb","advanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fquery-excel-files-with-sql-using-duckdb\u002Findex",{"title":1882,"path":1883,"stem":1884,"children":-1},"Refresh an Excel Report from a Database on a Schedule","\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Frefresh-an-excel-report-from-a-database-on-a-schedule","advanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Frefresh-an-excel-report-from-a-database-on-a-schedule\u002Findex",[1886,1888],{"path":1887,"title":1862},"\u002Fadvanced-data-transformation-and-cleaning",{"path":1889,"title":22},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases",[],[1892,1895,1898,1901,1904,1907],{"path":1893,"title":1894,"kind":1874},"\u002Fautomating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage\u002Fread-an-excel-file-directly-from-amazon-s3-with-pandas","Read an Excel File Directly from Amazon S3 with pandas",{"path":1896,"title":1897,"kind":1874},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fcache-api-responses-when-building-excel-reports","Cache API Responses When Building Excel Reports",{"path":1899,"title":1900,"kind":1874},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fcompare-an-excel-file-with-a-database-table","Compare an Excel File with a Database Table",{"path":1902,"title":1903,"kind":1874},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fdownload-new-excel-files-from-an-sftp-server","Download New Excel Files from an SFTP Server",{"path":1905,"title":1906,"kind":1874},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fexport-a-large-database-table-to-excel-in-chunks","Export a Large Database Table to Excel in Chunks",{"path":1908,"title":1909,"kind":1874},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fexport-database-lookup-tables-as-excel-dropdown-lists","Export Database Lookup Tables as Excel Dropdown Lists",1790972096887]