[{"data":1,"prerenderedAt":1899},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fdownload-new-excel-files-from-an-sftp-server":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fdownload-new-excel-files-from-an-sftp-server":1869,"ancestors:\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fdownload-new-excel-files-from-an-sftp-server":1878,"children:\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fdownload-new-excel-files-from-an-sftp-server":1883,"related:\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fdownload-new-excel-files-from-an-sftp-server":1884},{"id":4,"title":5,"body":6,"dateModified":1836,"datePublished":1836,"description":1837,"extension":1838,"faq":1839,"keyword":1848,"kind":1849,"meta":1850,"navigation":91,"path":1858,"seo":1859,"slug":1862,"stem":1863,"topics":1864,"type":1867,"__hash__":1868},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fdownload-new-excel-files-from-an-sftp-server\u002Findex.md","Download New Excel Files from an SFTP Server",{"type":7,"value":8,"toc":1825},"minimark",[9,19,24,53,65,69,290,304,456,460,846,863,867,1304,1321,1518,1582,1586,1684,1695,1699,1741,1745,1757,1761,1767,1773,1782,1788,1792,1821],[10,11,12,13,18],"p",{},"Banks, logistics providers, payroll bureaus and marketplaces often deliver their reports by dropping files on an SFTP server: a new stock file every night, a settlement report every morning. The receiving side needs a job that runs on a schedule, picks up the files it has not seen before, and leaves the rest alone. The details are where these jobs fail: downloading a file that the partner is still uploading and getting half a workbook, fetching everything again on every run, missing a file the partner re-uploaded with corrections under the same name, or connecting to whatever server answers because the host key was never checked. This page builds a downloader with paramiko that handles each of those: a pinned host key, a listing with sizes and times, a rule for files still being written, a state file that records what was fetched, downloads that only appear under their final name when complete, and a check that each workbook opens. Everything is tested against an SFTP server running locally. It belongs to ",[14,15,17],"a",{"href":16},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002F","Moving Data Between Excel and Databases",".",[20,21,23],"h2",{"id":22},"prerequisites","Prerequisites",[25,26,31],"pre",{"className":27,"code":28,"language":29,"meta":30,"style":30},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install paramiko openpyxl\n","bash","",[32,33,34],"code",{"__ignoreMap":30},[35,36,39,43,47,50],"span",{"class":37,"line":38},"line",1,[35,40,42],{"class":41},"sMTad","pip",[35,44,46],{"class":45},"srMev"," install",[35,48,49],{"class":45}," paramiko",[35,51,52],{"class":45}," openpyxl\n",[10,54,55,56,60,61,64],{},"The partner publishes files in an ",[57,58,59],"em",{},"outgoing"," folder on their SFTP server. The job authenticates with its own SSH key, and the partner's host key has been received out of band — by email from the partner or read from their documentation — and stored in a ",[32,62,63],{},"known_hosts"," file. The local test server here is started with asyncssh; the downloader does not know or care that it is local.",[20,66,68],{"id":67},"step-1-connect-refusing-unknown-servers","Step 1 — Connect, refusing unknown servers",[25,70,74],{"className":71,"code":72,"language":73,"meta":30,"style":30},"language-python shiki shiki-themes github-light github-dark-high-contrast","import paramiko\n\ndef connect(host=\"127.0.0.1\", port=8022, user=\"reports\"):\n    client = paramiko.SSHClient()\n    client.load_host_keys(\"known_hosts\")                       # the partner's published host key\n    client.set_missing_host_key_policy(paramiko.RejectPolicy())\n    client.connect(host, port=port, username=user, key_filename=\"id_ed25519_reports\",\n                   look_for_keys=False, allow_agent=False, timeout=30)\n    return client\n\nclient = connect()\nsftp = client.open_sftp()\nprint(sorted(sftp.listdir(\"\u002Foutgoing\")))\n","python",[32,75,76,86,93,132,143,159,165,199,233,242,247,258,269],{"__ignoreMap":30},[35,77,78,82],{"class":37,"line":38},[35,79,81],{"class":80},"s-kum","import",[35,83,85],{"class":84},"skGVy"," paramiko\n",[35,87,89],{"class":37,"line":88},2,[35,90,92],{"emptyLinePlaceholder":91},true,"\n",[35,94,96,99,103,106,109,112,115,117,121,124,126,129],{"class":37,"line":95},3,[35,97,98],{"class":80},"def",[35,100,102],{"class":101},"s_Opv"," connect",[35,104,105],{"class":84},"(host",[35,107,108],{"class":80},"=",[35,110,111],{"class":45},"\"127.0.0.1\"",[35,113,114],{"class":84},", port",[35,116,108],{"class":80},[35,118,120],{"class":119},"sP0c6","8022",[35,122,123],{"class":84},", user",[35,125,108],{"class":80},[35,127,128],{"class":45},"\"reports\"",[35,130,131],{"class":84},"):\n",[35,133,135,138,140],{"class":37,"line":134},4,[35,136,137],{"class":84},"    client ",[35,139,108],{"class":80},[35,141,142],{"class":84}," paramiko.SSHClient()\n",[35,144,146,149,152,155],{"class":37,"line":145},5,[35,147,148],{"class":84},"    client.load_host_keys(",[35,150,151],{"class":45},"\"known_hosts\"",[35,153,154],{"class":84},")                       ",[35,156,158],{"class":157},"s-wDw","# the partner's published host key\n",[35,160,162],{"class":37,"line":161},6,[35,163,164],{"class":84},"    client.set_missing_host_key_policy(paramiko.RejectPolicy())\n",[35,166,168,171,175,177,180,183,185,188,191,193,196],{"class":37,"line":167},7,[35,169,170],{"class":84},"    client.connect(host, ",[35,172,174],{"class":173},"sa561","port",[35,176,108],{"class":80},[35,178,179],{"class":84},"port, ",[35,181,182],{"class":173},"username",[35,184,108],{"class":80},[35,186,187],{"class":84},"user, ",[35,189,190],{"class":173},"key_filename",[35,192,108],{"class":80},[35,194,195],{"class":45},"\"id_ed25519_reports\"",[35,197,198],{"class":84},",\n",[35,200,202,205,207,210,213,216,218,220,222,225,227,230],{"class":37,"line":201},8,[35,203,204],{"class":173},"                   look_for_keys",[35,206,108],{"class":80},[35,208,209],{"class":119},"False",[35,211,212],{"class":84},", ",[35,214,215],{"class":173},"allow_agent",[35,217,108],{"class":80},[35,219,209],{"class":119},[35,221,212],{"class":84},[35,223,224],{"class":173},"timeout",[35,226,108],{"class":80},[35,228,229],{"class":119},"30",[35,231,232],{"class":84},")\n",[35,234,236,239],{"class":37,"line":235},9,[35,237,238],{"class":80},"    return",[35,240,241],{"class":84}," client\n",[35,243,245],{"class":37,"line":244},10,[35,246,92],{"emptyLinePlaceholder":91},[35,248,250,253,255],{"class":37,"line":249},11,[35,251,252],{"class":84},"client ",[35,254,108],{"class":80},[35,256,257],{"class":84}," connect()\n",[35,259,261,264,266],{"class":37,"line":260},12,[35,262,263],{"class":84},"sftp ",[35,265,108],{"class":80},[35,267,268],{"class":84}," client.open_sftp()\n",[35,270,272,275,278,281,284,287],{"class":37,"line":271},13,[35,273,274],{"class":119},"print",[35,276,277],{"class":84},"(",[35,279,280],{"class":119},"sorted",[35,282,283],{"class":84},"(sftp.listdir(",[35,285,286],{"class":45},"\"\u002Foutgoing\"",[35,288,289],{"class":84},")))\n",[10,291,292,293,295,296,299,300,303],{},"The connection loads the partner's host key from ",[32,294,63],{}," and uses ",[32,297,298],{},"RejectPolicy",", which refuses to connect if the server presents any other key. That is the point of SFTP's security: without the check, the job would hand its credentials to, and accept files from, any machine that answered at that address. The job logs in with a dedicated key, not a person's, and turns off the agent and default key search so it only ever uses that one. A timeout stops a stuck connection from hanging the job. The listing shows what the partner has published: three finished workbooks, one workbook that was written moments ago, a partial upload with a ",[32,301,302],{},".part"," suffix and a text file.",[305,306,314,315,314,319,314,323,314,331,314,341,314,348,314,353,314,357,314,362,314,367,314,372,314,378,314,382,314,386,314,390,314,394,314,397,314,402,314,407,314,410,314,413,314,417,314,420,314,425,314,430,314,433,314,436,314,440,314,443,314,446,314,450,314,453],"svg",{"viewBox":307,"role":308,"ariaLabelledBy":309,"xmlns":312,"style":313},"4 0 752 126","img",[310,311],"sftp-flow-t","sftp-flow-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:752px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[316,317,318],"title",{"id":310},"One run of the downloader",[320,321,322],"desc",{"id":311},"The job connects with a pinned host key, lists the folder with sizes and times, keeps only finished Excel files not fetched before, downloads each to a temporary name, checks it opens, renames it, and records it in the state file.",[324,325],"rect",{"x":326,"y":327,"width":328,"height":329,"fill":330},"4","0","752","126","#ffffff",[324,332],{"x":333,"y":334,"width":335,"height":336,"rx":337,"fill":338,"stroke":339,"style":340},"20.0","16.0","115.2","94.0","12","#f0f4ff","var(--brand,#5b5cf0)","stroke-width:2px",[342,343,347],"text",{"x":344,"y":345,"style":346},"77.6","50.2","font-size:13.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Connect",[342,349,352],{"x":344,"y":350,"style":351},"69.8","font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","host key must",[342,354,356],{"x":344,"y":355,"style":351},"87.2","match",[37,358],{"x1":359,"y1":360,"x2":361,"y2":360,"stroke":339,"style":340},"140.2","63.0","162.2",[363,364],"polygon",{"points":365,"fill":366},"166.2,63.0 157.2,68.0 157.2,58.0","#5b5cf0",[324,368],{"x":369,"y":334,"width":335,"height":336,"rx":337,"fill":370,"stroke":371,"style":340},"171.2","#e7ebef","var(--line,#cdd5e6)",[342,373,377],{"x":374,"y":375,"style":376},"228.8","41.5","font-size:13.5px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","List with",[342,379,381],{"x":374,"y":380,"style":376},"59.0","attributes",[342,383,385],{"x":374,"y":384,"style":351},"78.5","name, size,",[342,387,389],{"x":374,"y":388,"style":351},"96.0","mtime",[37,391],{"x1":392,"y1":360,"x2":393,"y2":360,"stroke":339,"style":340},"291.4","313.4",[363,395],{"points":396,"fill":366},"317.4,63.0 308.4,68.0 308.4,58.0",[324,398],{"x":399,"y":334,"width":335,"height":336,"rx":337,"fill":400,"stroke":401,"style":340},"322.4","#fdefd8","var(--gold,#b4740a)",[342,403,406],{"x":404,"y":345,"style":405},"380.0","font-size:13.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","Filter",[342,408,409],{"x":404,"y":350,"style":351},"xlsx, settled,",[342,411,412],{"x":404,"y":355,"style":351},"new or changed",[37,414],{"x1":415,"y1":360,"x2":416,"y2":360,"stroke":339,"style":340},"442.6","464.6",[363,418],{"points":419,"fill":366},"468.6,63.0 459.6,68.0 459.6,58.0",[324,421],{"x":422,"y":334,"width":335,"height":336,"rx":337,"fill":423,"stroke":424,"style":340},"473.6","#d9f4f1","var(--teal,#0f9488)",[342,426,429],{"x":427,"y":345,"style":428},"531.2","font-size:13.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","Download to",[342,431,302],{"x":427,"y":432,"style":428},"67.8",[342,434,435],{"x":427,"y":355,"style":351},"then rename",[37,437],{"x1":438,"y1":360,"x2":439,"y2":360,"stroke":339,"style":340},"593.8","615.8",[363,441],{"points":442,"fill":366},"619.8,63.0 610.8,68.0 610.8,58.0",[324,444],{"x":445,"y":334,"width":335,"height":336,"rx":337,"fill":338,"stroke":339,"style":340},"624.8",[342,447,449],{"x":448,"y":345,"style":346},"682.4","Check +",[342,451,452],{"x":448,"y":432,"style":346},"record",[342,454,455],{"x":448,"y":355,"style":351},"opens; state file",[20,457,459],{"id":458},"step-2-decide-which-files-to-fetch","Step 2 — Decide which files to fetch",[25,461,463],{"className":71,"code":462,"language":73,"meta":30,"style":30},"import json, stat, time\nfrom pathlib import Path\n\nSTATE = Path(\"sftp_state.json\")\nseen = json.loads(STATE.read_text()) if STATE.exists() else {}\nSETTLE = 120                                                    # seconds without change before a file counts as finished\n\ndef candidates(sftp, folder=\"\u002Foutgoing\"):\n    now = time.time()\n    picked, skipped = [], []\n    for a in sftp.listdir_attr(folder):\n        name, reason = a.filename, None\n        if not stat.S_ISREG(a.st_mode):\n            reason = \"not a file\"\n        elif not name.lower().endswith(\".xlsx\") or name.startswith(\"~$\"):\n            reason = \"not a workbook\"\n        elif now - a.st_mtime \u003C SETTLE:\n            reason = \"still being written\"\n        elif seen.get(name) == [a.st_size, int(a.st_mtime)]:\n            reason = \"already downloaded\"\n        (skipped if reason else picked).append((name, a.st_size, int(a.st_mtime), reason))\n    return picked, skipped\n\npicked, skipped = candidates(sftp)\nprint(\"fetch:\", [p[0] for p in picked])\nprint(\"skip:\", [(s[0], s[3]) for s in skipped])\n",[32,464,465,472,485,489,505,535,548,552,568,578,588,602,615,626,637,665,675,698,708,728,738,759,767,772,783,812],{"__ignoreMap":30},[35,466,467,469],{"class":37,"line":38},[35,468,81],{"class":80},[35,470,471],{"class":84}," json, stat, time\n",[35,473,474,477,480,482],{"class":37,"line":88},[35,475,476],{"class":80},"from",[35,478,479],{"class":84}," pathlib ",[35,481,81],{"class":80},[35,483,484],{"class":84}," Path\n",[35,486,487],{"class":37,"line":95},[35,488,92],{"emptyLinePlaceholder":91},[35,490,491,494,497,500,503],{"class":37,"line":134},[35,492,493],{"class":119},"STATE",[35,495,496],{"class":80}," =",[35,498,499],{"class":84}," Path(",[35,501,502],{"class":45},"\"sftp_state.json\"",[35,504,232],{"class":84},[35,506,507,510,512,515,517,520,523,526,529,532],{"class":37,"line":145},[35,508,509],{"class":84},"seen ",[35,511,108],{"class":80},[35,513,514],{"class":84}," json.loads(",[35,516,493],{"class":119},[35,518,519],{"class":84},".read_text()) ",[35,521,522],{"class":80},"if",[35,524,525],{"class":119}," STATE",[35,527,528],{"class":84},".exists() ",[35,530,531],{"class":80},"else",[35,533,534],{"class":84}," {}\n",[35,536,537,540,542,545],{"class":37,"line":161},[35,538,539],{"class":119},"SETTLE",[35,541,496],{"class":80},[35,543,544],{"class":119}," 120",[35,546,547],{"class":157},"                                                    # seconds without change before a file counts as finished\n",[35,549,550],{"class":37,"line":167},[35,551,92],{"emptyLinePlaceholder":91},[35,553,554,556,559,562,564,566],{"class":37,"line":201},[35,555,98],{"class":80},[35,557,558],{"class":101}," candidates",[35,560,561],{"class":84},"(sftp, folder",[35,563,108],{"class":80},[35,565,286],{"class":45},[35,567,131],{"class":84},[35,569,570,573,575],{"class":37,"line":235},[35,571,572],{"class":84},"    now ",[35,574,108],{"class":80},[35,576,577],{"class":84}," time.time()\n",[35,579,580,583,585],{"class":37,"line":244},[35,581,582],{"class":84},"    picked, skipped ",[35,584,108],{"class":80},[35,586,587],{"class":84}," [], []\n",[35,589,590,593,596,599],{"class":37,"line":249},[35,591,592],{"class":80},"    for",[35,594,595],{"class":84}," a ",[35,597,598],{"class":80},"in",[35,600,601],{"class":84}," sftp.listdir_attr(folder):\n",[35,603,604,607,609,612],{"class":37,"line":260},[35,605,606],{"class":84},"        name, reason ",[35,608,108],{"class":80},[35,610,611],{"class":84}," a.filename, ",[35,613,614],{"class":119},"None\n",[35,616,617,620,623],{"class":37,"line":271},[35,618,619],{"class":80},"        if",[35,621,622],{"class":80}," not",[35,624,625],{"class":84}," stat.S_ISREG(a.st_mode):\n",[35,627,629,632,634],{"class":37,"line":628},14,[35,630,631],{"class":84},"            reason ",[35,633,108],{"class":80},[35,635,636],{"class":45}," \"not a file\"\n",[35,638,640,643,645,648,651,654,657,660,663],{"class":37,"line":639},15,[35,641,642],{"class":80},"        elif",[35,644,622],{"class":80},[35,646,647],{"class":84}," name.lower().endswith(",[35,649,650],{"class":45},"\".xlsx\"",[35,652,653],{"class":84},") ",[35,655,656],{"class":80},"or",[35,658,659],{"class":84}," name.startswith(",[35,661,662],{"class":45},"\"~$\"",[35,664,131],{"class":84},[35,666,668,670,672],{"class":37,"line":667},16,[35,669,631],{"class":84},[35,671,108],{"class":80},[35,673,674],{"class":45}," \"not a workbook\"\n",[35,676,678,680,683,686,689,692,695],{"class":37,"line":677},17,[35,679,642],{"class":80},[35,681,682],{"class":84}," now ",[35,684,685],{"class":80},"-",[35,687,688],{"class":84}," a.st_mtime ",[35,690,691],{"class":80},"\u003C",[35,693,694],{"class":119}," SETTLE",[35,696,697],{"class":84},":\n",[35,699,701,703,705],{"class":37,"line":700},18,[35,702,631],{"class":84},[35,704,108],{"class":80},[35,706,707],{"class":45}," \"still being written\"\n",[35,709,711,713,716,719,722,725],{"class":37,"line":710},19,[35,712,642],{"class":80},[35,714,715],{"class":84}," seen.get(name) ",[35,717,718],{"class":80},"==",[35,720,721],{"class":84}," [a.st_size, ",[35,723,724],{"class":119},"int",[35,726,727],{"class":84},"(a.st_mtime)]:\n",[35,729,731,733,735],{"class":37,"line":730},20,[35,732,631],{"class":84},[35,734,108],{"class":80},[35,736,737],{"class":45}," \"already downloaded\"\n",[35,739,741,744,746,749,751,754,756],{"class":37,"line":740},21,[35,742,743],{"class":84},"        (skipped ",[35,745,522],{"class":80},[35,747,748],{"class":84}," reason ",[35,750,531],{"class":80},[35,752,753],{"class":84}," picked).append((name, a.st_size, ",[35,755,724],{"class":119},[35,757,758],{"class":84},"(a.st_mtime), reason))\n",[35,760,762,764],{"class":37,"line":761},22,[35,763,238],{"class":80},[35,765,766],{"class":84}," picked, skipped\n",[35,768,770],{"class":37,"line":769},23,[35,771,92],{"emptyLinePlaceholder":91},[35,773,775,778,780],{"class":37,"line":774},24,[35,776,777],{"class":84},"picked, skipped ",[35,779,108],{"class":80},[35,781,782],{"class":84}," candidates(sftp)\n",[35,784,786,788,790,793,796,798,801,804,807,809],{"class":37,"line":785},25,[35,787,274],{"class":119},[35,789,277],{"class":84},[35,791,792],{"class":45},"\"fetch:\"",[35,794,795],{"class":84},", [p[",[35,797,327],{"class":119},[35,799,800],{"class":84},"] ",[35,802,803],{"class":80},"for",[35,805,806],{"class":84}," p ",[35,808,598],{"class":80},[35,810,811],{"class":84}," picked])\n",[35,813,815,817,819,822,825,827,830,833,836,838,841,843],{"class":37,"line":814},26,[35,816,274],{"class":119},[35,818,277],{"class":84},[35,820,821],{"class":45},"\"skip:\"",[35,823,824],{"class":84},", [(s[",[35,826,327],{"class":119},[35,828,829],{"class":84},"], s[",[35,831,832],{"class":119},"3",[35,834,835],{"class":84},"]) ",[35,837,803],{"class":80},[35,839,840],{"class":84}," s ",[35,842,598],{"class":80},[35,844,845],{"class":84}," skipped])\n",[10,847,848,851,852,855,856,858,859,862],{},[32,849,850],{},"listdir_attr"," returns each entry with its size, modification time and type in a single request, which is all the information the decision needs. Directories and other non-files are skipped, as is anything that is not an ",[32,853,854],{},".xlsx"," workbook — the partial upload ends in ",[32,857,302],{},", the readme is text. A workbook modified in the last two minutes is treated as ",[57,860,861],{},"still being written"," and left for the next run: there is no reliable way to know from the outside that an upload has finished, but a file whose time has not moved for a couple of minutes almost always has. Finally, the state file records each downloaded file's size and modification time; a file with the same name, size and time as before is skipped, while a file the partner re-uploaded with corrections — same name, new time or size — is fetched again.",[20,864,866],{"id":865},"step-3-download-safely-and-check-each-workbook","Step 3 — Download safely and check each workbook",[25,868,870],{"className":71,"code":869,"language":73,"meta":30,"style":30},"from openpyxl import load_workbook\n\ndest = Path(\"incoming\")\ndest.mkdir(exist_ok=True)\nfor name, size, mtime, _ in picked:\n    tmp = dest \u002F (name + \".part\")\n    sftp.get(f\"\u002Foutgoing\u002F{name}\", str(tmp))\n    if tmp.stat().st_size != size:\n        tmp.unlink()\n        print(f\"{name}: size changed during download, will retry next run\")\n        continue\n    try:\n        with open(tmp, \"rb\") as fh:                             # a file object: openpyxl rejects the .part name\n            load_workbook(fh, read_only=True).close()           # a quick check that it is a real workbook\n    except Exception as e:\n        tmp.rename(dest \u002F (name + \".bad\"))\n        print(f\"{name}: not a valid workbook ({type(e).__name__})\")\n        continue\n    final = dest \u002F name\n    if final.exists():                                          # a corrected re-upload: keep the earlier copy\n        final.rename(dest \u002F f\"{final.stem}.{time.strftime('%Y%m%d%H%M%S', time.localtime(seen[name][1]))}{final.suffix}\")\n    tmp.rename(final)\n    seen[name] = [size, mtime]\n    print(f\"{name}: downloaded ({size:,} bytes)\")\nSTATE.write_text(json.dumps(seen, indent=1))\nclient.close()\n",[32,871,872,884,888,902,917,929,953,985,999,1004,1026,1031,1038,1063,1081,1095,1112,1149,1153,1167,1177,1233,1238,1248,1283,1299],{"__ignoreMap":30},[35,873,874,876,879,881],{"class":37,"line":38},[35,875,476],{"class":80},[35,877,878],{"class":84}," openpyxl ",[35,880,81],{"class":80},[35,882,883],{"class":84}," load_workbook\n",[35,885,886],{"class":37,"line":88},[35,887,92],{"emptyLinePlaceholder":91},[35,889,890,893,895,897,900],{"class":37,"line":95},[35,891,892],{"class":84},"dest ",[35,894,108],{"class":80},[35,896,499],{"class":84},[35,898,899],{"class":45},"\"incoming\"",[35,901,232],{"class":84},[35,903,904,907,910,912,915],{"class":37,"line":134},[35,905,906],{"class":84},"dest.mkdir(",[35,908,909],{"class":173},"exist_ok",[35,911,108],{"class":80},[35,913,914],{"class":119},"True",[35,916,232],{"class":84},[35,918,919,921,924,926],{"class":37,"line":145},[35,920,803],{"class":80},[35,922,923],{"class":84}," name, size, mtime, _ ",[35,925,598],{"class":80},[35,927,928],{"class":84}," picked:\n",[35,930,931,934,936,939,942,945,948,951],{"class":37,"line":161},[35,932,933],{"class":84},"    tmp ",[35,935,108],{"class":80},[35,937,938],{"class":84}," dest ",[35,940,941],{"class":80},"\u002F",[35,943,944],{"class":84}," (name ",[35,946,947],{"class":80},"+",[35,949,950],{"class":45}," \".part\"",[35,952,232],{"class":84},[35,954,955,958,961,964,968,971,974,977,979,982],{"class":37,"line":167},[35,956,957],{"class":84},"    sftp.get(",[35,959,960],{"class":80},"f",[35,962,963],{"class":45},"\"\u002Foutgoing\u002F",[35,965,967],{"class":966},"sSjpA","{",[35,969,970],{"class":84},"name",[35,972,973],{"class":966},"}",[35,975,976],{"class":45},"\"",[35,978,212],{"class":84},[35,980,981],{"class":119},"str",[35,983,984],{"class":84},"(tmp))\n",[35,986,987,990,993,996],{"class":37,"line":201},[35,988,989],{"class":80},"    if",[35,991,992],{"class":84}," tmp.stat().st_size ",[35,994,995],{"class":80},"!=",[35,997,998],{"class":84}," size:\n",[35,1000,1001],{"class":37,"line":235},[35,1002,1003],{"class":84},"        tmp.unlink()\n",[35,1005,1006,1009,1011,1013,1015,1017,1019,1021,1024],{"class":37,"line":244},[35,1007,1008],{"class":119},"        print",[35,1010,277],{"class":84},[35,1012,960],{"class":80},[35,1014,976],{"class":45},[35,1016,967],{"class":966},[35,1018,970],{"class":84},[35,1020,973],{"class":966},[35,1022,1023],{"class":45},": size changed during download, will retry next run\"",[35,1025,232],{"class":84},[35,1027,1028],{"class":37,"line":249},[35,1029,1030],{"class":80},"        continue\n",[35,1032,1033,1036],{"class":37,"line":260},[35,1034,1035],{"class":80},"    try",[35,1037,697],{"class":84},[35,1039,1040,1043,1046,1049,1052,1054,1057,1060],{"class":37,"line":271},[35,1041,1042],{"class":80},"        with",[35,1044,1045],{"class":119}," open",[35,1047,1048],{"class":84},"(tmp, ",[35,1050,1051],{"class":45},"\"rb\"",[35,1053,653],{"class":84},[35,1055,1056],{"class":80},"as",[35,1058,1059],{"class":84}," fh:                             ",[35,1061,1062],{"class":157},"# a file object: openpyxl rejects the .part name\n",[35,1064,1065,1068,1071,1073,1075,1078],{"class":37,"line":628},[35,1066,1067],{"class":84},"            load_workbook(fh, ",[35,1069,1070],{"class":173},"read_only",[35,1072,108],{"class":80},[35,1074,914],{"class":119},[35,1076,1077],{"class":84},").close()           ",[35,1079,1080],{"class":157},"# a quick check that it is a real workbook\n",[35,1082,1083,1086,1089,1092],{"class":37,"line":639},[35,1084,1085],{"class":80},"    except",[35,1087,1088],{"class":119}," Exception",[35,1090,1091],{"class":80}," as",[35,1093,1094],{"class":84}," e:\n",[35,1096,1097,1100,1102,1104,1106,1109],{"class":37,"line":667},[35,1098,1099],{"class":84},"        tmp.rename(dest ",[35,1101,941],{"class":80},[35,1103,944],{"class":84},[35,1105,947],{"class":80},[35,1107,1108],{"class":45}," \".bad\"",[35,1110,1111],{"class":84},"))\n",[35,1113,1114,1116,1118,1120,1122,1124,1126,1128,1131,1133,1136,1139,1142,1144,1147],{"class":37,"line":677},[35,1115,1008],{"class":119},[35,1117,277],{"class":84},[35,1119,960],{"class":80},[35,1121,976],{"class":45},[35,1123,967],{"class":966},[35,1125,970],{"class":84},[35,1127,973],{"class":966},[35,1129,1130],{"class":45},": not a valid workbook (",[35,1132,967],{"class":966},[35,1134,1135],{"class":119},"type",[35,1137,1138],{"class":84},"(e).",[35,1140,1141],{"class":119},"__name__",[35,1143,973],{"class":966},[35,1145,1146],{"class":45},")\"",[35,1148,232],{"class":84},[35,1150,1151],{"class":37,"line":700},[35,1152,1030],{"class":80},[35,1154,1155,1158,1160,1162,1164],{"class":37,"line":710},[35,1156,1157],{"class":84},"    final ",[35,1159,108],{"class":80},[35,1161,938],{"class":84},[35,1163,941],{"class":80},[35,1165,1166],{"class":84}," name\n",[35,1168,1169,1171,1174],{"class":37,"line":730},[35,1170,989],{"class":80},[35,1172,1173],{"class":84}," final.exists():                                          ",[35,1175,1176],{"class":157},"# a corrected re-upload: keep the earlier copy\n",[35,1178,1179,1182,1184,1187,1189,1191,1194,1196,1198,1200,1203,1206,1209,1212,1215,1218,1221,1224,1227,1229,1231],{"class":37,"line":740},[35,1180,1181],{"class":84},"        final.rename(dest ",[35,1183,941],{"class":80},[35,1185,1186],{"class":80}," f",[35,1188,976],{"class":45},[35,1190,967],{"class":966},[35,1192,1193],{"class":84},"final.stem",[35,1195,973],{"class":966},[35,1197,18],{"class":45},[35,1199,967],{"class":966},[35,1201,1202],{"class":84},"time.strftime(",[35,1204,1205],{"class":45},"'%Y%m",[35,1207,1208],{"class":966},"%d",[35,1210,1211],{"class":45},"%H%M%S'",[35,1213,1214],{"class":84},", time.localtime(seen[name][",[35,1216,1217],{"class":119},"1",[35,1219,1220],{"class":84},"]))",[35,1222,1223],{"class":966},"}{",[35,1225,1226],{"class":84},"final.suffix",[35,1228,973],{"class":966},[35,1230,976],{"class":45},[35,1232,232],{"class":84},[35,1234,1235],{"class":37,"line":761},[35,1236,1237],{"class":84},"    tmp.rename(final)\n",[35,1239,1240,1243,1245],{"class":37,"line":769},[35,1241,1242],{"class":84},"    seen[name] ",[35,1244,108],{"class":80},[35,1246,1247],{"class":84}," [size, mtime]\n",[35,1249,1250,1253,1255,1257,1259,1261,1263,1265,1268,1270,1273,1276,1278,1281],{"class":37,"line":774},[35,1251,1252],{"class":119},"    print",[35,1254,277],{"class":84},[35,1256,960],{"class":80},[35,1258,976],{"class":45},[35,1260,967],{"class":966},[35,1262,970],{"class":84},[35,1264,973],{"class":966},[35,1266,1267],{"class":45},": downloaded (",[35,1269,967],{"class":966},[35,1271,1272],{"class":84},"size",[35,1274,1275],{"class":80},":,",[35,1277,973],{"class":966},[35,1279,1280],{"class":45}," bytes)\"",[35,1282,232],{"class":84},[35,1284,1285,1287,1290,1293,1295,1297],{"class":37,"line":785},[35,1286,493],{"class":119},[35,1288,1289],{"class":84},".write_text(json.dumps(seen, ",[35,1291,1292],{"class":173},"indent",[35,1294,108],{"class":80},[35,1296,1217],{"class":119},[35,1298,1111],{"class":84},[35,1300,1301],{"class":37,"line":814},[35,1302,1303],{"class":84},"client.close()\n",[10,1305,1306,1307,1309,1310,1313,1314,1316,1317,1320],{},"Each file is downloaded to a ",[32,1308,302],{}," name and renamed only when it is complete and checked, so anything that watches the ",[57,1311,1312],{},"incoming"," folder never sees half a file. Two checks run before the rename: the downloaded size must match the size in the listing — if the partner changed the file mid-download, the copy is discarded and the next run tries again — and the workbook must open with openpyxl in read-only mode, which takes a moment even for large files. The check opens the file through a file object rather than by path, because openpyxl refuses to open a path whose extension is not one it recognises, and ",[57,1315,302],{}," is not; an earlier version of this code, passing the path, rejected every download as invalid. A file that fails the second check is kept under a ",[32,1318,1319],{},".bad"," name for investigation rather than deleted. When a corrected version of a file already downloaded arrives, the earlier copy is renamed with its timestamp rather than overwritten, so nothing that has already been processed is lost. The state file is written at the end of the run with every file that made it.",[305,1322,314,1328,314,1331,314,1334,314,1339,314,1345,314,1349,314,1355,314,1359,314,1363,314,1367,314,1371,314,1375,314,1378,314,1381,314,1385,314,1387,314,1390,314,1392,314,1395,314,1398,314,1402,314,1404,314,1408,314,1410,314,1413,314,1415,314,1418,314,1420,314,1423,314,1425,314,1428,314,1430,314,1432,314,1434,314,1436,314,1439,314,1442,314,1444,314,1447,314,1449,314,1451,314,1453,314,1455,314,1458,314,1462,314,1464,314,1467,314,1469,314,1473,314,1475,314,1477,314,1480,314,1484,314,1486,314,1489,314,1491,314,1493,314,1495,314,1498,314,1501,314,1505,314,1507,314,1510,314,1512,314,1514,314,1516],{"viewBox":1323,"role":308,"ariaLabelledBy":1324,"xmlns":312,"style":1327},"140 0 480 252",[1325,1326],"sftp-decisions-t","sftp-decisions-d","width:100%;max-width:480px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[316,1329,1330],{"id":1325},"What the first run decides",[320,1332,1333],{"id":1326},"The two stock files and the settlement file are downloaded. The stock file for 3 October is skipped as still being written, the .part file and the readme are skipped as not workbooks.",[324,1335],{"x":1336,"y":327,"width":1337,"height":1338,"fill":330},"140","480","252",[324,1340],{"x":1341,"y":334,"width":1342,"height":1343,"rx":327,"fill":370,"stroke":371,"style":1344},"156.4","32.0","24.0","stroke-width:1px",[324,1346],{"x":1347,"y":334,"width":1348,"height":1343,"rx":327,"fill":370,"stroke":371,"style":1344},"188.4","200.2",[342,1350,1354],{"x":1351,"y":1352,"style":1353},"288.5","33.0","font-size:11px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","A",[324,1356],{"x":1357,"y":334,"width":1358,"height":1343,"rx":327,"fill":370,"stroke":371,"style":1344},"388.6","84.6",[342,1360,1362],{"x":1361,"y":1352,"style":1353},"430.9","B",[324,1364],{"x":1365,"y":334,"width":1366,"height":1343,"rx":327,"fill":370,"stroke":371,"style":1344},"473.2","130.4",[342,1368,1370],{"x":1369,"y":1352,"style":1353},"538.4","C",[324,1372],{"x":1341,"y":1373,"width":1342,"height":1374,"rx":327,"fill":370,"stroke":371,"style":1344},"40.0","28.0",[342,1376,1217],{"x":1377,"y":380,"style":1353},"172.4",[324,1379],{"x":1347,"y":1373,"width":1348,"height":1374,"rx":327,"fill":338,"stroke":339,"style":1380},"stroke-width:1.5px",[342,1382,1384],{"x":1351,"y":380,"style":1383},"font-size:12px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","File",[324,1386],{"x":1357,"y":1373,"width":1358,"height":1374,"rx":327,"fill":338,"stroke":339,"style":1380},[342,1388,1389],{"x":1361,"y":380,"style":1383},"Decision",[324,1391],{"x":1365,"y":1373,"width":1366,"height":1374,"rx":327,"fill":338,"stroke":339,"style":1380},[342,1393,1394],{"x":1369,"y":380,"style":1383},"Reason",[324,1396],{"x":1341,"y":1397,"width":1342,"height":1374,"rx":327,"fill":370,"stroke":371,"style":1344},"68.0",[342,1399,1401],{"x":1377,"y":1400,"style":1353},"87.0","2",[324,1403],{"x":1347,"y":1397,"width":1348,"height":1374,"rx":327,"fill":330,"stroke":371,"style":1344},[342,1405,1407],{"x":1351,"y":1400,"style":1406},"font-size:12px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","stock_2026-10-01.xlsx",[324,1409],{"x":1357,"y":1397,"width":1358,"height":1374,"rx":327,"fill":330,"stroke":371,"style":1344},[342,1411,1412],{"x":1361,"y":1400,"style":1406},"download",[324,1414],{"x":1365,"y":1397,"width":1366,"height":1374,"rx":327,"fill":330,"stroke":371,"style":1344},[342,1416,1417],{"x":1369,"y":1400,"style":1406},"new",[324,1419],{"x":1341,"y":388,"width":1342,"height":1374,"rx":327,"fill":370,"stroke":371,"style":1344},[342,1421,832],{"x":1377,"y":1422,"style":1353},"115.0",[324,1424],{"x":1347,"y":388,"width":1348,"height":1374,"rx":327,"fill":330,"stroke":371,"style":1344},[342,1426,1427],{"x":1351,"y":1422,"style":1406},"stock_2026-10-02.xlsx",[324,1429],{"x":1357,"y":388,"width":1358,"height":1374,"rx":327,"fill":330,"stroke":371,"style":1344},[342,1431,1412],{"x":1361,"y":1422,"style":1406},[324,1433],{"x":1365,"y":388,"width":1366,"height":1374,"rx":327,"fill":330,"stroke":371,"style":1344},[342,1435,1417],{"x":1369,"y":1422,"style":1406},[324,1437],{"x":1341,"y":1438,"width":1342,"height":1374,"rx":327,"fill":370,"stroke":371,"style":1344},"124.0",[342,1440,326],{"x":1377,"y":1441,"style":1353},"143.0",[324,1443],{"x":1347,"y":1438,"width":1348,"height":1374,"rx":327,"fill":330,"stroke":371,"style":1344},[342,1445,1446],{"x":1351,"y":1441,"style":1406},"settlement_2026-10-02.xlsx",[324,1448],{"x":1357,"y":1438,"width":1358,"height":1374,"rx":327,"fill":330,"stroke":371,"style":1344},[342,1450,1412],{"x":1361,"y":1441,"style":1406},[324,1452],{"x":1365,"y":1438,"width":1366,"height":1374,"rx":327,"fill":330,"stroke":371,"style":1344},[342,1454,1417],{"x":1369,"y":1441,"style":1406},[324,1456],{"x":1341,"y":1457,"width":1342,"height":1374,"rx":327,"fill":370,"stroke":371,"style":1344},"152.0",[342,1459,1461],{"x":1377,"y":1460,"style":1353},"171.0","5",[324,1463],{"x":1347,"y":1457,"width":1348,"height":1374,"rx":327,"fill":330,"stroke":371,"style":1344},[342,1465,1466],{"x":1351,"y":1460,"style":1406},"stock_2026-10-03.xlsx",[324,1468],{"x":1357,"y":1457,"width":1358,"height":1374,"rx":327,"fill":400,"stroke":401,"style":1380},[342,1470,1472],{"x":1361,"y":1460,"style":1471},"font-size:12px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","skip",[324,1474],{"x":1365,"y":1457,"width":1366,"height":1374,"rx":327,"fill":330,"stroke":371,"style":1344},[342,1476,861],{"x":1369,"y":1460,"style":1406},[324,1478],{"x":1341,"y":1479,"width":1342,"height":1374,"rx":327,"fill":370,"stroke":371,"style":1344},"180.0",[342,1481,1483],{"x":1377,"y":1482,"style":1353},"199.0","6",[324,1485],{"x":1347,"y":1479,"width":1348,"height":1374,"rx":327,"fill":330,"stroke":371,"style":1344},[342,1487,1488],{"x":1351,"y":1482,"style":1406},"stock_2026-10-03.xlsx.part",[324,1490],{"x":1357,"y":1479,"width":1358,"height":1374,"rx":327,"fill":330,"stroke":371,"style":1344},[342,1492,1472],{"x":1361,"y":1482,"style":1406},[324,1494],{"x":1365,"y":1479,"width":1366,"height":1374,"rx":327,"fill":330,"stroke":371,"style":1344},[342,1496,1497],{"x":1369,"y":1482,"style":1406},"not a workbook",[324,1499],{"x":1341,"y":1500,"width":1342,"height":1374,"rx":327,"fill":370,"stroke":371,"style":1344},"208.0",[342,1502,1504],{"x":1377,"y":1503,"style":1353},"227.0","7",[324,1506],{"x":1347,"y":1500,"width":1348,"height":1374,"rx":327,"fill":330,"stroke":371,"style":1344},[342,1508,1509],{"x":1351,"y":1503,"style":1406},"readme.txt",[324,1511],{"x":1357,"y":1500,"width":1358,"height":1374,"rx":327,"fill":330,"stroke":371,"style":1344},[342,1513,1472],{"x":1361,"y":1503,"style":1406},[324,1515],{"x":1365,"y":1500,"width":1366,"height":1374,"rx":327,"fill":330,"stroke":371,"style":1344},[342,1517,1497],{"x":1369,"y":1503,"style":1406},[305,1519,314,1524,314,1527,314,1530,314,1533,314,1538,314,1544,314,1549,314,1553,314,1557,314,1561,314,1564,314,1569,314,1573,314,1576,314,1579],{"viewBox":1520,"role":308,"ariaLabelledBy":1521,"xmlns":312,"style":313},"4 0 752 173",[1522,1523],"sftp-state-t","sftp-state-d",[316,1525,1526],{"id":1522},"Remembering what was fetched",[320,1528,1529],{"id":1523},"A local state file of name, size and modification time works with read-only access and also notices corrected re-uploads. Moving fetched files to a processed folder on the server makes the server the record but needs write access, which partners often do not give.",[324,1531],{"x":326,"y":327,"width":328,"height":1532,"fill":330},"173",[324,1534],{"x":333,"y":334,"width":1535,"height":1536,"rx":1537,"fill":423,"stroke":424,"style":340},"351.0","141.0","14",[342,1539,1543],{"x":1540,"y":1541,"style":1542},"195.5","44.0","font-size:14px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","Local state file",[37,1545],{"x1":1546,"y1":1547,"x2":1548,"y2":1547,"stroke":424,"style":1344},"36.0","57.0","355.0",[342,1550,1552],{"x":1540,"y":1551,"style":1406},"77.0","works with read-only access",[342,1554,1556],{"x":1540,"y":1555,"style":1406},"103.0","notices corrected re-uploads",[342,1558,1560],{"x":1540,"y":1559,"style":1406},"129.0","must be kept with the job",[324,1562],{"x":1563,"y":334,"width":1535,"height":1536,"rx":1537,"fill":338,"stroke":339,"style":340},"389.0",[342,1565,1568],{"x":1566,"y":1541,"style":1567},"564.5","font-size:14px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Move to processed\u002F on the server",[37,1570],{"x1":1571,"y1":1547,"x2":1572,"y2":1547,"stroke":339,"style":1344},"405.0","724.0",[342,1574,1575],{"x":1566,"y":1551,"style":1406},"the server is the record",[342,1577,1578],{"x":1566,"y":1555,"style":1406},"needs write access",[342,1580,1581],{"x":1566,"y":1559,"style":1406},"partner sees what was taken",[20,1583,1585],{"id":1584},"step-4-run-it-again","Step 4 — Run it again",[25,1587,1589],{"className":71,"code":1588,"language":73,"meta":30,"style":30},"client = connect()\nsftp = client.open_sftp()\npicked, skipped = candidates(sftp)\nprint(\"second run fetches:\", [p[0] for p in picked])\nprint(\"already downloaded:\", [s[0] for s in skipped if s[3] == \"already downloaded\"])\nclient.close()\n",[32,1590,1591,1599,1607,1615,1638,1680],{"__ignoreMap":30},[35,1592,1593,1595,1597],{"class":37,"line":38},[35,1594,252],{"class":84},[35,1596,108],{"class":80},[35,1598,257],{"class":84},[35,1600,1601,1603,1605],{"class":37,"line":88},[35,1602,263],{"class":84},[35,1604,108],{"class":80},[35,1606,268],{"class":84},[35,1608,1609,1611,1613],{"class":37,"line":95},[35,1610,777],{"class":84},[35,1612,108],{"class":80},[35,1614,782],{"class":84},[35,1616,1617,1619,1621,1624,1626,1628,1630,1632,1634,1636],{"class":37,"line":134},[35,1618,274],{"class":119},[35,1620,277],{"class":84},[35,1622,1623],{"class":45},"\"second run fetches:\"",[35,1625,795],{"class":84},[35,1627,327],{"class":119},[35,1629,800],{"class":84},[35,1631,803],{"class":80},[35,1633,806],{"class":84},[35,1635,598],{"class":80},[35,1637,811],{"class":84},[35,1639,1640,1642,1644,1647,1650,1652,1654,1656,1658,1660,1663,1665,1668,1670,1672,1674,1677],{"class":37,"line":145},[35,1641,274],{"class":119},[35,1643,277],{"class":84},[35,1645,1646],{"class":45},"\"already downloaded:\"",[35,1648,1649],{"class":84},", [s[",[35,1651,327],{"class":119},[35,1653,800],{"class":84},[35,1655,803],{"class":80},[35,1657,840],{"class":84},[35,1659,598],{"class":80},[35,1661,1662],{"class":84}," skipped ",[35,1664,522],{"class":80},[35,1666,1667],{"class":84}," s[",[35,1669,832],{"class":119},[35,1671,800],{"class":84},[35,1673,718],{"class":80},[35,1675,1676],{"class":45}," \"already downloaded\"",[35,1678,1679],{"class":84},"])\n",[35,1681,1682],{"class":37,"line":161},[35,1683,1303],{"class":84},[10,1685,1686,1687,1690,1691,1694],{},"A second run straight afterwards fetches nothing new: the three files from the first run are recognised from the state file, and the newest stock file is still within its settling time. Once it has been untouched for two minutes, the next scheduled run picks it up. That is the behaviour a scheduled job needs — safe to run every fifteen minutes, never fetching twice, never fetching early. If the partner allows it, an alternative to the state file is to move each fetched file into a ",[57,1688,1689],{},"processed"," folder on the server with ",[32,1692,1693],{},"sftp.rename",", which makes the server itself the record; many partners give read-only access, so the state file is the approach that always works.",[20,1696,1698],{"id":1697},"common-pitfalls","Common pitfalls",[1700,1701,1702,1714,1720,1726,1735],"ul",{},[1703,1704,1705,1711,1712,18],"li",{},[1706,1707,1708,18],"strong",{},[32,1709,1710],{},"AutoAddPolicy"," It trusts any server; pin the partner's host key and use ",[32,1713,298],{},[1703,1715,1716,1719],{},[1706,1717,1718],{},"Fetching files still being uploaded."," Wait until a file's time has not changed for a while.",[1703,1721,1722,1725],{},[1706,1723,1724],{},"Deduplicating by name only."," Corrected re-uploads keep the name; compare size and time too.",[1703,1727,1728,1731,1732,1734],{},[1706,1729,1730],{},"Writing straight to the final name."," Download to ",[32,1733,302],{}," and rename when complete.",[1703,1736,1737,1740],{},[1706,1738,1739],{},"Overwriting earlier versions."," Keep the previous copy when a corrected file arrives.",[20,1742,1744],{"id":1743},"conclusion","Conclusion",[10,1746,1747,1748,1750,1751,1753,1754,1756],{},"To pick up new Excel files from an SFTP server, connect with paramiko using a pinned host key and ",[32,1749,298],{},", list the folder with ",[32,1752,850],{},", and fetch only ",[32,1755,854],{}," files that have settled and whose name, size and time are not in the state file. Download each to a temporary name, check its size and that it opens, then rename it, keeping any earlier version, and record it in the state file so the job can run as often as needed without fetching anything twice.",[20,1758,1760],{"id":1759},"frequently-asked-questions","Frequently asked questions",[10,1762,1763,1766],{},[1706,1764,1765],{},"How do I download only new files from SFTP in Python?","\nKeep a record of each downloaded file's name, size and modification time, and fetch only files whose entry is missing or different.",[10,1768,1769,1772],{},[1706,1770,1771],{},"How do I avoid downloading a file that is still being uploaded?","\nSkip files whose modification time is within the last few minutes, and check the downloaded size against the listing.",[10,1774,1775,1778,1779,1781],{},[1706,1776,1777],{},"Should I use AutoAddPolicy with paramiko?","\nNo; load the partner's host key from a known_hosts file and use ",[32,1780,298],{}," so the job refuses unknown servers.",[10,1783,1784,1787],{},[1706,1785,1786],{},"How do I test SFTP code without a real server?","\nRun a local SFTP server, for example with asyncssh, with its own host key and an authorised client key.",[20,1789,1791],{"id":1790},"related","Related",[1700,1793,1794,1800,1807,1814],{},[1703,1795,1796,1797,1799],{},"Up one level: ",[14,1798,17],{"href":16}," — Excel and data sources.",[1703,1801,1802,1806],{},[14,1803,1805],{"href":1804},"\u002Fautomating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage\u002Fupload-an-excel-report-over-sftp\u002F","Upload an Excel Report over SFTP with Python"," — the sending side.",[1703,1808,1809,1813],{},[14,1810,1812],{"href":1811},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fread-excel-attachments-into-a-database-nightly\u002F","Read Excel Attachments into a Database Nightly"," — loading what arrives.",[1703,1815,1816,1820],{},[14,1817,1819],{"href":1818},"\u002Fautomating-reporting-workflows\u002Fscheduling-python-excel-scripts-with-cron\u002Fwait-for-upstream-data-before-running-a-report\u002F","Wait for Upstream Data Before Running a Report"," — when the file is late.",[1822,1823,1824],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}",{"title":30,"searchDepth":88,"depth":88,"links":1826},[1827,1828,1829,1830,1831,1832,1833,1834,1835],{"id":22,"depth":88,"text":23},{"id":67,"depth":88,"text":68},{"id":458,"depth":88,"text":459},{"id":865,"depth":88,"text":866},{"id":1584,"depth":88,"text":1585},{"id":1697,"depth":88,"text":1698},{"id":1743,"depth":88,"text":1744},{"id":1759,"depth":88,"text":1760},{"id":1790,"depth":88,"text":1791},"2026-10-02","Collect the spreadsheets a partner drops on an SFTP server with paramiko: connect with a pinned host key, list the folder with sizes and modification times, ignore partial uploads and files still being written, download only files not fetched before — or fetched before but changed since — into a temporary name and rename, check each workbook opens, keep a small state file, and test it all against a local SFTP server.","md",[1840,1842,1844,1846],{"q":1765,"a":1841},"Keep a record of each downloaded file's name, size and modification time, and fetch only files whose entry is missing or different.",{"q":1771,"a":1843},"Skip files whose modification time is within the last few minutes, and check the downloaded size against the listing.",{"q":1777,"a":1845},"No; load the partner's host key from a known_hosts file and use RejectPolicy so the job refuses unknown servers.",{"q":1786,"a":1847},"Run a local SFTP server, for example with asyncssh, with its own host key and an authorised client key.","download new files sftp python",null,{"breadcrumb":1851},[1852,1854,1857],{"name":1853,"item":941},"Home",{"name":1855,"item":1856},"Advanced Data Transformation and Cleaning for Excel Automation","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":17,"item":16},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fdownload-new-excel-files-from-an-sftp-server",{"title":1860,"description":1861},"Download New Excel Files from an SFTP Server with Python","Fetch new Excel files over SFTP with paramiko: pinned host keys, listdir_attr, skipping partial and in-progress uploads, a state file for already downloaded files, atomic local saves and a local test server.","download-new-excel-files-from-an-sftp-server","advanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fdownload-new-excel-files-from-an-sftp-server\u002Findex",[1865,1866],"data-sources","scheduling","how-to","uGNkzzGIZtRagWd3fjNdcNLlNaz7CWLdp_KYQ47H0FU",[1870,1874],{"title":1871,"path":1872,"stem":1873,"children":-1},"Compare an Excel File with a Database Table","\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fcompare-an-excel-file-with-a-database-table","advanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fcompare-an-excel-file-with-a-database-table\u002Findex",{"title":1875,"path":1876,"stem":1877,"children":-1},"Export a Large Database Table to Excel in Chunks","\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fexport-a-large-database-table-to-excel-in-chunks","advanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fexport-a-large-database-table-to-excel-in-chunks\u002Findex",[1879,1881],{"path":1880,"title":1855},"\u002Fadvanced-data-transformation-and-cleaning",{"path":1882,"title":17},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases",[],[1885,1888,1891,1892,1893,1896],{"path":1886,"title":1887,"kind":1867},"\u002Fautomating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage\u002Fread-an-excel-file-directly-from-amazon-s3-with-pandas","Read an Excel File Directly from Amazon S3 with pandas",{"path":1889,"title":1890,"kind":1867},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fcache-api-responses-when-building-excel-reports","Cache API Responses When Building Excel Reports",{"path":1872,"title":1871,"kind":1867},{"path":1876,"title":1875,"kind":1867},{"path":1894,"title":1895,"kind":1867},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fexport-database-lookup-tables-as-excel-dropdown-lists","Export Database Lookup Tables as Excel Dropdown Lists",{"path":1897,"title":1898,"kind":1867},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fexport-sql-query-results-to-excel-with-python","Export SQL Query Results to Excel with Python",1790972096607]