[{"data":1,"prerenderedAt":2288},["ShallowReactive",2],{"doc:\u002Fautomating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage\u002Fread-an-excel-file-directly-from-amazon-s3-with-pandas":3,"surround:\u002Fautomating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage\u002Fread-an-excel-file-directly-from-amazon-s3-with-pandas":2280},{"id":4,"title":5,"body":6,"dateModified":2253,"datePublished":2253,"description":2254,"extension":2255,"faq":2256,"meta":2265,"navigation":214,"path":2273,"seo":2274,"slug":2276,"stem":2277,"type":2278,"__hash__":2279},"docs\u002Fautomating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage\u002Fread-an-excel-file-directly-from-amazon-s3-with-pandas\u002Findex.md","Read an Excel File Directly from Amazon S3 with pandas",{"type":7,"value":8,"toc":2238},"minimark",[9,19,121,126,163,175,179,186,280,283,287,651,658,662,763,1050,1057,1061,1248,1255,1259,1266,1373,1382,1386,1393,1559,1566,1573,1632,1638,1642,1647,1754,1761,1765,1872,1876,1979,1985,1988,2137,2144,2148,2167,2171,2178,2184,2190,2196,2200,2234],[10,11,12,13,18],"p",{},"Reports increasingly source their input from object storage rather than a shared drive, and the\nnatural instinct — download the file, then open it — adds a disk write and a cleanup problem to what\ncan be a single call. This guide, part of\n",[14,15,17],"a",{"href":16},"\u002Fautomating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage\u002F","Publishing Excel Reports to Cloud Storage",",\ncovers reading a workbook from S3 straight into a DataFrame, choosing between in-memory and\non-disk, and the failure modes worth handling explicitly.",[20,21,29,30,29,34,29,38,29,45,29,52,29,62,29,68,29,73,29,80,29,85,29,89,29,93,29,96,29,100,29,103,29,108,29,113,29,116],"svg",{"viewBox":22,"role":23,"ariaLabelledBy":24,"xmlns":27,"style":28},"0 0 760 232","img",[25,26],"s3-read-t","s3-read-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:760px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[31,32,33],"title",{"id":25},"From object to DataFrame without touching disk",[35,36,37],"desc",{"id":26},"A get_object call returns a streaming body, its bytes wrap in a BytesIO buffer, and the parser reads that buffer exactly as it would read a file.",[39,40],"rect",{"x":41,"y":41,"width":42,"height":43,"fill":44},"0","760","232","#ffffff",[46,47,51],"text",{"x":48,"y":49,"style":50},"380.0","32","font-size:13px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","in-memory read",[39,53],{"x":54,"y":55,"width":56,"height":57,"rx":58,"fill":59,"stroke":60,"style":61},"24.0","74","208.0","96","12","#e7ebef","var(--line,#cdd5e6)","stroke-width:2px",[46,63,67],{"x":64,"y":65,"style":66},"128.0","114","font-size:14px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:middle","get_object",[46,69,72],{"x":64,"y":70,"style":71},"136","font-size:11.5px;font-weight:400;fill:var(--muted,#5b6780);text-anchor:middle","streaming body",[74,75],"line",{"x1":76,"y1":77,"x2":78,"y2":77,"stroke":79,"style":61},"237.0","122.0","269.0","var(--brand,#5b5cf0)",[81,82],"polygon",{"points":83,"fill":84},"269.0,122.0 260.0,117.0 260.0,127.0","#5b5cf0",[39,86],{"x":87,"y":55,"width":56,"height":57,"rx":58,"fill":88,"stroke":79,"style":61},"276.0","#f0f4ff",[46,90,92],{"x":48,"y":65,"style":91},"font-size:14px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","BytesIO",[46,94,95],{"x":48,"y":70,"style":71},"bytes in memory",[74,97],{"x1":98,"y1":77,"x2":99,"y2":77,"stroke":79,"style":61},"489.0","521.0",[81,101],{"points":102,"fill":84},"521.0,122.0 512.0,117.0 512.0,127.0",[39,104],{"x":105,"y":55,"width":56,"height":57,"rx":58,"fill":106,"stroke":107,"style":61},"528.0","#d9f4f1","var(--teal,#0f9488)",[46,109,112],{"x":110,"y":65,"style":111},"632.0","font-size:14px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","read_excel",[46,114,115],{"x":110,"y":70,"style":71},"a typed frame",[46,117,120],{"x":48,"y":118,"style":119},"210","font-size:12.5px;font-weight:400;fill:var(--muted,#5b6780);text-anchor:middle","no temporary file to write, name, or clean up",[122,123,125],"h2",{"id":124},"prerequisites","Prerequisites",[127,128,133],"pre",{"className":129,"code":130,"language":131,"meta":132,"style":132},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas boto3 s3fs openpyxl python-calamine\n","bash","",[134,135,136],"code",{"__ignoreMap":132},[137,138,140,144,148,151,154,157,160],"span",{"class":74,"line":139},1,[137,141,143],{"class":142},"sMTad","pip",[137,145,147],{"class":146},"srMev"," install",[137,149,150],{"class":146}," pandas",[137,152,153],{"class":146}," boto3",[137,155,156],{"class":146}," s3fs",[137,158,159],{"class":146}," openpyxl",[137,161,162],{"class":146}," python-calamine\n",[10,164,165,166,169,170,174],{},"Credentials from the standard chain — an instance role, a task role, or ",[134,167,168],{},"~\u002F.aws\u002Fcredentials",". Do not\nput keys in the script; the reasoning is the same as in\n",[14,171,173],{"href":172},"\u002Fautomating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage\u002Fupload-an-excel-report-to-amazon-s3-with-boto3\u002F","Upload an Excel Report to Amazon S3 with boto3",".",[122,176,178],{"id":177},"the-one-line-version","The one-line version",[10,180,181,182,185],{},"With ",[134,183,184],{},"s3fs"," installed, pandas accepts an S3 URI wherever it accepts a path.",[127,187,191],{"className":188,"code":189,"language":190,"meta":132,"style":132},"language-python shiki shiki-themes github-light github-dark-high-contrast","import pandas as pd\n\nframe = pd.read_excel(\n    \"s3:\u002F\u002Freporting-artifacts\u002Fexports\u002Forders-2026-09-01.xlsx\",\n    sheet_name=\"Detail\",\n    engine=\"calamine\",\n)\nprint(frame.shape)\n","python",[134,192,193,209,216,228,237,251,264,270],{"__ignoreMap":132},[137,194,195,199,203,206],{"class":74,"line":139},[137,196,198],{"class":197},"s-kum","import",[137,200,202],{"class":201},"skGVy"," pandas ",[137,204,205],{"class":197},"as",[137,207,208],{"class":201}," pd\n",[137,210,212],{"class":74,"line":211},2,[137,213,215],{"emptyLinePlaceholder":214},true,"\n",[137,217,219,222,225],{"class":74,"line":218},3,[137,220,221],{"class":201},"frame ",[137,223,224],{"class":197},"=",[137,226,227],{"class":201}," pd.read_excel(\n",[137,229,231,234],{"class":74,"line":230},4,[137,232,233],{"class":146},"    \"s3:\u002F\u002Freporting-artifacts\u002Fexports\u002Forders-2026-09-01.xlsx\"",[137,235,236],{"class":201},",\n",[137,238,240,244,246,249],{"class":74,"line":239},5,[137,241,243],{"class":242},"sa561","    sheet_name",[137,245,224],{"class":197},[137,247,248],{"class":146},"\"Detail\"",[137,250,236],{"class":201},[137,252,254,257,259,262],{"class":74,"line":253},6,[137,255,256],{"class":242},"    engine",[137,258,224],{"class":197},[137,260,261],{"class":146},"\"calamine\"",[137,263,236],{"class":201},[137,265,267],{"class":74,"line":266},7,[137,268,269],{"class":201},")\n",[137,271,273,277],{"class":74,"line":272},8,[137,274,276],{"class":275},"sP0c6","print",[137,278,279],{"class":201},"(frame.shape)\n",[10,281,282],{},"That is genuinely all it takes, and it is the right thing to reach for in a notebook. In a scheduled\njob it has two drawbacks worth knowing: the credential and permission errors surface as fsspec\nexceptions that are harder to read than boto3's, and you have no hook to add retries or a timeout.",[122,284,286],{"id":285},"the-explicit-version","The explicit version",[127,288,290],{"className":188,"code":289,"language":190,"meta":132,"style":132},"import io\nimport boto3\nimport pandas as pd\nfrom botocore.exceptions import ClientError\n\ns3 = boto3.client(\"s3\")\n\ndef read_workbook(bucket: str, key: str, **kwargs) -> pd.DataFrame:\n    try:\n        response = s3.get_object(Bucket=bucket, Key=key)\n    except ClientError as error:\n        code = error.response[\"Error\"][\"Code\"]\n        if code == \"NoSuchKey\":\n            raise FileNotFoundError(f\"s3:\u002F\u002F{bucket}\u002F{key} does not exist\") from error\n        if code in (\"AccessDenied\", \"403\"):\n            raise PermissionError(f\"no permission to read s3:\u002F\u002F{bucket}\u002F{key}\") from error\n        raise\n    return pd.read_excel(io.BytesIO(response[\"Body\"].read()), engine=\"calamine\", **kwargs)\n\nframe = read_workbook(\"reporting-artifacts\", \"exports\u002Forders-2026-09-01.xlsx\", sheet_name=\"Detail\")\n",[134,291,292,299,306,316,329,333,348,352,381,390,417,431,454,471,520,544,582,588,617,622],{"__ignoreMap":132},[137,293,294,296],{"class":74,"line":139},[137,295,198],{"class":197},[137,297,298],{"class":201}," io\n",[137,300,301,303],{"class":74,"line":211},[137,302,198],{"class":197},[137,304,305],{"class":201}," boto3\n",[137,307,308,310,312,314],{"class":74,"line":218},[137,309,198],{"class":197},[137,311,202],{"class":201},[137,313,205],{"class":197},[137,315,208],{"class":201},[137,317,318,321,324,326],{"class":74,"line":230},[137,319,320],{"class":197},"from",[137,322,323],{"class":201}," botocore.exceptions ",[137,325,198],{"class":197},[137,327,328],{"class":201}," ClientError\n",[137,330,331],{"class":74,"line":239},[137,332,215],{"emptyLinePlaceholder":214},[137,334,335,338,340,343,346],{"class":74,"line":253},[137,336,337],{"class":201},"s3 ",[137,339,224],{"class":197},[137,341,342],{"class":201}," boto3.client(",[137,344,345],{"class":146},"\"s3\"",[137,347,269],{"class":201},[137,349,350],{"class":74,"line":266},[137,351,215],{"emptyLinePlaceholder":214},[137,353,354,357,361,364,367,370,372,375,378],{"class":74,"line":272},[137,355,356],{"class":197},"def",[137,358,360],{"class":359},"s_Opv"," read_workbook",[137,362,363],{"class":201},"(bucket: ",[137,365,366],{"class":275},"str",[137,368,369],{"class":201},", key: ",[137,371,366],{"class":275},[137,373,374],{"class":201},", ",[137,376,377],{"class":197},"**",[137,379,380],{"class":201},"kwargs) -> pd.DataFrame:\n",[137,382,384,387],{"class":74,"line":383},9,[137,385,386],{"class":197},"    try",[137,388,389],{"class":201},":\n",[137,391,393,396,398,401,404,406,409,412,414],{"class":74,"line":392},10,[137,394,395],{"class":201},"        response ",[137,397,224],{"class":197},[137,399,400],{"class":201}," s3.get_object(",[137,402,403],{"class":242},"Bucket",[137,405,224],{"class":197},[137,407,408],{"class":201},"bucket, ",[137,410,411],{"class":242},"Key",[137,413,224],{"class":197},[137,415,416],{"class":201},"key)\n",[137,418,420,423,426,428],{"class":74,"line":419},11,[137,421,422],{"class":197},"    except",[137,424,425],{"class":201}," ClientError ",[137,427,205],{"class":197},[137,429,430],{"class":201}," error:\n",[137,432,434,437,439,442,445,448,451],{"class":74,"line":433},12,[137,435,436],{"class":201},"        code ",[137,438,224],{"class":197},[137,440,441],{"class":201}," error.response[",[137,443,444],{"class":146},"\"Error\"",[137,446,447],{"class":201},"][",[137,449,450],{"class":146},"\"Code\"",[137,452,453],{"class":201},"]\n",[137,455,457,460,463,466,469],{"class":74,"line":456},13,[137,458,459],{"class":197},"        if",[137,461,462],{"class":201}," code ",[137,464,465],{"class":197},"==",[137,467,468],{"class":146}," \"NoSuchKey\"",[137,470,389],{"class":201},[137,472,474,477,480,483,486,489,493,496,499,502,504,507,509,512,515,517],{"class":74,"line":473},14,[137,475,476],{"class":197},"            raise",[137,478,479],{"class":275}," FileNotFoundError",[137,481,482],{"class":201},"(",[137,484,485],{"class":197},"f",[137,487,488],{"class":146},"\"s3:\u002F\u002F",[137,490,492],{"class":491},"sSjpA","{",[137,494,495],{"class":201},"bucket",[137,497,498],{"class":491},"}",[137,500,501],{"class":146},"\u002F",[137,503,492],{"class":491},[137,505,506],{"class":201},"key",[137,508,498],{"class":491},[137,510,511],{"class":146}," does not exist\"",[137,513,514],{"class":201},") ",[137,516,320],{"class":197},[137,518,519],{"class":201}," error\n",[137,521,523,525,527,530,533,536,538,541],{"class":74,"line":522},15,[137,524,459],{"class":197},[137,526,462],{"class":201},[137,528,529],{"class":197},"in",[137,531,532],{"class":201}," (",[137,534,535],{"class":146},"\"AccessDenied\"",[137,537,374],{"class":201},[137,539,540],{"class":146},"\"403\"",[137,542,543],{"class":201},"):\n",[137,545,547,549,552,554,556,559,561,563,565,567,569,571,573,576,578,580],{"class":74,"line":546},16,[137,548,476],{"class":197},[137,550,551],{"class":275}," PermissionError",[137,553,482],{"class":201},[137,555,485],{"class":197},[137,557,558],{"class":146},"\"no permission to read s3:\u002F\u002F",[137,560,492],{"class":491},[137,562,495],{"class":201},[137,564,498],{"class":491},[137,566,501],{"class":146},[137,568,492],{"class":491},[137,570,506],{"class":201},[137,572,498],{"class":491},[137,574,575],{"class":146},"\"",[137,577,514],{"class":201},[137,579,320],{"class":197},[137,581,519],{"class":201},[137,583,585],{"class":74,"line":584},17,[137,586,587],{"class":197},"        raise\n",[137,589,591,594,597,600,603,606,608,610,612,614],{"class":74,"line":590},18,[137,592,593],{"class":197},"    return",[137,595,596],{"class":201}," pd.read_excel(io.BytesIO(response[",[137,598,599],{"class":146},"\"Body\"",[137,601,602],{"class":201},"].read()), ",[137,604,605],{"class":242},"engine",[137,607,224],{"class":197},[137,609,261],{"class":146},[137,611,374],{"class":201},[137,613,377],{"class":197},[137,615,616],{"class":201},"kwargs)\n",[137,618,620],{"class":74,"line":619},19,[137,621,215],{"emptyLinePlaceholder":214},[137,623,625,627,629,632,635,637,640,642,645,647,649],{"class":74,"line":624},20,[137,626,221],{"class":201},[137,628,224],{"class":197},[137,630,631],{"class":201}," read_workbook(",[137,633,634],{"class":146},"\"reporting-artifacts\"",[137,636,374],{"class":201},[137,638,639],{"class":146},"\"exports\u002Forders-2026-09-01.xlsx\"",[137,641,374],{"class":201},[137,643,644],{"class":242},"sheet_name",[137,646,224],{"class":197},[137,648,248],{"class":146},[137,650,269],{"class":201},[10,652,653,654,657],{},"Translating the two error codes people actually hit into Python's own exception types is the whole\npoint of the extra ten lines: a missing file and a missing permission produce very different\nremedies, and ",[134,655,656],{},"ClientError: An error occurred (403)"," does not distinguish them for the person reading\nthe log at seven in the morning.",[122,659,661],{"id":660},"reading-every-sheet-and-reading-several-files","Reading every sheet, and reading several files",[20,663,29,668,29,671,29,674,29,677,29,684,29,689,29,694,29,700,29,705,29,708,29,711,29,715,29,719,29,723,29,726,29,729,29,733,29,737,29,741,29,744,29,747,29,751,29,755,29,759],{"viewBox":664,"role":23,"ariaLabelledBy":665,"xmlns":27,"style":28},"0 0 760 308",[666,667],"s3-list-t","s3-list-d",[31,669,670],{"id":666},"Reading a whole prefix without missing files",[35,672,673],{"id":667},"List with a paginator because a single call returns at most a thousand keys, filter by extension, tag each frame with its source key, and raise when nothing matched.",[39,675],{"x":41,"y":41,"width":42,"height":676,"fill":44},"308",[39,678],{"x":679,"y":680,"width":681,"height":682,"rx":683,"fill":106,"stroke":107},"22","28","716","52","10",[685,686],"circle",{"cx":682,"cy":687,"r":688,"fill":84},"54.0","15",[46,690,693],{"x":682,"y":691,"style":692},"59.0","font-size:13px;font-weight:700;fill:#ffffff;text-anchor:middle","1",[46,695,699],{"x":696,"y":697,"style":698},"82","50","font-size:13px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:start","Use a paginator",[46,701,704],{"x":696,"y":702,"style":703},"68","font-size:11.5px;font-weight:400;fill:var(--muted,#5b6780);text-anchor:start","list_objects_v2 truncates at 1,000 keys",[39,706],{"x":679,"y":707,"width":681,"height":682,"rx":683,"fill":106,"stroke":107},"90",[685,709],{"cx":682,"cy":710,"r":688,"fill":84},"116.0",[46,712,714],{"x":682,"y":713,"style":692},"121.0","2",[46,716,718],{"x":696,"y":717,"style":698},"112","Filter by extension",[46,720,722],{"x":696,"y":721,"style":703},"130","prefixes collect more than workbooks",[39,724],{"x":679,"y":725,"width":681,"height":682,"rx":683,"fill":106,"stroke":107},"152",[685,727],{"cx":682,"cy":728,"r":688,"fill":84},"178.0",[46,730,732],{"x":682,"y":731,"style":692},"183.0","3",[46,734,736],{"x":696,"y":735,"style":698},"174","Tag each frame",[46,738,740],{"x":696,"y":739,"style":703},"192","assign the key so rows keep their origin",[39,742],{"x":679,"y":743,"width":681,"height":682,"rx":683,"fill":106,"stroke":107},"214",[685,745],{"cx":682,"cy":746,"r":688,"fill":84},"240.0",[46,748,750],{"x":682,"y":749,"style":692},"245.0","4",[46,752,754],{"x":696,"y":753,"style":698},"236","Raise on empty",[46,756,758],{"x":696,"y":757,"style":703},"254","an empty frame becomes an empty report",[46,760,762],{"x":48,"y":761,"style":119},"296","the truncation is silent, which is what makes it expensive",[127,764,766],{"className":188,"code":765,"language":190,"meta":132,"style":132},"tabs = read_workbook(\"reporting-artifacts\", \"exports\u002Forders.xlsx\", sheet_name=None)\nprint({name: frame.shape for name, frame in tabs.items()})\n\n# Every workbook under a prefix, concatenated with a source column\ndef read_prefix(bucket: str, prefix: str) -> pd.DataFrame:\n    paginator = s3.get_paginator(\"list_objects_v2\")\n    frames = []\n    for page in paginator.paginate(Bucket=bucket, Prefix=prefix):\n        for item in page.get(\"Contents\", []):\n            if item[\"Key\"].endswith((\".xlsx\", \".xlsm\")):\n                part = read_workbook(bucket, item[\"Key\"])\n                frames.append(part.assign(Source=item[\"Key\"].rsplit(\"\u002F\", 1)[-1]))\n    if not frames:\n        raise FileNotFoundError(f\"no workbooks under s3:\u002F\u002F{bucket}\u002F{prefix}\")\n    return pd.concat(frames, ignore_index=True)\n",[134,767,768,795,813,817,823,842,857,867,894,913,938,953,989,1000,1033],{"__ignoreMap":132},[137,769,770,773,775,777,779,781,784,786,788,790,793],{"class":74,"line":139},[137,771,772],{"class":201},"tabs ",[137,774,224],{"class":197},[137,776,631],{"class":201},[137,778,634],{"class":146},[137,780,374],{"class":201},[137,782,783],{"class":146},"\"exports\u002Forders.xlsx\"",[137,785,374],{"class":201},[137,787,644],{"class":242},[137,789,224],{"class":197},[137,791,792],{"class":275},"None",[137,794,269],{"class":201},[137,796,797,799,802,805,808,810],{"class":74,"line":211},[137,798,276],{"class":275},[137,800,801],{"class":201},"({name: frame.shape ",[137,803,804],{"class":197},"for",[137,806,807],{"class":201}," name, frame ",[137,809,529],{"class":197},[137,811,812],{"class":201}," tabs.items()})\n",[137,814,815],{"class":74,"line":218},[137,816,215],{"emptyLinePlaceholder":214},[137,818,819],{"class":74,"line":230},[137,820,822],{"class":821},"s-wDw","# Every workbook under a prefix, concatenated with a source column\n",[137,824,825,827,830,832,834,837,839],{"class":74,"line":239},[137,826,356],{"class":197},[137,828,829],{"class":359}," read_prefix",[137,831,363],{"class":201},[137,833,366],{"class":275},[137,835,836],{"class":201},", prefix: ",[137,838,366],{"class":275},[137,840,841],{"class":201},") -> pd.DataFrame:\n",[137,843,844,847,849,852,855],{"class":74,"line":253},[137,845,846],{"class":201},"    paginator ",[137,848,224],{"class":197},[137,850,851],{"class":201}," s3.get_paginator(",[137,853,854],{"class":146},"\"list_objects_v2\"",[137,856,269],{"class":201},[137,858,859,862,864],{"class":74,"line":266},[137,860,861],{"class":201},"    frames ",[137,863,224],{"class":197},[137,865,866],{"class":201}," []\n",[137,868,869,872,875,877,880,882,884,886,889,891],{"class":74,"line":272},[137,870,871],{"class":197},"    for",[137,873,874],{"class":201}," page ",[137,876,529],{"class":197},[137,878,879],{"class":201}," paginator.paginate(",[137,881,403],{"class":242},[137,883,224],{"class":197},[137,885,408],{"class":201},[137,887,888],{"class":242},"Prefix",[137,890,224],{"class":197},[137,892,893],{"class":201},"prefix):\n",[137,895,896,899,902,904,907,910],{"class":74,"line":383},[137,897,898],{"class":197},"        for",[137,900,901],{"class":201}," item ",[137,903,529],{"class":197},[137,905,906],{"class":201}," page.get(",[137,908,909],{"class":146},"\"Contents\"",[137,911,912],{"class":201},", []):\n",[137,914,915,918,921,924,927,930,932,935],{"class":74,"line":392},[137,916,917],{"class":197},"            if",[137,919,920],{"class":201}," item[",[137,922,923],{"class":146},"\"Key\"",[137,925,926],{"class":201},"].endswith((",[137,928,929],{"class":146},"\".xlsx\"",[137,931,374],{"class":201},[137,933,934],{"class":146},"\".xlsm\"",[137,936,937],{"class":201},")):\n",[137,939,940,943,945,948,950],{"class":74,"line":419},[137,941,942],{"class":201},"                part ",[137,944,224],{"class":197},[137,946,947],{"class":201}," read_workbook(bucket, item[",[137,949,923],{"class":146},[137,951,952],{"class":201},"])\n",[137,954,955,958,961,963,966,968,971,974,976,978,981,984,986],{"class":74,"line":433},[137,956,957],{"class":201},"                frames.append(part.assign(",[137,959,960],{"class":242},"Source",[137,962,224],{"class":197},[137,964,965],{"class":201},"item[",[137,967,923],{"class":146},[137,969,970],{"class":201},"].rsplit(",[137,972,973],{"class":146},"\"\u002F\"",[137,975,374],{"class":201},[137,977,693],{"class":275},[137,979,980],{"class":201},")[",[137,982,983],{"class":197},"-",[137,985,693],{"class":275},[137,987,988],{"class":201},"]))\n",[137,990,991,994,997],{"class":74,"line":456},[137,992,993],{"class":197},"    if",[137,995,996],{"class":197}," not",[137,998,999],{"class":201}," frames:\n",[137,1001,1002,1005,1007,1009,1011,1014,1016,1018,1020,1022,1024,1027,1029,1031],{"class":74,"line":473},[137,1003,1004],{"class":197},"        raise",[137,1006,479],{"class":275},[137,1008,482],{"class":201},[137,1010,485],{"class":197},[137,1012,1013],{"class":146},"\"no workbooks under s3:\u002F\u002F",[137,1015,492],{"class":491},[137,1017,495],{"class":201},[137,1019,498],{"class":491},[137,1021,501],{"class":146},[137,1023,492],{"class":491},[137,1025,1026],{"class":201},"prefix",[137,1028,498],{"class":491},[137,1030,575],{"class":146},[137,1032,269],{"class":201},[137,1034,1035,1037,1040,1043,1045,1048],{"class":74,"line":522},[137,1036,593],{"class":197},[137,1038,1039],{"class":201}," pd.concat(frames, ",[137,1041,1042],{"class":242},"ignore_index",[137,1044,224],{"class":197},[137,1046,1047],{"class":275},"True",[137,1049,269],{"class":201},[10,1051,1052,1053,1056],{},"The paginator is not optional detail: ",[134,1054,1055],{},"list_objects_v2"," returns at most a thousand keys per call, and\ncode that ignores the continuation token silently processes the first thousand. Raising when nothing\nmatched is the other half — an empty concat produces an empty frame, and an empty frame flows\ndownstream as a report with no rows rather than as an error.",[122,1058,1060],{"id":1059},"finding-the-newest-file","Finding the newest file",[127,1062,1064],{"className":188,"code":1063,"language":190,"meta":132,"style":132},"def newest_key(bucket: str, prefix: str, suffix: str = \".xlsx\") -> str:\n    paginator = s3.get_paginator(\"list_objects_v2\")\n    candidates = [\n        item for page in paginator.paginate(Bucket=bucket, Prefix=prefix)\n        for item in page.get(\"Contents\", [])\n        if item[\"Key\"].endswith(suffix)\n    ]\n    if not candidates:\n        raise FileNotFoundError(f\"nothing matching {suffix} under {prefix}\")\n    return max(candidates, key=lambda item: item[\"LastModified\"])[\"Key\"]\n",[134,1065,1066,1099,1111,1121,1147,1162,1173,1178,1187,1220],{"__ignoreMap":132},[137,1067,1068,1070,1073,1075,1077,1079,1081,1084,1086,1089,1092,1095,1097],{"class":74,"line":139},[137,1069,356],{"class":197},[137,1071,1072],{"class":359}," newest_key",[137,1074,363],{"class":201},[137,1076,366],{"class":275},[137,1078,836],{"class":201},[137,1080,366],{"class":275},[137,1082,1083],{"class":201},", suffix: ",[137,1085,366],{"class":275},[137,1087,1088],{"class":197}," =",[137,1090,1091],{"class":146}," \".xlsx\"",[137,1093,1094],{"class":201},") -> ",[137,1096,366],{"class":275},[137,1098,389],{"class":201},[137,1100,1101,1103,1105,1107,1109],{"class":74,"line":211},[137,1102,846],{"class":201},[137,1104,224],{"class":197},[137,1106,851],{"class":201},[137,1108,854],{"class":146},[137,1110,269],{"class":201},[137,1112,1113,1116,1118],{"class":74,"line":218},[137,1114,1115],{"class":201},"    candidates ",[137,1117,224],{"class":197},[137,1119,1120],{"class":201}," [\n",[137,1122,1123,1126,1128,1130,1132,1134,1136,1138,1140,1142,1144],{"class":74,"line":230},[137,1124,1125],{"class":201},"        item ",[137,1127,804],{"class":197},[137,1129,874],{"class":201},[137,1131,529],{"class":197},[137,1133,879],{"class":201},[137,1135,403],{"class":242},[137,1137,224],{"class":197},[137,1139,408],{"class":201},[137,1141,888],{"class":242},[137,1143,224],{"class":197},[137,1145,1146],{"class":201},"prefix)\n",[137,1148,1149,1151,1153,1155,1157,1159],{"class":74,"line":239},[137,1150,898],{"class":197},[137,1152,901],{"class":201},[137,1154,529],{"class":197},[137,1156,906],{"class":201},[137,1158,909],{"class":146},[137,1160,1161],{"class":201},", [])\n",[137,1163,1164,1166,1168,1170],{"class":74,"line":253},[137,1165,459],{"class":197},[137,1167,920],{"class":201},[137,1169,923],{"class":146},[137,1171,1172],{"class":201},"].endswith(suffix)\n",[137,1174,1175],{"class":74,"line":266},[137,1176,1177],{"class":201},"    ]\n",[137,1179,1180,1182,1184],{"class":74,"line":272},[137,1181,993],{"class":197},[137,1183,996],{"class":197},[137,1185,1186],{"class":201}," candidates:\n",[137,1188,1189,1191,1193,1195,1197,1200,1202,1205,1207,1210,1212,1214,1216,1218],{"class":74,"line":383},[137,1190,1004],{"class":197},[137,1192,479],{"class":275},[137,1194,482],{"class":201},[137,1196,485],{"class":197},[137,1198,1199],{"class":146},"\"nothing matching ",[137,1201,492],{"class":491},[137,1203,1204],{"class":201},"suffix",[137,1206,498],{"class":491},[137,1208,1209],{"class":146}," under ",[137,1211,492],{"class":491},[137,1213,1026],{"class":201},[137,1215,498],{"class":491},[137,1217,575],{"class":146},[137,1219,269],{"class":201},[137,1221,1222,1224,1227,1230,1232,1235,1238,1241,1244,1246],{"class":74,"line":392},[137,1223,593],{"class":197},[137,1225,1226],{"class":275}," max",[137,1228,1229],{"class":201},"(candidates, ",[137,1231,506],{"class":242},[137,1233,1234],{"class":197},"=lambda",[137,1236,1237],{"class":201}," item: item[",[137,1239,1240],{"class":146},"\"LastModified\"",[137,1242,1243],{"class":201},"])[",[137,1245,923],{"class":146},[137,1247,453],{"class":201},[10,1249,1250,1251,1254],{},"Scanning for the newest is convenient and does not scale: on a prefix with tens of thousands of\nobjects it is several API calls and a growing list. Where the pipeline controls the naming, putting\nthe date in the key and constructing it — ",[134,1252,1253],{},"exports\u002Forders-{date:%Y-%m-%d}.xlsx"," — is both faster and\nmore honest, because the job then fails when the expected file is missing rather than silently\nprocessing an older one.",[122,1256,1258],{"id":1257},"streaming-a-large-workbook-to-disk","Streaming a large workbook to disk",[10,1260,1261,1262,1265],{},"An ",[134,1263,1264],{},".xlsx"," file expands substantially when parsed, so for a large source it can be better to spool it\nto disk and let the parser stream from there rather than holding the compressed bytes and the parsed\nframe simultaneously.",[127,1267,1269],{"className":188,"code":1268,"language":190,"meta":132,"style":132},"import tempfile\nfrom pathlib import Path\n\ndef read_large(bucket: str, key: str, **kwargs) -> pd.DataFrame:\n    with tempfile.TemporaryDirectory() as directory:\n        target = Path(directory) \u002F Path(key).name\n        s3.download_file(bucket, key, str(target))\n        return pd.read_excel(target, engine=\"calamine\", **kwargs)\n",[134,1270,1271,1278,1290,1294,1315,1328,1343,1353],{"__ignoreMap":132},[137,1272,1273,1275],{"class":74,"line":139},[137,1274,198],{"class":197},[137,1276,1277],{"class":201}," tempfile\n",[137,1279,1280,1282,1285,1287],{"class":74,"line":211},[137,1281,320],{"class":197},[137,1283,1284],{"class":201}," pathlib ",[137,1286,198],{"class":197},[137,1288,1289],{"class":201}," Path\n",[137,1291,1292],{"class":74,"line":218},[137,1293,215],{"emptyLinePlaceholder":214},[137,1295,1296,1298,1301,1303,1305,1307,1309,1311,1313],{"class":74,"line":230},[137,1297,356],{"class":197},[137,1299,1300],{"class":359}," read_large",[137,1302,363],{"class":201},[137,1304,366],{"class":275},[137,1306,369],{"class":201},[137,1308,366],{"class":275},[137,1310,374],{"class":201},[137,1312,377],{"class":197},[137,1314,380],{"class":201},[137,1316,1317,1320,1323,1325],{"class":74,"line":239},[137,1318,1319],{"class":197},"    with",[137,1321,1322],{"class":201}," tempfile.TemporaryDirectory() ",[137,1324,205],{"class":197},[137,1326,1327],{"class":201}," directory:\n",[137,1329,1330,1333,1335,1338,1340],{"class":74,"line":253},[137,1331,1332],{"class":201},"        target ",[137,1334,224],{"class":197},[137,1336,1337],{"class":201}," Path(directory) ",[137,1339,501],{"class":197},[137,1341,1342],{"class":201}," Path(key).name\n",[137,1344,1345,1348,1350],{"class":74,"line":266},[137,1346,1347],{"class":201},"        s3.download_file(bucket, key, ",[137,1349,366],{"class":275},[137,1351,1352],{"class":201},"(target))\n",[137,1354,1355,1358,1361,1363,1365,1367,1369,1371],{"class":74,"line":272},[137,1356,1357],{"class":197},"        return",[137,1359,1360],{"class":201}," pd.read_excel(target, ",[137,1362,605],{"class":242},[137,1364,224],{"class":197},[137,1366,261],{"class":146},[137,1368,374],{"class":201},[137,1370,377],{"class":197},[137,1372,616],{"class":201},[10,1374,1375,1378,1379,1381],{},[134,1376,1377],{},"download_file"," uses a managed multipart transfer with retries built in, which for a large object is\nmore robust than a single ",[134,1380,67],{},". The temporary directory removes itself, which is what makes\nthis safe inside a container with limited disk.",[122,1383,1385],{"id":1384},"confirming-you-read-the-whole-file","Confirming you read the whole file",[10,1387,1388,1389,1392],{},"A truncated download produces a ",[134,1390,1391],{},"BadZipFile"," if you are lucky and a partially-read workbook if you\nare not. Comparing the bytes you received against the object's declared length turns that into a\nclear failure, and it costs one comparison.",[127,1394,1396],{"className":188,"code":1395,"language":190,"meta":132,"style":132},"def read_verified(bucket: str, key: str, **kwargs) -> pd.DataFrame:\n    response = s3.get_object(Bucket=bucket, Key=key)\n    declared = response[\"ContentLength\"]\n    payload = response[\"Body\"].read()\n    if len(payload) != declared:\n        raise IOError(f\"s3:\u002F\u002F{bucket}\u002F{key}: read {len(payload)} of {declared} bytes\")\n    return pd.read_excel(io.BytesIO(payload), engine=\"calamine\", **kwargs)\n",[134,1397,1398,1419,1440,1455,1469,1485,1540],{"__ignoreMap":132},[137,1399,1400,1402,1405,1407,1409,1411,1413,1415,1417],{"class":74,"line":139},[137,1401,356],{"class":197},[137,1403,1404],{"class":359}," read_verified",[137,1406,363],{"class":201},[137,1408,366],{"class":275},[137,1410,369],{"class":201},[137,1412,366],{"class":275},[137,1414,374],{"class":201},[137,1416,377],{"class":197},[137,1418,380],{"class":201},[137,1420,1421,1424,1426,1428,1430,1432,1434,1436,1438],{"class":74,"line":211},[137,1422,1423],{"class":201},"    response ",[137,1425,224],{"class":197},[137,1427,400],{"class":201},[137,1429,403],{"class":242},[137,1431,224],{"class":197},[137,1433,408],{"class":201},[137,1435,411],{"class":242},[137,1437,224],{"class":197},[137,1439,416],{"class":201},[137,1441,1442,1445,1447,1450,1453],{"class":74,"line":218},[137,1443,1444],{"class":201},"    declared ",[137,1446,224],{"class":197},[137,1448,1449],{"class":201}," response[",[137,1451,1452],{"class":146},"\"ContentLength\"",[137,1454,453],{"class":201},[137,1456,1457,1460,1462,1464,1466],{"class":74,"line":230},[137,1458,1459],{"class":201},"    payload ",[137,1461,224],{"class":197},[137,1463,1449],{"class":201},[137,1465,599],{"class":146},[137,1467,1468],{"class":201},"].read()\n",[137,1470,1471,1473,1476,1479,1482],{"class":74,"line":239},[137,1472,993],{"class":197},[137,1474,1475],{"class":275}," len",[137,1477,1478],{"class":201},"(payload) ",[137,1480,1481],{"class":197},"!=",[137,1483,1484],{"class":201}," declared:\n",[137,1486,1487,1489,1492,1494,1496,1498,1500,1502,1504,1506,1508,1510,1512,1515,1517,1520,1523,1525,1528,1530,1533,1535,1538],{"class":74,"line":253},[137,1488,1004],{"class":197},[137,1490,1491],{"class":275}," IOError",[137,1493,482],{"class":201},[137,1495,485],{"class":197},[137,1497,488],{"class":146},[137,1499,492],{"class":491},[137,1501,495],{"class":201},[137,1503,498],{"class":491},[137,1505,501],{"class":146},[137,1507,492],{"class":491},[137,1509,506],{"class":201},[137,1511,498],{"class":491},[137,1513,1514],{"class":146},": read ",[137,1516,492],{"class":491},[137,1518,1519],{"class":275},"len",[137,1521,1522],{"class":201},"(payload)",[137,1524,498],{"class":491},[137,1526,1527],{"class":146}," of ",[137,1529,492],{"class":491},[137,1531,1532],{"class":201},"declared",[137,1534,498],{"class":491},[137,1536,1537],{"class":146}," bytes\"",[137,1539,269],{"class":201},[137,1541,1542,1544,1547,1549,1551,1553,1555,1557],{"class":74,"line":266},[137,1543,593],{"class":197},[137,1545,1546],{"class":201}," pd.read_excel(io.BytesIO(payload), ",[137,1548,605],{"class":242},[137,1550,224],{"class":197},[137,1552,261],{"class":146},[137,1554,374],{"class":201},[137,1556,377],{"class":197},[137,1558,616],{"class":201},[10,1560,1561,1562,1565],{},"The same response carries an ",[134,1563,1564],{},"ETag",", which for a single-part upload is the object's MD5. Where the\nproducer records a checksum alongside the file, comparing it here catches corruption that a length\ncheck cannot — a rare failure, but a cheap one to rule out on data that will drive a report.",[10,1567,1568,1569,1572],{},"Reading ",[134,1570,1571],{},"ContentLength"," before deciding how to read is also how the in-memory and spool-to-disk paths\nchoose between themselves:",[127,1574,1576],{"className":188,"code":1575,"language":190,"meta":132,"style":132},"head = s3.head_object(Bucket=bucket, Key=key)\nframe = (read_large if head[\"ContentLength\"] > 50_000_000 else read_workbook)(bucket, key)\n",[134,1577,1578,1600],{"__ignoreMap":132},[137,1579,1580,1583,1585,1588,1590,1592,1594,1596,1598],{"class":74,"line":139},[137,1581,1582],{"class":201},"head ",[137,1584,224],{"class":197},[137,1586,1587],{"class":201}," s3.head_object(",[137,1589,403],{"class":242},[137,1591,224],{"class":197},[137,1593,408],{"class":201},[137,1595,411],{"class":242},[137,1597,224],{"class":197},[137,1599,416],{"class":201},[137,1601,1602,1604,1606,1609,1612,1615,1617,1620,1623,1626,1629],{"class":74,"line":211},[137,1603,221],{"class":201},[137,1605,224],{"class":197},[137,1607,1608],{"class":201}," (read_large ",[137,1610,1611],{"class":197},"if",[137,1613,1614],{"class":201}," head[",[137,1616,1452],{"class":146},[137,1618,1619],{"class":201},"] ",[137,1621,1622],{"class":197},">",[137,1624,1625],{"class":275}," 50_000_000",[137,1627,1628],{"class":197}," else",[137,1630,1631],{"class":201}," read_workbook)(bucket, key)\n",[10,1633,1634,1637],{},[134,1635,1636],{},"head_object"," is a cheap metadata call, so branching on it costs almost nothing and removes the guess\nabout which path a given file needs.",[122,1639,1641],{"id":1640},"reading-from-other-providers","Reading from other providers",[10,1643,1644,1645,174],{},"The pattern generalises, and the only part that changes is the client. Azure Blob Storage and Google\nCloud Storage both expose a bytes download that feeds the same ",[134,1646,92],{},[127,1648,1650],{"className":188,"code":1649,"language":190,"meta":132,"style":132},"# Azure\nfrom azure.storage.blob import BlobServiceClient\nblob = BlobServiceClient.from_connection_string(conn).get_blob_client(\"reports\", key)\nframe = pd.read_excel(io.BytesIO(blob.download_blob().readall()), engine=\"calamine\")\n\n# Google Cloud Storage\nfrom google.cloud import storage\nbucket = storage.Client().bucket(\"reporting-artifacts\")\nframe = pd.read_excel(io.BytesIO(bucket.blob(key).download_as_bytes()), engine=\"calamine\")\n",[134,1651,1652,1657,1669,1685,1702,1706,1711,1723,1737],{"__ignoreMap":132},[137,1653,1654],{"class":74,"line":139},[137,1655,1656],{"class":821},"# Azure\n",[137,1658,1659,1661,1664,1666],{"class":74,"line":211},[137,1660,320],{"class":197},[137,1662,1663],{"class":201}," azure.storage.blob ",[137,1665,198],{"class":197},[137,1667,1668],{"class":201}," BlobServiceClient\n",[137,1670,1671,1674,1676,1679,1682],{"class":74,"line":218},[137,1672,1673],{"class":201},"blob ",[137,1675,224],{"class":197},[137,1677,1678],{"class":201}," BlobServiceClient.from_connection_string(conn).get_blob_client(",[137,1680,1681],{"class":146},"\"reports\"",[137,1683,1684],{"class":201},", key)\n",[137,1686,1687,1689,1691,1694,1696,1698,1700],{"class":74,"line":230},[137,1688,221],{"class":201},[137,1690,224],{"class":197},[137,1692,1693],{"class":201}," pd.read_excel(io.BytesIO(blob.download_blob().readall()), ",[137,1695,605],{"class":242},[137,1697,224],{"class":197},[137,1699,261],{"class":146},[137,1701,269],{"class":201},[137,1703,1704],{"class":74,"line":239},[137,1705,215],{"emptyLinePlaceholder":214},[137,1707,1708],{"class":74,"line":253},[137,1709,1710],{"class":821},"# Google Cloud Storage\n",[137,1712,1713,1715,1718,1720],{"class":74,"line":266},[137,1714,320],{"class":197},[137,1716,1717],{"class":201}," google.cloud ",[137,1719,198],{"class":197},[137,1721,1722],{"class":201}," storage\n",[137,1724,1725,1728,1730,1733,1735],{"class":74,"line":272},[137,1726,1727],{"class":201},"bucket ",[137,1729,224],{"class":197},[137,1731,1732],{"class":201}," storage.Client().bucket(",[137,1734,634],{"class":146},[137,1736,269],{"class":201},[137,1738,1739,1741,1743,1746,1748,1750,1752],{"class":74,"line":383},[137,1740,221],{"class":201},[137,1742,224],{"class":197},[137,1744,1745],{"class":201}," pd.read_excel(io.BytesIO(bucket.blob(key).download_as_bytes()), ",[137,1747,605],{"class":242},[137,1749,224],{"class":197},[137,1751,261],{"class":146},[137,1753,269],{"class":201},[10,1755,1756,1757,174],{},"Keeping the parse behind one function that takes bytes — rather than one per provider — means a\npipeline that moves between clouds changes its fetch and nothing else. That is also what makes the\nread testable, since a test can hand the function bytes from a fixture without touching a network.\nThe write-side equivalents are in\n",[14,1758,1760],{"href":1759},"\u002Fautomating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage\u002Fsave-excel-reports-to-google-drive-with-python\u002F","Save Excel Reports to Google Drive with Python",[122,1762,1764],{"id":1763},"common-pitfalls","Common pitfalls",[1766,1767,1768,1784],"table",{},[1769,1770,1771],"thead",{},[1772,1773,1774,1778,1781],"tr",{},[1775,1776,1777],"th",{},"Symptom",[1775,1779,1780],{},"Cause",[1775,1782,1783],{},"Fix",[1785,1786,1787,1804,1822,1834,1850,1861],"tbody",{},[1772,1788,1789,1796,1799],{},[1790,1791,1792,1795],"td",{},[134,1793,1794],{},"NoSuchKey"," on a key that exists",[1790,1797,1798],{},"Wrong bucket, or a leading slash in the key",[1790,1800,1801,1802],{},"S3 keys never start with ",[134,1803,501],{},[1772,1805,1806,1812,1819],{},[1790,1807,1808,1811],{},[134,1809,1810],{},"403"," with valid credentials",[1790,1813,1814,1815,1818],{},"The role lacks ",[134,1816,1817],{},"s3:GetObject"," for that prefix",[1790,1820,1821],{},"Check the policy, not the credentials",[1772,1823,1824,1827,1832],{},[1790,1825,1826],{},"Only 1,000 files processed",[1790,1828,1829,1831],{},[134,1830,1055],{}," truncation ignored",[1790,1833,699],{},[1772,1835,1836,1841,1844],{},[1790,1837,1838,1840],{},[134,1839,1391],{}," on read",[1790,1842,1843],{},"The object is not a workbook, or the download was truncated",[1790,1845,1846,1847,1849],{},"Check ",[134,1848,1571],{}," against the bytes read",[1772,1851,1852,1855,1858],{},[1790,1853,1854],{},"Reads are slow and repeated",[1790,1856,1857],{},"The same file parsed on every run",[1790,1859,1860],{},"Convert to Parquet once and read that",[1772,1862,1863,1866,1869],{},[1790,1864,1865],{},"Memory spike on a large file",[1790,1867,1868],{},"Bytes and parsed frame held together",[1790,1870,1871],{},"Spool to a temporary file first",[122,1873,1875],{"id":1874},"performance-and-scale","Performance and scale",[20,1877,29,1883,29,1886,29,1889,29,1893,29,1899,29,1906,29,1913,29,1919,29,1923,29,1926,29,1932,29,1937,29,1941,29,1944,29,1948,29,1953,29,1957,29,1960,29,1965,29,1970,29,1974],{"viewBox":1878,"role":23,"ariaLabelledBy":1879,"xmlns":27,"style":1882},"0 0 720 240",[1880,1881],"s3-cost-t","s3-cost-d","width:100%;max-width:720px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[31,1884,1885],{"id":1880},"Where the time goes on an S3 workbook read",[35,1887,1888],{"id":1881},"Transferring a compressed workbook is quick, parsing it with openpyxl is the dominant cost, calamine cuts that substantially, and reading a cached Parquet copy makes it negligible.",[39,1890],{"x":41,"y":41,"width":1891,"height":1892,"fill":44},"720","240",[46,1894,1898],{"x":1895,"y":1896,"style":1897},"20","56","font-size:12px;font-weight:600;fill:var(--text,#172033);text-anchor:start","transfer",[39,1900],{"x":1901,"y":1902,"width":1903,"height":1904,"rx":1905,"fill":59,"stroke":60},"200","40","371.6","26","6",[39,1907],{"x":1908,"y":1909,"width":1910,"height":1911,"rx":1912,"fill":59,"stroke":60},"201","41","40.9","24","5",[46,1914,1918],{"x":1915,"y":1916,"style":1917},"583.6","58","font-size:12px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:start","compressed bytes",[46,1920,1922],{"x":1895,"y":1921,"style":1897},"100","parse with openpyxl",[39,1924],{"x":1901,"y":1925,"width":1903,"height":1904,"rx":1905,"fill":59,"stroke":60},"84",[39,1927],{"x":1908,"y":1928,"width":1929,"height":1911,"rx":1912,"fill":1930,"stroke":1931},"85","369.6","#fee8f2","var(--accent,#d81b73)",[46,1933,1936],{"x":1915,"y":1934,"style":1935},"102","font-size:12px;font-weight:700;fill:var(--accent,#d81b73);text-anchor:start","dominates",[46,1938,1940],{"x":1895,"y":1939,"style":1897},"144","parse with calamine",[39,1942],{"x":1901,"y":1943,"width":1903,"height":1904,"rx":1905,"fill":59,"stroke":60},"128",[39,1945],{"x":1908,"y":1946,"width":1947,"height":1911,"rx":1912,"fill":106,"stroke":107},"129","93.2",[46,1949,1952],{"x":1915,"y":1950,"style":1951},"146","font-size:12px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:start","same bytes",[46,1954,1956],{"x":1895,"y":1955,"style":1897},"188","read Parquet copy",[39,1958],{"x":1901,"y":1959,"width":1903,"height":1904,"rx":1905,"fill":59,"stroke":60},"172",[39,1961],{"x":1908,"y":1962,"width":680,"height":1911,"rx":1912,"fill":1963,"stroke":1964},"173","#fdefd8","var(--gold,#b4740a)",[46,1966,1969],{"x":1915,"y":1967,"style":1968},"190","font-size:12px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:start","converted once",[46,1971,1973],{"x":1895,"y":1895,"style":1972},"font-size:11.5px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:start","relative cost",[46,1975,1978],{"x":1976,"y":1977,"style":119},"360.0","230","the network is rarely the bottleneck; the parser is",[10,1980,1981,1982,1984],{},"Two costs make up an S3 read: the transfer and the parse. The transfer is usually the smaller of the\ntwo — ",[134,1983,1264],{}," is compressed — and the parse dominates, which means the engine choice matters as much\nhere as it does locally. calamine is several times faster than openpyxl on the same bytes.",[10,1986,1987],{},"The larger win is not reading the workbook at all on repeat runs:",[127,1989,1991],{"className":188,"code":1990,"language":190,"meta":132,"style":132},"def cached_parquet(bucket: str, key: str) -> pd.DataFrame:\n    parquet_key = key.rsplit(\".\", 1)[0] + \".parquet\"\n    try:\n        return pd.read_parquet(f\"s3:\u002F\u002F{bucket}\u002F{parquet_key}\")\n    except FileNotFoundError:\n        frame = read_workbook(bucket, key)\n        frame.to_parquet(f\"s3:\u002F\u002F{bucket}\u002F{parquet_key}\", index=False)\n        return frame\n",[134,1992,1993,2010,2039,2045,2075,2083,2093,2130],{"__ignoreMap":132},[137,1994,1995,1997,2000,2002,2004,2006,2008],{"class":74,"line":139},[137,1996,356],{"class":197},[137,1998,1999],{"class":359}," cached_parquet",[137,2001,363],{"class":201},[137,2003,366],{"class":275},[137,2005,369],{"class":201},[137,2007,366],{"class":275},[137,2009,841],{"class":201},[137,2011,2012,2015,2017,2020,2023,2025,2027,2029,2031,2033,2036],{"class":74,"line":211},[137,2013,2014],{"class":201},"    parquet_key ",[137,2016,224],{"class":197},[137,2018,2019],{"class":201}," key.rsplit(",[137,2021,2022],{"class":146},"\".\"",[137,2024,374],{"class":201},[137,2026,693],{"class":275},[137,2028,980],{"class":201},[137,2030,41],{"class":275},[137,2032,1619],{"class":201},[137,2034,2035],{"class":197},"+",[137,2037,2038],{"class":146}," \".parquet\"\n",[137,2040,2041,2043],{"class":74,"line":218},[137,2042,386],{"class":197},[137,2044,389],{"class":201},[137,2046,2047,2049,2052,2054,2056,2058,2060,2062,2064,2066,2069,2071,2073],{"class":74,"line":230},[137,2048,1357],{"class":197},[137,2050,2051],{"class":201}," pd.read_parquet(",[137,2053,485],{"class":197},[137,2055,488],{"class":146},[137,2057,492],{"class":491},[137,2059,495],{"class":201},[137,2061,498],{"class":491},[137,2063,501],{"class":146},[137,2065,492],{"class":491},[137,2067,2068],{"class":201},"parquet_key",[137,2070,498],{"class":491},[137,2072,575],{"class":146},[137,2074,269],{"class":201},[137,2076,2077,2079,2081],{"class":74,"line":239},[137,2078,422],{"class":197},[137,2080,479],{"class":275},[137,2082,389],{"class":201},[137,2084,2085,2088,2090],{"class":74,"line":253},[137,2086,2087],{"class":201},"        frame ",[137,2089,224],{"class":197},[137,2091,2092],{"class":201}," read_workbook(bucket, key)\n",[137,2094,2095,2098,2100,2102,2104,2106,2108,2110,2112,2114,2116,2118,2120,2123,2125,2128],{"class":74,"line":266},[137,2096,2097],{"class":201},"        frame.to_parquet(",[137,2099,485],{"class":197},[137,2101,488],{"class":146},[137,2103,492],{"class":491},[137,2105,495],{"class":201},[137,2107,498],{"class":491},[137,2109,501],{"class":146},[137,2111,492],{"class":491},[137,2113,2068],{"class":201},[137,2115,498],{"class":491},[137,2117,575],{"class":146},[137,2119,374],{"class":201},[137,2121,2122],{"class":242},"index",[137,2124,224],{"class":197},[137,2126,2127],{"class":275},"False",[137,2129,269],{"class":201},[137,2131,2132,2134],{"class":74,"line":272},[137,2133,1357],{"class":197},[137,2135,2136],{"class":201}," frame\n",[10,2138,2139,2140,174],{},"For a file read by several jobs a day, converting once turns a multi-second parse into a fraction of\na second for every consumer — the argument made in\n",[14,2141,2143],{"href":2142},"\u002Fadvanced-data-transformation-and-cleaning\u002Freading-excel-with-polars-and-arrow\u002Fconvert-excel-files-to-parquet-with-python\u002F","Convert Excel Files to Parquet with Python",[122,2145,2147],{"id":2146},"conclusion","Conclusion",[10,2149,2150,2153,2154,2156,2157,2159,2160,2162,2163,2166],{},[134,2151,2152],{},"pd.read_excel(\"s3:\u002F\u002F…\")"," is the right call in a notebook; in a job, use ",[134,2155,67],{}," with ",[134,2158,92],{},"\nso you can translate ",[134,2161,1794],{}," and ",[134,2164,2165],{},"AccessDenied"," into errors that name the problem. Paginate when\nlisting, construct keys from dates rather than scanning for the newest, spool large workbooks to a\ntemporary file, and cache a Parquet copy when the same source is read more than once.",[122,2168,2170],{"id":2169},"frequently-asked-questions","Frequently asked questions",[10,2172,2173,2177],{},[2174,2175,2176],"strong",{},"Does pandas read from s3:\u002F\u002F directly?","\nYes, when s3fs is installed — handing read_excel an s3:\u002F\u002F URI works, and it uses your usual AWS credential chain. It is convenient for exploration; for production, an explicit boto3 get_object gives clearer errors and lets you control retries.",[10,2179,2180,2183],{},[2174,2181,2182],{},"Should I download to a temporary file or read into memory?","\nRead into memory with BytesIO for anything that comfortably fits — it avoids a disk write and cleans itself up. Download to a file when the workbook is large enough that holding both the bytes and the parsed frame is a problem, or when a library insists on a path.",[10,2185,2186,2189],{},[2174,2187,2188],{},"How do I read only the newest file in a prefix?","\nList the prefix with a paginator and take the maximum LastModified. For anything more than a few hundred keys, prefer a naming convention with the date in the key so you can construct the name rather than scanning.",[10,2191,2192,2195],{},[2174,2193,2194],{},"Why is my read slow from a Lambda?","\nUsually cold-start plus a large workbook. Convert the source to Parquet once and have the function read that instead — it is smaller to transfer and much faster to parse, which matters most where the compute is billed by the millisecond.",[122,2197,2199],{"id":2198},"related","Related",[2201,2202,2203,2210,2215,2222,2227],"ul",{},[2204,2205,2206,2207,2209],"li",{},"Up one level: ",[14,2208,17],{"href":16}," — the write side and the other providers.",[2204,2211,2212,2214],{},[14,2213,173],{"href":172}," — credentials, content types and presigned links.",[2204,2216,2217,2221],{},[14,2218,2220],{"href":2219},"\u002Fgetting-started-with-python-excel-automation\u002Fhandling-excel-file-formats-and-conversions\u002Fread-an-excel-file-from-a-url-or-bytes-in-python\u002F","Read an Excel File from a URL or Bytes in Python"," — the same BytesIO pattern over HTTP.",[2204,2223,2224,2226],{},[14,2225,2143],{"href":2142}," — the caching step that removes repeat parsing.",[2204,2228,2229,2233],{},[14,2230,2232],{"href":2231},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fprocess-multiple-excel-files-in-parallel-with-python\u002F","Process Multiple Excel Files in Parallel with Python"," — fanning out a prefix read across workers.",[2235,2236,2237],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}",{"title":132,"searchDepth":211,"depth":211,"links":2239},[2240,2241,2242,2243,2244,2245,2246,2247,2248,2249,2250,2251,2252],{"id":124,"depth":211,"text":125},{"id":177,"depth":211,"text":178},{"id":285,"depth":211,"text":286},{"id":660,"depth":211,"text":661},{"id":1059,"depth":211,"text":1060},{"id":1257,"depth":211,"text":1258},{"id":1384,"depth":211,"text":1385},{"id":1640,"depth":211,"text":1641},{"id":1763,"depth":211,"text":1764},{"id":1874,"depth":211,"text":1875},{"id":2146,"depth":211,"text":2147},{"id":2169,"depth":211,"text":2170},{"id":2198,"depth":211,"text":2199},"2026-09-04","Load a workbook from S3 into a DataFrame without a temporary file: BytesIO reads, paginated prefix scans, readable error translation, and a Parquet cache for repeat runs.","md",[2257,2259,2261,2263],{"q":2176,"a":2258},"Yes, when s3fs is installed — handing read_excel an s3:\u002F\u002F URI works, and it uses your usual AWS credential chain. It is convenient for exploration; for production, an explicit boto3 get_object gives clearer errors and lets you control retries.",{"q":2182,"a":2260},"Read into memory with BytesIO for anything that comfortably fits — it avoids a disk write and cleans itself up. Download to a file when the workbook is large enough that holding both the bytes and the parsed frame is a problem, or when a library insists on a path.",{"q":2188,"a":2262},"List the prefix with a paginator and take the maximum LastModified. For anything more than a few hundred keys, prefer a naming convention with the date in the key so you can construct the name rather than scanning.",{"q":2194,"a":2264},"Usually cold-start plus a large workbook. Convert the source to Parquet once and have the function read that instead — it is smaller to transfer and much faster to parse, which matters most where the compute is billed by the millisecond.",{"breadcrumb":2266},[2267,2269,2272],{"name":2268,"item":501},"Home",{"name":2270,"item":2271},"Automating Reporting Workflows","\u002Fautomating-reporting-workflows\u002F",{"name":17,"item":16},"\u002Fautomating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage\u002Fread-an-excel-file-directly-from-amazon-s3-with-pandas",{"title":5,"description":2275},"Read .xlsx from S3 into pandas — the s3:\u002F\u002F one-liner, an explicit boto3 get_object with BytesIO, paginated prefix reads, newest-key selection and Parquet caching.","read-an-excel-file-directly-from-amazon-s3-with-pandas","automating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage\u002Fread-an-excel-file-directly-from-amazon-s3-with-pandas\u002Findex","how-to","imgv9iGiuFSZY7blmqFHVCoW2rbaVWV4UM9vGl2pcKw",[2281,2285],{"title":2282,"path":2283,"stem":2284,"children":-1},"Post an Excel Report to Microsoft Teams with Python","\u002Fautomating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage\u002Fpost-an-excel-report-to-microsoft-teams-with-python","automating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage\u002Fpost-an-excel-report-to-microsoft-teams-with-python\u002Findex",{"title":1760,"path":2286,"stem":2287,"children":-1},"\u002Fautomating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage\u002Fsave-excel-reports-to-google-drive-with-python","automating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage\u002Fsave-excel-reports-to-google-drive-with-python\u002Findex",1788710159064]