[{"data":1,"prerenderedAt":2693},["ShallowReactive",2],{"doc:\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Ftrack-run-history-and-step-timings-of-a-report-job":3,"surround:\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Ftrack-run-history-and-step-timings-of-a-report-job":2658,"ancestors:\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Ftrack-run-history-and-step-timings-of-a-report-job":2667,"children:\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Ftrack-run-history-and-step-timings-of-a-report-job":2672,"related:\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Ftrack-run-history-and-step-timings-of-a-report-job":2673},{"id":4,"title":5,"body":6,"dateModified":2626,"datePublished":2626,"description":2627,"extension":2628,"faq":2629,"keyword":2638,"kind":2639,"meta":2640,"navigation":109,"path":2648,"seo":2649,"slug":2652,"stem":2653,"topics":2654,"type":2656,"__hash__":2657},"docs\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Ftrack-run-history-and-step-timings-of-a-report-job\u002Findex.md","Track Run History and Step Timings of a Report Job",{"type":7,"value":8,"toc":2614},"minimark",[9,19,24,53,56,60,554,565,706,710,1058,1069,1073,1537,1545,1795,1799,2368,2383,2387,2390,2393,2476,2480,2531,2535,2541,2545,2555,2561,2567,2577,2581,2610],[10,11,12,13,18],"p",{},"A scheduled report job that \"takes about ten minutes\" usually has no record of how long it actually took last Tuesday, or which part of it has been getting slower. That missing history is what turns a slow drift into a surprise: the query that took two minutes in January takes nine by June, the job finishes after the 08:00 deadline one morning, and nobody can say when it started. The fix is cheap. Time each step of the job, write one row per step per run into a small SQLite database next to the job, and once a week turn that history into a workbook that shows failures, durations and trends. This page builds the timing wrapper, the run table and the history report, including a rule that flags runs much slower than usual. It belongs to ",[14,15,17],"a",{"href":16},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002F","Error Handling and Logging in Excel Automation",".",[20,21,23],"h2",{"id":22},"prerequisites","Prerequisites",[25,26,31],"pre",{"className":27,"code":28,"language":29,"meta":30,"style":30},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl\n","bash","",[32,33,34],"code",{"__ignoreMap":30},[35,36,39,43,47,50],"span",{"class":37,"line":38},"line",1,[35,40,42],{"class":41},"sMTad","pip",[35,44,46],{"class":45},"srMev"," install",[35,48,49],{"class":45}," pandas",[35,51,52],{"class":45}," openpyxl\n",[10,54,55],{},"SQLite is part of Python's standard library. The examples use a simulated report job with four steps — extract, transform, write workbook, send — so the history can be generated for three months without waiting three months.",[20,57,59],{"id":58},"step-1-a-context-manager-that-times-and-records-each-step","Step 1 — A context manager that times and records each step",[25,61,65],{"className":62,"code":63,"language":64,"meta":30,"style":30},"language-python shiki shiki-themes github-light github-dark-high-contrast","import sqlite3, time, uuid\nfrom contextlib import closing, contextmanager\nfrom datetime import datetime, timezone\n\nDB = \"job_history.sqlite\"\nwith closing(sqlite3.connect(DB)) as con:\n    con.executescript(\"\"\"\nCREATE TABLE IF NOT EXISTS steps (\n    run_id TEXT NOT NULL, job TEXT NOT NULL, step TEXT NOT NULL,\n    started_utc TEXT NOT NULL, seconds REAL NOT NULL, ok INTEGER NOT NULL,\n    rows INTEGER, error TEXT);\nCREATE INDEX IF NOT EXISTS steps_job_started ON steps(job, started_utc);\n\"\"\")\n\nclass Run:\n    def __init__(self, job, clock=None):\n        self.job, self.id = job, uuid.uuid4().hex[:12]\n        self.clock = clock or (lambda: datetime.now(timezone.utc))\n\n    @contextmanager\n    def step(self, name):\n        started, t0 = self.clock(), time.perf_counter()\n        info = {\"rows\": None}\n        try:\n            yield info\n        except Exception as e:\n            self._record(name, started, time.perf_counter() - t0, False, info[\"rows\"], f\"{type(e).__name__}: {e}\")\n            raise\n        else:\n            self._record(name, started, time.perf_counter() - t0, True, info[\"rows\"], None)\n\n    def _record(self, name, started, seconds, ok, rows, error):\n        with closing(sqlite3.connect(DB)) as c, c:      # close the file, commit the insert\n            c.execute(\"INSERT INTO steps VALUES (?,?,?,?,?,?,?,?)\",\n                      (self.id, self.job, name, started.isoformat(), round(seconds, 3), int(ok), rows, error))\n","python",[32,66,67,77,91,104,111,124,144,153,159,165,171,177,183,192,197,209,230,256,281,286,293,304,318,340,348,357,372,433,439,447,471,476,487,508,520],{"__ignoreMap":30},[35,68,69,73],{"class":37,"line":38},[35,70,72],{"class":71},"s-kum","import",[35,74,76],{"class":75},"skGVy"," sqlite3, time, uuid\n",[35,78,80,83,86,88],{"class":37,"line":79},2,[35,81,82],{"class":71},"from",[35,84,85],{"class":75}," contextlib ",[35,87,72],{"class":71},[35,89,90],{"class":75}," closing, contextmanager\n",[35,92,94,96,99,101],{"class":37,"line":93},3,[35,95,82],{"class":71},[35,97,98],{"class":75}," datetime ",[35,100,72],{"class":71},[35,102,103],{"class":75}," datetime, timezone\n",[35,105,107],{"class":37,"line":106},4,[35,108,110],{"emptyLinePlaceholder":109},true,"\n",[35,112,114,118,121],{"class":37,"line":113},5,[35,115,117],{"class":116},"sP0c6","DB",[35,119,120],{"class":71}," =",[35,122,123],{"class":45}," \"job_history.sqlite\"\n",[35,125,127,130,133,135,138,141],{"class":37,"line":126},6,[35,128,129],{"class":71},"with",[35,131,132],{"class":75}," closing(sqlite3.connect(",[35,134,117],{"class":116},[35,136,137],{"class":75},")) ",[35,139,140],{"class":71},"as",[35,142,143],{"class":75}," con:\n",[35,145,147,150],{"class":37,"line":146},7,[35,148,149],{"class":75},"    con.executescript(",[35,151,152],{"class":45},"\"\"\"\n",[35,154,156],{"class":37,"line":155},8,[35,157,158],{"class":45},"CREATE TABLE IF NOT EXISTS steps (\n",[35,160,162],{"class":37,"line":161},9,[35,163,164],{"class":45},"    run_id TEXT NOT NULL, job TEXT NOT NULL, step TEXT NOT NULL,\n",[35,166,168],{"class":37,"line":167},10,[35,169,170],{"class":45},"    started_utc TEXT NOT NULL, seconds REAL NOT NULL, ok INTEGER NOT NULL,\n",[35,172,174],{"class":37,"line":173},11,[35,175,176],{"class":45},"    rows INTEGER, error TEXT);\n",[35,178,180],{"class":37,"line":179},12,[35,181,182],{"class":45},"CREATE INDEX IF NOT EXISTS steps_job_started ON steps(job, started_utc);\n",[35,184,186,189],{"class":37,"line":185},13,[35,187,188],{"class":45},"\"\"\"",[35,190,191],{"class":75},")\n",[35,193,195],{"class":37,"line":194},14,[35,196,110],{"emptyLinePlaceholder":109},[35,198,200,203,206],{"class":37,"line":199},15,[35,201,202],{"class":71},"class",[35,204,205],{"class":41}," Run",[35,207,208],{"class":75},":\n",[35,210,212,215,218,221,224,227],{"class":37,"line":211},16,[35,213,214],{"class":71},"    def",[35,216,217],{"class":116}," __init__",[35,219,220],{"class":75},"(self, job, clock",[35,222,223],{"class":71},"=",[35,225,226],{"class":116},"None",[35,228,229],{"class":75},"):\n",[35,231,233,236,239,242,245,247,250,253],{"class":37,"line":232},17,[35,234,235],{"class":116},"        self",[35,237,238],{"class":75},".job, ",[35,240,241],{"class":116},"self",[35,243,244],{"class":75},".id ",[35,246,223],{"class":71},[35,248,249],{"class":75}," job, uuid.uuid4().hex[:",[35,251,252],{"class":116},"12",[35,254,255],{"class":75},"]\n",[35,257,259,261,264,266,269,272,275,278],{"class":37,"line":258},18,[35,260,235],{"class":116},[35,262,263],{"class":75},".clock ",[35,265,223],{"class":71},[35,267,268],{"class":75}," clock ",[35,270,271],{"class":71},"or",[35,273,274],{"class":75}," (",[35,276,277],{"class":71},"lambda",[35,279,280],{"class":75},": datetime.now(timezone.utc))\n",[35,282,284],{"class":37,"line":283},19,[35,285,110],{"emptyLinePlaceholder":109},[35,287,289],{"class":37,"line":288},20,[35,290,292],{"class":291},"s_Opv","    @contextmanager\n",[35,294,296,298,301],{"class":37,"line":295},21,[35,297,214],{"class":71},[35,299,300],{"class":291}," step",[35,302,303],{"class":75},"(self, name):\n",[35,305,307,310,312,315],{"class":37,"line":306},22,[35,308,309],{"class":75},"        started, t0 ",[35,311,223],{"class":71},[35,313,314],{"class":116}," self",[35,316,317],{"class":75},".clock(), time.perf_counter()\n",[35,319,321,324,326,329,332,335,337],{"class":37,"line":320},23,[35,322,323],{"class":75},"        info ",[35,325,223],{"class":71},[35,327,328],{"class":75}," {",[35,330,331],{"class":45},"\"rows\"",[35,333,334],{"class":75},": ",[35,336,226],{"class":116},[35,338,339],{"class":75},"}\n",[35,341,343,346],{"class":37,"line":342},24,[35,344,345],{"class":71},"        try",[35,347,208],{"class":75},[35,349,351,354],{"class":37,"line":350},25,[35,352,353],{"class":71},"            yield",[35,355,356],{"class":75}," info\n",[35,358,360,363,366,369],{"class":37,"line":359},26,[35,361,362],{"class":71},"        except",[35,364,365],{"class":116}," Exception",[35,367,368],{"class":71}," as",[35,370,371],{"class":75}," e:\n",[35,373,375,378,381,384,387,390,393,395,398,401,404,408,411,414,417,420,422,424,427,429,431],{"class":37,"line":374},27,[35,376,377],{"class":116},"            self",[35,379,380],{"class":75},"._record(name, started, time.perf_counter() ",[35,382,383],{"class":71},"-",[35,385,386],{"class":75}," t0, ",[35,388,389],{"class":116},"False",[35,391,392],{"class":75},", info[",[35,394,331],{"class":45},[35,396,397],{"class":75},"], ",[35,399,400],{"class":71},"f",[35,402,403],{"class":45},"\"",[35,405,407],{"class":406},"sSjpA","{",[35,409,410],{"class":116},"type",[35,412,413],{"class":75},"(e).",[35,415,416],{"class":116},"__name__",[35,418,419],{"class":406},"}",[35,421,334],{"class":45},[35,423,407],{"class":406},[35,425,426],{"class":75},"e",[35,428,419],{"class":406},[35,430,403],{"class":45},[35,432,191],{"class":75},[35,434,436],{"class":37,"line":435},28,[35,437,438],{"class":71},"            raise\n",[35,440,442,445],{"class":37,"line":441},29,[35,443,444],{"class":71},"        else",[35,446,208],{"class":75},[35,448,450,452,454,456,458,461,463,465,467,469],{"class":37,"line":449},30,[35,451,377],{"class":116},[35,453,380],{"class":75},[35,455,383],{"class":71},[35,457,386],{"class":75},[35,459,460],{"class":116},"True",[35,462,392],{"class":75},[35,464,331],{"class":45},[35,466,397],{"class":75},[35,468,226],{"class":116},[35,470,191],{"class":75},[35,472,474],{"class":37,"line":473},31,[35,475,110],{"emptyLinePlaceholder":109},[35,477,479,481,484],{"class":37,"line":478},32,[35,480,214],{"class":71},[35,482,483],{"class":291}," _record",[35,485,486],{"class":75},"(self, name, started, seconds, ok, rows, error):\n",[35,488,490,493,495,497,499,501,504],{"class":37,"line":489},33,[35,491,492],{"class":71},"        with",[35,494,132],{"class":75},[35,496,117],{"class":116},[35,498,137],{"class":75},[35,500,140],{"class":71},[35,502,503],{"class":75}," c, c:      ",[35,505,507],{"class":506},"s-wDw","# close the file, commit the insert\n",[35,509,511,514,517],{"class":37,"line":510},34,[35,512,513],{"class":75},"            c.execute(",[35,515,516],{"class":45},"\"INSERT INTO steps VALUES (?,?,?,?,?,?,?,?)\"",[35,518,519],{"class":75},",\n",[35,521,523,526,528,531,533,536,539,542,545,548,551],{"class":37,"line":522},35,[35,524,525],{"class":75},"                      (",[35,527,241],{"class":116},[35,529,530],{"class":75},".id, ",[35,532,241],{"class":116},[35,534,535],{"class":75},".job, name, started.isoformat(), ",[35,537,538],{"class":116},"round",[35,540,541],{"class":75},"(seconds, ",[35,543,544],{"class":116},"3",[35,546,547],{"class":75},"), ",[35,549,550],{"class":116},"int",[35,552,553],{"class":75},"(ok), rows, error))\n",[10,555,556,557,560,561,564],{},"Each step of the job is wrapped in ",[32,558,559],{},"with run.step(\"extract\") as s:",". The context manager notes the start time, measures the elapsed time with ",[32,562,563],{},"perf_counter"," — a monotonic clock that is not affected by the system clock being adjusted — and writes one row when the step finishes. If the step raises, the row is still written, marked as failed with the error message, and the exception carries on to whatever error handling the job already has, so timing never hides a failure. The step can also report how many rows it handled through the yielded dictionary, which makes it possible later to tell \"slower because there is more data\" from \"slower for no reason\". Each row is written in its own short transaction, so the history survives even if the job is killed halfway through.",[566,567,575,576,575,580,575,584,575,592,575,601,575,607,575,611,575,616,575,622,575,627,575,631,575,637,575,641,575,645,575,648,575,653,575,658,575,661,575,663,575,667,575,670,575,675,575,680,575,683,575,687,575,690,575,695,575,700,575,703],"svg",{"viewBox":568,"role":569,"ariaLabelledBy":570,"xmlns":573,"style":574},"4 0 752 108","img",[571,572],"rh-wrap-t","rh-wrap-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:752px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[577,578,579],"title",{"id":571},"What happens around each step",[581,582,583],"desc",{"id":572},"The context manager records the start time, runs the step, measures the elapsed seconds and writes one row to the SQLite steps table, marked as success or failure. A failure is recorded and then raised again so the job's own error handling still runs.",[585,586],"rect",{"x":587,"y":588,"width":589,"height":590,"fill":591},"4","0","752","108","#ffffff",[585,593],{"x":594,"y":595,"width":596,"height":597,"rx":252,"fill":598,"stroke":599,"style":600},"20.0","16.0","115.2","76.5","#e7ebef","var(--line,#cdd5e6)","stroke-width:2px",[602,603,129],"text",{"x":604,"y":605,"style":606},"77.6","41.5","font-size:13.5px;font-weight:700;fill:var(--text,#172033);text-anchor:middle",[602,608,610],{"x":604,"y":609,"style":606},"59.0","run.step()",[602,612,615],{"x":604,"y":613,"style":614},"78.5","font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","start time noted",[37,617],{"x1":618,"y1":619,"x2":620,"y2":619,"stroke":621,"style":600},"140.2","54.2","162.2","var(--brand,#5b5cf0)",[623,624],"polygon",{"points":625,"fill":626},"166.2,54.2 157.2,59.2 157.2,49.2","#5b5cf0",[585,628],{"x":629,"y":595,"width":596,"height":597,"rx":252,"fill":630,"stroke":621,"style":600},"171.2","#f0f4ff",[602,632,636],{"x":633,"y":634,"style":635},"228.8","50.2","font-size:13.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Step runs",[602,638,640],{"x":633,"y":639,"style":614},"69.8","extract or write",[37,642],{"x1":643,"y1":619,"x2":644,"y2":619,"stroke":621,"style":600},"291.4","313.4",[623,646],{"points":647,"fill":626},"317.4,54.2 308.4,59.2 308.4,49.2",[585,649],{"x":650,"y":595,"width":596,"height":597,"rx":252,"fill":651,"stroke":652,"style":600},"322.4","#fdefd8","var(--gold,#b4740a)",[602,654,657],{"x":655,"y":605,"style":656},"380.0","font-size:13.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","Elapsed",[602,659,660],{"x":655,"y":609,"style":656},"seconds",[602,662,563],{"x":655,"y":613,"style":614},[37,664],{"x1":665,"y1":619,"x2":666,"y2":619,"stroke":621,"style":600},"442.6","464.6",[623,668],{"points":669,"fill":626},"468.6,54.2 459.6,59.2 459.6,49.2",[585,671],{"x":672,"y":595,"width":596,"height":597,"rx":252,"fill":673,"stroke":674,"style":600},"473.6","#d9f4f1","var(--teal,#0f9488)",[602,676,679],{"x":677,"y":634,"style":678},"531.2","font-size:13.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","Row written",[602,681,682],{"x":677,"y":639,"style":614},"ok or error",[37,684],{"x1":685,"y1":619,"x2":686,"y2":619,"stroke":621,"style":600},"593.8","615.8",[623,688],{"points":689,"fill":626},"619.8,54.2 610.8,59.2 610.8,49.2",[585,691],{"x":692,"y":595,"width":596,"height":597,"rx":252,"fill":693,"stroke":694,"style":600},"624.8","#fce9e9","var(--danger,#dc2626)",[602,696,699],{"x":697,"y":605,"style":698},"682.4","font-size:13.5px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","Error",[602,701,702],{"x":697,"y":609,"style":698},"re-raised",[602,704,705],{"x":697,"y":613,"style":614},"if it failed",[20,707,709],{"id":708},"step-2-use-it-in-the-job","Step 2 — Use it in the job",[25,711,713],{"className":62,"code":712,"language":64,"meta":30,"style":30},"import pandas as pd\n\ndef report_job(send_fails=False):\n    run = Run(\"daily-sales\")\n    with run.step(\"extract\") as s:\n        df = pd.DataFrame({\"region\": [\"N\", \"S\"] * 5000, \"revenue\": range(10000)})\n        s[\"rows\"] = len(df)\n    with run.step(\"transform\") as s:\n        summary = df.groupby(\"region\", as_index=False)[\"revenue\"].sum()\n        s[\"rows\"] = len(summary)\n    with run.step(\"write workbook\"):\n        summary.to_excel(\"daily_sales.xlsx\", index=False)\n    with run.step(\"send\"):\n        if send_fails:\n            raise ConnectionError(\"SMTP server timed out\")\n\nreport_job()\ntry:\n    report_job(send_fails=True)\nexcept ConnectionError:\n    pass\nwith closing(sqlite3.connect(DB)) as c:\n    print(pd.read_sql(\"SELECT run_id, step, seconds, ok, rows, error FROM steps\", c).to_string(index=False))\n",[32,714,715,727,731,748,763,782,835,852,867,897,912,923,942,953,961,976,980,985,992,1006,1015,1020,1035],{"__ignoreMap":30},[35,716,717,719,722,724],{"class":37,"line":38},[35,718,72],{"class":71},[35,720,721],{"class":75}," pandas ",[35,723,140],{"class":71},[35,725,726],{"class":75}," pd\n",[35,728,729],{"class":37,"line":79},[35,730,110],{"emptyLinePlaceholder":109},[35,732,733,736,739,742,744,746],{"class":37,"line":93},[35,734,735],{"class":71},"def",[35,737,738],{"class":291}," report_job",[35,740,741],{"class":75},"(send_fails",[35,743,223],{"class":71},[35,745,389],{"class":116},[35,747,229],{"class":75},[35,749,750,753,755,758,761],{"class":37,"line":106},[35,751,752],{"class":75},"    run ",[35,754,223],{"class":71},[35,756,757],{"class":75}," Run(",[35,759,760],{"class":45},"\"daily-sales\"",[35,762,191],{"class":75},[35,764,765,768,771,774,777,779],{"class":37,"line":113},[35,766,767],{"class":71},"    with",[35,769,770],{"class":75}," run.step(",[35,772,773],{"class":45},"\"extract\"",[35,775,776],{"class":75},") ",[35,778,140],{"class":71},[35,780,781],{"class":75}," s:\n",[35,783,784,787,789,792,795,798,801,804,807,810,813,816,818,821,823,826,829,832],{"class":37,"line":126},[35,785,786],{"class":75},"        df ",[35,788,223],{"class":71},[35,790,791],{"class":75}," pd.DataFrame({",[35,793,794],{"class":45},"\"region\"",[35,796,797],{"class":75},": [",[35,799,800],{"class":45},"\"N\"",[35,802,803],{"class":75},", ",[35,805,806],{"class":45},"\"S\"",[35,808,809],{"class":75},"] ",[35,811,812],{"class":71},"*",[35,814,815],{"class":116}," 5000",[35,817,803],{"class":75},[35,819,820],{"class":45},"\"revenue\"",[35,822,334],{"class":75},[35,824,825],{"class":116},"range",[35,827,828],{"class":75},"(",[35,830,831],{"class":116},"10000",[35,833,834],{"class":75},")})\n",[35,836,837,840,842,844,846,849],{"class":37,"line":146},[35,838,839],{"class":75},"        s[",[35,841,331],{"class":45},[35,843,809],{"class":75},[35,845,223],{"class":71},[35,847,848],{"class":116}," len",[35,850,851],{"class":75},"(df)\n",[35,853,854,856,858,861,863,865],{"class":37,"line":155},[35,855,767],{"class":71},[35,857,770],{"class":75},[35,859,860],{"class":45},"\"transform\"",[35,862,776],{"class":75},[35,864,140],{"class":71},[35,866,781],{"class":75},[35,868,869,872,874,877,879,881,885,887,889,892,894],{"class":37,"line":161},[35,870,871],{"class":75},"        summary ",[35,873,223],{"class":71},[35,875,876],{"class":75}," df.groupby(",[35,878,794],{"class":45},[35,880,803],{"class":75},[35,882,884],{"class":883},"sa561","as_index",[35,886,223],{"class":71},[35,888,389],{"class":116},[35,890,891],{"class":75},")[",[35,893,820],{"class":45},[35,895,896],{"class":75},"].sum()\n",[35,898,899,901,903,905,907,909],{"class":37,"line":167},[35,900,839],{"class":75},[35,902,331],{"class":45},[35,904,809],{"class":75},[35,906,223],{"class":71},[35,908,848],{"class":116},[35,910,911],{"class":75},"(summary)\n",[35,913,914,916,918,921],{"class":37,"line":173},[35,915,767],{"class":71},[35,917,770],{"class":75},[35,919,920],{"class":45},"\"write workbook\"",[35,922,229],{"class":75},[35,924,925,928,931,933,936,938,940],{"class":37,"line":179},[35,926,927],{"class":75},"        summary.to_excel(",[35,929,930],{"class":45},"\"daily_sales.xlsx\"",[35,932,803],{"class":75},[35,934,935],{"class":883},"index",[35,937,223],{"class":71},[35,939,389],{"class":116},[35,941,191],{"class":75},[35,943,944,946,948,951],{"class":37,"line":185},[35,945,767],{"class":71},[35,947,770],{"class":75},[35,949,950],{"class":45},"\"send\"",[35,952,229],{"class":75},[35,954,955,958],{"class":37,"line":194},[35,956,957],{"class":71},"        if",[35,959,960],{"class":75}," send_fails:\n",[35,962,963,966,969,971,974],{"class":37,"line":199},[35,964,965],{"class":71},"            raise",[35,967,968],{"class":116}," ConnectionError",[35,970,828],{"class":75},[35,972,973],{"class":45},"\"SMTP server timed out\"",[35,975,191],{"class":75},[35,977,978],{"class":37,"line":211},[35,979,110],{"emptyLinePlaceholder":109},[35,981,982],{"class":37,"line":232},[35,983,984],{"class":75},"report_job()\n",[35,986,987,990],{"class":37,"line":258},[35,988,989],{"class":71},"try",[35,991,208],{"class":75},[35,993,994,997,1000,1002,1004],{"class":37,"line":283},[35,995,996],{"class":75},"    report_job(",[35,998,999],{"class":883},"send_fails",[35,1001,223],{"class":71},[35,1003,460],{"class":116},[35,1005,191],{"class":75},[35,1007,1008,1011,1013],{"class":37,"line":288},[35,1009,1010],{"class":71},"except",[35,1012,968],{"class":116},[35,1014,208],{"class":75},[35,1016,1017],{"class":37,"line":295},[35,1018,1019],{"class":71},"    pass\n",[35,1021,1022,1024,1026,1028,1030,1032],{"class":37,"line":306},[35,1023,129],{"class":71},[35,1025,132],{"class":75},[35,1027,117],{"class":116},[35,1029,137],{"class":75},[35,1031,140],{"class":71},[35,1033,1034],{"class":75}," c:\n",[35,1036,1037,1040,1043,1046,1049,1051,1053,1055],{"class":37,"line":320},[35,1038,1039],{"class":116},"    print",[35,1041,1042],{"class":75},"(pd.read_sql(",[35,1044,1045],{"class":45},"\"SELECT run_id, step, seconds, ok, rows, error FROM steps\"",[35,1047,1048],{"class":75},", c).to_string(",[35,1050,935],{"class":883},[35,1052,223],{"class":71},[35,1054,389],{"class":116},[35,1056,1057],{"class":75},"))\n",[10,1059,1060,1061,1063,1064,1068],{},"The job itself barely changes: each logical step sits inside a ",[32,1062,129],{}," block, and that is all. The second run fails in the send step, and the table shows exactly that — three successful steps and a failed one with the error text, all under the same run id. Steps should be the parts a person would name when describing the job, usually four to eight of them; timing every function call produces a table nobody reads. For jobs that already log to a file, as in ",[14,1065,1067],{"href":1066},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Flog-python-excel-script-output-to-a-file\u002F","Log Python Excel Script Output to a File",", the database complements the log: the log says what happened in one run, the table says how runs compare.",[20,1070,1072],{"id":1071},"step-3-turn-the-history-into-runs-trends-and-flags","Step 3 — Turn the history into runs, trends and flags",[25,1074,1076],{"className":62,"code":1075,"language":64,"meta":30,"style":30},"with closing(sqlite3.connect(DB)) as c:\n    steps = pd.read_sql(\"SELECT * FROM steps WHERE job = 'daily-sales'\", c)\nsteps[\"started_utc\"] = pd.to_datetime(steps[\"started_utc\"], format=\"ISO8601\", utc=True)\nruns = (steps.groupby(\"run_id\")\n             .agg(started=(\"started_utc\", \"min\"), seconds=(\"seconds\", \"sum\"), ok=(\"ok\", \"min\"))\n             .sort_values(\"started\"))\nruns[\"minutes\"] = (runs[\"seconds\"] \u002F 60).round(1)\n\n# Slow-run flag: more than 3 robust deviations above the median of the previous 20 runs\nmed = runs[\"seconds\"].rolling(20, min_periods=10).median().shift()\nmad = (runs[\"seconds\"] - med).abs().rolling(20, min_periods=10).median().shift()\nruns[\"slow\"] = runs[\"seconds\"] > med + 3 * 1.4826 * mad\n\nby_step = steps.pivot_table(index=steps[\"started_utc\"].dt.strftime(\"%Y-%m\").rename(\"month\"),\n                            columns=\"step\", values=\"seconds\", aggfunc=\"median\").round(0)\nprint(f\"{len(runs)} runs, {int((runs['ok'] == 0).sum())} failed, {int(runs['slow'].sum())} flagged slow\")\nprint(runs.loc[runs[\"slow\"], [\"started\", \"minutes\"]].to_string())\nprint(by_step.to_string())\n",[32,1077,1078,1092,1108,1146,1161,1214,1224,1257,1261,1266,1297,1327,1367,1371,1404,1439,1509,1530],{"__ignoreMap":30},[35,1079,1080,1082,1084,1086,1088,1090],{"class":37,"line":38},[35,1081,129],{"class":71},[35,1083,132],{"class":75},[35,1085,117],{"class":116},[35,1087,137],{"class":75},[35,1089,140],{"class":71},[35,1091,1034],{"class":75},[35,1093,1094,1097,1099,1102,1105],{"class":37,"line":79},[35,1095,1096],{"class":75},"    steps ",[35,1098,223],{"class":71},[35,1100,1101],{"class":75}," pd.read_sql(",[35,1103,1104],{"class":45},"\"SELECT * FROM steps WHERE job = 'daily-sales'\"",[35,1106,1107],{"class":75},", c)\n",[35,1109,1110,1113,1116,1118,1120,1123,1125,1127,1130,1132,1135,1137,1140,1142,1144],{"class":37,"line":93},[35,1111,1112],{"class":75},"steps[",[35,1114,1115],{"class":45},"\"started_utc\"",[35,1117,809],{"class":75},[35,1119,223],{"class":71},[35,1121,1122],{"class":75}," pd.to_datetime(steps[",[35,1124,1115],{"class":45},[35,1126,397],{"class":75},[35,1128,1129],{"class":883},"format",[35,1131,223],{"class":71},[35,1133,1134],{"class":45},"\"ISO8601\"",[35,1136,803],{"class":75},[35,1138,1139],{"class":883},"utc",[35,1141,223],{"class":71},[35,1143,460],{"class":116},[35,1145,191],{"class":75},[35,1147,1148,1151,1153,1156,1159],{"class":37,"line":106},[35,1149,1150],{"class":75},"runs ",[35,1152,223],{"class":71},[35,1154,1155],{"class":75}," (steps.groupby(",[35,1157,1158],{"class":45},"\"run_id\"",[35,1160,191],{"class":75},[35,1162,1163,1166,1169,1171,1173,1175,1177,1180,1182,1184,1186,1188,1191,1193,1196,1198,1201,1203,1205,1208,1210,1212],{"class":37,"line":113},[35,1164,1165],{"class":75},"             .agg(",[35,1167,1168],{"class":883},"started",[35,1170,223],{"class":71},[35,1172,828],{"class":75},[35,1174,1115],{"class":45},[35,1176,803],{"class":75},[35,1178,1179],{"class":45},"\"min\"",[35,1181,547],{"class":75},[35,1183,660],{"class":883},[35,1185,223],{"class":71},[35,1187,828],{"class":75},[35,1189,1190],{"class":45},"\"seconds\"",[35,1192,803],{"class":75},[35,1194,1195],{"class":45},"\"sum\"",[35,1197,547],{"class":75},[35,1199,1200],{"class":883},"ok",[35,1202,223],{"class":71},[35,1204,828],{"class":75},[35,1206,1207],{"class":45},"\"ok\"",[35,1209,803],{"class":75},[35,1211,1179],{"class":45},[35,1213,1057],{"class":75},[35,1215,1216,1219,1222],{"class":37,"line":126},[35,1217,1218],{"class":75},"             .sort_values(",[35,1220,1221],{"class":45},"\"started\"",[35,1223,1057],{"class":75},[35,1225,1226,1229,1232,1234,1236,1239,1241,1243,1246,1249,1252,1255],{"class":37,"line":146},[35,1227,1228],{"class":75},"runs[",[35,1230,1231],{"class":45},"\"minutes\"",[35,1233,809],{"class":75},[35,1235,223],{"class":71},[35,1237,1238],{"class":75}," (runs[",[35,1240,1190],{"class":45},[35,1242,809],{"class":75},[35,1244,1245],{"class":71},"\u002F",[35,1247,1248],{"class":116}," 60",[35,1250,1251],{"class":75},").round(",[35,1253,1254],{"class":116},"1",[35,1256,191],{"class":75},[35,1258,1259],{"class":37,"line":155},[35,1260,110],{"emptyLinePlaceholder":109},[35,1262,1263],{"class":37,"line":161},[35,1264,1265],{"class":506},"# Slow-run flag: more than 3 robust deviations above the median of the previous 20 runs\n",[35,1267,1268,1271,1273,1276,1278,1281,1284,1286,1289,1291,1294],{"class":37,"line":167},[35,1269,1270],{"class":75},"med ",[35,1272,223],{"class":71},[35,1274,1275],{"class":75}," runs[",[35,1277,1190],{"class":45},[35,1279,1280],{"class":75},"].rolling(",[35,1282,1283],{"class":116},"20",[35,1285,803],{"class":75},[35,1287,1288],{"class":883},"min_periods",[35,1290,223],{"class":71},[35,1292,1293],{"class":116},"10",[35,1295,1296],{"class":75},").median().shift()\n",[35,1298,1299,1302,1304,1306,1308,1310,1312,1315,1317,1319,1321,1323,1325],{"class":37,"line":173},[35,1300,1301],{"class":75},"mad ",[35,1303,223],{"class":71},[35,1305,1238],{"class":75},[35,1307,1190],{"class":45},[35,1309,809],{"class":75},[35,1311,383],{"class":71},[35,1313,1314],{"class":75}," med).abs().rolling(",[35,1316,1283],{"class":116},[35,1318,803],{"class":75},[35,1320,1288],{"class":883},[35,1322,223],{"class":71},[35,1324,1293],{"class":116},[35,1326,1296],{"class":75},[35,1328,1329,1331,1334,1336,1338,1340,1342,1344,1347,1350,1353,1356,1359,1362,1364],{"class":37,"line":179},[35,1330,1228],{"class":75},[35,1332,1333],{"class":45},"\"slow\"",[35,1335,809],{"class":75},[35,1337,223],{"class":71},[35,1339,1275],{"class":75},[35,1341,1190],{"class":45},[35,1343,809],{"class":75},[35,1345,1346],{"class":71},">",[35,1348,1349],{"class":75}," med ",[35,1351,1352],{"class":71},"+",[35,1354,1355],{"class":116}," 3",[35,1357,1358],{"class":71}," *",[35,1360,1361],{"class":116}," 1.4826",[35,1363,1358],{"class":71},[35,1365,1366],{"class":75}," mad\n",[35,1368,1369],{"class":37,"line":185},[35,1370,110],{"emptyLinePlaceholder":109},[35,1372,1373,1376,1378,1381,1383,1385,1387,1389,1392,1395,1398,1401],{"class":37,"line":194},[35,1374,1375],{"class":75},"by_step ",[35,1377,223],{"class":71},[35,1379,1380],{"class":75}," steps.pivot_table(",[35,1382,935],{"class":883},[35,1384,223],{"class":71},[35,1386,1112],{"class":75},[35,1388,1115],{"class":45},[35,1390,1391],{"class":75},"].dt.strftime(",[35,1393,1394],{"class":45},"\"%Y-%m\"",[35,1396,1397],{"class":75},").rename(",[35,1399,1400],{"class":45},"\"month\"",[35,1402,1403],{"class":75},"),\n",[35,1405,1406,1409,1411,1414,1416,1419,1421,1423,1425,1428,1430,1433,1435,1437],{"class":37,"line":199},[35,1407,1408],{"class":883},"                            columns",[35,1410,223],{"class":71},[35,1412,1413],{"class":45},"\"step\"",[35,1415,803],{"class":75},[35,1417,1418],{"class":883},"values",[35,1420,223],{"class":71},[35,1422,1190],{"class":45},[35,1424,803],{"class":75},[35,1426,1427],{"class":883},"aggfunc",[35,1429,223],{"class":71},[35,1431,1432],{"class":45},"\"median\"",[35,1434,1251],{"class":75},[35,1436,588],{"class":116},[35,1438,191],{"class":75},[35,1440,1441,1444,1446,1448,1450,1452,1455,1458,1460,1463,1465,1467,1470,1473,1475,1478,1481,1484,1486,1489,1491,1493,1496,1499,1502,1504,1507],{"class":37,"line":211},[35,1442,1443],{"class":116},"print",[35,1445,828],{"class":75},[35,1447,400],{"class":71},[35,1449,403],{"class":45},[35,1451,407],{"class":406},[35,1453,1454],{"class":116},"len",[35,1456,1457],{"class":75},"(runs)",[35,1459,419],{"class":406},[35,1461,1462],{"class":45}," runs, ",[35,1464,407],{"class":406},[35,1466,550],{"class":116},[35,1468,1469],{"class":75},"((runs[",[35,1471,1472],{"class":45},"'ok'",[35,1474,809],{"class":75},[35,1476,1477],{"class":71},"==",[35,1479,1480],{"class":116}," 0",[35,1482,1483],{"class":75},").sum())",[35,1485,419],{"class":406},[35,1487,1488],{"class":45}," failed, ",[35,1490,407],{"class":406},[35,1492,550],{"class":116},[35,1494,1495],{"class":75},"(runs[",[35,1497,1498],{"class":45},"'slow'",[35,1500,1501],{"class":75},"].sum())",[35,1503,419],{"class":406},[35,1505,1506],{"class":45}," flagged slow\"",[35,1508,191],{"class":75},[35,1510,1511,1513,1516,1518,1521,1523,1525,1527],{"class":37,"line":232},[35,1512,1443],{"class":116},[35,1514,1515],{"class":75},"(runs.loc[runs[",[35,1517,1333],{"class":45},[35,1519,1520],{"class":75},"], [",[35,1522,1221],{"class":45},[35,1524,803],{"class":75},[35,1526,1231],{"class":45},[35,1528,1529],{"class":75},"]].to_string())\n",[35,1531,1532,1534],{"class":37,"line":258},[35,1533,1443],{"class":116},[35,1535,1536],{"class":75},"(by_step.to_string())\n",[10,1538,1539,1540,1544],{},"Grouping the step rows by run id gives one row per run: when it started, how long it took in total and whether every step succeeded. The slow-run flag compares each run with the twenty runs before it, using the median and the median absolute deviation rather than the mean and standard deviation, so that one very slow run does not inflate the threshold for the runs after it. The monthly median per step is the most useful table of all: it shows which step is responsible when the total grows. Here ",[1541,1542,1543],"em",{},"extract"," has grown from a median of 117 seconds in June to 328 in October while the other steps are flat, and the one flagged run — 12.1 minutes on 7 September — was a one-off spike in the transform step rather than part of the trend — the data source is growing, and that step is where optimisation effort should go before the job runs into its deadline.",[566,1546,575,1552,575,1555,575,1558,575,1563,575,1569,575,1573,575,1579,575,1583,575,1587,575,1591,575,1595,575,1599,575,1603,575,1607,575,1611,575,1615,575,1618,575,1621,575,1625,575,1627,575,1629,575,1631,575,1634,575,1636,575,1639,575,1641,575,1644,575,1647,575,1651,575,1653,575,1657,575,1659,575,1662,575,1664,575,1667,575,1669,575,1672,575,1674,575,1677,575,1680,575,1683,575,1685,575,1688,575,1690,575,1693,575,1695,575,1697,575,1699,575,1702,575,1704,575,1706,575,1709,575,1712,575,1714,575,1717,575,1719,575,1722,575,1724,575,1727,575,1729,575,1731,575,1733,575,1735,575,1738,575,1742,575,1744,575,1747,575,1749,575,1752,575,1754,575,1757,575,1759,575,1761,575,1763,575,1765,575,1768,575,1771,575,1773,575,1776,575,1778,575,1782,575,1784,575,1786,575,1788,575,1791,575,1793],{"viewBox":1547,"role":569,"ariaLabelledBy":1548,"xmlns":573,"style":1551},"144 0 471 224",[1549,1550],"rh-steps-t","rh-steps-d","width:100%;max-width:471px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[577,1553,1554],{"id":1549},"Median step duration by month (seconds)",[581,1556,1557],{"id":1550},"Extract grows steadily from 117 seconds in June to 328 in October, while transform, write workbook and send stay roughly constant. The growth of one step accounts for almost all the increase in total run time.",[585,1559],{"x":1560,"y":588,"width":1561,"height":1562,"fill":591},"144","471","224",[585,1564],{"x":1565,"y":595,"width":1566,"height":1567,"rx":588,"fill":598,"stroke":599,"style":1568},"160.7","32.0","24.0","stroke-width:1px",[585,1570],{"x":1571,"y":595,"width":1572,"height":1567,"rx":588,"fill":598,"stroke":599,"style":1568},"192.7","79.1",[602,1574,1578],{"x":1575,"y":1576,"style":1577},"232.2","33.0","font-size:11px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","A",[585,1580],{"x":1581,"y":595,"width":1582,"height":1567,"rx":588,"fill":598,"stroke":599,"style":1568},"271.8","67.9",[602,1584,1586],{"x":1585,"y":1576,"style":1577},"305.8","B",[585,1588],{"x":1589,"y":595,"width":1590,"height":1567,"rx":588,"fill":598,"stroke":599,"style":1568},"339.7","85.3",[602,1592,1594],{"x":1593,"y":1576,"style":1577},"382.4","C",[585,1596],{"x":1597,"y":595,"width":1598,"height":1567,"rx":588,"fill":598,"stroke":599,"style":1568},"425.0","119.9",[602,1600,1602],{"x":1601,"y":1576,"style":1577},"485.0","D",[585,1604],{"x":1605,"y":595,"width":1606,"height":1567,"rx":588,"fill":598,"stroke":599,"style":1568},"545.0","54.4",[602,1608,1610],{"x":1609,"y":1576,"style":1577},"572.1","E",[585,1612],{"x":1565,"y":1613,"width":1566,"height":1614,"rx":588,"fill":598,"stroke":599,"style":1568},"40.0","28.0",[602,1616,1254],{"x":1617,"y":609,"style":1577},"176.7",[585,1619],{"x":1571,"y":1613,"width":1572,"height":1614,"rx":588,"fill":630,"stroke":621,"style":1620},"stroke-width:1.5px",[602,1622,1624],{"x":1575,"y":609,"style":1623},"font-size:12px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Month",[585,1626],{"x":1581,"y":1613,"width":1582,"height":1614,"rx":588,"fill":630,"stroke":621,"style":1620},[602,1628,1543],{"x":1585,"y":609,"style":1623},[585,1630],{"x":1589,"y":1613,"width":1590,"height":1614,"rx":588,"fill":630,"stroke":621,"style":1620},[602,1632,1633],{"x":1593,"y":609,"style":1623},"transform",[585,1635],{"x":1597,"y":1613,"width":1598,"height":1614,"rx":588,"fill":630,"stroke":621,"style":1620},[602,1637,1638],{"x":1601,"y":609,"style":1623},"write workbook",[585,1640],{"x":1605,"y":1613,"width":1606,"height":1614,"rx":588,"fill":630,"stroke":621,"style":1620},[602,1642,1643],{"x":1609,"y":609,"style":1623},"send",[585,1645],{"x":1565,"y":1646,"width":1566,"height":1614,"rx":588,"fill":598,"stroke":599,"style":1568},"68.0",[602,1648,1650],{"x":1617,"y":1649,"style":1577},"87.0","2",[585,1652],{"x":1571,"y":1646,"width":1572,"height":1614,"rx":588,"fill":591,"stroke":599,"style":1568},[602,1654,1656],{"x":1575,"y":1649,"style":1655},"font-size:12px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","2026-06",[585,1658],{"x":1581,"y":1646,"width":1582,"height":1614,"rx":588,"fill":591,"stroke":599,"style":1568},[602,1660,1661],{"x":1585,"y":1649,"style":1655},"117",[585,1663],{"x":1589,"y":1646,"width":1590,"height":1614,"rx":588,"fill":591,"stroke":599,"style":1568},[602,1665,1666],{"x":1593,"y":1649,"style":1655},"41",[585,1668],{"x":1597,"y":1646,"width":1598,"height":1614,"rx":588,"fill":591,"stroke":599,"style":1568},[602,1670,1671],{"x":1601,"y":1649,"style":1655},"57",[585,1673],{"x":1605,"y":1646,"width":1606,"height":1614,"rx":588,"fill":591,"stroke":599,"style":1568},[602,1675,1676],{"x":1609,"y":1649,"style":1655},"6",[585,1678],{"x":1565,"y":1679,"width":1566,"height":1614,"rx":588,"fill":598,"stroke":599,"style":1568},"96.0",[602,1681,544],{"x":1617,"y":1682,"style":1577},"115.0",[585,1684],{"x":1571,"y":1679,"width":1572,"height":1614,"rx":588,"fill":591,"stroke":599,"style":1568},[602,1686,1687],{"x":1575,"y":1682,"style":1655},"2026-07",[585,1689],{"x":1581,"y":1679,"width":1582,"height":1614,"rx":588,"fill":591,"stroke":599,"style":1568},[602,1691,1692],{"x":1585,"y":1682,"style":1655},"180",[585,1694],{"x":1589,"y":1679,"width":1590,"height":1614,"rx":588,"fill":591,"stroke":599,"style":1568},[602,1696,1666],{"x":1593,"y":1682,"style":1655},[585,1698],{"x":1597,"y":1679,"width":1598,"height":1614,"rx":588,"fill":591,"stroke":599,"style":1568},[602,1700,1701],{"x":1601,"y":1682,"style":1655},"56",[585,1703],{"x":1605,"y":1679,"width":1606,"height":1614,"rx":588,"fill":591,"stroke":599,"style":1568},[602,1705,1676],{"x":1609,"y":1682,"style":1655},[585,1707],{"x":1565,"y":1708,"width":1566,"height":1614,"rx":588,"fill":598,"stroke":599,"style":1568},"124.0",[602,1710,587],{"x":1617,"y":1711,"style":1577},"143.0",[585,1713],{"x":1571,"y":1708,"width":1572,"height":1614,"rx":588,"fill":591,"stroke":599,"style":1568},[602,1715,1716],{"x":1575,"y":1711,"style":1655},"2026-08",[585,1718],{"x":1581,"y":1708,"width":1582,"height":1614,"rx":588,"fill":591,"stroke":599,"style":1568},[602,1720,1721],{"x":1585,"y":1711,"style":1655},"235",[585,1723],{"x":1589,"y":1708,"width":1590,"height":1614,"rx":588,"fill":591,"stroke":599,"style":1568},[602,1725,1726],{"x":1593,"y":1711,"style":1655},"43",[585,1728],{"x":1597,"y":1708,"width":1598,"height":1614,"rx":588,"fill":591,"stroke":599,"style":1568},[602,1730,1701],{"x":1601,"y":1711,"style":1655},[585,1732],{"x":1605,"y":1708,"width":1606,"height":1614,"rx":588,"fill":591,"stroke":599,"style":1568},[602,1734,1676],{"x":1609,"y":1711,"style":1655},[585,1736],{"x":1565,"y":1737,"width":1566,"height":1614,"rx":588,"fill":598,"stroke":599,"style":1568},"152.0",[602,1739,1741],{"x":1617,"y":1740,"style":1577},"171.0","5",[585,1743],{"x":1571,"y":1737,"width":1572,"height":1614,"rx":588,"fill":591,"stroke":599,"style":1568},[602,1745,1746],{"x":1575,"y":1740,"style":1655},"2026-09",[585,1748],{"x":1581,"y":1737,"width":1582,"height":1614,"rx":588,"fill":591,"stroke":599,"style":1568},[602,1750,1751],{"x":1585,"y":1740,"style":1655},"298",[585,1753],{"x":1589,"y":1737,"width":1590,"height":1614,"rx":588,"fill":591,"stroke":599,"style":1568},[602,1755,1756],{"x":1593,"y":1740,"style":1655},"42",[585,1758],{"x":1597,"y":1737,"width":1598,"height":1614,"rx":588,"fill":591,"stroke":599,"style":1568},[602,1760,1671],{"x":1601,"y":1740,"style":1655},[585,1762],{"x":1605,"y":1737,"width":1606,"height":1614,"rx":588,"fill":591,"stroke":599,"style":1568},[602,1764,1676],{"x":1609,"y":1740,"style":1655},[585,1766],{"x":1565,"y":1767,"width":1566,"height":1614,"rx":588,"fill":598,"stroke":599,"style":1568},"180.0",[602,1769,1676],{"x":1617,"y":1770,"style":1577},"199.0",[585,1772],{"x":1571,"y":1767,"width":1572,"height":1614,"rx":588,"fill":591,"stroke":599,"style":1568},[602,1774,1775],{"x":1575,"y":1770,"style":1655},"2026-10",[585,1777],{"x":1581,"y":1767,"width":1582,"height":1614,"rx":588,"fill":693,"stroke":694,"style":1620},[602,1779,1781],{"x":1585,"y":1770,"style":1780},"font-size:12px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","328",[585,1783],{"x":1589,"y":1767,"width":1590,"height":1614,"rx":588,"fill":591,"stroke":599,"style":1568},[602,1785,1756],{"x":1593,"y":1770,"style":1655},[585,1787],{"x":1597,"y":1767,"width":1598,"height":1614,"rx":588,"fill":591,"stroke":599,"style":1568},[602,1789,1790],{"x":1601,"y":1770,"style":1655},"54",[585,1792],{"x":1605,"y":1767,"width":1606,"height":1614,"rx":588,"fill":591,"stroke":599,"style":1568},[602,1794,1676],{"x":1609,"y":1770,"style":1655},[20,1796,1798],{"id":1797},"step-4-the-run-history-workbook","Step 4 — The run-history workbook",[25,1800,1802],{"className":62,"code":1801,"language":64,"meta":30,"style":30},"from openpyxl import load_workbook\nfrom openpyxl.chart import LineChart, Reference\nfrom openpyxl.styles import PatternFill\n\nout = \"daily-sales_run_history.xlsx\"\nsheet_runs = runs.assign(started=runs[\"started\"].dt.tz_convert(None),\n                         status=runs[\"ok\"].map({1: \"ok\", 0: \"FAILED\"}))[[\"started\", \"minutes\", \"status\", \"slow\"]]\nfailures = steps.loc[steps[\"ok\"] == 0, [\"run_id\", \"started_utc\", \"step\", \"error\"]]\nfailures = failures.assign(started_utc=failures[\"started_utc\"].dt.tz_convert(None))\nwith pd.ExcelWriter(out, engine=\"openpyxl\") as xw:\n    sheet_runs.to_excel(xw, sheet_name=\"Runs\")\n    by_step.to_excel(xw, sheet_name=\"Median by step\")\n    failures.to_excel(xw, sheet_name=\"Failures\", index=False)\n\nwb = load_workbook(out)\nws = wb[\"Runs\"]\nred = PatternFill(\"solid\", fgColor=\"F8CBAD\")\nfor row in ws.iter_rows(min_row=2):\n    if row[3].value == \"FAILED\" or row[4].value is True:\n        for cell in row:\n            cell.fill = red\nchart = LineChart()\nchart.title, chart.height, chart.width = \"Run time (minutes)\", 7, 18\nchart.add_data(Reference(ws, min_col=3, min_row=1, max_row=ws.max_row), titles_from_data=True)\nchart.set_categories(Reference(ws, min_col=2, min_row=2, max_row=ws.max_row))\nchart.x_axis.number_format, chart.x_axis.delete, chart.y_axis.delete = \"dd mmm\", False, False\nws.add_chart(chart, \"G2\")\nwb.save(out)\nprint(wb.sheetnames, \"| failures:\", len(failures), \"| latest run:\", sheet_runs[\"minutes\"].iloc[-1], \"min\")\n",[32,1803,1804,1816,1828,1840,1844,1854,1879,1929,1967,1992,2014,2029,2043,2065,2069,2079,2093,2118,2141,2176,2189,2199,2209,2229,2268,2296,2315,2325,2330],{"__ignoreMap":30},[35,1805,1806,1808,1811,1813],{"class":37,"line":38},[35,1807,82],{"class":71},[35,1809,1810],{"class":75}," openpyxl ",[35,1812,72],{"class":71},[35,1814,1815],{"class":75}," load_workbook\n",[35,1817,1818,1820,1823,1825],{"class":37,"line":79},[35,1819,82],{"class":71},[35,1821,1822],{"class":75}," openpyxl.chart ",[35,1824,72],{"class":71},[35,1826,1827],{"class":75}," LineChart, Reference\n",[35,1829,1830,1832,1835,1837],{"class":37,"line":93},[35,1831,82],{"class":71},[35,1833,1834],{"class":75}," openpyxl.styles ",[35,1836,72],{"class":71},[35,1838,1839],{"class":75}," PatternFill\n",[35,1841,1842],{"class":37,"line":106},[35,1843,110],{"emptyLinePlaceholder":109},[35,1845,1846,1849,1851],{"class":37,"line":113},[35,1847,1848],{"class":75},"out ",[35,1850,223],{"class":71},[35,1852,1853],{"class":45}," \"daily-sales_run_history.xlsx\"\n",[35,1855,1856,1859,1861,1864,1866,1868,1870,1872,1875,1877],{"class":37,"line":126},[35,1857,1858],{"class":75},"sheet_runs ",[35,1860,223],{"class":71},[35,1862,1863],{"class":75}," runs.assign(",[35,1865,1168],{"class":883},[35,1867,223],{"class":71},[35,1869,1228],{"class":75},[35,1871,1221],{"class":45},[35,1873,1874],{"class":75},"].dt.tz_convert(",[35,1876,226],{"class":116},[35,1878,1403],{"class":75},[35,1880,1881,1884,1886,1888,1890,1893,1895,1897,1899,1901,1903,1905,1908,1911,1913,1915,1917,1919,1922,1924,1926],{"class":37,"line":146},[35,1882,1883],{"class":883},"                         status",[35,1885,223],{"class":71},[35,1887,1228],{"class":75},[35,1889,1207],{"class":45},[35,1891,1892],{"class":75},"].map({",[35,1894,1254],{"class":116},[35,1896,334],{"class":75},[35,1898,1207],{"class":45},[35,1900,803],{"class":75},[35,1902,588],{"class":116},[35,1904,334],{"class":75},[35,1906,1907],{"class":45},"\"FAILED\"",[35,1909,1910],{"class":75},"}))[[",[35,1912,1221],{"class":45},[35,1914,803],{"class":75},[35,1916,1231],{"class":45},[35,1918,803],{"class":75},[35,1920,1921],{"class":45},"\"status\"",[35,1923,803],{"class":75},[35,1925,1333],{"class":45},[35,1927,1928],{"class":75},"]]\n",[35,1930,1931,1934,1936,1939,1941,1943,1945,1947,1950,1952,1954,1956,1958,1960,1962,1965],{"class":37,"line":155},[35,1932,1933],{"class":75},"failures ",[35,1935,223],{"class":71},[35,1937,1938],{"class":75}," steps.loc[steps[",[35,1940,1207],{"class":45},[35,1942,809],{"class":75},[35,1944,1477],{"class":71},[35,1946,1480],{"class":116},[35,1948,1949],{"class":75},", [",[35,1951,1158],{"class":45},[35,1953,803],{"class":75},[35,1955,1115],{"class":45},[35,1957,803],{"class":75},[35,1959,1413],{"class":45},[35,1961,803],{"class":75},[35,1963,1964],{"class":45},"\"error\"",[35,1966,1928],{"class":75},[35,1968,1969,1971,1973,1976,1979,1981,1984,1986,1988,1990],{"class":37,"line":161},[35,1970,1933],{"class":75},[35,1972,223],{"class":71},[35,1974,1975],{"class":75}," failures.assign(",[35,1977,1978],{"class":883},"started_utc",[35,1980,223],{"class":71},[35,1982,1983],{"class":75},"failures[",[35,1985,1115],{"class":45},[35,1987,1874],{"class":75},[35,1989,226],{"class":116},[35,1991,1057],{"class":75},[35,1993,1994,1996,1999,2002,2004,2007,2009,2011],{"class":37,"line":167},[35,1995,129],{"class":71},[35,1997,1998],{"class":75}," pd.ExcelWriter(out, ",[35,2000,2001],{"class":883},"engine",[35,2003,223],{"class":71},[35,2005,2006],{"class":45},"\"openpyxl\"",[35,2008,776],{"class":75},[35,2010,140],{"class":71},[35,2012,2013],{"class":75}," xw:\n",[35,2015,2016,2019,2022,2024,2027],{"class":37,"line":173},[35,2017,2018],{"class":75},"    sheet_runs.to_excel(xw, ",[35,2020,2021],{"class":883},"sheet_name",[35,2023,223],{"class":71},[35,2025,2026],{"class":45},"\"Runs\"",[35,2028,191],{"class":75},[35,2030,2031,2034,2036,2038,2041],{"class":37,"line":179},[35,2032,2033],{"class":75},"    by_step.to_excel(xw, ",[35,2035,2021],{"class":883},[35,2037,223],{"class":71},[35,2039,2040],{"class":45},"\"Median by step\"",[35,2042,191],{"class":75},[35,2044,2045,2048,2050,2052,2055,2057,2059,2061,2063],{"class":37,"line":185},[35,2046,2047],{"class":75},"    failures.to_excel(xw, ",[35,2049,2021],{"class":883},[35,2051,223],{"class":71},[35,2053,2054],{"class":45},"\"Failures\"",[35,2056,803],{"class":75},[35,2058,935],{"class":883},[35,2060,223],{"class":71},[35,2062,389],{"class":116},[35,2064,191],{"class":75},[35,2066,2067],{"class":37,"line":194},[35,2068,110],{"emptyLinePlaceholder":109},[35,2070,2071,2074,2076],{"class":37,"line":199},[35,2072,2073],{"class":75},"wb ",[35,2075,223],{"class":71},[35,2077,2078],{"class":75}," load_workbook(out)\n",[35,2080,2081,2084,2086,2089,2091],{"class":37,"line":211},[35,2082,2083],{"class":75},"ws ",[35,2085,223],{"class":71},[35,2087,2088],{"class":75}," wb[",[35,2090,2026],{"class":45},[35,2092,255],{"class":75},[35,2094,2095,2098,2100,2103,2106,2108,2111,2113,2116],{"class":37,"line":232},[35,2096,2097],{"class":75},"red ",[35,2099,223],{"class":71},[35,2101,2102],{"class":75}," PatternFill(",[35,2104,2105],{"class":45},"\"solid\"",[35,2107,803],{"class":75},[35,2109,2110],{"class":883},"fgColor",[35,2112,223],{"class":71},[35,2114,2115],{"class":45},"\"F8CBAD\"",[35,2117,191],{"class":75},[35,2119,2120,2123,2126,2129,2132,2135,2137,2139],{"class":37,"line":258},[35,2121,2122],{"class":71},"for",[35,2124,2125],{"class":75}," row ",[35,2127,2128],{"class":71},"in",[35,2130,2131],{"class":75}," ws.iter_rows(",[35,2133,2134],{"class":883},"min_row",[35,2136,223],{"class":71},[35,2138,1650],{"class":116},[35,2140,229],{"class":75},[35,2142,2143,2146,2149,2151,2154,2156,2159,2162,2164,2166,2168,2171,2174],{"class":37,"line":283},[35,2144,2145],{"class":71},"    if",[35,2147,2148],{"class":75}," row[",[35,2150,544],{"class":116},[35,2152,2153],{"class":75},"].value ",[35,2155,1477],{"class":71},[35,2157,2158],{"class":45}," \"FAILED\"",[35,2160,2161],{"class":71}," or",[35,2163,2148],{"class":75},[35,2165,587],{"class":116},[35,2167,2153],{"class":75},[35,2169,2170],{"class":71},"is",[35,2172,2173],{"class":116}," True",[35,2175,208],{"class":75},[35,2177,2178,2181,2184,2186],{"class":37,"line":288},[35,2179,2180],{"class":71},"        for",[35,2182,2183],{"class":75}," cell ",[35,2185,2128],{"class":71},[35,2187,2188],{"class":75}," row:\n",[35,2190,2191,2194,2196],{"class":37,"line":295},[35,2192,2193],{"class":75},"            cell.fill ",[35,2195,223],{"class":71},[35,2197,2198],{"class":75}," red\n",[35,2200,2201,2204,2206],{"class":37,"line":306},[35,2202,2203],{"class":75},"chart ",[35,2205,223],{"class":71},[35,2207,2208],{"class":75}," LineChart()\n",[35,2210,2211,2214,2216,2219,2221,2224,2226],{"class":37,"line":320},[35,2212,2213],{"class":75},"chart.title, chart.height, chart.width ",[35,2215,223],{"class":71},[35,2217,2218],{"class":45}," \"Run time (minutes)\"",[35,2220,803],{"class":75},[35,2222,2223],{"class":116},"7",[35,2225,803],{"class":75},[35,2227,2228],{"class":116},"18\n",[35,2230,2231,2234,2237,2239,2241,2243,2245,2247,2249,2251,2254,2256,2259,2262,2264,2266],{"class":37,"line":342},[35,2232,2233],{"class":75},"chart.add_data(Reference(ws, ",[35,2235,2236],{"class":883},"min_col",[35,2238,223],{"class":71},[35,2240,544],{"class":116},[35,2242,803],{"class":75},[35,2244,2134],{"class":883},[35,2246,223],{"class":71},[35,2248,1254],{"class":116},[35,2250,803],{"class":75},[35,2252,2253],{"class":883},"max_row",[35,2255,223],{"class":71},[35,2257,2258],{"class":75},"ws.max_row), ",[35,2260,2261],{"class":883},"titles_from_data",[35,2263,223],{"class":71},[35,2265,460],{"class":116},[35,2267,191],{"class":75},[35,2269,2270,2273,2275,2277,2279,2281,2283,2285,2287,2289,2291,2293],{"class":37,"line":350},[35,2271,2272],{"class":75},"chart.set_categories(Reference(ws, ",[35,2274,2236],{"class":883},[35,2276,223],{"class":71},[35,2278,1650],{"class":116},[35,2280,803],{"class":75},[35,2282,2134],{"class":883},[35,2284,223],{"class":71},[35,2286,1650],{"class":116},[35,2288,803],{"class":75},[35,2290,2253],{"class":883},[35,2292,223],{"class":71},[35,2294,2295],{"class":75},"ws.max_row))\n",[35,2297,2298,2301,2303,2306,2308,2310,2312],{"class":37,"line":359},[35,2299,2300],{"class":75},"chart.x_axis.number_format, chart.x_axis.delete, chart.y_axis.delete ",[35,2302,223],{"class":71},[35,2304,2305],{"class":45}," \"dd mmm\"",[35,2307,803],{"class":75},[35,2309,389],{"class":116},[35,2311,803],{"class":75},[35,2313,2314],{"class":116},"False\n",[35,2316,2317,2320,2323],{"class":37,"line":374},[35,2318,2319],{"class":75},"ws.add_chart(chart, ",[35,2321,2322],{"class":45},"\"G2\"",[35,2324,191],{"class":75},[35,2326,2327],{"class":37,"line":435},[35,2328,2329],{"class":75},"wb.save(out)\n",[35,2331,2332,2334,2337,2340,2342,2344,2347,2350,2353,2355,2358,2360,2362,2364,2366],{"class":37,"line":441},[35,2333,1443],{"class":116},[35,2335,2336],{"class":75},"(wb.sheetnames, ",[35,2338,2339],{"class":45},"\"| failures:\"",[35,2341,803],{"class":75},[35,2343,1454],{"class":116},[35,2345,2346],{"class":75},"(failures), ",[35,2348,2349],{"class":45},"\"| latest run:\"",[35,2351,2352],{"class":75},", sheet_runs[",[35,2354,1231],{"class":45},[35,2356,2357],{"class":75},"].iloc[",[35,2359,383],{"class":71},[35,2361,1254],{"class":116},[35,2363,397],{"class":75},[35,2365,1179],{"class":45},[35,2367,191],{"class":75},[10,2369,2370,2371,2374,2375,2378,2379,2382],{},"The workbook has three sheets. ",[1541,2372,2373],{},"Runs"," lists every run with its start time, duration, status and slow flag, with failed and slow runs highlighted, and a line chart of run time beside it — the upward slope is obvious at a glance in a way that a list of numbers is not. ",[1541,2376,2377],{},"Median by step"," is the monthly table from step 3. ",[1541,2380,2381],{},"Failures"," lists each failed step with its error message, which is usually enough to see whether failures are random (a mail server timing out now and then) or have a pattern (every Monday, every month end). Time zones are removed before writing because Excel cannot store them; the times stay in UTC, which the column header should say if readers are in other zones.",[20,2384,2386],{"id":2385},"what-to-do-with-the-history","What to do with the history",[10,2388,2389],{},"The history pays for itself in three ways. First, it predicts trouble: if the run time grows by a fixed amount each week, a simple straight line fitted to recent runs says when the job will cross its deadline, which is a much better conversation to have in July than on the morning in October when the report is late. Second, it points at the cause: the step table shows which step grew, and the row counts recorded with it show whether the growth is explained by more data or is a genuine slowdown. Third, it gives honest answers to questions like \"how reliable is this job?\" — three failed runs out of ninety is a reliability of about 97%, all three in the send step, which tells you to look at the mail server, not at the report code.",[10,2391,2392],{},"Keep the database small: a row per step per run is a few hundred bytes, so even several jobs running daily for years stay well under a few megabytes. Delete rows older than a year or two if you want, but the long history is often the most interesting part.",[566,2394,575,2400,575,2403,575,2406,575,2411,575,2415,575,2418,575,2422,575,2426,575,2429,575,2432,575,2436,575,2440,575,2444,575,2447,575,2450,575,2454,575,2458,575,2462,575,2465,575,2468,575,2472],{"viewBox":2395,"role":569,"ariaLabelledBy":2396,"xmlns":573,"style":2399},"84 0 592 376",[2397,2398],"rh-uses-t","rh-uses-d","width:100%;max-width:592px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[577,2401,2402],{"id":2397},"What the run history answers",[581,2404,2405],{"id":2398},"The history answers when the job will miss its deadline from the trend of run times, which step is responsible from the step medians, whether more data explains it from the row counts, and how reliable the job is from the failure rate.",[585,2407],{"x":2408,"y":588,"width":2409,"height":2410,"fill":591},"84","592","376",[585,2412],{"x":2413,"y":595,"width":2414,"height":609,"rx":252,"fill":693,"stroke":694,"style":600},"100.0","560.0",[602,2416,2417],{"x":655,"y":605,"style":698},"When will it miss the deadline?",[602,2419,2421],{"x":655,"y":2420,"style":614},"61.0","run time trend",[37,2423],{"x1":655,"y1":2424,"x2":655,"y2":2425,"stroke":621,"style":600},"80.0","102.0",[623,2427],{"points":2428,"fill":626},"380.0,106.0 375.0,97.0 385.0,97.0",[585,2430],{"x":2413,"y":2431,"width":2414,"height":609,"rx":252,"fill":651,"stroke":652,"style":600},"111.0",[602,2433,2435],{"x":655,"y":2434,"style":656},"136.5","Which step is slowing?",[602,2437,2439],{"x":655,"y":2438,"style":614},"156.0","median by step",[37,2441],{"x1":655,"y1":2442,"x2":655,"y2":2443,"stroke":621,"style":600},"175.0","197.0",[623,2445],{"points":2446,"fill":626},"380.0,201.0 375.0,192.0 385.0,192.0",[585,2448],{"x":2413,"y":2449,"width":2414,"height":609,"rx":252,"fill":630,"stroke":621,"style":600},"206.0",[602,2451,2453],{"x":655,"y":2452,"style":635},"231.5","More data or slower code?",[602,2455,2457],{"x":655,"y":2456,"style":614},"251.0","rows per step",[37,2459],{"x1":655,"y1":2460,"x2":655,"y2":2461,"stroke":621,"style":600},"270.0","292.0",[623,2463],{"points":2464,"fill":626},"380.0,296.0 375.0,287.0 385.0,287.0",[585,2466],{"x":2413,"y":2467,"width":2414,"height":609,"rx":252,"fill":673,"stroke":674,"style":600},"301.0",[602,2469,2471],{"x":655,"y":2470,"style":678},"326.5","How reliable is it?",[602,2473,2475],{"x":655,"y":2474,"style":614},"346.0","failures by step",[20,2477,2479],{"id":2478},"common-pitfalls","Common pitfalls",[2481,2482,2483,2497,2506,2512,2518],"ul",{},[2484,2485,2486,2493,2494,2496],"li",{},[2487,2488,2489,2490,18],"strong",{},"Timing with ",[32,2491,2492],{},"time.time()"," Use ",[32,2495,563],{},"; the wall clock can jump.",[2484,2498,2499,2502,2503,2505],{},[2487,2500,2501],{},"Losing the record of failed steps."," Write the row in the ",[32,2504,1010],{}," branch, then re-raise.",[2484,2507,2508,2511],{},[2487,2509,2510],{},"Averages for the slow flag."," One huge run distorts a mean; use median and MAD.",[2484,2513,2514,2517],{},[2487,2515,2516],{},"Timing every function."," Time the four to eight steps a person would name.",[2484,2519,2520,2523,2524,2527,2528,18],{},[2487,2521,2522],{},"Unclosed connections."," ",[32,2525,2526],{},"with sqlite3.connect()"," commits but does not close; wrap it in ",[32,2529,2530],{},"closing()",[20,2532,2534],{"id":2533},"conclusion","Conclusion",[10,2536,2537,2538,2540],{},"To understand how a report job behaves over time, wrap each step in a context manager that measures it with ",[32,2539,563],{}," and writes one row to SQLite, recording failures before re-raising them. Group the rows into runs, flag runs far above the recent median, take monthly medians per step to see which part is growing, and write a run-history workbook with a duration chart and a list of failures.",[20,2542,2544],{"id":2543},"frequently-asked-questions","Frequently asked questions",[10,2546,2547,2550,2551,2554],{},[2487,2548,2549],{},"How do I time each step of a Python script?","\nWrap each step in a context manager that records ",[32,2552,2553],{},"time.perf_counter()"," before and after and stores the difference, including when the step raises.",[10,2556,2557,2560],{},[2487,2558,2559],{},"Where should I store job run history?","\nA small SQLite file next to the job is enough: it needs no server, survives crashes because each row is committed, and pandas can read it directly.",[10,2562,2563,2566],{},[2487,2564,2565],{},"How do I detect a run that was unusually slow?","\nCompare its duration with the median of recent runs and flag it when it exceeds the median by more than about three robust standard deviations (1.4826 × MAD).",[10,2568,2569,2572,2573,2576],{},[2487,2570,2571],{},"Why does sqlite3's with statement not close the connection?","\nThe connection's context manager commits or rolls back the transaction but leaves the connection open; use ",[32,2574,2575],{},"contextlib.closing"," to close it.",[20,2578,2580],{"id":2579},"related","Related",[2481,2582,2583,2589,2596,2603],{},[2484,2584,2585,2586,2588],{},"Up one level: ",[14,2587,17],{"href":16}," — robust report jobs.",[2484,2590,2591,2595],{},[14,2592,2594],{"href":2593},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Fdetect-a-scheduled-report-that-did-not-run\u002F","Detect a Scheduled Report That Did Not Run"," — when there is no run at all.",[2484,2597,2598,2602],{},[14,2599,2601],{"href":2600},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Fretry-a-failed-excel-report-job-in-python\u002F","Retry a Failed Excel Report Job in Python"," — retrying failed steps.",[2484,2604,2605,2609],{},[14,2606,2608],{"href":2607},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Fsend-an-alert-when-an-excel-report-job-fails\u002F","Send an Alert When an Excel Report Job Fails"," — alerting on failures.",[2611,2612,2613],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}",{"title":30,"searchDepth":79,"depth":79,"links":2615},[2616,2617,2618,2619,2620,2621,2622,2623,2624,2625],{"id":22,"depth":79,"text":23},{"id":58,"depth":79,"text":59},{"id":708,"depth":79,"text":709},{"id":1071,"depth":79,"text":1072},{"id":1797,"depth":79,"text":1798},{"id":2385,"depth":79,"text":2386},{"id":2478,"depth":79,"text":2479},{"id":2533,"depth":79,"text":2534},{"id":2543,"depth":79,"text":2544},{"id":2579,"depth":79,"text":2580},"2026-10-02","Record every run of a scheduled Excel report job and how long each step took in a small SQLite table, using a context manager around each step; then turn the history into a run-history workbook that shows failures, duration trends and the step that is slowing down, and flags runs much slower than usual before they hit a deadline.","md",[2630,2632,2634,2636],{"q":2549,"a":2631},"Wrap each step in a context manager that records time.perf_counter() before and after and stores the difference, including when the step raises.",{"q":2559,"a":2633},"A small SQLite file next to the job is enough: it needs no server, survives crashes because each row is committed, and pandas can read it directly.",{"q":2565,"a":2635},"Compare its duration with the median of recent runs and flag it when it exceeds the median by more than about three robust standard deviations (1.4826 × MAD).",{"q":2571,"a":2637},"The connection's context manager commits or rolls back the transaction but leaves the connection open; use contextlib.closing to close it.","report job run history step timings",null,{"breadcrumb":2641},[2642,2644,2647],{"name":2643,"item":1245},"Home",{"name":2645,"item":2646},"Automating Reporting Workflows with Python and Excel","\u002Fautomating-reporting-workflows\u002F",{"name":17,"item":16},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Ftrack-run-history-and-step-timings-of-a-report-job",{"title":2650,"description":2651},"Track Run History and Step Timings of a Python Report Job","Time each step of an Excel report job with a context manager, store runs in SQLite, and build a run-history workbook with failure rates, duration trends and slow-run flags.","track-run-history-and-step-timings-of-a-report-job","automating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Ftrack-run-history-and-step-timings-of-a-report-job\u002Findex",[2655],"reliable-jobs","how-to","loNbG_TCH0lbK_dedUPXgb3otoEiQEkLIfLKmzB53Nk",[2659,2663],{"title":2660,"path":2661,"stem":2662,"children":-1},"Skip a Report Run When the Input File Has Not Changed","\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Fskip-a-report-when-the-input-file-has-not-changed","automating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Fskip-a-report-when-the-input-file-has-not-changed\u002Findex",{"title":2664,"path":2665,"stem":2666,"children":-1},"Validate an Excel Report Before Sending It","\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Fvalidate-an-excel-report-before-sending-it","automating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Fvalidate-an-excel-report-before-sending-it\u002Findex",[2668,2670],{"path":2669,"title":17},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation",{"path":2671,"title":2645},"\u002Fautomating-reporting-workflows",[],[2674,2678,2681,2684,2687,2690],{"path":2675,"title":2676,"kind":2677},"\u002Fautomating-reporting-workflows\u002Ftesting-and-packaging-excel-automation-scripts","Testing and Packaging Excel Automation Scripts","guide",{"path":2679,"title":2680,"kind":2656},"\u002Fautomating-reporting-workflows\u002Ftesting-and-packaging-excel-automation-scripts\u002Fblock-large-or-sensitive-excel-files-with-a-pre-commit-hook","Block Large or Sensitive Excel Files with a Pre-commit Hook",{"path":2682,"title":2683,"kind":2656},"\u002Fautomating-reporting-workflows\u002Ftesting-and-packaging-excel-automation-scripts\u002Fbuild-a-command-line-tool-for-excel-reports-with-argparse","Build a Command-Line Tool for Excel Reports with argparse",{"path":2685,"title":2686,"kind":2656},"\u002Fautomating-reporting-workflows\u002Ftesting-and-packaging-excel-automation-scripts\u002Fcompare-excel-output-to-a-golden-file-in-tests","Compare Excel Output to a Golden File in Tests",{"path":2688,"title":2689,"kind":2656},"\u002Fautomating-reporting-workflows\u002Ftesting-and-packaging-excel-automation-scripts\u002Fcreate-excel-test-fixtures-in-memory","Create Excel Test Fixtures in Memory for pytest",{"path":2691,"title":2692,"kind":2656},"\u002Fautomating-reporting-workflows\u002Ftesting-and-packaging-excel-automation-scripts\u002Fdiff-excel-files-in-git-with-a-textconv-driver","Diff Excel Files in Git with a textconv Driver",1790972132244]