[{"data":1,"prerenderedAt":2237},["ShallowReactive",2],{"doc:\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Flog-python-excel-script-output-to-a-file":3,"surround:\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Flog-python-excel-script-output-to-a-file":2230},{"id":4,"title":5,"body":6,"dateModified":2200,"datePublished":2200,"description":2201,"extension":2202,"faq":2203,"meta":2213,"navigation":238,"path":2223,"seo":2224,"slug":2226,"stem":2227,"type":2228,"__hash__":2229},"docs\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Flog-python-excel-script-output-to-a-file\u002Findex.md","Log Python Excel Script Output to a File",{"type":7,"value":8,"toc":2185},"minimark",[9,27,150,155,158,185,189,591,610,614,617,964,974,978,1064,1079,1173,1177,1180,1285,1298,1302,1305,1472,1483,1487,1490,1554,1619,1622,1626,1752,1756,1759,2025,2036,2040,2056,2060,2079,2083,2095,2105,2120,2132,2136,2139,2148,2151,2181],[10,11,12,16,17,20,21,26],"p",{},[13,14,15],"code",{},"print()"," is fine while you are writing a script and useless the moment it runs unattended: no timestamps, no severity, and output that disappears wherever the scheduler sends it. Python's ",[13,18,19],{},"logging"," module fixes all three in about fifteen lines. This guide, part of ",[22,23,25],"a",{"href":24},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002F","Error Handling and Logging in Excel Automation",", sets it up and covers what an Excel job should actually record.",[28,29,37,38,37,42,37,46,37,53,37,62,37,69,37,74,37,82,37,87,37,92,37,97,37,101,37,107,37,111,37,116,37,120,37,127,37,133,37,137,37,141,37,146],"svg",{"viewBox":30,"role":31,"ariaLabelledBy":32,"xmlns":35,"style":36},"0 0 740 232","img",[33,34],"lf-arch-t","lf-arch-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:740px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[39,40,41],"title",{"id":33},"One logger, two handlers",[43,44,45],"desc",{"id":34},"Log calls from the script go to a single logger, which fans out to a rotating file handler that keeps history on disk and a stream handler that the scheduler captures. Both receive the same formatted line.",[47,48],"rect",{"x":49,"y":49,"width":50,"height":51,"fill":52},"0","740","232","#ffffff",[47,54],{"x":55,"y":56,"width":57,"height":58,"rx":59,"fill":60,"stroke":61},"40","88","180","66","12","#ebebfd","var(--line,#cdd5e6)",[63,64,68],"text",{"x":65,"y":66,"style":67},"130","116","font-size:12.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","log.info(...)",[63,70,73],{"x":65,"y":71,"style":72},"138","font-size:11.5px;fill:var(--muted,#5b6780);text-anchor:middle","from anywhere in the job",[75,76],"line",{"x1":77,"y1":78,"x2":79,"y2":78,"stroke":80,"style":81},"224","121","266","var(--muted,#5b6780)","stroke-width:2px",[83,84],"polygon",{"points":85,"fill":86},"270,121 258,115 258,127","#5b6780",[47,88],{"x":89,"y":56,"width":90,"height":58,"rx":59,"fill":91},"272","160","#5b5cf0",[63,93,96],{"x":94,"y":66,"style":95},"352","font-size:12.5px;font-weight:700;fill:#ffffff;text-anchor:middle","logger",[63,98,100],{"x":94,"y":71,"style":99},"font-size:11.5px;fill:rgba(255,255,255,0.88);text-anchor:middle","level + formatter",[75,102],{"x1":103,"y1":104,"x2":105,"y2":58,"stroke":106,"style":81},"436","106","486","var(--teal,#0f9488)",[83,108],{"points":109,"fill":110},"490,63 478,63 482,73","#0f766e",[75,112],{"x1":103,"y1":113,"x2":105,"y2":114,"stroke":115,"style":81},"136","176","var(--gold,#b4740a)",[83,117],{"points":118,"fill":119},"490,179 478,171 482,181","#8a5808",[47,121],{"x":122,"y":123,"width":124,"height":125,"rx":59,"fill":126,"stroke":106,"style":81},"492","30","228","72","#d9f4f1",[63,128,132],{"x":129,"y":130,"style":131},"606","58","font-size:12.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","RotatingFileHandler",[63,134,136],{"x":129,"y":135,"style":72},"82","history you can grep tomorrow",[47,138],{"x":122,"y":139,"width":124,"height":125,"rx":59,"fill":140,"stroke":115,"style":81},"146","#fdefd8",[63,142,145],{"x":129,"y":143,"style":144},"174","font-size:12.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","StreamHandler",[63,147,149],{"x":129,"y":148,"style":72},"198","what the scheduler captures",[151,152,154],"h2",{"id":153},"prerequisites","Prerequisites",[10,156,157],{},"Only the standard library is needed for logging itself:",[159,160,165],"pre",{"className":161,"code":162,"language":163,"meta":164,"style":164},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl\n","bash","",[13,166,167],{"__ignoreMap":164},[168,169,171,175,179,182],"span",{"class":75,"line":170},1,[168,172,174],{"class":173},"sMTad","pip",[168,176,178],{"class":177},"srMev"," install",[168,180,181],{"class":177}," pandas",[168,183,184],{"class":177}," openpyxl\n",[151,186,188],{"id":187},"a-setup-function-you-can-reuse","A setup function you can reuse",[159,190,194],{"className":191,"code":192,"language":193,"meta":164,"style":164},"language-python shiki shiki-themes github-light github-dark-high-contrast","import logging\nfrom logging.handlers import RotatingFileHandler\nfrom pathlib import Path\n\ndef setup_logging(name=\"report\", logdir=\"logs\", level=logging.INFO, console=True):\n    Path(logdir).mkdir(parents=True, exist_ok=True)\n    logger = logging.getLogger(name)\n    logger.setLevel(level)\n    logger.handlers.clear()          # safe to call twice, e.g. from tests\n    logger.propagate = False         # do not double-log through the root logger\n\n    fmt = logging.Formatter(\n        \"%(asctime)s %(levelname)-8s %(name)s %(message)s\",\n        datefmt=\"%Y-%m-%d %H:%M:%S\",\n    )\n\n    file_handler = RotatingFileHandler(\n        Path(logdir) \u002F f\"{name}.log\", maxBytes=5_000_000, backupCount=5, encoding=\"utf-8\"\n    )\n    file_handler.setFormatter(fmt)\n    logger.addHandler(file_handler)\n\n    if console:\n        stream = logging.StreamHandler()\n        stream.setFormatter(fmt)\n        logger.addHandler(stream)\n\n    return logger\n\nlog = setup_logging()\nlog.info(\"logging ready\")\n","python",[13,195,196,206,220,233,240,290,317,328,334,344,358,363,374,399,418,424,429,440,496,501,507,513,518,527,538,544,550,555,564,569,580],{"__ignoreMap":164},[168,197,198,202],{"class":75,"line":170},[168,199,201],{"class":200},"s-kum","import",[168,203,205],{"class":204},"skGVy"," logging\n",[168,207,209,212,215,217],{"class":75,"line":208},2,[168,210,211],{"class":200},"from",[168,213,214],{"class":204}," logging.handlers ",[168,216,201],{"class":200},[168,218,219],{"class":204}," RotatingFileHandler\n",[168,221,223,225,228,230],{"class":75,"line":222},3,[168,224,211],{"class":200},[168,226,227],{"class":204}," pathlib ",[168,229,201],{"class":200},[168,231,232],{"class":204}," Path\n",[168,234,236],{"class":75,"line":235},4,[168,237,239],{"emptyLinePlaceholder":238},true,"\n",[168,241,243,246,250,253,256,259,262,264,267,270,272,275,279,282,284,287],{"class":75,"line":242},5,[168,244,245],{"class":200},"def",[168,247,249],{"class":248},"s_Opv"," setup_logging",[168,251,252],{"class":204},"(name",[168,254,255],{"class":200},"=",[168,257,258],{"class":177},"\"report\"",[168,260,261],{"class":204},", logdir",[168,263,255],{"class":200},[168,265,266],{"class":177},"\"logs\"",[168,268,269],{"class":204},", level",[168,271,255],{"class":200},[168,273,274],{"class":204},"logging.",[168,276,278],{"class":277},"sP0c6","INFO",[168,280,281],{"class":204},", console",[168,283,255],{"class":200},[168,285,286],{"class":277},"True",[168,288,289],{"class":204},"):\n",[168,291,293,296,300,302,304,307,310,312,314],{"class":75,"line":292},6,[168,294,295],{"class":204},"    Path(logdir).mkdir(",[168,297,299],{"class":298},"sa561","parents",[168,301,255],{"class":200},[168,303,286],{"class":277},[168,305,306],{"class":204},", ",[168,308,309],{"class":298},"exist_ok",[168,311,255],{"class":200},[168,313,286],{"class":277},[168,315,316],{"class":204},")\n",[168,318,320,323,325],{"class":75,"line":319},7,[168,321,322],{"class":204},"    logger ",[168,324,255],{"class":200},[168,326,327],{"class":204}," logging.getLogger(name)\n",[168,329,331],{"class":75,"line":330},8,[168,332,333],{"class":204},"    logger.setLevel(level)\n",[168,335,337,340],{"class":75,"line":336},9,[168,338,339],{"class":204},"    logger.handlers.clear()          ",[168,341,343],{"class":342},"s-wDw","# safe to call twice, e.g. from tests\n",[168,345,347,350,352,355],{"class":75,"line":346},10,[168,348,349],{"class":204},"    logger.propagate ",[168,351,255],{"class":200},[168,353,354],{"class":277}," False",[168,356,357],{"class":342},"         # do not double-log through the root logger\n",[168,359,361],{"class":75,"line":360},11,[168,362,239],{"emptyLinePlaceholder":238},[168,364,366,369,371],{"class":75,"line":365},12,[168,367,368],{"class":204},"    fmt ",[168,370,255],{"class":200},[168,372,373],{"class":204}," logging.Formatter(\n",[168,375,377,380,384,387,390,393,396],{"class":75,"line":376},13,[168,378,379],{"class":177},"        \"",[168,381,383],{"class":382},"sSjpA","%(asctime)s",[168,385,386],{"class":382}," %(levelname)-8s",[168,388,389],{"class":382}," %(name)s",[168,391,392],{"class":382}," %(message)s",[168,394,395],{"class":177},"\"",[168,397,398],{"class":204},",\n",[168,400,402,405,407,410,413,416],{"class":75,"line":401},14,[168,403,404],{"class":298},"        datefmt",[168,406,255],{"class":200},[168,408,409],{"class":177},"\"%Y-%m-",[168,411,412],{"class":382},"%d",[168,414,415],{"class":177}," %H:%M:%S\"",[168,417,398],{"class":204},[168,419,421],{"class":75,"line":420},15,[168,422,423],{"class":204},"    )\n",[168,425,427],{"class":75,"line":426},16,[168,428,239],{"emptyLinePlaceholder":238},[168,430,432,435,437],{"class":75,"line":431},17,[168,433,434],{"class":204},"    file_handler ",[168,436,255],{"class":200},[168,438,439],{"class":204}," RotatingFileHandler(\n",[168,441,443,446,449,452,454,457,460,463,466,468,471,473,476,478,481,483,486,488,491,493],{"class":75,"line":442},18,[168,444,445],{"class":204},"        Path(logdir) ",[168,447,448],{"class":200},"\u002F",[168,450,451],{"class":200}," f",[168,453,395],{"class":177},[168,455,456],{"class":382},"{",[168,458,459],{"class":204},"name",[168,461,462],{"class":382},"}",[168,464,465],{"class":177},".log\"",[168,467,306],{"class":204},[168,469,470],{"class":298},"maxBytes",[168,472,255],{"class":200},[168,474,475],{"class":277},"5_000_000",[168,477,306],{"class":204},[168,479,480],{"class":298},"backupCount",[168,482,255],{"class":200},[168,484,485],{"class":277},"5",[168,487,306],{"class":204},[168,489,490],{"class":298},"encoding",[168,492,255],{"class":200},[168,494,495],{"class":177},"\"utf-8\"\n",[168,497,499],{"class":75,"line":498},19,[168,500,423],{"class":204},[168,502,504],{"class":75,"line":503},20,[168,505,506],{"class":204},"    file_handler.setFormatter(fmt)\n",[168,508,510],{"class":75,"line":509},21,[168,511,512],{"class":204},"    logger.addHandler(file_handler)\n",[168,514,516],{"class":75,"line":515},22,[168,517,239],{"emptyLinePlaceholder":238},[168,519,521,524],{"class":75,"line":520},23,[168,522,523],{"class":200},"    if",[168,525,526],{"class":204}," console:\n",[168,528,530,533,535],{"class":75,"line":529},24,[168,531,532],{"class":204},"        stream ",[168,534,255],{"class":200},[168,536,537],{"class":204}," logging.StreamHandler()\n",[168,539,541],{"class":75,"line":540},25,[168,542,543],{"class":204},"        stream.setFormatter(fmt)\n",[168,545,547],{"class":75,"line":546},26,[168,548,549],{"class":204},"        logger.addHandler(stream)\n",[168,551,553],{"class":75,"line":552},27,[168,554,239],{"emptyLinePlaceholder":238},[168,556,558,561],{"class":75,"line":557},28,[168,559,560],{"class":200},"    return",[168,562,563],{"class":204}," logger\n",[168,565,567],{"class":75,"line":566},29,[168,568,239],{"emptyLinePlaceholder":238},[168,570,572,575,577],{"class":75,"line":571},30,[168,573,574],{"class":204},"log ",[168,576,255],{"class":200},[168,578,579],{"class":204}," setup_logging()\n",[168,581,583,586,589],{"class":75,"line":582},31,[168,584,585],{"class":204},"log.info(",[168,587,588],{"class":177},"\"logging ready\"",[168,590,316],{"class":204},[10,592,593,594,597,598,601,602,605,606,609],{},"Three lines earn their place. ",[13,595,596],{},"handlers.clear()"," makes the function idempotent, so importing the module twice does not produce doubled output. ",[13,599,600],{},"propagate = False"," stops messages also reaching the root logger, which is what causes each line to appear twice when a library has configured logging as well. And ",[13,603,604],{},"encoding=\"utf-8\""," avoids a ",[13,607,608],{},"UnicodeEncodeError"," the first time a customer name with an accent reaches the log on Windows.",[151,611,613],{"id":612},"what-an-excel-job-should-record","What an Excel job should record",[10,615,616],{},"Log the boundaries — every point where data enters, changes shape, or leaves:",[159,618,620],{"className":191,"code":619,"language":193,"meta":164,"style":164},"import time\nfrom pathlib import Path\n\nimport pandas as pd\n\ndef build_report(source, target):\n    started = time.perf_counter()\n    log.info(\"run started source=%s target=%s\", source, target)\n\n    df = pd.read_excel(source, sheet_name=\"Orders\")\n    log.info(\"read rows=%d columns=%d sheet=%s\", len(df), len(df.columns), \"Orders\")\n\n    before = len(df)\n    df = df.dropna(subset=[\"Order_ID\"])\n    log.info(\"validated rows=%d dropped=%d\", len(df), before - len(df))\n\n    summary = df.groupby(\"Region\", observed=True)[\"Quantity\"].sum()\n    log.info(\"aggregated groups=%d total=%s\", len(summary), int(summary.sum()))\n\n    summary.to_frame().to_excel(target)\n    size_kb = Path(target).stat().st_size \u002F 1024\n    log.info(\"wrote %s size=%.1fKB elapsed=%.1fs\",\n             target, size_kb, time.perf_counter() - started)\n    return target\n",[13,621,622,629,639,643,656,660,670,680,701,705,725,763,767,780,803,834,838,869,898,902,907,922,947,957],{"__ignoreMap":164},[168,623,624,626],{"class":75,"line":170},[168,625,201],{"class":200},[168,627,628],{"class":204}," time\n",[168,630,631,633,635,637],{"class":75,"line":208},[168,632,211],{"class":200},[168,634,227],{"class":204},[168,636,201],{"class":200},[168,638,232],{"class":204},[168,640,641],{"class":75,"line":222},[168,642,239],{"emptyLinePlaceholder":238},[168,644,645,647,650,653],{"class":75,"line":235},[168,646,201],{"class":200},[168,648,649],{"class":204}," pandas ",[168,651,652],{"class":200},"as",[168,654,655],{"class":204}," pd\n",[168,657,658],{"class":75,"line":242},[168,659,239],{"emptyLinePlaceholder":238},[168,661,662,664,667],{"class":75,"line":292},[168,663,245],{"class":200},[168,665,666],{"class":248}," build_report",[168,668,669],{"class":204},"(source, target):\n",[168,671,672,675,677],{"class":75,"line":319},[168,673,674],{"class":204},"    started ",[168,676,255],{"class":200},[168,678,679],{"class":204}," time.perf_counter()\n",[168,681,682,685,688,691,694,696,698],{"class":75,"line":330},[168,683,684],{"class":204},"    log.info(",[168,686,687],{"class":177},"\"run started source=",[168,689,690],{"class":382},"%s",[168,692,693],{"class":177}," target=",[168,695,690],{"class":382},[168,697,395],{"class":177},[168,699,700],{"class":204},", source, target)\n",[168,702,703],{"class":75,"line":336},[168,704,239],{"emptyLinePlaceholder":238},[168,706,707,710,712,715,718,720,723],{"class":75,"line":346},[168,708,709],{"class":204},"    df ",[168,711,255],{"class":200},[168,713,714],{"class":204}," pd.read_excel(source, ",[168,716,717],{"class":298},"sheet_name",[168,719,255],{"class":200},[168,721,722],{"class":177},"\"Orders\"",[168,724,316],{"class":204},[168,726,727,729,732,734,737,739,742,744,746,748,751,754,756,759,761],{"class":75,"line":360},[168,728,684],{"class":204},[168,730,731],{"class":177},"\"read rows=",[168,733,412],{"class":382},[168,735,736],{"class":177}," columns=",[168,738,412],{"class":382},[168,740,741],{"class":177}," sheet=",[168,743,690],{"class":382},[168,745,395],{"class":177},[168,747,306],{"class":204},[168,749,750],{"class":277},"len",[168,752,753],{"class":204},"(df), ",[168,755,750],{"class":277},[168,757,758],{"class":204},"(df.columns), ",[168,760,722],{"class":177},[168,762,316],{"class":204},[168,764,765],{"class":75,"line":365},[168,766,239],{"emptyLinePlaceholder":238},[168,768,769,772,774,777],{"class":75,"line":376},[168,770,771],{"class":204},"    before ",[168,773,255],{"class":200},[168,775,776],{"class":277}," len",[168,778,779],{"class":204},"(df)\n",[168,781,782,784,786,789,792,794,797,800],{"class":75,"line":401},[168,783,709],{"class":204},[168,785,255],{"class":200},[168,787,788],{"class":204}," df.dropna(",[168,790,791],{"class":298},"subset",[168,793,255],{"class":200},[168,795,796],{"class":204},"[",[168,798,799],{"class":177},"\"Order_ID\"",[168,801,802],{"class":204},"])\n",[168,804,805,807,810,812,815,817,819,821,823,826,829,831],{"class":75,"line":420},[168,806,684],{"class":204},[168,808,809],{"class":177},"\"validated rows=",[168,811,412],{"class":382},[168,813,814],{"class":177}," dropped=",[168,816,412],{"class":382},[168,818,395],{"class":177},[168,820,306],{"class":204},[168,822,750],{"class":277},[168,824,825],{"class":204},"(df), before ",[168,827,828],{"class":200},"-",[168,830,776],{"class":277},[168,832,833],{"class":204},"(df))\n",[168,835,836],{"class":75,"line":426},[168,837,239],{"emptyLinePlaceholder":238},[168,839,840,843,845,848,851,853,856,858,860,863,866],{"class":75,"line":431},[168,841,842],{"class":204},"    summary ",[168,844,255],{"class":200},[168,846,847],{"class":204}," df.groupby(",[168,849,850],{"class":177},"\"Region\"",[168,852,306],{"class":204},[168,854,855],{"class":298},"observed",[168,857,255],{"class":200},[168,859,286],{"class":277},[168,861,862],{"class":204},")[",[168,864,865],{"class":177},"\"Quantity\"",[168,867,868],{"class":204},"].sum()\n",[168,870,871,873,876,878,881,883,885,887,889,892,895],{"class":75,"line":442},[168,872,684],{"class":204},[168,874,875],{"class":177},"\"aggregated groups=",[168,877,412],{"class":382},[168,879,880],{"class":177}," total=",[168,882,690],{"class":382},[168,884,395],{"class":177},[168,886,306],{"class":204},[168,888,750],{"class":277},[168,890,891],{"class":204},"(summary), ",[168,893,894],{"class":277},"int",[168,896,897],{"class":204},"(summary.sum()))\n",[168,899,900],{"class":75,"line":498},[168,901,239],{"emptyLinePlaceholder":238},[168,903,904],{"class":75,"line":503},[168,905,906],{"class":204},"    summary.to_frame().to_excel(target)\n",[168,908,909,912,914,917,919],{"class":75,"line":509},[168,910,911],{"class":204},"    size_kb ",[168,913,255],{"class":200},[168,915,916],{"class":204}," Path(target).stat().st_size ",[168,918,448],{"class":200},[168,920,921],{"class":277}," 1024\n",[168,923,924,926,929,931,934,937,940,942,945],{"class":75,"line":515},[168,925,684],{"class":204},[168,927,928],{"class":177},"\"wrote ",[168,930,690],{"class":382},[168,932,933],{"class":177}," size=",[168,935,936],{"class":382},"%.1f",[168,938,939],{"class":177},"KB elapsed=",[168,941,936],{"class":382},[168,943,944],{"class":177},"s\"",[168,946,398],{"class":204},[168,948,949,952,954],{"class":75,"line":520},[168,950,951],{"class":204},"             target, size_kb, time.perf_counter() ",[168,953,828],{"class":200},[168,955,956],{"class":204}," started)\n",[168,958,959,961],{"class":75,"line":529},[168,960,560],{"class":200},[168,962,963],{"class":204}," target\n",[10,965,966,967,969,970,973],{},"Each line carries a number, and the numbers are what make a log useful weeks later. \"read rows=48211 … dropped=0 … groups=4\" reconstructs a run without the source file; \"read rows=12\" on the following day identifies the problem instantly. Using ",[13,968,690],{}," placeholders rather than f-strings means the formatting cost is only paid if the level is enabled, which matters for ",[13,971,972],{},"DEBUG"," lines inside loops.",[151,975,977],{"id":976},"levels-and-when-to-use-each","Levels, and when to use each",[979,980,981,997],"table",{},[982,983,984],"thead",{},[985,986,987,991,994],"tr",{},[988,989,990],"th",{},"Level",[988,992,993],{},"Use for",[988,995,996],{},"Example",[998,999,1000,1013,1025,1038,1051],"tbody",{},[985,1001,1002,1007,1010],{},[1003,1004,1005],"td",{},[13,1006,972],{},[1003,1008,1009],{},"Detail you would only want while investigating",[1003,1011,1012],{},"per-file paths in a batch of 300",[985,1014,1015,1019,1022],{},[1003,1016,1017],{},[13,1018,278],{},[1003,1020,1021],{},"The narrative of a normal run",[1003,1023,1024],{},"rows read, groups aggregated, file written",[985,1026,1027,1032,1035],{},[1003,1028,1029],{},[13,1030,1031],{},"WARNING",[1003,1033,1034],{},"Something unexpected the job handled",[1003,1036,1037],{},"fallback filename used, 3 rows quarantined",[985,1039,1040,1045,1048],{},[1003,1041,1042],{},[13,1043,1044],{},"ERROR",[1003,1046,1047],{},"The job could not do what it was asked",[1003,1049,1050],{},"output validation failed, source missing",[985,1052,1053,1058,1061],{},[1003,1054,1055],{},[13,1056,1057],{},"CRITICAL",[1003,1059,1060],{},"Rare — the process cannot continue at all",[1003,1062,1063],{},"disk full, credentials rejected",[10,1065,1066,1067,1069,1070,1072,1073,1075,1076,1078],{},"The line between ",[13,1068,278],{}," and ",[13,1071,972],{}," decides whether the log is readable. A daily report should produce perhaps twenty ",[13,1074,278],{}," lines: enough to tell the story, few enough to scan. Anything per-row or per-file belongs at ",[13,1077,972],{},", where raising the level in an emergency gives you more detail without changing the code.",[28,1080,37,1085,37,1088,37,1091,37,1095,37,1100,37,1107,37,1113,37,1117,37,1120,37,1125,37,1129,37,1132,37,1136,37,1138,37,1141,37,1145,37,1147,37,1151,37,1155,37,1158,37,1161,37,1164,37,1169],{"viewBox":1081,"role":31,"ariaLabelledBy":1082,"xmlns":35,"style":36},"0 18 740 164",[1083,1084],"lf-level-t","lf-level-d",[39,1086,1087],{"id":1083},"How the configured level filters what is written",[43,1089,1090],{"id":1084},"At INFO level the debug lines are discarded and the info, warning and error lines are written. Switching to DEBUG for an investigation adds the detailed lines without changing any code.",[47,1092],{"x":49,"y":1093,"width":50,"height":1094,"fill":52},"18","164",[63,1096,1099],{"x":57,"y":1097,"style":1098},"34","font-size:12.5px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:middle","level = INFO (normal)",[47,1101],{"x":123,"y":1102,"width":1103,"height":1104,"rx":1105,"fill":1106,"stroke":61},"48","300","28","6","#f0f2f5",[63,1108,1112],{"x":1109,"y":1110,"style":1111},"46","67","font-size:11px;fill:var(--muted,#5b6780)","DEBUG file 118\u002F300 …",[63,1114,1116],{"x":1103,"y":1110,"style":1115},"font-size:11px;fill:var(--danger,#dc2626);text-anchor:end","dropped",[47,1118],{"x":123,"y":1119,"width":1103,"height":1104,"rx":1105,"fill":52,"stroke":61},"80",[63,1121,1124],{"x":1109,"y":1122,"style":1123},"99","font-size:11px;fill:var(--text,#172033)","INFO read rows=48211",[63,1126,1128],{"x":1103,"y":1122,"style":1127},"font-size:11px;fill:var(--teal,#0f9488);text-anchor:end","written",[47,1130],{"x":123,"y":1131,"width":1103,"height":1104,"rx":1105,"fill":52,"stroke":61},"112",[63,1133,1135],{"x":1109,"y":1134,"style":1123},"131","WARNING 3 rows quarantined",[63,1137,1128],{"x":1103,"y":1134,"style":1127},[47,1139],{"x":123,"y":1140,"width":1103,"height":1104,"rx":1105,"fill":52,"stroke":61},"144",[63,1142,1144],{"x":1109,"y":1143,"style":1123},"163","ERROR output validation failed",[63,1146,1128],{"x":1103,"y":1143,"style":1127},[63,1148,1150],{"x":1149,"y":1097,"style":1098},"540","level = DEBUG (investigating)",[47,1152],{"x":1153,"y":1102,"width":1154,"height":1104,"rx":1105,"fill":126,"stroke":106},"390","320",[63,1156,1112],{"x":1157,"y":1110,"style":1123},"406",[63,1159,1128],{"x":1160,"y":1110,"style":1127},"694",[47,1162],{"x":1153,"y":1119,"width":1154,"height":1163,"rx":1105,"fill":52,"stroke":61},"92",[63,1165,1168],{"x":1166,"y":1167,"style":72},"550","120","everything above, unchanged",[63,1170,1172],{"x":1166,"y":1140,"style":1171},"font-size:11.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","no code edits required",[151,1174,1176],{"id":1175},"tracebacks-and-library-warnings","Tracebacks and library warnings",[10,1178,1179],{},"Two things routinely go missing from a log and are exactly what you need after a failure:",[159,1181,1183],{"className":191,"code":1182,"language":193,"meta":164,"style":164},"import logging\nimport warnings\n\nlogging.captureWarnings(True)                      # route warnings.warn into logging\nlogging.getLogger(\"py.warnings\").addHandler(log.handlers[0])\n\ntry:\n    df = pd.read_excel(\"missing.xlsx\")\nexcept Exception:\n    log.exception(\"failed to read the source workbook\")   # ERROR + full traceback\n    raise\n",[13,1184,1185,1191,1198,1202,1215,1230,1234,1242,1256,1266,1280],{"__ignoreMap":164},[168,1186,1187,1189],{"class":75,"line":170},[168,1188,201],{"class":200},[168,1190,205],{"class":204},[168,1192,1193,1195],{"class":75,"line":208},[168,1194,201],{"class":200},[168,1196,1197],{"class":204}," warnings\n",[168,1199,1200],{"class":75,"line":222},[168,1201,239],{"emptyLinePlaceholder":238},[168,1203,1204,1207,1209,1212],{"class":75,"line":235},[168,1205,1206],{"class":204},"logging.captureWarnings(",[168,1208,286],{"class":277},[168,1210,1211],{"class":204},")                      ",[168,1213,1214],{"class":342},"# route warnings.warn into logging\n",[168,1216,1217,1220,1223,1226,1228],{"class":75,"line":242},[168,1218,1219],{"class":204},"logging.getLogger(",[168,1221,1222],{"class":177},"\"py.warnings\"",[168,1224,1225],{"class":204},").addHandler(log.handlers[",[168,1227,49],{"class":277},[168,1229,802],{"class":204},[168,1231,1232],{"class":75,"line":292},[168,1233,239],{"emptyLinePlaceholder":238},[168,1235,1236,1239],{"class":75,"line":319},[168,1237,1238],{"class":200},"try",[168,1240,1241],{"class":204},":\n",[168,1243,1244,1246,1248,1251,1254],{"class":75,"line":330},[168,1245,709],{"class":204},[168,1247,255],{"class":200},[168,1249,1250],{"class":204}," pd.read_excel(",[168,1252,1253],{"class":177},"\"missing.xlsx\"",[168,1255,316],{"class":204},[168,1257,1258,1261,1264],{"class":75,"line":336},[168,1259,1260],{"class":200},"except",[168,1262,1263],{"class":277}," Exception",[168,1265,1241],{"class":204},[168,1267,1268,1271,1274,1277],{"class":75,"line":346},[168,1269,1270],{"class":204},"    log.exception(",[168,1272,1273],{"class":177},"\"failed to read the source workbook\"",[168,1275,1276],{"class":204},")   ",[168,1278,1279],{"class":342},"# ERROR + full traceback\n",[168,1281,1282],{"class":75,"line":360},[168,1283,1284],{"class":200},"    raise\n",[10,1286,1287,1290,1291,1293,1294,1297],{},[13,1288,1289],{},"log.exception"," may only be called inside an ",[13,1292,1260],{}," block, and it appends the traceback automatically — there is no need to format the exception yourself. ",[13,1295,1296],{},"captureWarnings(True)"," matters more than it sounds for spreadsheet work: openpyxl warns about unsupported extensions, and pandas warns about inconsistent date parsing, and both are usually the first sign that a source file has changed.",[151,1299,1301],{"id":1300},"add-context-without-repeating-yourself","Add context without repeating yourself",[10,1303,1304],{},"When a job processes many units, put the unit in every line automatically rather than threading it through each call:",[159,1306,1308],{"className":191,"code":1307,"language":193,"meta":164,"style":164},"import logging\n\nclass RunContext(logging.LoggerAdapter):\n    def process(self, msg, kwargs):\n        return f\"[{self.extra['unit']}] {msg}\", kwargs\n\nfor region in [\"North\", \"South\", \"West\"]:\n    rlog = RunContext(log, {\"unit\": region})\n    rlog.info(\"started\")\n    rlog.info(\"rows=%d\", 1200)\n",[13,1309,1310,1316,1320,1342,1353,1394,1398,1428,1444,1454],{"__ignoreMap":164},[168,1311,1312,1314],{"class":75,"line":170},[168,1313,201],{"class":200},[168,1315,205],{"class":204},[168,1317,1318],{"class":75,"line":208},[168,1319,239],{"emptyLinePlaceholder":238},[168,1321,1322,1325,1328,1331,1334,1337,1340],{"class":75,"line":222},[168,1323,1324],{"class":200},"class",[168,1326,1327],{"class":173}," RunContext",[168,1329,1330],{"class":204},"(",[168,1332,19],{"class":1333},"sHg5J",[168,1335,1336],{"class":204},".",[168,1338,1339],{"class":1333},"LoggerAdapter",[168,1341,289],{"class":204},[168,1343,1344,1347,1350],{"class":75,"line":235},[168,1345,1346],{"class":200},"    def",[168,1348,1349],{"class":248}," process",[168,1351,1352],{"class":204},"(self, msg, kwargs):\n",[168,1354,1355,1358,1360,1363,1365,1368,1371,1374,1377,1379,1382,1384,1387,1389,1391],{"class":75,"line":242},[168,1356,1357],{"class":200},"        return",[168,1359,451],{"class":200},[168,1361,1362],{"class":177},"\"[",[168,1364,456],{"class":382},[168,1366,1367],{"class":277},"self",[168,1369,1370],{"class":204},".extra[",[168,1372,1373],{"class":177},"'unit'",[168,1375,1376],{"class":204},"]",[168,1378,462],{"class":382},[168,1380,1381],{"class":177},"] ",[168,1383,456],{"class":382},[168,1385,1386],{"class":204},"msg",[168,1388,462],{"class":382},[168,1390,395],{"class":177},[168,1392,1393],{"class":204},", kwargs\n",[168,1395,1396],{"class":75,"line":292},[168,1397,239],{"emptyLinePlaceholder":238},[168,1399,1400,1403,1406,1409,1412,1415,1417,1420,1422,1425],{"class":75,"line":319},[168,1401,1402],{"class":200},"for",[168,1404,1405],{"class":204}," region ",[168,1407,1408],{"class":200},"in",[168,1410,1411],{"class":204}," [",[168,1413,1414],{"class":177},"\"North\"",[168,1416,306],{"class":204},[168,1418,1419],{"class":177},"\"South\"",[168,1421,306],{"class":204},[168,1423,1424],{"class":177},"\"West\"",[168,1426,1427],{"class":204},"]:\n",[168,1429,1430,1433,1435,1438,1441],{"class":75,"line":330},[168,1431,1432],{"class":204},"    rlog ",[168,1434,255],{"class":200},[168,1436,1437],{"class":204}," RunContext(log, {",[168,1439,1440],{"class":177},"\"unit\"",[168,1442,1443],{"class":204},": region})\n",[168,1445,1446,1449,1452],{"class":75,"line":336},[168,1447,1448],{"class":204},"    rlog.info(",[168,1450,1451],{"class":177},"\"started\"",[168,1453,316],{"class":204},[168,1455,1456,1458,1461,1463,1465,1467,1470],{"class":75,"line":346},[168,1457,1448],{"class":204},[168,1459,1460],{"class":177},"\"rows=",[168,1462,412],{"class":382},[168,1464,395],{"class":177},[168,1466,306],{"class":204},[168,1468,1469],{"class":277},"1200",[168,1471,316],{"class":204},[10,1473,1474,1475,1478,1479,1482],{},"Every line now begins with ",[13,1476,1477],{},"[North]",", so ",[13,1480,1481],{},"grep North logs\u002Freport.log"," gives you one region's entire story. On a batch job this is the difference between a readable log and an interleaved one where three regions' messages are shuffled together.",[151,1484,1486],{"id":1485},"rotation-by-size-or-by-day","Rotation: by size or by day?",[10,1488,1489],{},"Both keep the log bounded; they suit different investigations. A scheduled report is almost always easier to debug with one file per day, because that is how the question arrives — \"what happened on the 3rd?\":",[28,1491,37,1496,37,1499,37,1502,37,1507,37,1512,37,1516,37,1521,37,1525,37,1530,37,1534,37,1537,37,1542,37,1545,37,1548,37,1551],{"viewBox":1492,"role":31,"ariaLabelledBy":1493,"xmlns":35,"style":36},"14 6 712 182",[1494,1495],"lf-rot-t","lf-rot-d",[39,1497,1498],{"id":1494},"Size-based and time-based rotation compared",[43,1500,1501],{"id":1495},"Size rotation produces report.log plus numbered backups that each hold an arbitrary slice of time, which suits jobs with variable output. Time rotation produces one file per day with the date in its name, which suits a scheduled report where questions arrive by date.",[47,1503],{"x":1504,"y":1105,"width":1505,"height":1506,"fill":52},"14","712","182",[47,1508],{"x":1509,"y":123,"width":1510,"height":1511,"rx":1504,"fill":60,"stroke":61},"24","330","156",[63,1513,132],{"x":1514,"y":130,"style":1515},"189","font-size:13px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle",[63,1517,1520],{"x":1514,"y":1518,"style":1519},"86","font-size:11.5px;fill:var(--text,#172033);text-anchor:middle","report.log · report.log.1 · .2",[63,1522,1524],{"x":1514,"y":1523,"style":72},"110","each file = 5 MB of whenever",[63,1526,1529],{"x":1514,"y":1527,"style":1528},"134","font-size:11.5px;fill:var(--teal,#0f9488);text-anchor:middle","bounded disk use, guaranteed",[63,1531,1533],{"x":1514,"y":1094,"style":1532},"font-size:11.5px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:middle","use when volume is unpredictable",[47,1535],{"x":1536,"y":123,"width":1510,"height":1511,"rx":1504,"fill":126,"stroke":106,"style":81},"386",[63,1538,1541],{"x":1539,"y":130,"style":1540},"551","font-size:13px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","TimedRotatingFileHandler",[63,1543,1544],{"x":1539,"y":1518,"style":1519},"report.log.2026-03-03",[63,1546,1547],{"x":1539,"y":1523,"style":72},"one file per day, named by date",[63,1549,1550],{"x":1539,"y":1527,"style":1528},"matches how questions are asked",[63,1552,1553],{"x":1539,"y":1094,"style":1532},"use for scheduled reports",[159,1555,1557],{"className":191,"code":1556,"language":193,"meta":164,"style":164},"from logging.handlers import TimedRotatingFileHandler\n\nhandler = TimedRotatingFileHandler(\n    \"logs\u002Freport.log\", when=\"midnight\", backupCount=30, encoding=\"utf-8\"\n)\n",[13,1558,1559,1570,1574,1584,1615],{"__ignoreMap":164},[168,1560,1561,1563,1565,1567],{"class":75,"line":170},[168,1562,211],{"class":200},[168,1564,214],{"class":204},[168,1566,201],{"class":200},[168,1568,1569],{"class":204}," TimedRotatingFileHandler\n",[168,1571,1572],{"class":75,"line":208},[168,1573,239],{"emptyLinePlaceholder":238},[168,1575,1576,1579,1581],{"class":75,"line":222},[168,1577,1578],{"class":204},"handler ",[168,1580,255],{"class":200},[168,1582,1583],{"class":204}," TimedRotatingFileHandler(\n",[168,1585,1586,1589,1591,1594,1596,1599,1601,1603,1605,1607,1609,1611,1613],{"class":75,"line":235},[168,1587,1588],{"class":177},"    \"logs\u002Freport.log\"",[168,1590,306],{"class":204},[168,1592,1593],{"class":298},"when",[168,1595,255],{"class":200},[168,1597,1598],{"class":177},"\"midnight\"",[168,1600,306],{"class":204},[168,1602,480],{"class":298},[168,1604,255],{"class":200},[168,1606,123],{"class":277},[168,1608,306],{"class":204},[168,1610,490],{"class":298},[168,1612,255],{"class":200},[168,1614,495],{"class":177},[168,1616,1617],{"class":75,"line":242},[168,1618,316],{"class":204},[10,1620,1621],{},"Thirty daily files is a month of history for a few megabytes, and the date in the filename means no searching at all for the common question. Where a job runs many times a day, keep time-based rotation and add the run key to each line — the file still bounds the search, and the key separates the runs inside it.",[151,1623,1625],{"id":1624},"common-pitfalls-and-gotchas","Common pitfalls and gotchas",[979,1627,1628,1641],{},[982,1629,1630],{},[985,1631,1632,1635,1638],{},[988,1633,1634],{},"Symptom",[988,1636,1637],{},"Cause",[988,1639,1640],{},"Fix",[998,1642,1643,1657,1672,1687,1704,1725,1738],{},[985,1644,1645,1648,1651],{},[1003,1646,1647],{},"Every line appears twice",[1003,1649,1650],{},"Handlers added twice, or propagation to root",[1003,1652,1653,1069,1655],{},[13,1654,596],{},[13,1656,600],{},[985,1658,1659,1662,1665],{},[1003,1660,1661],{},"Log file is empty",[1003,1663,1664],{},"Level set above the calls being made",[1003,1666,1667,1668,1671],{},"Check ",[13,1669,1670],{},"logger.setLevel"," and the handler level",[985,1673,1674,1679,1682],{},[1003,1675,1676,1678],{},[13,1677,608],{}," writing a name",[1003,1680,1681],{},"Default encoding on Windows",[1003,1683,1684,1686],{},[13,1685,604],{}," on the handler",[985,1688,1689,1692,1698],{},[1003,1690,1691],{},"Log grows without limit",[1003,1693,1694,1695],{},"Plain ",[13,1696,1697],{},"FileHandler",[1003,1699,1700,1701,1703],{},"Use ",[13,1702,132],{}," or the timed variant",[985,1705,1706,1709,1717],{},[1003,1707,1708],{},"Traceback missing",[1003,1710,1711,1714,1715],{},[13,1712,1713],{},"log.error(exc)"," instead of ",[13,1716,1289],{},[1003,1718,1719,1720,1722,1723],{},"Call ",[13,1721,1289],{}," inside the ",[13,1724,1260],{},[985,1726,1727,1730,1733],{},[1003,1728,1729],{},"pandas warnings invisible",[1003,1731,1732],{},"Warnings not routed to logging",[1003,1734,1735],{},[13,1736,1737],{},"logging.captureWarnings(True)",[985,1739,1740,1743,1746],{},[1003,1741,1742],{},"Nothing captured by the scheduler",[1003,1744,1745],{},"Only a file handler configured",[1003,1747,1748,1749,1751],{},"Keep the ",[13,1750,145],{}," too",[151,1753,1755],{"id":1754},"make-the-log-machine-readable-too","Make the log machine-readable too",[10,1757,1758],{},"A human reads the text; a monitoring system wants fields. Emitting a second, structured line per run gives you both without complicating the narrative:",[159,1760,1762],{"className":191,"code":1761,"language":193,"meta":164,"style":164},"import json\nimport logging\n\nclass JsonLine(logging.Formatter):\n    def format(self, record):\n        payload = {\n            \"ts\": self.formatTime(record, \"%Y-%m-%dT%H:%M:%S\"),\n            \"level\": record.levelname,\n            \"message\": record.getMessage(),\n        }\n        payload.update(getattr(record, \"fields\", {}))\n        return json.dumps(payload, ensure_ascii=False)\n\nmetrics = logging.getLogger(\"report.metrics\")\nmetrics.propagate = False\nhandler = logging.FileHandler(\"logs\u002Freport.jsonl\", encoding=\"utf-8\")\nhandler.setFormatter(JsonLine())\nmetrics.addHandler(handler)\nmetrics.setLevel(logging.INFO)\n\nmetrics.info(\"run finished\", extra={\"fields\": {\"rows\": 48211, \"seconds\": 12.4, \"status\": \"ok\"}})\n",[13,1763,1764,1771,1777,1781,1799,1809,1819,1842,1850,1858,1863,1880,1897,1901,1916,1926,1949,1954,1959,1968,1972],{"__ignoreMap":164},[168,1765,1766,1768],{"class":75,"line":170},[168,1767,201],{"class":200},[168,1769,1770],{"class":204}," json\n",[168,1772,1773,1775],{"class":75,"line":208},[168,1774,201],{"class":200},[168,1776,205],{"class":204},[168,1778,1779],{"class":75,"line":222},[168,1780,239],{"emptyLinePlaceholder":238},[168,1782,1783,1785,1788,1790,1792,1794,1797],{"class":75,"line":235},[168,1784,1324],{"class":200},[168,1786,1787],{"class":173}," JsonLine",[168,1789,1330],{"class":204},[168,1791,19],{"class":1333},[168,1793,1336],{"class":204},[168,1795,1796],{"class":1333},"Formatter",[168,1798,289],{"class":204},[168,1800,1801,1803,1806],{"class":75,"line":242},[168,1802,1346],{"class":200},[168,1804,1805],{"class":277}," format",[168,1807,1808],{"class":204},"(self, record):\n",[168,1810,1811,1814,1816],{"class":75,"line":292},[168,1812,1813],{"class":204},"        payload ",[168,1815,255],{"class":200},[168,1817,1818],{"class":204}," {\n",[168,1820,1821,1824,1827,1829,1832,1834,1836,1839],{"class":75,"line":319},[168,1822,1823],{"class":177},"            \"ts\"",[168,1825,1826],{"class":204},": ",[168,1828,1367],{"class":277},[168,1830,1831],{"class":204},".formatTime(record, ",[168,1833,409],{"class":177},[168,1835,412],{"class":382},[168,1837,1838],{"class":177},"T%H:%M:%S\"",[168,1840,1841],{"class":204},"),\n",[168,1843,1844,1847],{"class":75,"line":330},[168,1845,1846],{"class":177},"            \"level\"",[168,1848,1849],{"class":204},": record.levelname,\n",[168,1851,1852,1855],{"class":75,"line":336},[168,1853,1854],{"class":177},"            \"message\"",[168,1856,1857],{"class":204},": record.getMessage(),\n",[168,1859,1860],{"class":75,"line":346},[168,1861,1862],{"class":204},"        }\n",[168,1864,1865,1868,1871,1874,1877],{"class":75,"line":360},[168,1866,1867],{"class":204},"        payload.update(",[168,1869,1870],{"class":277},"getattr",[168,1872,1873],{"class":204},"(record, ",[168,1875,1876],{"class":177},"\"fields\"",[168,1878,1879],{"class":204},", {}))\n",[168,1881,1882,1884,1887,1890,1892,1895],{"class":75,"line":365},[168,1883,1357],{"class":200},[168,1885,1886],{"class":204}," json.dumps(payload, ",[168,1888,1889],{"class":298},"ensure_ascii",[168,1891,255],{"class":200},[168,1893,1894],{"class":277},"False",[168,1896,316],{"class":204},[168,1898,1899],{"class":75,"line":376},[168,1900,239],{"emptyLinePlaceholder":238},[168,1902,1903,1906,1908,1911,1914],{"class":75,"line":401},[168,1904,1905],{"class":204},"metrics ",[168,1907,255],{"class":200},[168,1909,1910],{"class":204}," logging.getLogger(",[168,1912,1913],{"class":177},"\"report.metrics\"",[168,1915,316],{"class":204},[168,1917,1918,1921,1923],{"class":75,"line":420},[168,1919,1920],{"class":204},"metrics.propagate ",[168,1922,255],{"class":200},[168,1924,1925],{"class":277}," False\n",[168,1927,1928,1930,1932,1935,1938,1940,1942,1944,1947],{"class":75,"line":426},[168,1929,1578],{"class":204},[168,1931,255],{"class":200},[168,1933,1934],{"class":204}," logging.FileHandler(",[168,1936,1937],{"class":177},"\"logs\u002Freport.jsonl\"",[168,1939,306],{"class":204},[168,1941,490],{"class":298},[168,1943,255],{"class":200},[168,1945,1946],{"class":177},"\"utf-8\"",[168,1948,316],{"class":204},[168,1950,1951],{"class":75,"line":431},[168,1952,1953],{"class":204},"handler.setFormatter(JsonLine())\n",[168,1955,1956],{"class":75,"line":442},[168,1957,1958],{"class":204},"metrics.addHandler(handler)\n",[168,1960,1961,1964,1966],{"class":75,"line":498},[168,1962,1963],{"class":204},"metrics.setLevel(logging.",[168,1965,278],{"class":277},[168,1967,316],{"class":204},[168,1969,1970],{"class":75,"line":503},[168,1971,239],{"emptyLinePlaceholder":238},[168,1973,1974,1977,1980,1982,1985,1987,1989,1991,1994,1997,1999,2002,2004,2007,2009,2012,2014,2017,2019,2022],{"class":75,"line":509},[168,1975,1976],{"class":204},"metrics.info(",[168,1978,1979],{"class":177},"\"run finished\"",[168,1981,306],{"class":204},[168,1983,1984],{"class":298},"extra",[168,1986,255],{"class":200},[168,1988,456],{"class":204},[168,1990,1876],{"class":177},[168,1992,1993],{"class":204},": {",[168,1995,1996],{"class":177},"\"rows\"",[168,1998,1826],{"class":204},[168,2000,2001],{"class":277},"48211",[168,2003,306],{"class":204},[168,2005,2006],{"class":177},"\"seconds\"",[168,2008,1826],{"class":204},[168,2010,2011],{"class":277},"12.4",[168,2013,306],{"class":204},[168,2015,2016],{"class":177},"\"status\"",[168,2018,1826],{"class":204},[168,2020,2021],{"class":177},"\"ok\"",[168,2023,2024],{"class":204},"}})\n",[10,2026,2027,2028,2031,2032,2035],{},"One JSON object per line — ",[13,2029,2030],{},".jsonl"," — is the format every log shipper and half the command-line tools already understand, and ",[13,2033,2034],{},"jq"," turns a month of runs into a table in one command. Keeping it in a separate logger means the readable log stays readable, and neither format has to compromise for the other.",[151,2037,2039],{"id":2038},"performance-and-scale-notes","Performance and scale notes",[10,2041,2042,2043,2045,2046,2048,2049,2051,2052,2055],{},"Logging costs a formatted string and a write per line, so twenty ",[13,2044,278],{}," lines per run are free. The pattern that does cost is logging inside a per-row loop: at 400,000 rows even a suppressed ",[13,2047,972],{}," call adds up, because the arguments are still evaluated. Use ",[13,2050,690],{}," placeholders so formatting is deferred, guard genuinely expensive messages with ",[13,2053,2054],{},"if log.isEnabledFor(logging.DEBUG)",", and prefer one summary line per chunk over one line per row. On a network share, keep the log on local disk — a blocked write to a mounted drive can stall the job it was meant to document.",[151,2057,2059],{"id":2058},"conclusion","Conclusion",[10,2061,2062,2063,2066,2067,2069,2070,2072,2073,2075,2076,2078],{},"Replace every ",[13,2064,2065],{},"print"," in a scheduled job with a logger that writes to both a rotating file and the console, and record the boundaries of the job with counts attached. Keep ",[13,2068,278],{}," to the narrative and push detail to ",[13,2071,972],{},", use ",[13,2074,1289],{}," so tracebacks land in the file, route library warnings into the same place, and add per-unit context with a ",[13,2077,1339],{}," when the job handles many things. The result is a log that answers questions without a re-run.",[151,2080,2082],{"id":2081},"frequently-asked-questions","Frequently asked questions",[10,2084,2085,2092,2094],{},[2086,2087,2088,2089,2091],"strong",{},"Why not just use ",[13,2090,2065],{}," and redirect the output?",[13,2093,2065],{}," has no levels, no timestamps and no way to send the same message to two places. Redirecting also loses everything if the scheduler discards stdout, which many do.",[10,2096,2097,2100,2101,2104],{},[2086,2098,2099],{},"Should I log to one file or one per run?","\nOne rotating file is easier to search and keeps history bounded. Add the run's date or key to each line rather than to the filename, so a single ",[13,2102,2103],{},"grep"," spans several runs.",[10,2106,2107,2110,2111,1722,2114,2116,2117,2119],{},[2086,2108,2109],{},"How do I record a traceback?","\nCall ",[13,2112,2113],{},"logger.exception",[13,2115,1260],{}," block. It logs at ",[13,2118,1044],{}," level and appends the full traceback automatically.",[10,2121,2122,2110,2125,2127,2128,2131],{},[2086,2123,2124],{},"How do I stop pandas warnings vanishing?",[13,2126,1737],{}," so ",[13,2129,2130],{},"warnings.warn"," output is routed through the logging system into the same file.",[151,2133,2135],{"id":2134},"related","Related",[10,2137,2138],{},"Up to the parent guide:",[2140,2141,2142],"ul",{},[2143,2144,2145,2147],"li",{},[22,2146,25],{"href":24}," — the failure classes these lines describe.",[10,2149,2150],{},"Related guides:",[2140,2152,2153,2160,2167,2174],{},[2143,2154,2155,2159],{},[22,2156,2158],{"href":2157},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Fretry-a-failed-excel-report-job-in-python\u002F","Retry a Failed Excel Report Job in Python"," — what to log around each attempt.",[2143,2161,2162,2166],{},[22,2163,2165],{"href":2164},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Fhandle-permission-denied-when-writing-excel-in-python\u002F","Handle Permission Denied When Writing Excel in Python"," — the diagnosis worth recording.",[2143,2168,2169,2173],{},[22,2170,2172],{"href":2171},"\u002Fautomating-reporting-workflows\u002Fscheduling-python-excel-scripts-with-cron\u002F","Scheduling Python Excel Scripts with cron"," — where the console handler's output ends up.",[2143,2175,2176,2180],{},[22,2177,2179],{"href":2178},"\u002Fautomating-reporting-workflows\u002Fscheduling-python-excel-scripts-with-cron\u002Frun-python-excel-script-on-windows-task-scheduler\u002F","Run a Python Excel Script on Windows Task Scheduler"," — logging when there is no terminal at all.",[2182,2183,2184],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}html pre.shiki code .sHg5J, html code.shiki .sHg5J{--shiki-default:#6F42C1;--shiki-dark:#91CBFF}",{"title":164,"searchDepth":208,"depth":208,"links":2186},[2187,2188,2189,2190,2191,2192,2193,2194,2195,2196,2197,2198,2199],{"id":153,"depth":208,"text":154},{"id":187,"depth":208,"text":188},{"id":612,"depth":208,"text":613},{"id":976,"depth":208,"text":977},{"id":1175,"depth":208,"text":1176},{"id":1300,"depth":208,"text":1301},{"id":1485,"depth":208,"text":1486},{"id":1624,"depth":208,"text":1625},{"id":1754,"depth":208,"text":1755},{"id":2038,"depth":208,"text":2039},{"id":2058,"depth":208,"text":2059},{"id":2081,"depth":208,"text":2082},{"id":2134,"depth":208,"text":2135},"2026-08-01","Replace print() with a logging setup that survives a scheduled run — file and console handlers, rotation, what to record at each stage, and capturing tracebacks and warnings.","md",[2204,2207,2209,2211],{"q":2205,"a":2206},"Why not just use print and redirect the output?","print has no levels, no timestamps and no way to send the same message to two places. Redirecting also loses everything if the scheduler discards stdout, which many do.",{"q":2099,"a":2208},"One rotating file is easier to search and keeps history bounded. Add the run's date or key to each line rather than to the filename, so a single grep spans several runs.",{"q":2109,"a":2210},"Call logger.exception inside the except block. It logs at ERROR level and appends the full traceback automatically.",{"q":2124,"a":2212},"Call logging.captureWarnings(True) so warnings.warn output is routed through the logging system into the same file.",{"breadcrumb":2214},[2215,2217,2220,2221],{"name":2216,"item":448},"Home",{"name":2218,"item":2219},"Automating Reporting Workflows","\u002Fautomating-reporting-workflows\u002F",{"name":25,"item":24},{"name":5,"item":2222},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Flog-python-excel-script-output-to-a-file\u002F","\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Flog-python-excel-script-output-to-a-file",{"title":5,"description":2225},"Set up Python logging for Excel jobs: FileHandler plus StreamHandler, rotation by size or day, per-stage row counts, logger.exception for tracebacks, and warning capture.","log-python-excel-script-output-to-a-file","automating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Flog-python-excel-script-output-to-a-file\u002Findex","how-to","JlqBCb3iiCqtV_37h5qYYMW6iFkUSP-XZZ1YexRONtw",[2231,2234],{"title":2165,"path":2232,"stem":2233,"children":-1},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Fhandle-permission-denied-when-writing-excel-in-python","automating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Fhandle-permission-denied-when-writing-excel-in-python\u002Findex",{"title":2158,"path":2235,"stem":2236,"children":-1},"\u002Fautomating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Fretry-a-failed-excel-report-job-in-python","automating-reporting-workflows\u002Ferror-handling-and-logging-in-excel-automation\u002Fretry-a-failed-excel-report-job-in-python\u002Findex",1785584462870]