[{"data":1,"prerenderedAt":1724},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fdetect-outliers-in-excel-data-with-pandas":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fdetect-outliers-in-excel-data-with-pandas":1690,"ancestors:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fdetect-outliers-in-excel-data-with-pandas":1699,"children:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fdetect-outliers-in-excel-data-with-pandas":1704,"related:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fdetect-outliers-in-excel-data-with-pandas":1705},{"id":4,"title":5,"body":6,"dateModified":1655,"datePublished":1655,"description":1656,"extension":1657,"faq":1658,"keyword":1667,"kind":1668,"meta":1669,"navigation":249,"path":1678,"seo":1679,"slug":1682,"stem":1683,"topics":1684,"type":1688,"__hash__":1689},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fdetect-outliers-in-excel-data-with-pandas\u002Findex.md","Detect Outliers in Excel Data with pandas",{"type":7,"value":8,"toc":1640},"minimark",[9,19,171,176,204,210,214,221,297,300,304,307,521,532,536,539,607,610,683,687,690,766,769,773,780,939,942,946,949,1084,1092,1258,1262,1265,1497,1505,1509,1544,1548,1565,1569,1575,1579,1585,1591,1597,1603,1607,1636],[10,11,12,13,18],"p",{},"A column of invoice amounts that normally sit around 400 contains one for 48,000. A quantity column has a -40. A delivery log says one parcel took 90 days. Some of these are typos — an extra zero, a sign keyed wrongly, a date in the wrong year — and some are real events that someone should know about. Either way they distort averages and charts, and they are easy to miss in a sheet of thousands of rows. This page, part of ",[14,15,17],"a",{"href":16},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002F","Cleaning Excel Data With Pandas",", compares three statistical ways of finding outliers on the same data, adds per-group detection and business rules, and flags the results in the workbook rather than deleting anything.",[20,21,29,30,29,34,29,38,29,46,29,54,29,60,29,64,29,68,29,71,29,75,29,78,29,82,29,86,29,92,29,96,29,100,29,104,29,108,29,112,29,116,29,120,29,129,29,134,29,137,29,140,29,143,29,146,29,149,29,154,29,159,29,166],"svg",{"viewBox":22,"role":23,"ariaLabelledBy":24,"xmlns":27,"style":28},"4 0 748 232","img",[25,26],"out-chart-t","out-chart-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:748px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[31,32,33],"title",{"id":25},"North region invoice amounts with one extreme value",[35,36,37],"desc",{"id":26},"A column chart of eight North invoices. Seven sit between 388 and 430; the eighth is 48,000, dwarfing the others so they appear as a flat line near the axis. That single value moves the mean from about 406 to over 6,300.",[39,40],"rect",{"x":41,"y":42,"width":43,"height":44,"fill":45},"4","0","748","232","#ffffff",[47,48],"line",{"x1":49,"y1":50,"x2":51,"y2":50,"stroke":52,"style":53},"53.1","194.0","736.0","var(--line,#cdd5e6)","stroke-width:1px",[55,56,42],"text",{"x":57,"y":58,"style":59},"30.5","198.0","font-size:11px;font-weight:400;fill:var(--muted,#5b6780);text-anchor:middle",[47,61],{"x1":49,"y1":62,"x2":51,"y2":62,"stroke":52,"style":63},"137.3","stroke-width:1px;stroke-dasharray:5 4",[55,65,67],{"x":57,"y":66,"style":59},"141.3","20k",[47,69],{"x1":49,"y1":70,"x2":51,"y2":70,"stroke":52,"style":63},"80.7",[55,72,74],{"x":57,"y":73,"style":59},"84.7","40k",[47,76],{"x1":49,"y1":77,"x2":51,"y2":77,"stroke":52,"style":63},"24.0",[55,79,81],{"x":57,"y":80,"style":59},"28.0","60k",[47,83],{"x1":49,"y1":50,"x2":51,"y2":50,"stroke":84,"style":85},"var(--muted,#5b6780)","stroke-width:1.5px",[55,87,91],{"x":88,"y":89,"style":90},"95.8","212.0","font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","N001",[55,93,95],{"x":94,"y":89,"style":90},"181.1","N002",[55,97,99],{"x":98,"y":89,"style":90},"266.5","N003",[55,101,103],{"x":102,"y":89,"style":90},"351.9","N004",[55,105,107],{"x":106,"y":89,"style":90},"437.2","N005",[55,109,111],{"x":110,"y":89,"style":90},"522.6","N006",[55,113,115],{"x":114,"y":89,"style":90},"608.0","N007",[55,117,119],{"x":118,"y":89,"style":90},"693.3","N008",[39,121],{"x":122,"y":123,"width":124,"height":125,"rx":126,"fill":127,"stroke":128,"style":85},"77.8","192.8","36.0","1.2","3","#f0f4ff","var(--brand,#5b5cf0)",[39,130],{"x":131,"y":132,"width":124,"height":133,"rx":126,"fill":127,"stroke":128,"style":85},"163.1","192.9","1.1",[39,135],{"x":136,"y":123,"width":124,"height":125,"rx":126,"fill":127,"stroke":128,"style":85},"248.5",[39,138],{"x":139,"y":132,"width":124,"height":133,"rx":126,"fill":127,"stroke":128,"style":85},"333.9",[39,141],{"x":142,"y":132,"width":124,"height":133,"rx":126,"fill":127,"stroke":128,"style":85},"419.2",[39,144],{"x":145,"y":123,"width":124,"height":125,"rx":126,"fill":127,"stroke":128,"style":85},"504.6",[39,147],{"x":148,"y":132,"width":124,"height":133,"rx":126,"fill":127,"stroke":128,"style":85},"590.0",[39,150],{"x":151,"y":152,"width":124,"height":153,"rx":126,"fill":127,"stroke":128,"style":85},"675.3","58.0","136.0",[47,155],{"x1":118,"y1":156,"x2":118,"y2":50,"stroke":157,"style":158},"20.0","var(--accent,#f43f8f)","stroke-width:1.5px;stroke-dasharray:5 4",[39,160],{"x":161,"y":162,"width":163,"height":156,"rx":164,"fill":165,"stroke":157,"style":53},"664.0","8.0","58.5","10","#fee8f2",[55,167,170],{"x":118,"y":168,"style":169},"22.0","font-size:11px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","48,000",[172,173,175],"h2",{"id":174},"prerequisites","Prerequisites",[177,178,183],"pre",{"className":179,"code":180,"language":181,"meta":182,"style":182},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl\n","bash","",[184,185,186],"code",{"__ignoreMap":182},[187,188,190,194,198,201],"span",{"class":47,"line":189},1,[187,191,193],{"class":192},"sMTad","pip",[187,195,197],{"class":196},"srMev"," install",[187,199,200],{"class":196}," pandas",[187,202,203],{"class":196}," openpyxl\n",[10,205,206,209],{},[184,207,208],{},"invoices.xlsx"," has sixteen invoices in two regions. North's are around 400 with one of 48,000; South's are around 125 with one of 690 and one of -40.",[172,211,213],{"id":212},"step-1-look-before-you-test","Step 1 — Look before you test",[10,215,216,217,220],{},"A statistical rule is only useful once you know what normal looks like. ",[184,218,219],{},"describe()"," per group gives the shape of the data in one line each:",[177,222,226],{"className":223,"code":224,"language":225,"meta":182,"style":182},"language-python shiki shiki-themes github-light github-dark-high-contrast","import pandas as pd\n\ninv = pd.read_excel(\"invoices.xlsx\")\nprint(inv.groupby(\"region\")[\"amount\"].describe().round(1))\n","python",[184,227,228,244,251,269],{"__ignoreMap":182},[187,229,230,234,238,241],{"class":47,"line":189},[187,231,233],{"class":232},"s-kum","import",[187,235,237],{"class":236},"skGVy"," pandas ",[187,239,240],{"class":232},"as",[187,242,243],{"class":236}," pd\n",[187,245,247],{"class":47,"line":246},2,[187,248,250],{"emptyLinePlaceholder":249},true,"\n",[187,252,254,257,260,263,266],{"class":47,"line":253},3,[187,255,256],{"class":236},"inv ",[187,258,259],{"class":232},"=",[187,261,262],{"class":236}," pd.read_excel(",[187,264,265],{"class":196},"\"invoices.xlsx\"",[187,267,268],{"class":236},")\n",[187,270,272,276,279,282,285,288,291,294],{"class":47,"line":271},4,[187,273,275],{"class":274},"sP0c6","print",[187,277,278],{"class":236},"(inv.groupby(",[187,280,281],{"class":196},"\"region\"",[187,283,284],{"class":236},")[",[187,286,287],{"class":196},"\"amount\"",[187,289,290],{"class":236},"].describe().round(",[187,292,293],{"class":274},"1",[187,295,296],{"class":236},"))\n",[10,298,299],{},"North's mean is over 6,000 while its median is about 400 — a gap that large always means extreme values. South's minimum is negative, which for invoices may be a credit note or a mistake. Neither fact needs a formula to see, but both need rules to catch automatically on next month's data.",[172,301,303],{"id":302},"step-2-flag-values-outside-the-iqr-fences","Step 2 — Flag values outside the IQR fences",[10,305,306],{},"The interquartile range method uses the 25th and 75th percentiles, which ignore extreme values, and flags anything more than 1.5 IQRs beyond them:",[177,308,310],{"className":223,"code":309,"language":225,"meta":182,"style":182},"def iqr_flags(s, k=1.5):\n    q1, q3 = s.quantile([0.25, 0.75])\n    iqr = q3 - q1\n    low, high = q1 - k * iqr, q3 + k * iqr\n    return (s \u003C low) | (s > high), (low, high)\n\nflags, fences = iqr_flags(inv[\"amount\"])\nprint(\"fences:\", tuple(round(f, 1) for f in fences))\nprint(inv.loc[flags, [\"invoice\", \"region\", \"amount\"]].to_string(index=False))\n",[184,311,312,332,354,370,401,427,432,447,488],{"__ignoreMap":182},[187,313,314,317,321,324,326,329],{"class":47,"line":189},[187,315,316],{"class":232},"def",[187,318,320],{"class":319},"s_Opv"," iqr_flags",[187,322,323],{"class":236},"(s, k",[187,325,259],{"class":232},[187,327,328],{"class":274},"1.5",[187,330,331],{"class":236},"):\n",[187,333,334,337,339,342,345,348,351],{"class":47,"line":246},[187,335,336],{"class":236},"    q1, q3 ",[187,338,259],{"class":232},[187,340,341],{"class":236}," s.quantile([",[187,343,344],{"class":274},"0.25",[187,346,347],{"class":236},", ",[187,349,350],{"class":274},"0.75",[187,352,353],{"class":236},"])\n",[187,355,356,359,361,364,367],{"class":47,"line":253},[187,357,358],{"class":236},"    iqr ",[187,360,259],{"class":232},[187,362,363],{"class":236}," q3 ",[187,365,366],{"class":232},"-",[187,368,369],{"class":236}," q1\n",[187,371,372,375,377,380,382,385,388,391,394,396,398],{"class":47,"line":271},[187,373,374],{"class":236},"    low, high ",[187,376,259],{"class":232},[187,378,379],{"class":236}," q1 ",[187,381,366],{"class":232},[187,383,384],{"class":236}," k ",[187,386,387],{"class":232},"*",[187,389,390],{"class":236}," iqr, q3 ",[187,392,393],{"class":232},"+",[187,395,384],{"class":236},[187,397,387],{"class":232},[187,399,400],{"class":236}," iqr\n",[187,402,404,407,410,413,416,419,421,424],{"class":47,"line":403},5,[187,405,406],{"class":232},"    return",[187,408,409],{"class":236}," (s ",[187,411,412],{"class":232},"\u003C",[187,414,415],{"class":236}," low) ",[187,417,418],{"class":232},"|",[187,420,409],{"class":236},[187,422,423],{"class":232},">",[187,425,426],{"class":236}," high), (low, high)\n",[187,428,430],{"class":47,"line":429},6,[187,431,250],{"emptyLinePlaceholder":249},[187,433,435,438,440,443,445],{"class":47,"line":434},7,[187,436,437],{"class":236},"flags, fences ",[187,439,259],{"class":232},[187,441,442],{"class":236}," iqr_flags(inv[",[187,444,287],{"class":196},[187,446,353],{"class":236},[187,448,450,452,455,458,460,463,465,468,471,473,476,479,482,485],{"class":47,"line":449},8,[187,451,275],{"class":274},[187,453,454],{"class":236},"(",[187,456,457],{"class":196},"\"fences:\"",[187,459,347],{"class":236},[187,461,462],{"class":274},"tuple",[187,464,454],{"class":236},[187,466,467],{"class":274},"round",[187,469,470],{"class":236},"(f, ",[187,472,293],{"class":274},[187,474,475],{"class":236},") ",[187,477,478],{"class":232},"for",[187,480,481],{"class":236}," f ",[187,483,484],{"class":232},"in",[187,486,487],{"class":236}," fences))\n",[187,489,491,493,496,499,501,503,505,507,510,514,516,519],{"class":47,"line":490},9,[187,492,275],{"class":274},[187,494,495],{"class":236},"(inv.loc[flags, [",[187,497,498],{"class":196},"\"invoice\"",[187,500,347],{"class":236},[187,502,281],{"class":196},[187,504,347],{"class":236},[187,506,287],{"class":196},[187,508,509],{"class":236},"]].to_string(",[187,511,513],{"class":512},"sa561","index",[187,515,259],{"class":232},[187,517,518],{"class":274},"False",[187,520,296],{"class":236},[10,522,523,524,527,528,531],{},"Across the whole company, only the 48,000 invoice falls outside the fences — the fences are wide because the two regions have different typical amounts. The 690 and the -40 are inside, which is the first sign that the test should run within each region (Step 5). ",[184,525,526],{},"k=1.5"," is the conventional multiplier; ",[184,529,530],{},"k=3"," flags only extreme values.",[172,533,535],{"id":534},"step-3-see-why-z-scores-miss-what-you-are-looking-for","Step 3 — See why z-scores miss what you are looking for",[10,537,538],{},"The z-score measures distance from the mean in standard deviations, and a common rule flags anything beyond 3. On this data it is a lesson in why that rule is unreliable:",[177,540,542],{"className":223,"code":541,"language":225,"meta":182,"style":182},"north = inv[inv[\"region\"] == \"North\"][\"amount\"]\nz = (north - north.mean()) \u002F north.std()\nprint(z.round(2).tolist())\n",[184,543,544,573,594],{"__ignoreMap":182},[187,545,546,549,551,554,556,559,562,565,568,570],{"class":47,"line":189},[187,547,548],{"class":236},"north ",[187,550,259],{"class":232},[187,552,553],{"class":236}," inv[inv[",[187,555,281],{"class":196},[187,557,558],{"class":236},"] ",[187,560,561],{"class":232},"==",[187,563,564],{"class":196}," \"North\"",[187,566,567],{"class":236},"][",[187,569,287],{"class":196},[187,571,572],{"class":236},"]\n",[187,574,575,578,580,583,585,588,591],{"class":47,"line":246},[187,576,577],{"class":236},"z ",[187,579,259],{"class":232},[187,581,582],{"class":236}," (north ",[187,584,366],{"class":232},[187,586,587],{"class":236}," north.mean()) ",[187,589,590],{"class":232},"\u002F",[187,592,593],{"class":236}," north.std()\n",[187,595,596,598,601,604],{"class":47,"line":253},[187,597,275],{"class":274},[187,599,600],{"class":236},"(z.round(",[187,602,603],{"class":274},"2",[187,605,606],{"class":236},").tolist())\n",[10,608,609],{},"The 48,000 invoice has a z-score of only about 2.5. The outlier inflates both the mean and the standard deviation it is measured against, pulling its own score down; with eight values, no single value can reach a z-score of 3 at all. Z-scores work for large samples from roughly bell-shaped data with modest outliers. For finding the extreme values themselves, use a method whose yardstick the outliers cannot move.",[20,611,29,617,29,620,29,623,29,627,29,633,29,639,29,645,29,649,29,654,29,657,29,661,29,666,29,671,29,674,29,678,29,680],{"viewBox":612,"role":23,"ariaLabelledBy":613,"xmlns":27,"style":616},"4 0 744 172",[614,615],"out-z-t","out-z-d","width:100%;max-width:744px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[31,618,619],{"id":614},"The same invoice measured three ways",[35,621,622],{"id":615},"Bars comparing how strongly the 48,000 North invoice is flagged. Its z-score is about 2.5, below the usual threshold of 3. Its robust score based on the median and MAD is in the thousands. Its distance beyond the upper IQR fence is also far past the threshold.",[39,624],{"x":41,"y":42,"width":625,"height":626,"fill":45},"744","172",[55,628,632],{"x":629,"y":630,"style":631},"380.0","31.0","font-size:14px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","North invoice N008 = 48,000",[39,634],{"x":156,"y":635,"width":636,"height":80,"rx":637,"fill":638,"stroke":52,"style":53},"48.0","208.1","7","#e7ebef",[55,640,644],{"x":641,"y":642,"style":643},"124.0","66.5","font-size:12.5px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","z-score (threshold 3)",[39,646],{"x":647,"y":635,"width":648,"height":80,"rx":637,"fill":165,"stroke":157,"style":85},"238.1","212.1",[55,650,653],{"x":651,"y":642,"style":652},"458.1","font-size:12px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:start","2.47 — missed",[39,655],{"x":156,"y":656,"width":636,"height":80,"rx":637,"fill":638,"stroke":52,"style":53},"88.0",[55,658,660],{"x":641,"y":659,"style":643},"106.5","robust score (threshold 3.5)",[39,662],{"x":647,"y":656,"width":663,"height":80,"rx":637,"fill":664,"stroke":665,"style":85},"300.5","#d9f4f1","var(--teal,#0f9488)",[55,667,670],{"x":668,"y":659,"style":669},"546.6","font-size:12px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:start","≈ 3,200 — flagged",[39,672],{"x":156,"y":673,"width":636,"height":80,"rx":637,"fill":638,"stroke":52,"style":53},"128.0",[55,675,677],{"x":641,"y":676,"style":643},"146.5","IQR rule",[39,679],{"x":647,"y":673,"width":663,"height":80,"rx":637,"fill":664,"stroke":665,"style":85},[55,681,682],{"x":668,"y":676,"style":669},"far above the fence — flagged",[172,684,686],{"id":685},"step-4-use-a-robust-score-based-on-the-median","Step 4 — Use a robust score based on the median",[10,688,689],{},"A robust z-score replaces the mean with the median and the standard deviation with the median absolute deviation (MAD), neither of which an outlier can drag far. The 0.6745 constant scales MAD to match the standard deviation for normal data, and a threshold of 3.5 is the usual choice:",[177,691,693],{"className":223,"code":692,"language":225,"meta":182,"style":182},"def robust_scores(s):\n    median = s.median()\n    mad = (s - median).abs().median()\n    return 0.6745 * (s - median) \u002F mad\n\nprint(robust_scores(north).round(1).tolist())\n",[184,694,695,705,715,729,751,755],{"__ignoreMap":182},[187,696,697,699,702],{"class":47,"line":189},[187,698,316],{"class":232},[187,700,701],{"class":319}," robust_scores",[187,703,704],{"class":236},"(s):\n",[187,706,707,710,712],{"class":47,"line":246},[187,708,709],{"class":236},"    median ",[187,711,259],{"class":232},[187,713,714],{"class":236}," s.median()\n",[187,716,717,720,722,724,726],{"class":47,"line":253},[187,718,719],{"class":236},"    mad ",[187,721,259],{"class":232},[187,723,409],{"class":236},[187,725,366],{"class":232},[187,727,728],{"class":236}," median).abs().median()\n",[187,730,731,733,736,739,741,743,746,748],{"class":47,"line":271},[187,732,406],{"class":232},[187,734,735],{"class":274}," 0.6745",[187,737,738],{"class":232}," *",[187,740,409],{"class":236},[187,742,366],{"class":232},[187,744,745],{"class":236}," median) ",[187,747,590],{"class":232},[187,749,750],{"class":236}," mad\n",[187,752,753],{"class":47,"line":403},[187,754,250],{"emptyLinePlaceholder":249},[187,756,757,759,762,764],{"class":47,"line":429},[187,758,275],{"class":274},[187,760,761],{"class":236},"(robust_scores(north).round(",[187,763,293],{"class":274},[187,765,606],{"class":236},[10,767,768],{},"The 48,000 invoice now scores in the thousands, and the ordinary invoices stay close to zero. One caution: if more than half the values are identical, MAD is zero and the score divides by zero. Guard against it by falling back to the IQR rule for such columns.",[172,770,772],{"id":771},"step-5-detect-outliers-within-each-group","Step 5 — Detect outliers within each group",[10,774,775,776,779],{},"A 690 invoice is unremarkable company-wide and very unusual for South, where invoices sit around 125. Compare each row with its own group by computing the statistics with ",[184,777,778],{},"groupby().transform",", which returns values aligned to the original rows:",[177,781,783],{"className":223,"code":782,"language":225,"meta":182,"style":182},"g = inv.groupby(\"region\")[\"amount\"]\nq1 = g.transform(lambda s: s.quantile(0.25))\nq3 = g.transform(lambda s: s.quantile(0.75))\niqr = q3 - q1\ninv[\"outlier\"] = (inv[\"amount\"] \u003C q1 - 1.5 * iqr) | (inv[\"amount\"] > q3 + 1.5 * iqr)\nprint(inv.loc[inv[\"outlier\"], [\"invoice\", \"region\", \"amount\"]].to_string(index=False))\n",[184,784,785,803,823,840,853,907],{"__ignoreMap":182},[187,786,787,790,792,795,797,799,801],{"class":47,"line":189},[187,788,789],{"class":236},"g ",[187,791,259],{"class":232},[187,793,794],{"class":236}," inv.groupby(",[187,796,281],{"class":196},[187,798,284],{"class":236},[187,800,287],{"class":196},[187,802,572],{"class":236},[187,804,805,808,810,813,816,819,821],{"class":47,"line":246},[187,806,807],{"class":236},"q1 ",[187,809,259],{"class":232},[187,811,812],{"class":236}," g.transform(",[187,814,815],{"class":232},"lambda",[187,817,818],{"class":236}," s: s.quantile(",[187,820,344],{"class":274},[187,822,296],{"class":236},[187,824,825,828,830,832,834,836,838],{"class":47,"line":253},[187,826,827],{"class":236},"q3 ",[187,829,259],{"class":232},[187,831,812],{"class":236},[187,833,815],{"class":232},[187,835,818],{"class":236},[187,837,350],{"class":274},[187,839,296],{"class":236},[187,841,842,845,847,849,851],{"class":47,"line":271},[187,843,844],{"class":236},"iqr ",[187,846,259],{"class":232},[187,848,363],{"class":236},[187,850,366],{"class":232},[187,852,369],{"class":236},[187,854,855,858,861,863,865,868,870,872,874,876,878,881,883,886,888,890,892,894,896,898,900,902,904],{"class":47,"line":403},[187,856,857],{"class":236},"inv[",[187,859,860],{"class":196},"\"outlier\"",[187,862,558],{"class":236},[187,864,259],{"class":232},[187,866,867],{"class":236}," (inv[",[187,869,287],{"class":196},[187,871,558],{"class":236},[187,873,412],{"class":232},[187,875,379],{"class":236},[187,877,366],{"class":232},[187,879,880],{"class":274}," 1.5",[187,882,738],{"class":232},[187,884,885],{"class":236}," iqr) ",[187,887,418],{"class":232},[187,889,867],{"class":236},[187,891,287],{"class":196},[187,893,558],{"class":236},[187,895,423],{"class":232},[187,897,363],{"class":236},[187,899,393],{"class":232},[187,901,880],{"class":274},[187,903,738],{"class":232},[187,905,906],{"class":236}," iqr)\n",[187,908,909,911,914,916,919,921,923,925,927,929,931,933,935,937],{"class":47,"line":429},[187,910,275],{"class":274},[187,912,913],{"class":236},"(inv.loc[inv[",[187,915,860],{"class":196},[187,917,918],{"class":236},"], [",[187,920,498],{"class":196},[187,922,347],{"class":236},[187,924,281],{"class":196},[187,926,347],{"class":236},[187,928,287],{"class":196},[187,930,509],{"class":236},[187,932,513],{"class":512},[187,934,259],{"class":232},[187,936,518],{"class":274},[187,938,296],{"class":236},[10,940,941],{},"Per region, the test flags North's 48,000, South's 690 and South's -40 — the three values a person looking at each region would question. Grouped detection is almost always the right default for business data, where \"normal\" depends on the product, region, customer or month.",[172,943,945],{"id":944},"step-6-add-the-rules-statistics-cannot-know","Step 6 — Add the rules statistics cannot know",[10,947,948],{},"Some values are wrong whatever the distribution says. A negative invoice is either a credit note, which belongs in a different document type, or an error; a quantity of zero on an order line is suspicious; a delivery date before the order date is impossible. Encode those as explicit rules and report them alongside the statistical flags, each with a reason:",[177,950,952],{"className":223,"code":951,"language":225,"meta":182,"style":182},"reasons = pd.Series(\"\", index=inv.index)\nreasons[inv[\"outlier\"]] += \"unusual for region; \"\nreasons[inv[\"amount\"] \u003C= 0] += \"not a positive amount; \"\ninv[\"reason\"] = reasons.str.rstrip(\"; \")\nreview = inv[inv[\"reason\"] != \"\"]\nprint(review[[\"invoice\", \"region\", \"amount\", \"reason\"]].to_string(index=False))\n",[184,953,954,976,992,1013,1032,1053],{"__ignoreMap":182},[187,955,956,959,961,964,967,969,971,973],{"class":47,"line":189},[187,957,958],{"class":236},"reasons ",[187,960,259],{"class":232},[187,962,963],{"class":236}," pd.Series(",[187,965,966],{"class":196},"\"\"",[187,968,347],{"class":236},[187,970,513],{"class":512},[187,972,259],{"class":232},[187,974,975],{"class":236},"inv.index)\n",[187,977,978,981,983,986,989],{"class":47,"line":246},[187,979,980],{"class":236},"reasons[inv[",[187,982,860],{"class":196},[187,984,985],{"class":236},"]] ",[187,987,988],{"class":232},"+=",[187,990,991],{"class":196}," \"unusual for region; \"\n",[187,993,994,996,998,1000,1003,1006,1008,1010],{"class":47,"line":253},[187,995,980],{"class":236},[187,997,287],{"class":196},[187,999,558],{"class":236},[187,1001,1002],{"class":232},"\u003C=",[187,1004,1005],{"class":274}," 0",[187,1007,558],{"class":236},[187,1009,988],{"class":232},[187,1011,1012],{"class":196}," \"not a positive amount; \"\n",[187,1014,1015,1017,1020,1022,1024,1027,1030],{"class":47,"line":271},[187,1016,857],{"class":236},[187,1018,1019],{"class":196},"\"reason\"",[187,1021,558],{"class":236},[187,1023,259],{"class":232},[187,1025,1026],{"class":236}," reasons.str.rstrip(",[187,1028,1029],{"class":196},"\"; \"",[187,1031,268],{"class":236},[187,1033,1034,1037,1039,1041,1043,1045,1048,1051],{"class":47,"line":403},[187,1035,1036],{"class":236},"review ",[187,1038,259],{"class":232},[187,1040,553],{"class":236},[187,1042,1019],{"class":196},[187,1044,558],{"class":236},[187,1046,1047],{"class":232},"!=",[187,1049,1050],{"class":196}," \"\"",[187,1052,572],{"class":236},[187,1054,1055,1057,1060,1062,1064,1066,1068,1070,1072,1074,1076,1078,1080,1082],{"class":47,"line":429},[187,1056,275],{"class":274},[187,1058,1059],{"class":236},"(review[[",[187,1061,498],{"class":196},[187,1063,347],{"class":236},[187,1065,281],{"class":196},[187,1067,347],{"class":236},[187,1069,287],{"class":196},[187,1071,347],{"class":236},[187,1073,1019],{"class":196},[187,1075,509],{"class":236},[187,1077,513],{"class":512},[187,1079,259],{"class":232},[187,1081,518],{"class":274},[187,1083,296],{"class":236},[10,1085,1086,1087,1091],{},"The -40 invoice now carries both reasons, which tells the reviewer why it was flagged twice. Rules are the part of outlier detection that improves most over time: each confirmed error suggests a rule that catches the next one without statistics. ",[14,1088,1090],{"href":1089},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fvalidate-excel-data-with-pandera-schemas\u002F","Validate Excel Data with pandera Schemas"," shows how to keep such rules in a declarative schema.",[20,1093,29,1099,29,1102,29,1105,29,1110,29,1113,29,1117,29,1121,29,1127,29,1131,29,1135,29,1139,29,1143,29,1147,29,1151,29,1154,29,1158,29,1160,29,1164,29,1166,29,1169,29,1171,29,1174,29,1176,29,1179,29,1182,29,1185,29,1187,29,1190,29,1192,29,1195,29,1197,29,1201,29,1203,29,1206,29,1208,29,1211,29,1213,29,1216,29,1218,29,1221,29,1225,29,1229,29,1231,29,1233,29,1236,29,1239,29,1241,29,1244,29,1246,29,1248,29,1250,29,1253,29,1255],{"viewBox":1094,"role":23,"ariaLabelledBy":1095,"xmlns":27,"style":1098},"169 0 421 200",[1096,1097],"out-review-t","out-review-d","width:100%;max-width:421px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[31,1100,1101],{"id":1096},"Flagged invoices with their reasons",[35,1103,1104],{"id":1097},"A review table of three invoices. N008 for 48,000 is unusual for its region. S006 for 690 is unusual for its region. S008 for -40 is both unusual for its region and not a positive amount.",[39,1106],{"x":1107,"y":42,"width":1108,"height":1109,"fill":45},"169","421","200",[55,1111,1112],{"x":629,"y":630,"style":631},"Review sheet — flag, do not delete",[39,1114],{"x":1115,"y":635,"width":1116,"height":77,"rx":42,"fill":638,"stroke":52,"style":53},"185.9","32.0",[39,1118],{"x":1119,"y":635,"width":1120,"height":77,"rx":42,"fill":638,"stroke":52,"style":53},"217.9","68.9",[55,1122,1126],{"x":1123,"y":1124,"style":1125},"252.3","65.0","font-size:11px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","A",[39,1128],{"x":1129,"y":635,"width":1130,"height":77,"rx":42,"fill":638,"stroke":52,"style":53},"286.8","63.6",[55,1132,1134],{"x":1133,"y":1124,"style":1125},"318.6","B",[39,1136],{"x":1137,"y":635,"width":1138,"height":77,"rx":42,"fill":638,"stroke":52,"style":53},"350.4","70.9",[55,1140,1142],{"x":1141,"y":1124,"style":1125},"385.8","C",[39,1144],{"x":1145,"y":635,"width":1146,"height":77,"rx":42,"fill":638,"stroke":52,"style":53},"421.2","152.9",[55,1148,1150],{"x":1149,"y":1124,"style":1125},"497.7","D",[39,1152],{"x":1115,"y":1153,"width":1116,"height":80,"rx":42,"fill":638,"stroke":52,"style":53},"72.0",[55,1155,293],{"x":1156,"y":1157,"style":1125},"201.9","91.0",[39,1159],{"x":1119,"y":1153,"width":1120,"height":80,"rx":42,"fill":45,"stroke":52,"style":53},[55,1161,1163],{"x":1123,"y":1157,"style":1162},"font-size:12px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","invoice",[39,1165],{"x":1129,"y":1153,"width":1130,"height":80,"rx":42,"fill":45,"stroke":52,"style":53},[55,1167,1168],{"x":1133,"y":1157,"style":1162},"region",[39,1170],{"x":1137,"y":1153,"width":1138,"height":80,"rx":42,"fill":45,"stroke":52,"style":53},[55,1172,1173],{"x":1141,"y":1157,"style":1162},"amount",[39,1175],{"x":1145,"y":1153,"width":1146,"height":80,"rx":42,"fill":45,"stroke":52,"style":53},[55,1177,1178],{"x":1149,"y":1157,"style":1162},"reason",[39,1180],{"x":1115,"y":1181,"width":1116,"height":80,"rx":42,"fill":638,"stroke":52,"style":53},"100.0",[55,1183,603],{"x":1156,"y":1184,"style":1125},"119.0",[39,1186],{"x":1119,"y":1181,"width":1120,"height":80,"rx":42,"fill":45,"stroke":52,"style":53},[55,1188,119],{"x":1123,"y":1184,"style":1189},"font-size:12px;font-weight:400;fill:var(--text,#172033);text-anchor:middle",[39,1191],{"x":1129,"y":1181,"width":1130,"height":80,"rx":42,"fill":45,"stroke":52,"style":53},[55,1193,1194],{"x":1133,"y":1184,"style":1189},"North",[39,1196],{"x":1137,"y":1181,"width":1138,"height":80,"rx":42,"fill":165,"stroke":157,"style":85},[55,1198,1200],{"x":1141,"y":1184,"style":1199},"font-size:12px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","48000",[39,1202],{"x":1145,"y":1181,"width":1146,"height":80,"rx":42,"fill":45,"stroke":52,"style":53},[55,1204,1205],{"x":1149,"y":1184,"style":1189},"unusual for region",[39,1207],{"x":1115,"y":673,"width":1116,"height":80,"rx":42,"fill":638,"stroke":52,"style":53},[55,1209,126],{"x":1156,"y":1210,"style":1125},"147.0",[39,1212],{"x":1119,"y":673,"width":1120,"height":80,"rx":42,"fill":45,"stroke":52,"style":53},[55,1214,1215],{"x":1123,"y":1210,"style":1189},"S006",[39,1217],{"x":1129,"y":673,"width":1130,"height":80,"rx":42,"fill":45,"stroke":52,"style":53},[55,1219,1220],{"x":1133,"y":1210,"style":1189},"South",[39,1222],{"x":1137,"y":673,"width":1138,"height":80,"rx":42,"fill":1223,"stroke":1224,"style":85},"#fdefd8","var(--gold,#b4740a)",[55,1226,1228],{"x":1141,"y":1210,"style":1227},"font-size:12px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","690",[39,1230],{"x":1145,"y":673,"width":1146,"height":80,"rx":42,"fill":45,"stroke":52,"style":53},[55,1232,1205],{"x":1149,"y":1210,"style":1189},[39,1234],{"x":1115,"y":1235,"width":1116,"height":80,"rx":42,"fill":638,"stroke":52,"style":53},"156.0",[55,1237,41],{"x":1156,"y":1238,"style":1125},"175.0",[39,1240],{"x":1119,"y":1235,"width":1120,"height":80,"rx":42,"fill":45,"stroke":52,"style":53},[55,1242,1243],{"x":1123,"y":1238,"style":1189},"S008",[39,1245],{"x":1129,"y":1235,"width":1130,"height":80,"rx":42,"fill":45,"stroke":52,"style":53},[55,1247,1220],{"x":1133,"y":1238,"style":1189},[39,1249],{"x":1137,"y":1235,"width":1138,"height":80,"rx":42,"fill":165,"stroke":157,"style":85},[55,1251,1252],{"x":1141,"y":1238,"style":1199},"-40",[39,1254],{"x":1145,"y":1235,"width":1146,"height":80,"rx":42,"fill":45,"stroke":52,"style":53},[55,1256,1257],{"x":1149,"y":1238,"style":1189},"unusual; not positive",[172,1259,1261],{"id":1260},"step-7-flag-them-in-the-workbook","Step 7 — Flag them in the workbook",[10,1263,1264],{},"Write the data with a flag column and colour the flagged rows, so the people who own the data can review them where they already work. Keep every row; nothing is deleted:",[177,1266,1268],{"className":223,"code":1267,"language":225,"meta":182,"style":182},"from openpyxl import load_workbook\nfrom openpyxl.styles import PatternFill\n\ninv.drop(columns=\"outlier\").to_excel(\"invoices_flagged.xlsx\", index=False)\nwb = load_workbook(\"invoices_flagged.xlsx\")\nws = wb.active\namber = PatternFill(\"solid\", fgColor=\"FFEB9C\")\nfor row in ws.iter_rows(min_row=2):\n    if row[3].value:                         # the reason column\n        for cell in row:\n            cell.fill = amber\nwb.save(\"invoices_flagged.xlsx\")\nprint(sum(1 for r in ws.iter_rows(min_row=2) if r[3].value), \"rows flagged\")\n",[184,1269,1270,1283,1295,1299,1327,1341,1351,1376,1397,1414,1428,1439,1449],{"__ignoreMap":182},[187,1271,1272,1275,1278,1280],{"class":47,"line":189},[187,1273,1274],{"class":232},"from",[187,1276,1277],{"class":236}," openpyxl ",[187,1279,233],{"class":232},[187,1281,1282],{"class":236}," load_workbook\n",[187,1284,1285,1287,1290,1292],{"class":47,"line":246},[187,1286,1274],{"class":232},[187,1288,1289],{"class":236}," openpyxl.styles ",[187,1291,233],{"class":232},[187,1293,1294],{"class":236}," PatternFill\n",[187,1296,1297],{"class":47,"line":253},[187,1298,250],{"emptyLinePlaceholder":249},[187,1300,1301,1304,1307,1309,1311,1314,1317,1319,1321,1323,1325],{"class":47,"line":271},[187,1302,1303],{"class":236},"inv.drop(",[187,1305,1306],{"class":512},"columns",[187,1308,259],{"class":232},[187,1310,860],{"class":196},[187,1312,1313],{"class":236},").to_excel(",[187,1315,1316],{"class":196},"\"invoices_flagged.xlsx\"",[187,1318,347],{"class":236},[187,1320,513],{"class":512},[187,1322,259],{"class":232},[187,1324,518],{"class":274},[187,1326,268],{"class":236},[187,1328,1329,1332,1334,1337,1339],{"class":47,"line":403},[187,1330,1331],{"class":236},"wb ",[187,1333,259],{"class":232},[187,1335,1336],{"class":236}," load_workbook(",[187,1338,1316],{"class":196},[187,1340,268],{"class":236},[187,1342,1343,1346,1348],{"class":47,"line":429},[187,1344,1345],{"class":236},"ws ",[187,1347,259],{"class":232},[187,1349,1350],{"class":236}," wb.active\n",[187,1352,1353,1356,1358,1361,1364,1366,1369,1371,1374],{"class":47,"line":434},[187,1354,1355],{"class":236},"amber ",[187,1357,259],{"class":232},[187,1359,1360],{"class":236}," PatternFill(",[187,1362,1363],{"class":196},"\"solid\"",[187,1365,347],{"class":236},[187,1367,1368],{"class":512},"fgColor",[187,1370,259],{"class":232},[187,1372,1373],{"class":196},"\"FFEB9C\"",[187,1375,268],{"class":236},[187,1377,1378,1380,1383,1385,1388,1391,1393,1395],{"class":47,"line":449},[187,1379,478],{"class":232},[187,1381,1382],{"class":236}," row ",[187,1384,484],{"class":232},[187,1386,1387],{"class":236}," ws.iter_rows(",[187,1389,1390],{"class":512},"min_row",[187,1392,259],{"class":232},[187,1394,603],{"class":274},[187,1396,331],{"class":236},[187,1398,1399,1402,1405,1407,1410],{"class":47,"line":490},[187,1400,1401],{"class":232},"    if",[187,1403,1404],{"class":236}," row[",[187,1406,126],{"class":274},[187,1408,1409],{"class":236},"].value:                         ",[187,1411,1413],{"class":1412},"s-wDw","# the reason column\n",[187,1415,1417,1420,1423,1425],{"class":47,"line":1416},10,[187,1418,1419],{"class":232},"        for",[187,1421,1422],{"class":236}," cell ",[187,1424,484],{"class":232},[187,1426,1427],{"class":236}," row:\n",[187,1429,1431,1434,1436],{"class":47,"line":1430},11,[187,1432,1433],{"class":236},"            cell.fill ",[187,1435,259],{"class":232},[187,1437,1438],{"class":236}," amber\n",[187,1440,1442,1445,1447],{"class":47,"line":1441},12,[187,1443,1444],{"class":236},"wb.save(",[187,1446,1316],{"class":196},[187,1448,268],{"class":236},[187,1450,1452,1454,1456,1459,1461,1463,1466,1469,1471,1473,1475,1477,1479,1481,1484,1487,1489,1492,1495],{"class":47,"line":1451},13,[187,1453,275],{"class":274},[187,1455,454],{"class":236},[187,1457,1458],{"class":274},"sum",[187,1460,454],{"class":236},[187,1462,293],{"class":274},[187,1464,1465],{"class":232}," for",[187,1467,1468],{"class":236}," r ",[187,1470,484],{"class":232},[187,1472,1387],{"class":236},[187,1474,1390],{"class":512},[187,1476,259],{"class":232},[187,1478,603],{"class":274},[187,1480,475],{"class":236},[187,1482,1483],{"class":232},"if",[187,1485,1486],{"class":236}," r[",[187,1488,126],{"class":274},[187,1490,1491],{"class":236},"].value), ",[187,1493,1494],{"class":196},"\"rows flagged\"",[187,1496,268],{"class":236},[10,1498,1499,1500,1504],{},"Once reviewed, corrections go back to the source system; a note in the reason column, such as \"confirmed: bulk order\", lets the next run skip a value that is unusual but real. ",[14,1501,1503],{"href":1502},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fhighlight-invalid-cells-in-excel-with-python\u002F","Highlight Invalid Cells in Excel with Python"," covers highlighting individual cells rather than rows.",[172,1506,1508],{"id":1507},"common-pitfalls","Common pitfalls",[1510,1511,1512,1520,1526,1532,1538],"ul",{},[1513,1514,1515,1519],"li",{},[1516,1517,1518],"strong",{},"Z-scores on small samples."," Outliers inflate the yardstick; use IQR or robust scores.",[1513,1521,1522,1525],{},[1516,1523,1524],{},"Company-wide thresholds on grouped data."," Compare each row with its own group.",[1513,1527,1528,1531],{},[1516,1529,1530],{},"Deleting flagged rows."," Totals become wrong; flag and review instead.",[1513,1533,1534,1537],{},[1516,1535,1536],{},"MAD of zero."," Columns with many identical values need a fallback rule.",[1513,1539,1540,1543],{},[1516,1541,1542],{},"Re-flagging confirmed values every run."," Keep a list of reviewed invoice numbers and exclude them, or reviewers stop reading the list.",[172,1545,1547],{"id":1546},"performance-notes","Performance notes",[10,1549,1550,1553,1554,1557,1558,1560,1561,1564],{},[184,1551,1552],{},"quantile"," and ",[184,1555,1556],{},"median"," are vectorised and fast on millions of rows. ",[184,1559,778],{}," with a lambda calls Python once per group, which is fine for hundreds of groups; for tens of thousands of groups, compute the quartiles once with ",[184,1562,1563],{},"g.quantile([0.25, 0.75]).unstack()"," and merge them back on the group key.",[172,1566,1568],{"id":1567},"conclusion","Conclusion",[10,1570,1571,1572,1574],{},"Look at the data per group first, then flag outliers with IQR fences or median-based robust scores — not plain z-scores, which the outliers themselves distort. Run the tests within each group with ",[184,1573,778],{},", add business rules that no statistic can infer, record a reason for every flag, and mark the rows in the workbook for review instead of deleting them.",[172,1576,1578],{"id":1577},"frequently-asked-questions","Frequently asked questions",[10,1580,1581,1584],{},[1516,1582,1583],{},"What is the IQR method?","\nIt flags values below Q1 - 1.5 x IQR or above Q3 + 1.5 x IQR, where Q1 and Q3 are the 25th and 75th percentiles and IQR is the gap between them. Because quartiles ignore extreme values, one huge outlier does not stretch the fences.",[10,1586,1587,1590],{},[1516,1588,1589],{},"Why did the z-score miss an obvious outlier?","\nA large outlier inflates the standard deviation it is measured against, so its own z-score shrinks. In small samples a single extreme value cannot reach a z-score of 3 at all. Use IQR or a median-based score when outliers are what you are looking for.",[10,1592,1593,1596],{},[1516,1594,1595],{},"Should I delete outliers?","\nNot automatically. An outlier may be a typo, a unit error or a genuine large order. Flag it, show it to someone who knows the data, and decide per case. Deleting real data to make charts look tidy produces wrong totals.",[10,1598,1599,1602],{},[1516,1600,1601],{},"How do I find outliers within each category?","\nCompute the statistics per group with groupby().transform so every row is compared with its own group. A 690 invoice is unusual in a region where invoices are around 125, even if it is ordinary for the company overall.",[172,1604,1606],{"id":1605},"related","Related",[1510,1608,1609,1615,1622,1629],{},[1513,1610,1611,1612,1614],{},"Up one level: ",[14,1613,17],{"href":16}," — the full cleaning workflow.",[1513,1616,1617,1621],{},[14,1618,1620],{"href":1619},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002F","Validating Excel Data with Python"," — rule-based checks before import.",[1513,1623,1624,1628],{},[14,1625,1627],{"href":1626},"\u002Fadvanced-data-transformation-and-cleaning\u002Fexcel-formula-equivalents-in-pandas\u002Frank-and-percentile-formulas-in-pandas\u002F","RANK and PERCENTILE Formulas in pandas"," — quantiles in more depth.",[1513,1630,1631,1635],{},[14,1632,1634],{"href":1633},"\u002Fadvanced-data-transformation-and-cleaning\u002Fapplying-conditional-formatting-with-openpyxl\u002Fhighlight-top-and-bottom-values-with-openpyxl\u002F","Highlight Top and Bottom Values in Excel with openpyxl"," — letting the workbook mark extremes itself.",[1637,1638,1639],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}",{"title":182,"searchDepth":246,"depth":246,"links":1641},[1642,1643,1644,1645,1646,1647,1648,1649,1650,1651,1652,1653,1654],{"id":174,"depth":246,"text":175},{"id":212,"depth":246,"text":213},{"id":302,"depth":246,"text":303},{"id":534,"depth":246,"text":535},{"id":685,"depth":246,"text":686},{"id":771,"depth":246,"text":772},{"id":944,"depth":246,"text":945},{"id":1260,"depth":246,"text":1261},{"id":1507,"depth":246,"text":1508},{"id":1546,"depth":246,"text":1547},{"id":1567,"depth":246,"text":1568},{"id":1577,"depth":246,"text":1578},{"id":1605,"depth":246,"text":1606},"2026-09-25","Find the invoice for 48,000 in a column of 400s, the negative quantity and the delivery that took 90 days. Compare IQR fences, z-scores and median-based robust scores on the same data, detect outliers within each group, add business rules, and flag rows in the workbook for review instead of deleting them.","md",[1659,1661,1663,1665],{"q":1583,"a":1660},"It flags values below Q1 - 1.5 x IQR or above Q3 + 1.5 x IQR, where Q1 and Q3 are the 25th and 75th percentiles and IQR is the gap between them. Because quartiles ignore extreme values, one huge outlier does not stretch the fences.",{"q":1589,"a":1662},"A large outlier inflates the standard deviation it is measured against, so its own z-score shrinks. In small samples a single extreme value cannot reach a z-score of 3 at all. Use IQR or a median-based score when outliers are what you are looking for.",{"q":1595,"a":1664},"Not automatically. An outlier may be a typo, a unit error or a genuine large order. Flag it, show it to someone who knows the data, and decide per case. Deleting real data to make charts look tidy produces wrong totals.",{"q":1601,"a":1666},"Compute the statistics per group with groupby().transform so every row is compared with its own group. A 690 invoice is unusual in a region where invoices are around 125, even if it is ordinary for the company overall.","detect outliers in excel data with pandas",null,{"breadcrumb":1670},[1671,1673,1676],{"name":1672,"item":590},"Home",{"name":1674,"item":1675},"Advanced Data Transformation and Cleaning for Excel Automation","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":1677,"item":16},"Cleaning Excel Data With Pandas: A Production Workflow","\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fdetect-outliers-in-excel-data-with-pandas",{"title":1680,"description":1681},"Find Outliers in Excel Data with pandas (IQR, Z-Score, MAD)","Detect outliers in Excel data with pandas: IQR fences with quantile, z-scores and why large outliers hide from them, robust median\u002FMAD scores, per-group outliers with groupby transform, business rules, and highlighting flagged rows.","detect-outliers-in-excel-data-with-pandas","advanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fdetect-outliers-in-excel-data-with-pandas\u002Findex",[1685,1686,1687],"data-validation","data-cleaning","pandas","how-to","LcAPds3PsmJQfeD7dq9lypgCiA0n_zKkD8zP2ExBcOQ",[1691,1695],{"title":1692,"path":1693,"stem":1694,"children":-1},"Convert Trailing Minus and CR\u002FDR Amounts with pandas","\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fconvert-trailing-minus-and-cr-dr-amounts-with-pandas","advanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fconvert-trailing-minus-and-cr-dr-amounts-with-pandas\u002Findex",{"title":1696,"path":1697,"stem":1698,"children":-1},"Extract Numbers and Codes from Text with pandas","\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fextract-numbers-and-codes-from-text-with-pandas","advanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fextract-numbers-and-codes-from-text-with-pandas\u002Findex",[1700,1702],{"path":1701,"title":1677},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas",{"path":1703,"title":1674},"\u002Fadvanced-data-transformation-and-cleaning",[],[1706,1709,1712,1715,1718,1721],{"path":1707,"title":1620,"kind":1708},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python","guide",{"path":1710,"title":1711,"kind":1688},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fadd-dropdown-data-validation-to-excel-with-openpyxl","Add Dropdown Data Validation to Excel with openpyxl",{"path":1713,"title":1714,"kind":1688},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fanonymise-personal-data-in-an-excel-file-with-python","Anonymise Personal Data in an Excel File with Python",{"path":1716,"title":1717,"kind":1688},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fbuild-a-data-quality-scorecard-for-an-excel-file","Build a Data Quality Scorecard for an Excel File",{"path":1719,"title":1720,"kind":1688},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fcheck-excel-data-types-with-pandas","Check Excel Data Types with pandas",{"path":1722,"title":1723,"kind":1688},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fcheck-references-between-excel-sheets","Check References Between Excel Sheets with pandas",1790972080646]