[{"data":1,"prerenderedAt":2221},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fremove-duplicate-words-and-extra-punctuation-from-text":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fremove-duplicate-words-and-extra-punctuation-from-text":2187,"ancestors:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fremove-duplicate-words-and-extra-punctuation-from-text":2196,"children:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fremove-duplicate-words-and-extra-punctuation-from-text":2201,"related:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fremove-duplicate-words-and-extra-punctuation-from-text":2202},{"id":4,"title":5,"body":6,"dateModified":2152,"datePublished":2152,"description":2153,"extension":2154,"faq":2155,"keyword":2164,"kind":2165,"meta":2166,"navigation":123,"path":2176,"seo":2177,"slug":2180,"stem":2181,"topics":2182,"type":2185,"__hash__":2186},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fremove-duplicate-words-and-extra-punctuation-from-text\u002Findex.md","Remove Duplicate Words and Extra Punctuation from Text",{"type":7,"value":8,"toc":2140},"minimark",[9,46,51,80,83,87,418,457,574,578,1044,1066,1070,1488,1499,1655,1659,1666,1862,1896,1965,1969,1984,1988,2034,2038,2041,2045,2059,2071,2083,2103,2107,2136],[10,11,12,13,17,18,17,21,17,24,27,28,31,32,35,36,39,40,45],"p",{},"Free-text columns — product descriptions, delivery notes, ticket summaries, survey comments — collect small mistakes that make reports look careless: ",[14,15,16],"em",{},"Delivered to to the warehouse",", ",[14,19,20],{},"Fragile!!!",[14,22,23],{},"Leave with neighbour , no 12",[14,25,26],{},"“Urgent”"," with curly quotes in one row and straight ones in the next. None of them changes the meaning, but they spoil printed reports, break exact matching and inflate the count of distinct values. They are also easy to fix with a handful of regular expressions, provided the rules are careful about what they change: a blanket rule that deletes every repeated word would also turn the island ",[14,29,30],{},"Bora Bora"," into ",[14,33,34],{},"Bora"," and the company ",[14,37,38],{},"Duran Duran Ltd"," into something else. This page builds a small text-cleaning function for pandas that removes accidental repeated words with an allowlist for genuine ones, collapses repeated punctuation, fixes spacing around punctuation, normalises quotes and dashes, and records every cell it changes so the cleaning can be reviewed. It belongs to ",[41,42,44],"a",{"href":43},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002F","Cleaning Excel Data with pandas",".",[47,48,50],"h2",{"id":49},"prerequisites","Prerequisites",[52,53,58],"pre",{"className":54,"code":55,"language":56,"meta":57,"style":57},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl\n","bash","",[59,60,61],"code",{"__ignoreMap":57},[62,63,66,70,74,77],"span",{"class":64,"line":65},"line",1,[62,67,69],{"class":68},"sMTad","pip",[62,71,73],{"class":72},"srMev"," install",[62,75,76],{"class":72}," pandas",[62,78,79],{"class":72}," openpyxl\n",[10,81,82],{},"The sample is ten delivery notes with the usual problems: repeated words, runs of exclamation and question marks, a space before a comma, curly quotes and an en dash, surrounding spaces, leading and trailing hyphens — and two legitimate repeats that must survive.",[47,84,86],{"id":85},"step-1-repeated-words-with-an-allowlist","Step 1 — Repeated words, with an allowlist",[52,88,92],{"className":89,"code":90,"language":91,"meta":57,"style":57},"language-python shiki shiki-themes github-light github-dark-high-contrast","import re\nimport pandas as pd\n\nnotes = pd.read_excel(\"ticket_notes.xlsx\")\nKEEP = {\"bora bora\", \"that that\", \"ok ok\", \"duran duran\"}          # genuine repeats\n\ndef drop_repeats(text):\n    def repl(m):\n        pair = f\"{m.group(1)} {m.group(2)}\".lower()\n        return m.group(0) if pair in KEEP else m.group(1)\n    return re.sub(r\"\\b(\\w+)\\s+(\\1)\\b\", repl, text, flags=re.IGNORECASE)\n\nfor t in notes[\"Note\"].head(7):\n    print(f\"{t!r:45} -> {drop_repeats(t)!r}\")\n","python",[59,93,94,104,118,125,143,181,186,199,211,257,293,347,352,378],{"__ignoreMap":57},[62,95,96,100],{"class":64,"line":65},[62,97,99],{"class":98},"s-kum","import",[62,101,103],{"class":102},"skGVy"," re\n",[62,105,107,109,112,115],{"class":64,"line":106},2,[62,108,99],{"class":98},[62,110,111],{"class":102}," pandas ",[62,113,114],{"class":98},"as",[62,116,117],{"class":102}," pd\n",[62,119,121],{"class":64,"line":120},3,[62,122,124],{"emptyLinePlaceholder":123},true,"\n",[62,126,128,131,134,137,140],{"class":64,"line":127},4,[62,129,130],{"class":102},"notes ",[62,132,133],{"class":98},"=",[62,135,136],{"class":102}," pd.read_excel(",[62,138,139],{"class":72},"\"ticket_notes.xlsx\"",[62,141,142],{"class":102},")\n",[62,144,146,150,153,156,159,161,164,166,169,171,174,177],{"class":64,"line":145},5,[62,147,149],{"class":148},"sP0c6","KEEP",[62,151,152],{"class":98}," =",[62,154,155],{"class":102}," {",[62,157,158],{"class":72},"\"bora bora\"",[62,160,17],{"class":102},[62,162,163],{"class":72},"\"that that\"",[62,165,17],{"class":102},[62,167,168],{"class":72},"\"ok ok\"",[62,170,17],{"class":102},[62,172,173],{"class":72},"\"duran duran\"",[62,175,176],{"class":102},"}          ",[62,178,180],{"class":179},"s-wDw","# genuine repeats\n",[62,182,184],{"class":64,"line":183},6,[62,185,124],{"emptyLinePlaceholder":123},[62,187,189,192,196],{"class":64,"line":188},7,[62,190,191],{"class":98},"def",[62,193,195],{"class":194},"s_Opv"," drop_repeats",[62,197,198],{"class":102},"(text):\n",[62,200,202,205,208],{"class":64,"line":201},8,[62,203,204],{"class":98},"    def",[62,206,207],{"class":194}," repl",[62,209,210],{"class":102},"(m):\n",[62,212,214,217,219,222,225,229,232,235,238,241,243,245,248,250,252,254],{"class":64,"line":213},9,[62,215,216],{"class":102},"        pair ",[62,218,133],{"class":98},[62,220,221],{"class":98}," f",[62,223,224],{"class":72},"\"",[62,226,228],{"class":227},"sSjpA","{",[62,230,231],{"class":102},"m.group(",[62,233,234],{"class":148},"1",[62,236,237],{"class":102},")",[62,239,240],{"class":227},"}",[62,242,155],{"class":227},[62,244,231],{"class":102},[62,246,247],{"class":148},"2",[62,249,237],{"class":102},[62,251,240],{"class":227},[62,253,224],{"class":72},[62,255,256],{"class":102},".lower()\n",[62,258,260,263,266,269,272,275,278,281,284,287,289,291],{"class":64,"line":259},10,[62,261,262],{"class":98},"        return",[62,264,265],{"class":102}," m.group(",[62,267,268],{"class":148},"0",[62,270,271],{"class":102},") ",[62,273,274],{"class":98},"if",[62,276,277],{"class":102}," pair ",[62,279,280],{"class":98},"in",[62,282,283],{"class":148}," KEEP",[62,285,286],{"class":98}," else",[62,288,265],{"class":102},[62,290,234],{"class":148},[62,292,142],{"class":102},[62,294,296,299,302,305,307,310,313,316,318,321,325,328,330,333,337,339,342,345],{"class":64,"line":295},11,[62,297,298],{"class":98},"    return",[62,300,301],{"class":102}," re.sub(",[62,303,304],{"class":98},"r",[62,306,224],{"class":72},[62,308,309],{"class":148},"\\b(\\w",[62,311,312],{"class":98},"+",[62,314,315],{"class":148},")\\s",[62,317,312],{"class":98},[62,319,320],{"class":148},"(",[62,322,324],{"class":323},"sx1sP","\\1",[62,326,327],{"class":148},")\\b",[62,329,224],{"class":72},[62,331,332],{"class":102},", repl, text, ",[62,334,336],{"class":335},"sa561","flags",[62,338,133],{"class":98},[62,340,341],{"class":102},"re.",[62,343,344],{"class":148},"IGNORECASE",[62,346,142],{"class":102},[62,348,350],{"class":64,"line":349},12,[62,351,124],{"emptyLinePlaceholder":123},[62,353,355,358,361,363,366,369,372,375],{"class":64,"line":354},13,[62,356,357],{"class":98},"for",[62,359,360],{"class":102}," t ",[62,362,280],{"class":98},[62,364,365],{"class":102}," notes[",[62,367,368],{"class":72},"\"Note\"",[62,370,371],{"class":102},"].head(",[62,373,374],{"class":148},"7",[62,376,377],{"class":102},"):\n",[62,379,381,384,386,389,391,393,396,399,401,404,406,409,412,414,416],{"class":64,"line":380},14,[62,382,383],{"class":148},"    print",[62,385,320],{"class":102},[62,387,388],{"class":98},"f",[62,390,224],{"class":72},[62,392,228],{"class":227},[62,394,395],{"class":102},"t",[62,397,398],{"class":98},"!r:45",[62,400,240],{"class":227},[62,402,403],{"class":72}," -> ",[62,405,228],{"class":227},[62,407,408],{"class":102},"drop_repeats(t)",[62,410,411],{"class":98},"!r",[62,413,240],{"class":227},[62,415,224],{"class":72},[62,417,142],{"class":102},[10,419,420,421,424,425,428,429,432,433,17,435,438,439,17,441,444,445,448,449,452,453,456],{},"The pattern ",[59,422,423],{},"\\b(\\w+)\\s+(\\1)\\b"," finds a word followed by whitespace and the same word again, as whole words thanks to the ",[59,426,427],{},"\\b"," boundaries — so ",[14,430,431],{},"the theatre"," is not touched. With ",[59,434,344],{},[14,436,437],{},"The the"," is caught as well. Instead of replacing every match blindly, the replacement is a function that looks the pair up in an allowlist and leaves genuine repeats alone: ",[14,440,30],{},[14,442,443],{},"that that"," in \"said that that was fine\", which is grammatical English, and ",[14,446,447],{},"OK OK"," style notes are kept, while ",[14,450,451],{},"to to"," and ",[14,454,455],{},"was was"," lose their second word. Keeping the allowlist small and visible in the code is the point — anyone reviewing the cleaning can see which repeats are treated as real.",[458,459,467,468,467,472,467,476,467,483,467,493,467,500,467,504,467,509,467,515,467,520,467,525,467,531,467,535,467,539,467,542,467,547,467,552,467,555,467,559,467,562,467,566,467,571],"svg",{"viewBox":460,"role":461,"ariaLabelledBy":462,"xmlns":465,"style":466},"4 0 752 108","img",[463,464],"txt-repeat-t","txt-repeat-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:752px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[469,470,471],"title",{"id":463},"Handling a repeated word",[473,474,475],"desc",{"id":464},"The regular expression finds a whole word followed by the same word. If the pair is in the allowlist, such as Bora Bora or that that, it is kept unchanged. Otherwise the second copy is removed.",[477,478],"rect",{"x":479,"y":268,"width":480,"height":481,"fill":482},"4","752","108","#ffffff",[477,484],{"x":485,"y":486,"width":487,"height":488,"rx":489,"fill":490,"stroke":491,"style":492},"20.0","16.0","153.0","76.5","12","#e7ebef","var(--line,#cdd5e6)","stroke-width:2px",[494,495,499],"text",{"x":496,"y":497,"style":498},"96.5","41.5","font-size:13.5px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","Find word + same",[494,501,503],{"x":496,"y":502,"style":498},"59.0","word",[494,505,508],{"x":496,"y":506,"style":507},"78.5","font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","\\b(\\w+)\\s+\\1\\b",[64,510],{"x1":511,"y1":512,"x2":513,"y2":512,"stroke":514,"style":492},"178.0","54.2","200.0","var(--brand,#5b5cf0)",[516,517],"polygon",{"points":518,"fill":519},"204.0,54.2 195.0,59.2 195.0,49.2","#5b5cf0",[477,521],{"x":522,"y":486,"width":487,"height":488,"rx":489,"fill":523,"stroke":524,"style":492},"209.0","#fdefd8","var(--gold,#b4740a)",[494,526,530],{"x":527,"y":528,"style":529},"285.5","50.2","font-size:13.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","In the allowlist?",[494,532,534],{"x":527,"y":533,"style":507},"69.8","Bora Bora, that that",[64,536],{"x1":537,"y1":512,"x2":538,"y2":512,"stroke":514,"style":492},"367.0","389.0",[516,540],{"points":541,"fill":519},"393.0,54.2 384.0,59.2 384.0,49.2",[477,543],{"x":544,"y":486,"width":487,"height":488,"rx":489,"fill":545,"stroke":546,"style":492},"398.0","#d9f4f1","var(--teal,#0f9488)",[494,548,551],{"x":549,"y":528,"style":550},"474.5","font-size:13.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","yes",[494,553,554],{"x":549,"y":533,"style":507},"keep as written",[64,556],{"x1":557,"y1":512,"x2":558,"y2":512,"stroke":514,"style":492},"556.0","578.0",[516,560],{"points":561,"fill":519},"582.0,54.2 573.0,59.2 573.0,49.2",[477,563],{"x":564,"y":486,"width":487,"height":488,"rx":489,"fill":565,"stroke":514,"style":492},"587.0","#f0f4ff",[494,567,570],{"x":568,"y":528,"style":569},"663.5","font-size:13.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","no",[494,572,573],{"x":568,"y":533,"style":507},"drop the repeat",[47,575,577],{"id":576},"step-2-punctuation-quotes-dashes-and-spaces","Step 2 — Punctuation, quotes, dashes and spaces",[52,579,581],{"className":89,"code":580,"language":91,"meta":57,"style":57},"RULES = [\n    (r\"[“”„]\", '\"'), (r\"[‘’]\", \"'\"),                     # curly quotes -> straight\n    (r\"\\s*[–—]\\s*\", \" - \"),                              # en and em dashes -> spaced hyphen\n    (r\"\\s+([,.;:!?])\", r\"\\1\"),                           # no space before punctuation\n    (r\"([!?])\\1+\", r\"\\1\"),                               # !!! -> !   ?? -> ?\n    (r\"(?\u003C!\\.)\\.{2}(?!\\.)\", \".\"),                        # exactly two dots -> one (keep ... as an ellipsis)\n    (r\",{2,}\", \",\"),                                     # ,, -> ,\n    (r\"([,;:])(?=\\S)\", r\"\\1 \"),                          # space after , ; : when missing\n    (r\"\\s{2,}\", \" \"),                                    # runs of spaces -> one\n]\n\ndef clean_text(text):\n    if not isinstance(text, str):\n        return text\n    out = drop_repeats(text)\n    for pattern, repl in RULES:\n        out = re.sub(pattern, repl, out)\n    return out.strip(\" -–—,;:\")                      # keep a final full stop\n\nnotes[\"Clean\"] = notes[\"Note\"].map(clean_text)\nprint(notes[[\"Note\", \"Clean\"]].to_string(index=False))\n",[59,582,583,593,645,682,721,758,797,822,864,889,894,898,907,926,933,944,961,972,989,994,1015],{"__ignoreMap":57},[62,584,585,588,590],{"class":64,"line":65},[62,586,587],{"class":148},"RULES",[62,589,152],{"class":98},[62,591,592],{"class":102}," [\n",[62,594,595,598,600,602,605,608,611,613,615,618,621,623,625,627,630,632,634,636,639,642],{"class":64,"line":106},[62,596,597],{"class":102},"    (",[62,599,304],{"class":98},[62,601,224],{"class":72},[62,603,604],{"class":148},"[",[62,606,607],{"class":227},"“”„",[62,609,610],{"class":148},"]",[62,612,224],{"class":72},[62,614,17],{"class":102},[62,616,617],{"class":72},"'\"'",[62,619,620],{"class":102},"), (",[62,622,304],{"class":98},[62,624,224],{"class":72},[62,626,604],{"class":148},[62,628,629],{"class":227},"‘’",[62,631,610],{"class":148},[62,633,224],{"class":72},[62,635,17],{"class":102},[62,637,638],{"class":72},"\"'\"",[62,640,641],{"class":102},"),                     ",[62,643,644],{"class":179},"# curly quotes -> straight\n",[62,646,647,649,651,653,656,659,661,664,667,669,671,673,676,679],{"class":64,"line":120},[62,648,597],{"class":102},[62,650,304],{"class":98},[62,652,224],{"class":72},[62,654,655],{"class":148},"\\s",[62,657,658],{"class":98},"*",[62,660,604],{"class":148},[62,662,663],{"class":227},"–—",[62,665,666],{"class":148},"]\\s",[62,668,658],{"class":98},[62,670,224],{"class":72},[62,672,17],{"class":102},[62,674,675],{"class":72},"\" - \"",[62,677,678],{"class":102},"),                              ",[62,680,681],{"class":179},"# en and em dashes -> spaced hyphen\n",[62,683,684,686,688,690,692,694,697,700,703,705,707,709,711,713,715,718],{"class":64,"line":127},[62,685,597],{"class":102},[62,687,304],{"class":98},[62,689,224],{"class":72},[62,691,655],{"class":148},[62,693,312],{"class":98},[62,695,696],{"class":148},"([",[62,698,699],{"class":227},",.;:!?",[62,701,702],{"class":148},"])",[62,704,224],{"class":72},[62,706,17],{"class":102},[62,708,304],{"class":98},[62,710,224],{"class":72},[62,712,324],{"class":323},[62,714,224],{"class":72},[62,716,717],{"class":102},"),                           ",[62,719,720],{"class":179},"# no space before punctuation\n",[62,722,723,725,727,729,731,734,736,738,740,742,744,746,748,750,752,755],{"class":64,"line":145},[62,724,597],{"class":102},[62,726,304],{"class":98},[62,728,224],{"class":72},[62,730,696],{"class":148},[62,732,733],{"class":227},"!?",[62,735,702],{"class":148},[62,737,324],{"class":323},[62,739,312],{"class":98},[62,741,224],{"class":72},[62,743,17],{"class":102},[62,745,304],{"class":98},[62,747,224],{"class":72},[62,749,324],{"class":323},[62,751,224],{"class":72},[62,753,754],{"class":102},"),                               ",[62,756,757],{"class":179},"# !!! -> !   ?? -> ?\n",[62,759,760,762,764,766,769,773,775,777,780,782,784,786,788,791,794],{"class":64,"line":183},[62,761,597],{"class":102},[62,763,304],{"class":98},[62,765,224],{"class":72},[62,767,768],{"class":98},"(?\u003C!",[62,770,772],{"class":771},"s_b0D","\\.",[62,774,237],{"class":98},[62,776,772],{"class":771},[62,778,779],{"class":98},"{2}(?!",[62,781,772],{"class":771},[62,783,237],{"class":98},[62,785,224],{"class":72},[62,787,17],{"class":102},[62,789,790],{"class":72},"\".\"",[62,792,793],{"class":102},"),                        ",[62,795,796],{"class":179},"# exactly two dots -> one (keep ... as an ellipsis)\n",[62,798,799,801,803,806,809,811,813,816,819],{"class":64,"line":188},[62,800,597],{"class":102},[62,802,304],{"class":98},[62,804,805],{"class":72},"\",",[62,807,808],{"class":98},"{2,}",[62,810,224],{"class":72},[62,812,17],{"class":102},[62,814,815],{"class":72},"\",\"",[62,817,818],{"class":102},"),                                     ",[62,820,821],{"class":179},"# ,, -> ,\n",[62,823,824,826,828,830,832,835,837,840,843,845,847,849,851,853,855,858,861],{"class":64,"line":201},[62,825,597],{"class":102},[62,827,304],{"class":98},[62,829,224],{"class":72},[62,831,696],{"class":148},[62,833,834],{"class":227},",;:",[62,836,702],{"class":148},[62,838,839],{"class":98},"(?=",[62,841,842],{"class":148},"\\S",[62,844,237],{"class":98},[62,846,224],{"class":72},[62,848,17],{"class":102},[62,850,304],{"class":98},[62,852,224],{"class":72},[62,854,324],{"class":323},[62,856,857],{"class":72}," \"",[62,859,860],{"class":102},"),                          ",[62,862,863],{"class":179},"# space after , ; : when missing\n",[62,865,866,868,870,872,874,876,878,880,883,886],{"class":64,"line":213},[62,867,597],{"class":102},[62,869,304],{"class":98},[62,871,224],{"class":72},[62,873,655],{"class":148},[62,875,808],{"class":98},[62,877,224],{"class":72},[62,879,17],{"class":102},[62,881,882],{"class":72},"\" \"",[62,884,885],{"class":102},"),                                    ",[62,887,888],{"class":179},"# runs of spaces -> one\n",[62,890,891],{"class":64,"line":259},[62,892,893],{"class":102},"]\n",[62,895,896],{"class":64,"line":295},[62,897,124],{"emptyLinePlaceholder":123},[62,899,900,902,905],{"class":64,"line":349},[62,901,191],{"class":98},[62,903,904],{"class":194}," clean_text",[62,906,198],{"class":102},[62,908,909,912,915,918,921,924],{"class":64,"line":354},[62,910,911],{"class":98},"    if",[62,913,914],{"class":98}," not",[62,916,917],{"class":148}," isinstance",[62,919,920],{"class":102},"(text, ",[62,922,923],{"class":148},"str",[62,925,377],{"class":102},[62,927,928,930],{"class":64,"line":380},[62,929,262],{"class":98},[62,931,932],{"class":102}," text\n",[62,934,936,939,941],{"class":64,"line":935},15,[62,937,938],{"class":102},"    out ",[62,940,133],{"class":98},[62,942,943],{"class":102}," drop_repeats(text)\n",[62,945,947,950,953,955,958],{"class":64,"line":946},16,[62,948,949],{"class":98},"    for",[62,951,952],{"class":102}," pattern, repl ",[62,954,280],{"class":98},[62,956,957],{"class":148}," RULES",[62,959,960],{"class":102},":\n",[62,962,964,967,969],{"class":64,"line":963},17,[62,965,966],{"class":102},"        out ",[62,968,133],{"class":98},[62,970,971],{"class":102}," re.sub(pattern, repl, out)\n",[62,973,975,977,980,983,986],{"class":64,"line":974},18,[62,976,298],{"class":98},[62,978,979],{"class":102}," out.strip(",[62,981,982],{"class":72},"\" -–—,;:\"",[62,984,985],{"class":102},")                      ",[62,987,988],{"class":179},"# keep a final full stop\n",[62,990,992],{"class":64,"line":991},19,[62,993,124],{"emptyLinePlaceholder":123},[62,995,997,1000,1003,1006,1008,1010,1012],{"class":64,"line":996},20,[62,998,999],{"class":102},"notes[",[62,1001,1002],{"class":72},"\"Clean\"",[62,1004,1005],{"class":102},"] ",[62,1007,133],{"class":98},[62,1009,365],{"class":102},[62,1011,368],{"class":72},[62,1013,1014],{"class":102},"].map(clean_text)\n",[62,1016,1018,1021,1024,1026,1028,1030,1033,1036,1038,1041],{"class":64,"line":1017},21,[62,1019,1020],{"class":148},"print",[62,1022,1023],{"class":102},"(notes[[",[62,1025,368],{"class":72},[62,1027,17],{"class":102},[62,1029,1002],{"class":72},[62,1031,1032],{"class":102},"]].to_string(",[62,1034,1035],{"class":335},"index",[62,1037,133],{"class":98},[62,1039,1040],{"class":148},"False",[62,1042,1043],{"class":102},"))\n",[10,1045,1046,1047,1050,1051,1054,1055,31,1058,1061,1062,1065],{},"Each rule fixes one thing, and their order matters a little. Quotes come first, so later rules see plain characters. Dashes are turned into a spaced hyphen. Spaces ",[14,1048,1049],{},"before"," punctuation are removed and a single space is added ",[14,1052,1053],{},"after"," commas, semicolons and colons where it is missing. Runs of exclamation and question marks collapse to one. Two full stops become one, but three are left alone, because an ellipsis is intentional. Repeated commas collapse, and runs of spaces become one. Finally, leading and trailing spaces, dashes, commas and colons are stripped — which is what turns ",[14,1056,1057],{},"- Collected from depot -",[14,1059,1060],{},"Collected from depot"," — while a final full stop is kept, since it ends a sentence rather than being junk. Non-text values, such as a blank cell read as ",[59,1063,1064],{},"NaN",", are passed through untouched.",[47,1067,1069],{"id":1068},"step-3-log-what-changed","Step 3 — Log what changed",[52,1071,1073],{"className":89,"code":1072,"language":91,"meta":57,"style":57},"changed = notes[notes[\"Note\"] != notes[\"Clean\"]]\nlog = changed.assign(Change=[\n    \", \".join(name for name, pattern in [(\"repeated word\", r\"\\b(\\w+)\\s+\\1\\b\"), (\"punctuation\", r\"[!?]{2}|\\.\\.|,,|\\s[,.]\"),\n                                          (\"quotes\u002Fdashes\", r\"[“”‘’–—]\"), (\"spacing\", r\"^\\s|\\s$|\\s{2}\"),\n                                          (\"leading\u002Ftrailing junk\", r\"^[-–—,;:]|[-–—,;:]$\")]\n              if re.search(pattern, before, flags=re.IGNORECASE))\n    for before in changed[\"Note\"]])\nprint(f\"{len(changed)} of {len(notes)} notes changed\")\nprint(log[[\"Ticket\", \"Note\", \"Clean\", \"Change\"]].to_string(index=False))\n\nwith pd.ExcelWriter(\"ticket_notes_clean.xlsx\") as xw:\n    notes.drop(columns=\"Note\").rename(columns={\"Clean\": \"Note\"}).to_excel(xw, sheet_name=\"Notes\", index=False)\n    log.to_excel(xw, sheet_name=\"Changes\", index=False)\n",[59,1074,1075,1099,1117,1202,1255,1290,1308,1325,1362,1395,1399,1417,1466],{"__ignoreMap":57},[62,1076,1077,1080,1082,1085,1087,1089,1092,1094,1096],{"class":64,"line":65},[62,1078,1079],{"class":102},"changed ",[62,1081,133],{"class":98},[62,1083,1084],{"class":102}," notes[notes[",[62,1086,368],{"class":72},[62,1088,1005],{"class":102},[62,1090,1091],{"class":98},"!=",[62,1093,365],{"class":102},[62,1095,1002],{"class":72},[62,1097,1098],{"class":102},"]]\n",[62,1100,1101,1104,1106,1109,1112,1114],{"class":64,"line":106},[62,1102,1103],{"class":102},"log ",[62,1105,133],{"class":98},[62,1107,1108],{"class":102}," changed.assign(",[62,1110,1111],{"class":335},"Change",[62,1113,133],{"class":98},[62,1115,1116],{"class":102},"[\n",[62,1118,1119,1122,1125,1127,1130,1132,1135,1138,1140,1142,1144,1146,1148,1150,1152,1154,1156,1158,1160,1163,1165,1167,1169,1171,1173,1175,1178,1181,1184,1187,1189,1192,1195,1197,1199],{"class":64,"line":120},[62,1120,1121],{"class":72},"    \", \"",[62,1123,1124],{"class":102},".join(name ",[62,1126,357],{"class":98},[62,1128,1129],{"class":102}," name, pattern ",[62,1131,280],{"class":98},[62,1133,1134],{"class":102}," [(",[62,1136,1137],{"class":72},"\"repeated word\"",[62,1139,17],{"class":102},[62,1141,304],{"class":98},[62,1143,224],{"class":72},[62,1145,309],{"class":148},[62,1147,312],{"class":98},[62,1149,315],{"class":148},[62,1151,312],{"class":98},[62,1153,324],{"class":323},[62,1155,427],{"class":148},[62,1157,224],{"class":72},[62,1159,620],{"class":102},[62,1161,1162],{"class":72},"\"punctuation\"",[62,1164,17],{"class":102},[62,1166,304],{"class":98},[62,1168,224],{"class":72},[62,1170,604],{"class":148},[62,1172,733],{"class":227},[62,1174,610],{"class":148},[62,1176,1177],{"class":98},"{2}|",[62,1179,1180],{"class":771},"\\.\\.",[62,1182,1183],{"class":98},"|",[62,1185,1186],{"class":72},",,",[62,1188,1183],{"class":98},[62,1190,1191],{"class":148},"\\s[",[62,1193,1194],{"class":227},",.",[62,1196,610],{"class":148},[62,1198,224],{"class":72},[62,1200,1201],{"class":102},"),\n",[62,1203,1204,1207,1210,1212,1214,1216,1218,1221,1223,1225,1227,1230,1232,1234,1236,1239,1241,1244,1246,1248,1251,1253],{"class":64,"line":127},[62,1205,1206],{"class":102},"                                          (",[62,1208,1209],{"class":72},"\"quotes\u002Fdashes\"",[62,1211,17],{"class":102},[62,1213,304],{"class":98},[62,1215,224],{"class":72},[62,1217,604],{"class":148},[62,1219,1220],{"class":227},"“”‘’–—",[62,1222,610],{"class":148},[62,1224,224],{"class":72},[62,1226,620],{"class":102},[62,1228,1229],{"class":72},"\"spacing\"",[62,1231,17],{"class":102},[62,1233,304],{"class":98},[62,1235,224],{"class":72},[62,1237,1238],{"class":148},"^\\s",[62,1240,1183],{"class":98},[62,1242,1243],{"class":148},"\\s$",[62,1245,1183],{"class":98},[62,1247,655],{"class":148},[62,1249,1250],{"class":98},"{2}",[62,1252,224],{"class":72},[62,1254,1201],{"class":102},[62,1256,1257,1259,1262,1264,1266,1268,1271,1274,1276,1278,1280,1282,1285,1287],{"class":64,"line":145},[62,1258,1206],{"class":102},[62,1260,1261],{"class":72},"\"leading\u002Ftrailing junk\"",[62,1263,17],{"class":102},[62,1265,304],{"class":98},[62,1267,224],{"class":72},[62,1269,1270],{"class":148},"^[",[62,1272,1273],{"class":227},"-–—,;:",[62,1275,610],{"class":148},[62,1277,1183],{"class":98},[62,1279,604],{"class":148},[62,1281,1273],{"class":227},[62,1283,1284],{"class":148},"]$",[62,1286,224],{"class":72},[62,1288,1289],{"class":102},")]\n",[62,1291,1292,1295,1298,1300,1302,1304,1306],{"class":64,"line":183},[62,1293,1294],{"class":98},"              if",[62,1296,1297],{"class":102}," re.search(pattern, before, ",[62,1299,336],{"class":335},[62,1301,133],{"class":98},[62,1303,341],{"class":102},[62,1305,344],{"class":148},[62,1307,1043],{"class":102},[62,1309,1310,1312,1315,1317,1320,1322],{"class":64,"line":188},[62,1311,949],{"class":98},[62,1313,1314],{"class":102}," before ",[62,1316,280],{"class":98},[62,1318,1319],{"class":102}," changed[",[62,1321,368],{"class":72},[62,1323,1324],{"class":102},"]])\n",[62,1326,1327,1329,1331,1333,1335,1337,1340,1343,1345,1348,1350,1352,1355,1357,1360],{"class":64,"line":201},[62,1328,1020],{"class":148},[62,1330,320],{"class":102},[62,1332,388],{"class":98},[62,1334,224],{"class":72},[62,1336,228],{"class":227},[62,1338,1339],{"class":148},"len",[62,1341,1342],{"class":102},"(changed)",[62,1344,240],{"class":227},[62,1346,1347],{"class":72}," of ",[62,1349,228],{"class":227},[62,1351,1339],{"class":148},[62,1353,1354],{"class":102},"(notes)",[62,1356,240],{"class":227},[62,1358,1359],{"class":72}," notes changed\"",[62,1361,142],{"class":102},[62,1363,1364,1366,1369,1372,1374,1376,1378,1380,1382,1385,1387,1389,1391,1393],{"class":64,"line":213},[62,1365,1020],{"class":148},[62,1367,1368],{"class":102},"(log[[",[62,1370,1371],{"class":72},"\"Ticket\"",[62,1373,17],{"class":102},[62,1375,368],{"class":72},[62,1377,17],{"class":102},[62,1379,1002],{"class":72},[62,1381,17],{"class":102},[62,1383,1384],{"class":72},"\"Change\"",[62,1386,1032],{"class":102},[62,1388,1035],{"class":335},[62,1390,133],{"class":98},[62,1392,1040],{"class":148},[62,1394,1043],{"class":102},[62,1396,1397],{"class":64,"line":259},[62,1398,124],{"emptyLinePlaceholder":123},[62,1400,1401,1404,1407,1410,1412,1414],{"class":64,"line":295},[62,1402,1403],{"class":98},"with",[62,1405,1406],{"class":102}," pd.ExcelWriter(",[62,1408,1409],{"class":72},"\"ticket_notes_clean.xlsx\"",[62,1411,271],{"class":102},[62,1413,114],{"class":98},[62,1415,1416],{"class":102}," xw:\n",[62,1418,1419,1422,1425,1427,1429,1432,1434,1436,1438,1440,1443,1445,1448,1451,1453,1456,1458,1460,1462,1464],{"class":64,"line":349},[62,1420,1421],{"class":102},"    notes.drop(",[62,1423,1424],{"class":335},"columns",[62,1426,133],{"class":98},[62,1428,368],{"class":72},[62,1430,1431],{"class":102},").rename(",[62,1433,1424],{"class":335},[62,1435,133],{"class":98},[62,1437,228],{"class":102},[62,1439,1002],{"class":72},[62,1441,1442],{"class":102},": ",[62,1444,368],{"class":72},[62,1446,1447],{"class":102},"}).to_excel(xw, ",[62,1449,1450],{"class":335},"sheet_name",[62,1452,133],{"class":98},[62,1454,1455],{"class":72},"\"Notes\"",[62,1457,17],{"class":102},[62,1459,1035],{"class":335},[62,1461,133],{"class":98},[62,1463,1040],{"class":148},[62,1465,142],{"class":102},[62,1467,1468,1471,1473,1475,1478,1480,1482,1484,1486],{"class":64,"line":354},[62,1469,1470],{"class":102},"    log.to_excel(xw, ",[62,1472,1450],{"class":335},[62,1474,133],{"class":98},[62,1476,1477],{"class":72},"\"Changes\"",[62,1479,17],{"class":102},[62,1481,1035],{"class":335},[62,1483,133],{"class":98},[62,1485,1040],{"class":148},[62,1487,142],{"class":102},[10,1489,1490,1491,1494,1495,1498],{},"Text cleaning changes what people wrote, so it should leave a trail. Comparing the original and cleaned columns gives every row that changed, and a few quick pattern checks on the original label ",[14,1492,1493],{},"why"," — repeated word, punctuation, quotes or dashes, spacing. The cleaned notes go to the output file under the original column name, and the change log goes on its own sheet. In the sample, seven of the ten notes change; the three that do not are the two genuine repeats and ",[14,1496,1497],{},"OK. OK.",", whose full stops keep the words from counting as a repeat. A reviewer can read the log in a minute and spot a rule that went too far; if one did, the fix is an addition to the allowlist or a tighter pattern, and the cleaning is rerun from the original data.",[458,1500,467,1506,467,1509,467,1512,467,1517,467,1523,467,1527,467,1533,467,1536,467,1540,467,1544,467,1547,467,1550,467,1554,467,1556,467,1559,467,1562,467,1565,467,1567,467,1570,467,1572,467,1575,467,1578,467,1582,467,1584,467,1587,467,1589,467,1592,467,1595,467,1598,467,1600,467,1602,467,1604,467,1607,467,1610,467,1614,467,1616,467,1619,467,1621,467,1624,467,1627,467,1631,467,1633,467,1636,467,1638,467,1641,467,1644,467,1647,467,1649,467,1651,467,1653],{"viewBox":1501,"role":461,"ariaLabelledBy":1502,"xmlns":465,"style":1505},"137 0 485 252",[1503,1504],"txt-before-after-t","txt-before-after-d","width:100%;max-width:485px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[469,1507,1508],{"id":1503},"Before and after",[473,1510,1511],{"id":1504},"Delivered to to the warehouse loses the repeated to. Fragile with three exclamation marks keeps one. The space before the comma is removed. Curly quotes and the dash are normalised and the surrounding spaces stripped. Bora Bora and that that are kept, and the leading and trailing hyphens are removed.",[477,1513],{"x":1514,"y":268,"width":1515,"height":1516,"fill":482},"137","485","252",[477,1518],{"x":1519,"y":486,"width":1520,"height":1521,"rx":268,"fill":490,"stroke":491,"style":1522},"153.5","32.0","24.0","stroke-width:1px",[477,1524],{"x":1525,"y":486,"width":1526,"height":1521,"rx":268,"fill":490,"stroke":491,"style":1522},"185.5","210.5",[494,1528,1532],{"x":1529,"y":1530,"style":1531},"290.8","33.0","font-size:11px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","A",[477,1534],{"x":1535,"y":486,"width":1526,"height":1521,"rx":268,"fill":490,"stroke":491,"style":1522},"396.0",[494,1537,1539],{"x":1538,"y":1530,"style":1531},"501.2","B",[477,1541],{"x":1519,"y":1542,"width":1520,"height":1543,"rx":268,"fill":490,"stroke":491,"style":1522},"40.0","28.0",[494,1545,234],{"x":1546,"y":502,"style":1531},"169.5",[477,1548],{"x":1525,"y":1542,"width":1526,"height":1543,"rx":268,"fill":565,"stroke":514,"style":1549},"stroke-width:1.5px",[494,1551,1553],{"x":1529,"y":502,"style":1552},"font-size:12px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Before",[477,1555],{"x":1535,"y":1542,"width":1526,"height":1543,"rx":268,"fill":565,"stroke":514,"style":1549},[494,1557,1558],{"x":1538,"y":502,"style":1552},"After",[477,1560],{"x":1519,"y":1561,"width":1520,"height":1543,"rx":268,"fill":490,"stroke":491,"style":1522},"68.0",[494,1563,247],{"x":1546,"y":1564,"style":1531},"87.0",[477,1566],{"x":1525,"y":1561,"width":1526,"height":1543,"rx":268,"fill":482,"stroke":491,"style":1522},[494,1568,16],{"x":1529,"y":1564,"style":1569},"font-size:12px;font-weight:400;fill:var(--text,#172033);text-anchor:middle",[477,1571],{"x":1535,"y":1561,"width":1526,"height":1543,"rx":268,"fill":482,"stroke":491,"style":1522},[494,1573,1574],{"x":1538,"y":1564,"style":1569},"Delivered to the warehouse",[477,1576],{"x":1519,"y":1577,"width":1520,"height":1543,"rx":268,"fill":490,"stroke":491,"style":1522},"96.0",[494,1579,1581],{"x":1546,"y":1580,"style":1531},"115.0","3",[477,1583],{"x":1525,"y":1577,"width":1526,"height":1543,"rx":268,"fill":482,"stroke":491,"style":1522},[494,1585,1586],{"x":1529,"y":1580,"style":1569},"Fragile!!! Handle with care..",[477,1588],{"x":1535,"y":1577,"width":1526,"height":1543,"rx":268,"fill":482,"stroke":491,"style":1522},[494,1590,1591],{"x":1538,"y":1580,"style":1569},"Fragile! Handle with care.",[477,1593],{"x":1519,"y":1594,"width":1520,"height":1543,"rx":268,"fill":490,"stroke":491,"style":1522},"124.0",[494,1596,479],{"x":1546,"y":1597,"style":1531},"143.0",[477,1599],{"x":1525,"y":1594,"width":1526,"height":1543,"rx":268,"fill":482,"stroke":491,"style":1522},[494,1601,23],{"x":1529,"y":1597,"style":1569},[477,1603],{"x":1535,"y":1594,"width":1526,"height":1543,"rx":268,"fill":482,"stroke":491,"style":1522},[494,1605,1606],{"x":1538,"y":1597,"style":1569},"Leave with neighbour, no 12",[477,1608],{"x":1519,"y":1609,"width":1520,"height":1543,"rx":268,"fill":490,"stroke":491,"style":1522},"152.0",[494,1611,1613],{"x":1546,"y":1612,"style":1531},"171.0","5",[477,1615],{"x":1525,"y":1609,"width":1526,"height":1543,"rx":268,"fill":482,"stroke":491,"style":1522},[494,1617,1618],{"x":1529,"y":1612,"style":1569},"“Urgent” – call before delivery",[477,1620],{"x":1535,"y":1609,"width":1526,"height":1543,"rx":268,"fill":482,"stroke":491,"style":1522},[494,1622,1623],{"x":1538,"y":1612,"style":1569},"\"Urgent\" - call before delivery",[477,1625],{"x":1519,"y":1626,"width":1520,"height":1543,"rx":268,"fill":490,"stroke":491,"style":1522},"180.0",[494,1628,1630],{"x":1546,"y":1629,"style":1531},"199.0","6",[477,1632],{"x":1525,"y":1626,"width":1526,"height":1543,"rx":268,"fill":482,"stroke":491,"style":1522},[494,1634,1635],{"x":1529,"y":1629,"style":1569},"Holiday booking for Bora Bora",[477,1637],{"x":1535,"y":1626,"width":1526,"height":1543,"rx":268,"fill":545,"stroke":546,"style":1549},[494,1639,1635],{"x":1538,"y":1629,"style":1640},"font-size:12px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle",[477,1642],{"x":1519,"y":1643,"width":1520,"height":1543,"rx":268,"fill":490,"stroke":491,"style":1522},"208.0",[494,1645,374],{"x":1546,"y":1646,"style":1531},"227.0",[477,1648],{"x":1525,"y":1643,"width":1526,"height":1543,"rx":268,"fill":482,"stroke":491,"style":1522},[494,1650,1057],{"x":1529,"y":1646,"style":1569},[477,1652],{"x":1535,"y":1643,"width":1526,"height":1543,"rx":268,"fill":482,"stroke":491,"style":1522},[494,1654,1060],{"x":1538,"y":1646,"style":1569},[47,1656,1658],{"id":1657},"pin-the-rules-down-with-test-cases","Pin the rules down with test cases",[10,1660,1661,1662,1665],{},"Regular expressions are easy to get slightly wrong and easy to break later with a well-meant tweak, so the cleaning function deserves a short list of examples that must keep working. Each case is an input and the exact output expected, chosen to cover one rule — and, just as important, the cases that must ",[14,1663,1664],{},"not"," change.",[52,1667,1669],{"className":89,"code":1668,"language":91,"meta":57,"style":57},"CASES = [\n    (\"Delivered to to the warehouse\", \"Delivered to the warehouse\"),\n    (\"The the box\", \"The box\"),\n    (\"Holiday booking for Bora Bora\", \"Holiday booking for Bora Bora\"),\n    (\"Visit the theatre\", \"Visit the theatre\"),\n    (\"Wait...\", \"Wait...\"),\n    (\"Call me , please\", \"Call me, please\"),\n    (\"Price: £12,50\", \"Price: £12, 50\"),\n]\nfor given, expected in CASES:\n    got = clean_text(given)\n    print(f\"{'ok ' if got == expected else 'BAD'} {given!r} -> {got!r}\")\n",[59,1670,1671,1680,1694,1708,1721,1734,1747,1761,1775,1779,1793,1803],{"__ignoreMap":57},[62,1672,1673,1676,1678],{"class":64,"line":65},[62,1674,1675],{"class":148},"CASES",[62,1677,152],{"class":98},[62,1679,592],{"class":102},[62,1681,1682,1684,1687,1689,1692],{"class":64,"line":106},[62,1683,597],{"class":102},[62,1685,1686],{"class":72},"\"Delivered to to the warehouse\"",[62,1688,17],{"class":102},[62,1690,1691],{"class":72},"\"Delivered to the warehouse\"",[62,1693,1201],{"class":102},[62,1695,1696,1698,1701,1703,1706],{"class":64,"line":120},[62,1697,597],{"class":102},[62,1699,1700],{"class":72},"\"The the box\"",[62,1702,17],{"class":102},[62,1704,1705],{"class":72},"\"The box\"",[62,1707,1201],{"class":102},[62,1709,1710,1712,1715,1717,1719],{"class":64,"line":127},[62,1711,597],{"class":102},[62,1713,1714],{"class":72},"\"Holiday booking for Bora Bora\"",[62,1716,17],{"class":102},[62,1718,1714],{"class":72},[62,1720,1201],{"class":102},[62,1722,1723,1725,1728,1730,1732],{"class":64,"line":145},[62,1724,597],{"class":102},[62,1726,1727],{"class":72},"\"Visit the theatre\"",[62,1729,17],{"class":102},[62,1731,1727],{"class":72},[62,1733,1201],{"class":102},[62,1735,1736,1738,1741,1743,1745],{"class":64,"line":183},[62,1737,597],{"class":102},[62,1739,1740],{"class":72},"\"Wait...\"",[62,1742,17],{"class":102},[62,1744,1740],{"class":72},[62,1746,1201],{"class":102},[62,1748,1749,1751,1754,1756,1759],{"class":64,"line":188},[62,1750,597],{"class":102},[62,1752,1753],{"class":72},"\"Call me , please\"",[62,1755,17],{"class":102},[62,1757,1758],{"class":72},"\"Call me, please\"",[62,1760,1201],{"class":102},[62,1762,1763,1765,1768,1770,1773],{"class":64,"line":201},[62,1764,597],{"class":102},[62,1766,1767],{"class":72},"\"Price: £12,50\"",[62,1769,17],{"class":102},[62,1771,1772],{"class":72},"\"Price: £12, 50\"",[62,1774,1201],{"class":102},[62,1776,1777],{"class":64,"line":213},[62,1778,893],{"class":102},[62,1780,1781,1783,1786,1788,1791],{"class":64,"line":259},[62,1782,357],{"class":98},[62,1784,1785],{"class":102}," given, expected ",[62,1787,280],{"class":98},[62,1789,1790],{"class":148}," CASES",[62,1792,960],{"class":102},[62,1794,1795,1798,1800],{"class":64,"line":295},[62,1796,1797],{"class":102},"    got ",[62,1799,133],{"class":98},[62,1801,1802],{"class":102}," clean_text(given)\n",[62,1804,1805,1807,1809,1811,1813,1815,1818,1821,1824,1827,1830,1833,1836,1838,1840,1843,1845,1847,1849,1851,1854,1856,1858,1860],{"class":64,"line":349},[62,1806,383],{"class":148},[62,1808,320],{"class":102},[62,1810,388],{"class":98},[62,1812,224],{"class":72},[62,1814,228],{"class":227},[62,1816,1817],{"class":72},"'ok '",[62,1819,1820],{"class":98}," if",[62,1822,1823],{"class":102}," got ",[62,1825,1826],{"class":98},"==",[62,1828,1829],{"class":102}," expected ",[62,1831,1832],{"class":98},"else",[62,1834,1835],{"class":72}," 'BAD'",[62,1837,240],{"class":227},[62,1839,155],{"class":227},[62,1841,1842],{"class":102},"given",[62,1844,411],{"class":98},[62,1846,240],{"class":227},[62,1848,403],{"class":72},[62,1850,228],{"class":227},[62,1852,1853],{"class":102},"got",[62,1855,411],{"class":98},[62,1857,240],{"class":227},[62,1859,224],{"class":72},[62,1861,142],{"class":102},[10,1863,1864,1865,1868,1869,1872,1873,1876,1877,1880,1881,1884,1885,31,1888,1891,1892,1895],{},"The ellipsis case earned its place while this page was being written: the first version of the two-dots rule had only a lookahead, ",[59,1866,1867],{},"\\.{2}(?!\\.)",", which happily matched the ",[14,1870,1871],{},"last"," two dots of ",[14,1874,1875],{},"Wait..."," and turned the ellipsis into two dots. Adding a lookbehind, ",[59,1878,1879],{},"(?\u003C!\\.)",", so the pair must not be preceded by a dot either, fixed it — and the case stays in the list so the bug cannot come back. The last case is there on purpose too: it shows a rule doing something unwanted. The ",[14,1882,1883],{},"space after a comma"," rule turns the European decimal ",[14,1886,1887],{},"£12,50",[14,1889,1890],{},"£12, 50",", because it cannot tell a decimal comma from a list separator. Writing the case down makes the decision explicit — either the rule is restricted to commas followed by a letter, ",[59,1893,1894],{},",(?=[A-Za-z])",", or text columns containing prices are excluded from cleaning. The test list is where such trade-offs are recorded, and running it after every change keeps the rules honest.",[458,1897,467,1902,467,1905,467,1908,467,1911,467,1916,467,1922,467,1927,467,1931,467,1935,467,1939,467,1943,467,1948,467,1952,467,1955,467,1958,467,1961],{"viewBox":1898,"role":461,"ariaLabelledBy":1899,"xmlns":465,"style":466},"4 0 752 199",[1900,1901],"txt-scope-t","txt-scope-d",[469,1903,1904],{"id":1900},"Where to apply the cleaning",[473,1906,1907],{"id":1901},"Free-text notes, descriptions and comments that people read are safe to tidy. Codes, addresses used for delivery, prices written as text and verbatim quotes should be left exactly as entered.",[477,1909],{"x":479,"y":268,"width":480,"height":1910,"fill":482},"199",[477,1912],{"x":485,"y":486,"width":1913,"height":1914,"rx":1915,"fill":545,"stroke":546,"style":492},"351.0","167.0","14",[494,1917,1921],{"x":1918,"y":1919,"style":1920},"195.5","44.0","font-size:14px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","Safe to tidy",[64,1923],{"x1":1924,"y1":1925,"x2":1926,"y2":1925,"stroke":546,"style":1522},"36.0","57.0","355.0",[494,1928,1930],{"x":1918,"y":1929,"style":1569},"77.0","delivery and ticket notes",[494,1932,1934],{"x":1918,"y":1933,"style":1569},"103.0","product descriptions",[494,1936,1938],{"x":1918,"y":1937,"style":1569},"129.0","survey comments",[477,1940],{"x":538,"y":486,"width":1913,"height":1914,"rx":1915,"fill":1941,"stroke":1942,"style":492},"#fce9e9","var(--danger,#dc2626)",[494,1944,1947],{"x":1945,"y":1919,"style":1946},"564.5","font-size:14px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","Leave as entered",[64,1949],{"x1":1950,"y1":1925,"x2":1951,"y2":1925,"stroke":1942,"style":1522},"405.0","724.0",[494,1953,1954],{"x":1945,"y":1929,"style":1569},"product codes and IDs",[494,1956,1957],{"x":1945,"y":1933,"style":1569},"delivery addresses",[494,1959,1960],{"x":1945,"y":1937,"style":1569},"prices written as text",[494,1962,1964],{"x":1945,"y":1963,"style":1569},"155.0","verbatim complaint quotes",[47,1966,1968],{"id":1967},"where-to-stop","Where to stop",[10,1970,1971,1972,452,1975,1978,1979,1983],{},"These rules fix typing accidents; they are not a spell-checker or a grammar tool, and they should not try to be. Misspellings, inconsistent abbreviations (",[14,1973,1974],{},"St",[14,1976,1977],{},"Street",") and casing are separate jobs with their own risks — see ",[41,1980,1982],{"href":1981},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffix-the-capitalisation-of-names-with-exceptions-in-pandas\u002F","Fix the Capitalisation of Names with Exceptions in pandas",". Some fields should not be touched at all: product codes, addresses used for delivery, and anything quoted verbatim in a legal or complaint context. Apply the function to the columns where tidy text helps a reader, keep the original data, and treat the change log as part of the output.",[47,1985,1987],{"id":1986},"common-pitfalls","Common pitfalls",[1989,1990,1991,2004,2016,2022,2028],"ul",{},[1992,1993,1994,1998,1999,2001,2002,45],"li",{},[1995,1996,1997],"strong",{},"Deleting every repeated word."," Use an allowlist for genuine repeats like ",[14,2000,30],{}," or ",[14,2003,443],{},[1992,2005,2006,2009,2010,2012,2013,2015],{},[1995,2007,2008],{},"Matching inside words."," Use ",[59,2011,427],{}," boundaries so ",[14,2014,431],{}," is not changed.",[1992,2017,2018,2021],{},[1995,2019,2020],{},"Turning an ellipsis into a full stop."," Collapse two dots, not three.",[1992,2023,2024,2027],{},[1995,2025,2026],{},"Cleaning non-text cells."," Pass through anything that is not a string.",[1992,2029,2030,2033],{},[1995,2031,2032],{},"No record of changes."," Log every changed cell with the reason.",[47,2035,2037],{"id":2036},"conclusion","Conclusion",[10,2039,2040],{},"To tidy free text from Excel, remove repeated words with a whole-word regular expression and a replacement function that keeps an allowlist of genuine repeats, then apply a short ordered list of rules for quotes, dashes, spacing around punctuation and repeated punctuation, and strip junk from the ends. Write the cleaned text under the original column name and log every changed cell with the reason, keeping the original data so any rule can be revised and rerun.",[47,2042,2044],{"id":2043},"frequently-asked-questions","Frequently asked questions",[10,2046,2047,2050,2051,2054,2055,2058],{},[1995,2048,2049],{},"How do I remove repeated words like \"the the\" in pandas?","\nUse ",[59,2052,2053],{},"re.sub(r\"\\b(\\w+)\\s+\\1\\b\", r\"\\1\", text, flags=re.IGNORECASE)"," inside ",[59,2056,2057],{},"Series.map",", with an allowlist for genuine repeats.",[10,2060,2061,2064,2065,2068,2069,45],{},[1995,2062,2063],{},"How do I replace repeated exclamation marks with one?","\nUse the pattern ",[59,2066,2067],{},"([!?])\\1+"," with the replacement ",[59,2070,324],{},[10,2072,2073,2076,2077,2080,2081,45],{},[1995,2074,2075],{},"How do I remove spaces before commas and full stops?","\nReplace ",[59,2078,2079],{},"\\s+([,.;:!?])"," with ",[59,2082,324],{},[10,2084,2085,2076,2088,2080,2091,452,2093,2080,2096,2099,2100,45],{},[1995,2086,2087],{},"How do I convert curly quotes to straight quotes in pandas?",[59,2089,2090],{},"[“”„]",[59,2092,224],{},[59,2094,2095],{},"[‘’]",[59,2097,2098],{},"'"," using ",[59,2101,2102],{},"str.replace(..., regex=True)",[47,2104,2106],{"id":2105},"related","Related",[1989,2108,2109,2115,2122,2129],{},[1992,2110,2111,2112,2114],{},"Up one level: ",[41,2113,44],{"href":43}," — cleaning workflows.",[1992,2116,2117,2121],{},[41,2118,2120],{"href":2119},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fstrip-whitespace-and-normalise-text-columns-with-pandas\u002F","Strip Whitespace and Normalise Text Columns with pandas"," — whitespace and invisible characters.",[1992,2123,2124,2128],{},[41,2125,2127],{"href":2126},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fmap-free-text-job-titles-to-standard-categories\u002F","Map Free-Text Job Titles to Standard Categories"," — the next step for free text.",[1992,2130,2131,2135],{},[41,2132,2134],{"href":2133},"\u002Fgetting-started-with-python-excel-automation\u002Ftroubleshooting-common-python-excel-errors\u002Ffix-keyerror-from-spaces-in-excel-column-names\u002F","Fix KeyError from Spaces in Excel Column Names"," — the same problem in headings.",[2137,2138,2139],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}html pre.shiki code .sx1sP, html code.shiki .sx1sP{--shiki-default:#22863A;--shiki-dark:#72F088}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .s_b0D, html code.shiki .s_b0D{--shiki-default:#22863A;--shiki-default-font-weight:bold;--shiki-dark:#72F088;--shiki-dark-font-weight:bold}",{"title":57,"searchDepth":106,"depth":106,"links":2141},[2142,2143,2144,2145,2146,2147,2148,2149,2150,2151],{"id":49,"depth":106,"text":50},{"id":85,"depth":106,"text":86},{"id":576,"depth":106,"text":577},{"id":1068,"depth":106,"text":1069},{"id":1657,"depth":106,"text":1658},{"id":1967,"depth":106,"text":1968},{"id":1986,"depth":106,"text":1987},{"id":2036,"depth":106,"text":2037},{"id":2043,"depth":106,"text":2044},{"id":2105,"depth":106,"text":2106},"2026-10-02","Tidy free-text columns from Excel with pandas: remove accidentally repeated words such as \"the the\" while keeping genuine repeats like \"Bora Bora\", collapse runs of punctuation, fix spaces before commas and full stops, normalise curly quotes and dashes, strip leading and trailing junk, and keep a log of exactly which cells changed and how.","md",[2156,2158,2160,2162],{"q":2049,"a":2157},"Use re.sub(r\"\\b(\\w+)\\s+\\1\\b\", r\"\\1\", text, flags=re.IGNORECASE) inside Series.map, with an allowlist for genuine repeats.",{"q":2063,"a":2159},"Use the pattern ([!?])\\1+ with the replacement \\1.",{"q":2075,"a":2161},"Replace \\s+([,.;:!?]) with \\1.",{"q":2087,"a":2163},"Replace [“”„] with \" and [‘’] with ' using str.replace(..., regex=True).","remove duplicate words pandas",null,{"breadcrumb":2167},[2168,2171,2174],{"name":2169,"item":2170},"Home","\u002F",{"name":2172,"item":2173},"Advanced Data Transformation and Cleaning for Excel Automation","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":2175,"item":43},"Cleaning Excel Data With Pandas: A Production Workflow","\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fremove-duplicate-words-and-extra-punctuation-from-text",{"title":2178,"description":2179},"Remove Duplicate Words and Extra Punctuation with pandas","Clean text in Excel data with pandas: remove repeated words with an allowlist, collapse repeated punctuation, fix spacing around punctuation, normalise quotes and dashes, and log every change.","remove-duplicate-words-and-extra-punctuation-from-text","advanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fremove-duplicate-words-and-extra-punctuation-from-text\u002Findex",[2183,2184],"data-cleaning","pandas","how-to","DOoRprZuohbtzFxzn793aje_oYyIR3_TE0Ku-mrpH2Y",[2188,2192],{"title":2189,"path":2190,"stem":2191,"children":-1},"Remove Blank Rows From Excel With Pandas","\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fremove-blank-rows-from-excel-with-pandas","advanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fremove-blank-rows-from-excel-with-pandas\u002Findex",{"title":2193,"path":2194,"stem":2195,"children":-1},"Remove Non-Printable Characters from Excel Data with Python","\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fremove-non-printable-characters-from-excel-data","advanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fremove-non-printable-characters-from-excel-data\u002Findex",[2197,2199],{"path":2198,"title":2175},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas",{"path":2200,"title":2172},"\u002Fadvanced-data-transformation-and-cleaning",[],[2203,2206,2209,2212,2215,2218],{"path":2204,"title":2205,"kind":2185},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Ffind-duplicate-rows-in-excel-with-python","Find Duplicate Rows in Excel with Python",{"path":2207,"title":2208,"kind":2185},"\u002Fadvanced-data-transformation-and-cleaning\u002Fapplying-conditional-formatting-with-openpyxl\u002Fhighlight-duplicate-values-in-excel-with-openpyxl","Highlight Duplicate Values in Excel with openpyxl",{"path":2210,"title":2211,"kind":2185},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fclean-mixed-currency-amounts-into-amount-and-currency-columns","Clean Mixed-Currency Amounts into Amount and Currency Columns",{"path":2213,"title":2214,"kind":2185},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fcoalesce-several-excel-columns-into-one-with-pandas","Coalesce Several Excel Columns into One with pandas",{"path":2216,"title":2217,"kind":2185},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fconvert-currency-strings-to-numbers-with-pandas","Convert Currency Strings Like \"$1,234.56\" and \"(500)\" to Numbers with pandas",{"path":2219,"title":2220,"kind":2185},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fconvert-excel-text-columns-to-numbers-with-pandas","Convert Excel Text Columns to Numbers with pandas",1790972085036]