[{"data":1,"prerenderedAt":2265},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffind-near-duplicate-names-within-one-excel-column":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffind-near-duplicate-names-within-one-excel-column":2232,"ancestors:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffind-near-duplicate-names-within-one-excel-column":2241,"children:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffind-near-duplicate-names-within-one-excel-column":2246,"related:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffind-near-duplicate-names-within-one-excel-column":2247},{"id":4,"title":5,"body":6,"dateModified":2197,"datePublished":2197,"description":2198,"extension":2199,"faq":2200,"keyword":2209,"kind":2210,"meta":2211,"navigation":117,"path":2221,"seo":2222,"slug":2225,"stem":2226,"topics":2227,"type":2230,"__hash__":2231},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffind-near-duplicate-names-within-one-excel-column\u002Findex.md","Find Near-Duplicate Names Within One Excel Column",{"type":7,"value":8,"toc":2185},"minimark",[9,27,32,63,69,73,439,442,587,591,929,943,955,1048,1052,1055,1276,1279,1432,1436,1439,1718,1725,1729,1739,2058,2065,2069,2110,2114,2120,2124,2130,2136,2142,2148,2152,2181],[10,11,12,16,17,20,21,26],"p",{},[13,14,15],"code",{},"drop_duplicates"," removes rows that are identical. It does nothing about \"ACME Ltd\", \"Acme Limited\" and \"A.C.M.E. Ltd.\" — three spellings of one customer, typed by three people over five years. Those near-duplicates split revenue across three rows in every report, send three statements to the same company, and make the customer count wrong. Finding them is a two-part job: first remove the differences that never matter (case, punctuation, \"Ltd\" versus \"Limited\"), then measure how similar the remaining strings are and group the close ones. The ",[13,18,19],{},"rapidfuzz"," library does the measuring quickly. Because fuzzy matching always produces some false positives, the output should be a list for a person to confirm, not an automatic merge. This page belongs to ",[22,23,25],"a",{"href":24},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002F","Cleaning Excel Data with pandas",".",[28,29,31],"h2",{"id":30},"prerequisites","Prerequisites",[33,34,39],"pre",{"className":35,"code":36,"language":37,"meta":38,"style":38},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl rapidfuzz\n","bash","",[13,40,41],{"__ignoreMap":38},[42,43,46,50,54,57,60],"span",{"class":44,"line":45},"line",1,[42,47,49],{"class":48},"sMTad","pip",[42,51,53],{"class":52},"srMev"," install",[42,55,56],{"class":52}," pandas",[42,58,59],{"class":52}," openpyxl",[42,61,62],{"class":52}," rapidfuzz\n",[10,64,65,68],{},[13,66,67],{},"customers.xlsx"," lists twenty customer names with the variants that real lists accumulate: legal suffixes added or dropped, punctuation, spaces inside words, accents, \"&\" versus \"and\", plurals — and two names that look alike but are different companies (Brightwater and Brightwell).",[28,70,72],{"id":71},"step-1-normalise-away-the-differences-that-never-matter","Step 1 — Normalise away the differences that never matter",[33,74,78],{"className":75,"code":76,"language":77,"meta":38,"style":38},"language-python shiki shiki-themes github-light github-dark-high-contrast","import re\nimport unicodedata\nimport pandas as pd\n\nLEGAL = r\"\\b(ltd|limited|gmbh|inc|llc|plc|co|company|corp|corporation)\\b\"\n\ndef normalise(name):\n    s = unicodedata.normalize(\"NFKD\", str(name)).encode(\"ascii\", \"ignore\").decode()\n    s = s.lower().replace(\"&\", \" and \")\n    s = re.sub(r\"[^\\w\\s]\", \"\", s)            # A.C.M.E. -> acme\n    s = re.sub(LEGAL, \" \", s)\n    return \" \".join(s.split())\n\ncust = pd.read_excel(\"customers.xlsx\")\ncust[\"Key\"] = cust[\"Customer\"].map(normalise)\nprint(cust[[\"Customer\", \"Key\"]].to_string(index=False))\nprint(\"exact duplicates after normalising:\", cust[\"Key\"].duplicated(keep=False).sum())\n","python",[13,79,80,90,98,112,119,193,198,211,246,267,305,324,336,341,357,380,410],{"__ignoreMap":38},[42,81,82,86],{"class":44,"line":45},[42,83,85],{"class":84},"s-kum","import",[42,87,89],{"class":88},"skGVy"," re\n",[42,91,93,95],{"class":44,"line":92},2,[42,94,85],{"class":84},[42,96,97],{"class":88}," unicodedata\n",[42,99,101,103,106,109],{"class":44,"line":100},3,[42,102,85],{"class":84},[42,104,105],{"class":88}," pandas ",[42,107,108],{"class":84},"as",[42,110,111],{"class":88}," pd\n",[42,113,115],{"class":44,"line":114},4,[42,116,118],{"emptyLinePlaceholder":117},true,"\n",[42,120,122,126,129,132,135,138,141,144,147,149,152,154,157,159,162,164,167,169,172,174,177,179,182,184,187,190],{"class":44,"line":121},5,[42,123,125],{"class":124},"sP0c6","LEGAL",[42,127,128],{"class":84}," =",[42,130,131],{"class":84}," r",[42,133,134],{"class":52},"\"",[42,136,137],{"class":124},"\\b(",[42,139,140],{"class":52},"ltd",[42,142,143],{"class":84},"|",[42,145,146],{"class":52},"limited",[42,148,143],{"class":84},[42,150,151],{"class":52},"gmbh",[42,153,143],{"class":84},[42,155,156],{"class":52},"inc",[42,158,143],{"class":84},[42,160,161],{"class":52},"llc",[42,163,143],{"class":84},[42,165,166],{"class":52},"plc",[42,168,143],{"class":84},[42,170,171],{"class":52},"co",[42,173,143],{"class":84},[42,175,176],{"class":52},"company",[42,178,143],{"class":84},[42,180,181],{"class":52},"corp",[42,183,143],{"class":84},[42,185,186],{"class":52},"corporation",[42,188,189],{"class":124},")\\b",[42,191,192],{"class":52},"\"\n",[42,194,196],{"class":44,"line":195},6,[42,197,118],{"emptyLinePlaceholder":117},[42,199,201,204,208],{"class":44,"line":200},7,[42,202,203],{"class":84},"def",[42,205,207],{"class":206},"s_Opv"," normalise",[42,209,210],{"class":88},"(name):\n",[42,212,214,217,220,223,226,229,232,235,238,240,243],{"class":44,"line":213},8,[42,215,216],{"class":88},"    s ",[42,218,219],{"class":84},"=",[42,221,222],{"class":88}," unicodedata.normalize(",[42,224,225],{"class":52},"\"NFKD\"",[42,227,228],{"class":88},", ",[42,230,231],{"class":124},"str",[42,233,234],{"class":88},"(name)).encode(",[42,236,237],{"class":52},"\"ascii\"",[42,239,228],{"class":88},[42,241,242],{"class":52},"\"ignore\"",[42,244,245],{"class":88},").decode()\n",[42,247,249,251,253,256,259,261,264],{"class":44,"line":248},9,[42,250,216],{"class":88},[42,252,219],{"class":84},[42,254,255],{"class":88}," s.lower().replace(",[42,257,258],{"class":52},"\"&\"",[42,260,228],{"class":88},[42,262,263],{"class":52},"\" and \"",[42,265,266],{"class":88},")\n",[42,268,270,272,274,277,280,282,285,288,291,293,295,298,301],{"class":44,"line":269},10,[42,271,216],{"class":88},[42,273,219],{"class":84},[42,275,276],{"class":88}," re.sub(",[42,278,279],{"class":84},"r",[42,281,134],{"class":52},[42,283,284],{"class":124},"[",[42,286,287],{"class":84},"^",[42,289,290],{"class":124},"\\w\\s]",[42,292,134],{"class":52},[42,294,228],{"class":88},[42,296,297],{"class":52},"\"\"",[42,299,300],{"class":88},", s)            ",[42,302,304],{"class":303},"s-wDw","# A.C.M.E. -> acme\n",[42,306,308,310,312,314,316,318,321],{"class":44,"line":307},11,[42,309,216],{"class":88},[42,311,219],{"class":84},[42,313,276],{"class":88},[42,315,125],{"class":124},[42,317,228],{"class":88},[42,319,320],{"class":52},"\" \"",[42,322,323],{"class":88},", s)\n",[42,325,327,330,333],{"class":44,"line":326},12,[42,328,329],{"class":84},"    return",[42,331,332],{"class":52}," \" \"",[42,334,335],{"class":88},".join(s.split())\n",[42,337,339],{"class":44,"line":338},13,[42,340,118],{"emptyLinePlaceholder":117},[42,342,344,347,349,352,355],{"class":44,"line":343},14,[42,345,346],{"class":88},"cust ",[42,348,219],{"class":84},[42,350,351],{"class":88}," pd.read_excel(",[42,353,354],{"class":52},"\"customers.xlsx\"",[42,356,266],{"class":88},[42,358,360,363,366,369,371,374,377],{"class":44,"line":359},15,[42,361,362],{"class":88},"cust[",[42,364,365],{"class":52},"\"Key\"",[42,367,368],{"class":88},"] ",[42,370,219],{"class":84},[42,372,373],{"class":88}," cust[",[42,375,376],{"class":52},"\"Customer\"",[42,378,379],{"class":88},"].map(normalise)\n",[42,381,383,386,389,391,393,395,398,402,404,407],{"class":44,"line":382},16,[42,384,385],{"class":124},"print",[42,387,388],{"class":88},"(cust[[",[42,390,376],{"class":52},[42,392,228],{"class":88},[42,394,365],{"class":52},[42,396,397],{"class":88},"]].to_string(",[42,399,401],{"class":400},"sa561","index",[42,403,219],{"class":84},[42,405,406],{"class":124},"False",[42,408,409],{"class":88},"))\n",[42,411,413,415,418,421,424,426,429,432,434,436],{"class":44,"line":412},17,[42,414,385],{"class":124},[42,416,417],{"class":88},"(",[42,419,420],{"class":52},"\"exact duplicates after normalising:\"",[42,422,423],{"class":88},", cust[",[42,425,365],{"class":52},[42,427,428],{"class":88},"].duplicated(",[42,430,431],{"class":400},"keep",[42,433,219],{"class":84},[42,435,406],{"class":124},[42,437,438],{"class":88},").sum())\n",[10,440,441],{},"Decomposing accents and dropping the marks turns \"Café Señor\" into \"cafe senor\"; removing punctuation turns \"A.C.M.E.\" into \"acme\"; dropping legal suffixes makes \"ACME Ltd\" and \"Acme Limited\" both \"acme\". Normalisation alone already makes several groups identical, which is cheap and certain. What remains — \"bright water supplies\" against \"brightwater supplies\", \"oak ridge farm\" against \"oakridge farms\" — needs a similarity measure. Keep the normalised key in its own column; the original spelling stays untouched for the report.",[443,444,452,453,452,457,452,461,452,469,452,477,452,484,452,487,452,492,452,495,452,499,452,502,452,506,452,509,452,513,452,516,452,520,452,524,452,528,452,533,452,537,452,539,452,541,452,543,452,545,452,549,452,553,452,555,452,557,452,561,452,563,452,568,452,573,452,581],"svg",{"viewBox":445,"role":446,"ariaLabelledBy":447,"xmlns":450,"style":451},"188 0 383 224","img",[448,449],"nd-normalise-t","nd-normalise-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:383px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[454,455,456],"title",{"id":448},"Normalisation makes many variants identical",[458,459,460],"desc",{"id":449},"On the left five raw names — ACME Ltd, Acme Limited, A.C.M.E. Ltd., Café Señor and Cafe Senor. On the right their normalised keys — acme three times and cafe senor twice.",[462,463],"rect",{"x":464,"y":465,"width":466,"height":467,"fill":468},"188","0","383","224","#ffffff",[462,470],{"x":471,"y":472,"width":473,"height":474,"rx":465,"fill":468,"stroke":475,"style":476},"204.8","40.0","101.3","28.0","var(--line,#cdd5e6)","stroke-width:1px",[478,479,483],"text",{"x":480,"y":481,"style":482},"255.5","59.0","font-size:11px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","Customer",[462,485],{"x":471,"y":486,"width":473,"height":474,"rx":465,"fill":468,"stroke":475,"style":476},"68.0",[478,488,491],{"x":480,"y":489,"style":490},"87.0","font-size:11px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","ACME Ltd",[462,493],{"x":471,"y":494,"width":473,"height":474,"rx":465,"fill":468,"stroke":475,"style":476},"96.0",[478,496,498],{"x":480,"y":497,"style":490},"115.0","Acme Limited",[462,500],{"x":471,"y":501,"width":473,"height":474,"rx":465,"fill":468,"stroke":475,"style":476},"124.0",[478,503,505],{"x":480,"y":504,"style":490},"143.0","A.C.M.E. Ltd.",[462,507],{"x":471,"y":508,"width":473,"height":474,"rx":465,"fill":468,"stroke":475,"style":476},"152.0",[478,510,512],{"x":480,"y":511,"style":490},"171.0","Café Señor",[462,514],{"x":471,"y":515,"width":473,"height":474,"rx":465,"fill":468,"stroke":475,"style":476},"180.0",[478,517,519],{"x":480,"y":518,"style":490},"199.0","Cafe Senor",[462,521],{"x":522,"y":472,"width":523,"height":474,"rx":465,"fill":468,"stroke":475,"style":476},"471.1","84.0",[478,525,527],{"x":526,"y":481,"style":482},"513.1","Key",[462,529],{"x":522,"y":486,"width":523,"height":474,"rx":465,"fill":530,"stroke":531,"style":532},"#d9f4f1","var(--teal,#0f9488)","stroke-width:1.5px",[478,534,536],{"x":526,"y":489,"style":535},"font-size:11px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","acme",[462,538],{"x":522,"y":494,"width":523,"height":474,"rx":465,"fill":530,"stroke":531,"style":532},[478,540,536],{"x":526,"y":497,"style":535},[462,542],{"x":522,"y":501,"width":523,"height":474,"rx":465,"fill":530,"stroke":531,"style":532},[478,544,536],{"x":526,"y":504,"style":535},[462,546],{"x":522,"y":508,"width":523,"height":474,"rx":465,"fill":547,"stroke":548,"style":532},"#f0f4ff","var(--brand,#5b5cf0)",[478,550,552],{"x":526,"y":511,"style":551},"font-size:11px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","cafe senor",[462,554],{"x":522,"y":515,"width":523,"height":474,"rx":465,"fill":547,"stroke":548,"style":532},[478,556,552],{"x":526,"y":518,"style":551},[478,558,483],{"x":480,"y":559,"style":560},"29.0","font-size:12.5px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:middle",[478,562,527],{"x":526,"y":559,"style":560},[44,564],{"x1":565,"y1":501,"x2":566,"y2":501,"stroke":548,"style":567},"316.2","457.1","stroke-width:2px",[569,570],"polygon",{"points":571,"fill":572},"461.1,124.0 452.1,129.0 452.1,119.0","#5b5cf0",[462,574],{"x":575,"y":576,"width":577,"height":578,"rx":579,"fill":468,"stroke":475,"style":580},"338.6","91.0","100.0","22.0","11","stroke-width:1.2px",[478,582,586],{"x":583,"y":584,"style":585},"388.6","106.0","font-size:11.5px;font-weight:400;fill:var(--brand-strong,#4338ca);text-anchor:middle;font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace","normalise()",[28,588,590],{"id":589},"step-2-score-every-pair-with-rapidfuzz","Step 2 — Score every pair with rapidfuzz",[33,592,594],{"className":75,"code":593,"language":77,"meta":38,"style":38},"import numpy as np\nfrom rapidfuzz import fuzz, process\n\nkeys = cust[\"Key\"].tolist()\nsquashed = [k.replace(\" \", \"\") for k in keys]\nby_words = process.cdist(keys, keys, scorer=fuzz.token_sort_ratio, workers=1)\nby_letters = process.cdist(squashed, squashed, scorer=fuzz.ratio, workers=1)\nscores = np.maximum(by_words, by_letters)\npairs = [(i, j, scores[i][j]) for i in range(len(keys)) for j in range(i + 1, len(keys)) if scores[i][j] >= 85]\nfor i, j, s in sorted(pairs, key=lambda p: -p[2]):\n    print(f\"{s:5.1f}  words {by_words[i][j]:5.1f}  letters {by_letters[i][j]:5.1f}  \"\n          f\"{cust.at[i, 'Customer']:26} ~ {cust.at[j, 'Customer']}\")\n",[13,595,596,608,621,625,639,670,698,723,733,800,836,888],{"__ignoreMap":38},[42,597,598,600,603,605],{"class":44,"line":45},[42,599,85],{"class":84},[42,601,602],{"class":88}," numpy ",[42,604,108],{"class":84},[42,606,607],{"class":88}," np\n",[42,609,610,613,616,618],{"class":44,"line":92},[42,611,612],{"class":84},"from",[42,614,615],{"class":88}," rapidfuzz ",[42,617,85],{"class":84},[42,619,620],{"class":88}," fuzz, process\n",[42,622,623],{"class":44,"line":100},[42,624,118],{"emptyLinePlaceholder":117},[42,626,627,630,632,634,636],{"class":44,"line":114},[42,628,629],{"class":88},"keys ",[42,631,219],{"class":84},[42,633,373],{"class":88},[42,635,365],{"class":52},[42,637,638],{"class":88},"].tolist()\n",[42,640,641,644,646,649,651,653,655,658,661,664,667],{"class":44,"line":121},[42,642,643],{"class":88},"squashed ",[42,645,219],{"class":84},[42,647,648],{"class":88}," [k.replace(",[42,650,320],{"class":52},[42,652,228],{"class":88},[42,654,297],{"class":52},[42,656,657],{"class":88},") ",[42,659,660],{"class":84},"for",[42,662,663],{"class":88}," k ",[42,665,666],{"class":84},"in",[42,668,669],{"class":88}," keys]\n",[42,671,672,675,677,680,683,685,688,691,693,696],{"class":44,"line":195},[42,673,674],{"class":88},"by_words ",[42,676,219],{"class":84},[42,678,679],{"class":88}," process.cdist(keys, keys, ",[42,681,682],{"class":400},"scorer",[42,684,219],{"class":84},[42,686,687],{"class":88},"fuzz.token_sort_ratio, ",[42,689,690],{"class":400},"workers",[42,692,219],{"class":84},[42,694,695],{"class":124},"1",[42,697,266],{"class":88},[42,699,700,703,705,708,710,712,715,717,719,721],{"class":44,"line":200},[42,701,702],{"class":88},"by_letters ",[42,704,219],{"class":84},[42,706,707],{"class":88}," process.cdist(squashed, squashed, ",[42,709,682],{"class":400},[42,711,219],{"class":84},[42,713,714],{"class":88},"fuzz.ratio, ",[42,716,690],{"class":400},[42,718,219],{"class":84},[42,720,695],{"class":124},[42,722,266],{"class":88},[42,724,725,728,730],{"class":44,"line":213},[42,726,727],{"class":88},"scores ",[42,729,219],{"class":84},[42,731,732],{"class":88}," np.maximum(by_words, by_letters)\n",[42,734,735,738,740,743,745,748,750,753,755,758,761,763,766,768,770,773,776,779,781,783,785,788,791,794,797],{"class":44,"line":248},[42,736,737],{"class":88},"pairs ",[42,739,219],{"class":84},[42,741,742],{"class":88}," [(i, j, scores[i][j]) ",[42,744,660],{"class":84},[42,746,747],{"class":88}," i ",[42,749,666],{"class":84},[42,751,752],{"class":124}," range",[42,754,417],{"class":88},[42,756,757],{"class":124},"len",[42,759,760],{"class":88},"(keys)) ",[42,762,660],{"class":84},[42,764,765],{"class":88}," j ",[42,767,666],{"class":84},[42,769,752],{"class":124},[42,771,772],{"class":88},"(i ",[42,774,775],{"class":84},"+",[42,777,778],{"class":124}," 1",[42,780,228],{"class":88},[42,782,757],{"class":124},[42,784,760],{"class":88},[42,786,787],{"class":84},"if",[42,789,790],{"class":88}," scores[i][j] ",[42,792,793],{"class":84},">=",[42,795,796],{"class":124}," 85",[42,798,799],{"class":88},"]\n",[42,801,802,804,807,809,812,815,818,821,824,827,830,833],{"class":44,"line":269},[42,803,660],{"class":84},[42,805,806],{"class":88}," i, j, s ",[42,808,666],{"class":84},[42,810,811],{"class":124}," sorted",[42,813,814],{"class":88},"(pairs, ",[42,816,817],{"class":400},"key",[42,819,820],{"class":84},"=lambda",[42,822,823],{"class":88}," p: ",[42,825,826],{"class":84},"-",[42,828,829],{"class":88},"p[",[42,831,832],{"class":124},"2",[42,834,835],{"class":88},"]):\n",[42,837,838,841,843,846,848,852,855,858,861,864,866,869,871,873,876,878,881,883,885],{"class":44,"line":307},[42,839,840],{"class":124},"    print",[42,842,417],{"class":88},[42,844,845],{"class":84},"f",[42,847,134],{"class":52},[42,849,851],{"class":850},"sSjpA","{",[42,853,854],{"class":88},"s",[42,856,857],{"class":84},":5.1f",[42,859,860],{"class":850},"}",[42,862,863],{"class":52},"  words ",[42,865,851],{"class":850},[42,867,868],{"class":88},"by_words[i][j]",[42,870,857],{"class":84},[42,872,860],{"class":850},[42,874,875],{"class":52},"  letters ",[42,877,851],{"class":850},[42,879,880],{"class":88},"by_letters[i][j]",[42,882,857],{"class":84},[42,884,860],{"class":850},[42,886,887],{"class":52},"  \"\n",[42,889,890,893,895,897,900,903,906,909,911,914,916,919,921,923,925,927],{"class":44,"line":326},[42,891,892],{"class":84},"          f",[42,894,134],{"class":52},[42,896,851],{"class":850},[42,898,899],{"class":88},"cust.at[i, ",[42,901,902],{"class":52},"'Customer'",[42,904,905],{"class":88},"]",[42,907,908],{"class":84},":26",[42,910,860],{"class":850},[42,912,913],{"class":52}," ~ ",[42,915,851],{"class":850},[42,917,918],{"class":88},"cust.at[j, ",[42,920,902],{"class":52},[42,922,905],{"class":88},[42,924,860],{"class":850},[42,926,134],{"class":52},[42,928,266],{"class":88},[10,930,931,934,935,938,939,942],{},[13,932,933],{},"process.cdist"," computes the full similarity matrix in compiled code; scores run from 0 (nothing in common) to 100 (identical). Two scorers are combined, because names differ in two ways. ",[13,936,937],{},"token_sort_ratio"," sorts the words before comparing, so word order does not matter — but it treats \"bright water\" and \"brightwater\" as different words. Comparing the keys with all spaces removed using plain ",[13,940,941],{},"fuzz.ratio"," catches exactly those cases: \"Bright Water Supplies Ltd\" and \"Pine Crest Ltd\" now match their run-together spellings at 100. The pair score is the better of the two.",[10,944,945,946,950,951,26],{},"The list at 85 or above contains the true variants and one deliberate trap: \"brightwater supplies\" against \"brightwell supplies\" scores about 87, because only a few characters differ, yet they are different businesses. \"North Wind Trading\" against \"Northwind Traders\" scores about 81 and misses the cut — a near miss of the other kind. No threshold separates every true match from every false one, which is why the next steps end in a review rather than a merge. Matching ",[947,948,949],"em",{},"between"," two lists instead of within one is covered in ",[22,952,954],{"href":953},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Ffuzzy-match-names-between-two-excel-files-with-python\u002F","Fuzzy Match Names Between Two Excel Files with Python",[443,956,452,962,452,965,452,968,452,973,452,980,452,986,452,990,452,995,452,998,452,1002,452,1005,452,1009,452,1011,452,1015,452,1020,452,1025,452,1028,452,1032,452,1037,452,1042],{"viewBox":957,"role":446,"ariaLabelledBy":958,"xmlns":450,"style":961},"4 0 714 206",[959,960],"nd-scores-t","nd-scores-d","width:100%;max-width:714px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[454,963,964],{"id":959},"Pair scores around the 85 threshold",[458,966,967],{"id":960},"Similarity scores for selected pairs. Zenith Logistics versus Zenith Logistic scores 97 and Oakridge Farms versus Oak Ridge Farm 96, both true matches. Brightwater versus Brightwell scores 87, above the threshold but a different company. Northwind Traders versus North Wind Trading scores 81, below the threshold although it is probably the same company.",[462,969],{"x":970,"y":465,"width":971,"height":972,"fill":468},"4","714","206",[462,974],{"x":975,"y":976,"width":977,"height":474,"rx":978,"fill":979,"stroke":475,"style":476},"20.0","16.0","290.5","7","#e7ebef",[478,981,985],{"x":982,"y":983,"style":984},"165.3","34.5","font-size:12.5px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","Zenith Logistics ~ Logistic",[462,987],{"x":988,"y":976,"width":989,"height":474,"rx":978,"fill":530,"stroke":531,"style":532},"320.5","288.7",[478,991,994],{"x":992,"y":983,"style":993},"617.2","font-size:12px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:start","97 · match",[462,996],{"x":975,"y":997,"width":977,"height":474,"rx":978,"fill":979,"stroke":475,"style":476},"56.0",[478,999,1001],{"x":982,"y":1000,"style":984},"74.5","Oakridge Farms ~ Oak Ridge Farm",[462,1003],{"x":988,"y":997,"width":1004,"height":474,"rx":978,"fill":530,"stroke":531,"style":532},"285.7",[478,1006,1008],{"x":1007,"y":1000,"style":993},"614.2","96 · match",[462,1010],{"x":975,"y":494,"width":977,"height":474,"rx":978,"fill":979,"stroke":475,"style":476},[478,1012,1014],{"x":982,"y":1013,"style":984},"114.5","Brightwater ~ Brightwell",[462,1016],{"x":988,"y":494,"width":1017,"height":474,"rx":978,"fill":1018,"stroke":1019,"style":532},"258.9","#fce9e9","var(--danger,#dc2626)",[478,1021,1024],{"x":1022,"y":1013,"style":1023},"587.4","font-size:12px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:start","87 · different firms",[462,1026],{"x":975,"y":1027,"width":977,"height":474,"rx":978,"fill":979,"stroke":475,"style":476},"136.0",[478,1029,1031],{"x":982,"y":1030,"style":984},"154.5","Northwind Traders ~ North Wind Trading",[462,1033],{"x":988,"y":1027,"width":1034,"height":474,"rx":978,"fill":1035,"stroke":1036,"style":532},"241.1","#fdefd8","var(--gold,#b4740a)",[478,1038,1041],{"x":1039,"y":1030,"style":1040},"569.6","font-size:12px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:start","81 · missed",[478,1043,1047],{"x":1044,"y":1045,"style":1046},"380.0","186.0","font-size:12.5px;font-weight:400;fill:var(--muted,#5b6780);text-anchor:middle","The threshold of 85 lets one false match through and misses one true one — hence the review step.",[28,1049,1051],{"id":1050},"step-3-group-matches-into-clusters","Step 3 — Group matches into clusters",[10,1053,1054],{},"Pairs chain: if A matches B and B matches C, all three belong together even if A and C score lower. A small union-find structure turns the pairs into clusters:",[33,1056,1058],{"className":75,"code":1057,"language":77,"meta":38,"style":38},"parent = list(range(len(keys)))\n\ndef find(i):\n    while parent[i] != i:\n        parent[i] = parent[parent[i]]\n        i = parent[i]\n    return i\n\nfor i, j, _ in pairs:\n    parent[find(i)] = find(j)\n\ncust[\"Cluster\"] = [find(i) for i in range(len(keys))]\nsizes = cust[\"Cluster\"].map(cust[\"Cluster\"].value_counts())\ngroups = cust[sizes > 1].sort_values([\"Cluster\", \"Customer\"])\nfor _, g in groups.groupby(\"Cluster\"):\n    print(\" | \".join(g[\"Customer\"]))\n",[13,1059,1060,1082,1086,1096,1110,1120,1130,1137,1141,1153,1163,1167,1196,1215,1242,1259],{"__ignoreMap":38},[42,1061,1062,1065,1067,1070,1072,1075,1077,1079],{"class":44,"line":45},[42,1063,1064],{"class":88},"parent ",[42,1066,219],{"class":84},[42,1068,1069],{"class":124}," list",[42,1071,417],{"class":88},[42,1073,1074],{"class":124},"range",[42,1076,417],{"class":88},[42,1078,757],{"class":124},[42,1080,1081],{"class":88},"(keys)))\n",[42,1083,1084],{"class":44,"line":92},[42,1085,118],{"emptyLinePlaceholder":117},[42,1087,1088,1090,1093],{"class":44,"line":100},[42,1089,203],{"class":84},[42,1091,1092],{"class":206}," find",[42,1094,1095],{"class":88},"(i):\n",[42,1097,1098,1101,1104,1107],{"class":44,"line":114},[42,1099,1100],{"class":84},"    while",[42,1102,1103],{"class":88}," parent[i] ",[42,1105,1106],{"class":84},"!=",[42,1108,1109],{"class":88}," i:\n",[42,1111,1112,1115,1117],{"class":44,"line":121},[42,1113,1114],{"class":88},"        parent[i] ",[42,1116,219],{"class":84},[42,1118,1119],{"class":88}," parent[parent[i]]\n",[42,1121,1122,1125,1127],{"class":44,"line":195},[42,1123,1124],{"class":88},"        i ",[42,1126,219],{"class":84},[42,1128,1129],{"class":88}," parent[i]\n",[42,1131,1132,1134],{"class":44,"line":200},[42,1133,329],{"class":84},[42,1135,1136],{"class":88}," i\n",[42,1138,1139],{"class":44,"line":213},[42,1140,118],{"emptyLinePlaceholder":117},[42,1142,1143,1145,1148,1150],{"class":44,"line":248},[42,1144,660],{"class":84},[42,1146,1147],{"class":88}," i, j, _ ",[42,1149,666],{"class":84},[42,1151,1152],{"class":88}," pairs:\n",[42,1154,1155,1158,1160],{"class":44,"line":269},[42,1156,1157],{"class":88},"    parent[find(i)] ",[42,1159,219],{"class":84},[42,1161,1162],{"class":88}," find(j)\n",[42,1164,1165],{"class":44,"line":307},[42,1166,118],{"emptyLinePlaceholder":117},[42,1168,1169,1171,1174,1176,1178,1181,1183,1185,1187,1189,1191,1193],{"class":44,"line":326},[42,1170,362],{"class":88},[42,1172,1173],{"class":52},"\"Cluster\"",[42,1175,368],{"class":88},[42,1177,219],{"class":84},[42,1179,1180],{"class":88}," [find(i) ",[42,1182,660],{"class":84},[42,1184,747],{"class":88},[42,1186,666],{"class":84},[42,1188,752],{"class":124},[42,1190,417],{"class":88},[42,1192,757],{"class":124},[42,1194,1195],{"class":88},"(keys))]\n",[42,1197,1198,1201,1203,1205,1207,1210,1212],{"class":44,"line":338},[42,1199,1200],{"class":88},"sizes ",[42,1202,219],{"class":84},[42,1204,373],{"class":88},[42,1206,1173],{"class":52},[42,1208,1209],{"class":88},"].map(cust[",[42,1211,1173],{"class":52},[42,1213,1214],{"class":88},"].value_counts())\n",[42,1216,1217,1220,1222,1225,1228,1230,1233,1235,1237,1239],{"class":44,"line":343},[42,1218,1219],{"class":88},"groups ",[42,1221,219],{"class":84},[42,1223,1224],{"class":88}," cust[sizes ",[42,1226,1227],{"class":84},">",[42,1229,778],{"class":124},[42,1231,1232],{"class":88},"].sort_values([",[42,1234,1173],{"class":52},[42,1236,228],{"class":88},[42,1238,376],{"class":52},[42,1240,1241],{"class":88},"])\n",[42,1243,1244,1246,1249,1251,1254,1256],{"class":44,"line":359},[42,1245,660],{"class":84},[42,1247,1248],{"class":88}," _, g ",[42,1250,666],{"class":84},[42,1252,1253],{"class":88}," groups.groupby(",[42,1255,1173],{"class":52},[42,1257,1258],{"class":88},"):\n",[42,1260,1261,1263,1265,1268,1271,1273],{"class":44,"line":382},[42,1262,840],{"class":124},[42,1264,417],{"class":88},[42,1266,1267],{"class":52},"\" | \"",[42,1269,1270],{"class":88},".join(g[",[42,1272,376],{"class":52},[42,1274,1275],{"class":88},"]))\n",[10,1277,1278],{},"Each printed line is one suspected entity. Chaining is powerful and slightly dangerous: one borderline pair can join two otherwise separate clusters. Keeping the threshold fairly high and reviewing the clusters is the usual compromise.",[443,1280,452,1286,452,1289,452,1292,452,1296,452,1303,452,1314,452,1320,452,1325,452,1329,452,1336,452,1342,452,1345,452,1349,452,1352,452,1355,452,1358,452,1363,452,1367,452,1371,452,1375,452,1378,452,1380,452,1383,452,1390,452,1395,452,1399,452,1403,452,1406,452,1408,452,1411,452,1418,452,1423,452,1427,452,1430],{"viewBox":1281,"role":446,"ariaLabelledBy":1282,"xmlns":450,"style":1285},"4 0 752 487",[1283,1284],"nd-clusters-t","nd-clusters-d","width:100%;max-width:752px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[454,1287,1288],{"id":1283},"Twenty names reduced to suspected entities",[458,1290,1291],{"id":1284},"The twenty customer names on the left collapse into clusters such as ACME (three spellings), Northwind (two), Brightwater (two, plus a false match with Brightwell), Oakridge, Zenith, Harbour, Pinecrest and Café Señor, with Riverside Dental on its own.",[462,1293],{"x":970,"y":465,"width":1294,"height":1295,"fill":468},"752","487",[462,1297],{"x":1298,"y":1299,"width":1300,"height":1301,"rx":1302,"fill":547,"stroke":548,"style":567},"520.0","197.0","220.0","93.0","12",[1304,1305,1307],"g",{"transform":1306},"translate(618.0,210.0) scale(1.000)",[1308,1309],"path",{"d":1310,"fill":1311,"stroke":1312,"style":1313},"M9 11a3.5 3.5 0 1 0 0-7 3.5 3.5 0 0 0 0 7z M2.5 20c0-3.6 2.9-6 6.5-6s6.5 2.4 6.5 6 M16 4.5a3.3 3.3 0 0 1 0 6.3 M18 14.3c2.2.7 3.5 2.8 3.5 5.7","none","var(--brand-strong,#4338ca)","stroke-width:1.80px;stroke-linecap:round;stroke-linejoin:round",[478,1315,1319],{"x":1316,"y":1317,"style":1318},"630.0","256.5","font-size:13.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Clusters to review",[478,1321,1324],{"x":1316,"y":1322,"style":1323},"276.0","font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","one row per suspected entity",[462,1326],{"x":975,"y":976,"width":1327,"height":1328,"rx":1302,"fill":530,"stroke":531,"style":567},"250.0","109.0",[1304,1330,1332],{"transform":1331},"translate(133.0,29.0) scale(1.000)",[1308,1333],{"d":1334,"fill":1311,"stroke":1335,"style":1313},"M5 12.5l4.5 4.5L19 7","var(--teal-ink,#0b6157)",[478,1337,1341],{"x":1338,"y":1339,"style":1340},"145.0","75.0","font-size:13px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","ACME Ltd · Acme Limited ·",[478,1343,505],{"x":1338,"y":1344,"style":1340},"92.0",[478,1346,1348],{"x":1338,"y":1347,"style":490},"111.0","normalised to the same key",[1308,1350],{"d":1351,"fill":1311,"stroke":548,"style":567},"M274.0,70.5 C394.0,70.5 394.0,243.5 510.0,243.5",[569,1353],{"points":1354,"fill":572},"514.0,243.5 505.0,248.5 505.0,238.5",[462,1356],{"x":975,"y":1357,"width":1327,"height":1328,"rx":1302,"fill":530,"stroke":531,"style":567},"137.0",[1304,1359,1361],{"transform":1360},"translate(133.0,150.0) scale(1.000)",[1308,1362],{"d":1334,"fill":1311,"stroke":1335,"style":1313},[478,1364,1366],{"x":1338,"y":1365,"style":1340},"196.0","Northwind Traders · Northwind",[478,1368,1370],{"x":1338,"y":1369,"style":1340},"213.0","Traders GmbH",[478,1372,1374],{"x":1338,"y":1373,"style":490},"232.0","North Wind Trading just missed",[1308,1376],{"d":1377,"fill":1311,"stroke":548,"style":567},"M274.0,191.5 C394.0,191.5 394.0,243.5 510.0,243.5",[569,1379],{"points":1354,"fill":572},[462,1381],{"x":975,"y":1382,"width":1327,"height":1328,"rx":1302,"fill":1035,"stroke":1036,"style":567},"258.0",[1304,1384,1386],{"transform":1385},"translate(133.0,271.0) scale(1.000)",[1308,1387],{"d":1388,"fill":1311,"stroke":1389,"style":1313},"M12 3.5l9.5 17h-19z M12 10v5 M12 17.8v.4","var(--gold-ink,#7a4e06)",[478,1391,1394],{"x":1338,"y":1392,"style":1393},"317.0","font-size:13px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","Brightwater Supplies · Bright",[478,1396,1398],{"x":1338,"y":1397,"style":1393},"334.0","Water … · Brightwell?",[478,1400,1402],{"x":1338,"y":1401,"style":490},"353.0","check before merging",[1308,1404],{"d":1405,"fill":1311,"stroke":548,"style":567},"M274.0,312.5 C394.0,312.5 394.0,243.5 510.0,243.5",[569,1407],{"points":1354,"fill":572},[462,1409],{"x":975,"y":1410,"width":1327,"height":1344,"rx":1302,"fill":979,"stroke":475,"style":567},"379.0",[1304,1412,1414],{"transform":1413},"translate(133.0,392.0) scale(1.000)",[1308,1415],{"d":1416,"fill":1311,"stroke":1417,"style":1313},"M12 12a4 4 0 1 0 0-8 4 4 0 0 0 0 8z M4 21c0-4 3.6-7 8-7s8 3 8 7","var(--text,#172033)",[478,1419,1422],{"x":1338,"y":1420,"style":1421},"438.0","font-size:13px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","Riverside Dental",[478,1424,1426],{"x":1338,"y":1425,"style":490},"457.0","no near-duplicate",[1308,1428],{"d":1429,"fill":1311,"stroke":548,"style":567},"M274.0,425.0 C394.0,425.0 394.0,243.5 510.0,243.5",[569,1431],{"points":1354,"fill":572},[28,1433,1435],{"id":1434},"step-4-pick-a-canonical-name-and-write-a-review-sheet","Step 4 — Pick a canonical name and write a review sheet",[10,1437,1438],{},"For each cluster, propose one spelling — the most frequent, or the longest original as a stand-in for the most complete — and write everything to a sheet where a person approves or rejects each row:",[33,1440,1442],{"className":75,"code":1441,"language":77,"meta":38,"style":38},"def canonical(group):\n    return group.loc[group.str.len().idxmax()]\n\ncust[\"Proposed\"] = cust.groupby(\"Cluster\")[\"Customer\"].transform(canonical)\nreview = cust[sizes > 1][[\"CustomerID\", \"Customer\", \"Proposed\", \"Cluster\"]].sort_values([\"Cluster\", \"Customer\"])\nreview[\"Approve (Y\u002FN)\"] = \"\"\nwith pd.ExcelWriter(\"near_duplicates_review.xlsx\", engine=\"xlsxwriter\") as xw:\n    review.to_excel(xw, sheet_name=\"Review\", index=False)\n    ws = xw.sheets[\"Review\"]\n    ws.set_column(\"A:A\", 12)\n    ws.set_column(\"B:C\", 30)\n    ws.data_validation(1, 4, len(review), 4, {\"validate\": \"list\", \"source\": [\"Y\", \"N\"]})\n    ws.freeze_panes(1, 0)\nprint(review.to_string(index=False))\n",[13,1443,1444,1454,1461,1465,1489,1531,1546,1574,1597,1611,1625,1639,1690,1703],{"__ignoreMap":38},[42,1445,1446,1448,1451],{"class":44,"line":45},[42,1447,203],{"class":84},[42,1449,1450],{"class":206}," canonical",[42,1452,1453],{"class":88},"(group):\n",[42,1455,1456,1458],{"class":44,"line":92},[42,1457,329],{"class":84},[42,1459,1460],{"class":88}," group.loc[group.str.len().idxmax()]\n",[42,1462,1463],{"class":44,"line":100},[42,1464,118],{"emptyLinePlaceholder":117},[42,1466,1467,1469,1472,1474,1476,1479,1481,1484,1486],{"class":44,"line":114},[42,1468,362],{"class":88},[42,1470,1471],{"class":52},"\"Proposed\"",[42,1473,368],{"class":88},[42,1475,219],{"class":84},[42,1477,1478],{"class":88}," cust.groupby(",[42,1480,1173],{"class":52},[42,1482,1483],{"class":88},")[",[42,1485,376],{"class":52},[42,1487,1488],{"class":88},"].transform(canonical)\n",[42,1490,1491,1494,1496,1498,1500,1502,1505,1508,1510,1512,1514,1516,1518,1520,1523,1525,1527,1529],{"class":44,"line":121},[42,1492,1493],{"class":88},"review ",[42,1495,219],{"class":84},[42,1497,1224],{"class":88},[42,1499,1227],{"class":84},[42,1501,778],{"class":124},[42,1503,1504],{"class":88},"][[",[42,1506,1507],{"class":52},"\"CustomerID\"",[42,1509,228],{"class":88},[42,1511,376],{"class":52},[42,1513,228],{"class":88},[42,1515,1471],{"class":52},[42,1517,228],{"class":88},[42,1519,1173],{"class":52},[42,1521,1522],{"class":88},"]].sort_values([",[42,1524,1173],{"class":52},[42,1526,228],{"class":88},[42,1528,376],{"class":52},[42,1530,1241],{"class":88},[42,1532,1533,1536,1539,1541,1543],{"class":44,"line":195},[42,1534,1535],{"class":88},"review[",[42,1537,1538],{"class":52},"\"Approve (Y\u002FN)\"",[42,1540,368],{"class":88},[42,1542,219],{"class":84},[42,1544,1545],{"class":52}," \"\"\n",[42,1547,1548,1551,1554,1557,1559,1562,1564,1567,1569,1571],{"class":44,"line":200},[42,1549,1550],{"class":84},"with",[42,1552,1553],{"class":88}," pd.ExcelWriter(",[42,1555,1556],{"class":52},"\"near_duplicates_review.xlsx\"",[42,1558,228],{"class":88},[42,1560,1561],{"class":400},"engine",[42,1563,219],{"class":84},[42,1565,1566],{"class":52},"\"xlsxwriter\"",[42,1568,657],{"class":88},[42,1570,108],{"class":84},[42,1572,1573],{"class":88}," xw:\n",[42,1575,1576,1579,1582,1584,1587,1589,1591,1593,1595],{"class":44,"line":213},[42,1577,1578],{"class":88},"    review.to_excel(xw, ",[42,1580,1581],{"class":400},"sheet_name",[42,1583,219],{"class":84},[42,1585,1586],{"class":52},"\"Review\"",[42,1588,228],{"class":88},[42,1590,401],{"class":400},[42,1592,219],{"class":84},[42,1594,406],{"class":124},[42,1596,266],{"class":88},[42,1598,1599,1602,1604,1607,1609],{"class":44,"line":248},[42,1600,1601],{"class":88},"    ws ",[42,1603,219],{"class":84},[42,1605,1606],{"class":88}," xw.sheets[",[42,1608,1586],{"class":52},[42,1610,799],{"class":88},[42,1612,1613,1616,1619,1621,1623],{"class":44,"line":269},[42,1614,1615],{"class":88},"    ws.set_column(",[42,1617,1618],{"class":52},"\"A:A\"",[42,1620,228],{"class":88},[42,1622,1302],{"class":124},[42,1624,266],{"class":88},[42,1626,1627,1629,1632,1634,1637],{"class":44,"line":307},[42,1628,1615],{"class":88},[42,1630,1631],{"class":52},"\"B:C\"",[42,1633,228],{"class":88},[42,1635,1636],{"class":124},"30",[42,1638,266],{"class":88},[42,1640,1641,1644,1646,1648,1650,1652,1654,1657,1659,1662,1665,1668,1671,1673,1676,1679,1682,1684,1687],{"class":44,"line":326},[42,1642,1643],{"class":88},"    ws.data_validation(",[42,1645,695],{"class":124},[42,1647,228],{"class":88},[42,1649,970],{"class":124},[42,1651,228],{"class":88},[42,1653,757],{"class":124},[42,1655,1656],{"class":88},"(review), ",[42,1658,970],{"class":124},[42,1660,1661],{"class":88},", {",[42,1663,1664],{"class":52},"\"validate\"",[42,1666,1667],{"class":88},": ",[42,1669,1670],{"class":52},"\"list\"",[42,1672,228],{"class":88},[42,1674,1675],{"class":52},"\"source\"",[42,1677,1678],{"class":88},": [",[42,1680,1681],{"class":52},"\"Y\"",[42,1683,228],{"class":88},[42,1685,1686],{"class":52},"\"N\"",[42,1688,1689],{"class":88},"]})\n",[42,1691,1692,1695,1697,1699,1701],{"class":44,"line":338},[42,1693,1694],{"class":88},"    ws.freeze_panes(",[42,1696,695],{"class":124},[42,1698,228],{"class":88},[42,1700,465],{"class":124},[42,1702,266],{"class":88},[42,1704,1705,1707,1710,1712,1714,1716],{"class":44,"line":343},[42,1706,385],{"class":124},[42,1708,1709],{"class":88},"(review.to_string(",[42,1711,401],{"class":400},[42,1713,219],{"class":84},[42,1715,406],{"class":124},[42,1717,409],{"class":88},[10,1719,1720,1721,26],{},"The reviewer sees each original name next to the proposed canonical spelling and picks Y or N from a dropdown. Cluster 5 shows why this matters: Brightwell Supplies has been pulled into the Brightwater cluster through its high score, and the reviewer — who knows these are two different suppliers — marks that row N while approving the two Brightwater spellings. Once the sheet comes back, read it, apply only the approved mappings to the source data, and keep the review file as the record of what was merged and why — the audit trail that an automatic merge would lose. Dropdowns are covered in ",[22,1722,1724],{"href":1723},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fadd-dropdown-data-validation-to-excel-with-openpyxl\u002F","Add Dropdown Data Validation to Excel with openpyxl",[28,1726,1728],{"id":1727},"step-5-scale-with-blocking","Step 5 — Scale with blocking",[10,1730,1731,1734,1735,1738],{},[13,1732,1733],{},"cdist"," compares every name with every other: 20 names make 400 comparisons, 50,000 names make 2.5 billion — too many even for rapidfuzz. ",[947,1736,1737],{},"Blocking"," only compares names that share something cheap to compute, such as the first letter of the key:",[33,1740,1742],{"className":75,"code":1741,"language":77,"meta":38,"style":38},"def fuzzy_pairs(keys, threshold=85, block=lambda k: k[:1]):\n    out = []\n    frame = pd.DataFrame({\"key\": keys})\n    for _, idx in frame.groupby(frame[\"key\"].map(block)).groups.items():\n        idx = list(idx)\n        if len(idx) \u003C 2:\n            continue\n        sub = [keys[i] for i in idx]\n        m = np.maximum(process.cdist(sub, sub, scorer=fuzz.token_sort_ratio, workers=1),\n                       process.cdist([k.replace(\" \", \"\") for k in sub], [k.replace(\" \", \"\") for k in sub],\n                                     scorer=fuzz.ratio, workers=1))\n        out += [(idx[a], idx[b], m[a][b]) for a in range(len(sub)) for b in range(a + 1, len(sub)) if m[a][b] >= threshold]\n    return out\n\nblocked = fuzzy_pairs(keys)\nprint(len(pairs), \"pairs without blocking,\", len(blocked), \"with first-letter blocking\")\n",[13,1743,1744,1771,1781,1797,1815,1827,1847,1852,1871,1896,1935,1952,2011,2018,2022,2032],{"__ignoreMap":38},[42,1745,1746,1748,1751,1754,1756,1759,1762,1764,1767,1769],{"class":44,"line":45},[42,1747,203],{"class":84},[42,1749,1750],{"class":206}," fuzzy_pairs",[42,1752,1753],{"class":88},"(keys, threshold",[42,1755,219],{"class":84},[42,1757,1758],{"class":124},"85",[42,1760,1761],{"class":88},", block",[42,1763,820],{"class":84},[42,1765,1766],{"class":88}," k: k[:",[42,1768,695],{"class":124},[42,1770,835],{"class":88},[42,1772,1773,1776,1778],{"class":44,"line":92},[42,1774,1775],{"class":88},"    out ",[42,1777,219],{"class":84},[42,1779,1780],{"class":88}," []\n",[42,1782,1783,1786,1788,1791,1794],{"class":44,"line":100},[42,1784,1785],{"class":88},"    frame ",[42,1787,219],{"class":84},[42,1789,1790],{"class":88}," pd.DataFrame({",[42,1792,1793],{"class":52},"\"key\"",[42,1795,1796],{"class":88},": keys})\n",[42,1798,1799,1802,1805,1807,1810,1812],{"class":44,"line":114},[42,1800,1801],{"class":84},"    for",[42,1803,1804],{"class":88}," _, idx ",[42,1806,666],{"class":84},[42,1808,1809],{"class":88}," frame.groupby(frame[",[42,1811,1793],{"class":52},[42,1813,1814],{"class":88},"].map(block)).groups.items():\n",[42,1816,1817,1820,1822,1824],{"class":44,"line":121},[42,1818,1819],{"class":88},"        idx ",[42,1821,219],{"class":84},[42,1823,1069],{"class":124},[42,1825,1826],{"class":88},"(idx)\n",[42,1828,1829,1832,1835,1838,1841,1844],{"class":44,"line":195},[42,1830,1831],{"class":84},"        if",[42,1833,1834],{"class":124}," len",[42,1836,1837],{"class":88},"(idx) ",[42,1839,1840],{"class":84},"\u003C",[42,1842,1843],{"class":124}," 2",[42,1845,1846],{"class":88},":\n",[42,1848,1849],{"class":44,"line":200},[42,1850,1851],{"class":84},"            continue\n",[42,1853,1854,1857,1859,1862,1864,1866,1868],{"class":44,"line":213},[42,1855,1856],{"class":88},"        sub ",[42,1858,219],{"class":84},[42,1860,1861],{"class":88}," [keys[i] ",[42,1863,660],{"class":84},[42,1865,747],{"class":88},[42,1867,666],{"class":84},[42,1869,1870],{"class":88}," idx]\n",[42,1872,1873,1876,1878,1881,1883,1885,1887,1889,1891,1893],{"class":44,"line":248},[42,1874,1875],{"class":88},"        m ",[42,1877,219],{"class":84},[42,1879,1880],{"class":88}," np.maximum(process.cdist(sub, sub, ",[42,1882,682],{"class":400},[42,1884,219],{"class":84},[42,1886,687],{"class":88},[42,1888,690],{"class":400},[42,1890,219],{"class":84},[42,1892,695],{"class":124},[42,1894,1895],{"class":88},"),\n",[42,1897,1898,1901,1903,1905,1907,1909,1911,1913,1915,1918,1920,1922,1924,1926,1928,1930,1932],{"class":44,"line":269},[42,1899,1900],{"class":88},"                       process.cdist([k.replace(",[42,1902,320],{"class":52},[42,1904,228],{"class":88},[42,1906,297],{"class":52},[42,1908,657],{"class":88},[42,1910,660],{"class":84},[42,1912,663],{"class":88},[42,1914,666],{"class":84},[42,1916,1917],{"class":88}," sub], [k.replace(",[42,1919,320],{"class":52},[42,1921,228],{"class":88},[42,1923,297],{"class":52},[42,1925,657],{"class":88},[42,1927,660],{"class":84},[42,1929,663],{"class":88},[42,1931,666],{"class":84},[42,1933,1934],{"class":88}," sub],\n",[42,1936,1937,1940,1942,1944,1946,1948,1950],{"class":44,"line":307},[42,1938,1939],{"class":400},"                                     scorer",[42,1941,219],{"class":84},[42,1943,714],{"class":88},[42,1945,690],{"class":400},[42,1947,219],{"class":84},[42,1949,695],{"class":124},[42,1951,409],{"class":88},[42,1953,1954,1957,1960,1963,1965,1968,1970,1972,1974,1976,1979,1981,1984,1986,1988,1991,1993,1995,1997,1999,2001,2003,2006,2008],{"class":44,"line":326},[42,1955,1956],{"class":88},"        out ",[42,1958,1959],{"class":84},"+=",[42,1961,1962],{"class":88}," [(idx[a], idx[b], m[a][b]) ",[42,1964,660],{"class":84},[42,1966,1967],{"class":88}," a ",[42,1969,666],{"class":84},[42,1971,752],{"class":124},[42,1973,417],{"class":88},[42,1975,757],{"class":124},[42,1977,1978],{"class":88},"(sub)) ",[42,1980,660],{"class":84},[42,1982,1983],{"class":88}," b ",[42,1985,666],{"class":84},[42,1987,752],{"class":124},[42,1989,1990],{"class":88},"(a ",[42,1992,775],{"class":84},[42,1994,778],{"class":124},[42,1996,228],{"class":88},[42,1998,757],{"class":124},[42,2000,1978],{"class":88},[42,2002,787],{"class":84},[42,2004,2005],{"class":88}," m[a][b] ",[42,2007,793],{"class":84},[42,2009,2010],{"class":88}," threshold]\n",[42,2012,2013,2015],{"class":44,"line":338},[42,2014,329],{"class":84},[42,2016,2017],{"class":88}," out\n",[42,2019,2020],{"class":44,"line":343},[42,2021,118],{"emptyLinePlaceholder":117},[42,2023,2024,2027,2029],{"class":44,"line":359},[42,2025,2026],{"class":88},"blocked ",[42,2028,219],{"class":84},[42,2030,2031],{"class":88}," fuzzy_pairs(keys)\n",[42,2033,2034,2036,2038,2040,2043,2046,2048,2050,2053,2056],{"class":44,"line":382},[42,2035,385],{"class":124},[42,2037,417],{"class":88},[42,2039,757],{"class":124},[42,2041,2042],{"class":88},"(pairs), ",[42,2044,2045],{"class":52},"\"pairs without blocking,\"",[42,2047,228],{"class":88},[42,2049,757],{"class":124},[42,2051,2052],{"class":88},"(blocked), ",[42,2054,2055],{"class":52},"\"with first-letter blocking\"",[42,2057,266],{"class":88},[10,2059,2060,2061,2064],{},"With first-letter blocking the number of comparisons drops roughly by the number of blocks, and here no pair is lost because every matching pair starts with the same letter. The trade-off is that names differing in their first character — \"The Acme Company\" against \"Acme\" — can no longer match; normalisation that strips leading words such as \"the\" reduces that risk. For lists of tens of thousands, also pass ",[13,2062,2063],{},"workers=-1"," to use every CPU core.",[28,2066,2068],{"id":2067},"common-pitfalls","Common pitfalls",[2070,2071,2072,2080,2086,2092,2098,2104],"ul",{},[2073,2074,2075,2079],"li",{},[2076,2077,2078],"strong",{},"Merging automatically."," Fuzzy scores produce false positives (Brightwater versus Brightwell); always review.",[2073,2081,2082,2085],{},[2076,2083,2084],{},"Skipping normalisation."," Without it, legal suffixes and punctuation dominate the scores.",[2073,2087,2088,2091],{},[2076,2089,2090],{},"Over-normalising."," Stripping too many words (\"supplies\", \"farm\") makes different businesses identical.",[2073,2093,2094,2097],{},[2076,2095,2096],{},"Chaining clusters."," One weak pair can glue two clusters together; check large clusters closely.",[2073,2099,2100,2103],{},[2076,2101,2102],{},"Comparing everything with everything."," Quadratic growth; block for large lists.",[2073,2105,2106,2109],{},[2076,2107,2108],{},"Picking the canonical name blindly."," The longest spelling is a heuristic; let the reviewer overwrite the proposed name, or choose the spelling used on the most recent invoice.",[28,2111,2113],{"id":2112},"conclusion","Conclusion",[10,2115,2116,2117,2119],{},"Near-duplicates in one column need two passes: normalise away case, accents, punctuation and legal suffixes, which turns many variants into exact matches; then score the remaining pairs with ",[13,2118,19],{}," and group matches into clusters with union-find. Propose a canonical name per cluster, write a review sheet with an approval dropdown, apply only what a person approves, and block by a cheap key so the comparison scales beyond a few thousand names.",[28,2121,2123],{"id":2122},"frequently-asked-questions","Frequently asked questions",[10,2125,2126,2129],{},[2076,2127,2128],{},"How do I find similar but not identical names in an Excel column?","\nNormalise the names (lower case, remove punctuation, accents and legal suffixes), then compare every pair with rapidfuzz, for example process.cdist with fuzz.token_sort_ratio, plus fuzz.ratio on the names with spaces removed, and keep pairs above a threshold such as 85.",[10,2131,2132,2135],{},[2076,2133,2134],{},"What similarity threshold should I use?","\nStart around 85 for token_sort_ratio on normalised company names and adjust after looking at the results. Lower thresholds find more variants but more false matches; there is no value that avoids review entirely.",[10,2137,2138,2141],{},[2076,2139,2140],{},"How do I group fuzzy matches into one entity?","\nTreat each matching pair as an edge and find connected components, for example with a small union-find structure. Every name in a component is a suspected duplicate of the others.",[10,2143,2144,2147],{},[2076,2145,2146],{},"How do I de-duplicate a list of 50,000 names efficiently?","\nAvoid comparing every pair: block the names by a cheap key such as the first letter of the normalised name and compare only within blocks, and use rapidfuzz with workers=-1 to use all CPU cores.",[28,2149,2151],{"id":2150},"related","Related",[2070,2153,2154,2160,2167,2174],{},[2073,2155,2156,2157,2159],{},"Up one level: ",[22,2158,25],{"href":24}," — cleaning messy spreadsheet data.",[2073,2161,2162,2166],{},[22,2163,2165],{"href":2164},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fpandas-drop-duplicates-from-excel-column\u002F","pandas drop_duplicates on an Excel Column"," — exact duplicates first.",[2073,2168,2169,2173],{},[22,2170,2172],{"href":2171},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fstandardise-category-values-with-a-mapping-table\u002F","Standardise Category Values with a Mapping Table"," — applying approved mappings.",[2073,2175,2176,2180],{},[22,2177,2179],{"href":2178},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Ffind-duplicate-rows-in-excel-with-python\u002F","Find Duplicate Rows in Excel with Python"," — duplicate detection across several columns.",[2182,2183,2184],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}",{"title":38,"searchDepth":92,"depth":92,"links":2186},[2187,2188,2189,2190,2191,2192,2193,2194,2195,2196],{"id":30,"depth":92,"text":31},{"id":71,"depth":92,"text":72},{"id":589,"depth":92,"text":590},{"id":1050,"depth":92,"text":1051},{"id":1434,"depth":92,"text":1435},{"id":1727,"depth":92,"text":1728},{"id":2067,"depth":92,"text":2068},{"id":2112,"depth":92,"text":2113},{"id":2122,"depth":92,"text":2123},{"id":2150,"depth":92,"text":2151},"2026-10-02","Customer and supplier lists collect variants of the same name — \"ACME Ltd\", \"Acme Limited\", \"A.C.M.E. Ltd.\" — that exact de-duplication misses. Normalise names, compare every pair with rapidfuzz, group matches into clusters, choose a canonical spelling, block by first letter to scale to tens of thousands of names, and write a review sheet so a person confirms each merge.","md",[2201,2203,2205,2207],{"q":2128,"a":2202},"Normalise the names (lower case, remove punctuation, accents and legal suffixes), then compare every pair with rapidfuzz, for example process.cdist with fuzz.token_sort_ratio, plus fuzz.ratio on the names with spaces removed, and keep pairs above a threshold such as 85.",{"q":2134,"a":2204},"Start around 85 for token_sort_ratio on normalised company names and adjust after looking at the results. Lower thresholds find more variants but more false matches; there is no value that avoids review entirely.",{"q":2140,"a":2206},"Treat each matching pair as an edge and find connected components, for example with a small union-find structure. Every name in a component is a suspected duplicate of the others.",{"q":2146,"a":2208},"Avoid comparing every pair: block the names by a cheap key such as the first letter of the normalised name and compare only within blocks, and use rapidfuzz with workers=-1 to use all CPU cores.","find near duplicate names within one excel column",null,{"breadcrumb":2212},[2213,2216,2219],{"name":2214,"item":2215},"Home","\u002F",{"name":2217,"item":2218},"Advanced Data Transformation and Cleaning for Excel Automation","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":2220,"item":24},"Cleaning Excel Data With Pandas: A Production Workflow","\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffind-near-duplicate-names-within-one-excel-column",{"title":2223,"description":2224},"Find Near-Duplicate Names in One Excel Column with Python","Fuzzy de-duplication of a single Excel column: normalise case, punctuation and legal suffixes, rapidfuzz cdist similarity, union-find clusters, a canonical name per cluster, blocking for large lists, and an Excel review sheet before merging.","find-near-duplicate-names-within-one-excel-column","advanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffind-near-duplicate-names-within-one-excel-column\u002Findex",[2228,2229],"data-cleaning","data-validation","how-to","KaOCwzf29GfrlK4mjhIaNRFwas_f1n5RkpAW2XFDlR0",[2233,2237],{"title":2234,"path":2235,"stem":2236,"children":-1},"Filter Excel Rows by Condition with pandas","\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffilter-excel-rows-by-condition-with-pandas","advanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffilter-excel-rows-by-condition-with-pandas\u002Findex",{"title":2238,"path":2239,"stem":2240,"children":-1},"Fix Garbled Characters Like \"Café\" and \"’\" in Excel Data","\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffix-garbled-characters-in-excel-data","advanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Ffix-garbled-characters-in-excel-data\u002Findex",[2242,2244],{"path":2243,"title":2220},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas",{"path":2245,"title":2217},"\u002Fadvanced-data-transformation-and-cleaning",[],[2248,2250,2253,2256,2259,2262],{"path":2249,"title":2179,"kind":2230},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Ffind-duplicate-rows-in-excel-with-python",{"path":2251,"title":2252,"kind":2230},"\u002Fadvanced-data-transformation-and-cleaning\u002Fapplying-conditional-formatting-with-openpyxl\u002Fhighlight-duplicate-values-in-excel-with-openpyxl","Highlight Duplicate Values in Excel with openpyxl",{"path":2254,"title":2255,"kind":2230},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fclean-mixed-currency-amounts-into-amount-and-currency-columns","Clean Mixed-Currency Amounts into Amount and Currency Columns",{"path":2257,"title":2258,"kind":2230},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fcoalesce-several-excel-columns-into-one-with-pandas","Coalesce Several Excel Columns into One with pandas",{"path":2260,"title":2261,"kind":2230},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fconvert-currency-strings-to-numbers-with-pandas","Convert Currency Strings Like \"$1,234.56\" and \"(500)\" to Numbers with pandas",{"path":2263,"title":2264,"kind":2230},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fconvert-excel-text-columns-to-numbers-with-pandas","Convert Excel Text Columns to Numbers with pandas",1790972080729]