[{"data":1,"prerenderedAt":1967},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fvalidate-a-merge-to-avoid-duplicate-rows":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fvalidate-a-merge-to-avoid-duplicate-rows":1934,"ancestors:\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fvalidate-a-merge-to-avoid-duplicate-rows":1942,"children:\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fvalidate-a-merge-to-avoid-duplicate-rows":1947,"related:\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fvalidate-a-merge-to-avoid-duplicate-rows":1948},{"id":4,"title":5,"body":6,"dateModified":1898,"datePublished":1898,"description":1899,"extension":1900,"faq":1901,"keyword":1910,"kind":1911,"meta":1912,"navigation":396,"path":1922,"seo":1923,"slug":1926,"stem":1927,"topics":1928,"type":1932,"__hash__":1933},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fvalidate-a-merge-to-avoid-duplicate-rows\u002Findex.md","Validate a pandas Merge to Avoid Duplicate Rows",{"type":7,"value":8,"toc":1885},"minimark",[9,19,332,337,365,368,686,690,768,771,775,778,848,867,880,1012,1016,1019,1080,1095,1099,1102,1212,1219,1223,1233,1306,1313,1469,1473,1478,1745,1751,1754,1758,1807,1811,1820,1824,1830,1836,1842,1848,1852,1881],[10,11,12,13,18],"p",{},"Merging an order sheet with a customer sheet should give one row per order. Then someone adds a second row for customer C2 to the customer workbook — a changed region, a typo, a copy-paste — and the next merge quietly returns extra rows. Nothing fails. The order count in the report goes up, revenue is overstated, and the error surfaces weeks later in a reconciliation. This guide, part of ",[14,15,17],"a",{"href":16},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002F","Merging and Joining Excel DataFrames",", shows how to make pandas refuse such merges, and which checks to put around every merge in a reporting pipeline.",[20,21,29,30,29,34,29,38,29,46,29,55,29,59,29,66,29,70,29,74,29,78,29,82,29,86,29,91,29,93,29,97,29,99,29,102,29,104,29,107,29,110,29,114,29,116,29,119,29,121,29,124,29,126,29,129,29,132,29,136,29,138,29,140,29,142,29,144,29,146,29,149,29,152,29,156,29,158,29,161,29,163,29,165,29,168,29,171,29,174,29,177,29,181,29,184,29,187,29,190,29,192,29,195,29,197,29,199,29,201,29,204,29,206,29,208,29,210,29,212,29,214,29,216,29,218,29,221,29,223,29,225,29,227,29,229,29,231,29,233,29,235,29,238,29,240,29,242,29,244,29,246,29,248,29,250,29,252,29,254,29,256,29,258,29,261,29,265,29,267,29,269,29,271,29,273,29,275,29,277,29,280,29,284,29,286,29,288,29,290,29,292,29,295,29,301,29,305,29,313,29,318,29,326],"svg",{"viewBox":22,"role":23,"ariaLabelledBy":24,"xmlns":27,"style":28},"40 0 680 248","img",[25,26],"vm-explode-t","vm-explode-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:680px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[31,32,33],"title",{"id":25},"One duplicate key multiplies the matching rows",[35,36,37],"desc",{"id":26},"The orders table has two orders for customer C2, worth 50 and 70. The customer table lists C2 twice, once as South and once as South-East. A left merge pairs each C2 order with both customer rows, so two orders become four rows and their 120 of revenue is counted as 240.",[39,40],"rect",{"x":41,"y":42,"width":43,"height":44,"fill":45},"40","0","680","248","#ffffff",[39,47],{"x":48,"y":49,"width":50,"height":51,"rx":42,"fill":52,"stroke":53,"style":54},"56.3","40.0","32.0","24.0","#e7ebef","var(--line,#cdd5e6)","stroke-width:1px",[39,56],{"x":57,"y":49,"width":58,"height":51,"rx":42,"fill":52,"stroke":53,"style":54},"88.3","57.4",[60,61,65],"text",{"x":62,"y":63,"style":64},"117.0","57.0","font-size:11px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","A",[39,67],{"x":68,"y":49,"width":69,"height":51,"rx":42,"fill":52,"stroke":53,"style":54},"145.7","50.9",[60,71,73],{"x":72,"y":63,"style":64},"171.2","B",[39,75],{"x":76,"y":49,"width":77,"height":51,"rx":42,"fill":52,"stroke":53,"style":54},"196.6","70.9",[60,79,81],{"x":80,"y":63,"style":64},"232.1","C",[39,83],{"x":48,"y":84,"width":50,"height":85,"rx":42,"fill":52,"stroke":53,"style":54},"64.0","28.0",[60,87,90],{"x":88,"y":89,"style":64},"72.3","83.0","1",[39,92],{"x":57,"y":84,"width":58,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,94,96],{"x":62,"y":89,"style":95},"font-size:12px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","order",[39,98],{"x":68,"y":84,"width":69,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,100,101],{"x":72,"y":89,"style":95},"cust",[39,103],{"x":76,"y":84,"width":77,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,105,106],{"x":80,"y":89,"style":95},"amount",[39,108],{"x":48,"y":109,"width":50,"height":85,"rx":42,"fill":52,"stroke":53,"style":54},"92.0",[60,111,113],{"x":88,"y":112,"style":64},"111.0","2",[39,115],{"x":57,"y":109,"width":58,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,117,113],{"x":62,"y":112,"style":118},"font-size:12px;font-weight:400;fill:var(--text,#172033);text-anchor:middle",[39,120],{"x":68,"y":109,"width":69,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,122,123],{"x":72,"y":112,"style":118},"C2",[39,125],{"x":76,"y":109,"width":77,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,127,128],{"x":80,"y":112,"style":118},"50",[39,130],{"x":48,"y":131,"width":50,"height":85,"rx":42,"fill":52,"stroke":53,"style":54},"120.0",[60,133,135],{"x":88,"y":134,"style":64},"139.0","3",[39,137],{"x":57,"y":131,"width":58,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,139,135],{"x":62,"y":134,"style":118},[39,141],{"x":68,"y":131,"width":69,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,143,123],{"x":72,"y":134,"style":118},[39,145],{"x":76,"y":131,"width":77,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,147,148],{"x":80,"y":134,"style":118},"70",[39,150],{"x":48,"y":151,"width":50,"height":85,"rx":42,"fill":52,"stroke":53,"style":54},"148.0",[60,153,155],{"x":88,"y":154,"style":64},"167.0","4",[39,157],{"x":57,"y":151,"width":58,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,159,160],{"x":62,"y":154,"style":118},"total",[39,162],{"x":68,"y":151,"width":69,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[39,164],{"x":76,"y":151,"width":77,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,166,167],{"x":80,"y":154,"style":118},"120",[39,169],{"x":170,"y":49,"width":50,"height":51,"rx":42,"fill":52,"stroke":53,"style":54},"450.6",[39,172],{"x":173,"y":49,"width":58,"height":51,"rx":42,"fill":52,"stroke":53,"style":54},"482.6",[60,175,65],{"x":176,"y":63,"style":64},"511.3",[39,178],{"x":179,"y":49,"width":180,"height":51,"rx":42,"fill":52,"stroke":53,"style":54},"540.0","92.8",[60,182,73],{"x":183,"y":63,"style":64},"586.5",[39,185],{"x":186,"y":49,"width":77,"height":51,"rx":42,"fill":52,"stroke":53,"style":54},"632.9",[60,188,81],{"x":189,"y":63,"style":64},"668.3",[39,191],{"x":170,"y":84,"width":50,"height":85,"rx":42,"fill":52,"stroke":53,"style":54},[60,193,90],{"x":194,"y":89,"style":64},"466.6",[39,196],{"x":173,"y":84,"width":58,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,198,96],{"x":176,"y":89,"style":95},[39,200],{"x":179,"y":84,"width":180,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,202,203],{"x":183,"y":89,"style":95},"region",[39,205],{"x":186,"y":84,"width":77,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,207,106],{"x":189,"y":89,"style":95},[39,209],{"x":170,"y":109,"width":50,"height":85,"rx":42,"fill":52,"stroke":53,"style":54},[60,211,113],{"x":194,"y":112,"style":64},[39,213],{"x":173,"y":109,"width":58,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,215,113],{"x":176,"y":112,"style":118},[39,217],{"x":179,"y":109,"width":180,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,219,220],{"x":183,"y":112,"style":118},"South",[39,222],{"x":186,"y":109,"width":77,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,224,128],{"x":189,"y":112,"style":118},[39,226],{"x":170,"y":131,"width":50,"height":85,"rx":42,"fill":52,"stroke":53,"style":54},[60,228,135],{"x":194,"y":134,"style":64},[39,230],{"x":173,"y":131,"width":58,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,232,113],{"x":176,"y":134,"style":118},[39,234],{"x":179,"y":131,"width":180,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,236,237],{"x":183,"y":134,"style":118},"South-East",[39,239],{"x":186,"y":131,"width":77,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,241,128],{"x":189,"y":134,"style":118},[39,243],{"x":170,"y":151,"width":50,"height":85,"rx":42,"fill":52,"stroke":53,"style":54},[60,245,155],{"x":194,"y":154,"style":64},[39,247],{"x":173,"y":151,"width":58,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,249,135],{"x":176,"y":154,"style":118},[39,251],{"x":179,"y":151,"width":180,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,253,220],{"x":183,"y":154,"style":118},[39,255],{"x":186,"y":151,"width":77,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,257,148],{"x":189,"y":154,"style":118},[39,259],{"x":170,"y":260,"width":50,"height":85,"rx":42,"fill":52,"stroke":53,"style":54},"176.0",[60,262,264],{"x":194,"y":263,"style":64},"195.0","5",[39,266],{"x":173,"y":260,"width":58,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,268,135],{"x":176,"y":263,"style":118},[39,270],{"x":179,"y":260,"width":180,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,272,237],{"x":183,"y":263,"style":118},[39,274],{"x":186,"y":260,"width":77,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,276,148],{"x":189,"y":263,"style":118},[39,278],{"x":170,"y":279,"width":50,"height":85,"rx":42,"fill":52,"stroke":53,"style":54},"204.0",[60,281,283],{"x":194,"y":282,"style":64},"223.0","6",[39,285],{"x":173,"y":279,"width":58,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,287,160],{"x":176,"y":282,"style":118},[39,289],{"x":179,"y":279,"width":180,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[39,291],{"x":186,"y":279,"width":77,"height":85,"rx":42,"fill":45,"stroke":53,"style":54},[60,293,294],{"x":189,"y":282,"style":118},"240",[60,296,300],{"x":297,"y":298,"style":299},"161.9","29.0","font-size:12.5px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:middle","before",[60,302,304],{"x":303,"y":298,"style":299},"577.2","after",[306,307],"line",{"x1":308,"y1":309,"x2":310,"y2":309,"stroke":311,"style":312},"277.5","108.0","436.6","var(--brand,#5b5cf0)","stroke-width:2px",[314,315],"polygon",{"points":316,"fill":317},"440.6,108.0 431.6,113.0 431.6,103.0","#5b5cf0",[39,319],{"x":320,"y":321,"width":322,"height":323,"rx":324,"fill":45,"stroke":53,"style":325},"301.8","75.0","114.5","22.0","11","stroke-width:1.2px",[60,327,331],{"x":328,"y":329,"style":330},"359.1","90.0","font-size:11.5px;font-weight:400;fill:var(--brand-strong,#4338ca);text-anchor:middle;font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace","merge on cust",[333,334,336],"h2",{"id":335},"prerequisites","Prerequisites",[338,339,344],"pre",{"className":340,"code":341,"language":342,"meta":343,"style":343},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl\n","bash","",[345,346,347],"code",{"__ignoreMap":343},[348,349,351,355,359,362],"span",{"class":306,"line":350},1,[348,352,354],{"class":353},"sMTad","pip",[348,356,358],{"class":357},"srMev"," install",[348,360,361],{"class":357}," pandas",[348,363,364],{"class":357}," openpyxl\n",[10,366,367],{},"An order sheet and a customer sheet with the three classic problems — a duplicated customer, an order for an unknown customer and an order with no customer ID:",[338,369,373],{"className":370,"code":371,"language":372,"meta":343,"style":343},"language-python shiki shiki-themes github-light github-dark-high-contrast","import pandas as pd\n\norders = pd.DataFrame({\"order\": [1, 2, 3, 4, 5],\n                       \"cust\": [\"C1\", \"C2\", \"C2\", \"C3\", None],\n                       \"amount\": [100, 50, 70, 20, 10]})\ncustomers = pd.DataFrame({\"cust\": [\"C1\", \"C2\", \"C2\", \"C4\", None],\n                          \"region\": [\"North\", \"South\", \"South-East\", \"West\", \"Unknown\"]})\nwith pd.ExcelWriter(\"sales.xlsx\") as writer:\n    orders.to_excel(writer, sheet_name=\"Orders\", index=False)\n    customers.to_excel(writer, sheet_name=\"Customers\", index=False)\norders = pd.read_excel(\"sales.xlsx\", sheet_name=\"Orders\")\ncustomers = pd.read_excel(\"sales.xlsx\", sheet_name=\"Customers\")\n","python",[345,374,375,391,398,439,471,503,539,572,592,620,643,665],{"__ignoreMap":343},[348,376,377,381,385,388],{"class":306,"line":350},[348,378,380],{"class":379},"s-kum","import",[348,382,384],{"class":383},"skGVy"," pandas ",[348,386,387],{"class":379},"as",[348,389,390],{"class":383}," pd\n",[348,392,394],{"class":306,"line":393},2,[348,395,397],{"emptyLinePlaceholder":396},true,"\n",[348,399,401,404,407,410,413,416,419,422,424,426,428,430,432,434,436],{"class":306,"line":400},3,[348,402,403],{"class":383},"orders ",[348,405,406],{"class":379},"=",[348,408,409],{"class":383}," pd.DataFrame({",[348,411,412],{"class":357},"\"order\"",[348,414,415],{"class":383},": [",[348,417,90],{"class":418},"sP0c6",[348,420,421],{"class":383},", ",[348,423,113],{"class":418},[348,425,421],{"class":383},[348,427,135],{"class":418},[348,429,421],{"class":383},[348,431,155],{"class":418},[348,433,421],{"class":383},[348,435,264],{"class":418},[348,437,438],{"class":383},"],\n",[348,440,442,445,447,450,452,455,457,459,461,464,466,469],{"class":306,"line":441},4,[348,443,444],{"class":357},"                       \"cust\"",[348,446,415],{"class":383},[348,448,449],{"class":357},"\"C1\"",[348,451,421],{"class":383},[348,453,454],{"class":357},"\"C2\"",[348,456,421],{"class":383},[348,458,454],{"class":357},[348,460,421],{"class":383},[348,462,463],{"class":357},"\"C3\"",[348,465,421],{"class":383},[348,467,468],{"class":418},"None",[348,470,438],{"class":383},[348,472,474,477,479,482,484,486,488,490,492,495,497,500],{"class":306,"line":473},5,[348,475,476],{"class":357},"                       \"amount\"",[348,478,415],{"class":383},[348,480,481],{"class":418},"100",[348,483,421],{"class":383},[348,485,128],{"class":418},[348,487,421],{"class":383},[348,489,148],{"class":418},[348,491,421],{"class":383},[348,493,494],{"class":418},"20",[348,496,421],{"class":383},[348,498,499],{"class":418},"10",[348,501,502],{"class":383},"]})\n",[348,504,506,509,511,513,516,518,520,522,524,526,528,530,533,535,537],{"class":306,"line":505},6,[348,507,508],{"class":383},"customers ",[348,510,406],{"class":379},[348,512,409],{"class":383},[348,514,515],{"class":357},"\"cust\"",[348,517,415],{"class":383},[348,519,449],{"class":357},[348,521,421],{"class":383},[348,523,454],{"class":357},[348,525,421],{"class":383},[348,527,454],{"class":357},[348,529,421],{"class":383},[348,531,532],{"class":357},"\"C4\"",[348,534,421],{"class":383},[348,536,468],{"class":418},[348,538,438],{"class":383},[348,540,542,545,547,550,552,555,557,560,562,565,567,570],{"class":306,"line":541},7,[348,543,544],{"class":357},"                          \"region\"",[348,546,415],{"class":383},[348,548,549],{"class":357},"\"North\"",[348,551,421],{"class":383},[348,553,554],{"class":357},"\"South\"",[348,556,421],{"class":383},[348,558,559],{"class":357},"\"South-East\"",[348,561,421],{"class":383},[348,563,564],{"class":357},"\"West\"",[348,566,421],{"class":383},[348,568,569],{"class":357},"\"Unknown\"",[348,571,502],{"class":383},[348,573,575,578,581,584,587,589],{"class":306,"line":574},8,[348,576,577],{"class":379},"with",[348,579,580],{"class":383}," pd.ExcelWriter(",[348,582,583],{"class":357},"\"sales.xlsx\"",[348,585,586],{"class":383},") ",[348,588,387],{"class":379},[348,590,591],{"class":383}," writer:\n",[348,593,595,598,602,604,607,609,612,614,617],{"class":306,"line":594},9,[348,596,597],{"class":383},"    orders.to_excel(writer, ",[348,599,601],{"class":600},"sa561","sheet_name",[348,603,406],{"class":379},[348,605,606],{"class":357},"\"Orders\"",[348,608,421],{"class":383},[348,610,611],{"class":600},"index",[348,613,406],{"class":379},[348,615,616],{"class":418},"False",[348,618,619],{"class":383},")\n",[348,621,623,626,628,630,633,635,637,639,641],{"class":306,"line":622},10,[348,624,625],{"class":383},"    customers.to_excel(writer, ",[348,627,601],{"class":600},[348,629,406],{"class":379},[348,631,632],{"class":357},"\"Customers\"",[348,634,421],{"class":383},[348,636,611],{"class":600},[348,638,406],{"class":379},[348,640,616],{"class":418},[348,642,619],{"class":383},[348,644,646,648,650,653,655,657,659,661,663],{"class":306,"line":645},11,[348,647,403],{"class":383},[348,649,406],{"class":379},[348,651,652],{"class":383}," pd.read_excel(",[348,654,583],{"class":357},[348,656,421],{"class":383},[348,658,601],{"class":600},[348,660,406],{"class":379},[348,662,606],{"class":357},[348,664,619],{"class":383},[348,666,668,670,672,674,676,678,680,682,684],{"class":306,"line":667},12,[348,669,508],{"class":383},[348,671,406],{"class":379},[348,673,652],{"class":383},[348,675,583],{"class":357},[348,677,421],{"class":383},[348,679,601],{"class":600},[348,681,406],{"class":379},[348,683,632],{"class":357},[348,685,619],{"class":383},[333,687,689],{"id":688},"step-1-see-the-damage-an-unchecked-merge-does","Step 1 — See the damage an unchecked merge does",[338,691,693],{"className":370,"code":692,"language":372,"meta":343,"style":343},"merged = orders.merge(customers, on=\"cust\", how=\"left\")\nprint(len(orders), len(merged))                        # 5 7\nprint(orders[\"amount\"].sum(), merged[\"amount\"].sum())  # 250 370\n",[345,694,695,724,747],{"__ignoreMap":343},[348,696,697,700,702,705,708,710,712,714,717,719,722],{"class":306,"line":350},[348,698,699],{"class":383},"merged ",[348,701,406],{"class":379},[348,703,704],{"class":383}," orders.merge(customers, ",[348,706,707],{"class":600},"on",[348,709,406],{"class":379},[348,711,515],{"class":357},[348,713,421],{"class":383},[348,715,716],{"class":600},"how",[348,718,406],{"class":379},[348,720,721],{"class":357},"\"left\"",[348,723,619],{"class":383},[348,725,726,729,732,735,738,740,743],{"class":306,"line":393},[348,727,728],{"class":418},"print",[348,730,731],{"class":383},"(",[348,733,734],{"class":418},"len",[348,736,737],{"class":383},"(orders), ",[348,739,734],{"class":418},[348,741,742],{"class":383},"(merged))                        ",[348,744,746],{"class":745},"s-wDw","# 5 7\n",[348,748,749,751,754,757,760,762,765],{"class":306,"line":400},[348,750,728],{"class":418},[348,752,753],{"class":383},"(orders[",[348,755,756],{"class":357},"\"amount\"",[348,758,759],{"class":383},"].sum(), merged[",[348,761,756],{"class":357},[348,763,764],{"class":383},"].sum())  ",[348,766,767],{"class":745},"# 250 370\n",[10,769,770],{},"Five orders became seven rows and 250 of revenue became 370. Orders 2 and 3 each appear twice, once per C2 row. The last order, which has no customer ID at all, picked up the region \"Unknown\" — because pandas, unlike SQL, treats missing keys as equal, and the customer sheet has a row with a blank ID too. Neither problem raises an error or a warning.",[333,772,774],{"id":773},"step-2-declare-the-relationship-with-validate","Step 2 — Declare the relationship with validate",[10,776,777],{},"Every lookup merge has an expected shape: many orders to one customer. Say so, and pandas checks it:",[338,779,781],{"className":370,"code":780,"language":372,"meta":343,"style":343},"try:\n    orders.merge(customers, on=\"cust\", how=\"left\", validate=\"many_to_one\")\nexcept pd.errors.MergeError as exc:\n    print(exc)\n# Merge keys are not unique in right dataset; not a many-to-one merge\n",[345,782,783,791,822,835,843],{"__ignoreMap":343},[348,784,785,788],{"class":306,"line":350},[348,786,787],{"class":379},"try",[348,789,790],{"class":383},":\n",[348,792,793,796,798,800,802,804,806,808,810,812,815,817,820],{"class":306,"line":393},[348,794,795],{"class":383},"    orders.merge(customers, ",[348,797,707],{"class":600},[348,799,406],{"class":379},[348,801,515],{"class":357},[348,803,421],{"class":383},[348,805,716],{"class":600},[348,807,406],{"class":379},[348,809,721],{"class":357},[348,811,421],{"class":383},[348,813,814],{"class":600},"validate",[348,816,406],{"class":379},[348,818,819],{"class":357},"\"many_to_one\"",[348,821,619],{"class":383},[348,823,824,827,830,832],{"class":306,"line":400},[348,825,826],{"class":379},"except",[348,828,829],{"class":383}," pd.errors.MergeError ",[348,831,387],{"class":379},[348,833,834],{"class":383}," exc:\n",[348,836,837,840],{"class":306,"line":441},[348,838,839],{"class":418},"    print",[348,841,842],{"class":383},"(exc)\n",[348,844,845],{"class":306,"line":473},[348,846,847],{"class":745},"# Merge keys are not unique in right dataset; not a many-to-one merge\n",[10,849,850,851,854,855,858,859,862,863,866],{},"The four options describe which side must have unique keys: ",[345,852,853],{},"one_to_one"," checks both, ",[345,856,857],{},"one_to_many"," the left, ",[345,860,861],{},"many_to_one"," the right and ",[345,864,865],{},"many_to_many"," neither. The check costs one pass over the key columns and turns a silent inflation into an immediate, explained failure. Make it a habit on every merge whose shape you know — which is nearly all of them in reporting code.",[10,868,869,870,872,873,876,877,879],{},"Know what ",[345,871,814],{}," does not check. It says nothing about left rows that find no partner, so an order for a customer missing from the lookup passes silently with empty columns. It does not stop blank keys from matching each other, as long as there is only one blank on the right. And it cannot tell that ",[345,874,875],{},"\"C2 \""," with a trailing space and ",[345,878,454],{}," are meant to be the same customer; to the check they are two unique keys. The remaining steps cover those gaps.",[20,881,29,887,29,890,29,893,29,897,29,902,29,907,29,913,29,916,29,920,29,924,29,928,29,930,29,941,29,943,29,948,29,950,29,953,29,955,29,960,29,962,29,970,29,973,29,977,29,979,29,984,29,986,29,991,29,994,29,998,29,1000,29,1005,29,1007],{"viewBox":882,"role":23,"ariaLabelledBy":883,"xmlns":27,"style":886},"4 0 752 208",[884,885],"vm-options-t","vm-options-d","width:100%;max-width:752px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[31,888,889],{"id":884},"What each validate option requires",[35,891,892],{"id":885},"one_to_one requires unique keys on both sides. one_to_many requires unique keys on the left. many_to_one requires unique keys on the right, the usual shape when looking up attributes. many_to_many requires nothing and is the default.",[39,894],{"x":155,"y":42,"width":895,"height":896,"fill":45},"752","208",[39,898],{"x":899,"y":900,"width":901,"height":49,"rx":42,"fill":45,"stroke":53,"style":54},"20.0","16.0","183.2",[39,903],{"x":904,"y":900,"width":905,"height":49,"rx":42,"fill":906,"stroke":311,"style":325},"203.2","268.4","#f0f4ff",[60,908,912],{"x":909,"y":910,"style":911},"337.4","41.0","font-size:13px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Left keys unique",[39,914],{"x":915,"y":900,"width":905,"height":49,"rx":42,"fill":906,"stroke":311,"style":325},"471.6",[60,917,919],{"x":918,"y":910,"style":911},"605.8","Right keys unique",[39,921],{"x":899,"y":922,"width":901,"height":923,"rx":42,"fill":52,"stroke":53,"style":54},"56.0","34.0",[60,925,853],{"x":50,"y":926,"style":927},"78.0","font-size:12.5px;font-weight:700;fill:var(--text,#172033);text-anchor:start",[39,929],{"x":904,"y":922,"width":905,"height":923,"rx":42,"fill":45,"stroke":53,"style":54},[931,932,934],"g",{"transform":933},"translate(328.4,64.0) scale(0.750)",[935,936],"path",{"d":937,"fill":938,"stroke":939,"style":940},"M5 12.5l4.5 4.5L19 7","none","var(--teal,#0f9488)","stroke-width:2.40px;stroke-linecap:round;stroke-linejoin:round",[39,942],{"x":915,"y":922,"width":905,"height":923,"rx":42,"fill":45,"stroke":53,"style":54},[931,944,946],{"transform":945},"translate(596.8,64.0) scale(0.750)",[935,947],{"d":937,"fill":938,"stroke":939,"style":940},[39,949],{"x":899,"y":329,"width":901,"height":923,"rx":42,"fill":52,"stroke":53,"style":54},[60,951,857],{"x":50,"y":952,"style":927},"112.0",[39,954],{"x":904,"y":329,"width":905,"height":923,"rx":42,"fill":45,"stroke":53,"style":54},[931,956,958],{"transform":957},"translate(328.4,98.0) scale(0.750)",[935,959],{"d":937,"fill":938,"stroke":939,"style":940},[39,961],{"x":915,"y":329,"width":905,"height":923,"rx":42,"fill":45,"stroke":53,"style":54},[931,963,965],{"transform":964},"translate(597.8,99.0) scale(0.667)",[935,966],{"d":967,"fill":938,"stroke":968,"style":969},"M6 6l12 12 M18 6L6 18","var(--accent,#f43f8f)","stroke-width:2.70px;stroke-linecap:round;stroke-linejoin:round",[39,971],{"x":899,"y":972,"width":901,"height":923,"rx":42,"fill":52,"stroke":53,"style":54},"124.0",[60,974,976],{"x":50,"y":975,"style":927},"146.0","many_to_one (lookups)",[39,978],{"x":904,"y":972,"width":905,"height":923,"rx":42,"fill":45,"stroke":53,"style":54},[931,980,982],{"transform":981},"translate(329.4,133.0) scale(0.667)",[935,983],{"d":967,"fill":938,"stroke":968,"style":969},[39,985],{"x":915,"y":972,"width":905,"height":923,"rx":42,"fill":45,"stroke":53,"style":54},[931,987,989],{"transform":988},"translate(596.8,132.0) scale(0.750)",[935,990],{"d":937,"fill":938,"stroke":939,"style":940},[39,992],{"x":899,"y":993,"width":901,"height":923,"rx":42,"fill":52,"stroke":53,"style":54},"158.0",[60,995,997],{"x":50,"y":996,"style":927},"180.0","many_to_many (default)",[39,999],{"x":904,"y":993,"width":905,"height":923,"rx":42,"fill":45,"stroke":53,"style":54},[931,1001,1003],{"transform":1002},"translate(329.4,167.0) scale(0.667)",[935,1004],{"d":967,"fill":938,"stroke":968,"style":969},[39,1006],{"x":915,"y":993,"width":905,"height":923,"rx":42,"fill":45,"stroke":53,"style":54},[931,1008,1010],{"transform":1009},"translate(597.8,167.0) scale(0.667)",[935,1011],{"d":967,"fill":938,"stroke":968,"style":969},[333,1013,1015],{"id":1014},"step-3-find-and-resolve-the-duplicate-keys","Step 3 — Find and resolve the duplicate keys",[10,1017,1018],{},"The error tells you the right side has duplicates; list them so someone can decide which row is correct:",[338,1020,1022],{"className":370,"code":1021,"language":372,"meta":343,"style":343},"dupes = customers[customers.duplicated(\"cust\", keep=False) & customers[\"cust\"].notna()]\nprint(dupes)\n#   cust      region\n# 1   C2       South\n# 2   C2  South-East\n",[345,1023,1024,1058,1065,1070,1075],{"__ignoreMap":343},[348,1025,1026,1029,1031,1034,1036,1038,1041,1043,1045,1047,1050,1053,1055],{"class":306,"line":350},[348,1027,1028],{"class":383},"dupes ",[348,1030,406],{"class":379},[348,1032,1033],{"class":383}," customers[customers.duplicated(",[348,1035,515],{"class":357},[348,1037,421],{"class":383},[348,1039,1040],{"class":600},"keep",[348,1042,406],{"class":379},[348,1044,616],{"class":418},[348,1046,586],{"class":383},[348,1048,1049],{"class":379},"&",[348,1051,1052],{"class":383}," customers[",[348,1054,515],{"class":357},[348,1056,1057],{"class":383},"].notna()]\n",[348,1059,1060,1062],{"class":306,"line":393},[348,1061,728],{"class":418},[348,1063,1064],{"class":383},"(dupes)\n",[348,1066,1067],{"class":306,"line":400},[348,1068,1069],{"class":745},"#   cust      region\n",[348,1071,1072],{"class":306,"line":441},[348,1073,1074],{"class":745},"# 1   C2       South\n",[348,1076,1077],{"class":306,"line":473},[348,1078,1079],{"class":745},"# 2   C2  South-East\n",[10,1081,1082,1085,1086,1089,1090,1094],{},[345,1083,1084],{},"keep=False"," marks every copy, not just the second, so the report shows the conflict in full. Resolving it is a business decision — the most recent row, the row from the authoritative system, or a correction in the source workbook. Only after that should the code deduplicate, and it should do so explicitly, for example with ",[345,1087,1088],{},"drop_duplicates(\"cust\", keep=\"last\")"," when newer rows are appended below. Near-duplicates that differ by spaces or case need normalising first; see ",[14,1091,1093],{"href":1092},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fpandas-drop-duplicates-from-excel-column\u002F","Pandas: Drop Duplicates From an Excel Column",".",[333,1096,1098],{"id":1097},"step-4-keep-missing-keys-out-of-the-join","Step 4 — Keep missing keys out of the join",[10,1100,1101],{},"A blank key is not a key. Drop blank keys from the lookup table and leave the left side's blanks unmatched, so they show up as missing rather than joining each other:",[338,1103,1105],{"className":370,"code":1104,"language":372,"meta":343,"style":343},"lookup = customers.dropna(subset=[\"cust\"]).drop_duplicates(\"cust\", keep=\"last\")\nmerged = orders.merge(lookup, on=\"cust\", how=\"left\", validate=\"many_to_one\", indicator=True)\nprint(merged[[\"order\", \"cust\", \"region\", \"_merge\"]].values.tolist())\n",[345,1106,1107,1143,1186],{"__ignoreMap":343},[348,1108,1109,1112,1114,1117,1120,1122,1125,1127,1130,1132,1134,1136,1138,1141],{"class":306,"line":350},[348,1110,1111],{"class":383},"lookup ",[348,1113,406],{"class":379},[348,1115,1116],{"class":383}," customers.dropna(",[348,1118,1119],{"class":600},"subset",[348,1121,406],{"class":379},[348,1123,1124],{"class":383},"[",[348,1126,515],{"class":357},[348,1128,1129],{"class":383},"]).drop_duplicates(",[348,1131,515],{"class":357},[348,1133,421],{"class":383},[348,1135,1040],{"class":600},[348,1137,406],{"class":379},[348,1139,1140],{"class":357},"\"last\"",[348,1142,619],{"class":383},[348,1144,1145,1147,1149,1152,1154,1156,1158,1160,1162,1164,1166,1168,1170,1172,1174,1176,1179,1181,1184],{"class":306,"line":393},[348,1146,699],{"class":383},[348,1148,406],{"class":379},[348,1150,1151],{"class":383}," orders.merge(lookup, ",[348,1153,707],{"class":600},[348,1155,406],{"class":379},[348,1157,515],{"class":357},[348,1159,421],{"class":383},[348,1161,716],{"class":600},[348,1163,406],{"class":379},[348,1165,721],{"class":357},[348,1167,421],{"class":383},[348,1169,814],{"class":600},[348,1171,406],{"class":379},[348,1173,819],{"class":357},[348,1175,421],{"class":383},[348,1177,1178],{"class":600},"indicator",[348,1180,406],{"class":379},[348,1182,1183],{"class":418},"True",[348,1185,619],{"class":383},[348,1187,1188,1190,1193,1195,1197,1199,1201,1204,1206,1209],{"class":306,"line":400},[348,1189,728],{"class":418},[348,1191,1192],{"class":383},"(merged[[",[348,1194,412],{"class":357},[348,1196,421],{"class":383},[348,1198,515],{"class":357},[348,1200,421],{"class":383},[348,1202,1203],{"class":357},"\"region\"",[348,1205,421],{"class":383},[348,1207,1208],{"class":357},"\"_merge\"",[348,1210,1211],{"class":383},"]].values.tolist())\n",[10,1213,1214,1215,1218],{},"Order 5 now has no region, which is the truth: we do not know its customer. Blank keys often hide as empty strings or spaces rather than ",[345,1216,1217],{},"NaN","; convert those to missing values before this step.",[333,1220,1222],{"id":1221},"step-5-count-what-matched-with-indicator","Step 5 — Count what matched with indicator",[10,1224,1225,1228,1229,1232],{},[345,1226,1227],{},"indicator=True"," adds a ",[345,1230,1231],{},"_merge"," column saying whether each row found a partner. Count it and act on the result:",[338,1234,1236],{"className":370,"code":1235,"language":372,"meta":343,"style":343},"counts = merged[\"_merge\"].value_counts().to_dict()\nprint(counts)            # {'both': 3, 'left_only': 2, 'right_only': 0}\nunmatched = merged.loc[merged[\"_merge\"] == \"left_only\", [\"order\", \"cust\"]]\nprint(unmatched.values.tolist())      # [[4, 'C3'], [5, nan]]\n",[345,1237,1238,1253,1263,1296],{"__ignoreMap":343},[348,1239,1240,1243,1245,1248,1250],{"class":306,"line":350},[348,1241,1242],{"class":383},"counts ",[348,1244,406],{"class":379},[348,1246,1247],{"class":383}," merged[",[348,1249,1208],{"class":357},[348,1251,1252],{"class":383},"].value_counts().to_dict()\n",[348,1254,1255,1257,1260],{"class":306,"line":393},[348,1256,728],{"class":418},[348,1258,1259],{"class":383},"(counts)            ",[348,1261,1262],{"class":745},"# {'both': 3, 'left_only': 2, 'right_only': 0}\n",[348,1264,1265,1268,1270,1273,1275,1278,1281,1284,1287,1289,1291,1293],{"class":306,"line":400},[348,1266,1267],{"class":383},"unmatched ",[348,1269,406],{"class":379},[348,1271,1272],{"class":383}," merged.loc[merged[",[348,1274,1208],{"class":357},[348,1276,1277],{"class":383},"] ",[348,1279,1280],{"class":379},"==",[348,1282,1283],{"class":357}," \"left_only\"",[348,1285,1286],{"class":383},", [",[348,1288,412],{"class":357},[348,1290,421],{"class":383},[348,1292,515],{"class":357},[348,1294,1295],{"class":383},"]]\n",[348,1297,1298,1300,1303],{"class":306,"line":441},[348,1299,728],{"class":418},[348,1301,1302],{"class":383},"(unmatched.values.tolist())      ",[348,1304,1305],{"class":745},"# [[4, 'C3'], [5, nan]]\n",[10,1307,1308,1309,1094],{},"C3 is missing from the customer sheet and order 5 has no customer. Whether that should stop the run depends on the report: a revenue total can proceed with an \"Unassigned\" region, while an invoicing run should not. Either way, write the unmatched rows somewhere a person will see them. Finding rows missing from one file or the other is covered in ",[14,1310,1312],{"href":1311},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Ffind-rows-in-one-excel-file-missing-from-another\u002F","Find Rows in One Excel File Missing from Another",[20,1314,29,1319,29,1322,29,1325,29,1328,29,1333,29,1341,29,1347,29,1351,29,1356,29,1361,29,1364,29,1368,29,1375,29,1380,29,1383,29,1386,29,1390,29,1393,29,1397,29,1404,29,1409,29,1413,29,1415,29,1419,29,1422,29,1425,29,1430,29,1434,29,1437,29,1440,29,1444,29,1447,29,1451,29,1458,29,1463,29,1466],{"viewBox":1315,"role":23,"ariaLabelledBy":1316,"xmlns":27,"style":886},"4 0 752 142",[1317,1318],"vm-guard-t","vm-guard-d",[31,1320,1321],{"id":1317},"The checks to wrap around a lookup merge",[35,1323,1324],{"id":1318},"Drop blank keys from the lookup table, resolve duplicate keys explicitly, merge with validate many_to_one and indicator on, then assert that the row count and the amount total are unchanged and report the unmatched rows.",[39,1326],{"x":155,"y":42,"width":895,"height":1327,"fill":45},"142",[39,1329],{"x":899,"y":900,"width":1330,"height":1331,"rx":1332,"fill":906,"stroke":311,"style":312},"115.2","110.5","12",[931,1334,1336],{"transform":1335},"translate(65.6,29.0) scale(1.000)",[935,1337],{"d":1338,"fill":938,"stroke":1339,"style":1340},"M3 5h18l-7 8v6l-4 2v-8z","var(--brand-strong,#4338ca)","stroke-width:1.80px;stroke-linecap:round;stroke-linejoin:round",[60,1342,1346],{"x":1343,"y":1344,"style":1345},"77.6","75.5","font-size:13.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Drop blank",[60,1348,1350],{"x":1343,"y":1349,"style":1345},"93.0","keys",[60,1352,1355],{"x":1343,"y":1353,"style":1354},"112.5","font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","lookup side",[306,1357],{"x1":1358,"y1":1359,"x2":1360,"y2":1359,"stroke":311,"style":312},"140.2","71.2","162.2",[314,1362],{"points":1363,"fill":317},"166.2,71.2 157.2,76.2 157.2,66.2",[39,1365],{"x":72,"y":900,"width":1330,"height":1331,"rx":1332,"fill":1366,"stroke":1367,"style":312},"#fdefd8","var(--gold,#b4740a)",[931,1369,1371],{"transform":1370},"translate(216.8,29.0) scale(1.000)",[935,1372],{"d":1373,"fill":938,"stroke":1374,"style":1340},"M12 3.5l9.5 17h-19z M12 10v5 M12 17.8v.4","var(--gold-ink,#7a4e06)",[60,1376,1379],{"x":1377,"y":1344,"style":1378},"228.8","font-size:13.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","Resolve",[60,1381,1382],{"x":1377,"y":1349,"style":1378},"duplicates",[60,1384,1385],{"x":1377,"y":1353,"style":1354},"explicit keep",[306,1387],{"x1":1388,"y1":1359,"x2":1389,"y2":1359,"stroke":311,"style":312},"291.4","313.4",[314,1391],{"points":1392,"fill":317},"317.4,71.2 308.4,76.2 308.4,66.2",[39,1394],{"x":1395,"y":900,"width":1330,"height":1331,"rx":1332,"fill":1396,"stroke":939,"style":312},"322.4","#d9f4f1",[931,1398,1400],{"transform":1399},"translate(368.0,29.0) scale(1.000)",[935,1401],{"d":1402,"fill":938,"stroke":1403,"style":1340},"M6 4v5a5 5 0 0 0 5 5h7 M6 20v-5 M15 11l3 3-3 3","var(--teal-ink,#0b6157)",[60,1405,1408],{"x":1406,"y":1344,"style":1407},"380.0","font-size:13.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","Merge",[60,1410,1412],{"x":1406,"y":1411,"style":1354},"95.0","validate +",[60,1414,1178],{"x":1406,"y":1353,"style":1354},[306,1416],{"x1":1417,"y1":1359,"x2":1418,"y2":1359,"stroke":311,"style":312},"442.6","464.6",[314,1420],{"points":1421,"fill":317},"468.6,71.2 459.6,76.2 459.6,66.2",[39,1423],{"x":1424,"y":900,"width":1330,"height":1331,"rx":1332,"fill":906,"stroke":311,"style":312},"473.6",[931,1426,1428],{"transform":1427},"translate(519.2,29.0) scale(1.000)",[935,1429],{"d":937,"fill":938,"stroke":1339,"style":1340},[60,1431,1433],{"x":1432,"y":1344,"style":1345},"531.2","Assert",[60,1435,1436],{"x":1432,"y":1349,"style":1345},"counts",[60,1438,1439],{"x":1432,"y":1353,"style":1354},"rows and totals",[306,1441],{"x1":1442,"y1":1359,"x2":1443,"y2":1359,"stroke":311,"style":312},"593.8","615.8",[314,1445],{"points":1446,"fill":317},"619.8,71.2 610.8,76.2 610.8,66.2",[39,1448],{"x":1449,"y":900,"width":1330,"height":1331,"rx":1332,"fill":1450,"stroke":968,"style":312},"624.8","#fee8f2",[931,1452,1454],{"transform":1453},"translate(670.4,29.0) scale(1.000)",[935,1455],{"d":1456,"fill":938,"stroke":1457,"style":1340},"M11 4a7 7 0 1 0 .01 0z M16 16l5 5","var(--accent-ink,#be185d)",[60,1459,1462],{"x":1460,"y":1344,"style":1461},"682.4","font-size:13.5px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","Report",[60,1464,1465],{"x":1460,"y":1349,"style":1461},"misses",[60,1467,1468],{"x":1460,"y":1353,"style":1354},"left_only rows",[333,1470,1472],{"id":1471},"step-6-assert-row-counts-and-totals","Step 6 — Assert row counts and totals",[10,1474,1475,1477],{},[345,1476,814],{}," protects against duplicate keys; it cannot know what the result should add up to. For a left lookup merge the rule is simple: the row count and every numeric total of the left table must be unchanged. Wrap that in a function you use everywhere:",[338,1479,1481],{"className":370,"code":1480,"language":372,"meta":343,"style":343},"def lookup_merge(left, right, on, **kwargs):\n    right = right.dropna(subset=[on] if isinstance(on, str) else on)\n    result = left.merge(right, on=on, how=\"left\", validate=\"many_to_one\", indicator=True, **kwargs)\n    assert len(result) == len(left), f\"row count changed: {len(left)} -> {len(result)}\"\n    for col in left.select_dtypes(\"number\").columns:\n        assert result[col].sum() == left[col].sum(), f\"total of {col} changed\"\n    return result\n\nreport = lookup_merge(orders, customers.drop_duplicates(\"cust\", keep=\"last\"), on=\"cust\")\nprint(len(report), report[\"amount\"].sum())       # 5 250\n",[345,1482,1483,1501,1538,1584,1635,1655,1683,1691,1695,1726],{"__ignoreMap":343},[348,1484,1485,1488,1492,1495,1498],{"class":306,"line":350},[348,1486,1487],{"class":379},"def",[348,1489,1491],{"class":1490},"s_Opv"," lookup_merge",[348,1493,1494],{"class":383},"(left, right, on, ",[348,1496,1497],{"class":379},"**",[348,1499,1500],{"class":383},"kwargs):\n",[348,1502,1503,1506,1508,1511,1513,1515,1518,1521,1524,1527,1530,1532,1535],{"class":306,"line":393},[348,1504,1505],{"class":383},"    right ",[348,1507,406],{"class":379},[348,1509,1510],{"class":383}," right.dropna(",[348,1512,1119],{"class":600},[348,1514,406],{"class":379},[348,1516,1517],{"class":383},"[on] ",[348,1519,1520],{"class":379},"if",[348,1522,1523],{"class":418}," isinstance",[348,1525,1526],{"class":383},"(on, ",[348,1528,1529],{"class":418},"str",[348,1531,586],{"class":383},[348,1533,1534],{"class":379},"else",[348,1536,1537],{"class":383}," on)\n",[348,1539,1540,1543,1545,1548,1550,1552,1555,1557,1559,1561,1563,1565,1567,1569,1571,1573,1575,1577,1579,1581],{"class":306,"line":400},[348,1541,1542],{"class":383},"    result ",[348,1544,406],{"class":379},[348,1546,1547],{"class":383}," left.merge(right, ",[348,1549,707],{"class":600},[348,1551,406],{"class":379},[348,1553,1554],{"class":383},"on, ",[348,1556,716],{"class":600},[348,1558,406],{"class":379},[348,1560,721],{"class":357},[348,1562,421],{"class":383},[348,1564,814],{"class":600},[348,1566,406],{"class":379},[348,1568,819],{"class":357},[348,1570,421],{"class":383},[348,1572,1178],{"class":600},[348,1574,406],{"class":379},[348,1576,1183],{"class":418},[348,1578,421],{"class":383},[348,1580,1497],{"class":379},[348,1582,1583],{"class":383},"kwargs)\n",[348,1585,1586,1589,1592,1595,1597,1599,1602,1605,1608,1612,1614,1617,1620,1623,1625,1627,1630,1632],{"class":306,"line":441},[348,1587,1588],{"class":379},"    assert",[348,1590,1591],{"class":418}," len",[348,1593,1594],{"class":383},"(result) ",[348,1596,1280],{"class":379},[348,1598,1591],{"class":418},[348,1600,1601],{"class":383},"(left), ",[348,1603,1604],{"class":379},"f",[348,1606,1607],{"class":357},"\"row count changed: ",[348,1609,1611],{"class":1610},"sSjpA","{",[348,1613,734],{"class":418},[348,1615,1616],{"class":383},"(left)",[348,1618,1619],{"class":1610},"}",[348,1621,1622],{"class":357}," -> ",[348,1624,1611],{"class":1610},[348,1626,734],{"class":418},[348,1628,1629],{"class":383},"(result)",[348,1631,1619],{"class":1610},[348,1633,1634],{"class":357},"\"\n",[348,1636,1637,1640,1643,1646,1649,1652],{"class":306,"line":473},[348,1638,1639],{"class":379},"    for",[348,1641,1642],{"class":383}," col ",[348,1644,1645],{"class":379},"in",[348,1647,1648],{"class":383}," left.select_dtypes(",[348,1650,1651],{"class":357},"\"number\"",[348,1653,1654],{"class":383},").columns:\n",[348,1656,1657,1660,1663,1665,1668,1670,1673,1675,1678,1680],{"class":306,"line":505},[348,1658,1659],{"class":379},"        assert",[348,1661,1662],{"class":383}," result[col].sum() ",[348,1664,1280],{"class":379},[348,1666,1667],{"class":383}," left[col].sum(), ",[348,1669,1604],{"class":379},[348,1671,1672],{"class":357},"\"total of ",[348,1674,1611],{"class":1610},[348,1676,1677],{"class":383},"col",[348,1679,1619],{"class":1610},[348,1681,1682],{"class":357}," changed\"\n",[348,1684,1685,1688],{"class":306,"line":541},[348,1686,1687],{"class":379},"    return",[348,1689,1690],{"class":383}," result\n",[348,1692,1693],{"class":306,"line":574},[348,1694,397],{"emptyLinePlaceholder":396},[348,1696,1697,1700,1702,1705,1707,1709,1711,1713,1715,1718,1720,1722,1724],{"class":306,"line":594},[348,1698,1699],{"class":383},"report ",[348,1701,406],{"class":379},[348,1703,1704],{"class":383}," lookup_merge(orders, customers.drop_duplicates(",[348,1706,515],{"class":357},[348,1708,421],{"class":383},[348,1710,1040],{"class":600},[348,1712,406],{"class":379},[348,1714,1140],{"class":357},[348,1716,1717],{"class":383},"), ",[348,1719,707],{"class":600},[348,1721,406],{"class":379},[348,1723,515],{"class":357},[348,1725,619],{"class":383},[348,1727,1728,1730,1732,1734,1737,1739,1742],{"class":306,"line":622},[348,1729,728],{"class":418},[348,1731,731],{"class":383},[348,1733,734],{"class":418},[348,1735,1736],{"class":383},"(report), report[",[348,1738,756],{"class":357},[348,1740,1741],{"class":383},"].sum())       ",[348,1743,1744],{"class":745},"# 5 250\n",[10,1746,1747,1748,1750],{},"With ",[345,1749,814],{}," in place the assertions should never fire for a lookup merge, which is the point: they document the expectation and guard against later edits, such as someone switching to an inner join or adding a second key. For merges that are supposed to change the row count — an inner join that filters, a one-to-many expansion — assert the expected count instead.",[10,1752,1753],{},"When a check fails in a scheduled job, the message is often the only thing anyone sees. Make it specific: name the lookup file, the key column, the number of duplicate keys and the first few of them. \"customers.xlsx has 1 duplicate cust key (C2)\" gets fixed in minutes by whoever maintains that workbook; \"AssertionError\" gets forwarded to the developer. The same goes for unmatched rows — a short sheet listing them, attached to the failure alert, turns a broken run into a to-do list.",[333,1755,1757],{"id":1756},"common-pitfalls","Common pitfalls",[1759,1760,1761,1779,1792,1801],"ul",{},[1762,1763,1764,1768,1769,1772,1773,1775,1776,1778],"li",{},[1765,1766,1767],"strong",{},"Keys that differ only by type."," ",[345,1770,1771],{},"\"100\""," in one sheet and ",[345,1774,481],{}," in the other never match, and ",[345,1777,814],{}," will not warn you. Convert both sides to the same type first.",[1762,1780,1781,1784,1785,1788,1789,1791],{},[1765,1782,1783],{},"Validating after deduplicating blindly."," A ",[345,1786,1787],{},"drop_duplicates"," added just to make ",[345,1790,814],{}," pass hides the conflict. Report duplicates before choosing a row.",[1762,1793,1794,1768,1797,1800],{},[1765,1795,1796],{},"Outer joins in reports.",[345,1798,1799],{},"how=\"outer\""," adds right-only rows with empty amounts, which changes counts in ways that look like data. Use left joins for lookups and report right-only keys separately.",[1762,1802,1803,1806],{},[1765,1804,1805],{},"Checking only the first merge."," Pipelines often merge several lookups in a row; each one can multiply rows. Guard each merge.",[333,1808,1810],{"id":1809},"conclusion","Conclusion",[10,1812,1813,1814,1816,1817,1819],{},"Duplicate keys make pandas merges multiply rows without any warning, and missing keys match each other. Declare every merge's shape with ",[345,1815,814],{},", list and resolve duplicate keys explicitly, drop blank keys from lookup tables, use ",[345,1818,1227],{}," to count and report unmatched rows, and assert that row counts and totals survive every lookup merge.",[333,1821,1823],{"id":1822},"frequently-asked-questions","Frequently asked questions",[10,1825,1826,1829],{},[1765,1827,1828],{},"Why does my merge have more rows than the original?","\nAt least one key appears more than once in the table you are merging in. Each matching left row is repeated once per duplicate, so row counts and sums grow. Pass validate=\"many_to_one\" to make pandas raise an error instead.",[10,1831,1832,1835],{},[1765,1833,1834],{},"What does validate=\"many_to_one\" check?","\nThat the keys in the right-hand table are unique, so each left row can match at most one right row. one_to_one checks both sides, one_to_many checks the left side, and many_to_many checks nothing.",[10,1837,1838,1841],{},[1765,1839,1840],{},"How do I see which rows did not match?","\nPass indicator=True. The merged frame gains a _merge column with the values both, left_only and right_only, which you can count or filter.",[10,1843,1844,1847],{},[1765,1845,1846],{},"Do missing keys match each other in a pandas merge?","\nYes. Unlike SQL, pandas treats NaN keys as equal, so a left row with no customer ID matches every right row with no ID. Drop or fill missing keys before merging.",[333,1849,1851],{"id":1850},"related","Related",[1759,1853,1854,1860,1867,1874],{},[1762,1855,1856,1857,1859],{},"Up one level: ",[14,1858,17],{"href":16}," — combining data from several workbooks.",[1762,1861,1862,1866],{},[14,1863,1865],{"href":1864},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fmerge-two-excel-files-on-common-column-python\u002F","Merge Two Excel Files on a Common Column in Python"," — the basic merge.",[1762,1868,1869,1873],{},[14,1870,1872],{"href":1871},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fvlookup-equivalent-in-pandas-for-excel-files\u002F","The VLOOKUP Equivalent in pandas for Excel Files"," — lookups translated from Excel.",[1762,1875,1876,1880],{},[14,1877,1879],{"href":1878},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Freconcile-excel-totals-with-control-totals\u002F","Reconcile Excel Totals with Control Totals in Python"," — checking results against known figures.",[1882,1883,1884],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}",{"title":343,"searchDepth":393,"depth":393,"links":1886},[1887,1888,1889,1890,1891,1892,1893,1894,1895,1896,1897],{"id":335,"depth":393,"text":336},{"id":688,"depth":393,"text":689},{"id":773,"depth":393,"text":774},{"id":1014,"depth":393,"text":1015},{"id":1097,"depth":393,"text":1098},{"id":1221,"depth":393,"text":1222},{"id":1471,"depth":393,"text":1472},{"id":1756,"depth":393,"text":1757},{"id":1809,"depth":393,"text":1810},{"id":1822,"depth":393,"text":1823},{"id":1850,"depth":393,"text":1851},"2026-09-23","A duplicate key in a lookup sheet makes a pandas merge silently add rows and inflate totals. Use merge's validate argument, indicator column and before-and-after checks to catch duplicate keys, unmatched rows and missing keys that match each other before a report goes out.","md",[1902,1904,1906,1908],{"q":1828,"a":1903},"At least one key appears more than once in the table you are merging in. Each matching left row is repeated once per duplicate, so row counts and sums grow. Pass validate=\"many_to_one\" to make pandas raise an error instead.",{"q":1834,"a":1905},"That the keys in the right-hand table are unique, so each left row can match at most one right row. one_to_one checks both sides, one_to_many checks the left side, and many_to_many checks nothing.",{"q":1840,"a":1907},"Pass indicator=True. The merged frame gains a _merge column with the values both, left_only and right_only, which you can count or filter.",{"q":1846,"a":1909},"Yes. Unlike SQL, pandas treats NaN keys as equal, so a left row with no customer ID matches every right row with no ID. Drop or fill missing keys before merging.","validate a pandas merge to avoid duplicate rows",null,{"breadcrumb":1913},[1914,1917,1920],{"name":1915,"item":1916},"Home","\u002F",{"name":1918,"item":1919},"Advanced Data Transformation and Cleaning for Excel Automation","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":1921,"item":16},"Merging and Joining Excel DataFrames with Pandas","\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fvalidate-a-merge-to-avoid-duplicate-rows",{"title":1924,"description":1925},"Validate a pandas Merge to Avoid Duplicate Rows (validate=, indicator)","Stop pandas merges from multiplying rows — validate=\"many_to_one\" and MergeError, finding duplicate keys, indicator=True for unmatched rows, NaN keys that join each other, and row-count and total checks around every merge.","validate-a-merge-to-avoid-duplicate-rows","advanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fvalidate-a-merge-to-avoid-duplicate-rows\u002Findex",[1929,1930,1931],"joins-and-lookups","data-validation","pandas","how-to","eFHYmF6uGMgg4hc-Bspu1KQmvpnwm0KRVXnlQO0Sayk",[1935,1939],{"title":1936,"path":1937,"stem":1938,"children":-1},"Update One Excel File from Another with pandas","\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fupdate-one-excel-file-from-another-with-pandas","advanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fupdate-one-excel-file-from-another-with-pandas\u002Findex",{"title":1872,"path":1940,"stem":1941,"children":-1},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fvlookup-equivalent-in-pandas-for-excel-files","advanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fvlookup-equivalent-in-pandas-for-excel-files\u002Findex",[1943,1945],{"path":1944,"title":1918},"\u002Fadvanced-data-transformation-and-cleaning",{"path":1946,"title":1921},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes",[],[1949,1952,1955,1958,1961,1964],{"path":1950,"title":1951,"kind":1932},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fcompare-two-excel-files-for-differences-with-python","Compare Two Excel Files for Differences with Python",{"path":1953,"title":1954,"kind":1932},"\u002Fautomating-reporting-workflows\u002Fbuilding-finance-and-operations-reports\u002Fconvert-multi-currency-transactions-with-an-fx-rate-table","Convert Multi-Currency Transactions with an FX Rate Table",{"path":1956,"title":1957,"kind":1932},"\u002Fadvanced-data-transformation-and-cleaning\u002Fapplying-conditional-formatting-with-openpyxl\u002Fhighlight-differences-between-two-columns-with-openpyxl","Highlight Differences Between Two Columns with openpyxl",{"path":1959,"title":1960,"kind":1932},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fmap-free-text-job-titles-to-standard-categories","Map Free-Text Job Titles to Standard Categories",{"path":1962,"title":1963,"kind":1932},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fstandardise-category-values-with-a-mapping-table","Standardise Category Values with a Mapping Table in pandas",{"path":1965,"title":1966,"kind":1932},"\u002Fadvanced-data-transformation-and-cleaning\u002Fexcel-formula-equivalents-in-pandas\u002Fapproximate-match-vlookup-in-pandas","Approximate-Match VLOOKUP for Tiered Rates in pandas",1790972096521]