[{"data":1,"prerenderedAt":2071},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fupdate-one-excel-file-from-another-with-pandas":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fupdate-one-excel-file-from-another-with-pandas":2038,"ancestors:\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fupdate-one-excel-file-from-another-with-pandas":2047,"children:\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fupdate-one-excel-file-from-another-with-pandas":2052,"related:\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fupdate-one-excel-file-from-another-with-pandas":2053},{"id":4,"title":5,"body":6,"dateModified":2003,"datePublished":2003,"description":2004,"extension":2005,"faq":2006,"keyword":2015,"kind":2016,"meta":2017,"navigation":440,"path":2026,"seo":2027,"slug":2030,"stem":2031,"topics":2032,"type":2036,"__hash__":2037},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fupdate-one-excel-file-from-another-with-pandas\u002Findex.md","Update One Excel File from Another with pandas",{"type":7,"value":8,"toc":1989},"minimark",[9,29,371,376,403,412,485,488,492,498,527,540,544,550,572,582,752,756,759,1037,1040,1044,1054,1193,1213,1217,1223,1611,1619,1780,1784,1787,1847,1855,1859,1898,1902,1910,1914,1924,1928,1934,1940,1946,1952,1956,1985],[10,11,12,13,17,18,22,23,28],"p",{},"A master list — products, customers, cost centres — lives in one workbook, and changes arrive in another: new prices from purchasing, corrected contact details from sales, a few new items. The manual process is a ",[14,15,16],"code",{},"VLOOKUP"," column, a filter, careful copy-paste and hope. pandas can apply the changes in one step, but which step depends on what \"update\" means: overwrite with every new value, fill only the blanks, or overwrite ",[19,20,21],"em",{},"and"," add new rows. The three have different rules about missing values, and a naive write-back throws away the master's formatting. This page, part of ",[24,25,27],"a",{"href":26},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002F","Merging and Joining Excel DataFrames with Pandas",", covers all of it.",[30,31,39,40,39,44,39,48,39,56,39,65,39,69,39,76,39,80,39,84,39,88,39,92,39,96,39,101,39,103,39,107,39,109,39,112,39,114,39,117,39,120,39,124,39,126,39,130,39,132,39,135,39,137,39,140,39,143,39,147,39,149,39,152,39,154,39,157,39,159,39,161,39,164,39,168,39,170,39,173,39,175,39,178,39,180,39,183,39,187,39,189,39,192,39,194,39,197,39,199,39,202,39,205,39,208,39,211,39,214,39,217,39,220,39,223,39,225,39,228,39,230,39,232,39,234,39,236,39,238,39,240,39,242,39,244,39,246,39,248,39,250,39,252,39,254,39,256,39,258,39,260,39,262,39,264,39,269,39,273,39,275,39,277,39,279,39,281,39,283,39,285,39,287,39,289,39,291,39,294,39,296,39,298,39,300,39,302,39,304,39,306,39,308,39,310,39,313,39,317,39,321,39,325,39,327,39,330,39,332,39,334,39,340,39,344,39,352,39,357,39,365],"svg",{"viewBox":32,"role":33,"ariaLabelledBy":34,"xmlns":37,"style":38},"88 0 584 248","img",[35,36],"upd-apply-t","upd-apply-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:584px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[41,42,43],"title",{"id":35},"A changes file applied to a master list",[45,46,47],"desc",{"id":36},"Left: the master list with four products. Right: after applying the changes, B-2's price is 12.50, C-3 has the supplier Initech filled in, C-3's price is unchanged because the change file left it blank, and a new product E-5 has been added. Changed and added cells are highlighted.",[49,50],"rect",{"x":51,"y":52,"width":53,"height":54,"fill":55},"88","0","584","248","#ffffff",[49,57],{"x":58,"y":59,"width":60,"height":61,"rx":52,"fill":62,"stroke":63,"style":64},"104.3","40.0","32.0","24.0","#e7ebef","var(--line,#cdd5e6)","stroke-width:1px",[49,66],{"x":67,"y":59,"width":68,"height":61,"rx":52,"fill":62,"stroke":63,"style":64},"136.3","47.5",[70,71,75],"text",{"x":72,"y":73,"style":74},"160.0","57.0","font-size:11px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","A",[49,77],{"x":78,"y":59,"width":79,"height":61,"rx":52,"fill":62,"stroke":63,"style":64},"183.8","62.2",[70,81,83],{"x":82,"y":73,"style":74},"214.9","B",[49,85],{"x":86,"y":59,"width":87,"height":61,"rx":52,"fill":62,"stroke":63,"style":64},"246.0","74.2",[70,89,91],{"x":90,"y":73,"style":74},"283.1","C",[49,93],{"x":58,"y":94,"width":60,"height":95,"rx":52,"fill":62,"stroke":63,"style":64},"64.0","28.0",[70,97,100],{"x":98,"y":99,"style":74},"120.3","83.0","1",[49,102],{"x":67,"y":94,"width":68,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,104,106],{"x":72,"y":99,"style":105},"font-size:12px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","sku",[49,108],{"x":78,"y":94,"width":79,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,110,111],{"x":82,"y":99,"style":105},"price",[49,113],{"x":86,"y":94,"width":87,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,115,116],{"x":90,"y":99,"style":105},"supplier",[49,118],{"x":58,"y":119,"width":60,"height":95,"rx":52,"fill":62,"stroke":63,"style":64},"92.0",[70,121,123],{"x":98,"y":122,"style":74},"111.0","2",[49,125],{"x":67,"y":119,"width":68,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,127,129],{"x":72,"y":122,"style":128},"font-size:12px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","A-1",[49,131],{"x":78,"y":119,"width":79,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,133,134],{"x":82,"y":122,"style":128},"2.50",[49,136],{"x":86,"y":119,"width":87,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,138,139],{"x":90,"y":122,"style":128},"Acme",[49,141],{"x":58,"y":142,"width":60,"height":95,"rx":52,"fill":62,"stroke":63,"style":64},"120.0",[70,144,146],{"x":98,"y":145,"style":74},"139.0","3",[49,148],{"x":67,"y":142,"width":68,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,150,151],{"x":72,"y":145,"style":128},"B-2",[49,153],{"x":78,"y":142,"width":79,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,155,156],{"x":82,"y":145,"style":128},"12.00",[49,158],{"x":86,"y":142,"width":87,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,160,139],{"x":90,"y":145,"style":128},[49,162],{"x":58,"y":163,"width":60,"height":95,"rx":52,"fill":62,"stroke":63,"style":64},"148.0",[70,165,167],{"x":98,"y":166,"style":74},"167.0","4",[49,169],{"x":67,"y":163,"width":68,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,171,172],{"x":72,"y":166,"style":128},"C-3",[49,174],{"x":78,"y":163,"width":79,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,176,177],{"x":82,"y":166,"style":128},"40.00",[49,179],{"x":86,"y":163,"width":87,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[49,181],{"x":58,"y":182,"width":60,"height":95,"rx":52,"fill":62,"stroke":63,"style":64},"176.0",[70,184,186],{"x":98,"y":185,"style":74},"195.0","5",[49,188],{"x":67,"y":182,"width":68,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,190,191],{"x":72,"y":185,"style":128},"D-4",[49,193],{"x":78,"y":182,"width":79,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,195,196],{"x":82,"y":185,"style":128},"7.25",[49,198],{"x":86,"y":182,"width":87,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,200,201],{"x":90,"y":185,"style":128},"Globex",[49,203],{"x":204,"y":59,"width":60,"height":61,"rx":52,"fill":62,"stroke":63,"style":64},"439.8",[49,206],{"x":207,"y":59,"width":68,"height":61,"rx":52,"fill":62,"stroke":63,"style":64},"471.8",[70,209,75],{"x":210,"y":73,"style":74},"495.5",[49,212],{"x":213,"y":59,"width":79,"height":61,"rx":52,"fill":62,"stroke":63,"style":64},"519.3",[70,215,83],{"x":216,"y":73,"style":74},"550.4",[49,218],{"x":219,"y":59,"width":87,"height":61,"rx":52,"fill":62,"stroke":63,"style":64},"581.5",[70,221,91],{"x":222,"y":73,"style":74},"618.6",[49,224],{"x":204,"y":94,"width":60,"height":95,"rx":52,"fill":62,"stroke":63,"style":64},[70,226,100],{"x":227,"y":99,"style":74},"455.8",[49,229],{"x":207,"y":94,"width":68,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,231,106],{"x":210,"y":99,"style":105},[49,233],{"x":213,"y":94,"width":79,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,235,111],{"x":216,"y":99,"style":105},[49,237],{"x":219,"y":94,"width":87,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,239,116],{"x":222,"y":99,"style":105},[49,241],{"x":204,"y":119,"width":60,"height":95,"rx":52,"fill":62,"stroke":63,"style":64},[70,243,123],{"x":227,"y":122,"style":74},[49,245],{"x":207,"y":119,"width":68,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,247,129],{"x":210,"y":122,"style":128},[49,249],{"x":213,"y":119,"width":79,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,251,134],{"x":216,"y":122,"style":128},[49,253],{"x":219,"y":119,"width":87,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,255,139],{"x":222,"y":122,"style":128},[49,257],{"x":204,"y":142,"width":60,"height":95,"rx":52,"fill":62,"stroke":63,"style":64},[70,259,146],{"x":227,"y":145,"style":74},[49,261],{"x":207,"y":142,"width":68,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,263,151],{"x":210,"y":145,"style":128},[49,265],{"x":213,"y":142,"width":79,"height":95,"rx":52,"fill":266,"stroke":267,"style":268},"#d9f4f1","var(--teal,#0f9488)","stroke-width:1.5px",[70,270,272],{"x":216,"y":145,"style":271},"font-size:12px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","12.50",[49,274],{"x":219,"y":142,"width":87,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,276,139],{"x":222,"y":145,"style":128},[49,278],{"x":204,"y":163,"width":60,"height":95,"rx":52,"fill":62,"stroke":63,"style":64},[70,280,167],{"x":227,"y":166,"style":74},[49,282],{"x":207,"y":163,"width":68,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,284,172],{"x":210,"y":166,"style":128},[49,286],{"x":213,"y":163,"width":79,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,288,177],{"x":216,"y":166,"style":128},[49,290],{"x":219,"y":163,"width":87,"height":95,"rx":52,"fill":266,"stroke":267,"style":268},[70,292,293],{"x":222,"y":166,"style":271},"Initech",[49,295],{"x":204,"y":182,"width":60,"height":95,"rx":52,"fill":62,"stroke":63,"style":64},[70,297,186],{"x":227,"y":185,"style":74},[49,299],{"x":207,"y":182,"width":68,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,301,191],{"x":210,"y":185,"style":128},[49,303],{"x":213,"y":182,"width":79,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,305,196],{"x":216,"y":185,"style":128},[49,307],{"x":219,"y":182,"width":87,"height":95,"rx":52,"fill":55,"stroke":63,"style":64},[70,309,201],{"x":222,"y":185,"style":128},[49,311],{"x":204,"y":312,"width":60,"height":95,"rx":52,"fill":62,"stroke":63,"style":64},"204.0",[70,314,316],{"x":227,"y":315,"style":74},"223.0","6",[49,318],{"x":207,"y":312,"width":68,"height":95,"rx":52,"fill":319,"stroke":320,"style":268},"#fdefd8","var(--gold,#b4740a)",[70,322,324],{"x":210,"y":315,"style":323},"font-size:12px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","E-5",[49,326],{"x":213,"y":312,"width":79,"height":95,"rx":52,"fill":319,"stroke":320,"style":268},[70,328,329],{"x":216,"y":315,"style":323},"3.10",[49,331],{"x":219,"y":312,"width":87,"height":95,"rx":52,"fill":319,"stroke":320,"style":268},[70,333,139],{"x":222,"y":315,"style":323},[70,335,339],{"x":336,"y":337,"style":338},"212.3","29.0","font-size:12.5px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:middle","master.xlsx",[70,341,343],{"x":342,"y":337,"style":338},"547.7","after the changes",[345,346],"line",{"x1":347,"y1":348,"x2":349,"y2":348,"stroke":350,"style":351},"330.2","122.0","425.8","var(--brand,#5b5cf0)","stroke-width:2px",[353,354],"polygon",{"points":355,"fill":356},"429.8,122.0 420.8,127.0 420.8,117.0","#5b5cf0",[49,358],{"x":359,"y":360,"width":361,"height":362,"rx":363,"fill":55,"stroke":63,"style":364},"348.2","89.0","63.6","22.0","11","stroke-width:1.2px",[70,366,370],{"x":367,"y":368,"style":369},"380.0","104.0","font-size:11.5px;font-weight:400;fill:var(--brand-strong,#4338ca);text-anchor:middle;font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace","upsert",[372,373,375],"h2",{"id":374},"prerequisites","Prerequisites",[377,378,383],"pre",{"className":379,"code":380,"language":381,"meta":382,"style":382},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl\n","bash","",[14,384,385],{"__ignoreMap":382},[386,387,389,393,397,400],"span",{"class":345,"line":388},1,[386,390,392],{"class":391},"sMTad","pip",[386,394,396],{"class":395},"srMev"," install",[386,398,399],{"class":395}," pandas",[386,401,402],{"class":395}," openpyxl\n",[10,404,405,407,408,411],{},[14,406,339],{}," has four products with a formatted header row and a widened column. ",[14,409,410],{},"changes.xlsx"," has three rows: a new price for B-2, a supplier for C-3 (with the price left blank), and a new product E-5.",[377,413,417],{"className":414,"code":415,"language":416,"meta":382,"style":382},"language-python shiki shiki-themes github-light github-dark-high-contrast","import pandas as pd\n\nmaster = pd.read_excel(\"master.xlsx\").set_index(\"sku\")\nchanges = pd.read_excel(\"changes.xlsx\").set_index(\"sku\")\n","python",[14,418,419,435,442,466],{"__ignoreMap":382},[386,420,421,425,429,432],{"class":345,"line":388},[386,422,424],{"class":423},"s-kum","import",[386,426,428],{"class":427},"skGVy"," pandas ",[386,430,431],{"class":423},"as",[386,433,434],{"class":427}," pd\n",[386,436,438],{"class":345,"line":437},2,[386,439,441],{"emptyLinePlaceholder":440},true,"\n",[386,443,445,448,451,454,457,460,463],{"class":345,"line":444},3,[386,446,447],{"class":427},"master ",[386,449,450],{"class":423},"=",[386,452,453],{"class":427}," pd.read_excel(",[386,455,456],{"class":395},"\"master.xlsx\"",[386,458,459],{"class":427},").set_index(",[386,461,462],{"class":395},"\"sku\"",[386,464,465],{"class":427},")\n",[386,467,469,472,474,476,479,481,483],{"class":345,"line":468},4,[386,470,471],{"class":427},"changes ",[386,473,450],{"class":423},[386,475,453],{"class":427},[386,477,478],{"class":395},"\"changes.xlsx\"",[386,480,459],{"class":427},[386,482,462],{"class":395},[386,484,465],{"class":427},[10,486,487],{},"Setting the key as the index on both frames is what lets pandas line rows up by SKU rather than by position.",[372,489,491],{"id":490},"step-1-dataframeupdate-overwrite-with-the-non-missing-values","Step 1 — DataFrame.update: overwrite with the non-missing values",[10,493,494,497],{},[14,495,496],{},"update"," copies every non-missing value from the changes into the master, aligned on index and columns, and modifies the master in place:",[377,499,501],{"className":414,"code":500,"language":416,"meta":382,"style":382},"updated = master.copy()\nupdated.update(changes)\nprint(updated)\n",[14,502,503,513,518],{"__ignoreMap":382},[386,504,505,508,510],{"class":345,"line":388},[386,506,507],{"class":427},"updated ",[386,509,450],{"class":423},[386,511,512],{"class":427}," master.copy()\n",[386,514,515],{"class":345,"line":437},[386,516,517],{"class":427},"updated.update(changes)\n",[386,519,520,524],{"class":345,"line":444},[386,521,523],{"class":522},"sP0c6","print",[386,525,526],{"class":427},"(updated)\n",[10,528,529,530,532,533,536,537,539],{},"B-2's price becomes 12.50 and C-3 gets its supplier. Two rules decide the rest. C-3's price stays 40.00 because the change file's price cell is blank — ",[14,531,496],{}," never overwrites with a missing value, so a changes file cannot ",[19,534,535],{},"clear"," a field this way. And E-5 does not appear: ",[14,538,496],{}," only touches rows that already exist in the master. Both rules are sensible for applying corrections, and both surprise people who expected a full sync.",[372,541,543],{"id":542},"step-2-combine_first-fill-only-the-gaps","Step 2 — combine_first: fill only the gaps",[10,545,546,549],{},[14,547,548],{},"combine_first"," works the other way round: existing values win, and the other frame only fills blanks. It also adds rows that exist only in the other frame:",[377,551,553],{"className":414,"code":552,"language":416,"meta":382,"style":382},"filled = master.combine_first(changes)\nprint(filled)\n",[14,554,555,565],{"__ignoreMap":382},[386,556,557,560,562],{"class":345,"line":388},[386,558,559],{"class":427},"filled ",[386,561,450],{"class":423},[386,563,564],{"class":427}," master.combine_first(changes)\n",[386,566,567,569],{"class":345,"line":437},[386,568,523],{"class":522},[386,570,571],{"class":427},"(filled)\n",[10,573,574,575,577,578,581],{},"C-3 gets its supplier, because the master had none, and E-5 is added — but B-2 keeps the old price of 12.00, because the master already had a value. That makes ",[14,576,548],{}," right for enriching a list from a secondary source (\"fill in missing suppliers from the purchasing file\") and wrong for applying corrections. Note that the columns come back in alphabetical order, the union of both frames' columns; reorder with ",[14,579,580],{},"filled[master.columns]",".",[30,583,39,589,39,592,39,595,39,599,39,604,39,609,39,615,39,619,39,622,39,626,39,629,39,632,39,637,39,640,39,644,39,647,39,652,39,654,39,664,39,666,39,671,39,673,39,681,39,683,39,688,39,691,39,695,39,697,39,700,39,702,39,704,39,706,39,711,39,713,39,718,39,721,39,725,39,727,39,732,39,734,39,738,39,740,39,745,39,747],{"viewBox":584,"role":33,"ariaLabelledBy":585,"xmlns":37,"style":588},"4 0 752 174",[586,587],"upd-methods-t","upd-methods-d","width:100%;max-width:752px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[41,590,591],{"id":586},"Three ways to apply one file to another",[45,593,594],{"id":587},"A comparison of update, combine_first and an explicit upsert. update overwrites existing values with new ones, never with blanks, and ignores new rows. combine_first keeps existing values, fills only blanks and adds new rows. The upsert overwrites with new values, can be told whether blanks clear fields, adds new rows and produces a change log.",[49,596],{"x":167,"y":52,"width":597,"height":598,"fill":55},"752","174",[49,600],{"x":601,"y":602,"width":603,"height":59,"rx":52,"fill":55,"stroke":63,"style":64},"20.0","16.0","230.5",[49,605],{"x":606,"y":602,"width":607,"height":59,"rx":52,"fill":608,"stroke":350,"style":364},"250.5","122.4","#f0f4ff",[70,610,614],{"x":611,"y":612,"style":613},"311.7","34.0","font-size:13px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Overwrites",[70,616,618],{"x":611,"y":617,"style":613},"49.0","values",[49,620],{"x":621,"y":602,"width":607,"height":59,"rx":52,"fill":608,"stroke":350,"style":364},"372.8",[70,623,625],{"x":624,"y":612,"style":613},"434.0","Blank in",[70,627,628],{"x":624,"y":617,"style":613},"changes",[49,630],{"x":631,"y":602,"width":607,"height":59,"rx":52,"fill":608,"stroke":350,"style":364},"495.2",[70,633,636],{"x":634,"y":635,"style":613},"556.4","41.0","Adds new rows",[49,638],{"x":639,"y":602,"width":607,"height":59,"rx":52,"fill":608,"stroke":350,"style":364},"617.6",[70,641,643],{"x":642,"y":635,"style":613},"678.8","Change log",[49,645],{"x":601,"y":646,"width":603,"height":612,"rx":52,"fill":62,"stroke":63,"style":64},"56.0",[70,648,651],{"x":60,"y":649,"style":650},"78.0","font-size:12.5px;font-weight:700;fill:var(--text,#172033);text-anchor:start","master.update(changes)",[49,653],{"x":606,"y":646,"width":607,"height":612,"rx":52,"fill":55,"stroke":63,"style":64},[655,656,658],"g",{"transform":657},"translate(302.7,64.0) scale(0.750)",[659,660],"path",{"d":661,"fill":662,"stroke":267,"style":663},"M5 12.5l4.5 4.5L19 7","none","stroke-width:2.40px;stroke-linecap:round;stroke-linejoin:round",[49,665],{"x":621,"y":646,"width":607,"height":612,"rx":52,"fill":55,"stroke":63,"style":64},[70,667,670],{"x":624,"y":668,"style":669},"77.0","font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","ignored",[49,672],{"x":631,"y":646,"width":607,"height":612,"rx":52,"fill":55,"stroke":63,"style":64},[655,674,676],{"transform":675},"translate(548.4,65.0) scale(0.667)",[659,677],{"d":678,"fill":662,"stroke":679,"style":680},"M6 6l12 12 M18 6L6 18","var(--accent,#f43f8f)","stroke-width:2.70px;stroke-linecap:round;stroke-linejoin:round",[49,682],{"x":639,"y":646,"width":607,"height":612,"rx":52,"fill":55,"stroke":63,"style":64},[655,684,686],{"transform":685},"translate(670.8,65.0) scale(0.667)",[659,687],{"d":678,"fill":662,"stroke":679,"style":680},[49,689],{"x":601,"y":690,"width":603,"height":612,"rx":52,"fill":62,"stroke":63,"style":64},"90.0",[70,692,694],{"x":60,"y":693,"style":650},"112.0","master.combine_first(changes)",[49,696],{"x":606,"y":690,"width":607,"height":612,"rx":52,"fill":55,"stroke":63,"style":64},[70,698,699],{"x":611,"y":122,"style":669},"only blanks",[49,701],{"x":621,"y":690,"width":607,"height":612,"rx":52,"fill":55,"stroke":63,"style":64},[70,703,670],{"x":624,"y":122,"style":669},[49,705],{"x":631,"y":690,"width":607,"height":612,"rx":52,"fill":55,"stroke":63,"style":64},[655,707,709],{"transform":708},"translate(547.4,98.0) scale(0.750)",[659,710],{"d":661,"fill":662,"stroke":267,"style":663},[49,712],{"x":639,"y":690,"width":607,"height":612,"rx":52,"fill":55,"stroke":63,"style":64},[655,714,716],{"transform":715},"translate(670.8,99.0) scale(0.667)",[659,717],{"d":678,"fill":662,"stroke":679,"style":680},[49,719],{"x":601,"y":720,"width":603,"height":612,"rx":52,"fill":62,"stroke":63,"style":64},"124.0",[70,722,724],{"x":60,"y":723,"style":650},"146.0","Explicit upsert",[49,726],{"x":606,"y":720,"width":607,"height":612,"rx":52,"fill":55,"stroke":63,"style":64},[655,728,730],{"transform":729},"translate(302.7,132.0) scale(0.750)",[659,731],{"d":661,"fill":662,"stroke":267,"style":663},[49,733],{"x":621,"y":720,"width":607,"height":612,"rx":52,"fill":55,"stroke":63,"style":64},[70,735,737],{"x":624,"y":736,"style":669},"145.0","your rule",[49,739],{"x":631,"y":720,"width":607,"height":612,"rx":52,"fill":55,"stroke":63,"style":64},[655,741,743],{"transform":742},"translate(547.4,132.0) scale(0.750)",[659,744],{"d":661,"fill":662,"stroke":267,"style":663},[49,746],{"x":639,"y":720,"width":607,"height":612,"rx":52,"fill":55,"stroke":63,"style":64},[655,748,750],{"transform":749},"translate(669.8,132.0) scale(0.750)",[659,751],{"d":661,"fill":662,"stroke":267,"style":663},[372,753,755],{"id":754},"step-3-an-explicit-upsert-with-a-change-log","Step 3 — An explicit upsert with a change log",[10,757,758],{},"For a real master list you usually want both behaviours — update existing rows and add new ones — plus a record of what changed. Build it explicitly: stack the non-missing changes into long form, compare with the current values, and apply:",[377,760,762],{"className":414,"code":761,"language":416,"meta":382,"style":382},"def upsert(master, changes):\n    result = master.copy()\n    log = []\n    new_keys = changes.index.difference(master.index)\n    for key, row in changes.drop(index=new_keys).iterrows():\n        for col, new in row.dropna().items():\n            old = result.at[key, col]\n            if pd.isna(old) or old != new:\n                log.append({\"sku\": key, \"column\": col, \"old\": old, \"new\": new})\n                result.at[key, col] = new\n    added = changes.loc[new_keys].reindex(columns=master.columns)\n    for key in new_keys:\n        log.append({\"sku\": key, \"column\": \"(new row)\", \"old\": None, \"new\": None})\n    return pd.concat([result, added]), pd.DataFrame(log)\n\nresult, change_log = upsert(master, changes)\nprint(result)\nprint(change_log.to_string(index=False))\n",[14,763,764,776,785,795,805,829,843,854,875,904,915,934,947,986,995,1000,1011,1019],{"__ignoreMap":382},[386,765,766,769,773],{"class":345,"line":388},[386,767,768],{"class":423},"def",[386,770,772],{"class":771},"s_Opv"," upsert",[386,774,775],{"class":427},"(master, changes):\n",[386,777,778,781,783],{"class":345,"line":437},[386,779,780],{"class":427},"    result ",[386,782,450],{"class":423},[386,784,512],{"class":427},[386,786,787,790,792],{"class":345,"line":444},[386,788,789],{"class":427},"    log ",[386,791,450],{"class":423},[386,793,794],{"class":427}," []\n",[386,796,797,800,802],{"class":345,"line":468},[386,798,799],{"class":427},"    new_keys ",[386,801,450],{"class":423},[386,803,804],{"class":427}," changes.index.difference(master.index)\n",[386,806,808,811,814,817,820,824,826],{"class":345,"line":807},5,[386,809,810],{"class":423},"    for",[386,812,813],{"class":427}," key, row ",[386,815,816],{"class":423},"in",[386,818,819],{"class":427}," changes.drop(",[386,821,823],{"class":822},"sa561","index",[386,825,450],{"class":423},[386,827,828],{"class":427},"new_keys).iterrows():\n",[386,830,832,835,838,840],{"class":345,"line":831},6,[386,833,834],{"class":423},"        for",[386,836,837],{"class":427}," col, new ",[386,839,816],{"class":423},[386,841,842],{"class":427}," row.dropna().items():\n",[386,844,846,849,851],{"class":345,"line":845},7,[386,847,848],{"class":427},"            old ",[386,850,450],{"class":423},[386,852,853],{"class":427}," result.at[key, col]\n",[386,855,857,860,863,866,869,872],{"class":345,"line":856},8,[386,858,859],{"class":423},"            if",[386,861,862],{"class":427}," pd.isna(old) ",[386,864,865],{"class":423},"or",[386,867,868],{"class":427}," old ",[386,870,871],{"class":423},"!=",[386,873,874],{"class":427}," new:\n",[386,876,878,881,883,886,889,892,895,898,901],{"class":345,"line":877},9,[386,879,880],{"class":427},"                log.append({",[386,882,462],{"class":395},[386,884,885],{"class":427},": key, ",[386,887,888],{"class":395},"\"column\"",[386,890,891],{"class":427},": col, ",[386,893,894],{"class":395},"\"old\"",[386,896,897],{"class":427},": old, ",[386,899,900],{"class":395},"\"new\"",[386,902,903],{"class":427},": new})\n",[386,905,907,910,912],{"class":345,"line":906},10,[386,908,909],{"class":427},"                result.at[key, col] ",[386,911,450],{"class":423},[386,913,914],{"class":427}," new\n",[386,916,918,921,923,926,929,931],{"class":345,"line":917},11,[386,919,920],{"class":427},"    added ",[386,922,450],{"class":423},[386,924,925],{"class":427}," changes.loc[new_keys].reindex(",[386,927,928],{"class":822},"columns",[386,930,450],{"class":423},[386,932,933],{"class":427},"master.columns)\n",[386,935,937,939,942,944],{"class":345,"line":936},12,[386,938,810],{"class":423},[386,940,941],{"class":427}," key ",[386,943,816],{"class":423},[386,945,946],{"class":427}," new_keys:\n",[386,948,950,953,955,957,959,962,965,968,970,972,975,977,979,981,983],{"class":345,"line":949},13,[386,951,952],{"class":427},"        log.append({",[386,954,462],{"class":395},[386,956,885],{"class":427},[386,958,888],{"class":395},[386,960,961],{"class":427},": ",[386,963,964],{"class":395},"\"(new row)\"",[386,966,967],{"class":427},", ",[386,969,894],{"class":395},[386,971,961],{"class":427},[386,973,974],{"class":522},"None",[386,976,967],{"class":427},[386,978,900],{"class":395},[386,980,961],{"class":427},[386,982,974],{"class":522},[386,984,985],{"class":427},"})\n",[386,987,989,992],{"class":345,"line":988},14,[386,990,991],{"class":423},"    return",[386,993,994],{"class":427}," pd.concat([result, added]), pd.DataFrame(log)\n",[386,996,998],{"class":345,"line":997},15,[386,999,441],{"emptyLinePlaceholder":440},[386,1001,1003,1006,1008],{"class":345,"line":1002},16,[386,1004,1005],{"class":427},"result, change_log ",[386,1007,450],{"class":423},[386,1009,1010],{"class":427}," upsert(master, changes)\n",[386,1012,1014,1016],{"class":345,"line":1013},17,[386,1015,523],{"class":522},[386,1017,1018],{"class":427},"(result)\n",[386,1020,1022,1024,1027,1029,1031,1034],{"class":345,"line":1021},18,[386,1023,523],{"class":522},[386,1025,1026],{"class":427},"(change_log.to_string(",[386,1028,823],{"class":822},[386,1030,450],{"class":423},[386,1032,1033],{"class":522},"False",[386,1035,1036],{"class":427},"))\n",[10,1038,1039],{},"The result has the new price, the filled supplier and the new product, and the change log lists every modification with its old and new value — B-2's price from 12.0 to 12.5, C-3's supplier from missing to Initech, and E-5 as a new row. Only changed values are logged: a change file that repeats an unchanged value produces no entry. For large tables, the loop can be replaced by a merge-and-compare, but the explicit version is easy to read and fast for thousands of changes.",[372,1041,1043],{"id":1042},"step-4-decide-whether-a-blank-means-no-change-or-clear","Step 4 — Decide whether a blank means \"no change\" or \"clear\"",[10,1045,1046,1047,1049,1050,1053],{},"The upsert above treats a blank in the changes as \"no change\", like ",[14,1048,496],{},". Sometimes a change file uses a blank to mean \"clear this field\" — a customer who no longer has a fax number. The file format cannot tell the two apart, so agree on a marker with whoever sends the changes, such as the text ",[14,1051,1052],{},"CLEAR",":",[377,1055,1057],{"className":414,"code":1056,"language":416,"meta":382,"style":382},"CLEAR = \"CLEAR\"\nmarked = changes.astype(object)\nmarked.loc[\"C-3\", \"price\"] = CLEAR          # as typed into the changes sheet\n\ncleared, log2 = upsert(master.astype(object), marked)\ncleared = cleared.replace(CLEAR, float(\"nan\")).infer_objects()\nprint(cleared.loc[\"C-3\"].to_dict())\nprint(log2[log2[\"new\"] == CLEAR].to_string(index=False))\n",[14,1058,1059,1069,1084,1109,1113,1128,1154,1166],{"__ignoreMap":382},[386,1060,1061,1063,1066],{"class":345,"line":388},[386,1062,1052],{"class":522},[386,1064,1065],{"class":423}," =",[386,1067,1068],{"class":395}," \"CLEAR\"\n",[386,1070,1071,1074,1076,1079,1082],{"class":345,"line":437},[386,1072,1073],{"class":427},"marked ",[386,1075,450],{"class":423},[386,1077,1078],{"class":427}," changes.astype(",[386,1080,1081],{"class":522},"object",[386,1083,465],{"class":427},[386,1085,1086,1089,1092,1094,1097,1100,1102,1105],{"class":345,"line":444},[386,1087,1088],{"class":427},"marked.loc[",[386,1090,1091],{"class":395},"\"C-3\"",[386,1093,967],{"class":427},[386,1095,1096],{"class":395},"\"price\"",[386,1098,1099],{"class":427},"] ",[386,1101,450],{"class":423},[386,1103,1104],{"class":522}," CLEAR",[386,1106,1108],{"class":1107},"s-wDw","          # as typed into the changes sheet\n",[386,1110,1111],{"class":345,"line":468},[386,1112,441],{"emptyLinePlaceholder":440},[386,1114,1115,1118,1120,1123,1125],{"class":345,"line":807},[386,1116,1117],{"class":427},"cleared, log2 ",[386,1119,450],{"class":423},[386,1121,1122],{"class":427}," upsert(master.astype(",[386,1124,1081],{"class":522},[386,1126,1127],{"class":427},"), marked)\n",[386,1129,1130,1133,1135,1138,1140,1142,1145,1148,1151],{"class":345,"line":831},[386,1131,1132],{"class":427},"cleared ",[386,1134,450],{"class":423},[386,1136,1137],{"class":427}," cleared.replace(",[386,1139,1052],{"class":522},[386,1141,967],{"class":427},[386,1143,1144],{"class":522},"float",[386,1146,1147],{"class":427},"(",[386,1149,1150],{"class":395},"\"nan\"",[386,1152,1153],{"class":427},")).infer_objects()\n",[386,1155,1156,1158,1161,1163],{"class":345,"line":845},[386,1157,523],{"class":522},[386,1159,1160],{"class":427},"(cleared.loc[",[386,1162,1091],{"class":395},[386,1164,1165],{"class":427},"].to_dict())\n",[386,1167,1168,1170,1173,1175,1177,1180,1182,1185,1187,1189,1191],{"class":345,"line":856},[386,1169,523],{"class":522},[386,1171,1172],{"class":427},"(log2[log2[",[386,1174,900],{"class":395},[386,1176,1099],{"class":427},[386,1178,1179],{"class":423},"==",[386,1181,1104],{"class":522},[386,1183,1184],{"class":427},"].to_string(",[386,1186,823],{"class":822},[386,1188,450],{"class":423},[386,1190,1033],{"class":522},[386,1192,1036],{"class":427},[10,1194,1195,1196,1198,1199,1201,1202,1205,1206,1209,1210,1212],{},"C-3's price is now missing, deliberately, and the change log records that it was cleared. Working in ",[14,1197,1081],{}," columns while applying the marker is necessary in pandas 3, which refuses to put the text ",[14,1200,1052],{}," into a float column with a ",[14,1203,1204],{},"TypeError"," instead of silently converting the column as older versions did; ",[14,1207,1208],{},"infer_objects()"," restores the numeric type afterwards. In a real file the marker arrives the same way — a price column containing the word ",[14,1211,1052],{}," is read as text, and has to be converted back once the marker is applied. An explicit marker is far safer than treating every blank as a deletion, which would wipe out every field the sender simply did not fill in.",[372,1214,1216],{"id":1215},"step-5-write-the-changes-into-the-formatted-master","Step 5 — Write the changes into the formatted master",[10,1218,1219,1222],{},[14,1220,1221],{},"result.to_excel(\"master.xlsx\")"," would replace the workbook with a plain one: the formatted header, column widths and anything else in the file would be lost. To keep them, open the master with openpyxl and write only the changed cells and the new rows:",[377,1224,1226],{"className":414,"code":1225,"language":416,"meta":382,"style":382},"from openpyxl import load_workbook\nfrom openpyxl.styles import PatternFill\n\nwb = load_workbook(\"master.xlsx\")\nws = wb[\"Products\"]\nheaders = {c.value: c.column for c in ws[1]}\nrows = {ws.cell(row=r, column=headers[\"sku\"]).value: r for r in range(2, ws.max_row + 1)}\nmark = PatternFill(\"solid\", fgColor=\"FFF2CC\")\n\nfor entry in change_log.itertuples(index=False):\n    if entry.column == \"(new row)\":\n        values = result.loc[entry.sku]\n        ws.append([entry.sku, *[None if pd.isna(values[c]) else values[c] for c in master.columns]])\n        for cell in ws[ws.max_row]:\n            cell.fill = mark\n    else:\n        cell = ws.cell(row=rows[entry.sku], column=headers[entry.column], value=entry.new)\n        cell.fill = mark\n\nwb.save(\"master_updated.xlsx\")\nprint(pd.read_excel(\"master_updated.xlsx\").to_string(index=False))\nprint(load_workbook(\"master_updated.xlsx\")[\"Products\"][\"A1\"].font.b)\n",[14,1227,1228,1241,1253,1257,1271,1287,1313,1370,1395,1399,1420,1436,1446,1480,1492,1502,1509,1541,1550,1555,1566,1587],{"__ignoreMap":382},[386,1229,1230,1233,1236,1238],{"class":345,"line":388},[386,1231,1232],{"class":423},"from",[386,1234,1235],{"class":427}," openpyxl ",[386,1237,424],{"class":423},[386,1239,1240],{"class":427}," load_workbook\n",[386,1242,1243,1245,1248,1250],{"class":345,"line":437},[386,1244,1232],{"class":423},[386,1246,1247],{"class":427}," openpyxl.styles ",[386,1249,424],{"class":423},[386,1251,1252],{"class":427}," PatternFill\n",[386,1254,1255],{"class":345,"line":444},[386,1256,441],{"emptyLinePlaceholder":440},[386,1258,1259,1262,1264,1267,1269],{"class":345,"line":468},[386,1260,1261],{"class":427},"wb ",[386,1263,450],{"class":423},[386,1265,1266],{"class":427}," load_workbook(",[386,1268,456],{"class":395},[386,1270,465],{"class":427},[386,1272,1273,1276,1278,1281,1284],{"class":345,"line":807},[386,1274,1275],{"class":427},"ws ",[386,1277,450],{"class":423},[386,1279,1280],{"class":427}," wb[",[386,1282,1283],{"class":395},"\"Products\"",[386,1285,1286],{"class":427},"]\n",[386,1288,1289,1292,1294,1297,1300,1303,1305,1308,1310],{"class":345,"line":831},[386,1290,1291],{"class":427},"headers ",[386,1293,450],{"class":423},[386,1295,1296],{"class":427}," {c.value: c.column ",[386,1298,1299],{"class":423},"for",[386,1301,1302],{"class":427}," c ",[386,1304,816],{"class":423},[386,1306,1307],{"class":427}," ws[",[386,1309,100],{"class":522},[386,1311,1312],{"class":427},"]}\n",[386,1314,1315,1318,1320,1323,1326,1328,1331,1334,1336,1339,1341,1344,1346,1349,1351,1354,1356,1358,1361,1364,1367],{"class":345,"line":845},[386,1316,1317],{"class":427},"rows ",[386,1319,450],{"class":423},[386,1321,1322],{"class":427}," {ws.cell(",[386,1324,1325],{"class":822},"row",[386,1327,450],{"class":423},[386,1329,1330],{"class":427},"r, ",[386,1332,1333],{"class":822},"column",[386,1335,450],{"class":423},[386,1337,1338],{"class":427},"headers[",[386,1340,462],{"class":395},[386,1342,1343],{"class":427},"]).value: r ",[386,1345,1299],{"class":423},[386,1347,1348],{"class":427}," r ",[386,1350,816],{"class":423},[386,1352,1353],{"class":522}," range",[386,1355,1147],{"class":427},[386,1357,123],{"class":522},[386,1359,1360],{"class":427},", ws.max_row ",[386,1362,1363],{"class":423},"+",[386,1365,1366],{"class":522}," 1",[386,1368,1369],{"class":427},")}\n",[386,1371,1372,1375,1377,1380,1383,1385,1388,1390,1393],{"class":345,"line":856},[386,1373,1374],{"class":427},"mark ",[386,1376,450],{"class":423},[386,1378,1379],{"class":427}," PatternFill(",[386,1381,1382],{"class":395},"\"solid\"",[386,1384,967],{"class":427},[386,1386,1387],{"class":822},"fgColor",[386,1389,450],{"class":423},[386,1391,1392],{"class":395},"\"FFF2CC\"",[386,1394,465],{"class":427},[386,1396,1397],{"class":345,"line":877},[386,1398,441],{"emptyLinePlaceholder":440},[386,1400,1401,1403,1406,1408,1411,1413,1415,1417],{"class":345,"line":906},[386,1402,1299],{"class":423},[386,1404,1405],{"class":427}," entry ",[386,1407,816],{"class":423},[386,1409,1410],{"class":427}," change_log.itertuples(",[386,1412,823],{"class":822},[386,1414,450],{"class":423},[386,1416,1033],{"class":522},[386,1418,1419],{"class":427},"):\n",[386,1421,1422,1425,1428,1430,1433],{"class":345,"line":917},[386,1423,1424],{"class":423},"    if",[386,1426,1427],{"class":427}," entry.column ",[386,1429,1179],{"class":423},[386,1431,1432],{"class":395}," \"(new row)\"",[386,1434,1435],{"class":427},":\n",[386,1437,1438,1441,1443],{"class":345,"line":936},[386,1439,1440],{"class":427},"        values ",[386,1442,450],{"class":423},[386,1444,1445],{"class":427}," result.loc[entry.sku]\n",[386,1447,1448,1451,1454,1457,1459,1462,1465,1468,1471,1473,1475,1477],{"class":345,"line":949},[386,1449,1450],{"class":427},"        ws.append([entry.sku, ",[386,1452,1453],{"class":423},"*",[386,1455,1456],{"class":427},"[",[386,1458,974],{"class":522},[386,1460,1461],{"class":423}," if",[386,1463,1464],{"class":427}," pd.isna(values[c]) ",[386,1466,1467],{"class":423},"else",[386,1469,1470],{"class":427}," values[c] ",[386,1472,1299],{"class":423},[386,1474,1302],{"class":427},[386,1476,816],{"class":423},[386,1478,1479],{"class":427}," master.columns]])\n",[386,1481,1482,1484,1487,1489],{"class":345,"line":988},[386,1483,834],{"class":423},[386,1485,1486],{"class":427}," cell ",[386,1488,816],{"class":423},[386,1490,1491],{"class":427}," ws[ws.max_row]:\n",[386,1493,1494,1497,1499],{"class":345,"line":997},[386,1495,1496],{"class":427},"            cell.fill ",[386,1498,450],{"class":423},[386,1500,1501],{"class":427}," mark\n",[386,1503,1504,1507],{"class":345,"line":1002},[386,1505,1506],{"class":423},"    else",[386,1508,1435],{"class":427},[386,1510,1511,1514,1516,1519,1521,1523,1526,1528,1530,1533,1536,1538],{"class":345,"line":1013},[386,1512,1513],{"class":427},"        cell ",[386,1515,450],{"class":423},[386,1517,1518],{"class":427}," ws.cell(",[386,1520,1325],{"class":822},[386,1522,450],{"class":423},[386,1524,1525],{"class":427},"rows[entry.sku], ",[386,1527,1333],{"class":822},[386,1529,450],{"class":423},[386,1531,1532],{"class":427},"headers[entry.column], ",[386,1534,1535],{"class":822},"value",[386,1537,450],{"class":423},[386,1539,1540],{"class":427},"entry.new)\n",[386,1542,1543,1546,1548],{"class":345,"line":1021},[386,1544,1545],{"class":427},"        cell.fill ",[386,1547,450],{"class":423},[386,1549,1501],{"class":427},[386,1551,1553],{"class":345,"line":1552},19,[386,1554,441],{"emptyLinePlaceholder":440},[386,1556,1558,1561,1564],{"class":345,"line":1557},20,[386,1559,1560],{"class":427},"wb.save(",[386,1562,1563],{"class":395},"\"master_updated.xlsx\"",[386,1565,465],{"class":427},[386,1567,1569,1571,1574,1576,1579,1581,1583,1585],{"class":345,"line":1568},21,[386,1570,523],{"class":522},[386,1572,1573],{"class":427},"(pd.read_excel(",[386,1575,1563],{"class":395},[386,1577,1578],{"class":427},").to_string(",[386,1580,823],{"class":822},[386,1582,450],{"class":423},[386,1584,1033],{"class":522},[386,1586,1036],{"class":427},[386,1588,1590,1592,1595,1597,1600,1602,1605,1608],{"class":345,"line":1589},22,[386,1591,523],{"class":522},[386,1593,1594],{"class":427},"(load_workbook(",[386,1596,1563],{"class":395},[386,1598,1599],{"class":427},")[",[386,1601,1283],{"class":395},[386,1603,1604],{"class":427},"][",[386,1606,1607],{"class":395},"\"A1\"",[386,1609,1610],{"class":427},"].font.b)\n",[10,1612,1613,1614,1618],{},"The header keeps its bold white-on-blue style, and each changed or added cell gets a light highlight so reviewers can see what moved. Saving to a new file name first, and replacing the original only after a check, protects the master if anything goes wrong. ",[24,1615,1617],{"href":1616},"\u002Fautomating-reporting-workflows\u002Fgenerating-excel-reports-from-templates\u002Fpopulate-excel-template-without-losing-formatting\u002F","Populate an Excel Template Without Losing Formatting"," covers the same technique for templates.",[30,1620,39,1625,39,1628,39,1631,39,1634,39,1639,39,1647,39,1653,39,1657,39,1661,39,1666,39,1669,39,1672,39,1679,39,1685,39,1688,39,1692,39,1696,39,1700,39,1703,39,1706,39,1713,39,1717,39,1721,39,1723,39,1727,39,1730,39,1733,39,1740,39,1745,39,1748,39,1752,39,1755,39,1759,39,1762,39,1765,39,1770,39,1774,39,1777],{"viewBox":1621,"role":33,"ariaLabelledBy":1622,"xmlns":37,"style":588},"4 0 752 160",[1623,1624],"upd-flow-t","upd-flow-d",[41,1626,1627],{"id":1623},"Updating a master workbook safely",[45,1629,1630],{"id":1624},"Five steps: read both files with the key as index; check the changes for duplicate or unknown keys; upsert and build a change log; write only the changed cells into the master with openpyxl, highlighting them; save to a new file and keep the change log alongside.",[49,1632],{"x":167,"y":52,"width":597,"height":1633,"fill":55},"160",[49,1635],{"x":601,"y":602,"width":1636,"height":1637,"rx":1638,"fill":62,"stroke":63,"style":351},"115.2","128.0","12",[655,1640,1642],{"transform":1641},"translate(65.6,37.8) scale(1.000)",[659,1643],{"d":1644,"fill":662,"stroke":1645,"style":1646},"M3 5h18v14H3z M3 10h18 M3 15h18 M9 5v14","var(--text,#172033)","stroke-width:1.80px;stroke-linecap:round;stroke-linejoin:round",[70,1648,1652],{"x":1649,"y":1650,"style":1651},"77.6","84.2","font-size:13.5px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","Read with",[70,1654,1656],{"x":1649,"y":1655,"style":1651},"101.8","key index",[70,1658,1660],{"x":1649,"y":1659,"style":669},"121.2",".set_index(\"sku\")",[345,1662],{"x1":1663,"y1":1664,"x2":1665,"y2":1664,"stroke":350,"style":351},"140.2","80.0","162.2",[353,1667],{"points":1668,"fill":356},"166.2,80.0 157.2,85.0 157.2,75.0",[49,1670],{"x":1671,"y":602,"width":1636,"height":1637,"rx":1638,"fill":319,"stroke":320,"style":351},"171.2",[655,1673,1675],{"transform":1674},"translate(216.8,29.0) scale(1.000)",[659,1676],{"d":1677,"fill":662,"stroke":1678,"style":1646},"M11 4a7 7 0 1 0 .01 0z M16 16l5 5","var(--gold-ink,#7a4e06)",[70,1680,1684],{"x":1681,"y":1682,"style":1683},"228.8","75.5","font-size:13.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","Check the",[70,1686,628],{"x":1681,"y":1687,"style":1683},"93.0",[70,1689,1691],{"x":1681,"y":1690,"style":669},"112.5","duplicates,",[70,1693,1695],{"x":1681,"y":1694,"style":669},"130.0","unknown keys",[345,1697],{"x1":1698,"y1":1664,"x2":1699,"y2":1664,"stroke":350,"style":351},"291.4","313.4",[353,1701],{"points":1702,"fill":356},"317.4,80.0 308.4,85.0 308.4,75.0",[49,1704],{"x":1705,"y":602,"width":1636,"height":1637,"rx":1638,"fill":608,"stroke":350,"style":351},"322.4",[655,1707,1709],{"transform":1708},"translate(368.0,37.8) scale(1.000)",[659,1710],{"d":1711,"fill":662,"stroke":1712,"style":1646},"M20 11a8 8 0 1 0-2.3 5.7 M20 4v7h-7","var(--brand-strong,#4338ca)",[70,1714,1716],{"x":367,"y":1650,"style":1715},"font-size:13.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Upsert + log",[70,1718,1720],{"x":367,"y":1719,"style":669},"103.8","old and new",[70,1722,618],{"x":367,"y":1659,"style":669},[345,1724],{"x1":1725,"y1":1664,"x2":1726,"y2":1664,"stroke":350,"style":351},"442.6","464.6",[353,1728],{"points":1729,"fill":356},"468.6,80.0 459.6,85.0 459.6,75.0",[49,1731],{"x":1732,"y":602,"width":1636,"height":1637,"rx":1638,"fill":266,"stroke":267,"style":351},"473.6",[655,1734,1736],{"transform":1735},"translate(519.2,29.0) scale(1.000)",[659,1737],{"d":1738,"fill":662,"stroke":1739,"style":1646},"M4 4h16v16H4z M4 9h16 M4 14h16 M10 4v16","var(--teal-ink,#0b6157)",[70,1741,1744],{"x":1742,"y":1682,"style":1743},"531.2","font-size:13.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","Write",[70,1746,1747],{"x":1742,"y":1687,"style":1743},"changed",[70,1749,1751],{"x":1742,"y":1750,"style":1743},"110.5","cells",[70,1753,1754],{"x":1742,"y":1694,"style":669},"formatting kept",[345,1756],{"x1":1757,"y1":1664,"x2":1758,"y2":1664,"stroke":350,"style":351},"593.8","615.8",[353,1760],{"points":1761,"fill":356},"619.8,80.0 610.8,85.0 610.8,75.0",[49,1763],{"x":1764,"y":602,"width":1636,"height":1637,"rx":1638,"fill":266,"stroke":267,"style":351},"624.8",[655,1766,1768],{"transform":1767},"translate(670.4,37.8) scale(1.000)",[659,1769],{"d":661,"fill":662,"stroke":1739,"style":1646},[70,1771,1773],{"x":1772,"y":1650,"style":1743},"682.4","Save a copy",[70,1775,1776],{"x":1772,"y":1719,"style":669},"with the change",[70,1778,1779],{"x":1772,"y":1659,"style":669},"log",[372,1781,1783],{"id":1782},"check-the-changes-file-first","Check the changes file first",[10,1785,1786],{},"A changes file with a duplicated key — two different new prices for B-2 — makes the result depend on row order. Check before applying:",[377,1788,1790],{"className":414,"code":1789,"language":416,"meta":382,"style":382},"dups = changes.index[changes.index.duplicated()].tolist()\nunknown_cols = sorted(set(changes.columns) - set(master.columns))\nprint(\"duplicate keys:\", dups, \"| unknown columns:\", unknown_cols)\n",[14,1791,1792,1802,1829],{"__ignoreMap":382},[386,1793,1794,1797,1799],{"class":345,"line":388},[386,1795,1796],{"class":427},"dups ",[386,1798,450],{"class":423},[386,1800,1801],{"class":427}," changes.index[changes.index.duplicated()].tolist()\n",[386,1803,1804,1807,1809,1812,1814,1817,1820,1823,1826],{"class":345,"line":437},[386,1805,1806],{"class":427},"unknown_cols ",[386,1808,450],{"class":423},[386,1810,1811],{"class":522}," sorted",[386,1813,1147],{"class":427},[386,1815,1816],{"class":522},"set",[386,1818,1819],{"class":427},"(changes.columns) ",[386,1821,1822],{"class":423},"-",[386,1824,1825],{"class":522}," set",[386,1827,1828],{"class":427},"(master.columns))\n",[386,1830,1831,1833,1835,1838,1841,1844],{"class":345,"line":444},[386,1832,523],{"class":522},[386,1834,1147],{"class":427},[386,1836,1837],{"class":395},"\"duplicate keys:\"",[386,1839,1840],{"class":427},", dups, ",[386,1842,1843],{"class":395},"\"| unknown columns:\"",[386,1845,1846],{"class":427},", unknown_cols)\n",[10,1848,1849,1850,1854],{},"Both lists are empty here. In a scheduled job, fail when either is not: a column the master does not have usually means a renamed header in the sender's template. ",[24,1851,1853],{"href":1852},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fvalidate-excel-columns-before-import-with-pandas\u002F","Validate Excel Columns Before Import with pandas"," extends this to full column checks.",[372,1856,1858],{"id":1857},"common-pitfalls","Common pitfalls",[1860,1861,1862,1873,1881,1889],"ul",{},[1863,1864,1865,1869,1870,1872],"li",{},[1866,1867,1868],"strong",{},"Forgetting the index."," ",[14,1871,496],{}," aligns on the index; without the key as index it aligns rows by position.",[1863,1874,1875,1869,1878,1880],{},[1866,1876,1877],{},"Expecting blanks to clear values.",[14,1879,496],{}," and the upsert ignore them; use an agreed marker.",[1863,1882,1883,1888],{},[1866,1884,1885,1887],{},[14,1886,548],{}," for corrections."," It keeps old values; use it only for filling gaps.",[1863,1890,1891,1897],{},[1866,1892,1893,1896],{},[14,1894,1895],{},"to_excel"," over the master."," It discards formatting; write changed cells with openpyxl.",[372,1899,1901],{"id":1900},"performance-notes","Performance notes",[10,1903,1904,1906,1907,1909],{},[14,1905,496],{}," and ",[14,1908,548],{}," are vectorised and fast on large frames. The explicit upsert loops over changed cells, which is fine for thousands of changes; writing cells with openpyxl is similarly quick. For hundreds of thousands of changes, compute the changed cells with a vectorised comparison of the aligned frames, then write them in one pass.",[372,1911,1913],{"id":1912},"conclusion","Conclusion",[10,1915,1916,1918,1919,1921,1922,581],{},[14,1917,496],{}," overwrites existing rows with non-missing new values and ignores new rows; ",[14,1920,548],{}," fills only blanks and adds new rows; an explicit upsert does both and can log every change. Decide whether a blank means \"no change\" or \"clear\" and agree a marker for the latter, check the changes for duplicate keys and unknown columns, and write the changes into the master with openpyxl — highlighting what changed — instead of overwriting the formatted workbook with ",[14,1923,1895],{},[372,1925,1927],{"id":1926},"frequently-asked-questions","Frequently asked questions",[10,1929,1930,1933],{},[1866,1931,1932],{},"What does DataFrame.update do?","\nIt overwrites values in one DataFrame with the non-missing values of another, aligned on the index and columns, in place. Missing values in the update never overwrite, and rows that exist only in the update are ignored.",[10,1935,1936,1939],{},[1866,1937,1938],{},"How is combine_first different?","\ncombine_first fills the gaps in one DataFrame with values from another, and keeps existing values. It also adds rows and columns that exist only in the other frame. It is for filling blanks, not for applying corrections.",[10,1941,1942,1945],{},[1866,1943,1944],{},"How do I both update existing rows and add new ones?","\nSet the key as the index on both frames, apply update for the existing rows, then append the rows whose keys are new. Or merge with an indicator and build the result explicitly, which also gives you a change log.",[10,1947,1948,1951],{},[1866,1949,1950],{},"Will writing the result with to_excel keep the master file's formatting?","\nNo. to_excel writes a new workbook and discards fonts, fills, widths, formulas outside the data and other sheets' formatting. To keep them, open the master with openpyxl and write only the changed cells.",[372,1953,1955],{"id":1954},"related","Related",[1860,1957,1958,1964,1971,1978],{},[1863,1959,1960,1961,1963],{},"Up one level: ",[24,1962,27],{"href":26}," — every kind of join.",[1863,1965,1966,1970],{},[24,1967,1969],{"href":1968},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fupsert-excel-rows-into-a-database-table\u002F","Upsert Excel Rows into a Database Table with Python"," — the same operation against a database.",[1863,1972,1973,1977],{},[24,1974,1976],{"href":1975},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fkeep-the-latest-record-per-key-with-pandas\u002F","Keep the Latest Record per Key When Deduplicating with pandas"," — resolving duplicated keys before applying them.",[1863,1979,1980,1984],{},[24,1981,1983],{"href":1982},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fcompare-two-excel-files-for-differences-with-python\u002F","Compare Two Excel Files for Differences with Python"," — reviewing the result against the original.",[1986,1987,1988],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}",{"title":382,"searchDepth":437,"depth":437,"links":1990},[1991,1992,1993,1994,1995,1996,1997,1998,1999,2000,2001,2002],{"id":374,"depth":437,"text":375},{"id":490,"depth":437,"text":491},{"id":542,"depth":437,"text":543},{"id":754,"depth":437,"text":755},{"id":1042,"depth":437,"text":1043},{"id":1215,"depth":437,"text":1216},{"id":1782,"depth":437,"text":1783},{"id":1857,"depth":437,"text":1858},{"id":1900,"depth":437,"text":1901},{"id":1912,"depth":437,"text":1913},{"id":1926,"depth":437,"text":1927},{"id":1954,"depth":437,"text":1955},"2026-09-25","Apply a changes file to a master list — new prices, corrected emails, customers added — the way people do with copy-paste and VLOOKUP. Compare DataFrame.update, combine_first and an explicit upsert, keep a change log of old and new values, and write the updates into the existing workbook without destroying its formatting.","md",[2007,2009,2011,2013],{"q":1932,"a":2008},"It overwrites values in one DataFrame with the non-missing values of another, aligned on the index and columns, in place. Missing values in the update never overwrite, and rows that exist only in the update are ignored.",{"q":1938,"a":2010},"combine_first fills the gaps in one DataFrame with values from another, and keeps existing values. It also adds rows and columns that exist only in the other frame. It is for filling blanks, not for applying corrections.",{"q":1944,"a":2012},"Set the key as the index on both frames, apply update for the existing rows, then append the rows whose keys are new. Or merge with an indicator and build the result explicitly, which also gives you a change log.",{"q":1950,"a":2014},"No. to_excel writes a new workbook and discards fonts, fills, widths, formulas outside the data and other sheets' formatting. To keep them, open the master with openpyxl and write only the changed cells.","update values in one excel file from another",null,{"breadcrumb":2018},[2019,2022,2025],{"name":2020,"item":2021},"Home","\u002F",{"name":2023,"item":2024},"Advanced Data Transformation and Cleaning for Excel Automation","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":27,"item":26},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fupdate-one-excel-file-from-another-with-pandas",{"title":2028,"description":2029},"Update an Excel Master File from a Changes File with pandas","Update one Excel sheet from another in pandas: DataFrame.update and its NaN rule, combine_first for filling gaps, upserting new rows, a change log of old and new values, and writing changes into the formatted master workbook with openpyxl.","update-one-excel-file-from-another-with-pandas","advanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fupdate-one-excel-file-from-another-with-pandas\u002Findex",[2033,2034,2035],"joins-and-lookups","writing-excel","pandas","how-to","nVq4qk0ejSzKQKGCRhNQIURAeR_Mx7rX4GuHw8Vd6pc",[2039,2043],{"title":2040,"path":2041,"stem":2042,"children":-1},"Roll Up Totals Through a Parent-Child Hierarchy","\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Froll-up-totals-through-a-parent-child-hierarchy","advanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Froll-up-totals-through-a-parent-child-hierarchy\u002Findex",{"title":2044,"path":2045,"stem":2046,"children":-1},"Validate a pandas Merge to Avoid Duplicate Rows","\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fvalidate-a-merge-to-avoid-duplicate-rows","advanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fvalidate-a-merge-to-avoid-duplicate-rows\u002Findex",[2048,2050],{"path":2049,"title":2023},"\u002Fadvanced-data-transformation-and-cleaning",{"path":2051,"title":27},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes",[],[2054,2056,2059,2062,2065,2068],{"path":2055,"title":1983,"kind":2036},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fcompare-two-excel-files-for-differences-with-python",{"path":2057,"title":2058,"kind":2036},"\u002Fautomating-reporting-workflows\u002Fbuilding-finance-and-operations-reports\u002Fconvert-multi-currency-transactions-with-an-fx-rate-table","Convert Multi-Currency Transactions with an FX Rate Table",{"path":2060,"title":2061,"kind":2036},"\u002Fadvanced-data-transformation-and-cleaning\u002Fapplying-conditional-formatting-with-openpyxl\u002Fhighlight-differences-between-two-columns-with-openpyxl","Highlight Differences Between Two Columns with openpyxl",{"path":2063,"title":2064,"kind":2036},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fmap-free-text-job-titles-to-standard-categories","Map Free-Text Job Titles to Standard Categories",{"path":2066,"title":2067,"kind":2036},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fstandardise-category-values-with-a-mapping-table","Standardise Category Values with a Mapping Table in pandas",{"path":2069,"title":2070,"kind":2036},"\u002Fadvanced-data-transformation-and-cleaning\u002Fexcel-formula-equivalents-in-pandas\u002Fapproximate-match-vlookup-in-pandas","Approximate-Match VLOOKUP for Tiered Rates in pandas",1790972096501]