[{"data":1,"prerenderedAt":1910},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fjoin-rows-to-effective-date-ranges-with-pandas":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fjoin-rows-to-effective-date-ranges-with-pandas":1877,"ancestors:\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fjoin-rows-to-effective-date-ranges-with-pandas":1886,"children:\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fjoin-rows-to-effective-date-ranges-with-pandas":1891,"related:\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fjoin-rows-to-effective-date-ranges-with-pandas":1892},{"id":4,"title":5,"body":6,"dateModified":1842,"datePublished":1842,"description":1843,"extension":1844,"faq":1845,"keyword":1854,"kind":1855,"meta":1856,"navigation":123,"path":1866,"seo":1867,"slug":1870,"stem":1871,"topics":1872,"type":1875,"__hash__":1876},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fjoin-rows-to-effective-date-ranges-with-pandas\u002Findex.md","Join Rows to Effective Date Ranges with pandas",{"type":7,"value":8,"toc":1829},"minimark",[9,45,50,78,95,249,252,357,361,364,500,517,521,537,673,693,812,816,819,946,953,1148,1152,1155,1329,1337,1341,1344,1668,1674,1678,1696,1700,1748,1752,1764,1768,1774,1780,1786,1792,1796,1825],[10,11,12,13,17,18,22,23,26,27,30,31,34,35,38,39,44],"p",{},"Prices change, tax rates change, commission plans change — and a report that applies today's value to last quarter's transactions is wrong. Reference tables handle this with ",[14,15,16],"em",{},"effective dates",": each row says the value and the period it applies to, ",[19,20,21],"code",{},"ValidFrom"," to ",[19,24,25],{},"ValidTo",", often with a blank end meaning \"still valid\". Joining transactions to such a table is a range join: match on the product (or employee, or country) ",[14,28,29],{},"and"," on the date falling inside the period. A plain ",[19,32,33],{},"merge"," cannot express that, and Excel's ",[19,36,37],{},"VLOOKUP"," cannot either without helper columns. pandas can do it two ways, and both need a check that every row matched exactly once. This page is part of ",[40,41,43],"a",{"href":42},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002F","Merging and Joining Excel DataFrames",".",[46,47,49],"h2",{"id":48},"prerequisites","Prerequisites",[51,52,57],"pre",{"className":53,"code":54,"language":55,"meta":56,"style":56},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl\n","bash","",[19,58,59],{"__ignoreMap":56},[60,61,64,68,72,75],"span",{"class":62,"line":63},"line",1,[60,65,67],{"class":66},"sMTad","pip",[60,69,71],{"class":70},"srMev"," install",[60,73,74],{"class":70}," pandas",[60,76,77],{"class":70}," openpyxl\n",[10,79,80,83,84,87,88,90,91,94],{},[19,81,82],{},"pricing.xlsx"," has a ",[19,85,86],{},"Prices"," sheet — each SKU's price with the period it applies to, inclusive at both ends, a blank ",[19,89,25],{}," for the current price — and an ",[19,92,93],{},"Orders"," sheet with nine orders. The price list has the problems real ones have: a gap for B-200 between 16 and 31 May, two C-300 periods that overlap in August, and an order for a SKU that has no price at all.",[51,96,100],{"className":97,"code":98,"language":99,"meta":56,"style":56},"language-python shiki shiki-themes github-light github-dark-high-contrast","import pandas as pd\n\nprices = pd.read_excel(\"pricing.xlsx\", sheet_name=\"Prices\", parse_dates=[\"ValidFrom\", \"ValidTo\"])\norders = pd.read_excel(\"pricing.xlsx\", sheet_name=\"Orders\", parse_dates=[\"OrderDate\"])\nprices[\"ValidTo\"] = prices[\"ValidTo\"].fillna(pd.Timestamp.max.normalize())\nprint(prices.to_string(index=False))\n","python",[19,101,102,118,125,173,207,228],{"__ignoreMap":56},[60,103,104,108,112,115],{"class":62,"line":63},[60,105,107],{"class":106},"s-kum","import",[60,109,111],{"class":110},"skGVy"," pandas ",[60,113,114],{"class":106},"as",[60,116,117],{"class":110}," pd\n",[60,119,121],{"class":62,"line":120},2,[60,122,124],{"emptyLinePlaceholder":123},true,"\n",[60,126,128,131,134,137,140,143,147,149,152,154,157,159,162,165,167,170],{"class":62,"line":127},3,[60,129,130],{"class":110},"prices ",[60,132,133],{"class":106},"=",[60,135,136],{"class":110}," pd.read_excel(",[60,138,139],{"class":70},"\"pricing.xlsx\"",[60,141,142],{"class":110},", ",[60,144,146],{"class":145},"sa561","sheet_name",[60,148,133],{"class":106},[60,150,151],{"class":70},"\"Prices\"",[60,153,142],{"class":110},[60,155,156],{"class":145},"parse_dates",[60,158,133],{"class":106},[60,160,161],{"class":110},"[",[60,163,164],{"class":70},"\"ValidFrom\"",[60,166,142],{"class":110},[60,168,169],{"class":70},"\"ValidTo\"",[60,171,172],{"class":110},"])\n",[60,174,176,179,181,183,185,187,189,191,194,196,198,200,202,205],{"class":62,"line":175},4,[60,177,178],{"class":110},"orders ",[60,180,133],{"class":106},[60,182,136],{"class":110},[60,184,139],{"class":70},[60,186,142],{"class":110},[60,188,146],{"class":145},[60,190,133],{"class":106},[60,192,193],{"class":70},"\"Orders\"",[60,195,142],{"class":110},[60,197,156],{"class":145},[60,199,133],{"class":106},[60,201,161],{"class":110},[60,203,204],{"class":70},"\"OrderDate\"",[60,206,172],{"class":110},[60,208,210,213,215,218,220,223,225],{"class":62,"line":209},5,[60,211,212],{"class":110},"prices[",[60,214,169],{"class":70},[60,216,217],{"class":110},"] ",[60,219,133],{"class":106},[60,221,222],{"class":110}," prices[",[60,224,169],{"class":70},[60,226,227],{"class":110},"].fillna(pd.Timestamp.max.normalize())\n",[60,229,231,235,238,241,243,246],{"class":62,"line":230},6,[60,232,234],{"class":233},"sP0c6","print",[60,236,237],{"class":110},"(prices.to_string(",[60,239,240],{"class":145},"index",[60,242,133],{"class":106},[60,244,245],{"class":233},"False",[60,247,248],{"class":110},"))\n",[10,250,251],{},"A blank end date means \"until further notice\", so fill it with the largest date pandas supports. That keeps every comparison below simple — no special case for open-ended periods.",[253,254,262,263,262,267,262,271,262,279,262,286,262,296,262,303,262,308,262,312,262,316,262,322,262,328,262,332,262,336,262,338,262,340,262,345,262,350,262,353,262,355],"svg",{"viewBox":255,"role":256,"ariaLabelledBy":257,"xmlns":260,"style":261},"4 0 752 245","img",[258,259],"ed-periods-t","ed-periods-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:752px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[264,265,266],"title",{"id":258},"A-100's price over the year",[268,269,270],"desc",{"id":259},"A timeline of three price periods for product A-100. 12.50 from January to March, 13.20 from April to June, and 13.90 from July with no end date. Orders on 1 April and 30 June fall on period boundaries.",[272,273],"rect",{"x":274,"y":275,"width":276,"height":277,"fill":278},"4","0","752","245","#ffffff",[62,280],{"x1":281,"y1":282,"x2":283,"y2":282,"stroke":284,"style":285},"96.0","122.5","664.0","var(--muted,#5b6780)","stroke-width:2.5px",[272,287],{"x":288,"y":289,"width":290,"height":291,"rx":292,"fill":293,"stroke":294,"style":295},"20.0","16.0","176.0","74.5","12","#e7ebef","var(--line,#cdd5e6)","stroke-width:2px",[297,298,302],"text",{"x":299,"y":300,"style":301},"108.0","49.0","font-size:12.5px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","1 Jan – 31 Mar",[297,304,307],{"x":299,"y":305,"style":306},"67.5","font-size:11px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","12.50",[62,309],{"x1":299,"y1":310,"x2":299,"y2":282,"stroke":284,"style":311},"90.5","stroke-width:1.5px",[313,314],"circle",{"cx":299,"cy":282,"r":315,"fill":284,"stroke":278,"style":295},"7.0",[272,317],{"x":318,"y":319,"width":290,"height":291,"rx":292,"fill":320,"stroke":321,"style":295},"292.0","154.5","#f0f4ff","var(--brand,#5b5cf0)",[297,323,327],{"x":324,"y":325,"style":326},"380.0","179.0","font-size:12.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","1 Apr – 30 Jun",[297,329,331],{"x":324,"y":330,"style":306},"197.5","13.20 — 9002 and 9003 on",[297,333,335],{"x":324,"y":334,"style":306},"214.5","the edges",[62,337],{"x1":324,"y1":282,"x2":324,"y2":319,"stroke":284,"style":311},[313,339],{"cx":324,"cy":282,"r":315,"fill":321,"stroke":278,"style":295},[272,341],{"x":342,"y":289,"width":290,"height":291,"rx":292,"fill":343,"stroke":344,"style":295},"564.0","#d9f4f1","var(--teal,#0f9488)",[297,346,349],{"x":347,"y":300,"style":348},"652.0","font-size:12.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","1 Jul – open",[297,351,352],{"x":347,"y":305,"style":306},"13.90 — current price",[62,354],{"x1":347,"y1":310,"x2":347,"y2":282,"stroke":284,"style":311},[313,356],{"cx":347,"cy":282,"r":315,"fill":344,"stroke":278,"style":295},[46,358,360],{"id":359},"step-1-merge-on-the-key-then-filter-by-date","Step 1 — Merge on the key, then filter by date",[10,362,363],{},"The most direct method: join every order to every price row for its SKU, then keep the rows whose date falls inside the period:",[51,365,367],{"className":97,"code":366,"language":99,"meta":56,"style":56},"candidates = orders.merge(prices, on=\"SKU\", how=\"left\")\nin_range = candidates[\"OrderDate\"].between(candidates[\"ValidFrom\"], candidates[\"ValidTo\"])\npriced = candidates[in_range]\nprint(priced[[\"OrderID\", \"OrderDate\", \"SKU\", \"ValidFrom\", \"Price\"]].to_string(index=False))\nprint(len(orders), \"orders ->\", len(priced), \"priced rows\")\n",[19,368,369,400,424,434,472],{"__ignoreMap":56},[60,370,371,374,376,379,382,384,387,389,392,394,397],{"class":62,"line":63},[60,372,373],{"class":110},"candidates ",[60,375,133],{"class":106},[60,377,378],{"class":110}," orders.merge(prices, ",[60,380,381],{"class":145},"on",[60,383,133],{"class":106},[60,385,386],{"class":70},"\"SKU\"",[60,388,142],{"class":110},[60,390,391],{"class":145},"how",[60,393,133],{"class":106},[60,395,396],{"class":70},"\"left\"",[60,398,399],{"class":110},")\n",[60,401,402,405,407,410,412,415,417,420,422],{"class":62,"line":120},[60,403,404],{"class":110},"in_range ",[60,406,133],{"class":106},[60,408,409],{"class":110}," candidates[",[60,411,204],{"class":70},[60,413,414],{"class":110},"].between(candidates[",[60,416,164],{"class":70},[60,418,419],{"class":110},"], candidates[",[60,421,169],{"class":70},[60,423,172],{"class":110},[60,425,426,429,431],{"class":62,"line":127},[60,427,428],{"class":110},"priced ",[60,430,133],{"class":106},[60,432,433],{"class":110}," candidates[in_range]\n",[60,435,436,438,441,444,446,448,450,452,454,456,458,461,464,466,468,470],{"class":62,"line":175},[60,437,234],{"class":233},[60,439,440],{"class":110},"(priced[[",[60,442,443],{"class":70},"\"OrderID\"",[60,445,142],{"class":110},[60,447,204],{"class":70},[60,449,142],{"class":110},[60,451,386],{"class":70},[60,453,142],{"class":110},[60,455,164],{"class":70},[60,457,142],{"class":110},[60,459,460],{"class":70},"\"Price\"",[60,462,463],{"class":110},"]].to_string(",[60,465,240],{"class":145},[60,467,133],{"class":106},[60,469,245],{"class":233},[60,471,248],{"class":110},[60,473,474,476,479,482,485,488,490,492,495,498],{"class":62,"line":209},[60,475,234],{"class":233},[60,477,478],{"class":110},"(",[60,480,481],{"class":233},"len",[60,483,484],{"class":110},"(orders), ",[60,486,487],{"class":70},"\"orders ->\"",[60,489,142],{"class":110},[60,491,481],{"class":233},[60,493,494],{"class":110},"(priced), ",[60,496,497],{"class":70},"\"priced rows\"",[60,499,399],{"class":110},[10,501,502,505,506,508,509,512,513,516],{},[19,503,504],{},"between"," includes both ends, which matches inclusive ",[19,507,25],{}," dates: the order on 1 April gets the new price and the one on 30 June the old one. The method is easy to read and handles any shape of reference table. Its cost is the intermediate ",[19,510,511],{},"candidates"," frame, which has one row per order ",[14,514,515],{},"per price period of its SKU"," — fine for price lists with a handful of periods per key, heavy for a daily rate table joined to millions of rows. The row count already signals trouble: nine orders produced eight priced rows, and order 9008 appears twice — so two orders are missing and one is duplicated. Step 3 finds out which.",[46,518,520],{"id":519},"step-2-merge_asof-with-a-validity-check","Step 2 — merge_asof with a validity check",[10,522,523,526,527,529,530,533,534,536],{},[19,524,525],{},"merge_asof"," finds, for each order, the last price row whose ",[19,528,21],{}," is on or before the order date — per SKU with ",[19,531,532],{},"by="," — without building all the combinations. It knows nothing about ",[19,535,25],{},", so check that separately:",[51,538,540],{"className":97,"code":539,"language":99,"meta":56,"style":56},"asof = pd.merge_asof(orders.sort_values(\"OrderDate\"), prices.sort_values(\"ValidFrom\"),\n                     left_on=\"OrderDate\", right_on=\"ValidFrom\", by=\"SKU\", direction=\"backward\")\nasof[\"Valid\"] = asof[\"OrderDate\"] \u003C= asof[\"ValidTo\"]\nprint(asof[[\"OrderID\", \"OrderDate\", \"SKU\", \"ValidFrom\", \"ValidTo\", \"Price\", \"Valid\"]].to_string(index=False))\n",[19,541,542,562,601,630],{"__ignoreMap":56},[60,543,544,547,549,552,554,557,559],{"class":62,"line":63},[60,545,546],{"class":110},"asof ",[60,548,133],{"class":106},[60,550,551],{"class":110}," pd.merge_asof(orders.sort_values(",[60,553,204],{"class":70},[60,555,556],{"class":110},"), prices.sort_values(",[60,558,164],{"class":70},[60,560,561],{"class":110},"),\n",[60,563,564,567,569,571,573,576,578,580,582,585,587,589,591,594,596,599],{"class":62,"line":120},[60,565,566],{"class":145},"                     left_on",[60,568,133],{"class":106},[60,570,204],{"class":70},[60,572,142],{"class":110},[60,574,575],{"class":145},"right_on",[60,577,133],{"class":106},[60,579,164],{"class":70},[60,581,142],{"class":110},[60,583,584],{"class":145},"by",[60,586,133],{"class":106},[60,588,386],{"class":70},[60,590,142],{"class":110},[60,592,593],{"class":145},"direction",[60,595,133],{"class":106},[60,597,598],{"class":70},"\"backward\"",[60,600,399],{"class":110},[60,602,603,606,609,611,613,616,618,620,623,625,627],{"class":62,"line":127},[60,604,605],{"class":110},"asof[",[60,607,608],{"class":70},"\"Valid\"",[60,610,217],{"class":110},[60,612,133],{"class":106},[60,614,615],{"class":110}," asof[",[60,617,204],{"class":70},[60,619,217],{"class":110},[60,621,622],{"class":106},"\u003C=",[60,624,615],{"class":110},[60,626,169],{"class":70},[60,628,629],{"class":110},"]\n",[60,631,632,634,637,639,641,643,645,647,649,651,653,655,657,659,661,663,665,667,669,671],{"class":62,"line":175},[60,633,234],{"class":233},[60,635,636],{"class":110},"(asof[[",[60,638,443],{"class":70},[60,640,142],{"class":110},[60,642,204],{"class":70},[60,644,142],{"class":110},[60,646,386],{"class":70},[60,648,142],{"class":110},[60,650,164],{"class":70},[60,652,142],{"class":110},[60,654,169],{"class":70},[60,656,142],{"class":110},[60,658,460],{"class":70},[60,660,142],{"class":110},[60,662,608],{"class":70},[60,664,463],{"class":110},[60,666,240],{"class":145},[60,668,133],{"class":106},[60,670,245],{"class":233},[60,672,248],{"class":110},[10,674,675,676,678,679,682,683,686,687,689,690,692],{},"Both inputs must be sorted by their date column. Without the ",[19,677,25],{}," check, order 9006 on 20 May would silently get B-200's old price, because the period that started on 1 January is the last one to ",[14,680,681],{},"start"," before that date — even though it ",[14,684,685],{},"ended"," on 15 May. With the check, the row is marked invalid. ",[19,688,525],{}," always returns at most one match per order, which hides overlaps instead of revealing them: order 9008 quietly gets 7.60, the later of its two overlapping C-300 prices, and nothing in the output says there was a choice. The open-ended periods show ",[19,691,25],{}," as 11 April 2262 — the largest date pandas can store, filled in above.",[253,694,262,699,262,702,262,705,262,708,262,713,262,724,262,730,262,736,262,742,262,749,262,753,262,758,262,762,262,769,262,772,262,779,262,783,262,787,262,791,262,796,262,799,262,804,262,807],{"viewBox":695,"role":256,"ariaLabelledBy":696,"xmlns":260,"style":261},"4 0 752 207",[697,698],"ed-methods-t","ed-methods-d",[264,700,701],{"id":697},"Two ways to join on a date range",[268,703,704],{"id":698},"Comparison of merge-then-filter and merge_asof for effective-date joins. Merge-then-filter is simple and exposes gaps and overlaps but builds every combination; merge_asof is fast and memory-light but ignores the end date and hides overlaps.",[272,706],{"x":274,"y":275,"width":276,"height":707,"fill":278},"207",[272,709],{"x":288,"y":289,"width":710,"height":711,"rx":712,"fill":320,"stroke":321,"style":295},"351.0","175.0","14",[714,715,717],"g",{"transform":716},"translate(182.5,29.0) scale(1.083)",[718,719],"path",{"d":720,"fill":721,"stroke":722,"style":723},"M3 5h18l-7 8v6l-4 2v-8z","none","var(--brand-strong,#4338ca)","stroke-width:1.66px;stroke-linecap:round;stroke-linejoin:round",[297,725,729],{"x":726,"y":727,"style":728},"195.5","78.0","font-size:14px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","merge then filter",[62,731],{"x1":732,"y1":733,"x2":734,"y2":733,"stroke":321,"style":735},"36.0","91.0","355.0","stroke-width:1px",[297,737,741],{"x":738,"y":739,"style":740},"60.0","111.0","font-size:12px;font-weight:400;fill:var(--text,#172033);text-anchor:start","reads like the rule",[714,743,745],{"transform":744},"translate(38.5,99.5) scale(0.625)",[718,746],{"d":747,"fill":721,"stroke":344,"style":748},"M5 12.5l4.5 4.5L19 7","stroke-width:2.88px;stroke-linecap:round;stroke-linejoin:round",[297,750,752],{"x":738,"y":751,"style":740},"137.0","gaps and overlaps visible",[714,754,756],{"transform":755},"translate(38.5,125.5) scale(0.625)",[718,757],{"d":747,"fill":721,"stroke":344,"style":748},[297,759,761],{"x":738,"y":760,"style":740},"163.0","one row per key × period first",[714,763,765],{"transform":764},"translate(38.5,151.5) scale(0.625)",[718,766],{"d":767,"fill":721,"stroke":768,"style":748},"M6 6l12 12 M18 6L6 18","var(--accent,#f43f8f)",[272,770],{"x":771,"y":289,"width":710,"height":711,"rx":712,"fill":343,"stroke":344,"style":295},"389.0",[714,773,775],{"transform":774},"translate(551.5,29.0) scale(1.083)",[718,776],{"d":777,"fill":721,"stroke":778,"style":723},"M13 2L4 14h7l-1 8 9-12h-7z","var(--teal-ink,#0b6157)",[297,780,525],{"x":781,"y":727,"style":782},"564.5","font-size:14px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle",[62,784],{"x1":785,"y1":733,"x2":786,"y2":733,"stroke":344,"style":735},"405.0","724.0",[297,788,790],{"x":789,"y":739,"style":740},"429.0","one match per row, fast",[714,792,794],{"transform":793},"translate(407.5,99.5) scale(0.625)",[718,795],{"d":747,"fill":721,"stroke":344,"style":748},[297,797,798],{"x":789,"y":751,"style":740},"ignores ValidTo unless checked",[714,800,802],{"transform":801},"translate(407.5,125.5) scale(0.625)",[718,803],{"d":767,"fill":721,"stroke":768,"style":748},[297,805,806],{"x":789,"y":760,"style":740},"silently picks one of two overlaps",[714,808,810],{"transform":809},"translate(407.5,151.5) scale(0.625)",[718,811],{"d":767,"fill":721,"stroke":768,"style":748},[46,813,815],{"id":814},"step-3-every-order-must-match-exactly-once","Step 3 — Every order must match exactly once",[10,817,818],{},"Count the matches per order from the merge-and-filter result and classify them:",[51,820,822],{"className":97,"code":821,"language":99,"meta":56,"style":56},"matches = priced.groupby(\"OrderID\").size().reindex(orders[\"OrderID\"], fill_value=0)\nreport = orders.assign(Matches=matches.values)\nreport[\"Problem\"] = report[\"Matches\"].map({0: \"no price\", 1: \"\"}).fillna(\"several prices\")\nprint(report[report[\"Problem\"] != \"\"].to_string(index=False))\n",[19,823,824,853,871,918],{"__ignoreMap":56},[60,825,826,829,831,834,836,839,841,844,847,849,851],{"class":62,"line":63},[60,827,828],{"class":110},"matches ",[60,830,133],{"class":106},[60,832,833],{"class":110}," priced.groupby(",[60,835,443],{"class":70},[60,837,838],{"class":110},").size().reindex(orders[",[60,840,443],{"class":70},[60,842,843],{"class":110},"], ",[60,845,846],{"class":145},"fill_value",[60,848,133],{"class":106},[60,850,275],{"class":233},[60,852,399],{"class":110},[60,854,855,858,860,863,866,868],{"class":62,"line":120},[60,856,857],{"class":110},"report ",[60,859,133],{"class":106},[60,861,862],{"class":110}," orders.assign(",[60,864,865],{"class":145},"Matches",[60,867,133],{"class":106},[60,869,870],{"class":110},"matches.values)\n",[60,872,873,876,879,881,883,886,889,892,894,897,900,902,905,907,910,913,916],{"class":62,"line":127},[60,874,875],{"class":110},"report[",[60,877,878],{"class":70},"\"Problem\"",[60,880,217],{"class":110},[60,882,133],{"class":106},[60,884,885],{"class":110}," report[",[60,887,888],{"class":70},"\"Matches\"",[60,890,891],{"class":110},"].map({",[60,893,275],{"class":233},[60,895,896],{"class":110},": ",[60,898,899],{"class":70},"\"no price\"",[60,901,142],{"class":110},[60,903,904],{"class":233},"1",[60,906,896],{"class":110},[60,908,909],{"class":70},"\"\"",[60,911,912],{"class":110},"}).fillna(",[60,914,915],{"class":70},"\"several prices\"",[60,917,399],{"class":110},[60,919,920,922,925,927,929,932,935,938,940,942,944],{"class":62,"line":175},[60,921,234],{"class":233},[60,923,924],{"class":110},"(report[report[",[60,926,878],{"class":70},[60,928,217],{"class":110},[60,930,931],{"class":106},"!=",[60,933,934],{"class":70}," \"\"",[60,936,937],{"class":110},"].to_string(",[60,939,240],{"class":145},[60,941,133],{"class":106},[60,943,245],{"class":233},[60,945,248],{"class":110},[10,947,948,949,952],{},"Three orders need attention, and each is a different problem. Order 9006 falls in B-200's gap between 16 and 31 May — the price list is missing a period, or the old price should have run to the end of the month. Order 9008 on 15 August matches ",[14,950,951],{},"two"," C-300 prices, because the old period runs to 31 August while the new one starts on 1 August — an overlap in the reference data, which would double the order's revenue in any total. Order 9009 is for D-400, which has no price at all. None of these is a coding problem; all of them are questions for whoever owns the price list, and a report should raise them, not guess.",[253,954,262,960,262,963,262,966,262,971,262,976,262,980,262,986,262,990,262,994,262,998,262,1002,262,1006,262,1010,262,1014,262,1018,262,1022,262,1026,262,1028,262,1032,262,1034,262,1037,262,1039,262,1042,262,1044,262,1046,262,1048,262,1051,262,1054,262,1058,262,1060,262,1063,262,1065,262,1068,262,1070,262,1073,262,1075,262,1077,262,1081,262,1085,262,1087,262,1091,262,1093,262,1096,262,1098,262,1101,262,1103,262,1106,262,1108,262,1110,262,1114,262,1118,262,1121,262,1124,262,1126,262,1129,262,1131,262,1134,262,1136,262,1139,262,1141,262,1143,262,1145],{"viewBox":955,"role":256,"ariaLabelledBy":956,"xmlns":260,"style":959},"116 0 528 168",[957,958],"ed-problems-t","ed-problems-d","width:100%;max-width:528px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif",[264,961,962],{"id":957},"Orders that did not get exactly one price",[268,964,965],{"id":958},"A table of three problem orders. Order 9006 for B-200 on 20 May has no price because of a gap in the price list; order 9008 for C-300 on 15 August has two prices because two periods overlap; order 9009 for D-400 has no price because the SKU is missing.",[272,967],{"x":968,"y":275,"width":969,"height":970,"fill":278},"116","528","168",[272,972],{"x":973,"y":289,"width":974,"height":975,"rx":275,"fill":293,"stroke":294,"style":735},"132.2","32.0","24.0",[272,977],{"x":978,"y":289,"width":979,"height":975,"rx":275,"fill":293,"stroke":294,"style":735},"164.2","68.1",[297,981,985],{"x":982,"y":983,"style":984},"198.2","33.0","font-size:11px;font-weight:600;fill:var(--muted,#5b6780);text-anchor:middle","A",[272,987],{"x":988,"y":289,"width":989,"height":975,"rx":275,"fill":293,"stroke":294,"style":735},"232.2","95.1",[297,991,993],{"x":992,"y":983,"style":984},"279.8","B",[272,995],{"x":996,"y":289,"width":997,"height":975,"rx":275,"fill":293,"stroke":294,"style":735},"327.4","61.1",[297,999,1001],{"x":1000,"y":983,"style":984},"357.9","C",[272,1003],{"x":1004,"y":289,"width":1005,"height":975,"rx":275,"fill":293,"stroke":294,"style":735},"388.5","73.0",[297,1007,1009],{"x":1008,"y":983,"style":984},"425.0","D",[272,1011],{"x":1012,"y":289,"width":1013,"height":975,"rx":275,"fill":293,"stroke":294,"style":735},"461.5","166.3",[297,1015,1017],{"x":1016,"y":983,"style":984},"544.7","E",[272,1019],{"x":973,"y":1020,"width":974,"height":1021,"rx":275,"fill":293,"stroke":294,"style":735},"40.0","28.0",[297,1023,904],{"x":1024,"y":1025,"style":984},"148.2","59.0",[272,1027],{"x":978,"y":1020,"width":979,"height":1021,"rx":275,"fill":320,"stroke":321,"style":311},[297,1029,1031],{"x":982,"y":1025,"style":1030},"font-size:11px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","OrderID",[272,1033],{"x":988,"y":1020,"width":989,"height":1021,"rx":275,"fill":320,"stroke":321,"style":311},[297,1035,1036],{"x":992,"y":1025,"style":1030},"OrderDate",[272,1038],{"x":996,"y":1020,"width":997,"height":1021,"rx":275,"fill":320,"stroke":321,"style":311},[297,1040,1041],{"x":1000,"y":1025,"style":1030},"SKU",[272,1043],{"x":1004,"y":1020,"width":1005,"height":1021,"rx":275,"fill":320,"stroke":321,"style":311},[297,1045,865],{"x":1008,"y":1025,"style":1030},[272,1047],{"x":1012,"y":1020,"width":1013,"height":1021,"rx":275,"fill":320,"stroke":321,"style":311},[297,1049,1050],{"x":1016,"y":1025,"style":1030},"Problem",[272,1052],{"x":973,"y":1053,"width":974,"height":1021,"rx":275,"fill":293,"stroke":294,"style":735},"68.0",[297,1055,1057],{"x":1024,"y":1056,"style":984},"87.0","2",[272,1059],{"x":978,"y":1053,"width":979,"height":1021,"rx":275,"fill":278,"stroke":294,"style":735},[297,1061,1062],{"x":982,"y":1056,"style":306},"9006",[272,1064],{"x":988,"y":1053,"width":989,"height":1021,"rx":275,"fill":278,"stroke":294,"style":735},[297,1066,1067],{"x":992,"y":1056,"style":306},"2026-05-20",[272,1069],{"x":996,"y":1053,"width":997,"height":1021,"rx":275,"fill":278,"stroke":294,"style":735},[297,1071,1072],{"x":1000,"y":1056,"style":306},"B-200",[272,1074],{"x":1004,"y":1053,"width":1005,"height":1021,"rx":275,"fill":278,"stroke":294,"style":735},[297,1076,275],{"x":1008,"y":1056,"style":306},[272,1078],{"x":1012,"y":1053,"width":1013,"height":1021,"rx":275,"fill":1079,"stroke":1080,"style":311},"#fdefd8","var(--gold,#b4740a)",[297,1082,1084],{"x":1016,"y":1056,"style":1083},"font-size:11px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","no price — gap",[272,1086],{"x":973,"y":281,"width":974,"height":1021,"rx":275,"fill":293,"stroke":294,"style":735},[297,1088,1090],{"x":1024,"y":1089,"style":984},"115.0","3",[272,1092],{"x":978,"y":281,"width":979,"height":1021,"rx":275,"fill":278,"stroke":294,"style":735},[297,1094,1095],{"x":982,"y":1089,"style":306},"9008",[272,1097],{"x":988,"y":281,"width":989,"height":1021,"rx":275,"fill":278,"stroke":294,"style":735},[297,1099,1100],{"x":992,"y":1089,"style":306},"2026-08-15",[272,1102],{"x":996,"y":281,"width":997,"height":1021,"rx":275,"fill":278,"stroke":294,"style":735},[297,1104,1105],{"x":1000,"y":1089,"style":306},"C-300",[272,1107],{"x":1004,"y":281,"width":1005,"height":1021,"rx":275,"fill":278,"stroke":294,"style":735},[297,1109,1057],{"x":1008,"y":1089,"style":306},[272,1111],{"x":1012,"y":281,"width":1013,"height":1021,"rx":275,"fill":1112,"stroke":1113,"style":311},"#fce9e9","var(--danger,#dc2626)",[297,1115,1117],{"x":1016,"y":1089,"style":1116},"font-size:11px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","several prices — overlap",[272,1119],{"x":973,"y":1120,"width":974,"height":1021,"rx":275,"fill":293,"stroke":294,"style":735},"124.0",[297,1122,274],{"x":1024,"y":1123,"style":984},"143.0",[272,1125],{"x":978,"y":1120,"width":979,"height":1021,"rx":275,"fill":278,"stroke":294,"style":735},[297,1127,1128],{"x":982,"y":1123,"style":306},"9009",[272,1130],{"x":988,"y":1120,"width":989,"height":1021,"rx":275,"fill":278,"stroke":294,"style":735},[297,1132,1133],{"x":992,"y":1123,"style":306},"2026-03-03",[272,1135],{"x":996,"y":1120,"width":997,"height":1021,"rx":275,"fill":278,"stroke":294,"style":735},[297,1137,1138],{"x":1000,"y":1123,"style":306},"D-400",[272,1140],{"x":1004,"y":1120,"width":1005,"height":1021,"rx":275,"fill":278,"stroke":294,"style":735},[297,1142,275],{"x":1008,"y":1123,"style":306},[272,1144],{"x":1012,"y":1120,"width":1013,"height":1021,"rx":275,"fill":1079,"stroke":1080,"style":311},[297,1146,1147],{"x":1016,"y":1123,"style":1083},"no price — unknown SKU",[46,1149,1151],{"id":1150},"step-4-check-the-reference-table-itself","Step 4 — Check the reference table itself",[10,1153,1154],{},"Gaps and overlaps come from the price list, so check it directly as well. Sort each SKU's periods and compare each start with the previous period's end:",[51,1156,1158],{"className":97,"code":1157,"language":99,"meta":56,"style":56},"p = prices.sort_values([\"SKU\", \"ValidFrom\"]).copy()\np[\"PrevTo\"] = p.groupby(\"SKU\")[\"ValidTo\"].shift()\np[\"Issue\"] = \"\"\np.loc[p[\"ValidFrom\"] \u003C= p[\"PrevTo\"], \"Issue\"] = \"overlaps previous period\"\np.loc[p[\"ValidFrom\"] > p[\"PrevTo\"] + pd.Timedelta(days=1), \"Issue\"] = \"gap after previous period\"\nprint(p.loc[p[\"Issue\"] != \"\", [\"SKU\", \"ValidFrom\", \"PrevTo\", \"Issue\"]].to_string(index=False))\n",[19,1159,1160,1179,1204,1218,1245,1287],{"__ignoreMap":56},[60,1161,1162,1165,1167,1170,1172,1174,1176],{"class":62,"line":63},[60,1163,1164],{"class":110},"p ",[60,1166,133],{"class":106},[60,1168,1169],{"class":110}," prices.sort_values([",[60,1171,386],{"class":70},[60,1173,142],{"class":110},[60,1175,164],{"class":70},[60,1177,1178],{"class":110},"]).copy()\n",[60,1180,1181,1184,1187,1189,1191,1194,1196,1199,1201],{"class":62,"line":120},[60,1182,1183],{"class":110},"p[",[60,1185,1186],{"class":70},"\"PrevTo\"",[60,1188,217],{"class":110},[60,1190,133],{"class":106},[60,1192,1193],{"class":110}," p.groupby(",[60,1195,386],{"class":70},[60,1197,1198],{"class":110},")[",[60,1200,169],{"class":70},[60,1202,1203],{"class":110},"].shift()\n",[60,1205,1206,1208,1211,1213,1215],{"class":62,"line":127},[60,1207,1183],{"class":110},[60,1209,1210],{"class":70},"\"Issue\"",[60,1212,217],{"class":110},[60,1214,133],{"class":106},[60,1216,1217],{"class":70}," \"\"\n",[60,1219,1220,1223,1225,1227,1229,1232,1234,1236,1238,1240,1242],{"class":62,"line":175},[60,1221,1222],{"class":110},"p.loc[p[",[60,1224,164],{"class":70},[60,1226,217],{"class":110},[60,1228,622],{"class":106},[60,1230,1231],{"class":110}," p[",[60,1233,1186],{"class":70},[60,1235,843],{"class":110},[60,1237,1210],{"class":70},[60,1239,217],{"class":110},[60,1241,133],{"class":106},[60,1243,1244],{"class":70}," \"overlaps previous period\"\n",[60,1246,1247,1249,1251,1253,1256,1258,1260,1262,1265,1268,1271,1273,1275,1278,1280,1282,1284],{"class":62,"line":209},[60,1248,1222],{"class":110},[60,1250,164],{"class":70},[60,1252,217],{"class":110},[60,1254,1255],{"class":106},">",[60,1257,1231],{"class":110},[60,1259,1186],{"class":70},[60,1261,217],{"class":110},[60,1263,1264],{"class":106},"+",[60,1266,1267],{"class":110}," pd.Timedelta(",[60,1269,1270],{"class":145},"days",[60,1272,133],{"class":106},[60,1274,904],{"class":233},[60,1276,1277],{"class":110},"), ",[60,1279,1210],{"class":70},[60,1281,217],{"class":110},[60,1283,133],{"class":106},[60,1285,1286],{"class":70}," \"gap after previous period\"\n",[60,1288,1289,1291,1294,1296,1298,1300,1302,1305,1307,1309,1311,1313,1315,1317,1319,1321,1323,1325,1327],{"class":62,"line":230},[60,1290,234],{"class":233},[60,1292,1293],{"class":110},"(p.loc[p[",[60,1295,1210],{"class":70},[60,1297,217],{"class":110},[60,1299,931],{"class":106},[60,1301,934],{"class":70},[60,1303,1304],{"class":110},", [",[60,1306,386],{"class":70},[60,1308,142],{"class":110},[60,1310,164],{"class":70},[60,1312,142],{"class":110},[60,1314,1186],{"class":70},[60,1316,142],{"class":110},[60,1318,1210],{"class":70},[60,1320,463],{"class":110},[60,1322,240],{"class":145},[60,1324,133],{"class":106},[60,1326,245],{"class":233},[60,1328,248],{"class":110},[10,1330,1331,1332,1336],{},"A clean effective-dated table has each period starting the day after the previous one ends. Running this check whenever the price list is updated catches the problem before any order is priced against it. ",[40,1333,1335],{"href":1334},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-dates-and-times-in-excel-data\u002Ffind-overlapping-date-ranges-with-pandas\u002F","Find Overlapping Date Ranges with pandas"," covers overlap detection in more depth.",[46,1338,1340],{"id":1339},"step-5-price-the-clean-orders-and-report-the-rest","Step 5 — Price the clean orders and report the rest",[10,1342,1343],{},"Use the single matches, compute line values, and write the problems to their own sheet so they get fixed:",[51,1345,1347],{"className":97,"code":1346,"language":99,"meta":56,"style":56},"ok_ids = report.loc[report[\"Problem\"] == \"\", \"OrderID\"]\nlines = priced[priced[\"OrderID\"].isin(ok_ids)].copy()\nlines[\"Value\"] = lines[\"Qty\"] * lines[\"Price\"]\nprint(lines[[\"OrderID\", \"SKU\", \"Qty\", \"Price\", \"Value\"]].to_string(index=False))\nprint(f\"priced {len(lines)} of {len(orders)} orders, value {lines['Value'].sum():,.2f}\")\n\nwith pd.ExcelWriter(\"orders_priced.xlsx\", engine=\"xlsxwriter\", datetime_format=\"dd mmm yyyy\") as xw:\n    lines.drop(columns=[\"ValidTo\"]).to_excel(xw, sheet_name=\"Priced\", index=False)\n    report[report[\"Problem\"] != \"\"].to_excel(xw, sheet_name=\"Needs attention\", index=False)\n    p[p[\"Issue\"] != \"\"].drop(columns=\"PrevTo\").to_excel(xw, sheet_name=\"Price list issues\", index=False)\n",[19,1348,1349,1374,1389,1418,1453,1512,1516,1556,1591,1625],{"__ignoreMap":56},[60,1350,1351,1354,1356,1359,1361,1363,1366,1368,1370,1372],{"class":62,"line":63},[60,1352,1353],{"class":110},"ok_ids ",[60,1355,133],{"class":106},[60,1357,1358],{"class":110}," report.loc[report[",[60,1360,878],{"class":70},[60,1362,217],{"class":110},[60,1364,1365],{"class":106},"==",[60,1367,934],{"class":70},[60,1369,142],{"class":110},[60,1371,443],{"class":70},[60,1373,629],{"class":110},[60,1375,1376,1379,1381,1384,1386],{"class":62,"line":120},[60,1377,1378],{"class":110},"lines ",[60,1380,133],{"class":106},[60,1382,1383],{"class":110}," priced[priced[",[60,1385,443],{"class":70},[60,1387,1388],{"class":110},"].isin(ok_ids)].copy()\n",[60,1390,1391,1394,1397,1399,1401,1404,1407,1409,1412,1414,1416],{"class":62,"line":127},[60,1392,1393],{"class":110},"lines[",[60,1395,1396],{"class":70},"\"Value\"",[60,1398,217],{"class":110},[60,1400,133],{"class":106},[60,1402,1403],{"class":110}," lines[",[60,1405,1406],{"class":70},"\"Qty\"",[60,1408,217],{"class":110},[60,1410,1411],{"class":106},"*",[60,1413,1403],{"class":110},[60,1415,460],{"class":70},[60,1417,629],{"class":110},[60,1419,1420,1422,1425,1427,1429,1431,1433,1435,1437,1439,1441,1443,1445,1447,1449,1451],{"class":62,"line":175},[60,1421,234],{"class":233},[60,1423,1424],{"class":110},"(lines[[",[60,1426,443],{"class":70},[60,1428,142],{"class":110},[60,1430,386],{"class":70},[60,1432,142],{"class":110},[60,1434,1406],{"class":70},[60,1436,142],{"class":110},[60,1438,460],{"class":70},[60,1440,142],{"class":110},[60,1442,1396],{"class":70},[60,1444,463],{"class":110},[60,1446,240],{"class":145},[60,1448,133],{"class":106},[60,1450,245],{"class":233},[60,1452,248],{"class":110},[60,1454,1455,1457,1459,1462,1465,1469,1471,1474,1477,1480,1482,1484,1487,1489,1492,1494,1496,1499,1502,1505,1507,1510],{"class":62,"line":209},[60,1456,234],{"class":233},[60,1458,478],{"class":110},[60,1460,1461],{"class":106},"f",[60,1463,1464],{"class":70},"\"priced ",[60,1466,1468],{"class":1467},"sSjpA","{",[60,1470,481],{"class":233},[60,1472,1473],{"class":110},"(lines)",[60,1475,1476],{"class":1467},"}",[60,1478,1479],{"class":70}," of ",[60,1481,1468],{"class":1467},[60,1483,481],{"class":233},[60,1485,1486],{"class":110},"(orders)",[60,1488,1476],{"class":1467},[60,1490,1491],{"class":70}," orders, value ",[60,1493,1468],{"class":1467},[60,1495,1393],{"class":110},[60,1497,1498],{"class":70},"'Value'",[60,1500,1501],{"class":110},"].sum()",[60,1503,1504],{"class":106},":,.2f",[60,1506,1476],{"class":1467},[60,1508,1509],{"class":70},"\"",[60,1511,399],{"class":110},[60,1513,1514],{"class":62,"line":230},[60,1515,124],{"emptyLinePlaceholder":123},[60,1517,1519,1522,1525,1528,1530,1533,1535,1538,1540,1543,1545,1548,1551,1553],{"class":62,"line":1518},7,[60,1520,1521],{"class":106},"with",[60,1523,1524],{"class":110}," pd.ExcelWriter(",[60,1526,1527],{"class":70},"\"orders_priced.xlsx\"",[60,1529,142],{"class":110},[60,1531,1532],{"class":145},"engine",[60,1534,133],{"class":106},[60,1536,1537],{"class":70},"\"xlsxwriter\"",[60,1539,142],{"class":110},[60,1541,1542],{"class":145},"datetime_format",[60,1544,133],{"class":106},[60,1546,1547],{"class":70},"\"dd mmm yyyy\"",[60,1549,1550],{"class":110},") ",[60,1552,114],{"class":106},[60,1554,1555],{"class":110}," xw:\n",[60,1557,1559,1562,1565,1567,1569,1571,1574,1576,1578,1581,1583,1585,1587,1589],{"class":62,"line":1558},8,[60,1560,1561],{"class":110},"    lines.drop(",[60,1563,1564],{"class":145},"columns",[60,1566,133],{"class":106},[60,1568,161],{"class":110},[60,1570,169],{"class":70},[60,1572,1573],{"class":110},"]).to_excel(xw, ",[60,1575,146],{"class":145},[60,1577,133],{"class":106},[60,1579,1580],{"class":70},"\"Priced\"",[60,1582,142],{"class":110},[60,1584,240],{"class":145},[60,1586,133],{"class":106},[60,1588,245],{"class":233},[60,1590,399],{"class":110},[60,1592,1594,1597,1599,1601,1603,1605,1608,1610,1612,1615,1617,1619,1621,1623],{"class":62,"line":1593},9,[60,1595,1596],{"class":110},"    report[report[",[60,1598,878],{"class":70},[60,1600,217],{"class":110},[60,1602,931],{"class":106},[60,1604,934],{"class":70},[60,1606,1607],{"class":110},"].to_excel(xw, ",[60,1609,146],{"class":145},[60,1611,133],{"class":106},[60,1613,1614],{"class":70},"\"Needs attention\"",[60,1616,142],{"class":110},[60,1618,240],{"class":145},[60,1620,133],{"class":106},[60,1622,245],{"class":233},[60,1624,399],{"class":110},[60,1626,1628,1631,1633,1635,1637,1639,1642,1644,1646,1648,1651,1653,1655,1658,1660,1662,1664,1666],{"class":62,"line":1627},10,[60,1629,1630],{"class":110},"    p[p[",[60,1632,1210],{"class":70},[60,1634,217],{"class":110},[60,1636,931],{"class":106},[60,1638,934],{"class":70},[60,1640,1641],{"class":110},"].drop(",[60,1643,1564],{"class":145},[60,1645,133],{"class":106},[60,1647,1186],{"class":70},[60,1649,1650],{"class":110},").to_excel(xw, ",[60,1652,146],{"class":145},[60,1654,133],{"class":106},[60,1656,1657],{"class":70},"\"Price list issues\"",[60,1659,142],{"class":110},[60,1661,240],{"class":145},[60,1663,133],{"class":106},[60,1665,245],{"class":233},[60,1667,399],{"class":110},[10,1669,1670,1671,1673],{},"Keeping ",[19,1672,21],{}," in the output shows which price period each line used, which is the first thing anyone checking a disputed invoice will ask. The same pattern prices anything effective-dated: tax rates by country, commission plans by salesperson, cost rates by employee grade, discount agreements by customer.",[46,1675,1677],{"id":1676},"performance-notes","Performance notes",[10,1679,1680,1681,1683,1684,1686,1687,1690,1691,1695],{},"The merge-then-filter method multiplies each order by the number of periods for its key. A price list with a dozen periods per SKU and a million orders makes twelve million intermediate rows — several hundred megabytes, workable but slow. ",[19,1682,525],{}," stays at one row per order and runs in roughly the time of a sort, so it is the method for large volumes, as long as the ",[19,1685,25],{}," check follows it and the reference table has been checked for overlaps (Step 4) beforehand. For very large jobs where the data already sits in a database, the same range join in SQL — ",[19,1688,1689],{},"ON o.sku = p.sku AND o.order_date BETWEEN p.valid_from AND p.valid_to"," — lets the database do the work; ",[40,1692,1694],{"href":1693},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fquery-excel-files-with-sql-using-duckdb\u002F","Query Excel Files with SQL Using DuckDB"," shows how to run it directly against workbooks.",[46,1697,1699],{"id":1698},"common-pitfalls","Common pitfalls",[1701,1702,1703,1714,1727,1736,1742],"ul",{},[1704,1705,1706,1710,1711,1713],"li",{},[1707,1708,1709],"strong",{},"Open-ended periods as NaN."," A blank ",[19,1712,25],{}," fails every comparison; fill it with a far-future date first.",[1704,1715,1716,1719,1720,1722,1723,1726],{},[1707,1717,1718],{},"Inclusive versus exclusive ends."," If ",[19,1721,25],{}," means \"valid until, not including\", use ",[19,1724,1725],{},"\u003C"," for the upper bound.",[1704,1728,1729,1732,1733,1735],{},[1707,1730,1731],{},"Trusting merge_asof alone."," It ignores end dates and hides overlaps; always check ",[19,1734,25],{}," and the reference table.",[1704,1737,1738,1741],{},[1707,1739,1740],{},"Unsorted inputs to merge_asof."," Both sides must be sorted by the date column, or it raises.",[1704,1743,1744,1747],{},[1707,1745,1746],{},"Summing without counting matches."," An overlap doubles the row; validate one match per order before totalling.",[46,1749,1751],{"id":1750},"conclusion","Conclusion",[10,1753,1754,1755,1757,1758,1760,1761,1763],{},"Fill open-ended ",[19,1756,25],{}," dates, then either merge on the key and keep rows where the date is ",[19,1759,504],{}," the period's ends, or use ",[19,1762,525],{}," by key and check the end date separately. Count matches per transaction — zero means a gap or a missing key, more than one an overlap — and check the reference table for gaps and overlaps directly. Price the rows with exactly one match and send the rest back to whoever owns the reference data.",[46,1765,1767],{"id":1766},"frequently-asked-questions","Frequently asked questions",[10,1769,1770,1773],{},[1707,1771,1772],{},"How do I merge on a date range in pandas?","\nMerge on the key column, which gives every combination of transaction and period for that key, then keep the rows where the transaction date is between ValidFrom and ValidTo. For large tables, merge_asof on ValidFrom with by= the key, followed by a check against ValidTo, avoids the intermediate combinations.",[10,1775,1776,1779],{},[1707,1777,1778],{},"How do I handle a blank \"valid to\" date?","\nFill it with a far-future date, such as pd.Timestamp.max.normalize(), before comparing. Then open-ended periods behave like any other range.",[10,1781,1782,1785],{},[1707,1783,1784],{},"Why do some rows appear twice after the join?","\nTwo periods for the same key overlap, so the transaction date falls inside both. Fix the reference table so each period starts the day after the previous one ends, and count matches per transaction to catch it.",[10,1787,1788,1791],{},[1707,1789,1790],{},"Can merge_asof handle valid_to dates?","\nNo. It matches on the start date only and returns the last period that started before the transaction, even if that period has ended. Compare the transaction date with the matched ValidTo afterwards and treat failures as unmatched.",[46,1793,1795],{"id":1794},"related","Related",[1701,1797,1798,1804,1811,1818],{},[1704,1799,1800,1801,1803],{},"Up one level: ",[40,1802,43],{"href":42}," — every way to combine tables.",[1704,1805,1806,1810],{},[40,1807,1809],{"href":1808},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fmatch-rows-by-nearest-date-with-merge-asof\u002F","Match Rows by Nearest Date with merge_asof"," — merge_asof in full.",[1704,1812,1813,1817],{},[40,1814,1816],{"href":1815},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fvalidate-a-merge-to-avoid-duplicate-rows\u002F","Validate a Merge to Avoid Duplicate Rows"," — catching unexpected many-to-one joins.",[1704,1819,1820,1824],{},[40,1821,1823],{"href":1822},"\u002Fautomating-reporting-workflows\u002Fbuilding-finance-and-operations-reports\u002Fconvert-multi-currency-transactions-with-an-fx-rate-table\u002F","Convert Multi-Currency Transactions with an FX Rate Table"," — a daily rate table joined by date.",[1826,1827,1828],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}",{"title":56,"searchDepth":120,"depth":120,"links":1830},[1831,1832,1833,1834,1835,1836,1837,1838,1839,1840,1841],{"id":48,"depth":120,"text":49},{"id":359,"depth":120,"text":360},{"id":519,"depth":120,"text":520},{"id":814,"depth":120,"text":815},{"id":1150,"depth":120,"text":1151},{"id":1339,"depth":120,"text":1340},{"id":1676,"depth":120,"text":1677},{"id":1698,"depth":120,"text":1699},{"id":1750,"depth":120,"text":1751},{"id":1766,"depth":120,"text":1767},{"id":1794,"depth":120,"text":1795},"2026-09-29","Look up the price, rate or rule that was in force on each row's date: a price list with valid-from and valid-to dates joined to orders, open-ended ranges, the merge-and-filter method, merge_asof with a validity check, orders that fall in a gap, overlapping price periods that match twice, and a check that every order got exactly one price.","md",[1846,1848,1850,1852],{"q":1772,"a":1847},"Merge on the key column, which gives every combination of transaction and period for that key, then keep the rows where the transaction date is between ValidFrom and ValidTo. For large tables, merge_asof on ValidFrom with by= the key, followed by a check against ValidTo, avoids the intermediate combinations.",{"q":1778,"a":1849},"Fill it with a far-future date, such as pd.Timestamp.max.normalize(), before comparing. Then open-ended periods behave like any other range.",{"q":1784,"a":1851},"Two periods for the same key overlap, so the transaction date falls inside both. Fix the reference table so each period starts the day after the previous one ends, and count matches per transaction to catch it.",{"q":1790,"a":1853},"No. It matches on the start date only and returns the last period that started before the transaction, even if that period has ended. Compare the transaction date with the matched ValidTo afterwards and treat failures as unmatched.","join rows to effective date ranges with pandas",null,{"breadcrumb":1857},[1858,1861,1864],{"name":1859,"item":1860},"Home","\u002F",{"name":1862,"item":1863},"Advanced Data Transformation and Cleaning for Excel Automation","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":1865,"item":42},"Merging and Joining Excel DataFrames with Pandas","\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fjoin-rows-to-effective-date-ranges-with-pandas",{"title":1868,"description":1869},"Join on Date Ranges (Valid From\u002FTo) in pandas","Match each order to the price, tax rate or commission plan effective on its date in pandas: merge then filter between valid_from and valid_to, merge_asof by key with a valid_to check, open-ended ranges, gaps and overlaps, validated to one match per row.","join-rows-to-effective-date-ranges-with-pandas","advanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fjoin-rows-to-effective-date-ranges-with-pandas\u002Findex",[1873,1874],"joins-and-lookups","dates-and-times","how-to","Vj9ce6xEiqaE77okRxa6-4MKEpLUkBb7W_FrDCYqpig",[1878,1882],{"title":1879,"path":1880,"stem":1881,"children":-1},"Handle Many-to-Many Lookups Without Duplicating Rows","\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fhandle-many-to-many-lookups-without-duplicating-rows","advanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Fhandle-many-to-many-lookups-without-duplicating-rows\u002Findex",{"title":1883,"path":1884,"stem":1885,"children":-1},"Look Up All Matches for a Key Instead of the First","\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Flook-up-all-matches-for-a-key-instead-of-the-first","advanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes\u002Flook-up-all-matches-for-a-key-instead-of-the-first\u002Findex",[1887,1889],{"path":1888,"title":1862},"\u002Fadvanced-data-transformation-and-cleaning",{"path":1890,"title":1865},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmerging-and-joining-excel-dataframes",[],[1893,1896,1898,1901,1904,1907],{"path":1894,"title":1895,"kind":1875},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fcompare-two-excel-files-for-differences-with-python","Compare Two Excel Files for Differences with Python",{"path":1897,"title":1823,"kind":1875},"\u002Fautomating-reporting-workflows\u002Fbuilding-finance-and-operations-reports\u002Fconvert-multi-currency-transactions-with-an-fx-rate-table",{"path":1899,"title":1900,"kind":1875},"\u002Fadvanced-data-transformation-and-cleaning\u002Fapplying-conditional-formatting-with-openpyxl\u002Fhighlight-differences-between-two-columns-with-openpyxl","Highlight Differences Between Two Columns with openpyxl",{"path":1902,"title":1903,"kind":1875},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fmap-free-text-job-titles-to-standard-categories","Map Free-Text Job Titles to Standard Categories",{"path":1905,"title":1906,"kind":1875},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fstandardise-category-values-with-a-mapping-table","Standardise Category Values with a Mapping Table in pandas",{"path":1908,"title":1909,"kind":1875},"\u002Fadvanced-data-transformation-and-cleaning\u002Fexcel-formula-equivalents-in-pandas\u002Fapproximate-match-vlookup-in-pandas","Approximate-Match VLOOKUP for Tiered Rates in pandas",1790972096284]