[{"data":1,"prerenderedAt":2583},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fconvert-excel-text-columns-to-numbers-with-pandas":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fconvert-excel-text-columns-to-numbers-with-pandas":2574},{"id":4,"title":5,"body":6,"dateModified":2548,"datePublished":2548,"description":2549,"extension":2550,"faq":2551,"meta":2565,"navigation":210,"path":2566,"seo":2567,"slug":2570,"stem":2571,"type":2572,"__hash__":2573},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fconvert-excel-text-columns-to-numbers-with-pandas\u002Findex.md","Convert Excel Text Columns to Numbers with pandas",{"type":7,"value":8,"toc":2535},"minimark",[9,24,146,151,179,182,295,299,305,511,526,530,533,582,592,596,715,721,1211,1235,1238,1423,1426,1430,1433,1527,1825,1836,1840,1851,1961,1973,1976,2003,2010,2014,2158,2162,2171,2178,2261,2267,2383,2386,2401,2405,2417,2421,2437,2452,2460,2469,2487,2491,2531],[10,11,12,13,17,18,23],"p",{},"A column of prices comes out of Excel and ",[14,15,16],"code",{},"df[\"revenue\"].sum()"," returns one enormous string. The values look like numbers, right-aligned even, but they are text — because the export wrote them as text, or because a currency symbol, a thousands separator or a stray space made pandas give up on inference. This guide covers converting them properly: the cleaning steps in the right order, the formats that need special handling, and the reporting that stops a bad conversion from silently understating a total. It is part of ",[19,20,22],"a",{"href":21},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002F","Cleaning Excel Data with pandas",".",[25,26,35,36,35,40,35,44,35,51,35,58,35,62,35,71,35,77,35,83,35,88,35,93,35,98,35,100,35,103,35,108,35,113,35,115,35,118,35,123,35,128,35,130,35,133,35,136,35,141,35,143],"svg",{"viewBox":27,"role":28,"ariaLabel":29,"ariaLabelledBy":30,"xmlns":33,"style":34},"0 0 800 254","img","Five text representations of numbers and what each needs: a currency symbol and separator, a European decimal comma, a parenthesised negative, a percentage, and a plain number.",[31,32],"num-t","num-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:800px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[37,38,39],"title",{"id":31},"Five text formats, and what each conversion needs",[41,42,43],"desc",{"id":32},"Five rows pairing a text value with the handling it requires. A dollar amount with a thousands comma needs the symbol and separator stripped. A European figure with dots for thousands and a comma decimal needs the dots removed and the comma converted, in that order. A parenthesised value is an accounting negative and must be negated after stripping. A percentage must be divided by one hundred. A plain numeric string converts directly.",[45,46],"rect",{"x":47,"y":47,"width":48,"height":49,"fill":50},"0","800","254","#ffffff",[52,53,57],"text",{"x":54,"y":55,"style":56},"200","30","font-size:12px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:middle","as stored (text)",[52,59,61],{"x":60,"y":55,"style":56},"580","what it needs",[45,63],{"x":55,"y":64,"width":65,"height":66,"rx":67,"fill":68,"stroke":69,"style":70},"44","336","34","6","#ebebfd","var(--brand,#5b5cf0)","stroke-width:2px",[52,72,76],{"x":73,"y":74,"style":75},"198","66","font-size:11.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","\"$1,234.56\"",[45,78],{"x":79,"y":64,"width":80,"height":66,"rx":67,"fill":81,"stroke":82},"392","378","#f0f2f5","var(--line,#cdd5e6)",[52,84,87],{"x":85,"y":74,"style":86},"581","font-size:11px;fill:var(--text,#172033);text-anchor:middle","strip the symbol and the comma",[45,89],{"x":55,"y":90,"width":65,"height":66,"rx":67,"fill":91,"stroke":92,"style":70},"86","#fdefd8","var(--gold,#b4740a)",[52,94,97],{"x":73,"y":95,"style":96},"108","font-size:11.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","\"1.234,56\"",[45,99],{"x":79,"y":90,"width":80,"height":66,"rx":67,"fill":81,"stroke":82},[52,101,102],{"x":85,"y":95,"style":86},"remove dots FIRST, then comma to dot",[45,104],{"x":55,"y":105,"width":65,"height":66,"rx":67,"fill":106,"stroke":107,"style":70},"128","#fee8f2","var(--accent,#f43f8f)",[52,109,112],{"x":73,"y":110,"style":111},"150","font-size:11.5px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","\"(1,000.00)\"",[45,114],{"x":79,"y":105,"width":80,"height":66,"rx":67,"fill":81,"stroke":82},[52,116,117],{"x":85,"y":110,"style":86},"accounting negative — strip, then negate",[45,119],{"x":55,"y":120,"width":65,"height":66,"rx":67,"fill":121,"stroke":122,"style":70},"170","#d9f4f1","var(--teal,#0f9488)",[52,124,127],{"x":73,"y":125,"style":126},"192","font-size:11.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","\"12.5%\"",[45,129],{"x":79,"y":120,"width":80,"height":66,"rx":67,"fill":81,"stroke":82},[52,131,132],{"x":85,"y":125,"style":86},"strip the sign and divide by 100",[45,134],{"x":55,"y":135,"width":65,"height":66,"rx":67,"fill":50,"stroke":82,"style":70},"212",[52,137,140],{"x":73,"y":138,"style":139},"234","font-size:11.5px;fill:var(--text,#172033);text-anchor:middle","\"4268.5\"",[45,142],{"x":79,"y":135,"width":80,"height":66,"rx":67,"fill":81,"stroke":82},[52,144,145],{"x":85,"y":138,"style":86},"to_numeric handles it directly",[147,148,150],"h2",{"id":149},"prerequisites","Prerequisites",[152,153,158],"pre",{"className":154,"code":155,"language":156,"meta":157,"style":157},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl\n","bash","",[14,159,160],{"__ignoreMap":157},[161,162,165,169,173,176],"span",{"class":163,"line":164},"line",1,[161,166,168],{"class":167},"sMTad","pip",[161,170,172],{"class":171},"srMev"," install",[161,174,175],{"class":171}," pandas",[161,177,178],{"class":171}," openpyxl\n",[10,180,181],{},"A frame with each of the awkward formats:",[152,183,187],{"className":184,"code":185,"language":186,"meta":157,"style":157},"language-python shiki shiki-themes github-light github-dark-high-contrast","import pandas as pd\n\ndf = pd.DataFrame({\n    \"revenue\": [\"$1,234.56\", \"1.234,56\", \"(1,000.00)\", \"12.5%\", \"4268.5\",\n                \"n\u002Fa\", \"\"],\n})\nprint(df[\"revenue\"].dtype)      # object\n","python",[14,188,189,205,212,224,255,269,275],{"__ignoreMap":157},[161,190,191,195,199,202],{"class":163,"line":164},[161,192,194],{"class":193},"s-kum","import",[161,196,198],{"class":197},"skGVy"," pandas ",[161,200,201],{"class":193},"as",[161,203,204],{"class":197}," pd\n",[161,206,208],{"class":163,"line":207},2,[161,209,211],{"emptyLinePlaceholder":210},true,"\n",[161,213,215,218,221],{"class":163,"line":214},3,[161,216,217],{"class":197},"df ",[161,219,220],{"class":193},"=",[161,222,223],{"class":197}," pd.DataFrame({\n",[161,225,227,230,233,235,238,240,242,244,246,248,250,252],{"class":163,"line":226},4,[161,228,229],{"class":171},"    \"revenue\"",[161,231,232],{"class":197},": [",[161,234,76],{"class":171},[161,236,237],{"class":197},", ",[161,239,97],{"class":171},[161,241,237],{"class":197},[161,243,112],{"class":171},[161,245,237],{"class":197},[161,247,127],{"class":171},[161,249,237],{"class":197},[161,251,140],{"class":171},[161,253,254],{"class":197},",\n",[161,256,258,261,263,266],{"class":163,"line":257},5,[161,259,260],{"class":171},"                \"n\u002Fa\"",[161,262,237],{"class":197},[161,264,265],{"class":171},"\"\"",[161,267,268],{"class":197},"],\n",[161,270,272],{"class":163,"line":271},6,[161,273,274],{"class":197},"})\n",[161,276,278,282,285,288,291],{"class":163,"line":277},7,[161,279,281],{"class":280},"sP0c6","print",[161,283,284],{"class":197},"(df[",[161,286,287],{"class":171},"\"revenue\"",[161,289,290],{"class":197},"].dtype)      ",[161,292,294],{"class":293},"s-wDw","# object\n",[147,296,298],{"id":297},"step-1-confirm-the-problem","Step 1 — Confirm the problem",[10,300,301,304],{},[14,302,303],{},"dtypes"," is the first thing to look at when arithmetic behaves oddly:",[152,306,308],{"className":184,"code":307,"language":186,"meta":157,"style":157},"import pandas as pd\n\ndf = pd.read_excel(\"sales.xlsx\")\nprint(df.dtypes)\n\nsuspect = [\n    name for name in df.columns\n    if df[name].dtype == \"object\"\n    and df[name].astype(\"string\").str.match(r\"^\\s*[\\$€£(]?[\\d.,\\s]+\\)?%?\\s*$\",\n                                            na=False).mean() > 0.8\n]\nprint(\"probably numeric but stored as text:\", suspect)\n",[14,309,310,320,324,339,346,350,360,377,392,470,491,497],{"__ignoreMap":157},[161,311,312,314,316,318],{"class":163,"line":164},[161,313,194],{"class":193},[161,315,198],{"class":197},[161,317,201],{"class":193},[161,319,204],{"class":197},[161,321,322],{"class":163,"line":207},[161,323,211],{"emptyLinePlaceholder":210},[161,325,326,328,330,333,336],{"class":163,"line":214},[161,327,217],{"class":197},[161,329,220],{"class":193},[161,331,332],{"class":197}," pd.read_excel(",[161,334,335],{"class":171},"\"sales.xlsx\"",[161,337,338],{"class":197},")\n",[161,340,341,343],{"class":163,"line":226},[161,342,281],{"class":280},[161,344,345],{"class":197},"(df.dtypes)\n",[161,347,348],{"class":163,"line":257},[161,349,211],{"emptyLinePlaceholder":210},[161,351,352,355,357],{"class":163,"line":271},[161,353,354],{"class":197},"suspect ",[161,356,220],{"class":193},[161,358,359],{"class":197}," [\n",[161,361,362,365,368,371,374],{"class":163,"line":277},[161,363,364],{"class":197},"    name ",[161,366,367],{"class":193},"for",[161,369,370],{"class":197}," name ",[161,372,373],{"class":193},"in",[161,375,376],{"class":197}," df.columns\n",[161,378,380,383,386,389],{"class":163,"line":379},8,[161,381,382],{"class":193},"    if",[161,384,385],{"class":197}," df[name].dtype ",[161,387,388],{"class":193},"==",[161,390,391],{"class":171}," \"object\"\n",[161,393,395,398,401,404,407,410,413,416,419,422,426,430,433,436,439,442,445,448,451,453,456,458,461,463,466,468],{"class":163,"line":394},9,[161,396,397],{"class":193},"    and",[161,399,400],{"class":197}," df[name].astype(",[161,402,403],{"class":171},"\"string\"",[161,405,406],{"class":197},").str.match(",[161,408,409],{"class":193},"r",[161,411,412],{"class":171},"\"",[161,414,415],{"class":280},"^\\s",[161,417,418],{"class":193},"*",[161,420,421],{"class":280},"[",[161,423,425],{"class":424},"s_b0D","\\$",[161,427,429],{"class":428},"sSjpA","€£(",[161,431,432],{"class":280},"]",[161,434,435],{"class":193},"?",[161,437,438],{"class":280},"[\\d",[161,440,441],{"class":428},".,",[161,443,444],{"class":280},"\\s]",[161,446,447],{"class":193},"+",[161,449,450],{"class":424},"\\)",[161,452,435],{"class":193},[161,454,455],{"class":171},"%",[161,457,435],{"class":193},[161,459,460],{"class":280},"\\s",[161,462,418],{"class":193},[161,464,465],{"class":280},"$",[161,467,412],{"class":171},[161,469,254],{"class":197},[161,471,473,477,479,482,485,488],{"class":163,"line":472},10,[161,474,476],{"class":475},"sa561","                                            na",[161,478,220],{"class":193},[161,480,481],{"class":280},"False",[161,483,484],{"class":197},").mean() ",[161,486,487],{"class":193},">",[161,489,490],{"class":280}," 0.8\n",[161,492,494],{"class":163,"line":493},11,[161,495,496],{"class":197},"]\n",[161,498,500,502,505,508],{"class":163,"line":499},12,[161,501,281],{"class":280},[161,503,504],{"class":197},"(",[161,506,507],{"class":171},"\"probably numeric but stored as text:\"",[161,509,510],{"class":197},", suspect)\n",[10,512,513,514,517,518,521,522,23],{},"The ",[14,515,516],{},"mean() > 0.8"," test asks whether most values in the column look numeric. That is more useful than checking one value, because a genuinely textual column will score near zero while a numeric column with a few ",[14,519,520],{},"n\u002Fa"," entries still scores high. The broader dtype audit is covered in ",[19,523,525],{"href":524},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fcheck-excel-data-types-with-pandas\u002F","checking Excel data types with pandas",[147,527,529],{"id":528},"step-2-the-simple-case","Step 2 — The simple case",[10,531,532],{},"When the values are clean numeric strings, one call does it:",[152,534,536],{"className":184,"code":535,"language":186,"meta":157,"style":157},"import pandas as pd\n\ndf[\"revenue\"] = pd.to_numeric(df[\"revenue\"], errors=\"coerce\")\n",[14,537,538,548,552],{"__ignoreMap":157},[161,539,540,542,544,546],{"class":163,"line":164},[161,541,194],{"class":193},[161,543,198],{"class":197},[161,545,201],{"class":193},[161,547,204],{"class":197},[161,549,550],{"class":163,"line":207},[161,551,211],{"emptyLinePlaceholder":210},[161,553,554,557,559,562,564,567,569,572,575,577,580],{"class":163,"line":214},[161,555,556],{"class":197},"df[",[161,558,287],{"class":171},[161,560,561],{"class":197},"] ",[161,563,220],{"class":193},[161,565,566],{"class":197}," pd.to_numeric(df[",[161,568,287],{"class":171},[161,570,571],{"class":197},"], ",[161,573,574],{"class":475},"errors",[161,576,220],{"class":193},[161,578,579],{"class":171},"\"coerce\"",[161,581,338],{"class":197},[10,583,584,587,588,591],{},[14,585,586],{},"errors=\"coerce\""," turns anything unparseable into ",[14,589,590],{},"NaN"," instead of raising. That is right for a batch job — and dangerous on its own, because a column where half the values failed still sums happily, just to the wrong number.",[147,593,595],{"id":594},"step-3-clean-the-formats-in-order","Step 3 — Clean the formats, in order",[25,597,35,603,35,606,35,609,35,612,35,619,35,625,35,630,35,635,35,640,35,643,35,647,35,651,35,654,35,657,35,660,35,664,35,668,35,671,35,674,35,677,35,681,35,685,35,688,35,691,35,694,35,698,35,702,35,705,35,710],{"viewBox":598,"role":28,"ariaLabel":599,"ariaLabelledBy":600,"xmlns":33,"style":34},"0 0 800 228","The cleaning pipeline in order: detect accounting parentheses, note and strip percent signs, remove currency symbols, then handle the decimal convention before converting.",[601,602],"order-t","order-d",[37,604,605],{"id":601},"Five cleaning steps that must happen in this order",[41,607,608],{"id":602},"A five-stage pipeline. First accounting parentheses are detected and recorded before being stripped, because the information they carry is the sign. Second percentage signs are noted and removed so the value can be scaled afterwards. Third currency symbols and stray characters are stripped. Fourth the decimal convention is resolved — for European numbers the thousands dots must be removed before the comma is converted to a dot, never the reverse. Only then does to_numeric run.",[45,610],{"x":47,"y":47,"width":48,"height":611,"fill":50},"228",[45,613],{"x":614,"y":615,"width":616,"height":617,"rx":618,"fill":106,"stroke":107,"style":70},"14","72","140","76","12",[52,620,624],{"x":621,"y":622,"style":623},"84","56","font-size:10.5px;font-weight:700;fill:var(--muted,#5b6780);text-anchor:middle","1",[52,626,629],{"x":621,"y":627,"style":628},"100","font-size:11px;font-weight:700;fill:var(--accent-ink,#be185d);text-anchor:middle","parentheses",[52,631,634],{"x":621,"y":632,"style":633},"122","font-size:10px;fill:var(--muted,#5b6780);text-anchor:middle","record the sign",[636,637],"polygon",{"points":638,"fill":639},"170,110 158,104 158,116","#5b5cf0",[45,641],{"x":642,"y":615,"width":616,"height":617,"rx":618,"fill":121,"stroke":122,"style":70},"176",[52,644,646],{"x":645,"y":622,"style":623},"246","2",[52,648,650],{"x":645,"y":627,"style":649},"font-size:11px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","percent sign",[52,652,653],{"x":645,"y":632,"style":633},"note, then strip",[636,655],{"points":656,"fill":639},"332,110 320,104 320,116",[45,658],{"x":659,"y":615,"width":616,"height":617,"rx":618,"fill":68,"stroke":69,"style":70},"338",[52,661,663],{"x":662,"y":622,"style":623},"408","3",[52,665,667],{"x":662,"y":627,"style":666},"font-size:11px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","symbols",[52,669,670],{"x":662,"y":632,"style":633},"currency, spaces",[636,672],{"points":673,"fill":639},"494,110 482,104 482,116",[45,675],{"x":676,"y":615,"width":616,"height":617,"rx":618,"fill":91,"stroke":92,"style":70},"500",[52,678,680],{"x":679,"y":622,"style":623},"570","4",[52,682,684],{"x":679,"y":627,"style":683},"font-size:11px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","separators",[52,686,687],{"x":679,"y":632,"style":633},"dots BEFORE comma",[636,689],{"points":690,"fill":639},"656,110 644,104 644,116",[45,692],{"x":693,"y":615,"width":632,"height":617,"rx":618,"fill":81,"stroke":82,"style":70},"662",[52,695,697],{"x":696,"y":622,"style":623},"723","5",[52,699,701],{"x":696,"y":627,"style":700},"font-size:11px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","to_numeric",[52,703,704],{"x":696,"y":632,"style":633},"coerce",[52,706,709],{"x":707,"y":708,"style":111},"400","184","reverse steps 1 and 5 and the sign is lost; reverse the two halves of step 4 and the decimal is",[52,711,714],{"x":707,"y":712,"style":713},"206","font-size:10.5px;fill:var(--muted,#5b6780);text-anchor:middle","\"1.234,56\" → remove dots → \"1234,56\" → comma to dot → 1234.56",[10,716,717,718,720],{},"Currency symbols, separators and accounting negatives all need removing before ",[14,719,701],{}," sees the value. The order is not arbitrary:",[152,722,724],{"className":184,"code":723,"language":186,"meta":157,"style":157},"import pandas as pd\n\ndef to_number(series, decimal=\".\", percent_as_fraction=True):\n    \"\"\"Convert a text column of formatted numbers into real numerics.\"\"\"\n    text = series.astype(\"string\").str.strip()\n\n    # Accounting negatives: (1,000.00) means -1000.00\n    negative = text.str.match(r\"^\\(.*\\)$\", na=False)\n    text = text.str.replace(r\"^\\((.*)\\)$\", r\"\\1\", regex=True)\n\n    # Percentages: remember which, so we can scale afterwards.\n    percent = text.str.endswith(\"%\", na=False)\n    text = text.str.rstrip(\"%\")\n\n    # Currency symbols and spaces, including the non-breaking kind.\n    text = text.str.replace(r\"[^\\d.,\\-+eE]\", \"\", regex=True)\n\n    if decimal == \",\":\n        # European: dots are thousands separators, comma is the decimal.\n        text = text.str.replace(\".\", \"\", regex=False)\n        text = text.str.replace(\",\", \".\", regex=False)\n    else:\n        text = text.str.replace(\",\", \"\", regex=False)\n\n    numbers = pd.to_numeric(text, errors=\"coerce\")\n    numbers = numbers.where(~negative, -numbers)\n    if percent_as_fraction:\n        numbers = numbers.where(~percent, numbers \u002F 100)\n    return numbers\n\nprint(to_number(df[\"revenue\"]).tolist())\n",[14,725,726,736,740,768,773,788,792,797,838,892,896,901,924,938,943,949,995,1000,1016,1022,1048,1074,1082,1107,1112,1131,1153,1161,1184,1193,1198],{"__ignoreMap":157},[161,727,728,730,732,734],{"class":163,"line":164},[161,729,194],{"class":193},[161,731,198],{"class":197},[161,733,201],{"class":193},[161,735,204],{"class":197},[161,737,738],{"class":163,"line":207},[161,739,211],{"emptyLinePlaceholder":210},[161,741,742,745,749,752,754,757,760,762,765],{"class":163,"line":214},[161,743,744],{"class":193},"def",[161,746,748],{"class":747},"s_Opv"," to_number",[161,750,751],{"class":197},"(series, decimal",[161,753,220],{"class":193},[161,755,756],{"class":171},"\".\"",[161,758,759],{"class":197},", percent_as_fraction",[161,761,220],{"class":193},[161,763,764],{"class":280},"True",[161,766,767],{"class":197},"):\n",[161,769,770],{"class":163,"line":226},[161,771,772],{"class":171},"    \"\"\"Convert a text column of formatted numbers into real numerics.\"\"\"\n",[161,774,775,778,780,783,785],{"class":163,"line":257},[161,776,777],{"class":197},"    text ",[161,779,220],{"class":193},[161,781,782],{"class":197}," series.astype(",[161,784,403],{"class":171},[161,786,787],{"class":197},").str.strip()\n",[161,789,790],{"class":163,"line":271},[161,791,211],{"emptyLinePlaceholder":210},[161,793,794],{"class":163,"line":277},[161,795,796],{"class":293},"    # Accounting negatives: (1,000.00) means -1000.00\n",[161,798,799,802,804,807,809,811,814,817,819,821,823,825,827,829,832,834,836],{"class":163,"line":379},[161,800,801],{"class":197},"    negative ",[161,803,220],{"class":193},[161,805,806],{"class":197}," text.str.match(",[161,808,409],{"class":193},[161,810,412],{"class":171},[161,812,813],{"class":280},"^",[161,815,816],{"class":424},"\\(",[161,818,23],{"class":280},[161,820,418],{"class":193},[161,822,450],{"class":424},[161,824,465],{"class":280},[161,826,412],{"class":171},[161,828,237],{"class":197},[161,830,831],{"class":475},"na",[161,833,220],{"class":193},[161,835,481],{"class":280},[161,837,338],{"class":197},[161,839,840,842,844,847,849,851,853,855,858,860,863,865,867,869,871,873,875,879,881,883,886,888,890],{"class":163,"line":394},[161,841,777],{"class":197},[161,843,220],{"class":193},[161,845,846],{"class":197}," text.str.replace(",[161,848,409],{"class":193},[161,850,412],{"class":171},[161,852,813],{"class":280},[161,854,816],{"class":424},[161,856,857],{"class":280},"(.",[161,859,418],{"class":193},[161,861,862],{"class":280},")",[161,864,450],{"class":424},[161,866,465],{"class":280},[161,868,412],{"class":171},[161,870,237],{"class":197},[161,872,409],{"class":193},[161,874,412],{"class":171},[161,876,878],{"class":877},"sx1sP","\\1",[161,880,412],{"class":171},[161,882,237],{"class":197},[161,884,885],{"class":475},"regex",[161,887,220],{"class":193},[161,889,764],{"class":280},[161,891,338],{"class":197},[161,893,894],{"class":163,"line":472},[161,895,211],{"emptyLinePlaceholder":210},[161,897,898],{"class":163,"line":493},[161,899,900],{"class":293},"    # Percentages: remember which, so we can scale afterwards.\n",[161,902,903,906,908,911,914,916,918,920,922],{"class":163,"line":499},[161,904,905],{"class":197},"    percent ",[161,907,220],{"class":193},[161,909,910],{"class":197}," text.str.endswith(",[161,912,913],{"class":171},"\"%\"",[161,915,237],{"class":197},[161,917,831],{"class":475},[161,919,220],{"class":193},[161,921,481],{"class":280},[161,923,338],{"class":197},[161,925,927,929,931,934,936],{"class":163,"line":926},13,[161,928,777],{"class":197},[161,930,220],{"class":193},[161,932,933],{"class":197}," text.str.rstrip(",[161,935,913],{"class":171},[161,937,338],{"class":197},[161,939,941],{"class":163,"line":940},14,[161,942,211],{"emptyLinePlaceholder":210},[161,944,946],{"class":163,"line":945},15,[161,947,948],{"class":293},"    # Currency symbols and spaces, including the non-breaking kind.\n",[161,950,952,954,956,958,960,962,964,966,969,971,974,977,979,981,983,985,987,989,991,993],{"class":163,"line":951},16,[161,953,777],{"class":197},[161,955,220],{"class":193},[161,957,846],{"class":197},[161,959,409],{"class":193},[161,961,412],{"class":171},[161,963,421],{"class":280},[161,965,813],{"class":193},[161,967,968],{"class":280},"\\d",[161,970,441],{"class":428},[161,972,973],{"class":424},"\\-",[161,975,976],{"class":428},"+eE",[161,978,432],{"class":280},[161,980,412],{"class":171},[161,982,237],{"class":197},[161,984,265],{"class":171},[161,986,237],{"class":197},[161,988,885],{"class":475},[161,990,220],{"class":193},[161,992,764],{"class":280},[161,994,338],{"class":197},[161,996,998],{"class":163,"line":997},17,[161,999,211],{"emptyLinePlaceholder":210},[161,1001,1003,1005,1008,1010,1013],{"class":163,"line":1002},18,[161,1004,382],{"class":193},[161,1006,1007],{"class":197}," decimal ",[161,1009,388],{"class":193},[161,1011,1012],{"class":171}," \",\"",[161,1014,1015],{"class":197},":\n",[161,1017,1019],{"class":163,"line":1018},19,[161,1020,1021],{"class":293},"        # European: dots are thousands separators, comma is the decimal.\n",[161,1023,1025,1028,1030,1032,1034,1036,1038,1040,1042,1044,1046],{"class":163,"line":1024},20,[161,1026,1027],{"class":197},"        text ",[161,1029,220],{"class":193},[161,1031,846],{"class":197},[161,1033,756],{"class":171},[161,1035,237],{"class":197},[161,1037,265],{"class":171},[161,1039,237],{"class":197},[161,1041,885],{"class":475},[161,1043,220],{"class":193},[161,1045,481],{"class":280},[161,1047,338],{"class":197},[161,1049,1051,1053,1055,1057,1060,1062,1064,1066,1068,1070,1072],{"class":163,"line":1050},21,[161,1052,1027],{"class":197},[161,1054,220],{"class":193},[161,1056,846],{"class":197},[161,1058,1059],{"class":171},"\",\"",[161,1061,237],{"class":197},[161,1063,756],{"class":171},[161,1065,237],{"class":197},[161,1067,885],{"class":475},[161,1069,220],{"class":193},[161,1071,481],{"class":280},[161,1073,338],{"class":197},[161,1075,1077,1080],{"class":163,"line":1076},22,[161,1078,1079],{"class":193},"    else",[161,1081,1015],{"class":197},[161,1083,1085,1087,1089,1091,1093,1095,1097,1099,1101,1103,1105],{"class":163,"line":1084},23,[161,1086,1027],{"class":197},[161,1088,220],{"class":193},[161,1090,846],{"class":197},[161,1092,1059],{"class":171},[161,1094,237],{"class":197},[161,1096,265],{"class":171},[161,1098,237],{"class":197},[161,1100,885],{"class":475},[161,1102,220],{"class":193},[161,1104,481],{"class":280},[161,1106,338],{"class":197},[161,1108,1110],{"class":163,"line":1109},24,[161,1111,211],{"emptyLinePlaceholder":210},[161,1113,1115,1118,1120,1123,1125,1127,1129],{"class":163,"line":1114},25,[161,1116,1117],{"class":197},"    numbers ",[161,1119,220],{"class":193},[161,1121,1122],{"class":197}," pd.to_numeric(text, ",[161,1124,574],{"class":475},[161,1126,220],{"class":193},[161,1128,579],{"class":171},[161,1130,338],{"class":197},[161,1132,1134,1136,1138,1141,1144,1147,1150],{"class":163,"line":1133},26,[161,1135,1117],{"class":197},[161,1137,220],{"class":193},[161,1139,1140],{"class":197}," numbers.where(",[161,1142,1143],{"class":193},"~",[161,1145,1146],{"class":197},"negative, ",[161,1148,1149],{"class":193},"-",[161,1151,1152],{"class":197},"numbers)\n",[161,1154,1156,1158],{"class":163,"line":1155},27,[161,1157,382],{"class":193},[161,1159,1160],{"class":197}," percent_as_fraction:\n",[161,1162,1164,1167,1169,1171,1173,1176,1179,1182],{"class":163,"line":1163},28,[161,1165,1166],{"class":197},"        numbers ",[161,1168,220],{"class":193},[161,1170,1140],{"class":197},[161,1172,1143],{"class":193},[161,1174,1175],{"class":197},"percent, numbers ",[161,1177,1178],{"class":193},"\u002F",[161,1180,1181],{"class":280}," 100",[161,1183,338],{"class":197},[161,1185,1187,1190],{"class":163,"line":1186},29,[161,1188,1189],{"class":193},"    return",[161,1191,1192],{"class":197}," numbers\n",[161,1194,1196],{"class":163,"line":1195},30,[161,1197,211],{"emptyLinePlaceholder":210},[161,1199,1201,1203,1206,1208],{"class":163,"line":1200},31,[161,1202,281],{"class":280},[161,1204,1205],{"class":197},"(to_number(df[",[161,1207,287],{"class":171},[161,1209,1210],{"class":197},"]).tolist())\n",[10,1212,1213,1214,1218,1219,1222,1223,1226,1227,1230,1231,1234],{},"The European branch is where order matters most. Removing dots ",[1215,1216,1217],"em",{},"before"," converting the comma turns ",[14,1220,1221],{},"1.234,56"," into ",[14,1224,1225],{},"1234,56"," and then ",[14,1228,1229],{},"1234.56",". Doing it the other way round produces ",[14,1232,1233],{},"1.234.56",", which parses as nothing.",[10,1236,1237],{},"Detecting the convention automatically is possible when a column is internally consistent — the last separator in a well-formed number is the decimal one:",[152,1239,1241],{"className":184,"code":1240,"language":186,"meta":157,"style":157},"import pandas as pd\n\ndef detect_decimal(series, sample=200):\n    \"\"\"Guess whether a column uses ',' or '.' as its decimal separator.\"\"\"\n    text = series.astype(\"string\").dropna().head(sample)\n    comma_last = text.str.match(r\"^[^,]*\\d[.\\s]\\d{3},\\d+$\", na=False).sum()\n    dot_last = text.str.match(r\"^[^.]*\\d[,\\s]\\d{3}\\.\\d+$\", na=False).sum()\n    return \",\" if comma_last > dot_last else \".\"\n",[14,1242,1243,1253,1257,1273,1278,1291,1348,1400],{"__ignoreMap":157},[161,1244,1245,1247,1249,1251],{"class":163,"line":164},[161,1246,194],{"class":193},[161,1248,198],{"class":197},[161,1250,201],{"class":193},[161,1252,204],{"class":197},[161,1254,1255],{"class":163,"line":207},[161,1256,211],{"emptyLinePlaceholder":210},[161,1258,1259,1261,1264,1267,1269,1271],{"class":163,"line":214},[161,1260,744],{"class":193},[161,1262,1263],{"class":747}," detect_decimal",[161,1265,1266],{"class":197},"(series, sample",[161,1268,220],{"class":193},[161,1270,54],{"class":280},[161,1272,767],{"class":197},[161,1274,1275],{"class":163,"line":226},[161,1276,1277],{"class":171},"    \"\"\"Guess whether a column uses ',' or '.' as its decimal separator.\"\"\"\n",[161,1279,1280,1282,1284,1286,1288],{"class":163,"line":257},[161,1281,777],{"class":197},[161,1283,220],{"class":193},[161,1285,782],{"class":197},[161,1287,403],{"class":171},[161,1289,1290],{"class":197},").dropna().head(sample)\n",[161,1292,1293,1296,1298,1300,1302,1304,1307,1309,1312,1314,1316,1319,1321,1324,1327,1329,1331,1333,1335,1337,1339,1341,1343,1345],{"class":163,"line":271},[161,1294,1295],{"class":197},"    comma_last ",[161,1297,220],{"class":193},[161,1299,806],{"class":197},[161,1301,409],{"class":193},[161,1303,412],{"class":171},[161,1305,1306],{"class":280},"^[",[161,1308,813],{"class":193},[161,1310,1311],{"class":428},",",[161,1313,432],{"class":280},[161,1315,418],{"class":193},[161,1317,1318],{"class":280},"\\d[",[161,1320,23],{"class":428},[161,1322,1323],{"class":280},"\\s]\\d",[161,1325,1326],{"class":193},"{3}",[161,1328,1311],{"class":171},[161,1330,968],{"class":280},[161,1332,447],{"class":193},[161,1334,465],{"class":280},[161,1336,412],{"class":171},[161,1338,237],{"class":197},[161,1340,831],{"class":475},[161,1342,220],{"class":193},[161,1344,481],{"class":280},[161,1346,1347],{"class":197},").sum()\n",[161,1349,1350,1353,1355,1357,1359,1361,1363,1365,1367,1369,1371,1373,1375,1377,1379,1382,1384,1386,1388,1390,1392,1394,1396,1398],{"class":163,"line":277},[161,1351,1352],{"class":197},"    dot_last ",[161,1354,220],{"class":193},[161,1356,806],{"class":197},[161,1358,409],{"class":193},[161,1360,412],{"class":171},[161,1362,1306],{"class":280},[161,1364,813],{"class":193},[161,1366,23],{"class":428},[161,1368,432],{"class":280},[161,1370,418],{"class":193},[161,1372,1318],{"class":280},[161,1374,1311],{"class":428},[161,1376,1323],{"class":280},[161,1378,1326],{"class":193},[161,1380,1381],{"class":424},"\\.",[161,1383,968],{"class":280},[161,1385,447],{"class":193},[161,1387,465],{"class":280},[161,1389,412],{"class":171},[161,1391,237],{"class":197},[161,1393,831],{"class":475},[161,1395,220],{"class":193},[161,1397,481],{"class":280},[161,1399,1347],{"class":197},[161,1401,1402,1404,1406,1409,1412,1414,1417,1420],{"class":163,"line":379},[161,1403,1189],{"class":193},[161,1405,1012],{"class":171},[161,1407,1408],{"class":193}," if",[161,1410,1411],{"class":197}," comma_last ",[161,1413,487],{"class":193},[161,1415,1416],{"class":197}," dot_last ",[161,1418,1419],{"class":193},"else",[161,1421,1422],{"class":171}," \".\"\n",[10,1424,1425],{},"Treat that as a hint to confirm, not a decision to trust silently — a column of mixed conventions will fool it, and the consequence is values wrong by a factor of a thousand.",[147,1427,1429],{"id":1428},"step-4-report-what-failed","Step 4 — Report what failed",[10,1431,1432],{},"The step that separates a safe conversion from a dangerous one:",[25,1434,35,1440,35,1443,35,1446,35,1449,35,1451,35,1456,35,1459,35,1474,35,1485,35,1492,35,1498,35,1503,35,1506,35,1511,35,1515,35,1518,35,1523],{"viewBox":1435,"role":28,"ariaLabel":1436,"ariaLabelledBy":1437,"xmlns":33,"style":34},"0 0 800 232","Conversion outcomes split three ways: values that converted, cells that were already blank, and text that failed to parse and must be reported.",[1438,1439],"conv-t","conv-d",[37,1441,1442],{"id":1438},"Three outcomes, only one of which is a problem",[41,1444,1445],{"id":1439},"After coercion, the column divides into three groups. Converted values are the successful majority. Cells that were blank to begin with are expected missing data and simply need counting. Values that held text and failed to parse are the group that matters: they were data, and they are now silently absent from every total. Reporting the distinct failing values usually reveals a small number of causes such as n slash a or a footnote marker.",[45,1447],{"x":47,"y":47,"width":48,"height":1448,"fill":50},"232",[45,1450],{"x":614,"y":90,"width":642,"height":74,"rx":618,"fill":81,"stroke":82,"style":70},[52,1452,701],{"x":1453,"y":1454,"style":1455},"102","118","font-size:12px;font-weight:700;fill:var(--text,#172033);text-anchor:middle",[52,1457,586],{"x":1453,"y":1458,"style":713},"139",[1460,1461,1463,1464,1463,1468,1463,1471,35],"g",{"stroke":82,"style":70,"fill":1462},"none","\n    ",[1465,1466],"path",{"d":1467},"M190 104 H 226 V 46 H 262",[1465,1469],{"d":1470},"M190 119 H 262",[1465,1472],{"d":1473},"M190 134 H 226 V 190 H 262",[1460,1475,1463,1476,1463,1479,1463,1482,35],{"fill":639},[636,1477],{"points":1478},"270,46 258,40 258,52",[636,1480],{"points":1481},"270,119 258,113 258,125",[636,1483],{"points":1484},"270,190 258,184 258,196",[45,1486],{"x":1487,"y":1488,"width":1489,"height":1490,"rx":1491,"fill":121,"stroke":122,"style":70},"278","22","506","48","11",[52,1493,1497],{"x":1494,"y":1495,"style":1496},"302","43","font-size:11.5px;font-weight:700;fill:var(--teal-ink,#0b6157)","converted — real numbers",[52,1499,1502],{"x":1494,"y":1500,"style":1501},"61","font-size:10.5px;fill:var(--muted,#5b6780)","the expected majority; nothing to do",[45,1504],{"x":1487,"y":1505,"width":1489,"height":1490,"rx":1491,"fill":91,"stroke":92,"style":70},"95",[52,1507,1510],{"x":1494,"y":1508,"style":1509},"116","font-size:11.5px;font-weight:700;fill:var(--gold-ink,#7a4e06)","already blank — expected missing",[52,1512,1514],{"x":1494,"y":1513,"style":1501},"134","count them, decide fill or drop",[45,1516],{"x":1487,"y":1517,"width":1489,"height":1490,"rx":1491,"fill":106,"stroke":107,"style":70},"168",[52,1519,1522],{"x":1494,"y":1520,"style":1521},"189","font-size:11.5px;font-weight:700;fill:var(--accent-ink,#be185d)","was text, failed — the group that matters",[52,1524,1526],{"x":1494,"y":1525,"style":1501},"207","report the distinct values: usually \"n\u002Fa\", \"-\", a footnote marker",[152,1528,1530],{"className":184,"code":1529,"language":186,"meta":157,"style":157},"import pandas as pd\n\ndef convert_and_report(df, column, threshold=0.05, **kwargs):\n    \"\"\"Convert a column to numeric, reporting anything that failed.\"\"\"\n    raw = df[column].astype(\"string\").str.strip()\n    converted = to_number(df[column], **kwargs)\n\n    blank = raw.isna() | (raw == \"\")\n    failed = converted.isna() & ~blank\n\n    if failed.any():\n        print(f\"{column}: {int(failed.sum())} value(s) failed to convert\")\n        print(raw[failed].value_counts().head(10).to_string())\n\n    rate = failed.mean()\n    if rate > threshold:\n        raise ValueError(\n            f\"{column}: {rate:.1%} of values failed to convert — \"\n            \"check the expected format before trusting any total.\"\n        )\n\n    df[column] = converted\n    return df\n\ndf = convert_and_report(df, \"revenue\")\n",[14,1531,1532,1542,1546,1569,1574,1588,1603,1607,1630,1649,1653,1660,1699,1712,1716,1726,1738,1749,1777,1782,1787,1791,1801,1808,1812],{"__ignoreMap":157},[161,1533,1534,1536,1538,1540],{"class":163,"line":164},[161,1535,194],{"class":193},[161,1537,198],{"class":197},[161,1539,201],{"class":193},[161,1541,204],{"class":197},[161,1543,1544],{"class":163,"line":207},[161,1545,211],{"emptyLinePlaceholder":210},[161,1547,1548,1550,1553,1556,1558,1561,1563,1566],{"class":163,"line":214},[161,1549,744],{"class":193},[161,1551,1552],{"class":747}," convert_and_report",[161,1554,1555],{"class":197},"(df, column, threshold",[161,1557,220],{"class":193},[161,1559,1560],{"class":280},"0.05",[161,1562,237],{"class":197},[161,1564,1565],{"class":193},"**",[161,1567,1568],{"class":197},"kwargs):\n",[161,1570,1571],{"class":163,"line":226},[161,1572,1573],{"class":171},"    \"\"\"Convert a column to numeric, reporting anything that failed.\"\"\"\n",[161,1575,1576,1579,1581,1584,1586],{"class":163,"line":257},[161,1577,1578],{"class":197},"    raw ",[161,1580,220],{"class":193},[161,1582,1583],{"class":197}," df[column].astype(",[161,1585,403],{"class":171},[161,1587,787],{"class":197},[161,1589,1590,1593,1595,1598,1600],{"class":163,"line":271},[161,1591,1592],{"class":197},"    converted ",[161,1594,220],{"class":193},[161,1596,1597],{"class":197}," to_number(df[column], ",[161,1599,1565],{"class":193},[161,1601,1602],{"class":197},"kwargs)\n",[161,1604,1605],{"class":163,"line":277},[161,1606,211],{"emptyLinePlaceholder":210},[161,1608,1609,1612,1614,1617,1620,1623,1625,1628],{"class":163,"line":379},[161,1610,1611],{"class":197},"    blank ",[161,1613,220],{"class":193},[161,1615,1616],{"class":197}," raw.isna() ",[161,1618,1619],{"class":193},"|",[161,1621,1622],{"class":197}," (raw ",[161,1624,388],{"class":193},[161,1626,1627],{"class":171}," \"\"",[161,1629,338],{"class":197},[161,1631,1632,1635,1637,1640,1643,1646],{"class":163,"line":394},[161,1633,1634],{"class":197},"    failed ",[161,1636,220],{"class":193},[161,1638,1639],{"class":197}," converted.isna() ",[161,1641,1642],{"class":193},"&",[161,1644,1645],{"class":193}," ~",[161,1647,1648],{"class":197},"blank\n",[161,1650,1651],{"class":163,"line":472},[161,1652,211],{"emptyLinePlaceholder":210},[161,1654,1655,1657],{"class":163,"line":493},[161,1656,382],{"class":193},[161,1658,1659],{"class":197}," failed.any():\n",[161,1661,1662,1665,1667,1670,1672,1675,1678,1681,1684,1686,1689,1692,1694,1697],{"class":163,"line":499},[161,1663,1664],{"class":280},"        print",[161,1666,504],{"class":197},[161,1668,1669],{"class":193},"f",[161,1671,412],{"class":171},[161,1673,1674],{"class":428},"{",[161,1676,1677],{"class":197},"column",[161,1679,1680],{"class":428},"}",[161,1682,1683],{"class":171},": ",[161,1685,1674],{"class":428},[161,1687,1688],{"class":280},"int",[161,1690,1691],{"class":197},"(failed.sum())",[161,1693,1680],{"class":428},[161,1695,1696],{"class":171}," value(s) failed to convert\"",[161,1698,338],{"class":197},[161,1700,1701,1703,1706,1709],{"class":163,"line":926},[161,1702,1664],{"class":280},[161,1704,1705],{"class":197},"(raw[failed].value_counts().head(",[161,1707,1708],{"class":280},"10",[161,1710,1711],{"class":197},").to_string())\n",[161,1713,1714],{"class":163,"line":940},[161,1715,211],{"emptyLinePlaceholder":210},[161,1717,1718,1721,1723],{"class":163,"line":945},[161,1719,1720],{"class":197},"    rate ",[161,1722,220],{"class":193},[161,1724,1725],{"class":197}," failed.mean()\n",[161,1727,1728,1730,1733,1735],{"class":163,"line":951},[161,1729,382],{"class":193},[161,1731,1732],{"class":197}," rate ",[161,1734,487],{"class":193},[161,1736,1737],{"class":197}," threshold:\n",[161,1739,1740,1743,1746],{"class":163,"line":997},[161,1741,1742],{"class":193},"        raise",[161,1744,1745],{"class":280}," ValueError",[161,1747,1748],{"class":197},"(\n",[161,1750,1751,1754,1756,1758,1760,1762,1764,1766,1769,1772,1774],{"class":163,"line":1002},[161,1752,1753],{"class":193},"            f",[161,1755,412],{"class":171},[161,1757,1674],{"class":428},[161,1759,1677],{"class":197},[161,1761,1680],{"class":428},[161,1763,1683],{"class":171},[161,1765,1674],{"class":428},[161,1767,1768],{"class":197},"rate",[161,1770,1771],{"class":193},":.1%",[161,1773,1680],{"class":428},[161,1775,1776],{"class":171}," of values failed to convert — \"\n",[161,1778,1779],{"class":163,"line":1018},[161,1780,1781],{"class":171},"            \"check the expected format before trusting any total.\"\n",[161,1783,1784],{"class":163,"line":1024},[161,1785,1786],{"class":197},"        )\n",[161,1788,1789],{"class":163,"line":1050},[161,1790,211],{"emptyLinePlaceholder":210},[161,1792,1793,1796,1798],{"class":163,"line":1076},[161,1794,1795],{"class":197},"    df[column] ",[161,1797,220],{"class":193},[161,1799,1800],{"class":197}," converted\n",[161,1802,1803,1805],{"class":163,"line":1084},[161,1804,1189],{"class":193},[161,1806,1807],{"class":197}," df\n",[161,1809,1810],{"class":163,"line":1109},[161,1811,211],{"emptyLinePlaceholder":210},[161,1813,1814,1816,1818,1821,1823],{"class":163,"line":1114},[161,1815,217],{"class":197},[161,1817,220],{"class":193},[161,1819,1820],{"class":197}," convert_and_report(df, ",[161,1822,287],{"class":171},[161,1824,338],{"class":197},[10,1826,1827,1828,1831,1832,1835],{},"Printing the distinct failing values is what makes this actionable. Nine times out of ten the report reads ",[14,1829,1830],{},"n\u002Fa: 34"," or ",[14,1833,1834],{},"-: 12",", and the fix is one line added to the cleaning step rather than an investigation.",[147,1837,1839],{"id":1838},"step-5-choose-the-right-dtype","Step 5 — Choose the right dtype",[10,1841,1842,1843,1846,1847,1850],{},"A column of counts that can be missing should be ",[14,1844,1845],{},"Int64",", not ",[14,1848,1849],{},"int64",":",[152,1852,1854],{"className":184,"code":1853,"language":186,"meta":157,"style":157},"import pandas as pd\n\nunits = pd.Series([\"120\", \"\", \"88\", \"n\u002Fa\"])\n\nas_float = pd.to_numeric(units, errors=\"coerce\")\nprint(as_float.tolist())        # [120.0, nan, 88.0, nan]\n\nas_int = as_float.astype(\"Int64\")\nprint(as_int.tolist())          # [120, \u003CNA>, 88, \u003CNA>]\n",[14,1855,1856,1866,1870,1900,1904,1922,1932,1936,1951],{"__ignoreMap":157},[161,1857,1858,1860,1862,1864],{"class":163,"line":164},[161,1859,194],{"class":193},[161,1861,198],{"class":197},[161,1863,201],{"class":193},[161,1865,204],{"class":197},[161,1867,1868],{"class":163,"line":207},[161,1869,211],{"emptyLinePlaceholder":210},[161,1871,1872,1875,1877,1880,1883,1885,1887,1889,1892,1894,1897],{"class":163,"line":214},[161,1873,1874],{"class":197},"units ",[161,1876,220],{"class":193},[161,1878,1879],{"class":197}," pd.Series([",[161,1881,1882],{"class":171},"\"120\"",[161,1884,237],{"class":197},[161,1886,265],{"class":171},[161,1888,237],{"class":197},[161,1890,1891],{"class":171},"\"88\"",[161,1893,237],{"class":197},[161,1895,1896],{"class":171},"\"n\u002Fa\"",[161,1898,1899],{"class":197},"])\n",[161,1901,1902],{"class":163,"line":226},[161,1903,211],{"emptyLinePlaceholder":210},[161,1905,1906,1909,1911,1914,1916,1918,1920],{"class":163,"line":257},[161,1907,1908],{"class":197},"as_float ",[161,1910,220],{"class":193},[161,1912,1913],{"class":197}," pd.to_numeric(units, ",[161,1915,574],{"class":475},[161,1917,220],{"class":193},[161,1919,579],{"class":171},[161,1921,338],{"class":197},[161,1923,1924,1926,1929],{"class":163,"line":271},[161,1925,281],{"class":280},[161,1927,1928],{"class":197},"(as_float.tolist())        ",[161,1930,1931],{"class":293},"# [120.0, nan, 88.0, nan]\n",[161,1933,1934],{"class":163,"line":277},[161,1935,211],{"emptyLinePlaceholder":210},[161,1937,1938,1941,1943,1946,1949],{"class":163,"line":379},[161,1939,1940],{"class":197},"as_int ",[161,1942,220],{"class":193},[161,1944,1945],{"class":197}," as_float.astype(",[161,1947,1948],{"class":171},"\"Int64\"",[161,1950,338],{"class":197},[161,1952,1953,1955,1958],{"class":163,"line":394},[161,1954,281],{"class":280},[161,1956,1957],{"class":197},"(as_int.tolist())          ",[161,1959,1960],{"class":293},"# [120, \u003CNA>, 88, \u003CNA>]\n",[10,1962,1963,1965,1966,1969,1970,1972],{},[14,1964,1849],{}," cannot hold a missing value, so pandas promotes the column to float and every integer gains a decimal point — which then writes to Excel as ",[14,1967,1968],{},"120.0"," and reads badly in a report. ",[14,1971,1845],{}," holds both.",[10,1974,1975],{},"For money, the pragmatic choice is float with rounding at the point of presentation, since Excel itself stores doubles:",[152,1977,1979],{"className":184,"code":1978,"language":186,"meta":157,"style":157},"df[\"revenue\"] = to_number(df[\"revenue\"]).round(2)\n",[14,1980,1981],{"__ignoreMap":157},[161,1982,1983,1985,1987,1989,1991,1994,1996,1999,2001],{"class":163,"line":164},[161,1984,556],{"class":197},[161,1986,287],{"class":171},[161,1988,561],{"class":197},[161,1990,220],{"class":193},[161,1992,1993],{"class":197}," to_number(df[",[161,1995,287],{"class":171},[161,1997,1998],{"class":197},"]).round(",[161,2000,646],{"class":280},[161,2002,338],{"class":197},[10,2004,2005,2006,2009],{},"Where exact decimal arithmetic genuinely matters — reconciling to the cent across many rows — accumulate with ",[14,2007,2008],{},"Decimal"," in Python and convert to float only when writing, because a float sum of many two-decimal values drifts.",[147,2011,2013],{"id":2012},"common-pitfalls-and-fixes","Common pitfalls and fixes",[2015,2016,2017,2033],"table",{},[2018,2019,2020],"thead",{},[2021,2022,2023,2027,2030],"tr",{},[2024,2025,2026],"th",{},"Symptom",[2024,2028,2029],{},"Cause",[2024,2031,2032],{},"Fix",[2034,2035,2036,2058,2072,2087,2098,2109,2130,2147],"tbody",{},[2021,2037,2038,2045,2052],{},[2039,2040,2041,2044],"td",{},[14,2042,2043],{},"sum()"," concatenates strings",[2039,2046,2047,2048,2051],{},"Column is ",[14,2049,2050],{},"object"," dtype",[2039,2053,2054,2055,2057],{},"Convert with ",[14,2056,701],{}," first.",[2021,2059,2060,2063,2069],{},[2039,2061,2062],{},"Total far too low",[2039,2064,2065,2066,2068],{},"Failed values coerced to ",[14,2067,590],{}," silently",[2039,2070,2071],{},"Report the failures; raise above a threshold.",[2021,2073,2074,2081,2084],{},[2039,2075,2076,2078,2079],{},[14,2077,1221],{}," becomes ",[14,2080,590],{},[2039,2082,2083],{},"European format not handled",[2039,2085,2086],{},"Remove dots first, then comma to dot.",[2021,2088,2089,2092,2095],{},[2039,2090,2091],{},"Values off by 1000×",[2039,2093,2094],{},"Decimal convention detected wrongly",[2039,2096,2097],{},"Confirm the convention; do not auto-detect blindly.",[2021,2099,2100,2103,2106],{},[2039,2101,2102],{},"Negatives came through positive",[2039,2104,2105],{},"Accounting parentheses",[2039,2107,2108],{},"Detect, strip, then negate.",[2021,2110,2111,2119,2124],{},[2039,2112,2113,2116,2117],{},[14,2114,2115],{},"120"," written as ",[14,2118,1968],{},[2039,2120,2121,2123],{},[14,2122,590],{}," promoted the column to float",[2039,2125,2126,2127,2129],{},"Use the ",[14,2128,1845],{}," dtype.",[2021,2131,2132,2141,2144],{},[2039,2133,2134,2137,2138],{},[14,2135,2136],{},"12.5%"," became ",[14,2139,2140],{},"12.5",[2039,2142,2143],{},"Percentage not scaled",[2039,2145,2146],{},"Divide the percent rows by 100.",[2021,2148,2149,2152,2155],{},[2039,2150,2151],{},"Some values still fail after cleaning",[2039,2153,2154],{},"A character not in the strip pattern",[2039,2156,2157],{},"Print the failing values and extend it.",[147,2159,2161],{"id":2160},"performance-and-scale-notes","Performance and scale notes",[10,2163,2164,2166,2167,2170],{},[14,2165,701],{}," is fast; the regex cleaning around it is not. Each ",[14,2168,2169],{},"str.replace"," walks the column, and the helper above makes six passes.",[10,2172,2173,2177],{},[2174,2175,2176],"strong",{},"Clean the distinct values, not every row."," A million-row price column typically holds far fewer distinct strings, and the saving is proportional:",[152,2179,2181],{"className":184,"code":2180,"language":186,"meta":157,"style":157},"import pandas as pd\n\ndef to_number_via_lookup(series, **kwargs):\n    \"\"\"Convert each distinct text value once, then map.\"\"\"\n    distinct = pd.Series(series.dropna().unique())\n    lookup = dict(zip(distinct, to_number(distinct, **kwargs)))\n    return series.map(lookup).astype(\"float64\")\n",[14,2182,2183,2193,2197,2211,2216,2226,2249],{"__ignoreMap":157},[161,2184,2185,2187,2189,2191],{"class":163,"line":164},[161,2186,194],{"class":193},[161,2188,198],{"class":197},[161,2190,201],{"class":193},[161,2192,204],{"class":197},[161,2194,2195],{"class":163,"line":207},[161,2196,211],{"emptyLinePlaceholder":210},[161,2198,2199,2201,2204,2207,2209],{"class":163,"line":214},[161,2200,744],{"class":193},[161,2202,2203],{"class":747}," to_number_via_lookup",[161,2205,2206],{"class":197},"(series, ",[161,2208,1565],{"class":193},[161,2210,1568],{"class":197},[161,2212,2213],{"class":163,"line":226},[161,2214,2215],{"class":171},"    \"\"\"Convert each distinct text value once, then map.\"\"\"\n",[161,2217,2218,2221,2223],{"class":163,"line":257},[161,2219,2220],{"class":197},"    distinct ",[161,2222,220],{"class":193},[161,2224,2225],{"class":197}," pd.Series(series.dropna().unique())\n",[161,2227,2228,2231,2233,2236,2238,2241,2244,2246],{"class":163,"line":271},[161,2229,2230],{"class":197},"    lookup ",[161,2232,220],{"class":193},[161,2234,2235],{"class":280}," dict",[161,2237,504],{"class":197},[161,2239,2240],{"class":280},"zip",[161,2242,2243],{"class":197},"(distinct, to_number(distinct, ",[161,2245,1565],{"class":193},[161,2247,2248],{"class":197},"kwargs)))\n",[161,2250,2251,2253,2256,2259],{"class":163,"line":277},[161,2252,1189],{"class":193},[161,2254,2255],{"class":197}," series.map(lookup).astype(",[161,2257,2258],{"class":171},"\"float64\"",[161,2260,338],{"class":197},[10,2262,2263,2266],{},[2174,2264,2265],{},"Skip the cleaning when it is not needed."," Try the direct conversion first and only fall back to the expensive path for the values that failed:",[152,2268,2270],{"className":184,"code":2269,"language":186,"meta":157,"style":157},"import pandas as pd\n\ndef to_number_fast(series, **kwargs):\n    direct = pd.to_numeric(series, errors=\"coerce\")\n    if not direct.isna().any():\n        return direct                       # nothing needed cleaning\n\n    needs_work = direct.isna() & series.notna()\n    cleaned = to_number(series[needs_work], **kwargs)\n    return direct.where(~needs_work, cleaned)\n",[14,2271,2272,2282,2286,2299,2317,2327,2338,2342,2357,2371],{"__ignoreMap":157},[161,2273,2274,2276,2278,2280],{"class":163,"line":164},[161,2275,194],{"class":193},[161,2277,198],{"class":197},[161,2279,201],{"class":193},[161,2281,204],{"class":197},[161,2283,2284],{"class":163,"line":207},[161,2285,211],{"emptyLinePlaceholder":210},[161,2287,2288,2290,2293,2295,2297],{"class":163,"line":214},[161,2289,744],{"class":193},[161,2291,2292],{"class":747}," to_number_fast",[161,2294,2206],{"class":197},[161,2296,1565],{"class":193},[161,2298,1568],{"class":197},[161,2300,2301,2304,2306,2309,2311,2313,2315],{"class":163,"line":226},[161,2302,2303],{"class":197},"    direct ",[161,2305,220],{"class":193},[161,2307,2308],{"class":197}," pd.to_numeric(series, ",[161,2310,574],{"class":475},[161,2312,220],{"class":193},[161,2314,579],{"class":171},[161,2316,338],{"class":197},[161,2318,2319,2321,2324],{"class":163,"line":257},[161,2320,382],{"class":193},[161,2322,2323],{"class":193}," not",[161,2325,2326],{"class":197}," direct.isna().any():\n",[161,2328,2329,2332,2335],{"class":163,"line":271},[161,2330,2331],{"class":193},"        return",[161,2333,2334],{"class":197}," direct                       ",[161,2336,2337],{"class":293},"# nothing needed cleaning\n",[161,2339,2340],{"class":163,"line":277},[161,2341,211],{"emptyLinePlaceholder":210},[161,2343,2344,2347,2349,2352,2354],{"class":163,"line":379},[161,2345,2346],{"class":197},"    needs_work ",[161,2348,220],{"class":193},[161,2350,2351],{"class":197}," direct.isna() ",[161,2353,1642],{"class":193},[161,2355,2356],{"class":197}," series.notna()\n",[161,2358,2359,2362,2364,2367,2369],{"class":163,"line":394},[161,2360,2361],{"class":197},"    cleaned ",[161,2363,220],{"class":193},[161,2365,2366],{"class":197}," to_number(series[needs_work], ",[161,2368,1565],{"class":193},[161,2370,1602],{"class":197},[161,2372,2373,2375,2378,2380],{"class":163,"line":472},[161,2374,1189],{"class":193},[161,2376,2377],{"class":197}," direct.where(",[161,2379,1143],{"class":193},[161,2381,2382],{"class":197},"needs_work, cleaned)\n",[10,2384,2385],{},"On a column that is already numeric this returns after one vectorised pass, which matters when the same function runs across thirty columns of an import.",[10,2387,2388,2391,2392,2396,2397,23],{},[2174,2389,2390],{},"Push the fix upstream where you can."," A column arriving as text usually means the export wrote it that way, and fixing the export removes the whole problem — see ",[19,2393,2395],{"href":2394},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fexport-sql-query-results-to-excel-with-python\u002F","exporting SQL query results to Excel"," for controlling types at the source. Failing that, converting at ingest once is far cheaper than converting on every read, and for large workbooks it composes with the chunked approach in ",[19,2398,2400],{"href":2399},"\u002Fadvanced-data-transformation-and-cleaning\u002Fworking-with-large-excel-files-in-python\u002Fread-large-excel-file-in-chunks-with-pandas\u002F","reading large Excel files in chunks",[147,2402,2404],{"id":2403},"conclusion","Conclusion",[10,2406,2407,2408,2410,2411,2413,2414,2416],{},"A numeric column stored as text is a formatting problem with arithmetic consequences. Check ",[14,2409,303],{}," first, then clean in a fixed order: strip accounting parentheses and remember them, note and strip percentage signs, remove currency symbols and separators, and handle the European convention by removing dots before converting the comma. Convert with ",[14,2412,586],{},", then always report which values failed and refuse to continue when too many did. Use ",[14,2415,1845],{}," for counts that can be missing, and clean the distinct values rather than every row when the column is large.",[147,2418,2420],{"id":2419},"frequently-asked-questions","Frequently asked questions",[10,2422,2423,2426,2427,2429,2430,2433,2434,2436],{},[2174,2424,2425],{},"Why is my sum concatenating strings instead of adding?","\nThe column has ",[14,2428,2050],{}," dtype, so the plus operator joins strings. Check ",[14,2431,2432],{},"df.dtypes"," — anything that should be numeric and shows ",[14,2435,2050],{}," needs converting before any arithmetic.",[10,2438,2439,2445,2446,2448,2449,2451],{},[2174,2440,2441,2442,2444],{},"What does ",[14,2443,586],{}," do to values it cannot parse?","\nIt replaces them with ",[14,2447,590],{}," rather than raising. That keeps a batch job running, but you must inspect which rows became ",[14,2450,590],{}," afterwards, or a data-quality problem turns into a silently understated total.",[10,2453,2454,2459],{},[2174,2455,2456,2457,435],{},"How do I handle European numbers like ",[14,2458,1221],{},"\nRemove the dots as thousands separators, then replace the comma with a dot before converting. Doing it in the other order destroys the decimal, so the order is not optional.",[10,2461,2462,2465,2466,2468],{},[2174,2463,2464],{},"What about negatives shown in parentheses?","\nAccounting formats write minus one thousand as a parenthesised value. Detect the parentheses, strip them, convert, and negate the result — ",[14,2467,701],{}," does not understand the convention.",[10,2470,2471,2478,2480,2481,2483,2484,2486],{},[2174,2472,2473,2474,1831,2476,435],{},"Should I use ",[14,2475,1845],{},[14,2477,1849],{},[14,2479,1845],{}," with a capital I, the nullable integer type, whenever the column can contain a missing value. Plain ",[14,2482,1849],{}," cannot hold ",[14,2485,590],{},", so pandas silently promotes the column to float and integers gain a decimal point.",[147,2488,2490],{"id":2489},"related","Related",[2492,2493,2494,2501,2508,2514,2521],"ul",{},[2495,2496,2497,2498,2500],"li",{},"Up to the parent: ",[19,2499,22],{"href":21}," — the wider cleaning toolkit.",[2495,2502,2503,2507],{},[19,2504,2506],{"href":2505},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fstrip-whitespace-and-normalise-text-columns-with-pandas\u002F","Strip Whitespace and Normalise Text Columns with pandas"," — the text-side equivalent.",[2495,2509,2510,2513],{},[19,2511,2512],{"href":524},"Check Excel Data Types with pandas"," — auditing dtypes across a whole sheet.",[2495,2515,2516,2520],{},[19,2517,2519],{"href":2518},"\u002Fformatting-and-charting-excel-reports-with-python\u002Fapplying-number-and-date-formats-in-excel\u002Fformat-excel-cells-as-currency-with-python\u002F","Format Excel Cells as Currency with Python"," — writing the converted numbers back properly.",[2495,2522,2523,2527,2528,2530],{},[19,2524,2526],{"href":2525},"\u002Fadvanced-data-transformation-and-cleaning\u002Fhandling-missing-data-in-excel-reports\u002Ffill-missing-values-in-excel-with-pandas-fillna\u002F","Fill Missing Values in Excel with pandas fillna"," — deciding what to do with the coerced ",[14,2529,590],{},"s.",[2532,2533,2534],"style",{},"html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .s-wDw, html code.shiki .s-wDw{--shiki-default:#6A737D;--shiki-dark:#BDC4CC}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s_b0D, html code.shiki .s_b0D{--shiki-default:#22863A;--shiki-default-font-weight:bold;--shiki-dark:#72F088;--shiki-dark-font-weight:bold}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}html pre.shiki code .sx1sP, html code.shiki .sx1sP{--shiki-default:#22863A;--shiki-dark:#72F088}html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}",{"title":157,"searchDepth":207,"depth":207,"links":2536},[2537,2538,2539,2540,2541,2542,2543,2544,2545,2546,2547],{"id":149,"depth":207,"text":150},{"id":297,"depth":207,"text":298},{"id":528,"depth":207,"text":529},{"id":594,"depth":207,"text":595},{"id":1428,"depth":207,"text":1429},{"id":1838,"depth":207,"text":1839},{"id":2012,"depth":207,"text":2013},{"id":2160,"depth":207,"text":2161},{"id":2403,"depth":207,"text":2404},{"id":2419,"depth":207,"text":2420},{"id":2489,"depth":207,"text":2490},"2026-08-15","Numbers stored as text will not sum. Convert them in pandas — currency symbols, thousands separators, European decimals, parentheses negatives, and reporting what failed.","md",[2552,2554,2557,2560,2562],{"q":2425,"a":2553},"The column has object dtype, so the plus operator joins strings. Check df.dtypes — anything that should be numeric and shows object needs converting before any arithmetic.",{"q":2555,"a":2556},"What does errors=\"coerce\" do to values it cannot parse?","It replaces them with NaN rather than raising. That keeps a batch job running, but you must inspect which rows became NaN afterwards, or a data-quality problem turns into a silently understated total.",{"q":2558,"a":2559},"How do I handle European numbers like 1.234,56?","Remove the dots as thousands separators, then replace the comma with a dot before converting. Doing it in the other order destroys the decimal, so the order is not optional.",{"q":2464,"a":2561},"Accounting formats write minus one thousand as a parenthesised value. Detect the parentheses, strip them, convert, and negate the result — to_numeric does not understand the convention.",{"q":2563,"a":2564},"Should I use Int64 or int64?","Int64 with a capital I, the nullable integer type, whenever the column can contain a missing value. Plain int64 cannot hold NaN, so pandas silently promotes the column to float and integers gain a decimal point.",{},"\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fconvert-excel-text-columns-to-numbers-with-pandas",{"title":2568,"description":2569},"Convert Text to Numbers in pandas (Excel Columns)","Turn text-formatted Excel numbers into real numerics: to_numeric with coerce, stripping currency and separators, comma decimals, accounting negatives and a rejects report.","convert-excel-text-columns-to-numbers-with-pandas","advanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fconvert-excel-text-columns-to-numbers-with-pandas\u002Findex","how-to","4MxbmsmgQXhgzl0io7ct7zw_hCTFLZPyPW-N5miRzy8",[2575,2579],{"title":2576,"path":2577,"stem":2578,"children":-1},"Cleaning Excel Data With Pandas: A Production Workflow","\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas","advanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Findex",{"title":2580,"path":2581,"stem":2582,"children":-1},"Pandas: Drop Duplicates From an Excel Column","\u002Fadvanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fpandas-drop-duplicates-from-excel-column","advanced-data-transformation-and-cleaning\u002Fcleaning-excel-data-with-pandas\u002Fpandas-drop-duplicates-from-excel-column\u002Findex",1786800028602]