[{"data":1,"prerenderedAt":1940},["ShallowReactive",2],{"doc:\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fload-excel-into-sqlite-with-a-defined-schema":3,"surround:\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fload-excel-into-sqlite-with-a-defined-schema":1907,"ancestors:\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fload-excel-into-sqlite-with-a-defined-schema":1916,"children:\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fload-excel-into-sqlite-with-a-defined-schema":1921,"related:\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fload-excel-into-sqlite-with-a-defined-schema":1922},{"id":4,"title":5,"body":6,"dateModified":1873,"datePublished":1873,"description":1874,"extension":1875,"faq":1876,"keyword":1885,"kind":1886,"meta":1887,"navigation":137,"path":1896,"seo":1897,"slug":1900,"stem":1901,"topics":1902,"type":1905,"__hash__":1906},"docs\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fload-excel-into-sqlite-with-a-defined-schema\u002Findex.md","Load Excel into SQLite with a Defined Schema",{"type":7,"value":8,"toc":1860},"minimark",[9,44,49,77,94,101,297,312,316,552,575,685,689,692,1246,1274,1400,1508,1512,1699,1710,1714,1725,1729,1777,1781,1795,1799,1805,1811,1817,1823,1827,1856],[10,11,12,16,17,21,22,25,26,29,30,33,34,37,38,43],"p",{},[13,14,15],"code",{},"df.to_sql(\"orders\", conn)"," gets Excel data into SQLite in one line, and for a quick look that is fine. For a database other code will rely on, the table it creates is weak: it has no primary key, so loading the same file twice doubles the data; dates become text in whatever form pandas chose; an integer column with one blank becomes a column of floats; and nothing stops a negative quantity or an order for a customer that does not exist. A database is only as useful as the rules it enforces, so the better approach is to define the tables yourself — types, keys, constraints — and load the spreadsheet ",[18,19,20],"em",{},"into"," them, letting the database reject what does not fit. This page compares the table ",[13,23,24],{},"to_sql"," invents with a designed schema, creates SQLite ",[13,27,28],{},"STRICT"," tables with primary keys, ",[13,31,32],{},"NOT NULL",", ",[13,35,36],{},"CHECK"," and foreign-key constraints, converts Excel's values deliberately, loads everything in one transaction, and reports each rejected row with the rule it broke. It belongs to ",[39,40,42],"a",{"href":41},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002F","Moving Data Between Excel and Databases",".",[45,46,48],"h2",{"id":47},"prerequisites","Prerequisites",[50,51,56],"pre",{"className":52,"code":53,"language":54,"meta":55,"style":55},"language-bash shiki shiki-themes github-light github-dark-high-contrast","pip install pandas openpyxl\n","bash","",[13,57,58],{"__ignoreMap":55},[59,60,63,67,71,74],"span",{"class":61,"line":62},"line",1,[59,64,66],{"class":65},"sMTad","pip",[59,68,70],{"class":69},"srMev"," install",[59,72,73],{"class":69}," pandas",[59,75,76],{"class":69}," openpyxl\n",[10,78,79,82,83,85,86,89,90,93],{},[13,80,81],{},"sqlite3"," is in the standard library; ",[13,84,28],{}," tables need SQLite 3.37 or later, which current Python builds include. The workbook has a ",[18,87,88],{},"Customers"," sheet and an ",[18,91,92],{},"Orders"," sheet with three deliberate problems: an order for a customer not on the customer list, an order with a blank quantity and one with a negative quantity.",[45,95,97,98,100],{"id":96},"step-1-what-to_sql-invents","Step 1 — What ",[13,99,24],{}," invents",[50,102,106],{"className":103,"code":104,"language":105,"meta":55,"style":55},"language-python shiki shiki-themes github-light github-dark-high-contrast","import sqlite3\nimport pandas as pd\n\norders = pd.read_excel(\"sales.xlsx\", sheet_name=\"Orders\")\nquick = sqlite3.connect(\":memory:\")\norders.to_sql(\"orders\", quick, index=False)\nprint(quick.execute(\"SELECT sql FROM sqlite_master WHERE name='orders'\").fetchone()[0])\nprint(quick.execute(\"SELECT OrderDate, Qty FROM orders LIMIT 3\").fetchall())\norders.to_sql(\"orders\", quick, index=False, if_exists=\"append\")\nprint(\"rows after loading twice:\", quick.execute(\"SELECT COUNT(*) FROM orders\").fetchone()[0])\nquick.close()\n","python",[13,107,108,118,132,139,168,184,207,228,241,268,291],{"__ignoreMap":55},[59,109,110,114],{"class":61,"line":62},[59,111,113],{"class":112},"s-kum","import",[59,115,117],{"class":116},"skGVy"," sqlite3\n",[59,119,121,123,126,129],{"class":61,"line":120},2,[59,122,113],{"class":112},[59,124,125],{"class":116}," pandas ",[59,127,128],{"class":112},"as",[59,130,131],{"class":116}," pd\n",[59,133,135],{"class":61,"line":134},3,[59,136,138],{"emptyLinePlaceholder":137},true,"\n",[59,140,142,145,148,151,154,156,160,162,165],{"class":61,"line":141},4,[59,143,144],{"class":116},"orders ",[59,146,147],{"class":112},"=",[59,149,150],{"class":116}," pd.read_excel(",[59,152,153],{"class":69},"\"sales.xlsx\"",[59,155,33],{"class":116},[59,157,159],{"class":158},"sa561","sheet_name",[59,161,147],{"class":112},[59,163,164],{"class":69},"\"Orders\"",[59,166,167],{"class":116},")\n",[59,169,171,174,176,179,182],{"class":61,"line":170},5,[59,172,173],{"class":116},"quick ",[59,175,147],{"class":112},[59,177,178],{"class":116}," sqlite3.connect(",[59,180,181],{"class":69},"\":memory:\"",[59,183,167],{"class":116},[59,185,187,190,193,196,199,201,205],{"class":61,"line":186},6,[59,188,189],{"class":116},"orders.to_sql(",[59,191,192],{"class":69},"\"orders\"",[59,194,195],{"class":116},", quick, ",[59,197,198],{"class":158},"index",[59,200,147],{"class":112},[59,202,204],{"class":203},"sP0c6","False",[59,206,167],{"class":116},[59,208,210,213,216,219,222,225],{"class":61,"line":209},7,[59,211,212],{"class":203},"print",[59,214,215],{"class":116},"(quick.execute(",[59,217,218],{"class":69},"\"SELECT sql FROM sqlite_master WHERE name='orders'\"",[59,220,221],{"class":116},").fetchone()[",[59,223,224],{"class":203},"0",[59,226,227],{"class":116},"])\n",[59,229,231,233,235,238],{"class":61,"line":230},8,[59,232,212],{"class":203},[59,234,215],{"class":116},[59,236,237],{"class":69},"\"SELECT OrderDate, Qty FROM orders LIMIT 3\"",[59,239,240],{"class":116},").fetchall())\n",[59,242,244,246,248,250,252,254,256,258,261,263,266],{"class":61,"line":243},9,[59,245,189],{"class":116},[59,247,192],{"class":69},[59,249,195],{"class":116},[59,251,198],{"class":158},[59,253,147],{"class":112},[59,255,204],{"class":203},[59,257,33],{"class":116},[59,259,260],{"class":158},"if_exists",[59,262,147],{"class":112},[59,264,265],{"class":69},"\"append\"",[59,267,167],{"class":116},[59,269,271,273,276,279,282,285,287,289],{"class":61,"line":270},10,[59,272,212],{"class":203},[59,274,275],{"class":116},"(",[59,277,278],{"class":69},"\"rows after loading twice:\"",[59,280,281],{"class":116},", quick.execute(",[59,283,284],{"class":69},"\"SELECT COUNT(*) FROM orders\"",[59,286,221],{"class":116},[59,288,224],{"class":203},[59,290,227],{"class":116},[59,292,294],{"class":61,"line":293},11,[59,295,296],{"class":116},"quick.close()\n",[10,298,299,300,303,304,307,308,311],{},"The generated table has a type per column and nothing else. ",[13,301,302],{},"OrderDate"," is stored as text with a time of midnight attached; ",[13,305,306],{},"Qty"," is ",[13,309,310],{},"REAL"," because the blank made pandas read the column as float, so 12 is stored as 12.0; there is no primary key, so loading the file a second time silently doubles the table. None of the three data problems is noticed. The database accepted everything because it was given no rules.",[45,313,315],{"id":314},"step-2-design-the-schema-first","Step 2 — Design the schema first",[50,317,319],{"className":103,"code":318,"language":105,"meta":55,"style":55},"SCHEMA = \"\"\"\nPRAGMA foreign_keys = ON;\nCREATE TABLE customers (\n    customer_id TEXT PRIMARY KEY CHECK (length(customer_id) = 4),\n    name        TEXT NOT NULL,\n    country     TEXT NOT NULL CHECK (length(country) = 2)\n) STRICT;\nCREATE TABLE orders (\n    order_id    INTEGER PRIMARY KEY,\n    customer_id TEXT NOT NULL REFERENCES customers(customer_id),\n    order_date  TEXT NOT NULL CHECK (order_date GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]'),\n    qty         INTEGER NOT NULL CHECK (qty > 0),\n    amount      REAL NOT NULL CHECK (amount >= 0)\n) STRICT;\n\"\"\"\ndb = sqlite3.connect(\"sales.db\")\ndb.executescript(\"DROP TABLE IF EXISTS orders; DROP TABLE IF EXISTS customers;\" + SCHEMA)\nfor (name,) in db.execute(\"SELECT name FROM sqlite_master WHERE type='table'\"):\n    print(name, [(c[1], c[2], \"NOT NULL\" if c[3] else \"\", \"PK\" if c[5] else \"\") for c in db.execute(f\"PRAGMA table_info({name})\")])\n",[13,320,321,332,337,342,347,352,357,362,367,372,377,382,388,394,399,405,420,437,458],{"__ignoreMap":55},[59,322,323,326,329],{"class":61,"line":62},[59,324,325],{"class":203},"SCHEMA",[59,327,328],{"class":112}," =",[59,330,331],{"class":69}," \"\"\"\n",[59,333,334],{"class":61,"line":120},[59,335,336],{"class":69},"PRAGMA foreign_keys = ON;\n",[59,338,339],{"class":61,"line":134},[59,340,341],{"class":69},"CREATE TABLE customers (\n",[59,343,344],{"class":61,"line":141},[59,345,346],{"class":69},"    customer_id TEXT PRIMARY KEY CHECK (length(customer_id) = 4),\n",[59,348,349],{"class":61,"line":170},[59,350,351],{"class":69},"    name        TEXT NOT NULL,\n",[59,353,354],{"class":61,"line":186},[59,355,356],{"class":69},"    country     TEXT NOT NULL CHECK (length(country) = 2)\n",[59,358,359],{"class":61,"line":209},[59,360,361],{"class":69},") STRICT;\n",[59,363,364],{"class":61,"line":230},[59,365,366],{"class":69},"CREATE TABLE orders (\n",[59,368,369],{"class":61,"line":243},[59,370,371],{"class":69},"    order_id    INTEGER PRIMARY KEY,\n",[59,373,374],{"class":61,"line":270},[59,375,376],{"class":69},"    customer_id TEXT NOT NULL REFERENCES customers(customer_id),\n",[59,378,379],{"class":61,"line":293},[59,380,381],{"class":69},"    order_date  TEXT NOT NULL CHECK (order_date GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]'),\n",[59,383,385],{"class":61,"line":384},12,[59,386,387],{"class":69},"    qty         INTEGER NOT NULL CHECK (qty > 0),\n",[59,389,391],{"class":61,"line":390},13,[59,392,393],{"class":69},"    amount      REAL NOT NULL CHECK (amount >= 0)\n",[59,395,397],{"class":61,"line":396},14,[59,398,361],{"class":69},[59,400,402],{"class":61,"line":401},15,[59,403,404],{"class":69},"\"\"\"\n",[59,406,408,411,413,415,418],{"class":61,"line":407},16,[59,409,410],{"class":116},"db ",[59,412,147],{"class":112},[59,414,178],{"class":116},[59,416,417],{"class":69},"\"sales.db\"",[59,419,167],{"class":116},[59,421,423,426,429,432,435],{"class":61,"line":422},17,[59,424,425],{"class":116},"db.executescript(",[59,427,428],{"class":69},"\"DROP TABLE IF EXISTS orders; DROP TABLE IF EXISTS customers;\"",[59,430,431],{"class":112}," +",[59,433,434],{"class":203}," SCHEMA",[59,436,167],{"class":116},[59,438,440,443,446,449,452,455],{"class":61,"line":439},18,[59,441,442],{"class":112},"for",[59,444,445],{"class":116}," (name,) ",[59,447,448],{"class":112},"in",[59,450,451],{"class":116}," db.execute(",[59,453,454],{"class":69},"\"SELECT name FROM sqlite_master WHERE type='table'\"",[59,456,457],{"class":116},"):\n",[59,459,461,464,467,470,473,476,479,482,485,488,491,494,497,500,502,505,507,509,512,514,516,518,521,523,526,528,530,533,536,540,543,546,549],{"class":61,"line":460},19,[59,462,463],{"class":203},"    print",[59,465,466],{"class":116},"(name, [(c[",[59,468,469],{"class":203},"1",[59,471,472],{"class":116},"], c[",[59,474,475],{"class":203},"2",[59,477,478],{"class":116},"], ",[59,480,481],{"class":69},"\"NOT NULL\"",[59,483,484],{"class":112}," if",[59,486,487],{"class":116}," c[",[59,489,490],{"class":203},"3",[59,492,493],{"class":116},"] ",[59,495,496],{"class":112},"else",[59,498,499],{"class":69}," \"\"",[59,501,33],{"class":116},[59,503,504],{"class":69},"\"PK\"",[59,506,484],{"class":112},[59,508,487],{"class":116},[59,510,511],{"class":203},"5",[59,513,493],{"class":116},[59,515,496],{"class":112},[59,517,499],{"class":69},[59,519,520],{"class":116},") ",[59,522,442],{"class":112},[59,524,525],{"class":116}," c ",[59,527,448],{"class":112},[59,529,451],{"class":116},[59,531,532],{"class":112},"f",[59,534,535],{"class":69},"\"PRAGMA table_info(",[59,537,539],{"class":538},"sSjpA","{",[59,541,542],{"class":116},"name",[59,544,545],{"class":538},"}",[59,547,548],{"class":69},")\"",[59,550,551],{"class":116},")])\n",[10,553,554,555,557,558,560,561,563,564,566,567,570,571,574],{},"Every rule the data must follow is now part of the table: primary keys stop duplicate loads, ",[13,556,32],{}," rejects missing values, ",[13,559,36],{}," constraints reject impossible ones — a non-positive quantity, a date that is not in ISO form — and the foreign key ties each order to an existing customer. ",[13,562,28],{}," makes SQLite enforce the column types, which it otherwise treats as suggestions: a ",[13,565,28],{}," integer column refuses ",[13,568,569],{},"'twelve'"," instead of storing it. ",[13,572,573],{},"PRAGMA foreign_keys = ON"," has to be set on every connection, because SQLite leaves foreign keys unenforced by default. Dates are stored as ISO text, which SQLite's date functions understand and which sorts correctly.",[576,577,585,586,585,590,585,594,585,601,585,611,585,622,585,629,585,635,585,640,585,644,585,648,585,652,585,657,585,664,585,669,585,673,585,676,585,679,585,682],"svg",{"viewBox":578,"role":579,"ariaLabelledBy":580,"xmlns":583,"style":584},"4 0 752 233","img",[581,582],"sq-schema-t","sq-schema-d","http:\u002F\u002Fwww.w3.org\u002F2000\u002Fsvg","width:100%;max-width:752px;height:auto;display:block;margin:1.5rem auto;font-family:Inter,ui-sans-serif,system-ui,sans-serif","\n  ",[587,588,589],"title",{"id":581},"The table to_sql infers and the table you design",[591,592,593],"desc",{"id":582},"to_sql creates columns with guessed types and no constraints, so dates become text with times, integers with blanks become REAL, duplicate loads double the data and bad rows are accepted. A designed STRICT table has primary keys, NOT NULL, CHECK and foreign key constraints that reject bad rows.",[595,596],"rect",{"x":597,"y":224,"width":598,"height":599,"fill":600},"4","752","233","#ffffff",[595,602],{"x":603,"y":604,"width":605,"height":606,"rx":607,"fill":608,"stroke":609,"style":610},"20.0","16.0","351.0","201.0","14","#fdefd8","var(--gold,#b4740a)","stroke-width:2px",[612,613,615],"g",{"transform":614},"translate(182.5,29.0) scale(1.083)",[616,617],"path",{"d":618,"fill":619,"stroke":620,"style":621},"M5 6c0-1.7 3.1-3 7-3s7 1.3 7 3-3.1 3-7 3-7-1.3-7-3z M5 6v12c0 1.7 3.1 3 7 3s7-1.3 7-3V6 M5 12c0 1.7 3.1 3 7 3s7-1.3 7-3","none","var(--gold-ink,#7a4e06)","stroke-width:1.66px;stroke-linecap:round;stroke-linejoin:round",[623,624,628],"text",{"x":625,"y":626,"style":627},"195.5","78.0","font-size:14px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","df.to_sql(...)",[61,630],{"x1":631,"y1":632,"x2":633,"y2":632,"stroke":609,"style":634},"36.0","91.0","355.0","stroke-width:1px",[623,636,639],{"x":625,"y":637,"style":638},"111.0","font-size:12px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","guessed types",[623,641,643],{"x":625,"y":642,"style":638},"137.0","Qty as REAL (blank)",[623,645,647],{"x":625,"y":646,"style":638},"163.0","no primary key",[623,649,651],{"x":625,"y":650,"style":638},"189.0","accepts every row",[595,653],{"x":654,"y":604,"width":605,"height":606,"rx":607,"fill":655,"stroke":656,"style":610},"389.0","#d9f4f1","var(--teal,#0f9488)",[612,658,660],{"transform":659},"translate(551.5,29.0) scale(1.083)",[616,661],{"d":662,"fill":619,"stroke":663,"style":621},"M6 11h12v9H6z M8.5 11V8a3.5 3.5 0 0 1 7 0v3","var(--teal-ink,#0b6157)",[623,665,668],{"x":666,"y":626,"style":667},"564.5","font-size:14px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","Designed STRICT schema",[61,670],{"x1":671,"y1":632,"x2":672,"y2":632,"stroke":656,"style":634},"405.0","724.0",[623,674,675],{"x":666,"y":637,"style":638},"types enforced",[623,677,678],{"x":666,"y":642,"style":638},"qty INTEGER CHECK > 0",[623,680,681],{"x":666,"y":646,"style":638},"PRIMARY KEY, FOREIGN KEY",[623,683,684],{"x":666,"y":650,"style":638},"rejects bad rows",[45,686,688],{"id":687},"step-3-convert-values-deliberately-and-load-in-one-transaction","Step 3 — Convert values deliberately and load in one transaction",[10,690,691],{},"Convert each column to the type the schema expects, then insert inside a transaction. Insert row by row inside it, collecting the rows the database rejects, and commit only if there are none:",[50,693,695],{"className":103,"code":694,"language":105,"meta":55,"style":55},"customers = pd.read_excel(\"sales.xlsx\", sheet_name=\"Customers\")\n\ndef to_rows(df, columns, convert):\n    return [tuple(convert[c](v) for c, v in zip(columns, row)) for row in df[columns].itertuples(index=False)]\n\ndef nullable(f):\n    return lambda v: None if pd.isna(v) else f(v)\n\ncust_rows = to_rows(customers, [\"CustomerID\", \"Name\", \"Country\"], {\"CustomerID\": str, \"Name\": str, \"Country\": str})\norder_rows = to_rows(orders, [\"OrderID\", \"CustomerID\", \"OrderDate\", \"Qty\", \"Amount\"],\n                     {\"OrderID\": int, \"CustomerID\": str, \"OrderDate\": nullable(lambda d: d.date().isoformat()),\n                      \"Qty\": nullable(int), \"Amount\": nullable(lambda a: round(float(a), 2))})\n\ndef load(db, table, rows):\n    placeholders = \",\".join(\"?\" * len(rows[0]))\n    rejected = []\n    for row in rows:\n        try:\n            db.execute(f\"INSERT INTO {table} VALUES ({placeholders})\", row)\n        except sqlite3.IntegrityError as e:\n            rejected.append((row, str(e)))\n    return rejected\n\nwith db:\n    bad = load(db, \"customers\", cust_rows) + load(db, \"orders\", order_rows)\n    if bad:\n        db.rollback()\nfor row, err in bad:\n    print(f\"rejected {row[0]}: {err}\")\nprint(\"orders in database:\", db.execute(\"SELECT COUNT(*) FROM orders\").fetchone()[0])\n",[13,696,697,719,723,735,781,785,795,818,822,875,910,943,980,984,994,1024,1034,1046,1054,1086,1100,1111,1119,1124,1133,1160,1169,1175,1187,1225],{"__ignoreMap":55},[59,698,699,702,704,706,708,710,712,714,717],{"class":61,"line":62},[59,700,701],{"class":116},"customers ",[59,703,147],{"class":112},[59,705,150],{"class":116},[59,707,153],{"class":69},[59,709,33],{"class":116},[59,711,159],{"class":158},[59,713,147],{"class":112},[59,715,716],{"class":69},"\"Customers\"",[59,718,167],{"class":116},[59,720,721],{"class":61,"line":120},[59,722,138],{"emptyLinePlaceholder":137},[59,724,725,728,732],{"class":61,"line":134},[59,726,727],{"class":112},"def",[59,729,731],{"class":730},"s_Opv"," to_rows",[59,733,734],{"class":116},"(df, columns, convert):\n",[59,736,737,740,743,746,749,751,754,756,759,762,764,767,769,772,774,776,778],{"class":61,"line":141},[59,738,739],{"class":112},"    return",[59,741,742],{"class":116}," [",[59,744,745],{"class":203},"tuple",[59,747,748],{"class":116},"(convert[c](v) ",[59,750,442],{"class":112},[59,752,753],{"class":116}," c, v ",[59,755,448],{"class":112},[59,757,758],{"class":203}," zip",[59,760,761],{"class":116},"(columns, row)) ",[59,763,442],{"class":112},[59,765,766],{"class":116}," row ",[59,768,448],{"class":112},[59,770,771],{"class":116}," df[columns].itertuples(",[59,773,198],{"class":158},[59,775,147],{"class":112},[59,777,204],{"class":203},[59,779,780],{"class":116},")]\n",[59,782,783],{"class":61,"line":170},[59,784,138],{"emptyLinePlaceholder":137},[59,786,787,789,792],{"class":61,"line":186},[59,788,727],{"class":112},[59,790,791],{"class":730}," nullable",[59,793,794],{"class":116},"(f):\n",[59,796,797,799,802,805,808,810,813,815],{"class":61,"line":209},[59,798,739],{"class":112},[59,800,801],{"class":112}," lambda",[59,803,804],{"class":116}," v: ",[59,806,807],{"class":203},"None",[59,809,484],{"class":112},[59,811,812],{"class":116}," pd.isna(v) ",[59,814,496],{"class":112},[59,816,817],{"class":116}," f(v)\n",[59,819,820],{"class":61,"line":230},[59,821,138],{"emptyLinePlaceholder":137},[59,823,824,827,829,832,835,837,840,842,845,848,850,853,856,858,860,862,864,866,868,870,872],{"class":61,"line":243},[59,825,826],{"class":116},"cust_rows ",[59,828,147],{"class":112},[59,830,831],{"class":116}," to_rows(customers, [",[59,833,834],{"class":69},"\"CustomerID\"",[59,836,33],{"class":116},[59,838,839],{"class":69},"\"Name\"",[59,841,33],{"class":116},[59,843,844],{"class":69},"\"Country\"",[59,846,847],{"class":116},"], {",[59,849,834],{"class":69},[59,851,852],{"class":116},": ",[59,854,855],{"class":203},"str",[59,857,33],{"class":116},[59,859,839],{"class":69},[59,861,852],{"class":116},[59,863,855],{"class":203},[59,865,33],{"class":116},[59,867,844],{"class":69},[59,869,852],{"class":116},[59,871,855],{"class":203},[59,873,874],{"class":116},"})\n",[59,876,877,880,882,885,888,890,892,894,897,899,902,904,907],{"class":61,"line":270},[59,878,879],{"class":116},"order_rows ",[59,881,147],{"class":112},[59,883,884],{"class":116}," to_rows(orders, [",[59,886,887],{"class":69},"\"OrderID\"",[59,889,33],{"class":116},[59,891,834],{"class":69},[59,893,33],{"class":116},[59,895,896],{"class":69},"\"OrderDate\"",[59,898,33],{"class":116},[59,900,901],{"class":69},"\"Qty\"",[59,903,33],{"class":116},[59,905,906],{"class":69},"\"Amount\"",[59,908,909],{"class":116},"],\n",[59,911,912,915,917,919,922,924,926,928,930,932,934,937,940],{"class":61,"line":293},[59,913,914],{"class":116},"                     {",[59,916,887],{"class":69},[59,918,852],{"class":116},[59,920,921],{"class":203},"int",[59,923,33],{"class":116},[59,925,834],{"class":69},[59,927,852],{"class":116},[59,929,855],{"class":203},[59,931,33],{"class":116},[59,933,896],{"class":69},[59,935,936],{"class":116},": nullable(",[59,938,939],{"class":112},"lambda",[59,941,942],{"class":116}," d: d.date().isoformat()),\n",[59,944,945,948,950,952,955,957,959,961,964,967,969,972,975,977],{"class":61,"line":384},[59,946,947],{"class":69},"                      \"Qty\"",[59,949,936],{"class":116},[59,951,921],{"class":203},[59,953,954],{"class":116},"), ",[59,956,906],{"class":69},[59,958,936],{"class":116},[59,960,939],{"class":112},[59,962,963],{"class":116}," a: ",[59,965,966],{"class":203},"round",[59,968,275],{"class":116},[59,970,971],{"class":203},"float",[59,973,974],{"class":116},"(a), ",[59,976,475],{"class":203},[59,978,979],{"class":116},"))})\n",[59,981,982],{"class":61,"line":390},[59,983,138],{"emptyLinePlaceholder":137},[59,985,986,988,991],{"class":61,"line":396},[59,987,727],{"class":112},[59,989,990],{"class":730}," load",[59,992,993],{"class":116},"(db, table, rows):\n",[59,995,996,999,1001,1004,1007,1010,1013,1016,1019,1021],{"class":61,"line":401},[59,997,998],{"class":116},"    placeholders ",[59,1000,147],{"class":112},[59,1002,1003],{"class":69}," \",\"",[59,1005,1006],{"class":116},".join(",[59,1008,1009],{"class":69},"\"?\"",[59,1011,1012],{"class":112}," *",[59,1014,1015],{"class":203}," len",[59,1017,1018],{"class":116},"(rows[",[59,1020,224],{"class":203},[59,1022,1023],{"class":116},"]))\n",[59,1025,1026,1029,1031],{"class":61,"line":407},[59,1027,1028],{"class":116},"    rejected ",[59,1030,147],{"class":112},[59,1032,1033],{"class":116}," []\n",[59,1035,1036,1039,1041,1043],{"class":61,"line":422},[59,1037,1038],{"class":112},"    for",[59,1040,766],{"class":116},[59,1042,448],{"class":112},[59,1044,1045],{"class":116}," rows:\n",[59,1047,1048,1051],{"class":61,"line":439},[59,1049,1050],{"class":112},"        try",[59,1052,1053],{"class":116},":\n",[59,1055,1056,1059,1061,1064,1066,1069,1071,1074,1076,1079,1081,1083],{"class":61,"line":460},[59,1057,1058],{"class":116},"            db.execute(",[59,1060,532],{"class":112},[59,1062,1063],{"class":69},"\"INSERT INTO ",[59,1065,539],{"class":538},[59,1067,1068],{"class":116},"table",[59,1070,545],{"class":538},[59,1072,1073],{"class":69}," VALUES (",[59,1075,539],{"class":538},[59,1077,1078],{"class":116},"placeholders",[59,1080,545],{"class":538},[59,1082,548],{"class":69},[59,1084,1085],{"class":116},", row)\n",[59,1087,1089,1092,1095,1097],{"class":61,"line":1088},20,[59,1090,1091],{"class":112},"        except",[59,1093,1094],{"class":116}," sqlite3.IntegrityError ",[59,1096,128],{"class":112},[59,1098,1099],{"class":116}," e:\n",[59,1101,1103,1106,1108],{"class":61,"line":1102},21,[59,1104,1105],{"class":116},"            rejected.append((row, ",[59,1107,855],{"class":203},[59,1109,1110],{"class":116},"(e)))\n",[59,1112,1114,1116],{"class":61,"line":1113},22,[59,1115,739],{"class":112},[59,1117,1118],{"class":116}," rejected\n",[59,1120,1122],{"class":61,"line":1121},23,[59,1123,138],{"emptyLinePlaceholder":137},[59,1125,1127,1130],{"class":61,"line":1126},24,[59,1128,1129],{"class":112},"with",[59,1131,1132],{"class":116}," db:\n",[59,1134,1136,1139,1141,1144,1147,1150,1153,1155,1157],{"class":61,"line":1135},25,[59,1137,1138],{"class":116},"    bad ",[59,1140,147],{"class":112},[59,1142,1143],{"class":116}," load(db, ",[59,1145,1146],{"class":69},"\"customers\"",[59,1148,1149],{"class":116},", cust_rows) ",[59,1151,1152],{"class":112},"+",[59,1154,1143],{"class":116},[59,1156,192],{"class":69},[59,1158,1159],{"class":116},", order_rows)\n",[59,1161,1163,1166],{"class":61,"line":1162},26,[59,1164,1165],{"class":112},"    if",[59,1167,1168],{"class":116}," bad:\n",[59,1170,1172],{"class":61,"line":1171},27,[59,1173,1174],{"class":116},"        db.rollback()\n",[59,1176,1178,1180,1183,1185],{"class":61,"line":1177},28,[59,1179,442],{"class":112},[59,1181,1182],{"class":116}," row, err ",[59,1184,448],{"class":112},[59,1186,1168],{"class":116},[59,1188,1190,1192,1194,1196,1199,1201,1204,1206,1209,1211,1213,1215,1218,1220,1223],{"class":61,"line":1189},29,[59,1191,463],{"class":203},[59,1193,275],{"class":116},[59,1195,532],{"class":112},[59,1197,1198],{"class":69},"\"rejected ",[59,1200,539],{"class":538},[59,1202,1203],{"class":116},"row[",[59,1205,224],{"class":203},[59,1207,1208],{"class":116},"]",[59,1210,545],{"class":538},[59,1212,852],{"class":69},[59,1214,539],{"class":538},[59,1216,1217],{"class":116},"err",[59,1219,545],{"class":538},[59,1221,1222],{"class":69},"\"",[59,1224,167],{"class":116},[59,1226,1228,1230,1232,1235,1238,1240,1242,1244],{"class":61,"line":1227},30,[59,1229,212],{"class":203},[59,1231,275],{"class":116},[59,1233,1234],{"class":69},"\"orders in database:\"",[59,1236,1237],{"class":116},", db.execute(",[59,1239,284],{"class":69},[59,1241,221],{"class":116},[59,1243,224],{"class":203},[59,1245,227],{"class":116},[10,1247,1248,1249,1251,1252,1254,1255,1257,1258,1261,1262,1265,1266,1269,1270,1273],{},"Every value is converted by an explicit function: dates to ISO strings with the midnight time dropped, quantities to ",[13,1250,921],{}," with blanks turned into ",[13,1253,807],{}," so the ",[13,1256,32],{}," rule can catch them, amounts rounded to pennies. Each insert either succeeds or raises ",[13,1259,1260],{},"IntegrityError"," naming the rule — ",[13,1263,1264],{},"FOREIGN KEY constraint failed"," for the order from customer C004, ",[13,1267,1268],{},"NOT NULL constraint failed: orders.qty"," for the blank, ",[13,1271,1272],{},"CHECK constraint failed: qty > 0"," for the negative. Because rejected rows were found, the whole load is rolled back and the database is left exactly as it was: no half-loaded day of data. Choosing all-or-nothing makes reruns simple — fix the sheet and load again.",[576,1275,585,1280,585,1283,585,1286,585,1289,585,1296,585,1303,585,1309,585,1314,585,1319,585,1324,585,1329,585,1336,585,1341,585,1344,585,1347,585,1350,585,1353,585,1359,585,1365,585,1369,585,1373,585,1377,585,1380,585,1383,585,1389,585,1394,585,1397],{"viewBox":1276,"role":579,"ariaLabelledBy":1277,"xmlns":583,"style":584},"4 0 752 142",[1278,1279],"sq-load-t","sq-load-d",[587,1281,1282],{"id":1278},"A schema-first load",[591,1284,1285],{"id":1279},"Create STRICT tables with keys and constraints, read each sheet, convert every column explicitly, insert all rows in one transaction while collecting constraint errors, and commit only if no row was rejected, otherwise roll back and report the rejected rows with the rule they broke.",[595,1287],{"x":597,"y":224,"width":598,"height":1288,"fill":600},"142",[595,1290],{"x":603,"y":604,"width":1291,"height":1292,"rx":1293,"fill":1294,"stroke":1295,"style":610},"153.0","110.5","12","#f0f4ff","var(--brand,#5b5cf0)",[612,1297,1299],{"transform":1298},"translate(84.5,37.8) scale(1.000)",[616,1300],{"d":618,"fill":619,"stroke":1301,"style":1302},"var(--brand-strong,#4338ca)","stroke-width:1.80px;stroke-linecap:round;stroke-linejoin:round",[623,1304,1308],{"x":1305,"y":1306,"style":1307},"96.5","84.2","font-size:13.5px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","Create schema",[623,1310,1313],{"x":1305,"y":1311,"style":1312},"103.8","font-size:11.5px;font-weight:400;fill:var(--text,#172033);text-anchor:middle","STRICT, keys, CHECK",[61,1315],{"x1":1316,"y1":1317,"x2":1318,"y2":1317,"stroke":1295,"style":610},"178.0","71.2","200.0",[1320,1321],"polygon",{"points":1322,"fill":1323},"204.0,71.2 195.0,76.2 195.0,66.2","#5b5cf0",[595,1325],{"x":1326,"y":604,"width":1291,"height":1292,"rx":1293,"fill":1327,"stroke":1328,"style":610},"209.0","#e7ebef","var(--line,#cdd5e6)",[612,1330,1332],{"transform":1331},"translate(273.5,37.8) scale(1.000)",[616,1333],{"d":1334,"fill":619,"stroke":1335,"style":1302},"M20 11a8 8 0 1 0-2.3 5.7 M20 4v7h-7","var(--text,#172033)",[623,1337,1340],{"x":1338,"y":1306,"style":1339},"285.5","font-size:13.5px;font-weight:700;fill:var(--text,#172033);text-anchor:middle","Convert values",[623,1342,1343],{"x":1338,"y":1311,"style":1312},"ISO dates, int, None",[61,1345],{"x1":1346,"y1":1317,"x2":654,"y2":1317,"stroke":1295,"style":610},"367.0",[1320,1348],{"points":1349,"fill":1323},"393.0,71.2 384.0,76.2 384.0,66.2",[595,1351],{"x":1352,"y":604,"width":1291,"height":1292,"rx":1293,"fill":608,"stroke":609,"style":610},"398.0",[612,1354,1356],{"transform":1355},"translate(462.5,29.0) scale(1.000)",[616,1357],{"d":1358,"fill":619,"stroke":620,"style":1302},"M12 3l9 5-9 5-9-5z M3 13l9 5 9-5 M3 17l9 5 9-5",[623,1360,1364],{"x":1361,"y":1362,"style":1363},"474.5","75.5","font-size:13.5px;font-weight:700;fill:var(--gold-ink,#7a4e06);text-anchor:middle","Insert in",[623,1366,1368],{"x":1361,"y":1367,"style":1363},"93.0","transaction",[623,1370,1372],{"x":1361,"y":1371,"style":1312},"112.5","collect IntegrityErrors",[61,1374],{"x1":1375,"y1":1317,"x2":1376,"y2":1317,"stroke":1295,"style":610},"556.0","578.0",[1320,1378],{"points":1379,"fill":1323},"582.0,71.2 573.0,76.2 573.0,66.2",[595,1381],{"x":1382,"y":604,"width":1291,"height":1292,"rx":1293,"fill":655,"stroke":656,"style":610},"587.0",[612,1384,1386],{"transform":1385},"translate(651.5,29.0) scale(1.000)",[616,1387],{"d":1388,"fill":619,"stroke":663,"style":1302},"M5 12.5l4.5 4.5L19 7",[623,1390,1393],{"x":1391,"y":1362,"style":1392},"663.5","font-size:13.5px;font-weight:700;fill:var(--teal-ink,#0b6157);text-anchor:middle","Commit or roll",[623,1395,1396],{"x":1391,"y":1367,"style":1392},"back",[623,1398,1399],{"x":1391,"y":1371,"style":1312},"all or nothing",[576,1401,585,1406,585,1409,585,1412,585,1415,585,1419,585,1423,585,1429,585,1432,585,1436,585,1440,585,1445,585,1447,585,1451,585,1453,585,1456,585,1459,585,1463,585,1465,585,1468,585,1470,585,1473,585,1476,585,1480,585,1482,585,1486,585,1488,585,1491,585,1494,585,1498,585,1500,585,1503,585,1505],{"viewBox":1402,"role":579,"ariaLabelledBy":1403,"xmlns":583,"style":584},"4 0 752 208",[1404,1405],"sq-rejects-t","sq-rejects-d",[587,1407,1408],{"id":1404},"Which rule caught each bad order",[591,1410,1411],{"id":1405},"Order 5003 has a blank quantity and is rejected by the NOT NULL constraint. Order 5004 refers to customer C004, which does not exist, and is rejected by the foreign key. Order 5005 has a negative quantity and is rejected by the CHECK constraint. The other three orders load.",[595,1413],{"x":597,"y":224,"width":598,"height":1414,"fill":600},"208",[595,1416],{"x":603,"y":604,"width":1417,"height":1418,"rx":224,"fill":600,"stroke":1328,"style":634},"143.0","40.0",[595,1420],{"x":646,"y":604,"width":1421,"height":1418,"rx":224,"fill":1294,"stroke":1295,"style":1422},"288.5","stroke-width:1.2px",[623,1424,1428],{"x":1425,"y":1426,"style":1427},"307.3","41.0","font-size:13px;font-weight:700;fill:var(--brand-strong,#4338ca);text-anchor:middle","problem in the sheet",[595,1430],{"x":1431,"y":604,"width":1421,"height":1418,"rx":224,"fill":1294,"stroke":1295,"style":1422},"451.5",[623,1433,1435],{"x":1434,"y":1426,"style":1427},"595.8","rule that rejected it",[595,1437],{"x":603,"y":1438,"width":1417,"height":1439,"rx":224,"fill":1327,"stroke":1328,"style":634},"56.0","34.0",[623,1441,1444],{"x":1442,"y":626,"style":1443},"32.0","font-size:12.5px;font-weight:700;fill:var(--text,#172033);text-anchor:start","5003",[595,1446],{"x":646,"y":1438,"width":1421,"height":1439,"rx":224,"fill":600,"stroke":1328,"style":634},[623,1448,1450],{"x":1425,"y":1449,"style":1312},"77.0","blank Qty",[595,1452],{"x":1431,"y":1438,"width":1421,"height":1439,"rx":224,"fill":600,"stroke":1328,"style":634},[623,1454,1455],{"x":1434,"y":1449,"style":1312},"NOT NULL (qty)",[595,1457],{"x":603,"y":1458,"width":1417,"height":1439,"rx":224,"fill":1327,"stroke":1328,"style":634},"90.0",[623,1460,1462],{"x":1442,"y":1461,"style":1443},"112.0","5004",[595,1464],{"x":646,"y":1458,"width":1421,"height":1439,"rx":224,"fill":600,"stroke":1328,"style":634},[623,1466,1467],{"x":1425,"y":637,"style":1312},"customer C004 unknown",[595,1469],{"x":1431,"y":1458,"width":1421,"height":1439,"rx":224,"fill":600,"stroke":1328,"style":634},[623,1471,1472],{"x":1434,"y":637,"style":1312},"FOREIGN KEY",[595,1474],{"x":603,"y":1475,"width":1417,"height":1439,"rx":224,"fill":1327,"stroke":1328,"style":634},"124.0",[623,1477,1479],{"x":1442,"y":1478,"style":1443},"146.0","5005",[595,1481],{"x":646,"y":1475,"width":1421,"height":1439,"rx":224,"fill":600,"stroke":1328,"style":634},[623,1483,1485],{"x":1425,"y":1484,"style":1312},"145.0","Qty = -2",[595,1487],{"x":1431,"y":1475,"width":1421,"height":1439,"rx":224,"fill":600,"stroke":1328,"style":634},[623,1489,1490],{"x":1434,"y":1484,"style":1312},"CHECK (qty > 0)",[595,1492],{"x":603,"y":1493,"width":1417,"height":1439,"rx":224,"fill":1327,"stroke":1328,"style":634},"158.0",[623,1495,1497],{"x":1442,"y":1496,"style":1443},"180.0","5001, 5002, 5006",[595,1499],{"x":646,"y":1493,"width":1421,"height":1439,"rx":224,"fill":600,"stroke":1328,"style":634},[623,1501,619],{"x":1425,"y":1502,"style":1312},"179.0",[595,1504],{"x":1431,"y":1493,"width":1421,"height":1439,"rx":224,"fill":600,"stroke":1328,"style":634},[623,1506,1507],{"x":1434,"y":1502,"style":1312},"loaded",[45,1509,1511],{"id":1510},"step-4-fix-reload-and-verify","Step 4 — Fix, reload and verify",[50,1513,1515],{"className":103,"code":1514,"language":105,"meta":55,"style":55},"fixed = [r for r in order_rows if r[0] not in {row[0] for row, _ in bad}]\nwith db:\n    assert not load(db, \"customers\", cust_rows) and not load(db, \"orders\", fixed)\nprint(db.execute(\"\"\"SELECT c.name, COUNT(*), SUM(o.amount), MIN(o.order_date)\n                    FROM orders o JOIN customers c USING (customer_id) GROUP BY c.name ORDER BY c.name\"\"\").fetchall())\ntry:\n    with db:\n        load_again = load(db, \"orders\", fixed)\n        if load_again:\n            raise sqlite3.IntegrityError(load_again[0][1])\nexcept sqlite3.IntegrityError as e:\n    print(\"loading the same orders again:\", e)\ndb.close()\n",[13,1516,1517,1570,1576,1602,1612,1619,1626,1633,1646,1654,1671,1682,1694],{"__ignoreMap":55},[59,1518,1519,1522,1524,1527,1529,1532,1534,1537,1540,1543,1545,1547,1550,1553,1556,1558,1560,1562,1565,1567],{"class":61,"line":62},[59,1520,1521],{"class":116},"fixed ",[59,1523,147],{"class":112},[59,1525,1526],{"class":116}," [r ",[59,1528,442],{"class":112},[59,1530,1531],{"class":116}," r ",[59,1533,448],{"class":112},[59,1535,1536],{"class":116}," order_rows ",[59,1538,1539],{"class":112},"if",[59,1541,1542],{"class":116}," r[",[59,1544,224],{"class":203},[59,1546,493],{"class":116},[59,1548,1549],{"class":112},"not",[59,1551,1552],{"class":112}," in",[59,1554,1555],{"class":116}," {row[",[59,1557,224],{"class":203},[59,1559,493],{"class":116},[59,1561,442],{"class":112},[59,1563,1564],{"class":116}," row, _ ",[59,1566,448],{"class":112},[59,1568,1569],{"class":116}," bad}]\n",[59,1571,1572,1574],{"class":61,"line":120},[59,1573,1129],{"class":112},[59,1575,1132],{"class":116},[59,1577,1578,1581,1584,1586,1588,1590,1593,1595,1597,1599],{"class":61,"line":134},[59,1579,1580],{"class":112},"    assert",[59,1582,1583],{"class":112}," not",[59,1585,1143],{"class":116},[59,1587,1146],{"class":69},[59,1589,1149],{"class":116},[59,1591,1592],{"class":112},"and",[59,1594,1583],{"class":112},[59,1596,1143],{"class":116},[59,1598,192],{"class":69},[59,1600,1601],{"class":116},", fixed)\n",[59,1603,1604,1606,1609],{"class":61,"line":141},[59,1605,212],{"class":203},[59,1607,1608],{"class":116},"(db.execute(",[59,1610,1611],{"class":69},"\"\"\"SELECT c.name, COUNT(*), SUM(o.amount), MIN(o.order_date)\n",[59,1613,1614,1617],{"class":61,"line":170},[59,1615,1616],{"class":69},"                    FROM orders o JOIN customers c USING (customer_id) GROUP BY c.name ORDER BY c.name\"\"\"",[59,1618,240],{"class":116},[59,1620,1621,1624],{"class":61,"line":186},[59,1622,1623],{"class":112},"try",[59,1625,1053],{"class":116},[59,1627,1628,1631],{"class":61,"line":209},[59,1629,1630],{"class":112},"    with",[59,1632,1132],{"class":116},[59,1634,1635,1638,1640,1642,1644],{"class":61,"line":230},[59,1636,1637],{"class":116},"        load_again ",[59,1639,147],{"class":112},[59,1641,1143],{"class":116},[59,1643,192],{"class":69},[59,1645,1601],{"class":116},[59,1647,1648,1651],{"class":61,"line":243},[59,1649,1650],{"class":112},"        if",[59,1652,1653],{"class":116}," load_again:\n",[59,1655,1656,1659,1662,1664,1667,1669],{"class":61,"line":270},[59,1657,1658],{"class":112},"            raise",[59,1660,1661],{"class":116}," sqlite3.IntegrityError(load_again[",[59,1663,224],{"class":203},[59,1665,1666],{"class":116},"][",[59,1668,469],{"class":203},[59,1670,227],{"class":116},[59,1672,1673,1676,1678,1680],{"class":61,"line":293},[59,1674,1675],{"class":112},"except",[59,1677,1094],{"class":116},[59,1679,128],{"class":112},[59,1681,1099],{"class":116},[59,1683,1684,1686,1688,1691],{"class":61,"line":384},[59,1685,463],{"class":203},[59,1687,275],{"class":116},[59,1689,1690],{"class":69},"\"loading the same orders again:\"",[59,1692,1693],{"class":116},", e)\n",[59,1695,1696],{"class":61,"line":390},[59,1697,1698],{"class":116},"db.close()\n",[10,1700,1701,1702,1705,1706,43],{},"In a real process the three rejected orders would go back to whoever maintains the sheet; here they are simply left out, and the remaining three load cleanly. A join confirms the result — each customer's order count, total and first order date, with dates sorting correctly because they are ISO strings. Loading the same orders a second time now fails on the primary key, which is exactly the protection the inferred table lacked. Loads that should ",[18,1703,1704],{},"update"," existing rows rather than refuse them use an upsert, covered in ",[39,1707,1709],{"href":1708},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fupsert-excel-rows-into-a-database-table\u002F","Upsert Excel Rows into a Database Table with Python",[45,1711,1713],{"id":1712},"keeping-the-schema-with-the-code","Keeping the schema with the code",[10,1715,1716,1717,1720,1721,1724],{},"Put the ",[13,1718,1719],{},"CREATE TABLE"," statements in a ",[13,1722,1723],{},".sql"," file next to the loading script and in version control, rather than letting the first load decide the structure. Reviewers can then see the rules the data must meet, changes to them show up as diffs, and a fresh database can be built from the file in a test. When the spreadsheet gains a column, the load fails loudly until the schema is updated — an inconvenience that is far preferable to new data quietly not arriving. The same discipline carries over unchanged to PostgreSQL or SQL Server, where the constraint syntax is almost identical; SQLite is simply the quickest place to start and test.",[45,1726,1728],{"id":1727},"common-pitfalls","Common pitfalls",[1730,1731,1732,1743,1751,1759,1771],"ul",{},[1733,1734,1735,1742],"li",{},[1736,1737,1738,1739,1741],"strong",{},"Letting ",[13,1740,24],{}," create the table."," You get guessed types and no rules.",[1733,1744,1745,1750],{},[1736,1746,1747,1748,43],{},"Forgetting ",[13,1749,573],{}," SQLite ignores foreign keys without it.",[1733,1752,1753,1756,1757,43],{},[1736,1754,1755],{},"Non-STRICT tables."," SQLite stores wrong types happily unless the table is ",[13,1758,28],{},[1733,1760,1761,1764,1765,1767,1768,1770],{},[1736,1762,1763],{},"Blanks as NaN in integer columns."," Convert to ",[13,1766,807],{}," so ",[13,1769,32],{}," can catch them.",[1733,1772,1773,1776],{},[1736,1774,1775],{},"Partial loads."," Insert in one transaction and roll back on any rejected row.",[45,1778,1780],{"id":1779},"conclusion","Conclusion",[10,1782,1783,1784,1786,1787,33,1789,1791,1792,1794],{},"Define SQLite tables yourself — ",[13,1785,28],{},", with primary keys, ",[13,1788,32],{},[13,1790,36],{}," and foreign-key constraints, and foreign keys switched on — then convert each Excel column explicitly and insert inside one transaction, collecting ",[13,1793,1260],{},"s and committing only when no row was rejected. The database then refuses duplicates, blanks, impossible values and orphan records, and every rejection comes with the rule that caught it.",[45,1796,1798],{"id":1797},"frequently-asked-questions","Frequently asked questions",[10,1800,1801,1804],{},[1736,1802,1803],{},"How do I load an Excel sheet into SQLite with my own table schema?","\nCreate the table with CREATE TABLE first, then insert the rows with executemany or row by row, or use DataFrame.to_sql with if_exists=\"append\" into the existing table.",[10,1806,1807,1810],{},[1736,1808,1809],{},"Why does pandas to_sql store my integers as REAL?","\nA blank cell makes pandas read the column as float; convert the column explicitly and turn blanks into None before inserting.",[10,1812,1813,1816],{},[1736,1814,1815],{},"How do I make SQLite enforce column types?","\nDeclare the table as STRICT (SQLite 3.37 or later), so values of the wrong type are rejected.",[10,1818,1819,1822],{},[1736,1820,1821],{},"How do I see which Excel rows broke a constraint?","\nInsert rows individually inside one transaction, catch sqlite3.IntegrityError for each, record the row and message, and roll back if any were rejected.",[45,1824,1826],{"id":1825},"related","Related",[1730,1828,1829,1835,1842,1849],{},[1733,1830,1831,1832,1834],{},"Up one level: ",[39,1833,42],{"href":41}," — Excel and SQL.",[1733,1836,1837,1841],{},[39,1838,1840],{"href":1839},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fload-an-excel-file-into-a-sql-database-with-pandas\u002F","Load an Excel File into a SQL Database with pandas"," — the quick approach.",[1733,1843,1844,1848],{},[39,1845,1847],{"href":1846},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fcompare-an-excel-file-with-a-database-table\u002F","Compare an Excel File with a Database Table"," — checking what was loaded.",[1733,1850,1851,1855],{},[39,1852,1854],{"href":1853},"\u002Fadvanced-data-transformation-and-cleaning\u002Fvalidating-excel-data-with-python\u002Fvalidate-excel-data-with-pandera-schemas\u002F","Validate Excel Data with pandera Schemas"," — validating before the database.",[1857,1858,1859],"style",{},"html pre.shiki code .sMTad, html code.shiki .sMTad{--shiki-default:#6F42C1;--shiki-dark:#FFB757}html pre.shiki code .srMev, html code.shiki .srMev{--shiki-default:#032F62;--shiki-dark:#ADDCFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .s-kum, html code.shiki .s-kum{--shiki-default:#D73A49;--shiki-dark:#FF9492}html pre.shiki code .skGVy, html code.shiki .skGVy{--shiki-default:#24292E;--shiki-dark:#F0F3F6}html pre.shiki code .sa561, html code.shiki .sa561{--shiki-default:#E36209;--shiki-dark:#FFB757}html pre.shiki code .sP0c6, html code.shiki .sP0c6{--shiki-default:#005CC5;--shiki-dark:#91CBFF}html pre.shiki code .sSjpA, html code.shiki .sSjpA{--shiki-default:#005CC5;--shiki-dark:#FF9492}html pre.shiki code .s_Opv, html code.shiki .s_Opv{--shiki-default:#6F42C1;--shiki-dark:#DBB7FF}",{"title":55,"searchDepth":120,"depth":120,"links":1861},[1862,1863,1865,1866,1867,1868,1869,1870,1871,1872],{"id":47,"depth":120,"text":48},{"id":96,"depth":120,"text":1864},"Step 1 — What to_sql invents",{"id":314,"depth":120,"text":315},{"id":687,"depth":120,"text":688},{"id":1510,"depth":120,"text":1511},{"id":1712,"depth":120,"text":1713},{"id":1727,"depth":120,"text":1728},{"id":1779,"depth":120,"text":1780},{"id":1797,"depth":120,"text":1798},{"id":1825,"depth":120,"text":1826},"2026-10-02","Load Excel sheets into SQLite tables that you designed — proper types, primary keys, NOT NULL and CHECK constraints, foreign keys and STRICT typing — instead of whatever pandas.to_sql guesses: create the schema first, convert dates and decimals deliberately, insert in one transaction so a bad row rolls the whole load back, and report exactly which rows broke which rule.","md",[1877,1879,1881,1883],{"q":1803,"a":1878},"Create the table with CREATE TABLE first, then insert the rows with executemany or row by row, or use DataFrame.to_sql with if_exists=\"append\" into the existing table.",{"q":1809,"a":1880},"A blank cell makes pandas read the column as float; convert the column explicitly and turn blanks into None before inserting.",{"q":1815,"a":1882},"Declare the table as STRICT (SQLite 3.37 or later), so values of the wrong type are rejected.",{"q":1821,"a":1884},"Insert rows individually inside one transaction, catch sqlite3.IntegrityError for each, record the row and message, and roll back if any were rejected.","load excel into sqlite with a defined schema",null,{"breadcrumb":1888},[1889,1892,1895],{"name":1890,"item":1891},"Home","\u002F",{"name":1893,"item":1894},"Advanced Data Transformation and Cleaning for Excel Automation","\u002Fadvanced-data-transformation-and-cleaning\u002F",{"name":42,"item":41},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fload-excel-into-sqlite-with-a-defined-schema",{"title":1898,"description":1899},"Load Excel into SQLite with a Defined Schema in Python","Excel to SQLite with a real schema: why to_sql's inferred tables are weak, CREATE TABLE with STRICT, keys and CHECK constraints, foreign keys, ISO dates, transactional loads, constraint errors per row and verification queries.","load-excel-into-sqlite-with-a-defined-schema","advanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fload-excel-into-sqlite-with-a-defined-schema\u002Findex",[1903,1904],"data-sources","pandas","how-to","7jKaFnk58MRb6miGgHj3Ff2QRLhSRZYsB106GQavP94",[1908,1912],{"title":1909,"path":1910,"stem":1911,"children":-1},"Load Excel Data into MongoDB with Python","\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fload-excel-data-into-mongodb-with-python","advanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fload-excel-data-into-mongodb-with-python\u002Findex",{"title":1913,"path":1914,"stem":1915,"children":-1},"Query a Database for IDs Listed in an Excel Sheet","\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fquery-a-database-for-ids-listed-in-an-excel-sheet","advanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fquery-a-database-for-ids-listed-in-an-excel-sheet\u002Findex",[1917,1919],{"path":1918,"title":1893},"\u002Fadvanced-data-transformation-and-cleaning",{"path":1920,"title":42},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases",[],[1923,1926,1929,1931,1934,1937],{"path":1924,"title":1925,"kind":1905},"\u002Fautomating-reporting-workflows\u002Fpublishing-excel-reports-to-cloud-storage\u002Fread-an-excel-file-directly-from-amazon-s3-with-pandas","Read an Excel File Directly from Amazon S3 with pandas",{"path":1927,"title":1928,"kind":1905},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fcache-api-responses-when-building-excel-reports","Cache API Responses When Building Excel Reports",{"path":1930,"title":1847,"kind":1905},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fcompare-an-excel-file-with-a-database-table",{"path":1932,"title":1933,"kind":1905},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fdownload-new-excel-files-from-an-sftp-server","Download New Excel Files from an SFTP Server",{"path":1935,"title":1936,"kind":1905},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fexport-a-large-database-table-to-excel-in-chunks","Export a Large Database Table to Excel in Chunks",{"path":1938,"title":1939,"kind":1905},"\u002Fadvanced-data-transformation-and-cleaning\u002Fmoving-data-between-excel-and-databases\u002Fexport-database-lookup-tables-as-excel-dropdown-lists","Export Database Lookup Tables as Excel Dropdown Lists",1790972096814]