X-Git-Url: http://git.veekun.com/zzz-pokedex.git/blobdiff_plain/4fac12946635f9465eb89ad00eed302a867cf2c5..6df90db0baf023ab59a21f1ca73b79fb04f9e22b:/pokedex/db/load.py?ds=sidebyside diff --git a/pokedex/db/load.py b/pokedex/db/load.py index 4512bf6..6a548b8 100644 --- a/pokedex/db/load.py +++ b/pokedex/db/load.py @@ -140,12 +140,16 @@ def load(session, tables=[], directory=None, drop_tables=False, verbose=False, s # Drop all tables if requested if drop_tables: print_start('Dropping tables') - for table in reversed(table_objs): + for n, table in enumerate(reversed(table_objs)): table.drop(checkfirst=True) + print_status('%s/%s' % (n, len(table_objs))) print_done() - for table in table_objs: + print_start('Creating tables') + for n, table in enumerate(table_objs): table.create() + print_status('%s/%s' % (n, len(table_objs))) + print_done() connection = session.connection() # Okay, run through the tables and actually load the data now @@ -170,14 +174,10 @@ def load(session, tables=[], directory=None, drop_tables=False, verbose=False, s if not safe and session.connection().dialect.name == 'postgresql': """ - Postgres' CSV dialect is nearly the same as ours, except that it - treats completely empty values as NULL, and empty quoted - strings ("") as an empty strings. - Pokedex dump does not quote empty strings. So, both empty strings - and NULLs are read in as NULL. - For an empty string in a NOT NULL column, the load will fail, and - load will fall back to the cross-backend row-by-row loading. And in - nullable columns, we already load empty stings as NULL. + Postgres' CSV dialect works with our data, if we mark the not-null + columns with FORCE NOT NULL. + COPY is only allowed for DB superusers. If you're not one, use safe + loading (pokedex load -S). """ session.commit() not_null_cols = [c for c in column_names if not table_obj.c[c].nullable] @@ -213,6 +213,8 @@ def load(session, tables=[], directory=None, drop_tables=False, verbose=False, s new_rows = [] def insert_and_commit(): + if not new_rows: + return session.connection().execute(insert_stmt, new_rows) session.commit() new_rows[:] = []