When loading Pokédex data, commit every 1000 rows.

Mostly done to fix RAM usage issues, but seems to have increased speed slightly.

diff --git a/pokedex/db/load.py b/pokedex/db/load.py
index 7f8824a..044d15c 100644 (file)
--- a/pokedex/db/load.py
+++ b/pokedex/db/load.py
@@ -175,6 +175,11 @@ def load(session, directory=None, drop_tables=False, verbose=False):
 
             session.add(row)
 
+            # Remembering some zillion rows in the session consumes a lot of
+            # RAM.  Let's not do that.  Commit every 1000 rows
+            if len(session.new) > 1000:
+                session.commit()
+
         session.commit()
 
         # Attempt to add any spare rows we've collected