Make plumbing respect the same env vars as the CLI. #180

[zzz-pokedex.git] / pokedex / lookup.py

diff --git a/pokedex/lookup.py b/pokedex/lookup.py
index 7b10974..a75136b 100644 (file)
--- a/pokedex/lookup.py
+++ b/pokedex/lookup.py
@@ -26,7 +26,25 @@ __all__ = ['PokedexLookup']
 rx_is_number = re.compile('^\d+$')
 
 LookupResult = namedtuple('LookupResult',
 rx_is_number = re.compile('^\d+$')
 
 LookupResult = namedtuple('LookupResult',

-                          ['object', 'name', 'language', 'iso3166', 'exact'])
+    ['object', 'indexed_name', 'name', 'language', 'iso3166', 'exact'])
+
+class UninitializedIndex(object):
+    class UninitializedIndexError(Exception):
+        pass
+
+    def __nonzero__(self):
+        """Dummy object should identify itself as False."""
+        return False
+
+    def __bool__(self):
+        """Python 3000 version of the above.  Future-proofing rules!"""
+        return False
+
+    def __getattr__(self, *args, **kwargs):
+        raise self.UninitializedIndexError(
+            "The lookup index does not exist.  Please use `pokedex setup` "
+            "or lookup.rebuild_index() to create it."
+        )

 
 class LanguageWeighting(whoosh.scoring.Weighting):
     """A scoring class that forces otherwise-equal English results to come
 
 class LanguageWeighting(whoosh.scoring.Weighting):
     """A scoring class that forces otherwise-equal English results to come


@@ -58,14 +76,16 @@ class PokedexLookup(object):
         for cls in (
             tables.Ability,
             tables.Item,
         for cls in (
             tables.Ability,
             tables.Item,

+            tables.Location,

             tables.Move,
             tables.Move,

+            tables.Nature,

             tables.Pokemon,
             tables.Type,
         )
     )
 
 
             tables.Pokemon,
             tables.Type,
         )
     )
 
 

-    def __init__(self, directory=None, session=None, recreate=False):
+    def __init__(self, directory=None, session=None):

         """Opens the whoosh index stored in the named directory.  If the index
         doesn't already exist, it will be created.
 
         """Opens the whoosh index stored in the named directory.  If the index
         doesn't already exist, it will be created.
 


@@ -74,13 +94,9 @@ class PokedexLookup(object):
             `pokedex` egg directory.
 
         `session`
             `pokedex` egg directory.
 
         `session`

-            If the index needs to be created, this database session will be
-            used.  Defaults to an attempt to connect to the default SQLite
-            database installed by `pokedex setup`.
-
-        `recreate`
-            If set to True, the whoosh index will be created even if it already
-            exists.
+            Used for creating the index and retrieving objects.  Defaults to an
+            attempt to connect to the default SQLite database installed by
+            `pokedex setup`.

         """
 
         # By the time this returns, self.index, self.speller, and self.session
         """
 
         # By the time this returns, self.index, self.speller, and self.session


@@ -88,8 +104,12 @@ class PokedexLookup(object):
 
         # Defaults
         if not directory:
 
         # Defaults
         if not directory:

+            directory = os.environ.get('POKEDEX_INDEX_DIR', None)
+
+        if not directory:

             directory = pkg_resources.resource_filename('pokedex',
                                                         'data/whoosh-index')
             directory = pkg_resources.resource_filename('pokedex',
                                                         'data/whoosh-index')

+        self.directory = directory

 
         if session:
             self.session = session
 
         if session:
             self.session = session


@@ -97,31 +117,33 @@ class PokedexLookup(object):
             self.session = connect()
 
         # Attempt to open or create the index
             self.session = connect()
 
         # Attempt to open or create the index

-        directory_exists = os.path.exists(directory)
-        if directory_exists and not recreate:
-            # Already exists; should be an index!  Bam, done.
-            try:
-                self.index = whoosh.index.open_dir(directory, indexname='MAIN')
-                spell_store = whoosh.filedb.filestore.FileStorage(directory)
-                self.speller = whoosh.spelling.SpellChecker(spell_store)
-                return
-            except whoosh.index.EmptyIndexError as e:
-                # Apparently not a real index.  Fall out and create it
-                pass
-
-        # Delete and start over if we're going to bail anyway.
-        if directory_exists and recreate:
-            # Be safe and only delete if it looks like a whoosh index, i.e.,
-            # everything starts with _
-            if all(f[0] == '_' for f in os.listdir(directory)):
-                shutil.rmtree(directory)
-                directory_exists = False
-
-        if not directory_exists:
-            os.mkdir(directory)
-
-
-        ### Create index
+        if not os.path.exists(directory) or not os.listdir(directory):
+            # Directory doesn't exist OR is empty; caller needs to use
+            # rebuild_index before doing anything.  Provide a dummy object that
+            # complains when used
+            self.index = UninitializedIndex()
+            self.speller = UninitializedIndex()
+            return
+
+        # Otherwise, already exists; should be an index!  Bam, done.
+        # Note that this will explode if the directory exists but doesn't
+        # contain an index; that's a feature
+        try:
+            self.index = whoosh.index.open_dir(directory, indexname='MAIN')
+        except whoosh.index.EmptyIndexError:
+            raise IOError(
+                "The index directory already contains files.  "
+                "Please use a dedicated directory for the lookup index."
+            )
+
+        # Create speller, and done
+        spell_store = whoosh.filedb.filestore.FileStorage(directory)
+        self.speller = whoosh.spelling.SpellChecker(spell_store)
+
+
+    def rebuild_index(self):
+        """Creates the index from scratch."""
+

         schema = whoosh.fields.Schema(
             name=whoosh.fields.ID(stored=True),
             table=whoosh.fields.ID(stored=True),
         schema = whoosh.fields.Schema(
             name=whoosh.fields.ID(stored=True),
             table=whoosh.fields.ID(stored=True),


@@ -129,11 +151,13 @@ class PokedexLookup(object):
             language=whoosh.fields.STORED,
             iso3166=whoosh.fields.STORED,
             display_name=whoosh.fields.STORED,  # non-lowercased name
             language=whoosh.fields.STORED,
             iso3166=whoosh.fields.STORED,
             display_name=whoosh.fields.STORED,  # non-lowercased name

-            forme_name=whoosh.fields.ID,

         )
 
         )
 

-        self.index = whoosh.index.create_in(directory, schema=schema,
-                                            indexname='MAIN')
+        if not os.path.exists(self.directory):
+            os.mkdir(self.directory)
+
+        self.index = whoosh.index.create_in(self.directory, schema=schema,
+                                                            indexname='MAIN')

         writer = self.index.writer()
 
         # Index every name in all our tables of interest
         writer = self.index.writer()
 
         # Index every name in all our tables of interest


@@ -143,47 +167,42 @@ class PokedexLookup(object):
         # especially when e.g. German names are very similar to English names
         speller_entries = []
         for cls in self.indexed_tables.values():
         # especially when e.g. German names are very similar to English names
         speller_entries = []
         for cls in self.indexed_tables.values():

-            q = session.query(cls)
+            q = self.session.query(cls)

 
             for row in q.yield_per(5):
 
             for row in q.yield_per(5):

-                # Need to give forme_name a dummy value because I can't
-                # search for explicitly empty fields.  Boo.
-                row_keys = [
-                    dict(table=unicode(cls.__tablename__),
-                         row_id=unicode(row.id),
-                         forme_name=u'__empty__')
-                ]
-
-                # If this is a form, mark it as such
-                # XXX foreign form names...?
-                if getattr(row, 'forme_name', None):
-                    # ...but if it's also the *default* form, index the name
-                    # bare too
-                    if not getattr(row, 'forme_base_pokemon_id', None):
-                        new_key = row_keys[0].copy()
-                        row_keys.append(new_key)
-
-                    row_keys[0]['forme_name'] = row.forme_name
+                row_key = dict(table=unicode(cls.__tablename__),
+                               row_id=unicode(row.id))

 
                 def add(name, language, iso3166, score):
                     normalized_name = self.normalize_name(name)
 
                 def add(name, language, iso3166, score):
                     normalized_name = self.normalize_name(name)

-                    for row_key in row_keys:
-                        writer.add_document(
-                            name=normalized_name, display_name=name,
-                            language=language, iso3166=iso3166,
-                            **row_key
-                        )
+
+                    writer.add_document(
+                        name=normalized_name, display_name=name,
+                        language=language, iso3166=iso3166,
+                        **row_key
+                    )

 
                     speller_entries.append((normalized_name, score))
 
 
 
                     speller_entries.append((normalized_name, score))
 
 

-                name = row.name
-                add(name, None, u'us', 1)
+                # Add the basic English name to the index
+                if cls == tables.Pokemon:
+                    # Pokémon need their form name added
+                    # XXX kinda kludgy
+                    add(row.full_name, None, u'us', 1)
+
+                    # If this is a default form, ALSO add the unadorned name,
+                    # so 'Deoxys' alone will still do the right thing
+                    if row.forme_name and not row.forme_base_pokemon_id:
+                        add(row.name, None, u'us', 1)
+                else:
+                    add(row.name, None, u'us', 1)

 
 

-                # Pokemon also get other languages
+                # Some things also have other languages' names
+                # XXX other language form names..?

                 for foreign_name in getattr(row, 'foreign_names', []):
                     moonspeak = foreign_name.name
                 for foreign_name in getattr(row, 'foreign_names', []):
                     moonspeak = foreign_name.name

-                    if name == moonspeak:
+                    if row.name == moonspeak:

                         # Don't add the English name again as a different
                         # language; no point and it makes spell results
                         # confusing
                         # Don't add the English name again as a different
                         # language; no point and it makes spell results
                         # confusing


@@ -230,21 +249,81 @@ class PokedexLookup(object):
         return name
 
 
         return name
 
 

+    def _apply_valid_types(self, name, valid_types):
+        """Combines the enforced `valid_types` with any from the search string
+        itself and updates the query.
+
+        For example, a name of 'a,b:foo' and valid_types of b,c will search for
+        only `b`s named "foo".
+
+        Returns `(name, merged_valid_types, term)`, where `name` has had any type
+        prefix stripped, `merged_valid_types` combines the original
+        `valid_types` with the type prefix, and `term` is a query term for
+        limited to just the allowed types.  If there are no type restrictions
+        at all, `term` will be None.
+        """
+
+        # Remove any type prefix (pokemon:133) first
+        user_valid_types = []
+        if ':' in name:
+            prefix_chunk, name = name.split(':', 1)
+            name = name.strip()
+
+            prefixes = prefix_chunk.split(',')
+            user_valid_types = [_.strip() for _ in prefixes]
+
+        # Merge the valid types together.  Only types that appear in BOTH lists
+        # may be used.
+        # As a special case, if the user asked for types that are explicitly
+        # forbidden, completely ignore what the user requested
+        combined_valid_types = []
+        if user_valid_types and valid_types:
+            combined_valid_types = list(
+                set(user_valid_types) & set(combined_valid_types)
+            )
+
+            if not combined_valid_types:
+                # No overlap!  Just use the enforced ones
+                combined_valid_types = valid_types
+        else:
+            # One list or the other was blank, so just use the one that isn't
+            combined_valid_types = valid_types + user_valid_types
+
+        if not combined_valid_types:
+            # No restrictions
+            return name, [], None
+
+        # Construct the term
+        type_terms = []
+        final_valid_types = []
+        for valid_type in combined_valid_types:
+            table_name = self._parse_table_name(valid_type)
+
+            # Quietly ignore bogus valid_types; more likely to DTRT
+            if table_name:
+                final_valid_types.append(valid_type)
+                type_terms.append(whoosh.query.Term(u'table', table_name))
+
+        return name, final_valid_types, whoosh.query.Or(type_terms)
+
+

     def _parse_table_name(self, name):
         """Takes a singular table name, table name, or table object and returns
         the table name.
 
         Returns None for a bogus name.
         """
     def _parse_table_name(self, name):
         """Takes a singular table name, table name, or table object and returns
         the table name.
 
         Returns None for a bogus name.
         """

+        # Table object

         if hasattr(name, '__tablename__'):
             return getattr(name, '__tablename__')
         if hasattr(name, '__tablename__'):
             return getattr(name, '__tablename__')

-        elif name in self.indexed_tables:
-            return name
-        elif name + 's' in self.indexed_tables:
-            return name + 's'
-        else:
-            # Bogus.  Be nice and return dummy
-            return None
+
+        # Table name
+        for table in self.indexed_tables.values():
+            if name in (table.__tablename__, table.__singlename__):
+                return table.__tablename__
+
+        # Bogus.  Be nice and return dummy
+        return None

 
     def _whoosh_records_to_results(self, records, exact=True):
         """Converts a list of whoosh's indexed records to LookupResult tuples
 
     def _whoosh_records_to_results(self, records, exact=True):
         """Converts a list of whoosh's indexed records to LookupResult tuples


@@ -265,6 +344,7 @@ class PokedexLookup(object):
             obj = self.session.query(cls).get(record['row_id'])
 
             results.append(LookupResult(object=obj,
             obj = self.session.query(cls).get(record['row_id'])
 
             results.append(LookupResult(object=obj,

+                                        indexed_name=record['name'],

                                         name=record['display_name'],
                                         language=record['language'],
                                         iso3166=record['iso3166'],
                                         name=record['display_name'],
                                         language=record['language'],
                                         iso3166=record['iso3166'],


@@ -321,53 +401,37 @@ class PokedexLookup(object):
         exact = True
         form = None
 
         exact = True
         form = None
 

-        # Remove any type prefix (pokemon:133) before constructing a query
-        if ':' in name:
-            prefix_chunk, name = name.split(':', 1)
-            name = name.strip()
-
-            if not valid_types:
-                # Only use types from the query string if none were explicitly
-                # provided
-                prefixes = prefix_chunk.split(',')
-                valid_types = [_.strip() for _ in prefixes]
+        # Pop off any type prefix and merge with valid_types
+        name, merged_valid_types, type_term = \
+            self._apply_valid_types(name, valid_types)

 
         # Random lookup
         if name == 'random':
 
         # Random lookup
         if name == 'random':

-            return self.random_lookup(valid_types=valid_types)
+            return self.random_lookup(valid_types=merged_valid_types)

 
         # Do different things depending what the query looks like
         # Note: Term objects do an exact match, so we don't have to worry about
         # a query parser tripping on weird characters in the input
 
         # Do different things depending what the query looks like
         # Note: Term objects do an exact match, so we don't have to worry about
         # a query parser tripping on weird characters in the input

+        try:
+            # Let Python try to convert to a number, so 0xff works
+            name_as_number = int(name, base=0)
+        except ValueError:
+            # Oh well
+            name_as_number = None
+

         if '*' in name or '?' in name:
             exact_only = True
             query = whoosh.query.Wildcard(u'name', name)
         if '*' in name or '?' in name:
             exact_only = True
             query = whoosh.query.Wildcard(u'name', name)

-        elif rx_is_number.match(name):
+        elif name_as_number is not None:

             # Don't spell-check numbers!
             exact_only = True
             # Don't spell-check numbers!
             exact_only = True

-            query = whoosh.query.Term(u'row_id', name)
+            query = whoosh.query.Term(u'row_id', unicode(name_as_number))

         else:
             # Not an integer
         else:
             # Not an integer

-            query = whoosh.query.Term(u'name', name) \
-                  & whoosh.query.Term(u'forme_name', u'__empty__')
-
-            # If there's a space in the input, this might be a form
-            if ' ' in name:
-                form, formless_name = name.split(' ', 1)
-                form_query = whoosh.query.Term(u'name', formless_name) \
-                           & whoosh.query.Term(u'forme_name', form)
-                query = query | form_query
-
-        ### Filter by type of object
-        type_terms = []
-        for valid_type in valid_types:
-            table_name = self._parse_table_name(valid_type)
-            if table_name:
-                # Quietly ignore bogus valid_types; more likely to DTRT
-                type_terms.append(whoosh.query.Term(u'table', table_name))
+            query = whoosh.query.Term(u'name', name)

 
 

-        if type_terms:
-            query = query & whoosh.query.Or(type_terms)
+        if type_term:
+            query = query & type_term

 
 
         ### Actual searching
 
 
         ### Actual searching


@@ -437,15 +501,21 @@ class PokedexLookup(object):
 
         return self.lookup(unicode(n), valid_types=[ partitions[0][0] ])
 
 
         return self.lookup(unicode(n), valid_types=[ partitions[0][0] ])
 

-    def prefix_lookup(self, prefix):
+    def prefix_lookup(self, prefix, valid_types=[]):

         """Returns terms starting with the given exact prefix.
 
         """Returns terms starting with the given exact prefix.
 

-        No special magic is currently done with the name; type prefixes are not
-        recognized.
+        Type prefixes are recognized, but no other name munging is done.

         """
 
         """
 

+        # Pop off any type prefix and merge with valid_types
+        prefix, merged_valid_types, type_term = \
+            self._apply_valid_types(prefix, valid_types)
+

         query = whoosh.query.Prefix(u'name', self.normalize_name(prefix))
 
         query = whoosh.query.Prefix(u'name', self.normalize_name(prefix))
 

+        if type_term:
+            query = query & type_term
+

         searcher = self.index.searcher()
         searcher.weighting = LanguageWeighting()
         results = searcher.search(query)  # XXX , limit=self.MAX_LOOKUP_RESULTS)
         searcher = self.index.searcher()
         searcher.weighting = LanguageWeighting()
         results = searcher.search(query)  # XXX , limit=self.MAX_LOOKUP_RESULTS)