Add migration script to move English texts to language-specific tables

[zzz-pokedex.git] / pokedex / lookup.py

diff --git a/pokedex/lookup.py b/pokedex/lookup.py
index f35e335..1b99065 100644 (file)
--- a/pokedex/lookup.py
+++ b/pokedex/lookup.py
@@ -1,7 +1,5 @@
 # encoding: utf8
 # encoding: utf8

-from collections import namedtuple

 import os, os.path
 import os, os.path

-import pkg_resources

 import random
 import re
 import shutil
 import random
 import re
 import shutil


@@ -16,39 +14,83 @@ from whoosh.qparser import QueryParser
 import whoosh.scoring
 import whoosh.spelling
 
 import whoosh.scoring
 import whoosh.spelling
 

+from pokedex.util import namedtuple
+

 from pokedex.db import connect
 import pokedex.db.tables as tables
 from pokedex.roomaji import romanize
 from pokedex.db import connect
 import pokedex.db.tables as tables
 from pokedex.roomaji import romanize

+from pokedex.defaults import get_default_index_dir

 
 __all__ = ['PokedexLookup']
 
 
 rx_is_number = re.compile('^\d+$')
 
 
 __all__ = ['PokedexLookup']
 
 
 rx_is_number = re.compile('^\d+$')
 

-LookupResult = namedtuple('LookupResult',
-    ['object', 'indexed_name', 'name', 'language', 'iso3166', 'exact'])
+LookupResult = namedtuple('LookupResult', [
+    'object', 'indexed_name', 'name', 'language', 'iso639', 'iso3166', 'exact',
+])
+
+class UninitializedIndex(object):
+    class UninitializedIndexError(Exception):
+        pass
+
+    def __nonzero__(self):
+        """Dummy object should identify itself as False."""
+        return False
+
+    def __bool__(self):
+        """Python 3000 version of the above.  Future-proofing rules!"""
+        return False
+
+    def __getattr__(self, *args, **kwargs):
+        raise self.UninitializedIndexError(
+            "The lookup index does not exist.  Please use `pokedex setup` "
+            "or lookup.rebuild_index() to create it."
+        )

 
 class LanguageWeighting(whoosh.scoring.Weighting):
     """A scoring class that forces otherwise-equal English results to come
     before foreign results.
     """
 
 
 class LanguageWeighting(whoosh.scoring.Weighting):
     """A scoring class that forces otherwise-equal English results to come
     before foreign results.
     """
 

+    def __init__(self, extra_weights={}, *args, **kwargs):
+        """`extra_weights` may be a dictionary of weights which will be
+        factored in.
+
+        Intended for use with spelling corrections, which come along with their
+        own weightings.
+        """
+        self.extra_weights = extra_weights
+        super(LanguageWeighting, self).__init__(*args, **kwargs)
+

     def score(self, searcher, fieldnum, text, docnum, weight, QTF=1):
         doc = searcher.stored_fields(docnum)
     def score(self, searcher, fieldnum, text, docnum, weight, QTF=1):
         doc = searcher.stored_fields(docnum)

-        if doc['language'] == None:
+
+        # Apply extra weight
+        weight = weight * self.extra_weights.get(text, 1.0)
+
+        language = doc.get('language')
+        if language is None:

             # English (well, "default"); leave it at 1
             return weight
             # English (well, "default"); leave it at 1
             return weight

-        elif doc['language'] == u'Roomaji':
+        elif language == u'Roomaji':

             # Give Roomaji a little boost; it's most likely to be searched
             # Give Roomaji a little boost; it's most likely to be searched

-            return weight * 0.95
+            return weight * 0.9

         else:
             # Everything else can drop down the totem pole
         else:
             # Everything else can drop down the totem pole

-            return weight * 0.9
+            return weight * 0.8

 
 
 class PokedexLookup(object):
 
 
 class PokedexLookup(object):

-    INTERMEDIATE_LOOKUP_RESULTS = 25
-    MAX_LOOKUP_RESULTS = 10
+    MAX_FUZZY_RESULTS = 10
+    MAX_EXACT_RESULTS = 43
+    INTERMEDIATE_FACTOR = 2
+
+    # The speller only checks how much the input matches a word; there can be
+    # all manner of extra unmatched junk, and it won't affect the weighting.
+    # To compensate, greatly boost the weighting of matches at the beginning
+    # and end, so nearly-full-word-matches are much better
+    SPELLER_OPTIONS = dict(booststart=10.0, boostend=9.0)

 
     # Dictionary of table name => table class.
     # Need the table name so we can get the class from the table name after we
 
     # Dictionary of table name => table class.
     # Need the table name so we can get the class from the table name after we


@@ -58,14 +100,17 @@ class PokedexLookup(object):
         for cls in (
             tables.Ability,
             tables.Item,
         for cls in (
             tables.Ability,
             tables.Item,

+            tables.Location,

             tables.Move,
             tables.Move,

+            tables.Nature,

             tables.Pokemon,
             tables.Pokemon,

+            tables.PokemonForm,

             tables.Type,
         )
     )
 
 
             tables.Type,
         )
     )
 
 

-    def __init__(self, directory=None, session=None, recreate=False):
+    def __init__(self, directory=None, session=None):

         """Opens the whoosh index stored in the named directory.  If the index
         doesn't already exist, it will be created.
 
         """Opens the whoosh index stored in the named directory.  If the index
         doesn't already exist, it will be created.
 


@@ -74,22 +119,19 @@ class PokedexLookup(object):
             `pokedex` egg directory.
 
         `session`
             `pokedex` egg directory.
 
         `session`

-            If the index needs to be created, this database session will be
-            used.  Defaults to an attempt to connect to the default SQLite
-            database installed by `pokedex setup`.
-
-        `recreate`
-            If set to True, the whoosh index will be created even if it already
-            exists.
+            Used for creating the index and retrieving objects.  Defaults to an
+            attempt to connect to the default SQLite database installed by
+            `pokedex setup`.

         """
 
         # By the time this returns, self.index, self.speller, and self.session
         # must be set
 
         """
 
         # By the time this returns, self.index, self.speller, and self.session
         # must be set
 

-        # Defaults
-        if not directory:
-            directory = pkg_resources.resource_filename('pokedex',
-                                                        'data/whoosh-index')
+        # If a directory was not given, use the default
+        if directory is None:
+            directory = get_default_index_dir()
+
+        self.directory = directory

 
         if session:
             self.session = session
 
         if session:
             self.session = session


@@ -97,108 +139,117 @@ class PokedexLookup(object):
             self.session = connect()
 
         # Attempt to open or create the index
             self.session = connect()
 
         # Attempt to open or create the index

-        directory_exists = os.path.exists(directory)
-        if directory_exists and not recreate:
-            # Already exists; should be an index!  Bam, done.
-            try:
-                self.index = whoosh.index.open_dir(directory, indexname='MAIN')
-                spell_store = whoosh.filedb.filestore.FileStorage(directory)
-                self.speller = whoosh.spelling.SpellChecker(spell_store)
-                return
-            except whoosh.index.EmptyIndexError as e:
-                # Apparently not a real index.  Fall out and create it
-                pass
-
-        # Delete and start over if we're going to bail anyway.
-        if directory_exists and recreate:
-            # Be safe and only delete if it looks like a whoosh index, i.e.,
-            # everything starts with _
-            if all(f[0] == '_' for f in os.listdir(directory)):
-                shutil.rmtree(directory)
-                directory_exists = False
-
-        if not directory_exists:
-            os.mkdir(directory)
-
-
-        ### Create index
+        if not os.path.exists(directory) or not os.listdir(directory):
+            # Directory doesn't exist OR is empty; caller needs to use
+            # rebuild_index before doing anything.  Provide a dummy object that
+            # complains when used
+            self.index = UninitializedIndex()
+            self.speller = UninitializedIndex()
+            return
+
+        # Otherwise, already exists; should be an index!  Bam, done.
+        # Note that this will explode if the directory exists but doesn't
+        # contain an index; that's a feature
+        try:
+            self.index = whoosh.index.open_dir(directory, indexname='MAIN')
+        except whoosh.index.EmptyIndexError:
+            raise IOError(
+                "The index directory already contains files.  "
+                "Please use a dedicated directory for the lookup index."
+            )
+
+        # Create speller, and done
+        spell_store = whoosh.filedb.filestore.FileStorage(directory)
+        self.speller = whoosh.spelling.SpellChecker(spell_store,
+            **self.SPELLER_OPTIONS)
+
+
+    def rebuild_index(self):
+        """Creates the index from scratch."""
+

         schema = whoosh.fields.Schema(
             name=whoosh.fields.ID(stored=True),
             table=whoosh.fields.ID(stored=True),
             row_id=whoosh.fields.ID(stored=True),
             language=whoosh.fields.STORED,
         schema = whoosh.fields.Schema(
             name=whoosh.fields.ID(stored=True),
             table=whoosh.fields.ID(stored=True),
             row_id=whoosh.fields.ID(stored=True),
             language=whoosh.fields.STORED,

-            iso3166=whoosh.fields.STORED,
+            iso639=whoosh.fields.ID(stored=True),
+            iso3166=whoosh.fields.ID(stored=True),

             display_name=whoosh.fields.STORED,  # non-lowercased name
         )
 
             display_name=whoosh.fields.STORED,  # non-lowercased name
         )
 

-        self.index = whoosh.index.create_in(directory, schema=schema,
-                                            indexname='MAIN')
+        if os.path.exists(self.directory):
+            # create_in() isn't totally reliable, so just nuke whatever's there
+            # manually.  Try to be careful about this...
+            for f in os.listdir(self.directory):
+                if re.match('^_?(MAIN|SPELL)_', f):
+                    os.remove(os.path.join(self.directory, f))
+        else:
+            os.mkdir(self.directory)
+
+        self.index = whoosh.index.create_in(self.directory, schema=schema,
+                                                            indexname='MAIN')

         writer = self.index.writer()
 
         # Index every name in all our tables of interest
         writer = self.index.writer()
 
         # Index every name in all our tables of interest

-        # speller_entries becomes a list of (word, score) tuples; the score is
-        # 2 for English names, 1.5 for Roomaji, and 1 for everything else.  I
-        # think this biases the results in the direction most people expect,
-        # especially when e.g. German names are very similar to English names
-        speller_entries = []
+        speller_entries = set()

         for cls in self.indexed_tables.values():
         for cls in self.indexed_tables.values():

-            q = session.query(cls)
+            q = self.session.query(cls)

 
             for row in q.yield_per(5):
                 row_key = dict(table=unicode(cls.__tablename__),
                                row_id=unicode(row.id))
 
 
             for row in q.yield_per(5):
                 row_key = dict(table=unicode(cls.__tablename__),
                                row_id=unicode(row.id))
 

-                def add(name, language, iso3166, score):
+                def add(name, language, iso639, iso3166):

                     normalized_name = self.normalize_name(name)
 
                     writer.add_document(
                         name=normalized_name, display_name=name,
                     normalized_name = self.normalize_name(name)
 
                     writer.add_document(
                         name=normalized_name, display_name=name,

-                        language=language, iso3166=iso3166,
+                        language=language, iso639=iso639, iso3166=iso3166,

                         **row_key
                     )
 
                         **row_key
                     )
 

-                    speller_entries.append((normalized_name, score))
+                    speller_entries.add(normalized_name)

 
 
                 # Add the basic English name to the index
                 if cls == tables.Pokemon:
 
 
                 # Add the basic English name to the index
                 if cls == tables.Pokemon:

-                    # Pokémon need their form name added
-                    # XXX kinda kludgy
-                    add(row.full_name, None, u'us', 1)
-
-                    # If this is a default form, ALSO add the unadorned name,
-                    # so 'Deoxys' alone will still do the right thing
-                    if row.forme_name and not row.forme_base_pokemon_id:
-                        add(row.name, None, u'us', 1)
-                else:
-                    add(row.name, None, u'us', 1)
+                    # Don't re-add alternate forms of the same Pokémon; they'll
+                    # be added as Pokémon forms instead
+                    if not row.is_base_form:
+                        continue
+                elif cls == tables.PokemonForm:
+                    if row.name:
+                        add(row.pokemon_name, None, u'en', u'us')
+                    continue

 
                 # Some things also have other languages' names
                 # XXX other language form names..?
 
                 # Some things also have other languages' names
                 # XXX other language form names..?

-                for foreign_name in getattr(row, 'foreign_names', []):
-                    moonspeak = foreign_name.name
-                    if row.name == moonspeak:
-                        # Don't add the English name again as a different
+                seen = set()
+                for language, name in getattr(row, 'names', []).items():
+                    if name in seen:
+                        # Don't add the name again as a different

                         # language; no point and it makes spell results
                         # confusing
                         continue
                         # language; no point and it makes spell results
                         # confusing
                         continue

+                    seen.add(name)

 
 

-                    add(moonspeak, foreign_name.language.name,
-                                   foreign_name.language.iso3166,
-                                   3)
+                    add(name, language.name,
+                              language.iso639,
+                              language.iso3166)

 
                     # Add Roomaji too
 
                     # Add Roomaji too

-                    if foreign_name.language.name == 'Japanese':
-                        roomaji = romanize(foreign_name.name)
-                        add(roomaji, u'Roomaji', u'jp', 8)
+                    if language.identifier == 'ja':
+                        roomaji = romanize(name)
+                        add(roomaji, u'Roomaji', u'ja', u'jp')

 
         writer.commit()
 
         # Construct and populate a spell-checker index.  Quicker to do it all
         # at once, as every call to add_* does a commit(), and those seem to be
         # expensive
 
         writer.commit()
 
         # Construct and populate a spell-checker index.  Quicker to do it all
         # at once, as every call to add_* does a commit(), and those seem to be
         # expensive

-        self.speller = whoosh.spelling.SpellChecker(self.index.storage)
-        self.speller.add_scored_words(speller_entries)
+        self.speller = whoosh.spelling.SpellChecker(self.index.storage, mingram=2,
+            **self.SPELLER_OPTIONS)
+        self.speller.add_words(speller_entries)

 
 
     def normalize_name(self, name):
 
 
     def normalize_name(self, name):


@@ -224,21 +275,102 @@ class PokedexLookup(object):
         return name
 
 
         return name
 
 

+    def _apply_valid_types(self, name, valid_types):
+        """Combines the enforced `valid_types` with any from the search string
+        itself and updates the query.
+
+        For example, a name of 'a,b:foo' and valid_types of b,c will search for
+        only `b`s named "foo".
+
+        Returns `(name, merged_valid_types, term)`, where `name` has had any type
+        prefix stripped, `merged_valid_types` combines the original
+        `valid_types` with the type prefix, and `term` is a query term for
+        limited to just the allowed types.  If there are no type restrictions
+        at all, `term` will be None.
+        """
+
+        # Remove any type prefix (pokemon:133) first
+        user_valid_types = []
+        if ':' in name:
+            prefix_chunk, name = name.split(':', 1)
+            name = name.strip()
+
+            prefixes = prefix_chunk.split(',')
+            user_valid_types = []
+            for prefix in prefixes:
+                prefix = prefix.strip()
+                if prefix:
+                    user_valid_types.append(prefix)
+
+        # Merge the valid types together.  Only types that appear in BOTH lists
+        # may be used.
+        # As a special case, if the user asked for types that are explicitly
+        # forbidden, completely ignore what the user requested.
+        # And, just to complicate matters: "type" and language need to be
+        # considered separately.
+        def merge_requirements(func):
+            user = filter(func, user_valid_types)
+            system = filter(func, valid_types)
+
+            if user and system:
+                merged = list(set(user) & set(system))
+                if merged:
+                    return merged
+                else:
+                    # No overlap; use the system restrictions
+                    return system
+            else:
+                # One or the other is blank; use the one that's not
+                return user or system
+
+        # @foo means language must be foo; otherwise it's a table name
+        lang_requirements = merge_requirements(lambda req: req[0] == u'@')
+        type_requirements = merge_requirements(lambda req: req[0] != u'@')
+        all_requirements = lang_requirements + type_requirements
+
+        # Construct the term
+        lang_terms = []
+        for lang in lang_requirements:
+            # Allow for either country or language codes
+            lang_code = lang[1:]
+            lang_terms.append(whoosh.query.Term(u'iso639', lang_code))
+            lang_terms.append(whoosh.query.Term(u'iso3166', lang_code))
+
+        type_terms = []
+        for type in type_requirements:
+            table_name = self._parse_table_name(type)
+
+            # Quietly ignore bogus valid_types; more likely to DTRT
+            if table_name:
+                type_terms.append(whoosh.query.Term(u'table', table_name))
+
+        # Combine both kinds of restriction
+        all_terms = []
+        if type_terms:
+            all_terms.append(whoosh.query.Or(type_terms))
+        if lang_terms:
+            all_terms.append(whoosh.query.Or(lang_terms))
+
+        return name, all_requirements, whoosh.query.And(all_terms)
+
+

     def _parse_table_name(self, name):
         """Takes a singular table name, table name, or table object and returns
         the table name.
 
         Returns None for a bogus name.
         """
     def _parse_table_name(self, name):
         """Takes a singular table name, table name, or table object and returns
         the table name.
 
         Returns None for a bogus name.
         """

+        # Table object

         if hasattr(name, '__tablename__'):
             return getattr(name, '__tablename__')
         if hasattr(name, '__tablename__'):
             return getattr(name, '__tablename__')

-        elif name in self.indexed_tables:
-            return name
-        elif name + 's' in self.indexed_tables:
-            return name + 's'
-        else:
-            # Bogus.  Be nice and return dummy
-            return None
+
+        # Table name
+        for table in self.indexed_tables.values():
+            if name in (table.__tablename__, table.__singlename__):
+                return table.__tablename__
+
+        # Bogus.  Be nice and return dummy
+        return None

 
     def _whoosh_records_to_results(self, records, exact=True):
         """Converts a list of whoosh's indexed records to LookupResult tuples
 
     def _whoosh_records_to_results(self, records, exact=True):
         """Converts a list of whoosh's indexed records to LookupResult tuples


@@ -261,7 +393,8 @@ class PokedexLookup(object):
             results.append(LookupResult(object=obj,
                                         indexed_name=record['name'],
                                         name=record['display_name'],
             results.append(LookupResult(object=obj,
                                         indexed_name=record['name'],
                                         name=record['display_name'],

-                                        language=record['language'],
+                                        language=record.get('language'),
+                                        iso639=record['iso639'],

                                         iso3166=record['iso3166'],
                                         exact=exact))
 
                                         iso3166=record['iso3166'],
                                         exact=exact))
 


@@ -271,12 +404,11 @@ class PokedexLookup(object):
     def lookup(self, input, valid_types=[], exact_only=False):
         """Attempts to find some sort of object, given a name.
 
     def lookup(self, input, valid_types=[], exact_only=False):
         """Attempts to find some sort of object, given a name.
 

-        Returns a list of named (object, name, language, iso3166, exact)
-        tuples.  `object` is a database object, `name` is the name under which
-        the object was found, `language` and `iso3166` are the name and country
-        code of the language in which the name was found, and `exact` is True
-        iff this was an
-        exact match.
+        Returns a list of named (object, name, language, iso639, iso3166,
+        exact) tuples.  `object` is a database object, `name` is the name under
+        which the object was found, `language` and the two isos are the name
+        and country codes of the language in which the name was found, and
+        `exact` is True iff this was an exact match.

 
         This function currently ONLY does fuzzy matching if there are no exact
         matches.
 
         This function currently ONLY does fuzzy matching if there are no exact
         matches.


@@ -294,17 +426,19 @@ class PokedexLookup(object):
         Also:
         - Type restrictions.  "type:psychic" will only return the type.  This
           is how to make ID lookup useful.  Multiple type specs can be entered
         Also:
         - Type restrictions.  "type:psychic" will only return the type.  This
           is how to make ID lookup useful.  Multiple type specs can be entered

-          with commas, as "move,item:1".  If `valid_types` are provided, any
-          type prefix will be ignored.
+          with commas, as "move,item:1".
+        - Language restrictions.  "@fr:charge" will only return Tackle, which
+          is called "Charge" in French.  These can be combined with type
+          restrictions, e.g., "@fr,move:charge".

         - Alternate formes can be specified merely like "wash rotom".
 
         `input`
             Name of the thing to look for.
 
         `valid_types`
         - Alternate formes can be specified merely like "wash rotom".
 
         `input`
             Name of the thing to look for.
 
         `valid_types`

-            A list of table objects or names, e.g., `['pokemon', 'moves']`.  If
-            this is provided, only results in one of the given tables will be
-            returned.
+            A list of type or language restrictions, e.g., `['pokemon',
+            '@ja']`.  If this is provided, only results in one of the given
+            tables will be returned.

 
         `exact_only`
             If True, only exact matches are returned.  If set to False (the
 
         `exact_only`
             If True, only exact matches are returned.  If set to False (the


@@ -316,75 +450,96 @@ class PokedexLookup(object):
         exact = True
         form = None
 
         exact = True
         form = None
 

-        # Remove any type prefix (pokemon:133) before constructing a query
-        if ':' in name:
-            prefix_chunk, name = name.split(':', 1)
-            name = name.strip()
-
-            if not valid_types:
-                # Only use types from the query string if none were explicitly
-                # provided
-                prefixes = prefix_chunk.split(',')
-                valid_types = [_.strip() for _ in prefixes]
+        # Pop off any type prefix and merge with valid_types
+        name, merged_valid_types, type_term = \
+            self._apply_valid_types(name, valid_types)

 
         # Random lookup
         if name == 'random':
 
         # Random lookup
         if name == 'random':

-            return self.random_lookup(valid_types=valid_types)
+            return self.random_lookup(valid_types=merged_valid_types)

 
         # Do different things depending what the query looks like
         # Note: Term objects do an exact match, so we don't have to worry about
         # a query parser tripping on weird characters in the input
 
         # Do different things depending what the query looks like
         # Note: Term objects do an exact match, so we don't have to worry about
         # a query parser tripping on weird characters in the input

+        try:
+            # Let Python try to convert to a number, so 0xff works
+            name_as_number = int(name, base=0)
+        except ValueError:
+            # Oh well
+            name_as_number = None
+

         if '*' in name or '?' in name:
             exact_only = True
             query = whoosh.query.Wildcard(u'name', name)
         if '*' in name or '?' in name:
             exact_only = True
             query = whoosh.query.Wildcard(u'name', name)

-        elif rx_is_number.match(name):
+        elif name_as_number is not None:

             # Don't spell-check numbers!
             exact_only = True
             # Don't spell-check numbers!
             exact_only = True

-            query = whoosh.query.Term(u'row_id', name)
+            query = whoosh.query.Term(u'row_id', unicode(name_as_number))

         else:
             # Not an integer
             query = whoosh.query.Term(u'name', name)
 
         else:
             # Not an integer
             query = whoosh.query.Term(u'name', name)
 

-        ### Filter by type of object
-        type_terms = []
-        for valid_type in valid_types:
-            table_name = self._parse_table_name(valid_type)
-            if table_name:
-                # Quietly ignore bogus valid_types; more likely to DTRT
-                type_terms.append(whoosh.query.Term(u'table', table_name))
-
-        if type_terms:
-            query = query & whoosh.query.Or(type_terms)
+        if type_term:
+            query = query & type_term

 
 
         ### Actual searching
 
 
         ### Actual searching

-        searcher = self.index.searcher()
-        # XXX is this kosher?  docs say search() takes a weighting arg, but it
-        # certainly does not
-        searcher.weighting = LanguageWeighting()
-        results = searcher.search(query,
-                                  limit=self.INTERMEDIATE_LOOKUP_RESULTS)
+        # Limits; result limits are constants, and intermediate results (before
+        # duplicate items are stripped out) are capped at the result limit
+        # times another constant.
+        # Fuzzy are capped at 10, beyond which something is probably very
+        # wrong.  Exact matches -- that is, wildcards and ids -- are far less
+        # constrained.
+        # Also, exact matches are sorted by name, since weight doesn't matter.
+        sort_by = dict()
+        if exact_only:
+            max_results = self.MAX_EXACT_RESULTS
+            sort_by['sortedby'] = (u'table', u'name')
+        else:
+            max_results = self.MAX_FUZZY_RESULTS
+
+        searcher = self.index.searcher(weighting=LanguageWeighting())
+        results = searcher.search(
+            query,
+            limit=int(max_results * self.INTERMEDIATE_FACTOR),
+            **sort_by
+        )

 
         # Look for some fuzzy matches if necessary
         if not exact_only and not results:
             exact = False
             results = []
 
 
         # Look for some fuzzy matches if necessary
         if not exact_only and not results:
             exact = False
             results = []
 

-            for suggestion in self.speller.suggest(
-                name, self.INTERMEDIATE_LOOKUP_RESULTS):
+            fuzzy_query_parts = []
+            fuzzy_weights = {}
+            min_weight = [None]
+            for suggestion, _, weight in self.speller.suggestions_and_scores(name):
+                # Only allow the top 50% of scores; otherwise there will always
+                # be a lot of trailing junk
+                if min_weight[0] is None:
+                    min_weight[0] = weight * 0.5
+                elif weight < min_weight[0]:
+                    break
+
+                fuzzy_query_parts.append(whoosh.query.Term('name', suggestion))
+                fuzzy_weights[suggestion] = weight
+
+            if not fuzzy_query_parts:
+                # Nothing at all; don't try querying
+                return []

 
 

-                query = whoosh.query.Term('name', suggestion)
-                results.extend(searcher.search(query))
+            fuzzy_query = whoosh.query.Or(fuzzy_query_parts)
+            if type_term:
+                fuzzy_query = fuzzy_query & type_term
+
+            searcher.weighting = LanguageWeighting(extra_weights=fuzzy_weights)
+            results = searcher.search(fuzzy_query)

 
         ### Convert results to db objects
         objects = self._whoosh_records_to_results(results, exact=exact)
 
 
         ### Convert results to db objects
         objects = self._whoosh_records_to_results(results, exact=exact)
 

-        # Only return up to 10 matches; beyond that, something is wrong.  We
-        # strip out duplicate entries above, so it's remotely possible that we
-        # should have more than 10 here and lost a few.  The speller returns 25
-        # to give us some padding, and should avoid that problem.  Not a big
-        # deal if we lose the 25th-most-likely match anyway.
-        return objects[:self.MAX_LOOKUP_RESULTS]
+        # Truncate and return
+        return objects[:max_results]

 
 
     def random_lookup(self, valid_types=[]):
 
 
     def random_lookup(self, valid_types=[]):


@@ -392,17 +547,21 @@ class PokedexLookup(object):
         `valid_types`.
         """
 
         `valid_types`.
         """
 

-        tables = []
+        table_names = []

         for valid_type in valid_types:
             table_name = self._parse_table_name(valid_type)
         for valid_type in valid_types:
             table_name = self._parse_table_name(valid_type)

-            if table_name:
-                tables.append(self.indexed_tables[table_name])
+            # Skip anything not recognized.  Could be, say, a language code.
+            # XXX The vast majority of Pokémon forms are unnamed and unindexed,
+            #     which can produce blank results.  So skip them too for now.
+            if table_name and table_name != 'pokemon_forms':
+                table_names.append(table_name)

 
 

-        if not tables:
+        if not table_names:

             # n.b.: It's possible we got a list of valid_types and none of them
             # were valid, but this function is guaranteed to return
             # n.b.: It's possible we got a list of valid_types and none of them
             # were valid, but this function is guaranteed to return

-            # *something*, so it politely selects from the entire index isntead
-            tables = self.indexed_tables.values()
+            # *something*, so it politely selects from the entire index instead
+            table_names = self.indexed_tables.keys()
+            table_names.remove('pokemon_forms')

 
         # Rather than create an array of many hundred items and pick randomly
         # from it, just pick a number up to the total number of potential
 
         # Rather than create an array of many hundred items and pick randomly
         # from it, just pick a number up to the total number of potential


@@ -412,10 +571,10 @@ class PokedexLookup(object):
         # XXX ought to cache this (in the index?) if possible
         total = 0
         partitions = []
         # XXX ought to cache this (in the index?) if possible
         total = 0
         partitions = []

-        for table in tables:
-            count = self.session.query(table).count()
+        for table_name in table_names:
+            count = self.session.query(self.indexed_tables[table_name]).count()

             total += count
             total += count

-            partitions.append((table, count))
+            partitions.append((table_name, count))

 
         n = random.randint(1, total)
         while n > partitions[0][1]:
 
         n = random.randint(1, total)
         while n > partitions[0][1]:


@@ -424,15 +583,21 @@ class PokedexLookup(object):
 
         return self.lookup(unicode(n), valid_types=[ partitions[0][0] ])
 
 
         return self.lookup(unicode(n), valid_types=[ partitions[0][0] ])
 

-    def prefix_lookup(self, prefix):
+    def prefix_lookup(self, prefix, valid_types=[]):

         """Returns terms starting with the given exact prefix.
 
         """Returns terms starting with the given exact prefix.
 

-        No special magic is currently done with the name; type prefixes are not
-        recognized.
+        Type prefixes are recognized, but no other name munging is done.

         """
 
         """
 

+        # Pop off any type prefix and merge with valid_types
+        prefix, merged_valid_types, type_term = \
+            self._apply_valid_types(prefix, valid_types)
+

         query = whoosh.query.Prefix(u'name', self.normalize_name(prefix))
 
         query = whoosh.query.Prefix(u'name', self.normalize_name(prefix))
 

+        if type_term:
+            query = query & type_term
+

         searcher = self.index.searcher()
         searcher.weighting = LanguageWeighting()
         results = searcher.search(query)  # XXX , limit=self.MAX_LOOKUP_RESULTS)
         searcher = self.index.searcher()
         searcher.weighting = LanguageWeighting()
         results = searcher.search(query)  # XXX , limit=self.MAX_LOOKUP_RESULTS)