Load the sources only on startup. Fix local limit/age behavior.

diff --git a/splinext/frontpage/__init__.py b/splinext/frontpage/__init__.py
index ea80f05..e113e74 100644 (file)
--- a/splinext/frontpage/__init__.py
+++ b/splinext/frontpage/__init__.py
@@ -1,10 +1,13 @@
-from collections import namedtuple
-import datetime
+from collections import defaultdict, namedtuple

 from pkg_resources import resource_filename
 from pkg_resources import resource_filename

+import re

 import subprocess
 
 import subprocess
 

+from pylons import config
+

 from spline.lib import helpers
 from spline.lib.plugin import PluginBase, PluginLink, Priority
 from spline.lib import helpers
 from spline.lib.plugin import PluginBase, PluginLink, Priority

+from spline.lib.plugin.load import run_hooks

 
 import splinext.frontpage.controllers.frontpage
 from splinext.frontpage.sources import FeedSource, GitSource
 
 import splinext.frontpage.controllers.frontpage
 from splinext.frontpage.sources import FeedSource, GitSource


@@ -13,6 +16,56 @@ def add_routes_hook(map, *args, **kwargs):
     """Hook to inject some of our behavior into the routes configuration."""
     map.connect('/', controller='frontpage', action='index')
 
     """Hook to inject some of our behavior into the routes configuration."""
     map.connect('/', controller='frontpage', action='index')
 

+def load_sources_hook(*args, **kwargs):
+    """Hook to load all the known sources and stuff them in config.  Run once,
+    on server startup.
+    """
+    # Extract source definitions from config and store as source_name => config
+    update_config = defaultdict(dict)
+    key_rx = re.compile(
+        '(?x) ^ spline-frontpage [.] sources [.] (\w+) (?: [.] (\w+) )? $')
+    for key, val in config.iteritems():
+        # Match against spline-frontpage.source.(source).(key)
+        match = key_rx.match(key)
+        if not match:
+            continue
+
+        source_name, subkey = match.groups()
+        if not subkey:
+            # This is the type declaration; use a special key
+            subkey = '__type__'
+
+        update_config[source_name][subkey] = val
+
+    # Figure out the global limit and expiration time, with reasonable
+    # defaults.  Make sure they're integers.
+    global_limit = int(config.get('spline-frontpage.limit', 10))
+    # max_age is optional and can be None
+    try:
+        global_max_age = int(config['spline-frontpage.max_age'])
+    except KeyError:
+        global_max_age = None
+
+    config['spline-frontpage.limit'] = global_limit
+    config['spline-frontpage.max_age'] = global_max_age
+
+    # Ask plugins to turn configuration into source objects
+    sources = []
+    for source, source_config in update_config.iteritems():
+        hook_name = 'frontpage_updates_' + source_config['__type__']
+        del source_config['__type__']  # don't feed this to constructor!
+
+        # Default to global limit and max age.  Source takes care of making
+        # integers and whatnot
+        source_config.setdefault('limit', global_limit)
+        source_config.setdefault('max_age', global_max_age)
+
+        # Hooks return a list of sources; combine with running list
+        sources += run_hooks(hook_name, **source_config)
+
+    # Save the list of sources, and done
+    config['spline-frontpage.sources'] = sources
+

 
 class FrontPagePlugin(PluginBase):
     def controllers(self):
 
 class FrontPagePlugin(PluginBase):
     def controllers(self):


@@ -28,6 +81,7 @@ class FrontPagePlugin(PluginBase):
     def hooks(self):
         return [
             ('routes_mapping',          Priority.NORMAL,    add_routes_hook),
     def hooks(self):
         return [
             ('routes_mapping',          Priority.NORMAL,    add_routes_hook),

+            ('after_setup',             Priority.NORMAL,    load_sources_hook),

             ('frontpage_updates_rss',   Priority.NORMAL,    FeedSource),
             ('frontpage_updates_git',   Priority.NORMAL,    GitSource),
         ]
             ('frontpage_updates_rss',   Priority.NORMAL,    FeedSource),
             ('frontpage_updates_git',   Priority.NORMAL,    GitSource),
         ]

diff --git a/splinext/frontpage/controllers/frontpage.py b/splinext/frontpage/controllers/frontpage.py
index 04386fd..764b292 100644 (file)
--- a/splinext/frontpage/controllers/frontpage.py
+++ b/splinext/frontpage/controllers/frontpage.py
@@ -1,7 +1,4 @@
-from collections import defaultdict
-import datetime

 import logging
 import logging

-import re

 
 from pylons import config, request, response, session, tmpl_context as c, url
 from pylons.controllers.util import abort, redirect_to
 
 from pylons import config, request, response, session, tmpl_context as c, url
 from pylons.controllers.util import abort, redirect_to


@@ -10,7 +7,7 @@ from sqlalchemy.orm.exc import NoResultFound
 
 from spline.lib import helpers as h
 from spline.lib.base import BaseController, render
 
 from spline.lib import helpers as h
 from spline.lib.base import BaseController, render

-from spline.lib.plugin.load import run_hooks
+from splinext.frontpage.sources import max_age_to_datetime

 
 log = logging.getLogger(__name__)
 
 
 log = logging.getLogger(__name__)
 


@@ -49,72 +46,26 @@ class FrontPageController(BaseController):
         Local plugins can override the fairly simple index.mako template to
         customize the front page layout.
         """
         Local plugins can override the fairly simple index.mako template to
         customize the front page layout.
         """

-        # XXX no reason to do this on the fly; cache it on server startup
-        update_config = defaultdict(dict)  # source_name => config
-        key_rx = re.compile(
-            '(?x) ^ spline-frontpage [.] sources [.] (\w+) (?: [.] (\w+) )? $')
-        for key, val in config.iteritems():
-            match = key_rx.match(key)
-            if not match:
-                continue
-
-            source_name, subkey = match.groups()
-            if not subkey:
-                # This is the type declaration; use a special key
-                subkey = '__type__'
-
-            if subkey in ('limit', 'max_age'):
-                val = int(val)
-            update_config[source_name][subkey] = val
-
-        global_limit = int(config.get('spline-frontpage.limit', 10))
-        now = datetime.datetime.now()
-        try:
-            global_max_age = now - datetime.timedelta(
-                seconds=int(config['spline-frontpage.max_age']))
-        except KeyError:
-            global_max_age = None
-
-        # Ask plugins to deal with this stuff for us!
+

         updates = []
         updates = []

-        for source, source_config in update_config.iteritems():
-            hook_name = 'frontpage_updates_' + source_config['__type__']
-
-            # Merge with the global config
-            merged_config = source_config.copy()
-            del merged_config['__type__']
-
-            merged_config['limit'] = min(
-                merged_config.get('limit', global_limit),
-                global_limit,
-            )
-
-            try:
-                local_max_age = now - datetime.timedelta(
-                    seconds=merged_config['max_age'])
-            except KeyError:
-                local_max_age = None
-
-            if global_max_age and local_max_age:
-                merged_config['max_age'] = max(global_max_age, local_max_age)
-            else:
-                merged_config['max_age'] = global_max_age or local_max_age
-
-            # XXX bleh
-            updates_lol = run_hooks(hook_name, **merged_config)
-            source_obj = updates_lol[0]
-            updates += source_obj.poll(merged_config['limit'], merged_config['max_age'])
-
-            # Little optimization: maximum age effectively becomes the age of
-            # the oldest thing that would still appear on the page, as anything
-            # older would drop off the end no matter what.
-            # So sort by descending time and crop each iteration...
+        global_limit = config['spline-frontpage.limit']
+        global_max_age = max_age_to_datetime(
+            config['spline-frontpage.max_age'])
+
+        for source in config['spline-frontpage.sources']:
+            new_updates = source.poll(global_limit, global_max_age)
+            updates.extend(new_updates)
+
+            # Little optimization: once there are global_limit items, anything
+            # older than the oldest cannot possibly make it onto the list.  So,
+            # bump global_max_age to that oldest time if this is ever the case.

             updates.sort(key=lambda obj: obj.time, reverse=True)
             updates.sort(key=lambda obj: obj.time, reverse=True)

-            updates = updates[:global_limit]
+            del updates[global_limit:]

 
             if updates and len(updates) == global_limit:
                 global_max_age = updates[-1].time
 
 
             if updates and len(updates) == global_limit:
                 global_max_age = updates[-1].time
 

+        # Done!  Feed to template

         c.updates = updates
 
         return render('/index.mako')
         c.updates = updates
 
         return render('/index.mako')

diff --git a/splinext/frontpage/sources.py b/splinext/frontpage/sources.py
index 1afc484..d082ada 100644 (file)
--- a/splinext/frontpage/sources.py
+++ b/splinext/frontpage/sources.py
@@ -12,6 +12,16 @@ import lxml.html
 
 from spline.lib import helpers
 
 
 from spline.lib import helpers
 

+def max_age_to_datetime(max_age):
+    """``max_age`` is specified in config as a number of seconds old.  This
+    function takes that number and returns a corresponding datetime object.
+    """
+    if max_age == None:
+        return None
+
+    seconds = int(max_age)
+
+

 
 class Source(object):
     """Represents a source to be polled for updates.  Sources are populated
 
 class Source(object):
     """Represents a source to be polled for updates.  Sources are populated


@@ -44,12 +54,28 @@ class Source(object):
         self.title = title
         self.icon = icon
         self.link = link
         self.title = title
         self.icon = icon
         self.link = link

-        self.limit = limit
-        self.max_age = max_age
+        self.limit = int(limit)
+        self.max_age = max_age_to_datetime(max_age)
+
+    def poll(self, global_limit, global_max_age):
+        """Public wrapper that takes care of reconciling global and source item
+        limit and max age.

 
 

-    def poll(self):
-        """Poll for updates.  Must return an iterable.  Each element should be
-        an Update object.
+        Subclasses should implement ``_poll``, below.
+        """
+        # Smallest limit wins
+        limit = min(self.limit, global_limit)
+
+        # Latest max age wins.  Note that either could be None, but that's
+        # fine, because None is less than everything else
+        max_age = max(self.max_age, global_max_age)
+
+        return self._poll(limit, max_age)
+
+    def _poll(self, limit, max_age):
+        """Implementation of polling for updates.  Must return an iterable.
+        Each element should be an object with ``source`` and ``time``
+        properties.  A namedtuple works well.

         """
         raise NotImplementedError
 
         """
         raise NotImplementedError
 


@@ -74,7 +100,7 @@ class FeedSource(Source):
 
         self.feed_url = feed_url
 
 
         self.feed_url = feed_url
 

-    def poll(self, limit, max_age):
+    def _poll(self, limit, max_age):

         feed = feedparser.parse(self.feed_url)
 
         if not self.title:
         feed = feedparser.parse(self.feed_url)
 
         if not self.title:


@@ -187,8 +213,7 @@ class GitSource(Source):
         self.gitweb = gitweb
         self.tag_pattern = tag_pattern
 
         self.gitweb = gitweb
         self.tag_pattern = tag_pattern
 

-    def poll(self, limit, max_age):
-
+    def _poll(self, limit, max_age):

         # Fetch the main repo's git tags
         git_dir = '--git-dir=' + self.repo_paths[0]
         args = [
         # Fetch the main repo's git tags
         git_dir = '--git-dir=' + self.repo_paths[0]
         args = [