From 2ff43fe9bb4cb9425c192f336ac30804a11520a2 Mon Sep 17 00:00:00 2001 From: Eevee Date: Sat, 24 Jul 2010 21:44:39 -0700 Subject: [PATCH] Load the sources only on startup. Fix local limit/age behavior. --- splinext/frontpage/__init__.py | 58 ++++++++++++++++++++- splinext/frontpage/controllers/frontpage.py | 79 ++++++----------------------- splinext/frontpage/sources.py | 41 ++++++++++++--- 3 files changed, 104 insertions(+), 74 deletions(-) diff --git a/splinext/frontpage/__init__.py b/splinext/frontpage/__init__.py index ea80f05..e113e74 100644 --- a/splinext/frontpage/__init__.py +++ b/splinext/frontpage/__init__.py @@ -1,10 +1,13 @@ -from collections import namedtuple -import datetime +from collections import defaultdict, namedtuple from pkg_resources import resource_filename +import re import subprocess +from pylons import config + from spline.lib import helpers from spline.lib.plugin import PluginBase, PluginLink, Priority +from spline.lib.plugin.load import run_hooks import splinext.frontpage.controllers.frontpage from splinext.frontpage.sources import FeedSource, GitSource @@ -13,6 +16,56 @@ def add_routes_hook(map, *args, **kwargs): """Hook to inject some of our behavior into the routes configuration.""" map.connect('/', controller='frontpage', action='index') +def load_sources_hook(*args, **kwargs): + """Hook to load all the known sources and stuff them in config. Run once, + on server startup. + """ + # Extract source definitions from config and store as source_name => config + update_config = defaultdict(dict) + key_rx = re.compile( + '(?x) ^ spline-frontpage [.] sources [.] (\w+) (?: [.] (\w+) )? $') + for key, val in config.iteritems(): + # Match against spline-frontpage.source.(source).(key) + match = key_rx.match(key) + if not match: + continue + + source_name, subkey = match.groups() + if not subkey: + # This is the type declaration; use a special key + subkey = '__type__' + + update_config[source_name][subkey] = val + + # Figure out the global limit and expiration time, with reasonable + # defaults. Make sure they're integers. + global_limit = int(config.get('spline-frontpage.limit', 10)) + # max_age is optional and can be None + try: + global_max_age = int(config['spline-frontpage.max_age']) + except KeyError: + global_max_age = None + + config['spline-frontpage.limit'] = global_limit + config['spline-frontpage.max_age'] = global_max_age + + # Ask plugins to turn configuration into source objects + sources = [] + for source, source_config in update_config.iteritems(): + hook_name = 'frontpage_updates_' + source_config['__type__'] + del source_config['__type__'] # don't feed this to constructor! + + # Default to global limit and max age. Source takes care of making + # integers and whatnot + source_config.setdefault('limit', global_limit) + source_config.setdefault('max_age', global_max_age) + + # Hooks return a list of sources; combine with running list + sources += run_hooks(hook_name, **source_config) + + # Save the list of sources, and done + config['spline-frontpage.sources'] = sources + class FrontPagePlugin(PluginBase): def controllers(self): @@ -28,6 +81,7 @@ class FrontPagePlugin(PluginBase): def hooks(self): return [ ('routes_mapping', Priority.NORMAL, add_routes_hook), + ('after_setup', Priority.NORMAL, load_sources_hook), ('frontpage_updates_rss', Priority.NORMAL, FeedSource), ('frontpage_updates_git', Priority.NORMAL, GitSource), ] diff --git a/splinext/frontpage/controllers/frontpage.py b/splinext/frontpage/controllers/frontpage.py index 04386fd..764b292 100644 --- a/splinext/frontpage/controllers/frontpage.py +++ b/splinext/frontpage/controllers/frontpage.py @@ -1,7 +1,4 @@ -from collections import defaultdict -import datetime import logging -import re from pylons import config, request, response, session, tmpl_context as c, url from pylons.controllers.util import abort, redirect_to @@ -10,7 +7,7 @@ from sqlalchemy.orm.exc import NoResultFound from spline.lib import helpers as h from spline.lib.base import BaseController, render -from spline.lib.plugin.load import run_hooks +from splinext.frontpage.sources import max_age_to_datetime log = logging.getLogger(__name__) @@ -49,72 +46,26 @@ class FrontPageController(BaseController): Local plugins can override the fairly simple index.mako template to customize the front page layout. """ - # XXX no reason to do this on the fly; cache it on server startup - update_config = defaultdict(dict) # source_name => config - key_rx = re.compile( - '(?x) ^ spline-frontpage [.] sources [.] (\w+) (?: [.] (\w+) )? $') - for key, val in config.iteritems(): - match = key_rx.match(key) - if not match: - continue - - source_name, subkey = match.groups() - if not subkey: - # This is the type declaration; use a special key - subkey = '__type__' - - if subkey in ('limit', 'max_age'): - val = int(val) - update_config[source_name][subkey] = val - - global_limit = int(config.get('spline-frontpage.limit', 10)) - now = datetime.datetime.now() - try: - global_max_age = now - datetime.timedelta( - seconds=int(config['spline-frontpage.max_age'])) - except KeyError: - global_max_age = None - - # Ask plugins to deal with this stuff for us! + updates = [] - for source, source_config in update_config.iteritems(): - hook_name = 'frontpage_updates_' + source_config['__type__'] - - # Merge with the global config - merged_config = source_config.copy() - del merged_config['__type__'] - - merged_config['limit'] = min( - merged_config.get('limit', global_limit), - global_limit, - ) - - try: - local_max_age = now - datetime.timedelta( - seconds=merged_config['max_age']) - except KeyError: - local_max_age = None - - if global_max_age and local_max_age: - merged_config['max_age'] = max(global_max_age, local_max_age) - else: - merged_config['max_age'] = global_max_age or local_max_age - - # XXX bleh - updates_lol = run_hooks(hook_name, **merged_config) - source_obj = updates_lol[0] - updates += source_obj.poll(merged_config['limit'], merged_config['max_age']) - - # Little optimization: maximum age effectively becomes the age of - # the oldest thing that would still appear on the page, as anything - # older would drop off the end no matter what. - # So sort by descending time and crop each iteration... + global_limit = config['spline-frontpage.limit'] + global_max_age = max_age_to_datetime( + config['spline-frontpage.max_age']) + + for source in config['spline-frontpage.sources']: + new_updates = source.poll(global_limit, global_max_age) + updates.extend(new_updates) + + # Little optimization: once there are global_limit items, anything + # older than the oldest cannot possibly make it onto the list. So, + # bump global_max_age to that oldest time if this is ever the case. updates.sort(key=lambda obj: obj.time, reverse=True) - updates = updates[:global_limit] + del updates[global_limit:] if updates and len(updates) == global_limit: global_max_age = updates[-1].time + # Done! Feed to template c.updates = updates return render('/index.mako') diff --git a/splinext/frontpage/sources.py b/splinext/frontpage/sources.py index 1afc484..d082ada 100644 --- a/splinext/frontpage/sources.py +++ b/splinext/frontpage/sources.py @@ -12,6 +12,16 @@ import lxml.html from spline.lib import helpers +def max_age_to_datetime(max_age): + """``max_age`` is specified in config as a number of seconds old. This + function takes that number and returns a corresponding datetime object. + """ + if max_age == None: + return None + + seconds = int(max_age) + + class Source(object): """Represents a source to be polled for updates. Sources are populated @@ -44,12 +54,28 @@ class Source(object): self.title = title self.icon = icon self.link = link - self.limit = limit - self.max_age = max_age + self.limit = int(limit) + self.max_age = max_age_to_datetime(max_age) + + def poll(self, global_limit, global_max_age): + """Public wrapper that takes care of reconciling global and source item + limit and max age. - def poll(self): - """Poll for updates. Must return an iterable. Each element should be - an Update object. + Subclasses should implement ``_poll``, below. + """ + # Smallest limit wins + limit = min(self.limit, global_limit) + + # Latest max age wins. Note that either could be None, but that's + # fine, because None is less than everything else + max_age = max(self.max_age, global_max_age) + + return self._poll(limit, max_age) + + def _poll(self, limit, max_age): + """Implementation of polling for updates. Must return an iterable. + Each element should be an object with ``source`` and ``time`` + properties. A namedtuple works well. """ raise NotImplementedError @@ -74,7 +100,7 @@ class FeedSource(Source): self.feed_url = feed_url - def poll(self, limit, max_age): + def _poll(self, limit, max_age): feed = feedparser.parse(self.feed_url) if not self.title: @@ -187,8 +213,7 @@ class GitSource(Source): self.gitweb = gitweb self.tag_pattern = tag_pattern - def poll(self, limit, max_age): - + def _poll(self, limit, max_age): # Fetch the main repo's git tags git_dir = '--git-dir=' + self.repo_paths[0] args = [ -- 2.7.4