From f3fc85523d8288c4eb352bf9836660fe4a67781b Mon Sep 17 00:00:00 2001 From: Eevee Date: Sun, 25 Jul 2010 20:27:42 -0700 Subject: [PATCH] Support cached sources. --- splinext/frontpage/__init__.py | 7 +++++ splinext/frontpage/sources.py | 63 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/splinext/frontpage/__init__.py b/splinext/frontpage/__init__.py index e113e74..9661906 100644 --- a/splinext/frontpage/__init__.py +++ b/splinext/frontpage/__init__.py @@ -66,6 +66,12 @@ def load_sources_hook(*args, **kwargs): # Save the list of sources, and done config['spline-frontpage.sources'] = sources +def source_cron_hook(*args, **kwargs): + """Hook to pass on cron tics to all sources, should they need it for e.g. + caching. + """ + for source in config['spline-frontpage.sources']: + source.do_cron(*args, **kwargs) class FrontPagePlugin(PluginBase): def controllers(self): @@ -82,6 +88,7 @@ class FrontPagePlugin(PluginBase): return [ ('routes_mapping', Priority.NORMAL, add_routes_hook), ('after_setup', Priority.NORMAL, load_sources_hook), + ('cron', Priority.NORMAL, source_cron_hook), ('frontpage_updates_rss', Priority.NORMAL, FeedSource), ('frontpage_updates_git', Priority.NORMAL, GitSource), ] diff --git a/splinext/frontpage/sources.py b/splinext/frontpage/sources.py index d082ada..dc97bf3 100644 --- a/splinext/frontpage/sources.py +++ b/splinext/frontpage/sources.py @@ -10,6 +10,8 @@ from subprocess import PIPE import feedparser import lxml.html +from pylons import cache + from spline.lib import helpers def max_age_to_datetime(max_age): @@ -57,6 +59,9 @@ class Source(object): self.limit = int(limit) self.max_age = max_age_to_datetime(max_age) + def do_cron(self, *args, **kwargs): + return + def poll(self, global_limit, global_max_age): """Public wrapper that takes care of reconciling global and source item limit and max age. @@ -79,9 +84,55 @@ class Source(object): """ raise NotImplementedError +class CachedSource(Source): + """Supports caching a source's updates in memcache. + + On the surface, this functions just like any other ``Source``. Calling + ``poll`` still returns a list of updates. However, ``poll`` does not call + your ``_poll``; instead, your implementation is called by the spline cron, + and the results are cached. ``poll`` then returns the contents of the + cache. + + You must define a ``_cache_key`` method that returns a key uniquely + identifying this object. Your key will be combined with the class name, so + it only needs to be unique for that source, not globally. + + You may also override ``poll_frequency``, the number of minutes between + pollings. By default, this is a rather conservative 60. + + Note that it may take up to a minute after server startup for updates + from a cached source to appear. + """ + + poll_frequency = 60 + + def cache_key(self): + return repr(type(self)) + ':' + self._cache_key() + + def _cache_key(self): + raise NotImplementedError + + def do_cron(self, tic, *args, **kwargs): + if tic % self.poll_frequency != 0: + # Too early! + return + + updates = self._poll(self.limit, self.max_age) + cache.get_cache('spline-frontpage')[self.cache_key()] = updates + + return + + def poll(self, global_limit, global_max_age): + """Fetches cached updates.""" + try: + return cache.get_cache('spline-frontpage')[self.cache_key()] + except KeyError: + # Haven't cached anything yet, apparently + return [] + FrontPageRSS = namedtuple('FrontPageRSS', ['source', 'time', 'entry', 'content']) -class FeedSource(Source): +class FeedSource(CachedSource): """Represents an RSS or Atom feed. Extra properties: @@ -94,12 +145,17 @@ class FeedSource(Source): SUMMARY_LENGTH = 1000 + poll_frequency = 15 + def __init__(self, feed_url, **kwargs): kwargs.setdefault('title', None) super(FeedSource, self).__init__(**kwargs) self.feed_url = feed_url + def _cache_key(self): + return self.feed_url + def _poll(self, limit, max_age): feed = feedparser.parse(self.feed_url) @@ -173,7 +229,7 @@ FrontPageGit = namedtuple('FrontPageGit', ['source', 'time', 'log', 'tag']) FrontPageGitCommit = namedtuple('FrontPageGitCommit', ['hash', 'author', 'time', 'subject', 'repo']) -class GitSource(Source): +class GitSource(CachedSource): """Represents a git repository. The main repository is checked for annotated tags, and an update is @@ -213,6 +269,9 @@ class GitSource(Source): self.gitweb = gitweb self.tag_pattern = tag_pattern + def _cache_key(self): + return self.repo_paths[0] + def _poll(self, limit, max_age): # Fetch the main repo's git tags git_dir = '--git-dir=' + self.repo_paths[0] -- 2.7.4