From dcf6146e9914ff3c166f242b72f04a8934d65148 Mon Sep 17 00:00:00 2001 From: Eevee Date: Wed, 2 Dec 2009 22:12:46 -0800 Subject: [PATCH] Thumbnailing. Most notably, uploading a file will now generate a thumbnail (and a "medium" size image). Also tried to clean up use of storage. I'll revisit this again later, but the intent was to make it easier to switch from file storage to something like S3 or mogile or whatever in the future. The biggest change was to keep storage access out of the db model; Art no longer knows anything about storage whatsoever. --- development.ini | 3 ++ floof/controllers/art.py | 86 +++++++++++++++++++++++++++++++------------ floof/lib/file_storage.py | 68 +++++++++++++++++----------------- floof/lib/helpers.py | 10 ++++- floof/model/art.py | 19 +++------- floof/templates/art/show.mako | 3 +- floof/templates/macros.mako | 4 +- floof/websetup.py | 33 +++++++++++++---- setup.py | 1 + 9 files changed, 140 insertions(+), 87 deletions(-) diff --git a/development.ini b/development.ini index f184f66..7e0e3ed 100644 --- a/development.ini +++ b/development.ini @@ -39,6 +39,9 @@ sqlalchemy.url = sqlite:///%(here)s/development.db # execute malicious code after an exception is raised. # set debug = false +# Floof-specific configuration +thumbnail_size = 120 +medium_size = 960 # Logging configuration [loggers] diff --git a/floof/controllers/art.py b/floof/controllers/art.py index 41298c8..5b347b3 100644 --- a/floof/controllers/art.py +++ b/floof/controllers/art.py @@ -1,22 +1,26 @@ import logging -from pylons import request, response, session, tmpl_context as c, h +from pylons import config, request, response, session, tmpl_context as c, h from pylons.controllers.util import abort, redirect from pylons import url from floof.lib.base import BaseController, render log = logging.getLogger(__name__) -import elixir +from floof.lib import file_storage as storage from floof.model.users import User from floof.model import Art, Rating, UserRelation from floof.model.comments import Discussion from floof.model.users import User, UserRelationship +import elixir +#import magic +import os.path +import PIL +import PIL.Image from sqlalchemy import func from sqlalchemy.exceptions import IntegrityError from sqlalchemy.orm.exc import NoResultFound - from wtforms.validators import ValidationError from wtforms import * @@ -79,32 +83,66 @@ class ArtController(BaseController): # TODO: login required def create(self): c.form = ArtUploadForm(request.params) - if c.form.validate(): + if not c.form.validate(): + ## TODO: JavaScript should be added to the upload form so that it is + ## impossible to submit the form when it contains any invalid users, + ## so this never happens. Only autocompled usernames should be allowed. + return render("/art/new.mako") - c.art = Art(uploader=c.user, **request.params) - c.art.discussion = Discussion(count=0) + # Save the file + upload = request.params['file'] + hash = storage.save_file('art/original', upload.file) - for artist in c.form.by.data: - UserRelation(user=artist, kind="by", creator=c.user, art=c.art) + # Create a thumbnail and a medium view + img = PIL.Image.open( + config['app_conf']['static_root'] + + storage.get_path('art/original', hash)[1:] + ) + for subdir, config_size in [('medium', config['medium_size']), + ('thumbnail', config['thumbnail_size'])]: + path = config['app_conf']['static_root'] + storage.get_path("art/{0}".format(subdir), hash) - file = request.params['file'] + dir, _ = os.path.split(path) + if not os.path.exists(dir): + os.makedirs(dir) - try: - elixir.session.commit() - redirect(url('show_art', id=c.art.id)) - except IntegrityError: - # hurr, there must be a better way to do this but I am lazy right now - hash = c.art.hash - elixir.session.rollback() - duplicate_art = Art.get_by(hash=hash) - h.flash("We already have that one.") - redirect(url('show_art', id=duplicate_art.id)) + size = int(config_size) + shrunken_img = img.copy() + shrunken_img.thumbnail((size, size), PIL.Image.ANTIALIAS) + shrunken_img.save(path, img.format) + + #magicker = magic.Magic(mime=True) + buffer = upload.file.read(64) + upload.file.seek(0) + mimetype = 'image/unknown' #magickery.from_buffer(buffer) + + # XXX Ensure we can actually handle the mimetype + + print mimetype + c.art = Art( + uploader=c.user, + original_filename=upload.filename, + hash=hash, + mimetype=mimetype, + ) + c.art.discussion = Discussion(count=0) + + for artist in c.form.by.data: + UserRelation(user=artist, kind="by", creator=c.user, art=c.art) + + + try: + elixir.session.commit() + redirect(url('show_art', id=c.art.id)) + except IntegrityError: + # XXX Check this as early as possible, and clean up the filesystem! + # Right now this replaces the original file! + hash = c.art.hash + elixir.session.rollback() + duplicate_art = Art.get_by(hash=hash) + h.flash("We already have that one.") + redirect(url('show_art', id=duplicate_art.id)) - else: - ## TODO: JavaScript should be added to the upload form so that it is - ## impossible to submit the form when it contains any invalid users, - ## so this never happens. Only autocompled usernames should be allowed. - return render("/art/new.mako") def show(self, id): diff --git a/floof/lib/file_storage.py b/floof/lib/file_storage.py index 90561d0..b30a662 100644 --- a/floof/lib/file_storage.py +++ b/floof/lib/file_storage.py @@ -7,12 +7,6 @@ from pylons import config """ Notes: -# Here's one way to move stuff... -shutil.copyfileobj(temp.file, dest_file) - -# we should probably store the basename of the original filename in the database somewhere -temp_base = os.path.basename(temp.filename) - # You can get a mime type like so: from mimetypes import guess_type guess_type(temp.filename)[0] @@ -22,45 +16,49 @@ def get_path(space, hash): return "/" + os.path.join( space, hash[:2], hash[2:] ) -def save_file(space, temp): +def save_file(space, fileobj, hash=None): + """Saves the contents of fileobj to the given storage space. + If a hash is not provided, the SHA-1 sum of the file will be computed and + that will be used. The ideal scenario here is to let the hash of the + original file be computed automatically, then create thumbnails et al. with + the same hash in a different space. + + Returns the hashsum. + """ dest_root = os.path.join( config['app_conf']['static_root'], space ) - # we don't know where we're going to save this stuff yet, - # so I guess we'll write it to another tempfile. One we know the path of. - # I'm assuming the tempfile we get from pylons is set to delete itself - # when it closes, and has no visible path. Maybe I'm wrong? - intermediate_file_descriptor, intermediate_path = tempfile.mkstemp() + # The incoming fileobj could be a tempfile that's already been unlinked -- + # and probably is, as it's coming from a Pylons upload object. Thus we + # have to copy the data rather than the file, and we may have to read the + # file anyway to hash it, so do both at the same time. - # that function gives me an integer file descriptor for some reason. - intermediate_file = os.fdopen(intermediate_file_descriptor, "wb") + # Need a named temporary file to write to; it gets renamed once the hash is + # computed + temp_fd, temp_path = tempfile.mkstemp() + temp_file = os.fdopen(temp_fd, "wb") sha1 = hashlib.sha1() - while 1: - data = temp.file.read(chunk_size) + while True: + data = fileobj.read(chunk_size) if not data: break - sha1.update(data) - intermediate_file.write(data) - - temp.file.close() - intermediate_file.close() - hash = sha1.hexdigest() - # git convention: first two characters are the directory - dest_dir = os.path.join( dest_root, hash[:2] ) - dest_path = os.path.join( dest_dir, hash[2:] ) + if not hash: + sha1.update(data) + temp_file.write(data) - makedirs(dest_dir) - shutil.move(intermediate_path, dest_path) - - return hash + temp_file.close() + if not hash: + hash = sha1.hexdigest() + # Git convention: first two characters are the directory + dest_dir = os.path.join(dest_root, hash[:2]) + dest_path = os.path.join(dest_dir, hash[2:]) + if not os.path.exists(dest_dir): + os.makedirs(dest_dir) + print dest_path + shutil.move(temp_path, dest_path) -def makedirs(dir): - try: - os.makedirs(dir) - except OSError: - pass - + return hash diff --git a/floof/lib/helpers.py b/floof/lib/helpers.py index b4cac1f..14f92d0 100644 --- a/floof/lib/helpers.py +++ b/floof/lib/helpers.py @@ -8,7 +8,7 @@ available to Controllers. This module is available to both as 'h'. # from webhelpers.html.tags import checkbox, password from webhelpers import * from routes import url_for, redirect_to -from pylons import url +from pylons import config, url # Scaffolding helper imports from webhelpers.html.tags import * @@ -18,6 +18,8 @@ import sqlalchemy.types as types flash = Flash() # End of. +from floof.lib import file_storage as storage + def get_object_or_404(model, **kw): from pylons.controllers.util import abort """ @@ -34,10 +36,14 @@ def get_object_or_404(model, **kw): def get_comment_owner_url(**route): """Given a view route, returns the owner_url route parameter for generating comment URLs. - """ + """ if 'owner_url' in route: # We're already in a comments page. Reuse the existing owner URL return route['owner_url'] # url() returns URLs beginning with a slash. We just need to strip it. return url(**route)[1:] + +def storage_url(prefix, identifier): + """Returns a URL for the given object-in-storage.""" + return storage.get_path(prefix, identifier) diff --git a/floof/model/art.py b/floof/model/art.py index bb68c9f..8989e8f 100644 --- a/floof/model/art.py +++ b/floof/model/art.py @@ -21,7 +21,8 @@ import floof.model.comments class Art(Entity): title = Field(Unicode(120)) original_filename = Field(Unicode(120)) - hash = Field(String, unique=True, required=True) + hash = Field(Unicode(40), unique=True, required=True) + mimetype = Field(Unicode(32), required=True) uploader = ManyToOne('User', required=True) tags = OneToMany('Tag') @@ -29,16 +30,6 @@ class Art(Entity): user_relations = OneToMany('UserRelation') - - def set_file(self, file): - self.hash = save_file("art", file) - self.original_filename = file.filename - - file = property(get_path, set_file) - - def get_path(self): - if self.hash: - return get_path("art", self.hash) - - def __unicode__(self): - return self.get_path() + @property + def file_path(self): + return get_path("art", self.hash) diff --git a/floof/templates/art/show.mako b/floof/templates/art/show.mako index 05d1e42..a38806d 100644 --- a/floof/templates/art/show.mako +++ b/floof/templates/art/show.mako @@ -45,7 +45,6 @@ By: ${h.text('username')} ${h.submit('add','Add')} ${h.end_form()} - - + ${comments.comment_block(c.art.discussion.comments)} diff --git a/floof/templates/macros.mako b/floof/templates/macros.mako index 061b7b9..ca3933e 100644 --- a/floof/templates/macros.mako +++ b/floof/templates/macros.mako @@ -3,9 +3,9 @@ % for item in art:
  • - +
    • % endfor - \ No newline at end of file + diff --git a/floof/websetup.py b/floof/websetup.py index b1611c9..815832d 100644 --- a/floof/websetup.py +++ b/floof/websetup.py @@ -10,23 +10,40 @@ from pylons import config from elixir import * from floof import model as model +import os + def setup_app(command, conf, vars): """Place any commands to setup floof here""" load_environment(conf.global_conf, conf.local_conf) - model.metadata.create_all() - # Initialisation here ... this sort of stuff: + ### Database schema + model.metadata.create_all() + ### Sample data # Users from floof.model.users import IdentityURL, User identity_url = IdentityURL(url=u'http://eevee.livejournal.com/') user = User(name=u'Eevee') user.identity_urls.append(identity_url) - - # some_entity = model.Session.query(model..).get(1) - # e.g. foo = model.Session.query(model.identity.User).get(1) - # from datetime import datetime - # some_entity.poked_on = datetime.now() - # model.Session.add(some_entity) model.Session.commit() + + + ### Filesystem stuff + # Only necessary if we're using the filesystem for storage. + # And we are! + art_path = os.path.join(config['app_conf']['static_root'], 'art') + # Test that we can write to the art directory + if os.access(art_path, os.W_OK | os.X_OK): + # Create directories for various image sizes + for subdir in ['original', 'medium', 'thumbnail']: + path = os.path.join(art_path, subdir) + # If it exists, it should be writeable. If not, create it. + if os.path.exists(path): + if not os.access(path, os.W_OK | os.X_OK): + print "*** WARNING: Can't write to the art/{0} " \ + "directory!".format(subdir) + continue + os.makedirs(path) + else: + print "*** WARNING: Can't write to the art directory!" diff --git a/setup.py b/setup.py index 56e5af4..c71e6b2 100644 --- a/setup.py +++ b/setup.py @@ -17,6 +17,7 @@ setup( "Pylons>=0.9.7", "SQLAlchemy>=0.5", "elixir>=0.6", + 'pil', 'python-openid', 'shabti', 'wtforms', -- 2.7.4