adding celery to manage clone imprinting as a background task

app/__init__.py

@@ -22,3 +22,5 @@
 # Register blueprints
 from app import routes
 app.register_blueprint(routes.clones.bp)
+
+from app import logging

app/logging.py

@@ -0,0 +1,25 @@
+"""
+Logging
+-------------------------
+
+Sets up a mail handler for logging errors.
+"""
+
+from app import app
+
+cfg = app.config
+
+# Email error messages.
+if not app.debug:
+    import logging
+    from logging.handlers import SMTPHandler
+    mail_handler = SMTPHandler(
+            (cfg['MAIL_HOST'], cfg['MAIL_PORT']),
+            cfg['MAIL_USER'],
+            cfg['MAIL_TARGETS'],
+            'CLONES IN TROUBLE',
+            (cfg['MAIL_USER'], cfg['MAIL_PASS']),
+            secure=())
+    mail_handler.setLevel(logging.ERROR)
+    app.logger.addHandler(mail_handler)
+

app/models.py

@@ -9,6 +9,7 @@
 class Clone(db.Document):
     created_at = db.DateTimeField(default=datetime.datetime.now, required=True)
     username = db.StringField(required=True, unique=True)
+    imprinting = db.BooleanField(default=False)
 
     # Mad-lib Tweet patterns.
     patterns = db.ListField(db.StringField(), default=[])
@@ -24,7 +25,8 @@ class Clone(db.Document):
 
     def imprint(self):
         """
-        Generate a clone for a given Twitter user.
+        Generate a clone for a given Twitter user
+        by analyzing their Twitter history.
         """
         user_tweets = twitter.tweets(self.username, count=2000)
 

app/routes/clones.py

@@ -1,6 +1,7 @@
 from flask import Blueprint, request, render_template, redirect, url_for, flash
 from app.models import Clone
 from app.forms import CloneForm
+from app.tasks import imprint
 import re
 
 bp = Blueprint('clones', __name__, url_prefix = '/clones')
@@ -16,8 +17,9 @@ def clones():
     if form.validate_on_submit():
         username = form.username.data
         clone, was_created = Clone.objects.get_or_create(username=username)
-        clone.imprint()
+        clone.imprinting = True
         clone.save()
+        imprint.delay(username)
         return redirect(url_for('clones.clone', username=username))
     return render_template('clones/create.jade', form=form)
 

app/tasks.py

@@ -0,0 +1,55 @@
+"""
+Tasks
+-------------------------
+
+Configure worker tasks.
+"""
+
+from app.models import Clone
+from celery import Celery
+import config
+
+# For sending mail.
+import smtplib
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+
+celery = Celery()
+celery.config_from_object(config)
+
+
+@celery.task
+def imprint(username):
+    """
+    Imprint a user's Twitter activity onto the clone.
+    """
+    c = Clone.objects.get(username=username)
+    c.imprint()
+    c.imprinting = False
+    c.save()
+    notify.delay('Imprinting complete', 'Imprinting complete for user {0}'.format(username))
+
+
+@celery.task
+def notify(subject, body):
+    """
+    Send an e-mail notification.
+    """
+    from_addr = config.MAIL_USER
+
+    # Construct the message.
+    msg = MIMEMultipart()
+    msg['From'] = from_addr
+    msg['Subject'] = subject
+    msg.attach(MIMEText(body, 'plain'))
+
+    # Connect to the mail server.
+    server = smtplib.SMTP(config.MAIL_HOST, config.MAIL_PORT)
+    server.starttls()
+    server.login(from_addr, config.MAIL_PASS)
+
+    for target in config.MAIL_TARGETS:
+        msg['To'] = target
+        server.sendmail(from_addr, target, msg.as_string())
+
+    server.quit()

app/templates/clones/member.jade

@@ -3,21 +3,24 @@ extends layout.jade
 block content
     h1= clone.username
 
-    //- Generate a few samples.
-    ul
-        for i in range(10)
-            li= clone.speak()
+    if clone.imprinting
+        h2 Imprinting in progress...please check back later.
+    else
+        //- Generate a few samples.
+        ul
+            for i in range(10)
+                li= clone.speak()
 
-    h2 Patterns
-    ul
-        for pattern in clone.patterns
-            li= pattern|highlight_pattern|safe
+        h2 Patterns
+        ul
+            for pattern in clone.patterns
+                li= pattern|highlight_pattern|safe
 
-    h2 Vocabulary
-    ul
-        for pos in clone.vocabulary
-            li
-                h6= pos
-                ul
-                    for token in clone.vocabulary[pos]
-                        li= token
+        h2 Vocabulary
+        ul
+            for pos in clone.vocabulary
+                li
+                    h6= pos
+                    ul
+                        for token in clone.vocabulary[pos]
+                            li= token

config-sample.py

@@ -1,6 +1,13 @@
 CSRF_ENABLED = True
 SECRET_KEY = 'some-passphrase'
-MONGODB_SETTINGS = {'DB': 'youtwo'}
+MONGODB_SETTINGS = {
+    'DB': 'youtwo',
+    'HOST': 'localhost'
+
+    # If necessary:
+    #'USERNAME': 'username',
+    #'PASSWORD': 'pw'
+}
 
 MAIL_HOST = 'smtp.gmail.com'
 MAIL_PORT = 587
@@ -27,3 +34,44 @@
     'VB',
     'RB'    # adverbs
 ]
+
+# Celery config.
+# Broker (message queue) url.
+BROKER_URL = 'amqp://guest@localhost:5672//'
+
+# Try connecting ad infinitum.
+BROKER_CONNECTION_MAX_RETRIES = None
+
+# Result backend.
+CELERY_RESULT_BACKEND = 'mongodb'
+CELERY_MONGODB_BACKEND_SETTINGS = {
+    'host': 'localhost',
+    'port': 27017,
+    'database': 'celery',
+    'taskmeta_collection': 'my_taskmeta' # Collection name to use for task output
+}
+
+# What modules to import on start.
+# Note that in production environments you will want to
+# remove the 'tests' tasks module.
+CELERY_IMPORTS = ('app.tasks',)
+
+# Send emails on errors
+CELERY_SEND_TASK_ERROR_EMAILS = True
+ADMINS = [
+    ('Francis Tseng', '[email protected]')
+]
+
+SERVER_EMAIL = '[email protected]'
+EMAIL_HOST = 'smtp.gmail.com'
+EMAIL_PORT = 587
+EMAIL_HOST_USER = '[email protected]'
+EMAIL_HOST_PASSWORD = 'your-pass'
+EMAIL_USE_TLS = True
+
+# Setting a maximum amount of tasks per worker
+# so the worker processes get regularly killed
+# (to reclaim memory). Not sure if this is the best
+# approach, but see:
+# https://github.com/publicscience/argos/issues/112
+CELERYD_MAX_TASKS_PER_CHILD=100

requirements.txt

@@ -15,6 +15,7 @@ pymongo==2.7.1
 six==1.7.3
 textblob==0.8.4
 webassets==0.10.1
+celery==3.1.13
 
 git+git://github.com/nltk/nltk.git
 git+git://github.com/ze-phyr-us/tweepy.git