From b88cb7d1d76d2252c58fa419a6c240b5ed5a9a78 Mon Sep 17 00:00:00 2001 From: Francis Tseng Date: Fri, 18 Jul 2014 09:53:23 -0400 Subject: [PATCH] adding celery to manage clone imprinting as a background task --- app/__init__.py | 2 ++ app/logging.py | 25 +++++++++++++++ app/models.py | 4 ++- app/notify.py | 12 ------- app/routes/clones.py | 4 ++- app/tasks.py | 55 ++++++++++++++++++++++++++++++++ app/templates/clones/member.jade | 35 ++++++++++---------- config-sample.py | 50 ++++++++++++++++++++++++++++- requirements.txt | 1 + 9 files changed, 157 insertions(+), 31 deletions(-) create mode 100644 app/logging.py delete mode 100644 app/notify.py create mode 100644 app/tasks.py diff --git a/app/__init__.py b/app/__init__.py index 7dbc559..c4390c2 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -22,3 +22,5 @@ # Register blueprints from app import routes app.register_blueprint(routes.clones.bp) + +from app import logging diff --git a/app/logging.py b/app/logging.py new file mode 100644 index 0000000..d34c18b --- /dev/null +++ b/app/logging.py @@ -0,0 +1,25 @@ +""" +Logging +------------------------- + +Sets up a mail handler for logging errors. +""" + +from app import app + +cfg = app.config + +# Email error messages. +if not app.debug: + import logging + from logging.handlers import SMTPHandler + mail_handler = SMTPHandler( + (cfg['MAIL_HOST'], cfg['MAIL_PORT']), + cfg['MAIL_USER'], + cfg['MAIL_TARGETS'], + 'CLONES IN TROUBLE', + (cfg['MAIL_USER'], cfg['MAIL_PASS']), + secure=()) + mail_handler.setLevel(logging.ERROR) + app.logger.addHandler(mail_handler) + diff --git a/app/models.py b/app/models.py index 8ad843c..f7bb22d 100644 --- a/app/models.py +++ b/app/models.py @@ -9,6 +9,7 @@ class Clone(db.Document): created_at = db.DateTimeField(default=datetime.datetime.now, required=True) username = db.StringField(required=True, unique=True) + imprinting = db.BooleanField(default=False) # Mad-lib Tweet patterns. patterns = db.ListField(db.StringField(), default=[]) @@ -24,7 +25,8 @@ class Clone(db.Document): def imprint(self): """ - Generate a clone for a given Twitter user. + Generate a clone for a given Twitter user + by analyzing their Twitter history. """ user_tweets = twitter.tweets(self.username, count=2000) diff --git a/app/notify.py b/app/notify.py deleted file mode 100644 index 85cd1d4..0000000 --- a/app/notify.py +++ /dev/null @@ -1,12 +0,0 @@ -from app import app - -cfg = app.config - -# Email error messages. -if not app.debug: - import logging - from logging.handlers import SMTPHandler - mail_handler = SMTPHandler((cfg['MAIL_HOST'], cfg['MAIL_PORT']), 'dont-talk-back@'+cfg['MAIL_HOST'], cfg['MAIL_TARGETS'], 'brain is floudering!', (cfg['MAIL_USER'], cfg['MAIL_PASS'])) - mail_handler.setLevel(logging.ERROR) - app.logger.addHandler(mail_handler) - diff --git a/app/routes/clones.py b/app/routes/clones.py index ffee4a7..2d4dcf1 100644 --- a/app/routes/clones.py +++ b/app/routes/clones.py @@ -1,6 +1,7 @@ from flask import Blueprint, request, render_template, redirect, url_for, flash from app.models import Clone from app.forms import CloneForm +from app.tasks import imprint import re bp = Blueprint('clones', __name__, url_prefix = '/clones') @@ -16,8 +17,9 @@ def clones(): if form.validate_on_submit(): username = form.username.data clone, was_created = Clone.objects.get_or_create(username=username) - clone.imprint() + clone.imprinting = True clone.save() + imprint.delay(username) return redirect(url_for('clones.clone', username=username)) return render_template('clones/create.jade', form=form) diff --git a/app/tasks.py b/app/tasks.py new file mode 100644 index 0000000..4ce653e --- /dev/null +++ b/app/tasks.py @@ -0,0 +1,55 @@ +""" +Tasks +------------------------- + +Configure worker tasks. +""" + +from app.models import Clone +from celery import Celery +import config + +# For sending mail. +import smtplib +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText + +celery = Celery() +celery.config_from_object(config) + + +@celery.task +def imprint(username): + """ + Imprint a user's Twitter activity onto the clone. + """ + c = Clone.objects.get(username=username) + c.imprint() + c.imprinting = False + c.save() + notify.delay('Imprinting complete', 'Imprinting complete for user {0}'.format(username)) + + +@celery.task +def notify(subject, body): + """ + Send an e-mail notification. + """ + from_addr = config.MAIL_USER + + # Construct the message. + msg = MIMEMultipart() + msg['From'] = from_addr + msg['Subject'] = subject + msg.attach(MIMEText(body, 'plain')) + + # Connect to the mail server. + server = smtplib.SMTP(config.MAIL_HOST, config.MAIL_PORT) + server.starttls() + server.login(from_addr, config.MAIL_PASS) + + for target in config.MAIL_TARGETS: + msg['To'] = target + server.sendmail(from_addr, target, msg.as_string()) + + server.quit() diff --git a/app/templates/clones/member.jade b/app/templates/clones/member.jade index f41cc49..3d2ba0f 100644 --- a/app/templates/clones/member.jade +++ b/app/templates/clones/member.jade @@ -3,21 +3,24 @@ extends layout.jade block content h1= clone.username - //- Generate a few samples. - ul - for i in range(10) - li= clone.speak() + if clone.imprinting + h2 Imprinting in progress...please check back later. + else + //- Generate a few samples. + ul + for i in range(10) + li= clone.speak() - h2 Patterns - ul - for pattern in clone.patterns - li= pattern|highlight_pattern|safe + h2 Patterns + ul + for pattern in clone.patterns + li= pattern|highlight_pattern|safe - h2 Vocabulary - ul - for pos in clone.vocabulary - li - h6= pos - ul - for token in clone.vocabulary[pos] - li= token + h2 Vocabulary + ul + for pos in clone.vocabulary + li + h6= pos + ul + for token in clone.vocabulary[pos] + li= token diff --git a/config-sample.py b/config-sample.py index 239ea5d..d9a11d0 100644 --- a/config-sample.py +++ b/config-sample.py @@ -1,6 +1,13 @@ CSRF_ENABLED = True SECRET_KEY = 'some-passphrase' -MONGODB_SETTINGS = {'DB': 'youtwo'} +MONGODB_SETTINGS = { + 'DB': 'youtwo', + 'HOST': 'localhost' + + # If necessary: + #'USERNAME': 'username', + #'PASSWORD': 'pw' +} MAIL_HOST = 'smtp.gmail.com' MAIL_PORT = 587 @@ -27,3 +34,44 @@ 'VB', 'RB' # adverbs ] + +# Celery config. +# Broker (message queue) url. +BROKER_URL = 'amqp://guest@localhost:5672//' + +# Try connecting ad infinitum. +BROKER_CONNECTION_MAX_RETRIES = None + +# Result backend. +CELERY_RESULT_BACKEND = 'mongodb' +CELERY_MONGODB_BACKEND_SETTINGS = { + 'host': 'localhost', + 'port': 27017, + 'database': 'celery', + 'taskmeta_collection': 'my_taskmeta' # Collection name to use for task output +} + +# What modules to import on start. +# Note that in production environments you will want to +# remove the 'tests' tasks module. +CELERY_IMPORTS = ('app.tasks',) + +# Send emails on errors +CELERY_SEND_TASK_ERROR_EMAILS = True +ADMINS = [ + ('Francis Tseng', 'ftzeng@gmail.com') +] + +SERVER_EMAIL = 'clone.bot@gmail.com' +EMAIL_HOST = 'smtp.gmail.com' +EMAIL_PORT = 587 +EMAIL_HOST_USER = 'clone.bot@gmail.com' +EMAIL_HOST_PASSWORD = 'your-pass' +EMAIL_USE_TLS = True + +# Setting a maximum amount of tasks per worker +# so the worker processes get regularly killed +# (to reclaim memory). Not sure if this is the best +# approach, but see: +# https://github.com/publicscience/argos/issues/112 +CELERYD_MAX_TASKS_PER_CHILD=100 diff --git a/requirements.txt b/requirements.txt index 9759763..b686d0a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,6 +15,7 @@ pymongo==2.7.1 six==1.7.3 textblob==0.8.4 webassets==0.10.1 +celery==3.1.13 git+git://github.com/nltk/nltk.git git+git://github.com/ze-phyr-us/tweepy.git