diff --git a/.gitignore b/.gitignore index b213e9a..114b710 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ config.py .webassets-cache bower/ index.css -config.py \ No newline at end of file +config.py +data/ \ No newline at end of file diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..8238065 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1,12 @@ +from flask import Flask +from flask.ext.mongoengine import MongoEngine + +app = Flask(__name__, static_folder='static', static_url_path='') + +# Load config. +app.config.from_object('config') + +# Setup the database. +db = MongoEngine(app) + +from app import notify diff --git a/app/models.py b/app/models.py new file mode 100644 index 0000000..6a76f53 --- /dev/null +++ b/app/models.py @@ -0,0 +1,42 @@ +import datetime +from app import db + +class Verb(db.Document): + created_at = db.DateTimeField(default=datetime.datetime.now, required=True) + body = db.StringField(required=True, unique=True) + + meta = { + 'allow_inheritance': True, + 'indexes': ['-created_at'], + 'ordering': ['-created_at'] + } + +class Noun(db.Document): + created_at = db.DateTimeField(default=datetime.datetime.now, required=True) + body = db.StringField(required=True, unique=True) + + meta = { + 'allow_inheritance': True, + 'indexes': ['-created_at'], + 'ordering': ['-created_at'] + } + +class Adjective(db.Document): + created_at = db.DateTimeField(default=datetime.datetime.now, required=True) + body = db.StringField(required=True, unique=True) + + meta = { + 'allow_inheritance': True, + 'indexes': ['-created_at'], + 'ordering': ['-created_at'] + } + +class Adverb(db.Document): + created_at = db.DateTimeField(default=datetime.datetime.now, required=True) + body = db.StringField(required=True, unique=True) + + meta = { + 'allow_inheritance': True, + 'indexes': ['-created_at'], + 'ordering': ['-created_at'] + } diff --git a/app/notify.py b/app/notify.py new file mode 100644 index 0000000..85cd1d4 --- /dev/null +++ b/app/notify.py @@ -0,0 +1,12 @@ +from app import app + +cfg = app.config + +# Email error messages. +if not app.debug: + import logging + from logging.handlers import SMTPHandler + mail_handler = SMTPHandler((cfg['MAIL_HOST'], cfg['MAIL_PORT']), 'dont-talk-back@'+cfg['MAIL_HOST'], cfg['MAIL_TARGETS'], 'brain is floudering!', (cfg['MAIL_USER'], cfg['MAIL_PASS'])) + mail_handler.setLevel(logging.ERROR) + app.logger.addHandler(mail_handler) + diff --git a/application.py b/application.py new file mode 100644 index 0000000..4af28fb --- /dev/null +++ b/application.py @@ -0,0 +1,4 @@ +from app import app + +if __name__ == '__main__': + app.run(debug=False) diff --git a/config-sample.py b/config-sample.py index 841b910..a725e3e 100644 --- a/config-sample.py +++ b/config-sample.py @@ -1,3 +1,13 @@ +CSRF_ENABLED = True +SECRET_KEY = 'some-passphrase' +MONGODB_SETTINGS = {'DB': 'youtwo'} + +MAIL_HOST = 'smtp.gmail.com' +MAIL_PORT = 587 +MAIL_USER = 'some@email.com' +MAIL_PASS = 'somepass' +MAIL_TARGETS = ['someadmin@email.com'] + # You can get these by creating a new app at # https://apps.twitter.com/ TWITTER_CONSUMER_KEY = 'fill_me_in' diff --git a/requirements.txt b/requirements.txt index 568173b..a274cb9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,17 @@ -tweepy==2.3.0 -wsgiref==0.1.2 +Flask==0.10.1 +Flask-WTF==0.9.5 +Jinja2==2.7.3 +MarkupSafe==0.23 +PyYAML==3.11 +WTForms==1.0.5 +Werkzeug==0.9.6 +flask-mongoengine==0.7.0 +itsdangerous==0.24 +mongoengine==0.8.7 +nltk==3.0b1 +numpy==1.8.1 +pymongo==2.7.1 +textblob==0.8.4 + +git+git://github.com/nltk/nltk.git +git+git://github.com/ze-phyr-us/tweepy/git diff --git a/run.py b/run.py new file mode 100644 index 0000000..d1760a3 --- /dev/null +++ b/run.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- +import sys +import string +import random +import json +import twitter +import re +from textblob import TextBlob + +def main(): + try: + import config + except ImportError: + print('No config found. Have you renamed `config-sample.py` to `config.py` and filled in your info?') + return + + if len(sys.argv) < 2: + print('Please tell me what example to run!') + return + + try: + globals()[sys.argv[1]](); + except KeyError: + print('Doesn\'t seem to be an example by that name.') + return + +def tweets(): + user_tweets = [] + for i in range(10): + user_tweets += twitter.tweets('frnsys', page=i) + with open('data/data.txt', 'w') as outfile: + json.dump(user_tweets, outfile) + +def process(): + f = open('data/data.txt', 'r') + + # Words matched with POS tags. + speech_parts = {} + + # Chains of POS tags to build + # tweets out of. + speech_patterns = {} + for tweet in json.load(f): + pattern_ = [] + + if '@' not in tweet['body']: # Trying without any @ mentions. + text = tweet['body'] + + # Remove urls + text = re.sub(r"(?:\@|https?\://)\S+", "", text) + + for t in TextBlob(text).pos_tags: + token = t[0] + tag = t[1] + + pattern_.append(tag) + + if tag == '-NONE-': + continue + + if tag not in speech_parts: + speech_parts[tag] = [] + speech_parts[tag].append(token) + + pattern = '.'.join(pattern_) + if pattern not in speech_patterns: + speech_patterns[pattern] = 0 + speech_patterns[pattern] += 1 + + with open('data/speech_parts.json', 'w') as outfile: + json.dump(speech_parts, outfile) + + with open('data/speech_patterns.json', 'w') as outfile: + json.dump(speech_patterns, outfile) + +def generate(): + with open('data/speech_parts.json', 'r') as f: + speech_parts = json.load(f) + with open('data/speech_patterns.json', 'r') as f: + speech_patterns = json.load(f) + + pattern = _weighted_choice(speech_patterns) + + tweet = [] + for tag in pattern.split('.'): + token = random.choice(speech_parts[tag]) + tweet.append(token) + print(' '.join(tweet)) + + +def _weighted_choice(choices): + """ + Random selects a key from a dictionary, + where each key's value is its probability weight. + """ + # Randomly select a value between 0 and + # the sum of all the weights. + rand = random.uniform(0, sum(choices.values())) + + # Seek through the dict until a key is found + # resulting in the random value. + summ = 0.0 + for key, value in choices.items(): + summ += value + if rand < summ: return key + + # If this returns False, + # it's likely because the knowledge is empty. + return False + + +if __name__ == '__main__': + main() + diff --git a/twitter.py b/twitter.py index 26533fb..c6b0a20 100644 --- a/twitter.py +++ b/twitter.py @@ -17,7 +17,7 @@ def _api(): api = _api() -def tweets(username, count=200): +def tweets(username, count=200, page=0): """ Returns 200 last tweets for a user. """ @@ -26,7 +26,7 @@ def tweets(username, count=200): 'tid': tweet.id, 'protected': tweet.user.protected, 'retweeted': tweet.retweeted - } for tweet in api.user_timeline(screen_name=username, count=count)] + } for tweet in api.user_timeline(screen_name=username, count=count, page=page)] def retweet(id): """