some initial experiments

.gitignore

@@ -10,4 +10,5 @@ config.py
 .webassets-cache
 bower/
 index.css
-config.py
+config.py
+data/

app/__init__.py

@@ -0,0 +1,12 @@
+from flask import Flask
+from flask.ext.mongoengine import MongoEngine
+
+app = Flask(__name__, static_folder='static', static_url_path='')
+
+# Load config.
+app.config.from_object('config')
+
+# Setup the database.
+db = MongoEngine(app)
+
+from app import notify

app/models.py

@@ -0,0 +1,42 @@
+import datetime
+from app import db
+
+class Verb(db.Document):
+    created_at = db.DateTimeField(default=datetime.datetime.now, required=True)
+    body = db.StringField(required=True, unique=True)
+
+    meta = {
+            'allow_inheritance': True,
+            'indexes': ['-created_at'],
+            'ordering': ['-created_at']
+    }
+
+class Noun(db.Document):
+    created_at = db.DateTimeField(default=datetime.datetime.now, required=True)
+    body = db.StringField(required=True, unique=True)
+
+    meta = {
+            'allow_inheritance': True,
+            'indexes': ['-created_at'],
+            'ordering': ['-created_at']
+    }
+
+class Adjective(db.Document):
+    created_at = db.DateTimeField(default=datetime.datetime.now, required=True)
+    body = db.StringField(required=True, unique=True)
+
+    meta = {
+            'allow_inheritance': True,
+            'indexes': ['-created_at'],
+            'ordering': ['-created_at']
+    }
+
+class Adverb(db.Document):
+    created_at = db.DateTimeField(default=datetime.datetime.now, required=True)
+    body = db.StringField(required=True, unique=True)
+
+    meta = {
+            'allow_inheritance': True,
+            'indexes': ['-created_at'],
+            'ordering': ['-created_at']
+    }

app/notify.py

@@ -0,0 +1,12 @@
+from app import app
+
+cfg = app.config
+
+# Email error messages.
+if not app.debug:
+    import logging
+    from logging.handlers import SMTPHandler
+    mail_handler = SMTPHandler((cfg['MAIL_HOST'], cfg['MAIL_PORT']), 'dont-talk-back@'+cfg['MAIL_HOST'], cfg['MAIL_TARGETS'], 'brain is floudering!', (cfg['MAIL_USER'], cfg['MAIL_PASS']))
+    mail_handler.setLevel(logging.ERROR)
+    app.logger.addHandler(mail_handler)
+

application.py

@@ -0,0 +1,4 @@
+from app import app
+
+if __name__ == '__main__':
+    app.run(debug=False)

config-sample.py

@@ -1,3 +1,13 @@
+CSRF_ENABLED = True
+SECRET_KEY = 'some-passphrase'
+MONGODB_SETTINGS = {'DB': 'youtwo'}
+
+MAIL_HOST = 'smtp.gmail.com'
+MAIL_PORT = 587
+MAIL_USER = '[email protected]'
+MAIL_PASS = 'somepass'
+MAIL_TARGETS = ['[email protected]']
+
 # You can get these by creating a new app at
 # https://apps.twitter.com/
 TWITTER_CONSUMER_KEY = 'fill_me_in'

requirements.txt

@@ -1,2 +1,17 @@
-tweepy==2.3.0
-wsgiref==0.1.2
+Flask==0.10.1
+Flask-WTF==0.9.5
+Jinja2==2.7.3
+MarkupSafe==0.23
+PyYAML==3.11
+WTForms==1.0.5
+Werkzeug==0.9.6
+flask-mongoengine==0.7.0
+itsdangerous==0.24
+mongoengine==0.8.7
+nltk==3.0b1
+numpy==1.8.1
+pymongo==2.7.1
+textblob==0.8.4
+
+git+git://github.com/nltk/nltk.git
+git+git://github.com/ze-phyr-us/tweepy/git

run.py

@@ -0,0 +1,114 @@
+# -*- coding: utf-8 -*-
+import sys
+import string
+import random
+import json
+import twitter
+import re
+from textblob import TextBlob
+
+def main():
+    try:
+        import config
+    except ImportError:
+        print('No config found. Have you renamed `config-sample.py` to `config.py` and filled in your info?')
+        return
+
+    if len(sys.argv) < 2:
+        print('Please tell me what example to run!')
+        return
+
+    try:
+        globals()[sys.argv[1]]();
+    except KeyError:
+        print('Doesn\'t seem to be an example by that name.')
+        return
+
+def tweets():
+    user_tweets = []
+    for i in range(10):
+        user_tweets += twitter.tweets('frnsys', page=i)
+    with open('data/data.txt', 'w') as outfile:
+        json.dump(user_tweets, outfile)
+
+def process():
+    f = open('data/data.txt', 'r')
+
+    # Words matched with POS tags.
+    speech_parts = {}
+
+    # Chains of POS tags to build
+    # tweets out of.
+    speech_patterns = {}
+    for tweet in json.load(f):
+        pattern_ = []
+
+        if '@' not in tweet['body']:    # Trying without any @ mentions.
+            text = tweet['body']
+
+            # Remove urls
+            text = re.sub(r"(?:\@|https?\://)\S+", "", text)
+
+            for t in TextBlob(text).pos_tags:
+                token = t[0]
+                tag = t[1]
+
+                pattern_.append(tag)
+
+                if tag == '-NONE-':
+                    continue
+
+                if tag not in speech_parts:
+                    speech_parts[tag] = []
+                speech_parts[tag].append(token)
+
+            pattern = '.'.join(pattern_)
+            if pattern not in speech_patterns:
+                speech_patterns[pattern] = 0
+            speech_patterns[pattern] += 1
+
+    with open('data/speech_parts.json', 'w') as outfile:
+        json.dump(speech_parts, outfile)
+
+    with open('data/speech_patterns.json', 'w') as outfile:
+        json.dump(speech_patterns, outfile)
+
+def generate():
+    with open('data/speech_parts.json', 'r') as f:
+        speech_parts = json.load(f)
+    with open('data/speech_patterns.json', 'r') as f:
+        speech_patterns = json.load(f)
+
+    pattern = _weighted_choice(speech_patterns)
+
+    tweet = []
+    for tag in pattern.split('.'):
+        token = random.choice(speech_parts[tag])
+        tweet.append(token)
+    print(' '.join(tweet))
+
+
+def _weighted_choice(choices):
+    """
+    Random selects a key from a dictionary,
+    where each key's value is its probability weight.
+    """
+    # Randomly select a value between 0 and
+    # the sum of all the weights.
+    rand = random.uniform(0, sum(choices.values()))
+
+    # Seek through the dict until a key is found
+    # resulting in the random value.
+    summ = 0.0
+    for key, value in choices.items():
+        summ += value
+        if rand < summ: return key
+
+    # If this returns False,
+    # it's likely because the knowledge is empty.
+    return False
+
+
+if __name__ == '__main__':
+    main()
+

twitter.py

@@ -17,7 +17,7 @@ def _api():
 
 api = _api()
 
-def tweets(username, count=200):
+def tweets(username, count=200, page=0):
     """
     Returns 200 last tweets for a user.
     """
@@ -26,7 +26,7 @@ def tweets(username, count=200):
                 'tid': tweet.id,
                 'protected': tweet.user.protected,
                 'retweeted': tweet.retweeted
-            } for tweet in api.user_timeline(screen_name=username, count=count)]
+            } for tweet in api.user_timeline(screen_name=username, count=count, page=page)]
 
 def retweet(id):
     """