config scrapy-mongodb pipeline

raphapassini · Jul 23, 2014 · 46c737c · 46c737c
1 parent 1b31246
commit 46c737c
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 3 deletions.
diff --git a/.gitignore b/.gitignore
@@ -53,3 +53,8 @@ coverage.xml
 # Sphinx documentation
 docs/_build/
 
+# Scrapyd files
+dbs/
+items/
+logs/
+*.pid
diff --git a/pyjobs/settings.py b/pyjobs/settings.py
@@ -14,8 +14,16 @@
 NEWSPIDER_MODULE = 'pyjobs.spiders'
 
 ITEM_PIPELINES = {
-    'pyjobs.pipelines.MongoPipeline': 0,
+    'scrapy_mongodb.MongoDBPipeline': 0,
 }
 
-# Crawl responsibly by identifying yourself (and your website) on the user-agent
+# Config MongoDB
+MONGODB_URI = 'mongodb://localhost:27017'
+MONGODB_DATABASE = 'pyjobs'
+MONGODB_COLLECTION = 'jobs'
+MONGODB_UNIQUE_KEY = 'uid'
+
+
+# Crawl responsibly by identifying yourself (and your website)
+# on the user-agent
 #USER_AGENT = 'pyjobs (+http://www.yourdomain.com)'
diff --git a/requirements.txt b/requirements.txt
@@ -14,6 +14,7 @@ pyasn1-modules==0.0.5
 pycparser==2.10
 pymongo==2.7.1
 queuelib==1.1.1
+scrapy-mongodb==0.7.1
 service-identity==1.0.0
 six==1.7.3
 w3lib==1.6

diff --git a/scrapy.cfg b/scrapy.cfg
@@ -7,5 +7,5 @@
 default = pyjobs.settings
 
 [deploy]
-#url = http://localhost:6800/
+url = http://localhost:6800/
 project = pyjobs