new configs and fix worker naming

celeryconfig.py

@@ -16,8 +16,10 @@
 
 
 # these options will be defaults in future as per http://celery.readthedocs.org/en/latest/getting-started/brokers/redis.html
-BROKER_TRANSPORT_OPTIONS = {'fanout_prefix': True}
-BROKER_TRANSPORT_OPTIONS = {'fanout_patterns': True}
+BROKER_TRANSPORT_OPTIONS = {'fanout_prefix': True, 
+                            'fanout_patterns': True, 
+                            'visibility_timeout': 60  # one minute
+                            }
 
 CELERY_DEFAULT_QUEUE = 'core_high'
 CELERY_QUEUES = [
@@ -37,14 +39,18 @@
 CELERY_ACKS_LATE = True
 
 # remove this, might fix deadlocks as per https://github.com/celery/celery/issues/970
-# CELERYD_MAX_TASKS_PER_CHILD = 1000
+# CELERYD_MAX_TASKS_PER_CHILD = 100
 
 CELERYD_FORCE_EXECV = True
 CELERY_TRACK_STARTED = True
 
+# https://groups.google.com/forum/#!topic/celery-users/Y_ifty2l6Fc
+CELERYD_PREFETCH_MULTIPLIER=1
+
 # List of modules to import when celery starts.
 CELERY_IMPORTS = ("tasks",)
 
 CELERY_ANNOTATIONS = {
     'celery.chord_unlock': {'soft_time_limit': 60}  # 1 minute
 }
+

celeryworkers.sh

@@ -3,10 +3,10 @@
 
 for ((i=1; i<=$CELERY_MULTI_WORKERS; i++))
 do
-  COMMAND="celery worker --pool=$CELERY_POOL -n core-$DYNO:${i} -Q core_high --loglevel=$CELERY_LOGLEVEL --config=celeryconfig --events --concurrency=$CELERY_CONCURRENCY -Ofair" 
+  COMMAND="celery worker --pool=$CELERY_POOL -n core-high-$DYNO:${i} -Q core_high --loglevel=$CELERY_LOGLEVEL --config=celeryconfig --events --concurrency=$CELERY_CONCURRENCY -Ofair" 
   echo $COMMAND
   $COMMAND&
-  COMMAND="celery worker --pool=$CELERY_POOL -n core-$DYNO:${i} -Q core_low --loglevel=$CELERY_LOGLEVEL --config=celeryconfig --events --concurrency=$CELERY_CONCURRENCY -Ofair" 
+  COMMAND="celery worker --pool=$CELERY_POOL -n core-low-$DYNO:${i} -Q core_low --loglevel=$CELERY_LOGLEVEL --config=celeryconfig --events --concurrency=$CELERY_CONCURRENCY -Ofair" 
   echo $COMMAND
   $COMMAND&
 done