1%-3% of /g requests end with Contention

[rahulsom]

java.util.ConcurrentModificationException: too much contention on these datastore entities. please try again.
    at com.google.appengine.api.datastore.DatastoreApiHelper.translateError(DatastoreApiHelper.java:36)
    at com.google.appengine.api.datastore.DatastoreApiHelper$1.convertException(DatastoreApiHelper.java:70)
    at com.google.appengine.api.utils.FutureWrapper.get(FutureWrapper.java:94)
    at com.google.appengine.api.utils.FutureWrapper.get(FutureWrapper.java:86)
    at com.google.appengine.api.utils.FutureWrapper.get(FutureWrapper.java:86)
    at com.google.appengine.api.utils.FutureWrapper.get(FutureWrapper.java:86)
    at com.google.appengine.api.datastore.FutureHelper.getInternal(FutureHelper.java:71)
    at com.google.appengine.api.datastore.FutureHelper.quietGet(FutureHelper.java:32)
    at com.google.appengine.api.datastore.TransactionImpl.commit(TransactionImpl.java:115)
    at groovyx.gaelyk.extensions.DatastoreExtensions.withTransaction(DatastoreExtensions.groovy:344)
    at groovyx.gaelyk.extensions.DatastoreExtensions.withTransaction(DatastoreExtensions.groovy:319)
    at sun.reflect.GeneratedMethodAccessor62.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:45)
    at org.codehaus.groovy.runtime.metaclass.ReflectionMetaMethod.invoke(ReflectionMetaMethod.java:51)
    at org.codehaus.groovy.runtime.metaclass.NewInstanceMetaMethod.invoke(NewInstanceMetaMethod.java:54)
    at org.codehaus.groovy.runtime.callsite.PojoMetaMethodSite$PojoMetaMethodSiteNoUnwrapNoCoerce.invoke(PojoMetaMethodSite.java:271)
    at org.codehaus.groovy.runtime.callsite.PojoMetaMethodSite.call(PojoMetaMethodSite.java:53)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.call(AbstractCallSite.java:116)
    at random.run(random.groovy:19)
    at groovyx.gaelyk.GaelykServlet.runPrecompiled(GaelykServlet.groovy:161)
    at groovyx.gaelyk.GaelykServlet.doService(GaelykServlet.groovy:112)
    at groovyx.gaelyk.GaelykServlet.service(GaelykServlet.groovy:87)
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
    at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
    at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1166)
    at javax.servlet.FilterChain$doFilter.call(Unknown Source)
    at groovyx.gaelyk.routes.RoutesFilter.doFilter(RoutesFilter.groovy:185)
    at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
    at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:388)
    at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
    at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
    at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:765)
    at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:418)
    at org.mortbay.jetty.servlet.Dispatcher.forward(Dispatcher.java:327)
    at org.mortbay.jetty.servlet.Dispatcher.forward(Dispatcher.java:126)
    at groovyx.gaelyk.cache.CacheHandler.serve(CacheHandler.groovy:88)
    at groovyx.gaelyk.cache.CacheHandler$serve.call(Unknown Source)
    at groovyx.gaelyk.routes.RoutesFilter.doFilter(RoutesFilter.groovy:169)
    at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
    at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
    at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
    at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
    at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
    at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:388)
    at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
    at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
    at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:765)
    at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:418)
    at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
    at org.mortbay.jetty.Server.handle(Server.java:326)
    at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
    at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:923)
    at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
    at com.google.tracing.TraceContext$TraceContextRunnable.runInContext(TraceContext.java:437)
    at com.google.tracing.TraceContext$TraceContextRunnable$1.run(TraceContext.java:444)
    at com.google.tracing.CurrentContext.runInContext(CurrentContext.java:188)
    at com.google.tracing.TraceContext$AbstractTraceContextCallback.runInInheritedContextNoUnref(TraceContext.java:308)
    at com.google.tracing.TraceContext$AbstractTraceContextCallback.runInInheritedContext(TraceContext.java:300)
    at com.google.tracing.TraceContext$TraceContextRunnable.run(TraceContext.java:441)
    at java.lang.Thread.run(Thread.java:724)

Options:

• Sharding - https://developers.google.com/appengine/articles/sharding_counters
• Pull Queues

[rahulsom]

Hasn't been seen in a while.