Close out the rest of oauth compatibility issues (#186)

* Update client to accommodate oauth2client>=4.0

This changes has been needed for a while now. The main blocker
seems to be the use of locked_file for caching GCE credentials.
I've added a simple multiprocess lockable file cache that uses
a similar approach to that used in ouath2client's multiprocess
file storage.

Submission of this should close issue #162.

* Update test runner setup

.travis.yml

@@ -1,26 +1,32 @@
 language: python
 sudo: false
-env:
-  - TOX_ENV=py27
-  - TOX_ENV=py27oldoauth2client
-  - TOX_ENV=py27newoauth2client
-  - TOX_ENV=py34
-  - TOX_ENV=py35oauth2client15
-  - TOX_ENV=py35oauth2client30
-  - TOX_ENV=py35oauth2client41
-  - TOX_ENV=lint
+matrix:
+  include:
+  - python: "2.7"
+    env: TOX_ENV=lint
+  - python: "2.7"
+    env: TOX_ENV=py27-oauth2client1
+  - python: "2.7"
+    env: TOX_ENV=py27-oauth2client2
+  - python: "2.7"
+    env: TOX_ENV=py27-oauth2client3
+  - python: "2.7"
+    env: TOX_ENV=py27-oauth2client4
+  - python: "3.3"
+    env: TOX_ENV=py33-oauth2client4
+  - python: "3.4"
+    env: TOX_ENV=py34-oauth2client4
+  - python: "3.5"
+    env: TOX_ENV=py35-oauth2client1
+  - python: "3.5"
+    env: TOX_ENV=py35-oauth2client2
+  - python: "3.5"
+    env: TOX_ENV=py35-oauth2client3
+  - python: "3.5"
+    env: TOX_ENV=py35-oauth2client4
 install:
   - pip install tox
   - pip install . --allow-external argparse
 script: tox -e $TOX_ENV
 after_success:
-  - if [[ "${TOX_ENV}" == "py27" ]]; then tox -e coveralls; fi
-
-# Tweak for adding python3.5; see
-# https://github.com/travis-ci/travis-ci/issues/4794
-addons:
-  apt:
-    sources:
-      - deadsnakes
-    packages:
-      - python3.5
+  - if [[ "${TOX_ENV}" == "py27-oauth2client4" ]]; then tox -e coveralls; fi

apitools/base/py/http_wrapper.py

@@ -285,6 +285,7 @@ def HandleExceptionsAndRebuildHttpConnections(retry_args):
         logging.debug('Response content was invalid (%s), retrying',
                       retry_args.exc)
     elif (isinstance(retry_args.exc, TokenRefreshError) and
+          hasattr(retry_args.exc, 'status') and
           (retry_args.exc.status == TOO_MANY_REQUESTS or
            retry_args.exc.status >= 500)):
         logging.debug(
@@ -300,7 +301,7 @@ def HandleExceptionsAndRebuildHttpConnections(retry_args):
         logging.debug('Response returned a retry-after header, retrying')
         retry_after = retry_args.exc.retry_after
     else:
-        raise  # pylint: disable=misplaced-bare-raise
+        raise retry_args.exc
     RebuildHttpConnections(retry_args.http)
     logging.debug('Retrying request to url %s after exception %s',
                   retry_args.http_request.url, retry_args.exc)

apitools/base/py/http_wrapper_test.py

@@ -17,7 +17,6 @@
 import socket
 
 import httplib2
-import oauth2client
 from six.moves import http_client
 import unittest2
 
@@ -26,6 +25,15 @@
 from apitools.base.py import exceptions
 from apitools.base.py import http_wrapper
 
+# pylint: disable=ungrouped-imports
+try:
+    from oauth2client.client import HttpAccessTokenRefreshError
+    from oauth2client.client import AccessTokenRefreshError
+    _TOKEN_REFRESH_STATUS_AVAILABLE = True
+except ImportError:
+    from oauth2client.client import AccessTokenRefreshError
+    _TOKEN_REFRESH_STATUS_AVAILABLE = False
+
 
 class _MockHttpRequest(object):
 
@@ -57,6 +65,51 @@ def testRequestBodyUsesLengthProperty(self):
     def testRequestBodyWithLen(self):
         http_wrapper.Request(body='burrito')
 
+    @unittest2.skipIf(not _TOKEN_REFRESH_STATUS_AVAILABLE,
+                      'oauth2client<1.5 lacks HttpAccessTokenRefreshError.')
+    def testExceptionHandlerHttpAccessTokenError(self):
+        exception_arg = HttpAccessTokenRefreshError(status=503)
+        retry_args = http_wrapper.ExceptionRetryArgs(
+            http={'connections': {}}, http_request=_MockHttpRequest(),
+            exc=exception_arg, num_retries=0, max_retry_wait=0,
+            total_wait_sec=0)
+
+        # Disable time.sleep for this handler as it is called with
+        # a minimum value of 1 second.
+        with patch('time.sleep', return_value=None):
+            http_wrapper.HandleExceptionsAndRebuildHttpConnections(
+                retry_args)
+
+    @unittest2.skipIf(not _TOKEN_REFRESH_STATUS_AVAILABLE,
+                      'oauth2client<1.5 lacks HttpAccessTokenRefreshError.')
+    def testExceptionHandlerHttpAccessTokenErrorRaises(self):
+        exception_arg = HttpAccessTokenRefreshError(status=200)
+        retry_args = http_wrapper.ExceptionRetryArgs(
+            http={'connections': {}}, http_request=_MockHttpRequest(),
+            exc=exception_arg, num_retries=0, max_retry_wait=0,
+            total_wait_sec=0)
+
+        # Disable time.sleep for this handler as it is called with
+        # a minimum value of 1 second.
+        with self.assertRaises(HttpAccessTokenRefreshError):
+            with patch('time.sleep', return_value=None):
+                http_wrapper.HandleExceptionsAndRebuildHttpConnections(
+                    retry_args)
+
+    def testExceptionHandlerAccessTokenErrorRaises(self):
+        exception_arg = AccessTokenRefreshError()
+        retry_args = http_wrapper.ExceptionRetryArgs(
+            http={'connections': {}}, http_request=_MockHttpRequest(),
+            exc=exception_arg, num_retries=0, max_retry_wait=0,
+            total_wait_sec=0)
+
+        # Disable time.sleep for this handler as it is called with
+        # a minimum value of 1 second.
+        with self.assertRaises(AccessTokenRefreshError):
+            with patch('time.sleep', return_value=None):
+                http_wrapper.HandleExceptionsAndRebuildHttpConnections(
+                    retry_args)
+
     def testDefaultExceptionHandler(self):
         """Ensures exception handles swallows (retries)"""
         mock_http_content = 'content'.encode('utf8')
@@ -68,7 +121,6 @@ def testDefaultExceptionHandler(self):
                 socket.gaierror(),
                 httplib2.ServerNotFoundError(),
                 ValueError(),
-                oauth2client.client.HttpAccessTokenRefreshError(status=503),
                 exceptions.RequestError(),
                 exceptions.BadStatusCodeError(
                     {'status': 503}, mock_http_content, 'url'),

tox.ini

@@ -1,94 +1,30 @@
 [tox]
 envlist =
-    py26
-    py27
-    py27oldoauth2client
-    py27newoauth2client
-    pypy
-    py34
-    py35oauth2client15
-    py35oauth2client30
-    py35oauth2client41
-    lint
-    cover
+    py26-oauth2client4
+    py27-oauth2client{1,2,3,4}
+    py33-oauth2client41
+    py34-oauth2client41
+    py35-oauth2client{1,2,3,4}
 
 [testenv]
 deps =
     nose
     python-gflags==3.0.6
+    oauth2client1: oauth2client<1.5dev
+    oauth2client2: oauth2client>=2,<=3dev
+    oauth2client3: oauth2client>=3,<=4dev
+    oauth2client4: oauth2client>=4,<=5dev
 commands =
     pip install google-apitools[testing]
     nosetests []
 passenv = TRAVIS*
 
-[testenv:py27newoauth2client]
-commands =
-    pip install oauth2client==4.1.0
-    {[testenv]commands}
-deps = {[testenv]deps}
-
-[testenv:py27oldoauth2client]
-commands =
-    pip install oauth2client==1.5.2
-    {[testenv]commands}
-deps = {[testenv]deps}
-
-[testenv:py34]
-basepython = python3.4
-deps =
-    mock
-    nose
-    unittest2
-commands = nosetests []
-
-[testenv:py35oauth2client15]
-basepython = python3.5
-deps =
-    mock
-    nose
-    unittest2
-commands =
-    pip install oauth2client==1.5.2
-    nosetests []
-
-[testenv:py35oauth2client30]
-basepython = python3.5
-deps =
-    mock
-    nose
-    unittest2
-commands =
-    pip install oauth2client==3.0.0
-    nosetests []
-
-[testenv:py35oauth2client41]
-basepython = python3.5
-deps =
-    mock
-    nose
-    unittest2
-commands =
-    pip install oauth2client==4.1.0
-    nosetests []
-
-[testenv:py35]
-basepython = python3.5
-deps =
-    mock
-    nose
-    unittest2
-commands = nosetests []
-
-[pep8]
-exclude = samples/*_sample/*/*,*/testdata/*,*.egg/,*.egg-info/,.*/,ez_setup.py,build
-verbose = 1
-
 [testenv:lint]
 basepython =
     python2.7
 commands =
     pip install six google-apitools[testing]
-    pep8
+    pep8 apitools
     python run_pylint.py
 deps =
     pep8