Ensure client generation works smoothly in py3.

It turns out that generated clients had a few issues in python3:

* the discovery doc was returned as bytes, but `json.loads` wanted a string
* in python3, source files should be strings, not bytes
* \N, \u, and \U have special interpretations in python3 strings, which makes
  them invalid bare sequences in a docstring.

This commit updates these and re-enables tests for python3. The only remaining
py2-only check is the "did we generate the exact same source file" check,
which fails for trivial reasons (eg u'foo' vs. 'foo') for python3.

Since there are no more python27-only tests, I removed the "disable test in
2.7" helper.

apitools/gen/client_generation_test.py

@@ -18,15 +18,18 @@
 import importlib
 import logging
 import os
+import six
 import subprocess
 import sys
 import tempfile
 
-import unittest2
-
 from apitools.gen import gen_client
 from apitools.gen import test_utils
 
+if six.PY2:
+    import unittest2 as unittest
+else:
+    import unittest
 
 _API_LIST = [
     'bigquery.v2',
@@ -36,14 +39,13 @@
 ]
 
 
-class ClientGenerationTest(unittest2.TestCase):
+class ClientGenerationTest(unittest.TestCase):
 
     def setUp(self):
         super(ClientGenerationTest, self).setUp()
         self.gen_client_binary = 'gen_client'
 
     @test_utils.SkipOnWindows
-    @test_utils.RunOnlyOnPython27
     def testGeneration(self):
         for api in _API_LIST:
             with test_utils.TempDir(change_to=True):

apitools/gen/gen_client_test.py

@@ -32,7 +32,6 @@ def _GetContent(file_path):
         return f.read()
 
 
-@test_utils.RunOnlyOnPython27
 class ClientGenCliTest(unittest2.TestCase):
 
     def testHelp_NotEnoughArguments(self):

apitools/gen/test_utils.py

@@ -17,17 +17,14 @@
 
 import contextlib
 import os
-import tempfile
 import shutil
 import sys
+import tempfile
 
 import six
 import unittest2
 
 
-RunOnlyOnPython27 = unittest2.skipUnless(
-    sys.version_info[:2] == (2, 7), 'Only runs in Python 2.7')
-
 SkipOnWindows = unittest2.skipIf(
     os.name == 'nt', 'Does not run on windows')
 

apitools/gen/util.py

@@ -255,6 +255,11 @@ def CleanDescription(description):
     """Return a version of description safe for printing in a docstring."""
     if not isinstance(description, six.string_types):
         return description
+    if six.PY3:
+        # https://docs.python.org/3/reference/lexical_analysis.html#index-18
+        description = description.replace('\\N', '\\\\N')
+        description = description.replace('\\u', '\\\\u')
+        description = description.replace('\\U', '\\\\U')
     return description.replace('"""', '" " "')
 
 
@@ -298,7 +303,8 @@ def __call__(self, *args):
                 line = (args[0] % args[1:]).rstrip()
             else:
                 line = args[0].rstrip()
-            line = line.encode('ascii', 'backslashreplace')
+            if six.PY2:
+                line = line.encode('ascii', 'backslashreplace')
             print('%s%s' % (self.__indent, line), file=self.__out)
         else:
             print('', file=self.__out)
@@ -327,7 +333,10 @@ def FetchDiscoveryDoc(discovery_url, retries=5):
     for url in discovery_urls:
         for _ in range(retries):
             try:
-                discovery_doc = json.loads(urllib_request.urlopen(url).read())
+                content = urllib_request.urlopen(url).read()
+                if isinstance(content, bytes):
+                    content = content.decode('utf8')
+                discovery_doc = json.loads(content)
                 break
             except (urllib_error.HTTPError, urllib_error.URLError) as e:
                 logging.info(

samples/uptodate_check_test.py

@@ -15,6 +15,7 @@
 import os
 import difflib
 
+import six
 import unittest2
 
 from apitools.gen import gen_client
@@ -30,7 +31,6 @@ def _GetContent(file_path):
         return f.read()
 
 
-@test_utils.RunOnlyOnPython27
 class ClientGenCliTest(unittest2.TestCase):
 
     def AssertDiffEqual(self, expected, actual):
@@ -59,6 +59,10 @@ def _CheckGeneratedFiles(self, api_name, api_version):
                      prefix + '_messages.py',
                      '__init__.py']))
             self.assertEquals(expected_files, set(os.listdir(tmp_dir_path)))
+            if six.PY3:
+                # The source files won't be identical under python3,
+                # so we exit early.
+                return
             for expected_file in expected_files:
                 self.AssertDiffEqual(
                     _GetContent(GetSampleClientPath(