Move app engine tests over to pytest
MODIFY
.travis.yml
.travis.yml
@@ -14,7 +14,7 @@ notifications:
email: false
env:
global:
- - GAE_PYTHONPATH=${HOME}/.cache/google_appengine
+ - GAE_SDK_PATH=${HOME}/.cache/google_appengine
- PYTHONWARNINGS=always::DeprecationWarning
matrix:
- TOXENV=flake8-py3

MODIFY
_travis/install.sh
_travis/install.sh
@@ -57,5 +57,5 @@ pip install tox
if [[ "${TOXENV}" == "gae" ]]; then
pip install gcp-devrel-py-tools
- gcp-devrel-py-tools download-appengine-sdk ${GAE_PYTHONPATH}
+ gcp-devrel-py-tools download-appengine-sdk "$(dirname ${GAE_SDK_PATH})"
fi

MODIFY
dev-requirements.txt
dev-requirements.txt
@@ -11,3 +11,4 @@ pkginfo>=1.0,!=1.3.0
psutil==4.3.1
pytest-cov==2.5.1
pytest==3.1.0
+gcp-devrel-py-tools==0.0.6

MODIFY
test/appengine/__init__.py
test/appengine/__init__.py
-import os
-import sys
-import unittest
-from nose.plugins.skip import SkipTest
-
-
-def activate_sandbox():
- """
- Enables parts of the GAE sandbox that are relevant.
-
- Inserts the stub module import hook which causes the usage of appengine-specific
- httplib, httplib2, socket, etc.
- """
- from google.appengine.tools.devappserver2.python import sandbox
-
- for name in list(sys.modules):
- if name in sandbox.dist27.MODULE_OVERRIDES:
- del sys.modules[name]
- sys.meta_path.insert(0, sandbox.StubModuleImportHook())
- sys.path_importer_cache = {}
-
-
-def deactivate_sandbox():
- from google.appengine.tools.devappserver2.python import sandbox
-
- sys.meta_path = [
- x for x in sys.meta_path if not isinstance(x, sandbox.StubModuleImportHook)]
- sys.path_importer_cache = {}
-
- # Delete any instances of sandboxed modules.
- for name in list(sys.modules):
- if name in sandbox.dist27.MODULE_OVERRIDES:
- del sys.modules[name]
-
-
-class AppEngineSandboxTest(unittest.TestCase):
-
- @classmethod
- def setUpClass(cls):
-
- if sys.version_info[:2] != (2, 7):
- raise SkipTest("App Engine only tests on py2.7")
-
- if 'APPLICATION_ID' not in os.environ:
- raise SkipTest("NoseGAE plugin not used.")
-
- try:
- activate_sandbox()
- except ImportError:
- raise SkipTest("App Engine SDK not available.")
-
- @classmethod
- def tearDownClass(self):
- try:
- deactivate_sandbox()
- except ImportError:
- pass
-
-
-class MockResponse(object):
- def __init__(self, content, status_code, content_was_truncated, final_url, headers):
- import httplib
- from StringIO import StringIO
-
- self.content = content
- self.status_code = status_code
- self.content_was_truncated = content_was_truncated
- self.final_url = final_url
- self.header_msg = httplib.HTTPMessage(StringIO(''.join(
- ["%s: %s\n" % (k, v) for k, v in headers.iteritems()] + ["\n"])))
- self.headers = self.header_msg.items()

DELETE
test/appengine/app.yaml
-# dummy app.yaml for nosegae
-
-api_version: 1
-runtime: python27
-threadsafe: true
-
-handlers:
-- url: /
- static_files: README.md
- upload: README.md
- mime_type: text/plain

ADD
test/appengine/conftest.py
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+
+# Import py.test hooks and fixtures for App Engine
+from gcp_devrel.testing.appengine import (
+ pytest_configure,
+ pytest_runtest_call,
+ testbed)
+import pytest
+import six
+
+(pytest_configure)
+(pytest_runtest_call)
+(testbed)
+
+
+@pytest.fixture
+def sandbox(testbed):
+ """
+ Enables parts of the GAE sandbox that are relevant.
+ Inserts the stub module import hook which causes the usage of
+ appengine-specific httplib, httplib2, socket, etc.
+ """
+ from google.appengine.tools.devappserver2.python import sandbox
+
+ for name in list(sys.modules):
+ if name in sandbox.dist27.MODULE_OVERRIDES:
+ del sys.modules[name]
+ sys.meta_path.insert(0, sandbox.StubModuleImportHook())
+ sys.path_importer_cache = {}
+
+ yield testbed
+
+ sys.meta_path = [
+ x for x in sys.meta_path
+ if not isinstance(x, sandbox.StubModuleImportHook)]
+ sys.path_importer_cache = {}
+
+ # Delete any instances of sandboxed modules.
+ for name in list(sys.modules):
+ if name in sandbox.dist27.MODULE_OVERRIDES:
+ del sys.modules[name]
+
+
+def pytest_ignore_collect(path, config):
+ """Skip App Engine tests in python 3 or if no SDK is available."""
+ if 'appengine' in str(path):
+ if six.PY3:
+ return True
+ if not os.environ.get('GAE_SDK_PATH'):
+ return True
+ return False

DELETE
test/appengine/nose.cfg
-[nosetests]
-cover-min-percentage=0
-with-gae=1
-gae-application=test/appengine/app.yaml

DELETE
test/appengine/requirements.txt
-NoseGAE==0.5.10

RENAME
test/contrib/test_gae_manager.py
test/appengine/test_gae_manager.py
-import unittest
+import dummyserver.testcase
+import pytest
-from dummyserver.testcase import HTTPSDummyServerTestCase
-from nose.plugins.skip import SkipTest
+from urllib3.contrib import appengine
+import urllib3.exceptions
+import urllib3.util.url
+import urllib3.util.retry
-try:
- from google.appengine.api import urlfetch
- (urlfetch)
-except ImportError:
- raise SkipTest("App Engine SDK not available.")
-
-from urllib3.contrib.appengine import AppEngineManager, AppEnginePlatformError
-from urllib3.exceptions import (
- TimeoutError,
- ProtocolError,
- SSLError)
-from urllib3.util.url import Url
-from urllib3.util.retry import Retry, RequestHistory
-
-from test.with_dummyserver.test_connectionpool import (
- TestConnectionPool, TestRetry, TestRetryAfter)
-
-
-# Prevent nose from running these test.
-TestConnectionPool.__test__ = False
-TestRetry.__test__ = False
+from test.with_dummyserver import test_connectionpool
# This class is used so we can re-use the tests from the connection pool.
@@ -44,7 +27,7 @@ class MockPool(object):
return self.manager.urlopen(method, url, *args, **kwargs)
def _absolute_url(self, path):
- return Url(
+ return urllib3.util.url.Url(
scheme=self.scheme,
host=self.host,
port=self.port,
@@ -54,14 +37,11 @@ class MockPool(object):
# Note that this doesn't run in the sandbox, it only runs with the URLFetch
# API stub enabled. There's no need to enable the sandbox as we know for a fact
# that URLFetch is used by the connection manager.
-class TestGAEConnectionManager(TestConnectionPool):
- __test__ = True
-
- # Magic class variable that tells NoseGAE to enable the URLFetch stub.
- nosegae_urlfetch = True
+@pytest.mark.usefixtures('testbed')
+class TestGAEConnectionManager(test_connectionpool.TestConnectionPool):
def setUp(self):
- self.manager = AppEngineManager()
+ self.manager = appengine.AppEngineManager()
self.pool = MockPool(self.host, self.port, self.manager)
# Tests specific to AppEngineManager
@@ -69,7 +49,7 @@ class TestGAEConnectionManager(TestConnectionPool):
def test_exceptions(self):
# DeadlineExceededError -> TimeoutError
self.assertRaises(
- TimeoutError,
+ urllib3.exceptions.TimeoutError,
self.pool.request,
'GET',
'/sleep?seconds=0.005',
@@ -77,21 +57,21 @@ class TestGAEConnectionManager(TestConnectionPool):
# InvalidURLError -> ProtocolError
self.assertRaises(
- ProtocolError,
+ urllib3.exceptions.ProtocolError,
self.manager.request,
'GET',
'ftp://invalid/url')
# DownloadError -> ProtocolError
self.assertRaises(
- ProtocolError,
+ urllib3.exceptions.ProtocolError,
self.manager.request,
'GET',
'http://0.0.0.0')
# ResponseTooLargeError -> AppEnginePlatformError
self.assertRaises(
- AppEnginePlatformError,
+ appengine.AppEnginePlatformError,
self.pool.request,
'GET',
'/nbytes?length=33554433') # One byte over 32 megabtyes.
@@ -100,7 +80,7 @@ class TestGAEConnectionManager(TestConnectionPool):
# which maps to a AppEnginePlatformError.
body = b'1' * 10485761 # One byte over 10 megabytes.
self.assertRaises(
- AppEnginePlatformError,
+ appengine.AppEnginePlatformError,
self.manager.request,
'POST',
'/',
@@ -139,11 +119,12 @@ class TestGAEConnectionManager(TestConnectionPool):
test_dns_error = None
-class TestGAEConnectionManagerWithSSL(HTTPSDummyServerTestCase):
- nosegae_urlfetch = True
+@pytest.mark.usefixtures('testbed')
+class TestGAEConnectionManagerWithSSL(
+ dummyserver.testcase.HTTPSDummyServerTestCase):
def setUp(self):
- self.manager = AppEngineManager()
+ self.manager = appengine.AppEngineManager()
self.pool = MockPool(self.host, self.port, self.manager, 'https')
def test_exceptions(self):
@@ -151,25 +132,22 @@ class TestGAEConnectionManagerWithSSL(HTTPSDummyServerTestCase):
# SSLError is raised with dummyserver because URLFetch doesn't allow
# self-signed certs.
self.assertRaises(
- SSLError,
+ urllib3.exceptions.SSLError,
self.pool.request,
'GET',
'/')
-class TestGAERetry(TestRetry):
- __test__ = True
-
- # Magic class variable that tells NoseGAE to enable the URLFetch stub.
- nosegae_urlfetch = True
+@pytest.mark.usefixtures('testbed')
+class TestGAERetry(test_connectionpool.TestRetry):
def setUp(self):
- self.manager = AppEngineManager()
+ self.manager = appengine.AppEngineManager()
self.pool = MockPool(self.host, self.port, self.manager)
def test_default_method_whitelist_retried(self):
""" urllib3 should retry methods in the default method whitelist """
- retry = Retry(total=1, status_forcelist=[418])
+ retry = urllib3.util.retry.Retry(total=1, status_forcelist=[418])
# Use HEAD instead of OPTIONS, as URLFetch doesn't support OPTIONS
resp = self.pool.request(
'HEAD', '/successful_retry',
@@ -179,14 +157,16 @@ class TestGAERetry(TestRetry):
def test_retry_return_in_response(self):
headers = {'test-name': 'test_retry_return_in_response'}
- retry = Retry(total=2, status_forcelist=[418])
+ retry = urllib3.util.retry.Retry(total=2, status_forcelist=[418])
resp = self.pool.request('GET', '/successful_retry',
headers=headers, retries=retry)
self.assertEqual(resp.status, 200)
self.assertEqual(resp.retries.total, 1)
# URLFetch use absolute urls.
- self.assertEqual(resp.retries.history,
- (RequestHistory('GET',
+ self.assertEqual(
+ resp.retries.history,
+ (urllib3.util.retry.RequestHistory(
+ 'GET',
self.pool._absolute_url('/successful_retry'),
None, 418, None),))
@@ -197,17 +177,10 @@ class TestGAERetry(TestRetry):
test_multi_redirect_history = None
-class TestGAERetryAfter(TestRetryAfter):
- __test__ = True
-
- # Magic class variable that tells NoseGAE to enable the URLFetch stub.
- nosegae_urlfetch = True
+@pytest.mark.usefixtures('testbed')
+class TestGAERetryAfter(test_connectionpool.TestRetryAfter):
def setUp(self):
# Disable urlfetch which doesn't respect Retry-After header.
- self.manager = AppEngineManager(urlfetch_retries=False)
+ self.manager = appengine.AppEngineManager(urlfetch_retries=False)
self.pool = MockPool(self.host, self.port, self.manager)
-
-
-if __name__ == '__main__':
- unittest.main()

MODIFY
test/appengine/test_urlfetch.py
test/appengine/test_urlfetch.py
-from . import AppEngineSandboxTest, MockResponse
+"""These tests ensure that when running in App Engine standard with the
+App Engine sandbox enabled that urllib3 appropriately uses the App
+Engine-patched version of httplib to make requests."""
+
+import httplib
+import StringIO
+import unittest
from mock import patch
-from nose.plugins.skip import SkipTest
+import pytest
+
from ..test_no_ssl import TestWithoutSSL
-class TestHTTP(AppEngineSandboxTest, TestWithoutSSL):
- nosegae_urlfetch = True
+class MockResponse(object):
+ def __init__(self, content, status_code, content_was_truncated, final_url,
+ headers):
+
+ self.content = content
+ self.status_code = status_code
+ self.content_was_truncated = content_was_truncated
+ self.final_url = final_url
+ self.header_msg = httplib.HTTPMessage(StringIO.StringIO(''.join(
+ ["%s: %s\n" % (k, v) for k, v in headers.iteritems()] + ["\n"])))
+ self.headers = headers
+
+@pytest.mark.usefixtures('sandbox')
+class TestHTTP(TestWithoutSSL):
def test_urlfetch_called_with_http(self):
- """
- Check that URLFetch is used to fetch non-https resources
- """
+ """Check that URLFetch is used to fetch non-https resources."""
resp = MockResponse(
'OK',
200,
False,
'http://www.google.com',
{'content-type': 'text/plain'})
- with patch('google.appengine.api.urlfetch.fetch', return_value=resp) as fetchmock:
+ fetch_patch = patch(
+ 'google.appengine.api.urlfetch.fetch', return_value=resp)
+ with fetch_patch as fetch_mock:
import urllib3
pool = urllib3.HTTPConnectionPool('www.google.com', '80')
r = pool.request('GET', '/')
self.assertEqual(r.status, 200, r.data)
- self.assertEqual(fetchmock.call_count, 1)
-
+ self.assertEqual(fetch_mock.call_count, 1)
-class TestHTTPS(AppEngineSandboxTest):
- nosegae_urlfetch = True
+@pytest.mark.usefixtures('sandbox')
+class TestHTTPS(unittest.TestCase):
+ @pytest.mark.xfail(
+ reason='This is not yet supported by urlfetch, presence of the ssl '
+ 'module will bypass urlfetch.')
def test_urlfetch_called_with_https(self):
"""
Check that URLFetch is used when fetching https resources
"""
- raise SkipTest() # Skipped for now because it fails.
resp = MockResponse(
'OK',
200,
False,
'https://www.google.com',
{'content-type': 'text/plain'})
- with patch('google.appengine.api.urlfetch.fetch', return_value=resp) as fetchmock:
+ fetch_patch = patch(
+ 'google.appengine.api.urlfetch.fetch', return_value=resp)
+ with fetch_patch as fetch_mock:
import urllib3
pool = urllib3.HTTPSConnectionPool('www.google.com', '443')
pool.ConnectionCls = urllib3.connection.UnverifiedHTTPSConnection
r = pool.request('GET', '/')
self.assertEqual(r.status, 200, r.data)
- self.assertEqual(fetchmock.call_count, 1)
+ self.assertEqual(fetch_mock.call_count, 1)

MODIFY
tox.ini
tox.ini
@@ -20,20 +20,10 @@ deps=
basepython = python2.7
deps=
{[testenv]deps}
- -r{toxinidir}/test/appengine/requirements.txt
commands=
- nosetests \
- -c {toxinidir}/test/appengine/nose.cfg \
- test/appengine \
- []
- # For now, this test is ran separately because the sandbox activation in
- # the app engine tests will blow up everything.
- nosetests \
- -c {toxinidir}/test/appengine/nose.cfg \
- test/contrib/test_gae_manager.py \
- []
+ py.test test/appengine {posargs}
setenv =
- PYTHONPATH={env:GAE_PYTHONPATH:}/google_appengine
+ GAE_SDK_PATH={env:GAE_SDK_PATH:}
{[testenv]setenv}
[testenv:flake8-py3]

MODIFY
urllib3/connectionpool.py
urllib3/connectionpool.py
@@ -396,7 +396,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
try:
assert_header_parsing(httplib_response.msg)
- except HeaderParsingError as hpe: # Platform-specific: Python 3
+ except (HeaderParsingError, TypeError) as hpe: # Platform-specific: Python 3
log.warning(
'Failed to parse headers (url=%s): %s',
self._absolute_url(url), hpe, exc_info=True)

Move app engine tests over to pytest