Stop retrying tests with flaky
Experience has shown that tests that are really flaky will fail anyway,
and there's usually a better way to fix those tests. Let's try this as
an experiment.
MODIFY
dev-requirements.txt
dev-requirements.txt
@@ -4,7 +4,6 @@ PySocks==1.7.1
pytest==7.2.0
pytest-timeout==2.1.0
pytest-freezegun==0.4.2
-flaky==3.7.0
trustme==0.9.0
cryptography==39.0.0
backports.zoneinfo==0.2.1;python_version<"3.9"

MODIFY
noxfile.py
noxfile.py
@@ -56,7 +56,6 @@ def tests_impl(
"-ra",
f"--color={'yes' if 'GITHUB_ACTIONS' in os.environ else 'auto'}",
"--tb=native",
- "--no-success-flaky-report",
"--durations=10",
"--strict-config",
"--strict-markers",

MODIFY
test/with_dummyserver/test_chunked_transfer.py
test/with_dummyserver/test_chunked_transfer.py
@@ -13,9 +13,6 @@ from urllib3 import HTTPConnectionPool
from urllib3.util import SKIP_HEADER
from urllib3.util.retry import Retry
-# Retry failed tests
-pytestmark = pytest.mark.flaky
-
class TestChunkedTransfer(SocketDummyServerTestCase):
def start_chunked_handler(self) -> None:

MODIFY
test/with_dummyserver/test_connectionpool.py
test/with_dummyserver/test_connectionpool.py
@@ -35,8 +35,6 @@ from urllib3.util.timeout import _TYPE_TIMEOUT, Timeout
from .. import INVALID_SOURCE_ADDRESSES, TARPIT_HOST, VALID_SOURCE_ADDRESSES
from ..port_helpers import find_unused_port
-pytestmark = pytest.mark.flaky
-
def wait_for_socket(ready_event: Event) -> None:
ready_event.wait()

MODIFY
test/with_dummyserver/test_https.py
test/with_dummyserver/test_https.py
@@ -47,10 +47,6 @@ from urllib3.util.timeout import Timeout
from .. import has_alpn
-# Retry failed tests
-pytestmark = pytest.mark.flaky
-
-
TLSv1_CERTS = DEFAULT_CERTS.copy()
TLSv1_CERTS["ssl_version"] = getattr(ssl, "PROTOCOL_TLSv1", None)

MODIFY
test/with_dummyserver/test_no_ssl.py
test/with_dummyserver/test_no_ssl.py
@@ -5,16 +5,11 @@ Note: Import urllib3 inside the test functions to get the importblocker to work
"""
from __future__ import annotations
-import pytest
-
import urllib3
from dummyserver.testcase import HTTPDummyServerTestCase, HTTPSDummyServerTestCase
from ..test_no_ssl import TestWithoutSSL
-# Retry failed tests
-pytestmark = pytest.mark.flaky
-
class TestHTTPWithoutSSL(HTTPDummyServerTestCase, TestWithoutSSL):
def test_simple(self) -> None:

MODIFY
test/with_dummyserver/test_poolmanager.py
test/with_dummyserver/test_poolmanager.py
@@ -14,9 +14,6 @@ from urllib3.exceptions import MaxRetryError, URLSchemeUnknown
from urllib3.poolmanager import PoolManager
from urllib3.util.retry import Retry
-# Retry failed tests
-pytestmark = pytest.mark.flaky
-
class TestPoolManager(HTTPDummyServerTestCase):
@classmethod

MODIFY
test/with_dummyserver/test_proxy_poolmanager.py
test/with_dummyserver/test_proxy_poolmanager.py
@@ -37,9 +37,6 @@ from urllib3.util.timeout import Timeout
from .. import TARPIT_HOST, requires_network
-# Retry failed tests
-pytestmark = pytest.mark.flaky
-
class TestHTTPProxyManager(HTTPDummyProxyTestCase):
@classmethod

MODIFY
test/with_dummyserver/test_socketlevel.py
test/with_dummyserver/test_socketlevel.py
@@ -60,9 +60,6 @@ if typing.TYPE_CHECKING:
else:
StrOrBytesPath = object
-# Retry failed tests
-pytestmark = pytest.mark.flaky
-
class TestCookies(SocketDummyServerTestCase):
def test_multi_setcookie(self) -> None:

Stop retrying tests with flaky

Experience has shown that tests that are really flaky will fail anyway,
and there's usually a better way to fix those tests. Let's try this as
an experiment.

Co-authored-by: Quentin Pradet <quentin.pradet@gmail.com>