<NME> loadtests.py
<BEF> 
import os
import random
import hashlib

import fxa.errors
from fxa._utils import APIClient
from fxa.core import Client
from fxa.tests.utils import TestEmailAccount
from loads import TestCase

# Parameters to affect the proportion of various reqs in the loadtest.

ACCOUNT_CREATE_PERCENT = 20  # percent of runs that should create a new account
ACCOUNT_DELETE_PERCENT = 10  # percent of runs that should delete the account
SIGN_REQS_MIN = 10   # range for number of key-sign requests per run
SIGN_REQS_MAX = 100

# Error constants used by the fxa-auth-server API.

ERROR_ACCOUNT_EXISTS = 101
ERROR_UNKNOWN_ACCOUNT = 102


# The tests need a public key for the server to sign, but we don't actually
# The tests need a public key for the server to sign, but we don't actually
# do anything with it.  It suffices to use a fixed dummy key throughout.

DUMMY_PUBLIC_KEY = {
    'algorithm': 'RS',
    'n': '475938596723561050357149433919674961454460669256778579'
         '095393476820271428065297309134131686299358278907987200'
         '7974809511698859885077002492642203267408776123',
    'e': '65537',
}


def uniq(size=10):
    """Generate a short random hex string."""
    return os.urandom(size // 2 + 1).encode('hex')[:size]



    server_url = 'https://api-accounts.stage.mozaws.net'


    def setUp(self):
        super(LoadTest, self).setUp()
        self.client = Client(APIClient(self.server_url, session=self.session))

    def test_auth_server(self):
        # Authenticate as a new or existing user.
        if random.randint(0, 100) < ACCOUNT_CREATE_PERCENT:
            session = self._authenticate_as_new_user()
        else:
            session = self._authenticate_as_existing_user()
        # Fetch keys and make some number of signing requests.
        session.fetch_keys()
        session.check_session_status()
        session.get_random_bytes()
        num_sign_reqs = random.randint(SIGN_REQS_MIN, SIGN_REQS_MAX)
        for i in xrange(num_sign_reqs):
            session.sign_certificate(DUMMY_PUBLIC_KEY)
        # Teardown the session, and maybe the whole account.
        session.destroy_session()
        if random.randint(0, 100) < ACCOUNT_DELETE_PERCENT:
            self.client.destroy_account(
                email=session.email,
                stretchpwd=self._get_stretchpwd(session.email),
            )

    def _get_stretchpwd(self, email):
        return hashlib.sha256(email).hexdigest()

    def _authenticate_as_new_user(self):
        # Authenticate as a brand-new user account.
        # Assume it doesn't exist, try to create the account.
        # But it's not big deal if it happens to already exist.
        email = "loads-fxa-%s-new@restmail.lcip.org" % (uniq(),)
        kwds = {
            "email": email,
            "stretchpwd": self._get_stretchpwd(email),
        return TestEmailAccount(email)
            "preVerified": True,
        }
        try:
            return self.client.create_account(**kwds)
        except fxa.errors.ClientError, e:
            if e.errno != ERROR_ACCOUNT_EXISTS:
                raise
            kwds.pop("preVerified")
            return self.client.login(**kwds)

    def _authenticate_as_existing_user(self):
        # Authenticate as an existing user account.
        # We select from a small pool of known accounts, creating it
        # if it does not exist.  This should mean that all the accounts
        # are created quickly at the start of the loadtest run.
        email = "loads-fxa-%s-old@restmail.lcip.org" % (random.randint(1, 999),)
        kwds = {
            "email": email,
            "stretchpwd": self._get_stretchpwd(email),
        session.emailAcct = emailAcct
        }
        try:
            return self.client.login(**kwds)
        except fxa.errors.ClientError, e:
            if e.errno != ERROR_UNKNOWN_ACCOUNT:
                raise
            kwds["preVerified"] = True
            # Account creation might likewise fail due to a race.
            try:
                return self.client.create_account(**kwds)
            except fxa.errors.ClientError, e:
                if e.errno != ERROR_ACCOUNT_EXISTS:
                    raise
                # Assume a normal login will now succeed.
                kwds.pop("preVerified")
                return self.client.login(**kwds)
            try:
                pft.get_status()
            except fxa.errors.ClientError as e:
                if e.errno != ERROR_INVALID_TOKEN:
                    raise
            else:
                assert False, "password forgot token should have been consumed"

    def test_password_change_flow(self):
        email = self._get_existing_user_email_acct().email
        stretchpwd = self._get_stretchpwd(email)
        try:
            self.client.change_password(
                email,
                oldstretchpwd=stretchpwd,
                newstretchpwd=stretchpwd,
            )
        except fxa.errors.ClientError as e:
            if e.errno != ERROR_UNKNOWN_ACCOUNT:
                raise
            # Create the "existing" account if it doesn't yet exist.
            kwds = {
                "email": email,
                "stretchpwd": stretchpwd,
                "preVerified": True,
            }
            try:
                self.client.create_account(**kwds)
            except fxa.errors.ClientError as e:
                if e.errno != ERROR_ACCOUNT_EXISTS:
                    raise
            else:
                self.client.change_password(
                    email,
                    oldstretchpwd=stretchpwd,
                    newstretchpwd=stretchpwd,
                )

    def test_support_doc_flow(self):
        base_url = self.server_url[:-3]
        self.session.get(base_url + "/.well-known/browserid")

    def _get_code_from_email(self, emailAcct, header):
        # Due to possible race conditions, and previous test runs
        # leaving junk in the restmail account, we look for the last
        # matching email rather than the first.  There's still a small
        # chance that we pull out the wrong code here.
        num_retries = 10
        for _ in xrange(num_retries):
            emailAcct.fetch()
            for m in reversed(emailAcct.messages):
                if header in m["headers"]:
                    return m["headers"][header]
        return None

<MSG> Merge pull request #1040 from mozilla/rfk/make-loadtests-match-prod-traffic

Update loadtest to be more similar to production traffic breakdown - redux
<DFF> @@ -10,16 +10,55 @@ from fxa.core import Client
 from loads import TestCase
 
 # Parameters to affect the proportion of various reqs in the loadtest.
+#
+# This is loosely modelled on production traffic analysis performed in:
+#
+#    https://bugzilla.mozilla.org/show_bug.cgi?id=1097584
+#
+# Based on the above we aim for the following breakdown:
+#
+#    * 95% of tests are a login flow that does various session operations:
+#        * 70% of these use an existing account
+#        * 30% of these create a new account
+#            * 10% of those call the resend_code endpoint
+#            * 80% of those call the verify_code endpoint
+#            * 2% of those will delete the account when they're done
+#        * each flow does an average of 500 status poll operations
+#          (this is by far the most frequent request in current prod traffic)
+#        * each flow fetches the keys exactly once
+#        * each flow does an average of 50 cert sign requests
+#        * 20% of flows will explicitly check the session status
+#        * 10% of flows will explicitly tear down the session once complete
+#        * 5% of flows will generate some random bytes
+#    * 3% of tests exercise the password reset flow
+#    * 1% of tests exercise are the password change flow
+#    * 1% of are simple requests for the browserid support document
 
-ACCOUNT_CREATE_PERCENT = 20  # percent of runs that should create a new account
-ACCOUNT_DELETE_PERCENT = 10  # percent of runs that should delete the account
-SIGN_REQS_MIN = 10   # range for number of key-sign requests per run
-SIGN_REQS_MAX = 100
+PERCENT_TEST_LOGIN = 95
+PERCENT_TEST_RESET = 3
+PERCENT_TEST_CHANGE = 1
+PERCENT_TEST_SUPPORTDOC = 1
+
+PERCENT_LOGIN_CREATE = 30
+PERCENT_LOGIN_CREATE_RESEND = 10
+PERCENT_LOGIN_CREATE_VERIFY = 80
+PERCENT_LOGIN_CREATE_DESTROY = 2
+PERCENT_LOGIN_STATUS = 20
+PERCENT_LOGIN_TEARDOWN = 10
+PERCENT_LOGIN_RANDBYTES = 5
+
+LOGIN_POLL_REQS_MIN = 10
+LOGIN_POLL_REQS_MAX = 1000
+
+LOGIN_SIGN_REQS_MIN = 10
+LOGIN_SIGN_REQS_MAX = 90
 
 # Error constants used by the fxa-auth-server API.
 
 ERROR_ACCOUNT_EXISTS = 101
 ERROR_UNKNOWN_ACCOUNT = 102
+ERROR_INVALID_CODE = 105
+ERROR_INVALID_TOKEN = 110
 
 
 # The tests need a public key for the server to sign, but we don't actually
@@ -43,40 +82,94 @@ class LoadTest(TestCase):
 
     server_url = 'https://api-accounts.stage.mozaws.net'
 
-
     def setUp(self):
         super(LoadTest, self).setUp()
         self.client = Client(APIClient(self.server_url, session=self.session))
 
+    def _pick(self, *choices):
+        """Pick one from a list of (item, weighting) options."""
+        sum_weights = sum(choice[1] for choice in choices)
+        remainder = random.randint(0, sum_weights - 1)
+        for choice, weight in choices:
+            remainder -= weight
+            if remainder < 0:
+                return choice
+        assert False, "somehow failed to pick from {}".format(choices)
+
+    def _perc(self, percent):
+        """Decide whether to do something, given desired percentage of runs."""
+        return random.randint(0, 99) < percent
+
     def test_auth_server(self):
-        # Authenticate as a new or existing user.
-        if random.randint(0, 100) < ACCOUNT_CREATE_PERCENT:
+        """Top-level method to run a randomly-secleted auth-server test."""
+        which_test = self._pick(
+            (self.test_login_session_flow, PERCENT_TEST_LOGIN),
+            (self.test_password_reset_flow, PERCENT_TEST_RESET),
+            (self.test_password_change_flow, PERCENT_TEST_CHANGE),
+            (self.test_support_doc_flow, PERCENT_TEST_SUPPORTDOC),
+        )
+        which_test()
+
+    def test_login_session_flow(self):
+        """Do a full login-flow with cert signing etc."""
+        # Login as either a new or existing user.
+        if self._perc(PERCENT_LOGIN_CREATE):
             session = self._authenticate_as_new_user()
+            can_delete = True
         else:
             session = self._authenticate_as_existing_user()
-        # Fetch keys and make some number of signing requests.
-        session.fetch_keys()
-        session.check_session_status()
-        session.get_random_bytes()
-        num_sign_reqs = random.randint(SIGN_REQS_MIN, SIGN_REQS_MAX)
-        for i in xrange(num_sign_reqs):
-            session.sign_certificate(DUMMY_PUBLIC_KEY)
-        # Teardown the session, and maybe the whole account.
-        session.destroy_session()
-        if random.randint(0, 100) < ACCOUNT_DELETE_PERCENT:
-            self.client.destroy_account(
-                email=session.email,
-                stretchpwd=self._get_stretchpwd(session.email),
-            )
+            can_delete = False
+        try:
+            # Do a whole lot of account status polling.
+            n_poll_reqs = random.randint(LOGIN_POLL_REQS_MIN,
+                                         LOGIN_POLL_REQS_MAX)
+            for i in xrange(n_poll_reqs):
+                session.get_email_status()
+            # Always fetch the keys.
+            session.fetch_keys()
+            # Sometimes check the session status.
+            if self._perc(PERCENT_LOGIN_STATUS):
+                session.check_session_status()
+            # Sometimes get some random bytes.
+            if self._perc(PERCENT_LOGIN_RANDBYTES):
+                session.get_random_bytes()
+            # Always do some number of signing requests.
+            n_sign_reqs = random.randint(LOGIN_SIGN_REQS_MIN,
+                                         LOGIN_SIGN_REQS_MAX)
+            for i in xrange(n_sign_reqs):
+                session.sign_certificate(DUMMY_PUBLIC_KEY)
+            # Sometimes tear down the session.
+            if self._perc(PERCENT_LOGIN_TEARDOWN):
+                session.destroy_session()
+        except fxa.errors.ClientError as e:
+            # There's a small chance this could fail due to concurrent
+            # password change destroying the session token.
+            if e.errno != ERROR_INVALID_TOKEN:
+                raise
+        # Sometimes destroy the account.
+        if can_delete:
+            if self._perc(PERCENT_LOGIN_CREATE_DESTROY):
+                self.client.destroy_account(
+                    email=session.email,
+                    stretchpwd=self._get_stretchpwd(session.email),
+                )
 
     def _get_stretchpwd(self, email):
         return hashlib.sha256(email).hexdigest()
 
+    def _get_new_user_email(self):
+        uid = uniq()
+        return "loads-fxa-{}-new@restmail.lcip.org".format(uid)
+
+    def _get_existing_user_email(self):
+        uid = random.randint(1, 999)
+        return "loads-fxa-{}-old@restmail.lcip.org".format(uid)
+
     def _authenticate_as_new_user(self):
         # Authenticate as a brand-new user account.
-        # Assume it doesn't exist, try to create the account.
+        # Assume it doesn't exist, and try to create the account.
         # But it's not big deal if it happens to already exist.
-        email = "loads-fxa-%s-new@restmail.lcip.org" % (uniq(),)
+        email = self._get_new_user_email()
         kwds = {
             "email": email,
             "stretchpwd": self._get_stretchpwd(email),
@@ -84,19 +177,30 @@ class LoadTest(TestCase):
             "preVerified": True,
         }
         try:
-            return self.client.create_account(**kwds)
-        except fxa.errors.ClientError, e:
+            session = self.client.create_account(**kwds)
+        except fxa.errors.ClientError as e:
             if e.errno != ERROR_ACCOUNT_EXISTS:
                 raise
             kwds.pop("preVerified")
-            return self.client.login(**kwds)
+            session = self.client.login(**kwds)
+        # Sometimes resend the confirmation email.
+        if self._perc(PERCENT_LOGIN_CREATE_RESEND):
+            session.resend_email_code()
+        # Sometimes (pretend to) verify the confirmation code.
+        if self._perc(PERCENT_LOGIN_CREATE_VERIFY):
+            try:
+                session.verify_email_code(uniq(32))
+            except fxa.errors.ClientError as e:
+                if e.errno != ERROR_INVALID_CODE:
+                    raise
+        return session
 
     def _authenticate_as_existing_user(self):
         # Authenticate as an existing user account.
         # We select from a small pool of known accounts, creating it
         # if it does not exist.  This should mean that all the accounts
         # are created quickly at the start of the loadtest run.
-        email = "loads-fxa-%s-old@restmail.lcip.org" % (random.randint(1, 999),)
+        email = self._get_existing_user_email()
         kwds = {
             "email": email,
             "stretchpwd": self._get_stretchpwd(email),
@@ -104,16 +208,63 @@ class LoadTest(TestCase):
         }
         try:
             return self.client.login(**kwds)
-        except fxa.errors.ClientError, e:
+        except fxa.errors.ClientError as e:
             if e.errno != ERROR_UNKNOWN_ACCOUNT:
                 raise
             kwds["preVerified"] = True
             # Account creation might likewise fail due to a race.
             try:
                 return self.client.create_account(**kwds)
-            except fxa.errors.ClientError, e:
+            except fxa.errors.ClientError as e:
                 if e.errno != ERROR_ACCOUNT_EXISTS:
                     raise
                 # Assume a normal login will now succeed.
                 kwds.pop("preVerified")
                 return self.client.login(**kwds)
+
+    def test_password_reset_flow(self):
+        email = self._get_existing_user_email()
+        pft = self.client.send_reset_code(email)
+        # XXX TODO: how to get the reset code?
+        # I don't want to actually poll restmail during a loadtest...
+        pft.get_status()
+        try:
+            pft.verify_code("0" * 32)
+        except fxa.errors.ClientError as e:
+            if e.errno != ERROR_INVALID_CODE:
+                raise
+        pft.get_status()
+
+    def test_password_change_flow(self):
+        email = self._get_existing_user_email()
+        stretchpwd = self._get_stretchpwd(email)
+        try:
+            self.client.change_password(
+                email,
+                oldstretchpwd=stretchpwd,
+                newstretchpwd=stretchpwd,
+            )
+        except fxa.errors.ClientError as e:
+            if e.errno != ERROR_UNKNOWN_ACCOUNT:
+                raise
+            # Create the "existing" account if it doens't yet exist.
+            kwds = {
+                "email": email,
+                "stretchpwd": stretchpwd,
+                "preVerified": True,
+            }
+            try:
+                self.client.create_account(**kwds)
+            except fxa.errors.ClientError as e:
+                if e.errno != ERROR_UNKNOWN_ACCOUNT:
+                    raise
+            else:
+                self.client.change_password(
+                    email,
+                    oldstretchpwd=stretchpwd,
+                    newstretchpwd=stretchpwd,
+                )
+
+    def test_support_doc_flow(self):
+        base_url = self.server_url[:-3]
+        self.session.get(base_url + "/.well-known/browserid")

{'commit': 'diff --git a/test/load/loadtests.py b/test/load/loadtests.py\nindex ba6917e3..0e2f7021 100644\n--- a/test/load/loadtests.py\n+++ b/test/load/loadtests.py\n@@ -10,16 +10,55 @@ from fxa.core import Client\n from loads import TestCase\n \n # Parameters to affect the proportion of various reqs in the loadtest.\n+#\n+# This is loosely modelled on production traffic analysis performed in:\n+#\n+#    https://bugzilla.mozilla.org/show_bug.cgi?id=1097584\n+#\n+# Based on the above we aim for the following breakdown:\n+#\n+#    * 95% of tests are a login flow that does various session operations:\n+#        * 70% of these use an existing account\n+#        * 30% of these create a new account\n+#            * 10% of those call the resend_code endpoint\n+#            * 80% of those call the verify_code endpoint\n+#            * 2% of those will delete the account when they\'re done\n+#        * each flow does an average of 500 status poll operations\n+#          (this is by far the most frequent request in current prod traffic)\n+#        * each flow fetches the keys exactly once\n+#        * each flow does an average of 50 cert sign requests\n+#        * 20% of flows will explicitly check the session status\n+#        * 10% of flows will explicitly tear down the session once complete\n+#        * 5% of flows will generate some random bytes\n+#    * 3% of tests exercise the password reset flow\n+#    * 1% of tests exercise are the password change flow\n+#    * 1% of are simple requests for the browserid support document\n \n-ACCOUNT_CREATE_PERCENT = 20  # percent of runs that should create a new account\n-ACCOUNT_DELETE_PERCENT = 10  # percent of runs that should delete the account\n-SIGN_REQS_MIN = 10   # range for number of key-sign requests per run\n-SIGN_REQS_MAX = 100\n+PERCENT_TEST_LOGIN = 95\n+PERCENT_TEST_RESET = 3\n+PERCENT_TEST_CHANGE = 1\n+PERCENT_TEST_SUPPORTDOC = 1\n+\n+PERCENT_LOGIN_CREATE = 30\n+PERCENT_LOGIN_CREATE_RESEND = 10\n+PERCENT_LOGIN_CREATE_VERIFY = 80\n+PERCENT_LOGIN_CREATE_DESTROY = 2\n+PERCENT_LOGIN_STATUS = 20\n+PERCENT_LOGIN_TEARDOWN = 10\n+PERCENT_LOGIN_RANDBYTES = 5\n+\n+LOGIN_POLL_REQS_MIN = 10\n+LOGIN_POLL_REQS_MAX = 1000\n+\n+LOGIN_SIGN_REQS_MIN = 10\n+LOGIN_SIGN_REQS_MAX = 90\n \n # Error constants used by the fxa-auth-server API.\n \n ERROR_ACCOUNT_EXISTS = 101\n ERROR_UNKNOWN_ACCOUNT = 102\n+ERROR_INVALID_CODE = 105\n+ERROR_INVALID_TOKEN = 110\n \n \n # The tests need a public key for the server to sign, but we don\'t actually\n@@ -43,40 +82,94 @@ class LoadTest(TestCase):\n \n     server_url = \'https://api-accounts.stage.mozaws.net\'\n \n-\n     def setUp(self):\n         super(LoadTest, self).setUp()\n         self.client = Client(APIClient(self.server_url, session=self.session))\n \n+    def _pick(self, *choices):\n+        """Pick one from a list of (item, weighting) options."""\n+        sum_weights = sum(choice[1] for choice in choices)\n+        remainder = random.randint(0, sum_weights - 1)\n+        for choice, weight in choices:\n+            remainder -= weight\n+            if remainder < 0:\n+                return choice\n+        assert False, "somehow failed to pick from {}".format(choices)\n+\n+    def _perc(self, percent):\n+        """Decide whether to do something, given desired percentage of runs."""\n+        return random.randint(0, 99) < percent\n+\n     def test_auth_server(self):\n-        # Authenticate as a new or existing user.\n-        if random.randint(0, 100) < ACCOUNT_CREATE_PERCENT:\n+        """Top-level method to run a randomly-secleted auth-server test."""\n+        which_test = self._pick(\n+            (self.test_login_session_flow, PERCENT_TEST_LOGIN),\n+            (self.test_password_reset_flow, PERCENT_TEST_RESET),\n+            (self.test_password_change_flow, PERCENT_TEST_CHANGE),\n+            (self.test_support_doc_flow, PERCENT_TEST_SUPPORTDOC),\n+        )\n+        which_test()\n+\n+    def test_login_session_flow(self):\n+        """Do a full login-flow with cert signing etc."""\n+        # Login as either a new or existing user.\n+        if self._perc(PERCENT_LOGIN_CREATE):\n             session = self._authenticate_as_new_user()\n+            can_delete = True\n         else:\n             session = self._authenticate_as_existing_user()\n-        # Fetch keys and make some number of signing requests.\n-        session.fetch_keys()\n-        session.check_session_status()\n-        session.get_random_bytes()\n-        num_sign_reqs = random.randint(SIGN_REQS_MIN, SIGN_REQS_MAX)\n-        for i in xrange(num_sign_reqs):\n-            session.sign_certificate(DUMMY_PUBLIC_KEY)\n-        # Teardown the session, and maybe the whole account.\n-        session.destroy_session()\n-        if random.randint(0, 100) < ACCOUNT_DELETE_PERCENT:\n-            self.client.destroy_account(\n-                email=session.email,\n-                stretchpwd=self._get_stretchpwd(session.email),\n-            )\n+            can_delete = False\n+        try:\n+            # Do a whole lot of account status polling.\n+            n_poll_reqs = random.randint(LOGIN_POLL_REQS_MIN,\n+                                         LOGIN_POLL_REQS_MAX)\n+            for i in xrange(n_poll_reqs):\n+                session.get_email_status()\n+            # Always fetch the keys.\n+            session.fetch_keys()\n+            # Sometimes check the session status.\n+            if self._perc(PERCENT_LOGIN_STATUS):\n+                session.check_session_status()\n+            # Sometimes get some random bytes.\n+            if self._perc(PERCENT_LOGIN_RANDBYTES):\n+                session.get_random_bytes()\n+            # Always do some number of signing requests.\n+            n_sign_reqs = random.randint(LOGIN_SIGN_REQS_MIN,\n+                                         LOGIN_SIGN_REQS_MAX)\n+            for i in xrange(n_sign_reqs):\n+                session.sign_certificate(DUMMY_PUBLIC_KEY)\n+            # Sometimes tear down the session.\n+            if self._perc(PERCENT_LOGIN_TEARDOWN):\n+                session.destroy_session()\n+        except fxa.errors.ClientError as e:\n+            # There\'s a small chance this could fail due to concurrent\n+            # password change destroying the session token.\n+            if e.errno != ERROR_INVALID_TOKEN:\n+                raise\n+        # Sometimes destroy the account.\n+        if can_delete:\n+            if self._perc(PERCENT_LOGIN_CREATE_DESTROY):\n+                self.client.destroy_account(\n+                    email=session.email,\n+                    stretchpwd=self._get_stretchpwd(session.email),\n+                )\n \n     def _get_stretchpwd(self, email):\n         return hashlib.sha256(email).hexdigest()\n \n+    def _get_new_user_email(self):\n+        uid = uniq()\n+        return "loads-fxa-{}-new@restmail.lcip.org".format(uid)\n+\n+    def _get_existing_user_email(self):\n+        uid = random.randint(1, 999)\n+        return "loads-fxa-{}-old@restmail.lcip.org".format(uid)\n+\n     def _authenticate_as_new_user(self):\n         # Authenticate as a brand-new user account.\n-        # Assume it doesn\'t exist, try to create the account.\n+        # Assume it doesn\'t exist, and try to create the account.\n         # But it\'s not big deal if it happens to already exist.\n-        email = "loads-fxa-%s-new@restmail.lcip.org" % (uniq(),)\n+        email = self._get_new_user_email()\n         kwds = {\n             "email": email,\n             "stretchpwd": self._get_stretchpwd(email),\n@@ -84,19 +177,30 @@ class LoadTest(TestCase):\n             "preVerified": True,\n         }\n         try:\n-            return self.client.create_account(**kwds)\n-        except fxa.errors.ClientError, e:\n+            session = self.client.create_account(**kwds)\n+        except fxa.errors.ClientError as e:\n             if e.errno != ERROR_ACCOUNT_EXISTS:\n                 raise\n             kwds.pop("preVerified")\n-            return self.client.login(**kwds)\n+            session = self.client.login(**kwds)\n+        # Sometimes resend the confirmation email.\n+        if self._perc(PERCENT_LOGIN_CREATE_RESEND):\n+            session.resend_email_code()\n+        # Sometimes (pretend to) verify the confirmation code.\n+        if self._perc(PERCENT_LOGIN_CREATE_VERIFY):\n+            try:\n+                session.verify_email_code(uniq(32))\n+            except fxa.errors.ClientError as e:\n+                if e.errno != ERROR_INVALID_CODE:\n+                    raise\n+        return session\n \n     def _authenticate_as_existing_user(self):\n         # Authenticate as an existing user account.\n         # We select from a small pool of known accounts, creating it\n         # if it does not exist.  This should mean that all the accounts\n         # are created quickly at the start of the loadtest run.\n-        email = "loads-fxa-%s-old@restmail.lcip.org" % (random.randint(1, 999),)\n+        email = self._get_existing_user_email()\n         kwds = {\n             "email": email,\n             "stretchpwd": self._get_stretchpwd(email),\n@@ -104,16 +208,63 @@ class LoadTest(TestCase):\n         }\n         try:\n             return self.client.login(**kwds)\n-        except fxa.errors.ClientError, e:\n+        except fxa.errors.ClientError as e:\n             if e.errno != ERROR_UNKNOWN_ACCOUNT:\n                 raise\n             kwds["preVerified"] = True\n             # Account creation might likewise fail due to a race.\n             try:\n                 return self.client.create_account(**kwds)\n-            except fxa.errors.ClientError, e:\n+            except fxa.errors.ClientError as e:\n                 if e.errno != ERROR_ACCOUNT_EXISTS:\n                     raise\n                 # Assume a normal login will now succeed.\n                 kwds.pop("preVerified")\n                 return self.client.login(**kwds)\n+\n+    def test_password_reset_flow(self):\n+        email = self._get_existing_user_email()\n+        pft = self.client.send_reset_code(email)\n+        # XXX TODO: how to get the reset code?\n+        # I don\'t want to actually poll restmail during a loadtest...\n+        pft.get_status()\n+        try:\n+            pft.verify_code("0" * 32)\n+        except fxa.errors.ClientError as e:\n+            if e.errno != ERROR_INVALID_CODE:\n+                raise\n+        pft.get_status()\n+\n+    def test_password_change_flow(self):\n+        email = self._get_existing_user_email()\n+        stretchpwd = self._get_stretchpwd(email)\n+        try:\n+            self.client.change_password(\n+                email,\n+                oldstretchpwd=stretchpwd,\n+                newstretchpwd=stretchpwd,\n+            )\n+        except fxa.errors.ClientError as e:\n+            if e.errno != ERROR_UNKNOWN_ACCOUNT:\n+                raise\n+            # Create the "existing" account if it doens\'t yet exist.\n+            kwds = {\n+                "email": email,\n+                "stretchpwd": stretchpwd,\n+                "preVerified": True,\n+            }\n+            try:\n+                self.client.create_account(**kwds)\n+            except fxa.errors.ClientError as e:\n+                if e.errno != ERROR_UNKNOWN_ACCOUNT:\n+                    raise\n+            else:\n+                self.client.change_password(\n+                    email,\n+                    oldstretchpwd=stretchpwd,\n+                    newstretchpwd=stretchpwd,\n+                )\n+\n+    def test_support_doc_flow(self):\n+        base_url = self.server_url[:-3]\n+        self.session.get(base_url + "/.well-known/browserid")\n', 'commit_subject': 'Merge pull request #1040 from mozilla/rfk/make-loadtests-match-prod-traffic', 'repo_name': 'mozilla/fxa-auth-server', 'committer': {'name': 'Ryan Kelly', 'email': 'ea3cd978650417470535f3a4725b6b5042a6ab59@rfk.id.au', 'time_sec': '1441620194', 'tz_offset': '600', 'date': {'seconds': '1441620194'}}, 'commit_tree': 'a707299a74d8d3a563681e3c3322225951bede3e', 'commit_message': 'Merge pull request #1040 from mozilla/rfk/make-loadtests-match-prod-traffic\n\nUpdate loadtest to be more similar to production traffic breakdown - redux', 'language_name': 'Python', 'license': 'mpl-2.0', 'hunks': '@@ -10,16 +10,55 @@ from fxa.core import Client\n from loads import TestCase\n \n # Parameters to affect the proportion of various reqs in the loadtest.\n+#\n+# This is loosely modelled on production traffic analysis performed in:\n+#\n+#    https://bugzilla.mozilla.org/show_bug.cgi?id=1097584\n+#\n+# Based on the above we aim for the following breakdown:\n+#\n+#    * 95% of tests are a login flow that does various session operations:\n+#        * 70% of these use an existing account\n+#        * 30% of these create a new account\n+#            * 10% of those call the resend_code endpoint\n+#            * 80% of those call the verify_code endpoint\n+#            * 2% of those will delete the account when they\'re done\n+#        * each flow does an average of 500 status poll operations\n+#          (this is by far the most frequent request in current prod traffic)\n+#        * each flow fetches the keys exactly once\n+#        * each flow does an average of 50 cert sign requests\n+#        * 20% of flows will explicitly check the session status\n+#        * 10% of flows will explicitly tear down the session once complete\n+#        * 5% of flows will generate some random bytes\n+#    * 3% of tests exercise the password reset flow\n+#    * 1% of tests exercise are the password change flow\n+#    * 1% of are simple requests for the browserid support document\n \n-ACCOUNT_CREATE_PERCENT = 20  # percent of runs that should create a new account\n-ACCOUNT_DELETE_PERCENT = 10  # percent of runs that should delete the account\n-SIGN_REQS_MIN = 10   # range for number of key-sign requests per run\n-SIGN_REQS_MAX = 100\n+PERCENT_TEST_LOGIN = 95\n+PERCENT_TEST_RESET = 3\n+PERCENT_TEST_CHANGE = 1\n+PERCENT_TEST_SUPPORTDOC = 1\n+\n+PERCENT_LOGIN_CREATE = 30\n+PERCENT_LOGIN_CREATE_RESEND = 10\n+PERCENT_LOGIN_CREATE_VERIFY = 80\n+PERCENT_LOGIN_CREATE_DESTROY = 2\n+PERCENT_LOGIN_STATUS = 20\n+PERCENT_LOGIN_TEARDOWN = 10\n+PERCENT_LOGIN_RANDBYTES = 5\n+\n+LOGIN_POLL_REQS_MIN = 10\n+LOGIN_POLL_REQS_MAX = 1000\n+\n+LOGIN_SIGN_REQS_MIN = 10\n+LOGIN_SIGN_REQS_MAX = 90\n \n # Error constants used by the fxa-auth-server API.\n \n ERROR_ACCOUNT_EXISTS = 101\n ERROR_UNKNOWN_ACCOUNT = 102\n+ERROR_INVALID_CODE = 105\n+ERROR_INVALID_TOKEN = 110\n \n \n # The tests need a public key for the server to sign, but we don\'t actually\n@@ -43,40 +82,94 @@ class LoadTest(TestCase):\n \n     server_url = \'https://api-accounts.stage.mozaws.net\'\n \n-\n     def setUp(self):\n         super(LoadTest, self).setUp()\n         self.client = Client(APIClient(self.server_url, session=self.session))\n \n+    def _pick(self, *choices):\n+        """Pick one from a list of (item, weighting) options."""\n+        sum_weights = sum(choice[1] for choice in choices)\n+        remainder = random.randint(0, sum_weights - 1)\n+        for choice, weight in choices:\n+            remainder -= weight\n+            if remainder < 0:\n+                return choice\n+        assert False, "somehow failed to pick from {}".format(choices)\n+\n+    def _perc(self, percent):\n+        """Decide whether to do something, given desired percentage of runs."""\n+        return random.randint(0, 99) < percent\n+\n     def test_auth_server(self):\n-        # Authenticate as a new or existing user.\n-        if random.randint(0, 100) < ACCOUNT_CREATE_PERCENT:\n+        """Top-level method to run a randomly-secleted auth-server test."""\n+        which_test = self._pick(\n+            (self.test_login_session_flow, PERCENT_TEST_LOGIN),\n+            (self.test_password_reset_flow, PERCENT_TEST_RESET),\n+            (self.test_password_change_flow, PERCENT_TEST_CHANGE),\n+            (self.test_support_doc_flow, PERCENT_TEST_SUPPORTDOC),\n+        )\n+        which_test()\n+\n+    def test_login_session_flow(self):\n+        """Do a full login-flow with cert signing etc."""\n+        # Login as either a new or existing user.\n+        if self._perc(PERCENT_LOGIN_CREATE):\n             session = self._authenticate_as_new_user()\n+            can_delete = True\n         else:\n             session = self._authenticate_as_existing_user()\n-        # Fetch keys and make some number of signing requests.\n-        session.fetch_keys()\n-        session.check_session_status()\n-        session.get_random_bytes()\n-        num_sign_reqs = random.randint(SIGN_REQS_MIN, SIGN_REQS_MAX)\n-        for i in xrange(num_sign_reqs):\n-            session.sign_certificate(DUMMY_PUBLIC_KEY)\n-        # Teardown the session, and maybe the whole account.\n-        session.destroy_session()\n-        if random.randint(0, 100) < ACCOUNT_DELETE_PERCENT:\n-            self.client.destroy_account(\n-                email=session.email,\n-                stretchpwd=self._get_stretchpwd(session.email),\n-            )\n+            can_delete = False\n+        try:\n+            # Do a whole lot of account status polling.\n+            n_poll_reqs = random.randint(LOGIN_POLL_REQS_MIN,\n+                                         LOGIN_POLL_REQS_MAX)\n+            for i in xrange(n_poll_reqs):\n+                session.get_email_status()\n+            # Always fetch the keys.\n+            session.fetch_keys()\n+            # Sometimes check the session status.\n+            if self._perc(PERCENT_LOGIN_STATUS):\n+                session.check_session_status()\n+            # Sometimes get some random bytes.\n+            if self._perc(PERCENT_LOGIN_RANDBYTES):\n+                session.get_random_bytes()\n+            # Always do some number of signing requests.\n+            n_sign_reqs = random.randint(LOGIN_SIGN_REQS_MIN,\n+                                         LOGIN_SIGN_REQS_MAX)\n+            for i in xrange(n_sign_reqs):\n+                session.sign_certificate(DUMMY_PUBLIC_KEY)\n+            # Sometimes tear down the session.\n+            if self._perc(PERCENT_LOGIN_TEARDOWN):\n+                session.destroy_session()\n+        except fxa.errors.ClientError as e:\n+            # There\'s a small chance this could fail due to concurrent\n+            # password change destroying the session token.\n+            if e.errno != ERROR_INVALID_TOKEN:\n+                raise\n+        # Sometimes destroy the account.\n+        if can_delete:\n+            if self._perc(PERCENT_LOGIN_CREATE_DESTROY):\n+                self.client.destroy_account(\n+                    email=session.email,\n+                    stretchpwd=self._get_stretchpwd(session.email),\n+                )\n \n     def _get_stretchpwd(self, email):\n         return hashlib.sha256(email).hexdigest()\n \n+    def _get_new_user_email(self):\n+        uid = uniq()\n+        return "loads-fxa-{}-new@restmail.lcip.org".format(uid)\n+\n+    def _get_existing_user_email(self):\n+        uid = random.randint(1, 999)\n+        return "loads-fxa-{}-old@restmail.lcip.org".format(uid)\n+\n     def _authenticate_as_new_user(self):\n         # Authenticate as a brand-new user account.\n-        # Assume it doesn\'t exist, try to create the account.\n+        # Assume it doesn\'t exist, and try to create the account.\n         # But it\'s not big deal if it happens to already exist.\n-        email = "loads-fxa-%s-new@restmail.lcip.org" % (uniq(),)\n+        email = self._get_new_user_email()\n         kwds = {\n             "email": email,\n             "stretchpwd": self._get_stretchpwd(email),\n@@ -84,19 +177,30 @@ class LoadTest(TestCase):\n             "preVerified": True,\n         }\n         try:\n-            return self.client.create_account(**kwds)\n-        except fxa.errors.ClientError, e:\n+            session = self.client.create_account(**kwds)\n+        except fxa.errors.ClientError as e:\n             if e.errno != ERROR_ACCOUNT_EXISTS:\n                 raise\n             kwds.pop("preVerified")\n-            return self.client.login(**kwds)\n+            session = self.client.login(**kwds)\n+        # Sometimes resend the confirmation email.\n+        if self._perc(PERCENT_LOGIN_CREATE_RESEND):\n+            session.resend_email_code()\n+        # Sometimes (pretend to) verify the confirmation code.\n+        if self._perc(PERCENT_LOGIN_CREATE_VERIFY):\n+            try:\n+                session.verify_email_code(uniq(32))\n+            except fxa.errors.ClientError as e:\n+                if e.errno != ERROR_INVALID_CODE:\n+                    raise\n+        return session\n \n     def _authenticate_as_existing_user(self):\n         # Authenticate as an existing user account.\n         # We select from a small pool of known accounts, creating it\n         # if it does not exist.  This should mean that all the accounts\n         # are created quickly at the start of the loadtest run.\n-        email = "loads-fxa-%s-old@restmail.lcip.org" % (random.randint(1, 999),)\n+        email = self._get_existing_user_email()\n         kwds = {\n             "email": email,\n             "stretchpwd": self._get_stretchpwd(email),\n@@ -104,16 +208,63 @@ class LoadTest(TestCase):\n         }\n         try:\n             return self.client.login(**kwds)\n-        except fxa.errors.ClientError, e:\n+        except fxa.errors.ClientError as e:\n             if e.errno != ERROR_UNKNOWN_ACCOUNT:\n                 raise\n             kwds["preVerified"] = True\n             # Account creation might likewise fail due to a race.\n             try:\n                 return self.client.create_account(**kwds)\n-            except fxa.errors.ClientError, e:\n+            except fxa.errors.ClientError as e:\n                 if e.errno != ERROR_ACCOUNT_EXISTS:\n                     raise\n                 # Assume a normal login will now succeed.\n                 kwds.pop("preVerified")\n                 return self.client.login(**kwds)\n+\n+    def test_password_reset_flow(self):\n+        email = self._get_existing_user_email()\n+        pft = self.client.send_reset_code(email)\n+        # XXX TODO: how to get the reset code?\n+        # I don\'t want to actually poll restmail during a loadtest...\n+        pft.get_status()\n+        try:\n+            pft.verify_code("0" * 32)\n+        except fxa.errors.ClientError as e:\n+            if e.errno != ERROR_INVALID_CODE:\n+                raise\n+        pft.get_status()\n+\n+    def test_password_change_flow(self):\n+        email = self._get_existing_user_email()\n+        stretchpwd = self._get_stretchpwd(email)\n+        try:\n+            self.client.change_password(\n+                email,\n+                oldstretchpwd=stretchpwd,\n+                newstretchpwd=stretchpwd,\n+            )\n+        except fxa.errors.ClientError as e:\n+            if e.errno != ERROR_UNKNOWN_ACCOUNT:\n+                raise\n+            # Create the "existing" account if it doens\'t yet exist.\n+            kwds = {\n+                "email": email,\n+                "stretchpwd": stretchpwd,\n+                "preVerified": True,\n+            }\n+            try:\n+                self.client.create_account(**kwds)\n+            except fxa.errors.ClientError as e:\n+                if e.errno != ERROR_UNKNOWN_ACCOUNT:\n+                    raise\n+            else:\n+                self.client.change_password(\n+                    email,\n+                    oldstretchpwd=stretchpwd,\n+                    newstretchpwd=stretchpwd,\n+                )\n+\n+    def test_support_doc_flow(self):\n+        base_url = self.server_url[:-3]\n+        self.session.get(base_url + "/.well-known/browserid")\n', 'addition_count': 180, 'deletion_count': 29, 'src_file': 'a/test/load/loadtests.py', 'tgt_file': 'b/test/load/loadtests.py', 'file_extension': '.py', 'before_file': '\nimport os\nimport random\nimport hashlib\n\nimport fxa.errors\nfrom fxa._utils import APIClient\nfrom fxa.core import Client\nfrom fxa.tests.utils import TestEmailAccount\nfrom loads import TestCase\n\n# Parameters to affect the proportion of various reqs in the loadtest.\n\nACCOUNT_CREATE_PERCENT = 20  # percent of runs that should create a new account\nACCOUNT_DELETE_PERCENT = 10  # percent of runs that should delete the account\nSIGN_REQS_MIN = 10   # range for number of key-sign requests per run\nSIGN_REQS_MAX = 100\n\n# Error constants used by the fxa-auth-server API.\n\nERROR_ACCOUNT_EXISTS = 101\nERROR_UNKNOWN_ACCOUNT = 102\n\n\n# The tests need a public key for the server to sign, but we don\'t actually\n# The tests need a public key for the server to sign, but we don\'t actually\n# do anything with it.  It suffices to use a fixed dummy key throughout.\n\nDUMMY_PUBLIC_KEY = {\n    \'algorithm\': \'RS\',\n    \'n\': \'475938596723561050357149433919674961454460669256778579\'\n         \'095393476820271428065297309134131686299358278907987200\'\n         \'7974809511698859885077002492642203267408776123\',\n    \'e\': \'65537\',\n}\n\n\ndef uniq(size=10):\n    """Generate a short random hex string."""\n    return os.urandom(size // 2 + 1).encode(\'hex\')[:size]\n\n\n\n    server_url = \'https://api-accounts.stage.mozaws.net\'\n\n\n    def setUp(self):\n        super(LoadTest, self).setUp()\n        self.client = Client(APIClient(self.server_url, session=self.session))\n\n    def test_auth_server(self):\n        # Authenticate as a new or existing user.\n        if random.randint(0, 100) < ACCOUNT_CREATE_PERCENT:\n            session = self._authenticate_as_new_user()\n        else:\n            session = self._authenticate_as_existing_user()\n        # Fetch keys and make some number of signing requests.\n        session.fetch_keys()\n        session.check_session_status()\n        session.get_random_bytes()\n        num_sign_reqs = random.randint(SIGN_REQS_MIN, SIGN_REQS_MAX)\n        for i in xrange(num_sign_reqs):\n            session.sign_certificate(DUMMY_PUBLIC_KEY)\n        # Teardown the session, and maybe the whole account.\n        session.destroy_session()\n        if random.randint(0, 100) < ACCOUNT_DELETE_PERCENT:\n            self.client.destroy_account(\n                email=session.email,\n                stretchpwd=self._get_stretchpwd(session.email),\n            )\n\n    def _get_stretchpwd(self, email):\n        return hashlib.sha256(email).hexdigest()\n\n    def _authenticate_as_new_user(self):\n        # Authenticate as a brand-new user account.\n        # Assume it doesn\'t exist, try to create the account.\n        # But it\'s not big deal if it happens to already exist.\n        email = "loads-fxa-%s-new@restmail.lcip.org" % (uniq(),)\n        kwds = {\n            "email": email,\n            "stretchpwd": self._get_stretchpwd(email),\n        return TestEmailAccount(email)\n            "preVerified": True,\n        }\n        try:\n            return self.client.create_account(**kwds)\n        except fxa.errors.ClientError, e:\n            if e.errno != ERROR_ACCOUNT_EXISTS:\n                raise\n            kwds.pop("preVerified")\n            return self.client.login(**kwds)\n\n    def _authenticate_as_existing_user(self):\n        # Authenticate as an existing user account.\n        # We select from a small pool of known accounts, creating it\n        # if it does not exist.  This should mean that all the accounts\n        # are created quickly at the start of the loadtest run.\n        email = "loads-fxa-%s-old@restmail.lcip.org" % (random.randint(1, 999),)\n        kwds = {\n            "email": email,\n            "stretchpwd": self._get_stretchpwd(email),\n        session.emailAcct = emailAcct\n        }\n        try:\n            return self.client.login(**kwds)\n        except fxa.errors.ClientError, e:\n            if e.errno != ERROR_UNKNOWN_ACCOUNT:\n                raise\n            kwds["preVerified"] = True\n            # Account creation might likewise fail due to a race.\n            try:\n                return self.client.create_account(**kwds)\n            except fxa.errors.ClientError, e:\n                if e.errno != ERROR_ACCOUNT_EXISTS:\n                    raise\n                # Assume a normal login will now succeed.\n                kwds.pop("preVerified")\n                return self.client.login(**kwds)\n            try:\n                pft.get_status()\n            except fxa.errors.ClientError as e:\n                if e.errno != ERROR_INVALID_TOKEN:\n                    raise\n            else:\n                assert False, "password forgot token should have been consumed"\n\n    def test_password_change_flow(self):\n        email = self._get_existing_user_email_acct().email\n        stretchpwd = self._get_stretchpwd(email)\n        try:\n            self.client.change_password(\n                email,\n                oldstretchpwd=stretchpwd,\n                newstretchpwd=stretchpwd,\n            )\n        except fxa.errors.ClientError as e:\n            if e.errno != ERROR_UNKNOWN_ACCOUNT:\n                raise\n            # Create the "existing" account if it doesn\'t yet exist.\n            kwds = {\n                "email": email,\n                "stretchpwd": stretchpwd,\n                "preVerified": True,\n            }\n            try:\n                self.client.create_account(**kwds)\n            except fxa.errors.ClientError as e:\n                if e.errno != ERROR_ACCOUNT_EXISTS:\n                    raise\n            else:\n                self.client.change_password(\n                    email,\n                    oldstretchpwd=stretchpwd,\n                    newstretchpwd=stretchpwd,\n                )\n\n    def test_support_doc_flow(self):\n        base_url = self.server_url[:-3]\n        self.session.get(base_url + "/.well-known/browserid")\n\n    def _get_code_from_email(self, emailAcct, header):\n        # Due to possible race conditions, and previous test runs\n        # leaving junk in the restmail account, we look for the last\n        # matching email rather than the first.  There\'s still a small\n        # chance that we pull out the wrong code here.\n        num_retries = 10\n        for _ in xrange(num_retries):\n            emailAcct.fetch()\n            for m in reversed(emailAcct.messages):\n                if header in m["headers"]:\n                    return m["headers"][header]\n        return None\n', 'message': 'Merge pull request #1040 from mozilla/rfk/make-loadtests-match-prod-traffic\n\nUpdate loadtest to be more similar to production traffic breakdown - redux', 'author_name': 'Ryan Kelly', 'lang': 'py'}