text: # Copyright (c) 2013 Mirantis Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #   http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  # implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  import time
  
  import mock
  import six
  import testtools
  
  from stackalytics.processor import record_processor
  from stackalytics.processor import runtime_storage
  from stackalytics.processor import user_processor
  from stackalytics.processor import utils
  
  
  RELEASES = [
      {
          'release_name': 'prehistory',
          'end_date': utils.date_to_timestamp('2011-Apr-21')
      },
      {
          'release_name': 'Diablo',
          'end_date': utils.date_to_timestamp('2011-Sep-08')
      },
      {
          'release_name': 'Zoo',
          'end_date': utils.date_to_timestamp('2035-Sep-08')
      },
  ]
  
  REPOS = [
      {
          "branches": ["master"],
          "module": "stackalytics",
          "project_type": "stackforge",
          "uri": "git://git.openstack.org/stackforge/stackalytics.git"
      }
  ]
  
  
  class TestRecordProcessor(testtools.TestCase):
      def setUp(self):
          super(TestRecordProcessor, self).setUp()
          self.read_json_from_uri_patch = mock.patch(
              'stackalytics.processor.utils.read_json_from_uri')
          self.read_launchpad = self.read_json_from_uri_patch.start()
          self.lp_profile_by_launchpad_id_patch = mock.patch(
              'stackalytics.processor.launchpad_utils.'
              'lp_profile_by_launchpad_id')
          self.lp_profile_by_launchpad_id = (
              self.lp_profile_by_launchpad_id_patch.start())
          self.lp_profile_by_launchpad_id.return_value = None
          self.lp_profile_by_email_patch = mock.patch(
              'stackalytics.processor.launchpad_utils.lp_profile_by_email')
          self.lp_profile_by_email = (
              self.lp_profile_by_email_patch.start())
          self.lp_profile_by_email.return_value = None
  
      def tearDown(self):
          super(TestRecordProcessor, self).tearDown()
          self.read_json_from_uri_patch.stop()
          self.lp_profile_by_launchpad_id_patch.stop()
          self.lp_profile_by_email_patch.stop()
  
      # get_company_by_email
  
      def test_get_company_by_email_mapped(self):
          record_processor_inst = self.make_record_processor(
              companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}]
          )
          email = 'jdoe@ibm.com'
          res = record_processor_inst._get_company_by_email(email)
          self.assertEqual('IBM', res)
  
      def test_get_company_by_email_with_long_suffix_mapped(self):
          record_processor_inst = self.make_record_processor(
              companies=[{'company_name': 'NEC', 'domains': ['nec.co.jp']}]
          )
          email = 'man@mxw.nes.nec.co.jp'
          res = record_processor_inst._get_company_by_email(email)
          self.assertEqual('NEC', res)
  
      def test_get_company_by_email_with_long_suffix_mapped_2(self):
          record_processor_inst = self.make_record_processor(
              companies=[{'company_name': 'NEC',
                          'domains': ['nec.co.jp', 'nec.com']}]
          )
          email = 'man@mxw.nes.nec.com'
          res = record_processor_inst._get_company_by_email(email)
          self.assertEqual('NEC', res)
  
      def test_get_company_by_email_not_mapped(self):
          record_processor_inst = self.make_record_processor()
          email = 'foo@boo.com'
          res = record_processor_inst._get_company_by_email(email)
          self.assertEqual(None, res)
  
      # get_lp_info
  
      def test_get_lp_info_invalid_email(self):
          self.read_launchpad.return_value = None
          record_processor_inst = self.make_record_processor(users=[])
          self.assertEqual((None, None),
                           record_processor_inst._get_lp_info('error.root'))
  
      # commit processing
  
      def test_process_commit_existing_user(self):
          record_processor_inst = self.make_record_processor(
              users=[
                  {
                      'user_id': 'john_doe',
                      'launchpad_id': 'john_doe',
                      'user_name': 'John Doe',
                      'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'],
                      'companies': [
                          {'company_name': '*independent',
                           'end_date': 1234567890},
                          {'company_name': 'NEC',
                           'end_date': 0},
                      ]
                  }
              ])
  
          processed_commit = list(record_processor_inst.process(
              generate_commits(author_email='johndoe@gmail.com',
                               author_name='John Doe')))[0]
  
          expected_commit = {
              'user_id': 'john_doe',
              'author_email': 'johndoe@gmail.com',
              'author_name': 'John Doe',
              'company_name': 'NEC',
          }
  
          self.assertRecordsMatch(expected_commit, processed_commit)
  
      def test_process_commit_existing_user_old_job(self):
          record_processor_inst = self.make_record_processor(
              users=[
                  {
                      'user_id': 'john_doe',
                      'launchpad_id': 'john_doe',
                      'user_name': 'John Doe',
                      'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'],
                      'companies': [
                          {'company_name': '*independent',
                           'end_date': 1234567890},
                          {'company_name': 'NEC',
                           'end_date': 0},
                      ]
                  }
              ])
  
          processed_commit = list(record_processor_inst.process(
              generate_commits(author_email='johndoe@gmail.com',
                               author_name='John Doe',
                               date=1000000000)))[0]
  
          expected_commit = {
              'user_id': 'john_doe',
              'author_email': 'johndoe@gmail.com',
              'author_name': 'John Doe',
              'company_name': '*independent',
          }
  
          self.assertRecordsMatch(expected_commit, processed_commit)
  
      def test_process_commit_existing_user_new_email_known_company(self):
          # User is known to LP, his email is new to us, and maps to other
          # company. Should return other company instead of those mentioned
          # in user profile
          record_processor_inst = self.make_record_processor(
              users=[
                  {'user_id': 'john_doe',
                   'launchpad_id': 'john_doe',
                   'user_name': 'John Doe',
                   'emails': ['johndoe@nec.co.jp'],
                   'companies': [{'company_name': 'NEC', 'end_date': 0}]}
              ],
              companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}],
              lp_info={'johndoe@ibm.com':
                       {'name': 'john_doe', 'display_name': 'John Doe'}})
  
          processed_commit = list(record_processor_inst.process(
              generate_commits(author_email='johndoe@ibm.com',
                               author_name='John Doe')))[0]
  
          expected_commit = {
              'user_id': 'john_doe',
              'author_email': 'johndoe@ibm.com',
              'author_name': 'John Doe',
              'company_name': 'IBM',
          }
  
          self.assertRecordsMatch(expected_commit, processed_commit)
          self.assertIn('johndoe@ibm.com', user_processor.load_user(
              record_processor_inst.runtime_storage_inst,
              user_id='john_doe')['emails'])
  
      def test_process_commit_existing_user_new_email_known_company_static(self):
          # User profile is configured in default_data. Email is new to us,
          # and maps to other company. We still use a company specified
          # in the profile
          record_processor_inst = self.make_record_processor(
              users=[
                  {'user_id': 'john_doe',
                   'launchpad_id': 'john_doe',
                   'user_name': 'John Doe',
                   'static': True,
                   'emails': ['johndoe@nec.co.jp'],
                   'companies': [{'company_name': 'NEC', 'end_date': 0}]}
              ],
              companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}],
              lp_info={'johndoe@ibm.com':
                       {'name': 'john_doe', 'display_name': 'John Doe'}})
  
          processed_commit = list(record_processor_inst.process(
              generate_commits(author_email='johndoe@ibm.com',
                               author_name='John Doe')))[0]
  
          expected_commit = {
              'user_id': 'john_doe',
              'author_email': 'johndoe@ibm.com',
              'author_name': 'John Doe',
              'company_name': 'NEC',
          }
  
          self.assertRecordsMatch(expected_commit, processed_commit)
          self.assertIn('johndoe@ibm.com', user_processor.load_user(
              record_processor_inst.runtime_storage_inst,
              user_id='john_doe')['emails'])
  
      def test_process_commit_existing_user_old_job_not_overridden(self):
          # User is known to LP, his email is new to us, and maps to other
          # company. Have some record with new email, but from the period when
          # he worked for other company. Should return other company as mentioned
          # in profile instead of overriding
          record_processor_inst = self.make_record_processor(
              users=[
                  {'user_id': 'john_doe',
                   'launchpad_id': 'john_doe',
                   'user_name': 'John Doe',
                   'emails': ['johndoe@nec.co.jp'],
                   'companies': [{'company_name': 'IBM', 'end_date': 1200000000},
                                 {'company_name': 'NEC', 'end_date': 0}]}
              ],
              companies=[{'company_name': 'IBM', 'domains': ['ibm.com']},
                         {'company_name': 'NEC', 'domains': ['nec.com']}],
              lp_info={'johndoe@nec.com':
                       {'name': 'john_doe', 'display_name': 'John Doe'}})
  
          processed_commit = list(record_processor_inst.process(
              generate_commits(author_email='johndoe@nec.com',
                               author_name='John Doe',
                               date=1000000000)))[0]
  
          expected_commit = {
              'user_id': 'john_doe',
              'author_email': 'johndoe@nec.com',
              'author_name': 'John Doe',
              'company_name': 'IBM',
          }
  
          self.assertRecordsMatch(expected_commit, processed_commit)
  
      def test_process_commit_existing_user_new_email_unknown_company(self):
          # User is known to LP, but his email is new to us. Should match
          # the user and return company from user profile
          record_processor_inst = self.make_record_processor(
              users=[
                  {'user_id': 'john_doe',
                   'launchpad_id': 'john_doe',
                   'user_name': 'John Doe',
                   'emails': ['johndoe@nec.co.jp'],
                   'companies': [{'company_name': 'NEC', 'end_date': 0}]}
              ],
              companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}],
              lp_info={'johndoe@gmail.com':
                       {'name': 'john_doe', 'display_name': 'John Doe'}})
  
          processed_commit = list(record_processor_inst.process(
              generate_commits(author_email='johndoe@gmail.com',
                               author_name='John Doe')))[0]
  
          expected_commit = {
              'user_id': 'john_doe',
              'author_email': 'johndoe@gmail.com',
              'author_name': 'John Doe',
              'company_name': 'NEC',
          }
  
          self.assertRecordsMatch(expected_commit, processed_commit)
          self.assertIn('johndoe@gmail.com', user_processor.load_user(
              record_processor_inst.runtime_storage_inst,
              user_id='john_doe')['emails'])
  
      def test_process_commit_existing_user_new_email_known_company_update(self):
          record_processor_inst = self.make_record_processor(
              users=[
                  {'user_id': 'john_doe',
                   'launchpad_id': 'john_doe',
                   'user_name': 'John Doe',
                   'emails': ['johndoe@gmail.com'],
                   'companies': [{'company_name': '*independent',
                                  'end_date': 0}]}
              ],
              companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}],
              lp_info={'johndoe@ibm.com':
                       {'name': 'john_doe', 'display_name': 'John Doe'}})
  
          processed_commit = list(record_processor_inst.process(
              generate_commits(author_email='johndoe@ibm.com',
                               author_name='John Doe')))[0]
  
          expected_commit = {
              'user_id': 'john_doe',
              'author_email': 'johndoe@ibm.com',
              'author_name': 'John Doe',
              'company_name': 'IBM',
          }
  
          self.assertRecordsMatch(expected_commit, processed_commit)
          user = user_processor.load_user(
              record_processor_inst.runtime_storage_inst, user_id='john_doe')
          self.assertIn('johndoe@gmail.com', user['emails'])
          self.assertEqual('IBM', user['companies'][0]['company_name'],
                           message='User affiliation should be updated')
  
      def test_process_commit_new_user(self):
          # User is known to LP, but new to us
          # Should add new user and set company depending on email
          record_processor_inst = self.make_record_processor(
              companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}],
              lp_info={'johndoe@ibm.com':
                       {'name': 'john_doe', 'display_name': 'John Doe'}})
  
          processed_commit = list(record_processor_inst.process(
              generate_commits(author_email='johndoe@ibm.com',
                               author_name='John Doe')))[0]
  
          expected_commit = {
              'user_id': 'john_doe',
              'author_email': 'johndoe@ibm.com',
              'author_name': 'John Doe',
              'company_name': 'IBM',
          }
  
          self.assertRecordsMatch(expected_commit, processed_commit)
          user = user_processor.load_user(
              record_processor_inst.runtime_storage_inst, user_id='john_doe')
          self.assertIn('johndoe@ibm.com', user['emails'])
          self.assertEqual('IBM', user['companies'][0]['company_name'])
  
      def test_process_commit_new_user_unknown_to_lb(self):
          # User is new to us and not known to LP
          # Should set user name and empty LPid
          record_processor_inst = self.make_record_processor(
              companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}])
  
          processed_commit = list(record_processor_inst.process(
              generate_commits(author_email='johndoe@ibm.com',
                               author_name='John Doe')))[0]
  
          expected_commit = {
              'launchpad_id': None,
              'author_email': 'johndoe@ibm.com',
              'author_name': 'John Doe',
              'company_name': 'IBM',
          }
  
          self.assertRecordsMatch(expected_commit, processed_commit)
          user = user_processor.load_user(
              record_processor_inst.runtime_storage_inst,
              user_id='johndoe@ibm.com')
          self.assertIn('johndoe@ibm.com', user['emails'])
          self.assertEqual('IBM', user['companies'][0]['company_name'])
          self.assertEqual(None, user['launchpad_id'])
  
      def test_process_review_new_user(self):
          # User is known to LP, but new to us
          # Should add new user and set company depending on email
          record_processor_inst = self.make_record_processor(
              companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}],
              lp_info={'johndoe@ibm.com':
                       {'name': 'john_doe', 'display_name': 'John Doe'}})
  
          processed_review = list(record_processor_inst.process([
              {'record_type': 'review',
               'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
               'subject': 'Fix AttributeError in Keypair._add_details()',
               'owner': {'name': 'John Doe',
                         'email': 'johndoe@ibm.com',
                         'username': 'John_Doe'},
               'createdOn': 1379404951,
               'module': 'nova', 'branch': 'master'}
          ]))[0]
  
          expected_review = {
              'user_id': 'john_doe',
              'author_email': 'johndoe@ibm.com',
              'author_name': 'John Doe',
              'company_name': 'IBM',
          }
  
          self.assertRecordsMatch(expected_review, processed_review)
          user = user_processor.load_user(
              record_processor_inst.runtime_storage_inst, user_id='john_doe')
          self.assertEqual('John_Doe', user['gerrit_id'])
  
      def generate_bugs(self, assignee=None, date_fix_committed=None,
                        status='Confirmed'):
          yield {
              'record_type': 'bug',
              'id': 'bug_id',
              'owner': 'owner',
              'assignee': assignee,
              'date_created': 1234567890,
              'date_fix_committed': date_fix_committed,
              'module': 'nova',
              'status': status
          }
  
      def test_process_bug_not_fixed(self):
          record = self.generate_bugs()
          record_processor_inst = self.make_record_processor()
          bugs = list(record_processor_inst.process(record))
          self.assertEqual(len(bugs), 1)
          self.assertRecordsMatch({
              'primary_key': 'bugf:bug_id',
              'record_type': 'bugf',
              'launchpad_id': 'owner',
              'date': 1234567890,
          }, bugs[0])
  
      def test_process_bug_fix_committed(self):
          record = self.generate_bugs(status='Fix Committed',
                                      date_fix_committed=1234567891,
                                      assignee='assignee')
          record_processor_inst = self.make_record_processor()
          bugs = list(record_processor_inst.process(record))
          self.assertEqual(len(bugs), 2)
          self.assertRecordsMatch({
              'primary_key': 'bugf:bug_id',
              'record_type': 'bugf',
              'launchpad_id': 'owner',
              'date': 1234567890,
          }, bugs[0])
          self.assertRecordsMatch({
              'primary_key': 'bugr:bug_id',
              'record_type': 'bugr',
              'launchpad_id': 'assignee',
              'date': 1234567891,
          }, bugs[1])
  
      def test_process_bug_fix_released(self):
          record = self.generate_bugs(status='Fix Released',
                                      date_fix_committed=1234567891,
                                      assignee='assignee')
          record_processor_inst = self.make_record_processor()
          bugs = list(record_processor_inst.process(record))
          self.assertEqual(len(bugs), 2)
          self.assertRecordsMatch({
              'primary_key': 'bugf:bug_id',
              'record_type': 'bugf',
              'launchpad_id': 'owner',
              'date': 1234567890,
          }, bugs[0])
          self.assertRecordsMatch({
              'primary_key': 'bugr:bug_id',
              'record_type': 'bugr',
              'launchpad_id': 'assignee',
              'date': 1234567891,
          }, bugs[1])
  
      def test_process_bug_fix_committed_without_assignee(self):
          record = self.generate_bugs(status='Fix Committed',
                                      date_fix_committed=1234567891)
          record_processor_inst = self.make_record_processor()
          bugs = list(record_processor_inst.process(record))
          self.assertEqual(len(bugs), 2)
          self.assertRecordsMatch({
              'primary_key': 'bugf:bug_id',
              'record_type': 'bugf',
              'launchpad_id': 'owner',
              'date': 1234567890,
          }, bugs[0])
          self.assertRecordsMatch({
              'primary_key': 'bugr:bug_id',
              'record_type': 'bugr',
              'launchpad_id': '*unassigned',
              'date': 1234567891,
          }, bugs[1])
  
      # process records complex scenarios
  
      def test_process_blueprint_one_draft_spawned_lp_doesnt_know_user(self):
          # In: blueprint record
          #     LP doesn't know user
          # Out: blueprint-draft record
          #      new user profile created
          record_processor_inst = self.make_record_processor()
  
          processed_records = list(record_processor_inst.process([
              {'record_type': 'bp',
               'id': 'mod:blueprint',
               'self_link': 'http://launchpad.net/blueprint',
               'owner': 'john_doe',
               'date_created': 1234567890}
          ]))
  
          self.assertRecordsMatch(
              {'record_type': 'bpd',
               'launchpad_id': 'john_doe',
               'author_name': 'john_doe',
               'company_name': '*independent'},
              processed_records[0])
  
          user = user_processor.load_user(
              record_processor_inst.runtime_storage_inst, user_id='john_doe')
          self.assertEqual({
              'seq': 1,
              'user_id': 'john_doe',
              'launchpad_id': 'john_doe',
              'user_name': 'john_doe',
              'emails': [],
              'companies': [{'company_name': '*independent', 'end_date': 0}]
          }, user)
  
      def test_process_blueprint_one_draft_spawned_lp_knows_user(self):
          # In: blueprint record
          #     LP knows user
          # Out: blueprint-draft record
          #      new user profile created, name is taken from LP profile
          record_processor_inst = self.make_record_processor(
              lp_user_name={
                  'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'}})
  
          processed_records = list(record_processor_inst.process([
              {'record_type': 'bp',
               'id': 'mod:blueprint',
               'self_link': 'http://launchpad.net/blueprint',
               'owner': 'john_doe',
               'date_created': 1234567890}
          ]))
  
          self.assertRecordsMatch(
              {'record_type': 'bpd',
               'launchpad_id': 'john_doe',
               'author_name': 'John Doe',
               'company_name': '*independent'},
              processed_records[0])
  
          user = user_processor.load_user(
              record_processor_inst.runtime_storage_inst, user_id='john_doe')
          self.assertEqual({
              'seq': 1,
              'user_id': 'john_doe',
              'launchpad_id': 'john_doe',
              'user_name': 'John Doe',
              'emails': [],
              'companies': [{'company_name': '*independent', 'end_date': 0}]
          }, user)
  
      def test_process_blueprint_then_review(self):
          record_processor_inst = self.make_record_processor(
              lp_user_name={
                  'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'}})
  
          processed_records = list(record_processor_inst.process([
              {'record_type': 'bp',
               'id': 'mod:blueprint',
               'self_link': 'http://launchpad.net/blueprint',
               'owner': 'john_doe',
               'date_created': 1234567890},
              {'record_type': 'review',
               'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
               'subject': 'Fix AttributeError in Keypair._add_details()',
               'owner': {'name': 'John Doe',
                         'email': 'john_doe@gmail.com',
                         'username': 'john_doe'},
               'createdOn': 1379404951,
               'module': 'nova', 'branch': 'master'}
          ]))
  
          self.assertRecordsMatch(
              {'record_type': 'bpd',
               'user_id': 'john_doe',
               'author_name': 'John Doe',
               'company_name': '*independent'},
              processed_records[0])
  
          self.assertRecordsMatch(
              {'record_type': 'review',
               'user_id': 'john_doe',
               'author_name': 'John Doe',
               'author_email': 'john_doe@gmail.com',
               'company_name': '*independent'},
              processed_records[1])
  
          user = {'seq': 1,
                  'user_id': 'john_doe',
                  'launchpad_id': 'john_doe',
                  'gerrit_id': 'john_doe',
                  'user_name': 'John Doe',
                  'emails': ['john_doe@gmail.com'],
                  'companies': [{'company_name': '*independent', 'end_date': 0}]}
          self.assertEqual(user, user_processor.load_user(
              record_processor_inst.runtime_storage_inst,
              user_id='john_doe'))
          self.assertEqual(user, user_processor.load_user(
              record_processor_inst.runtime_storage_inst,
              email='john_doe@gmail.com'))
  
      def test_process_blueprint_then_commit(self):
          record_processor_inst = self.make_record_processor(
              lp_user_name={
                  'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'}},
              lp_info={'john_doe@gmail.com':
                       {'name': 'john_doe', 'display_name': 'John Doe'}})
  
          processed_records = list(record_processor_inst.process([
              {'record_type': 'bp',
               'id': 'mod:blueprint',
               'self_link': 'http://launchpad.net/blueprint',
               'owner': 'john_doe',
               'date_created': 1234567890},
              {'record_type': 'commit',
               'commit_id': 'de7e8f297c193fb310f22815334a54b9c76a0be1',
               'author_name': 'John Doe',
               'author_email': 'john_doe@gmail.com',
               'date': 1234567890,
               'lines_added': 25,
               'lines_deleted': 9,
               'release_name': 'havana'}
          ]))
  
          self.assertRecordsMatch(
              {'record_type': 'bpd',
               'launchpad_id': 'john_doe',
               'author_name': 'John Doe',
               'company_name': '*independent'},
              processed_records[0])
  
          self.assertRecordsMatch(
              {'record_type': 'commit',
               'user_id': 'john_doe',
               'author_name': 'John Doe',
               'author_email': 'john_doe@gmail.com',
               'company_name': '*independent'},
              processed_records[1])
  
          user = {'seq': 1,
                  'user_id': 'john_doe',
                  'launchpad_id': 'john_doe',
                  'user_name': 'John Doe',
                  'emails': ['john_doe@gmail.com'],
                  'companies': [{'company_name': '*independent', 'end_date': 0}]}
          self.assertEqual(user, user_processor.load_user(
              record_processor_inst.runtime_storage_inst,
              user_id='john_doe'))
          self.assertEqual(user, user_processor.load_user(
              record_processor_inst.runtime_storage_inst,
              email='john_doe@gmail.com'))
  
      def test_process_review_then_blueprint(self):
          record_processor_inst = self.make_record_processor(
              lp_user_name={
                  'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'}})
  
          processed_records = list(record_processor_inst.process([
              {'record_type': 'review',
               'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
               'subject': 'Fix AttributeError in Keypair._add_details()',
               'owner': {'name': 'John Doe',
                         'email': 'john_doe@gmail.com',
                         'username': 'john_doe'},
               'createdOn': 1379404951,
               'module': 'nova', 'branch': 'master'},
              {'record_type': 'bp',
               'id': 'mod:blueprint',
               'self_link': 'http://launchpad.net/blueprint',
               'owner': 'john_doe',
               'date_created': 1234567890}
          ]))
  
          self.assertRecordsMatch(
              {'record_type': 'review',
               'user_id': 'john_doe',
               'author_name': 'John Doe',
               'author_email': 'john_doe@gmail.com',
               'company_name': '*independent'},
              processed_records[0])
  
          self.assertRecordsMatch(
              {'record_type': 'bpd',
               'user_id': 'john_doe',
               'author_name': 'John Doe',
               'company_name': '*independent'},
              processed_records[1])
  
          user = {'seq': 1,
                  'user_id': 'john_doe',
                  'launchpad_id': 'john_doe',
                  'gerrit_id': 'john_doe',
                  'user_name': 'John Doe',
                  'emails': ['john_doe@gmail.com'],
                  'companies': [{'company_name': '*independent', 'end_date': 0}]}
          self.assertEqual(user, user_processor.load_user(
              record_processor_inst.runtime_storage_inst,
              user_id='john_doe'))
          self.assertEqual(user, user_processor.load_user(
              record_processor_inst.runtime_storage_inst,
              email='john_doe@gmail.com'))
  
      def test_create_member(self):
          member_record = {'member_id': '123456789',
                           'member_name': 'John Doe',
                           'member_uri': 'http://www.openstack.org/community'
                                         '/members/profile/123456789',
                           'date_joined': 'August 01, 2012 ',
                           'company_draft': 'Mirantis'}
  
          record_processor_inst = self.make_record_processor()
          result_member = next(record_processor_inst._process_member(
              member_record))
  
          self.assertEqual(result_member['primary_key'], 'member:123456789')
          self.assertEqual(result_member['date'], utils.member_date_to_timestamp(
              'August 01, 2012 '))
          self.assertEqual(result_member['author_name'], 'John Doe')
          self.assertEqual(result_member['company_name'], 'Mirantis')
  
          result_user = user_processor.load_user(
              record_processor_inst.runtime_storage_inst,
              member_id='123456789')
  
          self.assertEqual(result_user['user_name'], 'John Doe')
          self.assertEqual(result_user['company_name'], 'Mirantis')
          self.assertEqual(result_user['companies'],
                           [{'company_name': 'Mirantis', 'end_date': 0}])
  
      def test_update_member(self):
          member_record = {'member_id': '123456789',
                           'member_name': 'John Doe',
                           'member_uri': 'http://www.openstack.org/community'
                                         '/members/profile/123456789',
                           'date_joined': 'August 01, 2012 ',
                           'company_draft': 'Mirantis'}
  
          record_processor_inst = self.make_record_processor()
  
          updated_member_record = member_record
          updated_member_record['member_name'] = 'Bill Smith'
          updated_member_record['company_draft'] = 'Rackspace'
  
          result_member = next(record_processor_inst._process_member(
              updated_member_record))
          self.assertEqual(result_member['author_name'], 'Bill Smith')
          self.assertEqual(result_member['company_name'], 'Rackspace')
  
          result_user = user_processor.load_user(
              record_processor_inst.runtime_storage_inst,
              member_id='123456789')
  
          self.assertEqual(result_user['user_name'], 'Bill Smith')
          self.assertEqual(result_user['companies'],
                           [{'company_name': 'Rackspace', 'end_date': 0}])
  
      def test_process_email_then_review(self):
          # it is expected that the user profile will contain email and
          # gerrit id, while LP id will be None
          record_processor_inst = self.make_record_processor()
  
          list(record_processor_inst.process([
              {'record_type': 'email',
               'message_id': '<message-id>',
               'author_email': 'john_doe@gmail.com',
               'subject': 'hello, world!',
               'body': 'lorem ipsum',
               'date': 1234567890},
              {'record_type': 'review',
               'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
               'subject': 'Fix AttributeError in Keypair._add_details()',
               'owner': {'name': 'John Doe',
                         'email': 'john_doe@gmail.com',
                         'username': 'john_doe'},
               'createdOn': 1379404951,
               'module': 'nova', 'branch': 'master'}
          ]))
  
          user = {'seq': 1,
                  'user_id': 'john_doe@gmail.com',
                  'gerrit_id': 'john_doe',
                  'user_name': 'John Doe',
                  'emails': ['john_doe@gmail.com'],
                  'companies': [{'company_name': '*independent', 'end_date': 0}]}
          self.assertEqual(user, user_processor.load_user(
              record_processor_inst.runtime_storage_inst,
              email='john_doe@gmail.com'))
          self.assertEqual(user, user_processor.load_user(
              record_processor_inst.runtime_storage_inst,
              gerrit_id='john_doe'))
  
      def test_process_email_then_review_gerrit_id_same_as_launchpad_id(self):
          # it is expected that the user profile will contain email, LP id and
          # gerrit id
          record_processor_inst = self.make_record_processor(
              lp_user_name={'john_doe': {'name': 'john_doe',
                                         'display_name': 'John Doe'}}
          )
  
          list(record_processor_inst.process([
              {'record_type': 'email',
               'message_id': '<message-id>',
               'author_email': 'john_doe@gmail.com',
               'subject': 'hello, world!',
               'body': 'lorem ipsum',
               'date': 1234567890},
              {'record_type': 'review',
               'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
               'subject': 'Fix AttributeError in Keypair._add_details()',
               'owner': {'name': 'John Doe',
                         'email': 'john_doe@gmail.com',
                         'username': 'john_doe'},
               'createdOn': 1379404951,
               'module': 'nova', 'branch': 'master'}
          ]))
  
          user = {'seq': 1,
                  'user_id': 'john_doe',
                  'launchpad_id': 'john_doe',
                  'gerrit_id': 'john_doe',
                  'user_name': 'John Doe',
                  'emails': ['john_doe@gmail.com'],
                  'companies': [{'company_name': '*independent', 'end_date': 0}]}
          self.assertEqual(user, user_processor.load_user(
              record_processor_inst.runtime_storage_inst,
              email='john_doe@gmail.com'))
          self.assertEqual(user, user_processor.load_user(
              record_processor_inst.runtime_storage_inst,
              user_id='john_doe'))
          self.assertEqual(user, user_processor.load_user(
              record_processor_inst.runtime_storage_inst,
              gerrit_id='john_doe'))
  
      def test_process_commit_then_review_with_different_email(self):
          record_processor_inst = self.make_record_processor(
              lp_info={'john_doe@gmail.com':
                       {'name': 'john_doe', 'display_name': 'John Doe'}},
              lp_user_name={'john_doe': {'name': 'john_doe',
                                         'display_name': 'John Doe'}},
              companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}])
  
          list(record_processor_inst.process([
              {'record_type': 'commit',
               'commit_id': 'de7e8f297c193fb310f22815334a54b9c76a0be1',
               'author_name': 'John Doe', 'author_email': 'john_doe@gmail.com',
               'date': 1234567890, 'lines_added': 25, 'lines_deleted': 9,
               'release_name': 'havana'},
              {'record_type': 'review',
               'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
               'subject': 'Fix AttributeError in Keypair._add_details()',
               'owner': {'name': 'Bill Smith', 'email': 'bill@smith.to',
                         'username': 'bsmith'},
               'createdOn': 1379404951, 'module': 'nova', 'branch': 'master',
               'patchSets': [
                   {'number': '1',
                    'revision': '4d8984e92910c37b7d101c1ae8c8283a2e6f4a76',
                    'ref': 'refs/changes/16/58516/1',
                    'uploader': {'name': 'Bill Smith', 'email': 'bill@smith.to',
                                 'username': 'bsmith'},
                    'createdOn': 1385470730,
                    'approvals': [
                        {'type': 'Code-Review', 'description': 'Code Review',
                         'value': '1', 'grantedOn': 1385478464,
                         'by': {'name': 'John Doe', 'email': 'john_doe@ibm.com',
                                'username': 'john_doe'}}]}]}
          ]))
          user = {'seq': 1,
                  'user_id': 'john_doe',
                  'launchpad_id': 'john_doe',
                  'user_name': 'John Doe',
                  'emails': ['john_doe@ibm.com', 'john_doe@gmail.com'],
                  'companies': [{'company_name': 'IBM', 'end_date': 0}]}
          self.assertUsersMatch(user, user_processor.load_user(
              record_processor_inst.runtime_storage_inst,
              user_id='john_doe'))
          self.assertUsersMatch(user, user_processor.load_user(
              record_processor_inst.runtime_storage_inst,
              email='john_doe@gmail.com'))
          self.assertUsersMatch(user, user_processor.load_user(
              record_processor_inst.runtime_storage_inst,
              email='john_doe@ibm.com'))
  
      def test_merge_users(self):
          record_processor_inst = self.make_record_processor(
              lp_user_name={
                  'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'}
              },
              companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}],
          )
          runtime_storage_inst = record_processor_inst.runtime_storage_inst
  
          runtime_storage_inst.set_records(record_processor_inst.process([
              {'record_type': 'bp',
               'id': 'mod:blueprint',
               'self_link': 'http://launchpad.net/blueprint',
               'owner': 'john_doe',
               'date_created': 1234567890},
              {'record_type': 'email',
               'message_id': '<message-id>',
               'author_email': 'john_doe@ibm.com', 'author_name': 'John Doe',
               'subject': 'hello, world!',
               'body': 'lorem ipsum',
               'date': 1234567890},
              {'record_type': 'review',
               'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
               'subject': 'Fix AttributeError in Keypair._add_details()',
               'owner': {'name': 'John Doe',
                         'email': 'john_doe@ibm.com',
                         'username': 'john_doe'},
               'createdOn': 1379404951,
               'module': 'nova', 'branch': 'master'}
          ]))
  
          record_processor_inst.post_processing({})
  
          user = {'seq': 2,
                  'user_id': 'john_doe',
                  'launchpad_id': 'john_doe',
                  'gerrit_id': 'john_doe',
                  'user_name': 'John Doe',
                  'emails': ['john_doe@ibm.com'],
                  'companies': [{'company_name': 'IBM', 'end_date': 0}]}
          runtime_storage_inst = record_processor_inst.runtime_storage_inst
          self.assertEqual(2, runtime_storage_inst.get_by_key('user:count'))
          self.assertEqual(None, user_processor.load_user(
              runtime_storage_inst, 1))
          self.assertEqual(user, user_processor.load_user(
              runtime_storage_inst, 2))
          self.assertEqual(user, user_processor.load_user(
              runtime_storage_inst, user_id='john_doe'))
          self.assertEqual(user, user_processor.load_user(
              runtime_storage_inst, email='john_doe@ibm.com'))
          self.assertEqual(user, user_processor.load_user(
              runtime_storage_inst, gerrit_id='john_doe'))
  
          # all records should have the same user_id and company name
          for record in runtime_storage_inst.get_all_records():
              self.assertEqual('john_doe', record['user_id'],
                               message='Record %s' % record['primary_key'])
              self.assertEqual('IBM', record['company_name'],
                               message='Record %s' % record['primary_key'])
  
      def test_core_user_guess(self):
          record_processor_inst = self.make_record_processor(
              lp_user_name={
                  'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'},
                  'homer': {'name': 'homer', 'display_name': 'Homer Simpson'},
              },
              companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}],
          )
          runtime_storage_inst = record_processor_inst.runtime_storage_inst
  
          timestamp = int(time.time())
          runtime_storage_inst.set_records(record_processor_inst.process([
              {'record_type': 'review',
               'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
               'subject': 'Fix AttributeError in Keypair._add_details()',
               'owner': {'name': 'John Doe',
                         'email': 'john_doe@ibm.com',
                         'username': 'john_doe'},
               'createdOn': timestamp,
               'module': 'nova',
               'branch': 'master',
               'patchSets': [
                   {'number': '1',
                    'revision': '4d8984e92910c37b7d101c1ae8c8283a2e6f4a76',
                    'ref': 'refs/changes/16/58516/1',
                    'uploader': {
                        'name': 'Bill Smith',
                        'email': 'bill@smith.to',
                        'username': 'bsmith'},
                    'createdOn': timestamp,
                    'approvals': [
                        {'type': 'Code-Review', 'description': 'Code Review',
                         'value': '2', 'grantedOn': timestamp,
                         'by': {
                             'name': 'John Doe',
                             'email': 'john_doe@ibm.com',
                             'username': 'john_doe'}},
                        {'type': 'Code-Review', 'description': 'Code Review',
                         'value': '-1', 'grantedOn': timestamp - 1,  # differ
                         'by': {
                             'name': 'Homer Simpson',
                             'email': 'hsimpson@gmail.com',
                             'username': 'homer'}}
                    ]
                    }]}
          ]))
  
          record_processor_inst.post_processing({})
  
          user_1 = {'seq': 1, 'user_id': 'john_doe',
                    'launchpad_id': 'john_doe', 'user_name': 'John Doe',
                    'emails': ['john_doe@ibm.com'],
                    'core': [('nova', 'master')],
                    'companies': [{'company_name': 'IBM', 'end_date': 0}]}
          user_2 = {'seq': 3, 'user_id': 'homer',
                    'launchpad_id': 'homer', 'user_name': 'Homer Simpson',
                    'emails': ['hsimpson@gmail.com'],
                    'companies': [{'company_name': '*independent',
                                   'end_date': 0}]}
          runtime_storage_inst = record_processor_inst.runtime_storage_inst
          self.assertUsersMatch(user_1, user_processor.load_user(
              runtime_storage_inst, user_id='john_doe'))
          self.assertUsersMatch(user_2, user_processor.load_user(
              runtime_storage_inst, user_id='homer'))
  
      def test_process_commit_with_coauthors(self):
          record_processor_inst = self.make_record_processor(
              lp_info={'jimi.hendrix@openstack.com':
                       {'name': 'jimi', 'display_name': 'Jimi Hendrix'},
                       'tupac.shakur@openstack.com':
                       {'name': 'tupac', 'display_name': 'Tupac Shakur'},
                       'bob.dylan@openstack.com':
                       {'name': 'bob', 'display_name': 'Bob Dylan'}})
          processed_commits = list(record_processor_inst.process([
              {'record_type': 'commit',
               'commit_id': 'de7e8f297c193fb310f22815334a54b9c76a0be1',
               'author_name': 'Jimi Hendrix',
               'author_email': 'jimi.hendrix@openstack.com', 'date': 1234567890,
               'lines_added': 25, 'lines_deleted': 9, 'release_name': 'havana',
               'coauthor': [{'author_name': 'Tupac Shakur',
                             'author_email': 'tupac.shakur@openstack.com'},
                            {'author_name': 'Bob Dylan',
                             'author_email': 'bob.dylan@openstack.com'}]}]))
  
          self.assertEqual(3, len(processed_commits))
  
          self.assertRecordsMatch({
              'user_id': 'tupac',
              'author_email': 'tupac.shakur@openstack.com',
              'author_name': 'Tupac Shakur',
          }, processed_commits[0])
          self.assertRecordsMatch({
              'user_id': 'jimi',
              'author_email': 'jimi.hendrix@openstack.com',
              'author_name': 'Jimi Hendrix',
          }, processed_commits[2])
          self.assertEqual('tupac',
                           processed_commits[0]['coauthor'][0]['user_id'])
          self.assertEqual('bob',
                           processed_commits[0]['coauthor'][1]['user_id'])
          self.assertEqual('jimi',
                           processed_commits[0]['coauthor'][2]['user_id'])
  
      def test_process_commit_with_coauthors_no_dup_of_author(self):
          record_processor_inst = self.make_record_processor(
              lp_info={'jimi.hendrix@openstack.com':
                       {'name': 'jimi', 'display_name': 'Jimi Hendrix'},
                       'bob.dylan@openstack.com':
                       {'name': 'bob', 'display_name': 'Bob Dylan'}})
          processed_commits = list(record_processor_inst.process([
              {'record_type': 'commit',
               'commit_id': 'de7e8f297c193fb310f22815334a54b9c76a0be1',
               'author_name': 'Jimi Hendrix',
               'author_email': 'jimi.hendrix@openstack.com', 'date': 1234567890,
               'lines_added': 25, 'lines_deleted': 9, 'release_name': 'havana',
               'coauthor': [{'author_name': 'Jimi Hendrix',
                             'author_email': 'jimi.hendrix@openstack.com'},
                            {'author_name': 'Bob Dylan',
                             'author_email': 'bob.dylan@openstack.com'}]}]))
  
          self.assertEqual(2, len(processed_commits))
  
          self.assertEqual('jimi',
                           processed_commits[0]['coauthor'][0]['user_id'])
          self.assertEqual('bob',
                           processed_commits[0]['coauthor'][1]['user_id'])
  
      # record post-processing
  
      def test_blueprint_mention_count(self):
          record_processor_inst = self.make_record_processor()
          runtime_storage_inst = record_processor_inst.runtime_storage_inst
  
          runtime_storage_inst.set_records(record_processor_inst.process([
              {'record_type': 'bp',
               'id': 'mod:blueprint',
               'self_link': 'http://launchpad.net/blueprint',
               'owner': 'john_doe',
               'date_created': 1234567890},
              {'record_type': 'bp',
               'id': 'mod:ignored',
               'self_link': 'http://launchpad.net/ignored',
               'owner': 'john_doe',
               'date_created': 1234567890},
              {'record_type': 'email',
               'message_id': '<message-id>',
               'author_email': 'john_doe@gmail.com', 'author_name': 'John Doe',
               'subject': 'hello, world!',
               'body': 'lorem ipsum',
               'date': 1234567890,
               'blueprint_id': ['mod:blueprint']},
              {'record_type': 'email',
               'message_id': '<another-message-id>',
               'author_email': 'john_doe@gmail.com', 'author_name': 'John Doe',
               'subject': 'hello, world!',
               'body': 'lorem ipsum',
               'date': 1234567895,
               'blueprint_id': ['mod:blueprint', 'mod:invalid']},
          ]))
          record_processor_inst.post_processing({})
  
          bp1 = runtime_storage_inst.get_by_primary_key('bpd:mod:blueprint')
          self.assertEqual(2, bp1['mention_count'])
          self.assertEqual(1234567895, bp1['mention_date'])
  
          bp2 = runtime_storage_inst.get_by_primary_key('bpd:mod:ignored')
          self.assertEqual(0, bp2['mention_count'])
          self.assertEqual(0, bp2['mention_date'])
  
          email = runtime_storage_inst.get_by_primary_key('<another-message-id>')
          self.assertTrue('mod:blueprint' in email['blueprint_id'])
          self.assertFalse('mod:invalid' in email['blueprint_id'])
  
      def test_review_number(self):
          record_processor_inst = self.make_record_processor()
          runtime_storage_inst = record_processor_inst.runtime_storage_inst
  
          runtime_storage_inst.set_records(record_processor_inst.process([
              {'record_type': 'review',
               'id': 'I111',
               'subject': 'Fix AttributeError in Keypair._add_details()',
               'owner': {'name': 'John Doe',
                         'email': 'john_doe@gmail.com',
                         'username': 'john_doe'},
               'createdOn': 10,
               'module': 'nova', 'branch': 'master'},
              {'record_type': 'review',
               'id': 'I222',
               'subject': 'Fix AttributeError in Keypair._add_details()',
               'owner': {'name': 'John Doe',
                         'email': 'john_doe@gmail.com',
                         'username': 'john_doe'},
               'createdOn': 5,
               'module': 'glance', 'branch': 'master'},
          ]))
          record_processor_inst.post_processing({})
  
          review1 = runtime_storage_inst.get_by_primary_key('I111')
          self.assertEqual(2, review1['review_number'])
  
          review2 = runtime_storage_inst.get_by_primary_key('I222')
          self.assertEqual(1, review2['review_number'])
  
      def test_mark_disagreement(self):
          record_processor_inst = self.make_record_processor(
              users=[
                  {'user_id': 'john_doe',
                   'launchpad_id': 'john_doe',
                   'user_name': 'John Doe',
                   'emails': ['john_doe@ibm.com'],
                   'core': [('nova', 'master')],
                   'companies': [{'company_name': 'IBM', 'end_date': 0}]}
              ],
          )
          timestamp = int(time.time())
          runtime_storage_inst = record_processor_inst.runtime_storage_inst
          runtime_storage_inst.set_records(record_processor_inst.process([
              {'record_type': 'review',
               'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
               'subject': 'Fix AttributeError in Keypair._add_details()',
               'owner': {'name': 'John Doe',
                         'email': 'john_doe@ibm.com',
                         'username': 'john_doe'},
               'createdOn': timestamp,
               'module': 'nova',
               'branch': 'master',
               'status': 'NEW',
               'patchSets': [
                   {'number': '1',
                    'revision': '4d8984e92910c37b7d101c1ae8c8283a2e6f4a76',
                    'ref': 'refs/changes/16/58516/1',
                    'uploader': {
                        'name': 'Bill Smith',
                        'email': 'bill@smith.to',
                        'username': 'bsmith'},
                    'createdOn': timestamp,
                    'approvals': [
                        {'type': 'Code-Review', 'description': 'Code Review',
                         'value': '2', 'grantedOn': timestamp - 1,
                         'by': {
                             'name': 'Homer Simpson',
                             'email': 'hsimpson@gmail.com',
                             'username': 'homer'}},
                        {'type': 'Code-Review', 'description': 'Code Review',
                         'value': '-2', 'grantedOn': timestamp,
                         'by': {
                             'name': 'John Doe',
                             'email': 'john_doe@ibm.com',
                             'username': 'john_doe'}}
                    ]
                    },
                   {'number': '2',
                    'revision': '4d8984e92910c37b7d101c1ae8c8283a2e6f4a76',
                    'ref': 'refs/changes/16/58516/1',
                    'uploader': {
                        'name': 'Bill Smith',
                        'email': 'bill@smith.to',
                        'username': 'bsmith'},
                    'createdOn': timestamp + 1,
                    'approvals': [
                        {'type': 'Code-Review', 'description': 'Code Review',
                         'value': '1', 'grantedOn': timestamp + 2,
                         'by': {
                             'name': 'Homer Simpson',
                             'email': 'hsimpson@gmail.com',
                             'username': 'homer'}},
                        {'type': 'Code-Review', 'description': 'Code Review',
                         'value': '-1', 'grantedOn': timestamp + 3,
                         'by': {
                             'name': 'Bart Simpson',
                             'email': 'bsimpson@gmail.com',
                             'username': 'bart'}},
                        {'type': 'Code-Review', 'description': 'Code Review',
                         'value': '2', 'grantedOn': timestamp + 4,
                         'by': {
                             'name': 'John Doe',
                             'email': 'john_doe@ibm.com',
                             'username': 'john_doe'}}
                    ]
                    }
               ]}
          ]))
          record_processor_inst.post_processing({})
  
          marks = list([r for r in runtime_storage_inst.get_all_records()
                        if r['record_type'] == 'mark'])
  
          homer_mark = next(six.moves.filter(
              lambda x: x['date'] == (timestamp - 1), marks), None)
          self.assertTrue(homer_mark.get('disagreement'),
                          msg='Disagreement: core set -2 after +2')
  
          homer_mark = next(six.moves.filter(
              lambda x: x['date'] == (timestamp + 2), marks), None)
          self.assertFalse(homer_mark.get('disagreement'),
                           msg='No disagreement: core set +2 after +1')
  
          bart_mark = next(six.moves.filter(
              lambda x: x['date'] == (timestamp + 3), marks), None)
          self.assertTrue(bart_mark.get('disagreement'),
                          msg='Disagreement: core set +2 after -1')
  
      def test_commit_merge_date(self):
          record_processor_inst = self.make_record_processor()
          runtime_storage_inst = record_processor_inst.runtime_storage_inst
  
          runtime_storage_inst.set_records(record_processor_inst.process([
              {'record_type': 'commit',
               'commit_id': 'de7e8f2',
               'change_id': ['I104573'],
               'author_name': 'John Doe',
               'author_email': 'john_doe@gmail.com',
               'date': 1234567890,
               'lines_added': 25,
               'lines_deleted': 9,
               'release_name': 'havana'},
              {'record_type': 'review',
               'id': 'I104573',
               'subject': 'Fix AttributeError in Keypair._add_details()',
               'owner': {'name': 'John Doe',
                         'email': 'john_doe@gmail.com',
                         'username': 'john_doe'},
               'createdOn': 1385478465,
               'lastUpdated': 1385490000,
               'status': 'MERGED',
               'module': 'nova', 'branch': 'master'},
          ]))
          record_processor_inst.post_processing({})
  
          commit = runtime_storage_inst.get_by_primary_key('de7e8f2')
          self.assertEqual(1385490000, commit['date'])
  
      # update records
  
      def _generate_record_commit(self):
          yield {'commit_id': u'0afdc64bfd041b03943ceda7849c4443940b6053',
                 'lines_added': 9,
                 'module': u'stackalytics',
                 'record_type': 'commit',
                 'message': u'Closes bug 1212953\n\nChange-Id: '
                            u'I33f0f37b6460dc494abf2520dc109c9893ace9e6\n',
                 'subject': u'Fixed affiliation of Edgar and Sumit',
                 'loc': 10,
                 'user_id': u'john_doe',
                 'primary_key': u'0afdc64bfd041b03943ceda7849c4443940b6053',
                 'author_email': u'jdoe@super.no',
                 'company_name': u'SuperCompany',
                 'record_id': 6,
                 'lines_deleted': 1,
                 'week': 2275,
                 'blueprint_id': None,
                 'bug_id': u'1212953',
                 'files_changed': 1,
                 'author_name': u'John Doe',
                 'date': 1376737923,
                 'launchpad_id': u'john_doe',
                 'branches': set([u'master']),
                 'change_id': u'I33f0f37b6460dc494abf2520dc109c9893ace9e6',
                 'release': u'havana'}
  
      # mail processing
  
      def test_process_mail(self):
          record_processor_inst = self.make_record_processor(
              users=[
                  {
                      'user_id': 'john_doe',
                      'launchpad_id': 'john_doe',
                      'user_name': 'John Doe',
                      'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'],
                      'companies': [
                          {'company_name': 'NEC', 'end_date': 0},
                      ]
                  }
              ],
              repos=[{"module": "stackalytics"}]
          )
  
          processed_commit = list(record_processor_inst.process(
              generate_emails(
                  author_email='johndoe@gmail.com',
                  author_name='John Doe',
                  subject='[openstack-dev] [Stackalytics] Configuration files')
          ))[0]
  
          expected_commit = {
              'user_id': 'john_doe',
              'author_email': 'johndoe@gmail.com',
              'author_name': 'John Doe',
              'company_name': 'NEC',
              'module': 'stackalytics',
          }
  
          self.assertRecordsMatch(expected_commit, processed_commit)
  
      def test_process_mail_guessed(self):
          record_processor_inst = self.make_record_processor(
              users=[
                  {
                      'user_id': 'john_doe',
                      'launchpad_id': 'john_doe',
                      'user_name': 'John Doe',
                      'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'],
                      'companies': [
                          {'company_name': 'NEC', 'end_date': 0},
                      ]
                  }
              ],
              repos=[{'module': 'nova'}, {'module': 'neutron'}]
          )
  
          processed_commit = list(record_processor_inst.process(
              generate_emails(
                  author_email='johndoe@gmail.com',
                  author_name='John Doe',
                  subject='[openstack-dev] [Neutron] [Nova] Integration issue')
          ))[0]
  
          expected_commit = {
              'user_id': 'john_doe',
              'author_email': 'johndoe@gmail.com',
              'author_name': 'John Doe',
              'company_name': 'NEC',
              'module': 'neutron',
          }
  
          self.assertRecordsMatch(expected_commit, processed_commit)
  
      def test_process_mail_guessed_module_in_body_override(self):
          record_processor_inst = self.make_record_processor(
              users=[
                  {
                      'user_id': 'john_doe',
                      'launchpad_id': 'john_doe',
                      'user_name': 'John Doe',
                      'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'],
                      'companies': [
                          {'company_name': 'NEC', 'end_date': 0},
                      ]
                  }
              ],
              repos=[{'module': 'nova'}, {'module': 'neutron'}]
          )
  
          processed_commit = list(record_processor_inst.process(
              generate_emails(
                  author_email='johndoe@gmail.com',
                  author_name='John Doe',
                  module='nova',
                  subject='[openstack-dev] [neutron] Comments/questions on the')
          ))[0]
  
          expected_commit = {
              'user_id': 'john_doe',
              'author_email': 'johndoe@gmail.com',
              'author_name': 'John Doe',
              'company_name': 'NEC',
              'module': 'neutron',
          }
  
          self.assertRecordsMatch(expected_commit, processed_commit)
  
      def test_process_mail_guessed_module_in_body(self):
          record_processor_inst = self.make_record_processor(
              users=[
                  {
                      'user_id': 'john_doe',
                      'launchpad_id': 'john_doe',
                      'user_name': 'John Doe',
                      'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'],
                      'companies': [
                          {'company_name': 'NEC', 'end_date': 0},
                      ]
                  }
              ],
              repos=[{'module': 'nova'}, {'module': 'neutron'}]
          )
  
          processed_commit = list(record_processor_inst.process(
              generate_emails(
                  author_email='johndoe@gmail.com',
                  author_name='John Doe',
                  module='nova',
                  subject='[openstack-dev] Comments/questions on the')
          ))[0]
  
          expected_commit = {
              'user_id': 'john_doe',
              'author_email': 'johndoe@gmail.com',
              'author_name': 'John Doe',
              'company_name': 'NEC',
              'module': 'nova',
          }
  
          self.assertRecordsMatch(expected_commit, processed_commit)
  
      def test_process_mail_unmatched(self):
          record_processor_inst = self.make_record_processor(
              users=[
                  {
                      'user_id': 'john_doe',
                      'launchpad_id': 'john_doe',
                      'user_name': 'John Doe',
                      'emails': ['johndoe@gmail.com', 'johndoe@nec.co.jp'],
                      'companies': [
                          {'company_name': 'NEC', 'end_date': 0},
                      ]
                  }
              ],
              repos=[{'module': 'nova'}, {'module': 'neutron'}]
          )
  
          processed_commit = list(record_processor_inst.process(
              generate_emails(
                  author_email='johndoe@gmail.com',
                  author_name='John Doe',
                  subject='[openstack-dev] Comments/questions on the')
          ))[0]
  
          expected_commit = {
              'user_id': 'john_doe',
              'author_email': 'johndoe@gmail.com',
              'author_name': 'John Doe',
              'company_name': 'NEC',
              'module': 'unknown',
          }
  
          self.assertRecordsMatch(expected_commit, processed_commit)
  
      def test_get_modules(self):
          record_processor_inst = self.make_record_processor()
          with mock.patch('stackalytics.processor.utils.load_repos') as patch:
              patch.return_value = [{'module': 'nova'},
                                    {'module': 'python-novaclient'},
                                    {'module': 'neutron'},
                                    {'module': 'sahara', 'aliases': ['savanna']}]
              modules, module_alias_map = record_processor_inst._get_modules()
              self.assertEqual(set(['nova', 'neutron', 'sahara', 'savanna']),
                               set(modules))
              self.assertEqual({'savanna': 'sahara'}, module_alias_map)
  
      def test_guess_module(self):
          record_processor_inst = self.make_record_processor()
          with mock.patch('stackalytics.processor.utils.load_repos') as patch:
              patch.return_value = [{'module': 'sahara', 'aliases': ['savanna']}]
              record = {'subject': '[savanna] T'}
              record_processor_inst._guess_module(record)
              self.assertEqual({'subject': '[savanna] T', 'module': 'sahara'},
                               record)
  
      def assertRecordsMatch(self, expected, actual):
          for key, value in six.iteritems(expected):
              self.assertEqual(value, actual.get(key),
                               'Values for key %s do not match' % key)
  
      def assertUsersMatch(self, expected, actual):
          self.assertIsNotNone(actual, 'User should not be None')
          match = True
          for key, value in six.iteritems(expected):
              if key == 'emails':
                  match = (set(value) == set(actual.get(key)))
              else:
                  match = (value == actual.get(key))
  
          self.assertTrue(match, 'User %s should match %s' % (actual, expected))
  
      # Helpers
  
      def make_record_processor(self, users=None, companies=None, releases=None,
                                repos=None, lp_info=None, lp_user_name=None):
          rp = record_processor.RecordProcessor(make_runtime_storage(
              users=users, companies=companies, releases=releases, repos=repos))
  
          if lp_info is not None:
              self.lp_profile_by_email.side_effect = (
                  lambda x: lp_info.get(x))
  
          if lp_user_name is not None:
              self.lp_profile_by_launchpad_id.side_effect = (
                  lambda x: lp_user_name.get(x))
  
          return rp
  
  
  def generate_commits(author_name='John Doe', author_email='johndoe@gmail.com',
                       date=1999999999):
      yield {
          'record_type': 'commit',
          'commit_id': 'de7e8f297c193fb310f22815334a54b9c76a0be1',
          'author_name': author_name,
          'author_email': author_email,
          'date': date,
          'lines_added': 25,
          'lines_deleted': 9,
          'release_name': 'havana',
      }
  
  
  def generate_emails(author_name='John Doe', author_email='johndoe@gmail.com',
                      date=1999999999, subject='[openstack-dev]', module=None):
      yield {
          'record_type': 'email',
          'message_id': 'de7e8f297c193fb310f22815334a54b9c76a0be1',
          'author_name': author_name,
          'author_email': author_email,
          'date': date,
          'subject': subject,
          'module': module,
          'body': 'lorem ipsum',
      }
  
  
  def make_runtime_storage(users=None, companies=None, releases=None,
                           repos=None):
      runtime_storage_cache = {}
      runtime_storage_record_keys = []
  
      def get_by_key(key):
          if key == 'companies':
              return _make_companies(companies or [
                  {"company_name": "*independent", "domains": [""]},
              ])
          elif key == 'users':
              return _make_users(users or [])
          elif key == 'releases':
              return releases or RELEASES
          elif key == 'repos':
              return repos or REPOS
          else:
              return runtime_storage_cache.get(key)
  
      def set_by_key(key, value):
          runtime_storage_cache[key] = value
  
      def delete_by_key(key):
          del runtime_storage_cache[key]
  
      def inc_user_count():
          count = runtime_storage_cache.get('user:count') or 0
          count += 1
          runtime_storage_cache['user:count'] = count
          return count
  
      def get_all_users():
          for n in six.moves.range(
                  0, (runtime_storage_cache.get('user:count') or 0) + 1):
              u = runtime_storage_cache.get('user:%s' % n)
              if u:
                  yield u
  
      def set_records(records_iterator):
          for record in records_iterator:
              runtime_storage_cache[record['primary_key']] = record
              runtime_storage_record_keys.append(record['primary_key'])
  
      def get_all_records():
          return [runtime_storage_cache[key]
                  for key in runtime_storage_record_keys]
  
      def get_by_primary_key(primary_key):
          return runtime_storage_cache.get(primary_key)
  
      rs = mock.Mock(runtime_storage.RuntimeStorage)
      rs.get_by_key = mock.Mock(side_effect=get_by_key)
      rs.set_by_key = mock.Mock(side_effect=set_by_key)
      rs.delete_by_key = mock.Mock(side_effect=delete_by_key)
      rs.inc_user_count = mock.Mock(side_effect=inc_user_count)
      rs.get_all_users = mock.Mock(side_effect=get_all_users)
      rs.set_records = mock.Mock(side_effect=set_records)
      rs.get_all_records = mock.Mock(side_effect=get_all_records)
      rs.get_by_primary_key = mock.Mock(side_effect=get_by_primary_key)
  
      if users:
          for user in users:
              set_by_key('user:%s' % user['user_id'], user)
              if user.get('launchpad_id'):
                  set_by_key('user:%s' % user['launchpad_id'], user)
              for email in user.get('emails') or []:
                  set_by_key('user:%s' % email, user)
  
      return rs
  
  
  def _make_users(users):
      users_index = {}
      for user in users:
          if 'user_id' in user:
              users_index[user['user_id']] = user
          if 'launchpad_id' in user:
              users_index[user['launchpad_id']] = user
          for email in user['emails']:
              users_index[email] = user
      return users_index
  
  
  def _make_companies(companies):
      domains_index = {}
      for company in companies:
          for domain in company['domains']:
              domains_index[domain] = company['company_name']
      return domains_index
  