text: #!/usr/bin/env python
  # Copyright (c) 2012 The Chromium Authors. All rights reserved.
  # Use of this source code is governed by a BSD-style license that can be
  # found in the LICENSE file.
  
  """Main functions for the Layout Test Analyzer module."""
  
  from datetime import datetime
  import optparse
  import os
  import sys
  import time
  
  import layouttest_analyzer_helpers
  from layouttest_analyzer_helpers import DEFAULT_REVISION_VIEW_URL
  import layouttests
  from layouttests import DEFAULT_LAYOUTTEST_SVN_VIEW_LOCATION
  
  from test_expectations import TestExpectations
  from trend_graph import TrendGraph
  
  # Predefined result directory.
  DEFAULT_RESULT_DIR = 'result'
  # TODO(shadi): Remove graph functions as they are not used any more.
  DEFAULT_GRAPH_FILE = os.path.join('graph', 'graph.html')
  # TODO(shadi): Check if these files are needed any more.
  DEFAULT_STATS_CSV_FILENAME = 'stats.csv'
  DEFAULT_ISSUES_CSV_FILENAME = 'issues.csv'
  # TODO(shadi): These are used only for |debug| mode. What is debug mode for?
  #              AFAIK, we don't run debug mode, should be safe to remove.
  # Predefined result files for debug.
  CUR_TIME_FOR_DEBUG = '2011-09-11-19'
  CURRENT_RESULT_FILE_FOR_DEBUG = os.path.join(DEFAULT_RESULT_DIR,
                                               CUR_TIME_FOR_DEBUG)
  PREV_TIME_FOR_DEBUG = '2011-09-11-18'
  
  # Text to append at the end of every analyzer result email.
  DEFAULT_EMAIL_APPEND_TEXT = (
      '<b><a href="https://groups.google.com/a/google.com/group/'
      'layout-test-analyzer-result/topics">Email History</a></b><br>'
    )
  
  
  def ParseOption():
    """Parse command-line options using OptionParser.
  
    Returns:
        an object containing all command-line option information.
    """
    option_parser = optparse.OptionParser()
  
    option_parser.add_option('-r', '--receiver-email-address',
                             dest='receiver_email_address',
                             help=('receiver\'s email address. '
                                   'Result email is not sent if this is not '
                                   'specified.'))
    option_parser.add_option('-g', '--debug-mode', dest='debug',
                             help=('Debug mode is used when you want to debug '
                                   'the analyzer by using local file rather '
                                   'than getting data from SVN. This shortens '
                                   'the debugging time (off by default).'),
                             action='store_true', default=False)
    option_parser.add_option('-t', '--trend-graph-location',
                             dest='trend_graph_location',
                             help=('Location of the bug trend file; '
                                   'file is expected to be in Google '
                                   'Visualization API trend-line format '
                                   '(defaults to %default).'),
                             default=DEFAULT_GRAPH_FILE)
    option_parser.add_option('-n', '--test-group-file-location',
                             dest='test_group_file_location',
                             help=('Location of the test group file; '
                                   'file is expected to be in CSV format '
                                   'and lists all test name patterns. '
                                   'When this option is not specified, '
                                   'the value of --test-group-name is used '
                                   'for a test name pattern.'),
                             default=None)
    option_parser.add_option('-x', '--test-group-name',
                             dest='test_group_name',
                             help=('A name of test group. Either '
                                   '--test_group_file_location or this option '
                                   'needs to be specified.'))
    option_parser.add_option('-d', '--result-directory-location',
                             dest='result_directory_location',
                             help=('Name of result directory location '
                                   '(default to %default).'),
                             default=DEFAULT_RESULT_DIR)
    option_parser.add_option('-b', '--email-appended-text-file-location',
                             dest='email_appended_text_file_location',
                             help=('File location of the email appended text. '
                                   'The text is appended in the status email. '
                                   '(default to %default and no text is '
                                   'appended in that case).'),
                             default=None)
    option_parser.add_option('-c', '--email-only-change-mode',
                             dest='email_only_change_mode',
                             help=('With this mode, email is sent out '
                                   'only when there is a change in the '
                                   'analyzer result compared to the previous '
                                   'result (off by default)'),
                             action='store_true', default=False)
    option_parser.add_option('-q', '--dashboard-file-location',
                             dest='dashboard_file_location',
                             help=('Location of dashboard file. The results are '
                                   'not reported to the dashboard if this '
                                   'option is not specified.'))
    option_parser.add_option('-z', '--issue-detail-mode',
                             dest='issue_detail_mode',
                             help=('With this mode, email includes issue details '
                                   '(links to the flakiness dashboard)'
                                   ' (off by default)'),
                             action='store_true', default=False)
    return option_parser.parse_args()[0]
  
  
  def GetCurrentAndPreviousResults(debug, test_group_file_location,
                                   test_group_name, result_directory_location):
    """Get current and the latest previous analyzer results.
  
    In debug mode, they are read from predefined files. In non-debug mode,
    current analyzer results are dynamically obtained from Blink SVN and
    the latest previous result is read from the corresponding file.
  
    Args:
      debug: please refer to |options|.
      test_group_file_location: please refer to |options|.
      test_group_name: please refer to |options|.
      result_directory_location: please refer to |options|.
  
    Returns:
      a tuple of the following:
         prev_time: the previous time string that is compared against.
         prev_analyzer_result_map: previous analyzer result map. Please refer to
            layouttest_analyzer_helpers.AnalyzerResultMap.
         analyzer_result_map: current analyzer result map. Please refer to
            layouttest_analyzer_helpers.AnalyzerResultMap.
    """
    if not debug:
      if not test_group_file_location and not test_group_name:
        print ('Either --test-group-name or --test_group_file_location must be '
               'specified. Exiting this program.')
        sys.exit()
      filter_names = []
      if test_group_file_location and os.path.exists(test_group_file_location):
        filter_names = layouttests.LayoutTests.GetLayoutTestNamesFromCSV(
            test_group_file_location)
        parent_location_list = (
            layouttests.LayoutTests.GetParentDirectoryList(filter_names))
        recursion = True
      else:
        # When test group CSV file is not specified, test group name
        # (e.g., 'media') is used for getting layout tests.
        # The tests are in
        #     http://src.chromium.org/blink/trunk/LayoutTests/media
        # Filtering is not set so all HTML files are considered as valid tests.
        # Also, we look for the tests recursively.
        if not test_group_file_location or (
            not os.path.exists(test_group_file_location)):
          print ('Warning: CSV file (%s) does not exist. So it is ignored and '
                 '%s is used for obtaining test names') % (
                     test_group_file_location, test_group_name)
        if not test_group_name.endswith('/'):
          test_group_name += '/'
        parent_location_list = [test_group_name]
        filter_names = None
        recursion = True
      layouttests_object = layouttests.LayoutTests(
          parent_location_list=parent_location_list, recursion=recursion,
          filter_names=filter_names)
      analyzer_result_map = layouttest_analyzer_helpers.AnalyzerResultMap(
          layouttests_object.JoinWithTestExpectation(TestExpectations()))
      result = layouttest_analyzer_helpers.FindLatestResult(
          result_directory_location)
      if result:
        (prev_time, prev_analyzer_result_map) = result
      else:
        prev_time = None
        prev_analyzer_result_map = None
    else:
      analyzer_result_map = layouttest_analyzer_helpers.AnalyzerResultMap.Load(
          CURRENT_RESULT_FILE_FOR_DEBUG)
      prev_time = PREV_TIME_FOR_DEBUG
      prev_analyzer_result_map = (
          layouttest_analyzer_helpers.AnalyzerResultMap.Load(
              os.path.join(DEFAULT_RESULT_DIR, prev_time)))
    return (prev_time, prev_analyzer_result_map, analyzer_result_map)
  
  
  def SendEmail(prev_time, prev_analyzer_result_map, analyzer_result_map,
                appended_text_to_email, email_only_change_mode, debug,
                receiver_email_address, test_group_name, issue_detail_mode):
    """Send result status email.
  
    Args:
      prev_time: the previous time string that is compared against.
      prev_analyzer_result_map: previous analyzer result map. Please refer to
          layouttest_analyzer_helpers.AnalyzerResultMap.
      analyzer_result_map: current analyzer result map. Please refer to
          layouttest_analyzer_helpers.AnalyzerResultMap.
      appended_text_to_email: the text string to append to the status email.
      email_only_change_mode: please refer to |options|.
      debug: please refer to |options|.
      receiver_email_address: please refer to |options|.
      test_group_name: please refer to |options|.
      issue_detail_mode: please refer to |options|.
  
    Returns:
      a tuple of the following:
          result_change: a boolean indicating whether there is a change in the
              result compared with the latest past result.
          diff_map: please refer to
              layouttest_analyzer_helpers.SendStatusEmail().
          simple_rev_str: a simple version of revision string that is sent in
              the email.
          rev: the latest revision number for the given test group.
          rev_date: the latest revision date for the given test group.
          email_content:  email content string (without
              |appended_text_to_email|) that will be shown on the dashboard.
    """
    rev = ''
    rev_date = ''
    email_content = ''
    if prev_analyzer_result_map:
      diff_map = analyzer_result_map.CompareToOtherResultMap(
          prev_analyzer_result_map)
      result_change = (any(diff_map['whole']) or any(diff_map['skip']) or
                       any(diff_map['nonskip']))
      # Email only when |email_only_change_mode| is False or there
      # is a change in the result compared to the last result.
      simple_rev_str = ''
      if not email_only_change_mode or result_change:
        prev_time_in_float = datetime.strptime(prev_time, '%Y-%m-%d-%H')
        prev_time_in_float = time.mktime(prev_time_in_float.timetuple())
        if debug:
          cur_time_in_float = datetime.strptime(CUR_TIME_FOR_DEBUG,
                                                '%Y-%m-%d-%H')
          cur_time_in_float = time.mktime(cur_time_in_float.timetuple())
        else:
          cur_time_in_float = time.time()
        (rev_str, simple_rev_str, rev, rev_date) = (
            layouttest_analyzer_helpers.GetRevisionString(prev_time_in_float,
                                                          cur_time_in_float,
                                                          diff_map))
        email_content = analyzer_result_map.ConvertToString(prev_time,
                                                            diff_map,
                                                            issue_detail_mode)
        if receiver_email_address:
          layouttest_analyzer_helpers.SendStatusEmail(
              prev_time, analyzer_result_map, diff_map,
              receiver_email_address, test_group_name,
              appended_text_to_email, email_content, rev_str,
              email_only_change_mode)
      if simple_rev_str:
        simple_rev_str = '\'' + simple_rev_str + '\''
      else:
        simple_rev_str = 'undefined'  # GViz uses undefined for NONE.
    else:
      # Initial result should be written to tread-graph if there are no previous
      # results.
      result_change = True
      diff_map = None
      simple_rev_str = 'undefined'
      email_content = analyzer_result_map.ConvertToString(None, diff_map,
                                                          issue_detail_mode)
    return (result_change, diff_map, simple_rev_str, rev, rev_date,
            email_content)
  
  
  def UpdateTrendGraph(start_time, analyzer_result_map, diff_map, simple_rev_str,
                       trend_graph_location):
    """Update trend graph in GViz.
  
    Annotate the graph with revision information.
  
    Args:
      start_time: the script starting time as a float value.
      analyzer_result_map: current analyzer result map. Please refer to
          layouttest_analyzer_helpers.AnalyzerResultMap.
      diff_map: a map that has 'whole', 'skip' and 'nonskip' as keys.
          Please refer to |diff_map| in
          |layouttest_analyzer_helpers.SendStatusEmail()|.
      simple_rev_str: a simple version of revision string that is sent in
          the email.
      trend_graph_location: the location of the trend graph that needs to be
          updated.
  
    Returns:
       a dictionary that maps result data category ('whole', 'skip', 'nonskip',
           'passingrate') to information tuple (a dictionary that maps test name
           to its description, annotation, simple_rev_string) of the given result
           data category. These tuples are used for trend graph update.
    """
    # Trend graph update (if specified in the command-line argument) when
    # there is change from the last result.
    # Currently, there are two graphs (graph1 is for 'whole', 'skip',
    # 'nonskip' and the graph2 is for 'passingrate'). Please refer to
    # graph/graph.html.
    # Sample JS annotation for graph1:
    #   [new Date(2011,8,12,10,41,32),224,undefined,'',52,undefined,
    #    undefined, 12, 'test1,','<a href="http://t</a>,',],
    # This example lists 'whole' triple and 'skip' triple and
    # 'nonskip' triple. Each triple is (the number of tests that belong to
    # the test group, linked text, a link). The following code generates this
    # automatically based on rev_string etc.
    trend_graph = TrendGraph(trend_graph_location)
    datetime_string = start_time.strftime('%Y,%m,%d,%H,%M,%S')
    data_map = {}
    passingrate_anno = ''
    for test_group in ['whole', 'skip', 'nonskip']:
      anno = 'undefined'
      # Extract test description.
      test_map = {}
      for (test_name, value) in (
          analyzer_result_map.result_map[test_group].iteritems()):
        test_map[test_name] = value['desc']
      test_str = ''
      links = ''
      if diff_map and diff_map[test_group]:
        for i in [0, 1]:
          for (name, _) in diff_map[test_group][i]:
            test_str += name + ','
            # This is link to test HTML in SVN.
            links += ('<a href="%s%s">%s</a>' %
                      (DEFAULT_LAYOUTTEST_SVN_VIEW_LOCATION, name, name))
        if test_str:
          anno = '\'' + test_str + '\''
          # The annotation of passing rate is a union of all annotations.
          passingrate_anno += anno
      if links:
        links = '\'' + links + '\''
      else:
        links = 'undefined'
      if test_group is 'whole':
        data_map[test_group] = (test_map, anno, links)
      else:
        data_map[test_group] = (test_map, anno, simple_rev_str)
    if not passingrate_anno:
      passingrate_anno = 'undefined'
    data_map['passingrate'] = (
        str(analyzer_result_map.GetPassingRate()), passingrate_anno,
        simple_rev_str)
    trend_graph.Update(datetime_string, data_map)
    return data_map
  
  
  def UpdateDashboard(dashboard_file_location, test_group_name, data_map,
                      layouttest_root_path, rev, rev_date, email,
                      email_content):
    """Update dashboard HTML file.
  
    Args:
      dashboard_file_location: the file location for the dashboard file.
      test_group_name: please refer to |options|.
      data_map: a dictionary that maps result data category ('whole', 'skip',
          'nonskip', 'passingrate') to information tuple (a dictionary that maps
          test name to its description, annotation, simple_rev_string) of the
          given result data category. These tuples are used for trend graph
          update.
      layouttest_root_path: A location string where layout tests are stored.
      rev: the latest revision number for the given test group.
      rev_date: the latest revision date for the given test group.
      email: email address of the owner for the given test group.
      email_content:  email content string (without |appended_text_to_email|)
          that will be shown on the dashboard.
    """
    # Generate a HTML file that contains all test names for each test group.
    escaped_tg_name = test_group_name.replace('/', '_')
    for tg in ['whole', 'skip', 'nonskip']:
      file_name = os.path.join(
          os.path.dirname(dashboard_file_location),
          escaped_tg_name + '_' + tg + '.html')
      file_object = open(file_name, 'wb')
      file_object.write('<table border="1">')
      sorted_testnames = data_map[tg][0].keys()
      sorted_testnames.sort()
      for testname in sorted_testnames:
        file_object.write((
            '<tr><td><a href="%s">%s</a></td><td><a href="%s">dashboard</a>'
            '</td><td>%s</td></tr>') % (
                layouttest_root_path + testname, testname,
                ('http://test-results.appspot.com/dashboards/'
                 'flakiness_dashboard.html#tests=%s') % testname,
                data_map[tg][0][testname]))
      file_object.write('</table>')
      file_object.close()
    email_content_with_link = ''
    if email_content:
      file_name = os.path.join(os.path.dirname(dashboard_file_location),
                               escaped_tg_name + '_email.html')
      file_object = open(file_name, 'wb')
      file_object.write(email_content)
      file_object.close()
      email_content_with_link = '<a href="%s_email.html">info</a>' % (
          escaped_tg_name)
    test_group_str = (
        '<td><a href="%(test_group_path)s">%(test_group_name)s</a></td>'
        '<td><a href="%(graph_path)s">graph</a></td>'
        '<td><a href="%(all_tests_path)s">%(all_tests_count)d</a></td>'
        '<td><a href="%(skip_tests_path)s">%(skip_tests_count)d</a></td>'
        '<td><a href="%(nonskip_tests_path)s">%(nonskip_tests_count)d</a></td>'
        '<td>%(fail_rate)d%%</td>'
        '<td>%(passing_rate)d%%</td>'
        '<td><a href="%(rev_url)s">%(rev)s</a></td>'
        '<td>%(rev_date)s</td>'
        '<td><a href="mailto:%(email)s">%(email)s</a></td>'
        '<td>%(email_content)s</td>\n') % {
            # Dashboard file and graph must be in the same directory
            # to make the following link work.
            'test_group_path': layouttest_root_path + '/' + test_group_name,
            'test_group_name': test_group_name,
            'graph_path': escaped_tg_name + '.html',
            'all_tests_path': escaped_tg_name + '_whole.html',
            'all_tests_count': len(data_map['whole'][0]),
            'skip_tests_path': escaped_tg_name + '_skip.html',
            'skip_tests_count': len(data_map['skip'][0]),
            'nonskip_tests_path': escaped_tg_name + '_nonskip.html',
            'nonskip_tests_count': len(data_map['nonskip'][0]),
            'fail_rate': 100 - float(data_map['passingrate'][0]),
            'passing_rate': float(data_map['passingrate'][0]),
            'rev_url': DEFAULT_REVISION_VIEW_URL % rev,
            'rev': rev,
            'rev_date': rev_date,
            'email': email,
            'email_content': email_content_with_link
        }
    layouttest_analyzer_helpers.ReplaceLineInFile(
        dashboard_file_location, '<td>' + test_group_name + '</td>',
        test_group_str)
  
  
  def main():
    """A main function for the analyzer."""
    options = ParseOption()
    start_time = datetime.now()
  
    (prev_time, prev_analyzer_result_map, analyzer_result_map) = (
        GetCurrentAndPreviousResults(options.debug,
                                     options.test_group_file_location,
                                     options.test_group_name,
                                     options.result_directory_location))
    (result_change, diff_map, simple_rev_str, rev, rev_date, email_content) = (
        SendEmail(prev_time, prev_analyzer_result_map, analyzer_result_map,
                  DEFAULT_EMAIL_APPEND_TEXT,
                  options.email_only_change_mode, options.debug,
                  options.receiver_email_address, options.test_group_name,
                  options.issue_detail_mode))
  
    # Create CSV texts and save them for bug spreadsheet.
    (stats, issues_txt) = analyzer_result_map.ConvertToCSVText(
        start_time.strftime('%Y-%m-%d-%H'))
    file_object = open(os.path.join(options.result_directory_location,
                                    DEFAULT_STATS_CSV_FILENAME), 'wb')
    file_object.write(stats)
    file_object.close()
    file_object = open(os.path.join(options.result_directory_location,
                                    DEFAULT_ISSUES_CSV_FILENAME), 'wb')
    file_object.write(issues_txt)
    file_object.close()
  
    if not options.debug and (result_change or not prev_analyzer_result_map):
      # Save the current result when result is changed or the script is
      # executed for the first time.
      date = start_time.strftime('%Y-%m-%d-%H')
      file_path = os.path.join(options.result_directory_location, date)
      analyzer_result_map.Save(file_path)
    if result_change or not prev_analyzer_result_map:
      data_map = UpdateTrendGraph(start_time, analyzer_result_map, diff_map,
                                  simple_rev_str, options.trend_graph_location)
      # Report the result to dashboard.
      if options.dashboard_file_location:
        UpdateDashboard(options.dashboard_file_location, options.test_group_name,
                        data_map, layouttests.DEFAULT_LAYOUTTEST_LOCATION, rev,
                        rev_date, options.receiver_email_address,
                        email_content)
  
  
  if '__main__' == __name__:
    main()
  