text: # @@@ START COPYRIGHT @@@
  #
  # Licensed to the Apache Software Foundation (ASF) under one
  # or more contributor license agreements.  See the NOTICE file
  # distributed with this work for additional information
  # regarding copyright ownership.  The ASF licenses this file
  # to you under the Apache License, Version 2.0 (the
  # "License"); you may not use this file except in compliance
  # with the License.  You may obtain a copy of the License at
  #
  #   http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing,
  # software distributed under the License is distributed on an
  # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  # KIND, either express or implied.  See the License for the
  # specific language governing permissions and limitations
  # under the License.
  #
  # @@@ END COPYRIGHT @@@
  #
  #  This script checks the consistency of the Messages Guide
  #  against the code. It looks for messages whose text differs
  #  between the Messages Guide and code. It looks for messages
  #  that are no longer in use. It looks for messages that are
  #  in use but not in the Messages Guide. It produces a report
  #  of these.
  #
  #  The report includes detailed information for each message
  #  that differs between the Messages Guide and code. It lists
  #  the enum used for the message if there is one. It lists
  #  the code modules that may have a reference to the message
  #  (it uses grep to determine this, so there may be false
  #  positives or more rarely false negatives). It lists the
  #  regression test expected files that contain examples of the
  #  error message.
  #
  import os
  import sys
  import subprocess 
  import sets
  import datetime
  import argparse  # requires Python 2.7
  
  
  # The script does the following:
  #
  # 1. Parses the Messages Guide chapters into a table containing message number
  # + message text.
  # 2. Parses the bin/SqlciErrors.txt file into a table containing message number
  # + message text. Merges that into the table from item 1.
  # 3. Parses the various *.h files with error message enums. Merges that 
  # information into the table from item 1.
  # 4. Compares the message texts in items 1 and 2. The comparison has to be
  # smart; parameters appear as angle-bracketed items in the Messages Guide but
  # contain $0-String0 notation in the SqlciErrors.txt text. When there are
  # mismatches, this is noted. If one or the other text is missing, this notation
  # is marked "false".
  # 5. Searches the sql *.cpp files for references to the error message number
  # and/or enum, and when found, maintains a list for each, merged into the
  # table from item 1. We only do this for mismatched messages as it takes a
  # lot of time.
  # 6. Searches the regression tests LOG* files for examples of each message
  # number, and when found, maintains a list for each, attached to the table
  # constructed in 1. We only do this for mismatched messages as it takes a
  # lot of time.
  # 7. Lists error messages that mismatch between Messages Guide and
  # SqlciErrors.txt.
  # 8. Lists error messages that are missing from the Messages Guide but present
  # in SqlciErrors.txt. Mentions whether these are actually used anywhere (in
  # *.cpp files or regression tests).
  # 9. Lists error messages that are present in the Messages Guide or
  # SqlciErrors.txt but apparently are no longer used (that is, don't appear in
  # *.cpp files or regression tests).
  #
  # The table resulting from items 1 through 6 has the following shape:
  # a. Message number (key)
  # b. Messages Guide text (if any)
  # c. SqlciErrors.txt message text (if any)
  # d. Enum (if any)
  # e. Enum *.h file name (if any)
  # f. List of *.cpp files with references to message number
  # g. List of *.cpp files with references to enum name
  # h. List of regression tests with messages number examples
  # i. Boolean indicating whether SqlciErrors.txt text matches Messages Guide
  #
  # Item 1 is implemented by a function that parses a single *.adoc file
  # producing table entries. It is driven by a loop over the set of *.adoc file
  # names. The function updates the table directly.
  # Item 2 is implemented by a function that parses a *.txt file, driven by a
  # caller supplying the bin/SqlciErrors.txt name. The function updates the table
  # directly.
  # Item 3 is implemented by a function that parses a *.h file looking for error
  # message enums. It is driven by a loop over a hard-coded set of known *.h
  # files, perhaps with the name of the enum passed in. The function updates the
  # table directly.
  # Item 4 is implemented by a function that does a grep of the *.cpp files, then
  # processes the results. The function updates the table directly.
  # Item 5 is implemented by a function that does a grep of the LOG* files, then
  # processes the results. The function updates the table directly.
  # Item 6 is implemented by a function that crawls the table, comparing the
  # retrieved texts. The function updates the table directly.
  # Items 7, 8, 9 are implemented by a function that reads the table and reports
  # results.
  #
  # We model the table as a Python object and implement these functions as
  # methods on that object.
  
  class MessagesTable:
  
      # The structure of self.dict is a dictionary of dictionaries.
      # The key to the top level dictionary is the message number.
      # The value in each top level dictionary is essentially a
      # relation, modeled by a Python dictionary. The key in this
      # value is the attribute name, as follows:
      #
      # 'messageGuideText'
      # 'errorMessageFileText'
      # 'enumSymbol'
      # 'enumFile'
      # 'listOfCodeReferences'
      # 'listOfEnumSymbolReferences'
      # 'listOfTestReferences'
      # 'textsMatch'
  
      def __init__(self):
          self.dict = {}
  
      def mergeEntry(self,key,values):
          if key in self.dict:
              self.dict[key].update(values)
          else:
              self.dict[key] = values
  
      def parseAdoc(self,adocFileName):
          #
          # Message entries look like this:
          #
          # [[SQL-1002]]
          # == SQL 1002
          #
          # ```
          # Catalog <catalog> does not exist.
          # ```
          #
          # Where <catalog> is the ANSI name of the target catalog.
          #
          # *Cause:* The catalog does not exist.
          #
          # *Effect:* The operation fails.
          #
          # *Recovery:* Enter a valid catalog name and resubmit.
          #
          # The stuff we are interested in is the message number and
          # the message text. We use a tiny state machine to figure
          # out what lines to look for.
          try:
              f = open(adocFileName)
              state = 0
              messageNumberStr = None
              messageText = None
              for line in f:
                  line = line.rstrip('\n')  # get rid of trailing return character
                  if line.startswith('== SQL '):
                      words = line.split()
                      if len(words) == 3:
                          try:
                              messageNumber = int(words[2])
                          except ValueError:
                              messageNumber = 0
                          if messageNumber > 0:
                              messageNumberStr = words[2]
                              state = 1 # look for first backticks
                  elif state == 1:
                      if line == '```':
                          state = 2 # start capturing text
                  elif state == 2:
                      messageText = line
                      state = 3 # continue capturing text until backticks
                  elif state == 3:
                      if line == '```':
                          state = 0 # look for next heading
                          values = { 'messageGuideText': messageText } 
                          self.mergeEntry(messageNumberStr,values)
                      else:
                          messageText = ' '.join([messageText,line])                    
              f.close()
           
          except IOError as detail:
              print "Could not open " + adocFileName
              print detail
  
  
  
      def parseMessagesFile(self,messagesFileName):
          try:
              f = open(messagesFileName)
              for line in f:
                  words = str.split(line)
                  if len(words) >= 7:
                      try:
                          messageNumber = int(words[0])
                      except ValueError:
                          messageNumber = 0
                      if messageNumber > 0:
                          # filter out "unused" messages
                          if not (words[6].startswith('--') or words[6].startswith('***')):
                              values = { 'errorMessageFileText': ' '.join(words[6:]) } 
                              self.mergeEntry(words[0],values)                    
              f.close()
           
          except IOError as detail:
              print "Could not open " + messagesFileName
              print detail
  
  
      def Cscreener(self,line):
          # remove any C or C++ comments from the line, returning the line
          # (not precise; we don't check for C strings for example)
          commentState = 0
          result = ''
          for c in line:
              if commentState == 0:
                  if c == '/':
                      commentState = 1  # seen '/'
                  else:
                      result = result + c
              elif commentState == 1:
                  if c == '/':
                      commentState = 4 # in C++ comment, ignore rest of line
                  elif c == '*':
                      commentState = 2 # in C comment, look for '*'
                  else:
                      result = result + '/' + c  # false alarm
                      commentState = 0
              elif commentState == 2 and c == '*':
                  commentState = 3 # in C comment, look for '/' ending comment
              elif commentState == 3:
                  if c == '/':
                      result = result + ' '  # so we don't glue two tokens together
                      commentState = 0
                  elif c != '*':
                      commentState = 2
          return result
                         
  
      def Ctokenize(self,line,screenOutComments):
          # break a line of text into a list of C-like tokens (not 
          # precise, just good enough for our purposes)
          if screenOutComments:
              line = self.Cscreener(line)  # remove C, C++ comments
          result = []
          currentToken = ''
          for c in line:
              if c.isspace():
                  if len(currentToken) > 0:
                      result.append(currentToken)
                      currentToken = ''
              elif c.isalnum() or c == '_':
                  currentToken = currentToken + c                
              else:
                  if len(currentToken) > 0:
                      result.append(currentToken)
                      currentToken = ''
                  result.append(c)
          return result                  
          
  
      def parseEnumFile(self,enumFileName,enumName):
          #
          # We are looking for a particular enum. The format we expect
          # is like this:
          #
          # enum <enumname> { <symbol> = <value>,
          #                   <symbol> = <value>,
          #                   ...
          #                   <symbol> = <value> } ;
          #
          # Of course this can freely flow across lines and there may be
          # C or C++ comments to navigate past. So, we essentially have
          # to tokenize and use a state machine to parse.
          state = 0
          symbol = None
          messageNumber = None
          try:
              f = open(enumFileName)
              for line in f:
                  tokens = self.Ctokenize(line,True)  # screen out comments
                  for token in tokens:
                      if state == 0 and token == 'enum':
                          state = 1
                      elif state == 1:
                          if token == enumName:
                              state = 2
                          else:
                              state = 0
                      elif state == 2:
                          if token == '{':
                              state = 3;
                          else:
                              state = 0;
                      elif state == 3:
                          if len(token) > 1: 
                              state = 4
                              symbol = token
                          elif token == '}':
                              state = 10 # ignore rest of file
                      elif state == 4:
                          if token == '=':
                              state = 5
                          elif token == '}':
                              state = 10 # ignore rest of file
                          else:
                              state = 3
                      elif state == 5:
                          if token.isdigit():
                              values = { 'enumSymbol': symbol, 'enumFile': enumFileName }
                              if int(token) > 0:  # ignore enums for 0
                                  self.mergeEntry(token,values)
                              state = 6
                          elif token == '-':
                              state = 5  # skip unary minus sign before digits
                          else:
                              state = 3
                      elif state == 6:
                          if token == '}':
                              state = 10 # ignore rest of file
                          else:  # probably a comma
                              state = 3
                    
              f.close()
           
          except IOError as detail:
              print "Could not open " + enumFileName
              print detail
  
   
      def analyzeCodeReferences(self,directory):
          fileString = directory + "/*/*.cpp " + directory + "/*/*.h " + directory + "/*/*.y" 
          for key in self.dict:
              tableEntry = self.dict[key]
              if tableEntry['textsMatch'] == False:
                  if 'enumSymbol' in tableEntry:
                      patternString = '"[' + key + "|" + tableEntry['enumSymbol'] + ']"'
                  else:
                      patternString = key
                  shellCmd = 'grep -H ' + patternString + " " + fileString
                  p1 = subprocess.Popen(shellCmd, shell=True, stdout=subprocess.PIPE, close_fds=True)
                  prevFileName = ""
                  fileName = ""
                  valueSet = set()
                  for line in p1.stdout:
                      fileName = line[:line.find(':')]
  
                      if fileName != prevFileName and len(prevFileName) > 0:
                          entry = { 'listOfCodeReferences': valueSet }
                          self.mergeEntry(key,entry)
                          prevFileName = fileName
                          valueSet = set()
                          
                      line = line[len(fileName)+1:]  # remove file name part and colon
                      tokens1 = self.Ctokenize(line,False)  # don't screen out comments
                      found = False
                      if key in tokens1:
                          found = True
                      elif 'enumSymbol' in tableEntry and tableEntry['enumSymbol'] in tokens1:
                          found = True
                      if found:                   
                          # remove directory part of the name
                          valueSet.add(fileName[len(directory):].rstrip(':').strip('/'))
         
                  # do the last one (if there was one)
                  if len(fileName) > 0:
                      entry = { 'listOfCodeReferences': valueSet }
                      self.mergeEntry(key,entry)
                  
  
  
      def analyzeTestReferences(self,directory):
          # print "analyzeTestReferences called for directory " + directory
          fileString = directory + "/*/EXPECTED*"
          for key in self.dict:
              if self.dict[key]['textsMatch'] == False:
                  patternString = "[ERROR|WARNING]\[" + key + "\]"
                  shellCmd = 'grep -l "' + patternString + '" ' + fileString
                  p1 = subprocess.Popen(shellCmd, shell=True, stdout=subprocess.PIPE, close_fds=True)
                  valueList = []
                  for fileName in p1.stdout:
                      # remove directory part of the name and trailing '\n'
                      valueList.append(fileName[len(directory):].rstrip('\n').strip('/'))
  
                  entry = { 'listOfTestReferences': valueList }
                  self.mergeEntry(key,entry)
                  
  
      def removeAngleBracketTerms(self,line):
          # replaces any text of the form "<stuff>"
          result = ''
          throwAway = ''
          i = 0
          state = 0
          while i < len(line):
              if state == 0:
                  if line[i] == '<':
                      state = 1
                      throwAway = line[i]
                  else:
                      result = result + line[i]
              elif state == 1:
                  if line[i] == '>':
                      result = result + throwAway + line[i]
                      state = 0
                  else:
                      state = 2
                      throwAway = throwAway + line[i]
              elif state == 2:
                  if line[i] == '>':
                      state = 0
                  else:
                      throwAway = throwAway + '>' + line[i]
              i = i + 1
  
          # if we reached the end of the line after a '<', put the
          # throwaway text back in
          if state == 2:
              result = result + ' <' + throwAway
          #print "Before<: " + line
          #print "After<: " + result
          return result 
  
      def removeDollarTerms(self,line):
          # removes text of the form $0~Datatype0 (where Datatype might
          # be String, Int, TableName etc.)
          result = ''
          throwAway = ''
          i = 0
          state = 0
          while i < len(line):
              if state == 0:
                  if line[i] == '$':
                      state = 1;
                      throwAway = line[i]
                  else:
                      result = result + line[i]
              elif state == 1:
                  if line[i].isdigit():
                      state = 2
                      throwAway = throwAway + line[i]
                  else:
                      result = result + throwAway + line[i]
                      throwAway = ''
                      state = 0
              elif state == 2:
                  if line[i] == '~':
                      state = 3
                      throwAway = throwAway + line[i]
                  else:
                      result = result + throwAway + line[i]
                      throwAway = ''
                      state = 0
              elif state == 3:
                  if line[i].isalpha():
                      state = 4
                      throwAway = throwAway + line[i]
                  else:
                      result = result + throwAway + line[i]
                      throwAway = ''
                      state = 0
              elif state == 4:
                  if line[i].isalpha():
                      throwAway = throwAway + line[i]
                  elif line[i].isdigit():
                      state = 0  # we reached the end of the dollar text                  
                  else:
                      result = result + line[i]
                      state = 0  # we reached the end of the dollar text
              i = i + 1
  
          # if we reached the end of the line then put the throwaway text
          # back in
          if state > 0 and state < 4:
              result = result + throwAway
          #print "Before$: " + line
          #print "After$: " + result
          return result           
  
  
      def compareText(self):
          # print "compareText called"
          for key in self.dict:
              attributes = self.dict[key]
              comparison = False
              if 'messageGuideText' in attributes:
                  if 'errorMessageFileText' in attributes:
                      if attributes['messageGuideText'] == attributes['errorMessageFileText']:
                          comparison = True
                      else:
                          temp1 = self.removeAngleBracketTerms(attributes['messageGuideText'])
                          temp2 = self.removeDollarTerms(attributes['errorMessageFileText'])
                          if temp1 == temp2:
                              comparison = True
              self.mergeEntry(key,{ 'textsMatch': comparison })
  
      
      # Iterating through a Python dictionary gets keys out in hash order
      # which isn't useful to humans. This helper function gets the keys,
      # and places them in a list in numeric order
      def sortedNumericKeys(self,dictionary):
          numericKeys = []
          for key in self.dict:
              numericKeys.append(int(key))
  
          result = sorted(numericKeys)
          return result
              
  
      def reportResults(self,withCodeRefs):
          matchedCount = 0
          mismatchedCount = 0
          for keyN in self.sortedNumericKeys(self.dict):
              key = str(keyN)
              value = self.dict[key]
              if value['textsMatch']:
                  matchedCount = matchedCount + 1
              else:
                  mismatchedCount = mismatchedCount + 1
                  if 'messageGuideText' in value:
                      if 'errorMessageFileText' in value:
                          print "Message " + key + " differs between code and Messages Guide:"
                          print "SqlciErrors.txt:    " + value['errorMessageFileText']
                          
                      else:
                          print "Message " + key + " appears in the Messages Guide but not the SqlciErrors.txt file:"
                      print "Message Guide text: " + value['messageGuideText']
                  else:
                      if 'errorMessageFileText' in value:
                          print "Message " + key + " appears in the SqlciErrors.txt file but not the Messages Guide:"
                          print "SqlciErrors.txt:    " + value['errorMessageFileText']
                      else:
                          # must exist only in an enum
                          print "Message " + key + " does not appear in either the SqlciErrors.txt file nor the Messages Guide."
                          
                  if 'enumSymbol' in value:
                      print "Enum symbol: " + value['enumSymbol'] + " (file " + value['enumFile'] + ")"
  
                  # these tests shouldn't be necessary but do make the following code safe
                  if not 'listOfCodeReferences' in value:
                      value['listOfCodeReferences'] = set()
                  if not 'listOfTestReferences' in value:
                      value['listOfTestReferences'] = []
                          
                  if withCodeRefs:
                      text = "This message "
                      if 'enumSymbol' in value:
                          text = text + "(or its enum symbol) "                            
                      if len(value['listOfCodeReferences']) == 0:
                          text = text + "does not seem to be referenced in C++ code." 
                      else:
                          text = text + "has possible references in " + str(len(value['listOfCodeReferences'])) + " files:" 
                      print text
  
                      for codeReference in value['listOfCodeReferences']:
                          print "  " + codeReference
    
                  if len(value['listOfTestReferences']) == 0:
                      print "This message does not appear to be in any regress expected file."
                  else:
                      print "This message possibly appears in the following " + str(len(value['listOfTestReferences'])) + " regress expected files:"
                      
                  for testReference in value['listOfTestReferences']:
                      print "  " + testReference
  
                  print " "
  
          print "Summary: There are " + str(matchedCount) + " matching messages and " + str(mismatchedCount) + " mismatching messages."
                          
  
  
  
  
  # beginning of main
  
  
  # process command line arguments
  
  parser = argparse.ArgumentParser(
      description='This script checks the consistency of the Messages Guide and the code.')
  parser.add_argument("--codeRefs", help='Looks for code references to mismatched messages; this option is quite slow and can take up to 10 seconds per message. Today there are about 2800 mismatched messages so count on six or seven hours.', action="store_true")
  
  args = parser.parse_args()  # exits and prints help if args are incorrect
  
  exitCode = 0
  
  messagesTable = MessagesTable()
  
  # check that $TRAF_HOME is set
  mySQroot = os.getenv('TRAF_HOME')
  if not mySQroot:
      print "$TRAF_HOME is not defined. Exiting."
      exit(1)
  
  # parse the Messages Guide files
  print
  print datetime.datetime.ctime(datetime.datetime.now()) + ": reading Messages Guide"
  messagesGuideChaptersDirectory = mySQroot + '/../../docs/messages_guide/src/asciidoc/_chapters'
  for subdir, dirs, files in os.walk(messagesGuideChaptersDirectory):
      for file in files:
          filepath = subdir + os.sep + file
          if filepath.endswith(".adoc"):
              messagesTable.parseAdoc(filepath)
  
  # parse the SqlciErrors.txt file
  print
  print datetime.datetime.ctime(datetime.datetime.now()) + ": reading SqlciErrors.txt"
  eTextFileName = mySQroot + '/../sql/bin/SqlciErrors.txt'
  messagesTable.parseMessagesFile(eTextFileName)
  
  # parse the enum files
  print
  print datetime.datetime.ctime(datetime.datetime.now()) + ": reading enum files"
  enumFileList = ( [ ['ustat/hs_const.h','USTAT_ERROR_CODES'],
      ['sqlcomp/CmpDDLCatErrorCodes.h','CatErrorCode'],
      ['optimizer/opt_error.h','OptimizerSQLErrorCode'],
      ['exp/ExpErrorEnums.h','ExeErrorCode'] ] )
  for entry in enumFileList:
      fileName = mySQroot + '/../sql/' + entry[0]
      messagesTable.parseEnumFile(fileName,entry[1])
  
  # compare Messages Guide and code text
  print
  print datetime.datetime.ctime(datetime.datetime.now()) + ": comparing Messages Guide and SqlciError.txt text"
  messagesTable.compareText()
  
  # analyze code references
  if args.codeRefs:
      print
      print datetime.datetime.ctime(datetime.datetime.now()) + ": looking for code references for mismatched messages (this may take a while)"
      sqlCodeDirectory = mySQroot + '/../sql'
      messagesTable.analyzeCodeReferences(sqlCodeDirectory)
  
  # analyze test references
  print
  print datetime.datetime.ctime(datetime.datetime.now()) + ": looking for test examples for mismatched messages (this may take a while)"
  regressDirectory = mySQroot + '/../sql/regress'
  messagesTable.analyzeTestReferences(regressDirectory)
  
  
  # report results
  print
  print datetime.datetime.ctime(datetime.datetime.now()) + ": generating report"
  print
  messagesTable.reportResults(args.codeRefs)
  
  print
  print datetime.datetime.ctime(datetime.datetime.now()) + ": done"
  
  exit(exitCode)   
  
  
  