text: #!/usr/bin/env python2
  #
  #  Check various source code policy rules and issue warnings for offenders.
  #
  #  Usage:
  #
  #    $ python check_code_policy.py src-input/*.c
  #
  
  import os
  import sys
  import re
  import optparse
  
  class Problem:
      filename = None
      linenumber = None
      line = None
      reason = None
  
      def __init__(self, filename, linenumber, line, reason):
          self.filename = filename
          self.linenumber = linenumber
          self.line = line
          self.reason = reason
  
  re_debuglog_callsite = re.compile(r'^.*?(DUK_D+PRINT).*?$')
  re_trailing_ws = re.compile(r'^.*?\s$')
  re_only_ws = re.compile(r'^\s*$')
  re_nonleading_tab = re.compile(r'^.*?[^\t]\t.*?$')  # tabs are only used for indent
  re_identifier = re.compile(r'[A-Za-z0-9_]+')
  re_nonascii = re.compile(r'^.*?[\x80-\xff].*?$')
  re_func_decl_or_def = re.compile(r'^(\w+)\s+(?:\w+\s+)*(\w+)\(.*?.*?$')  # may not finish on same line
  re_cpp_comment = re.compile(r'^.*?//.*?$')
  re_longlong_constant_dec = re.compile(r'[0-9]+(LL|ULL)')
  re_longlong_constant_hex = re.compile(r'0x[0-9a-fA-F]+(LL|ULL)')
  re_float_compare = re.compile(r'==\s*\d*\.\d+')
  
  fixmeString = 'FIX' + 'ME'  # avoid triggering a code policy check warning :)
  
  # These identifiers are wrapped in duk_config.h, and should only be used
  # through the wrappers elsewhere.
  rejected_plain_identifiers_list = [
      # math classification
      'fpclassify',
      'signbit',
      'isfinite',
      'isnan',
      'isinf',
      'FP_NAN',
      'FP_INFINITE',
      'FP_ZERO',
      'FP_SUBNORMAL',
      'FP_NORMAL',
  
      # math functions
      'fabs',
      'fmin',
      'fmax',
      'floor',
      'ceil',
      'fmod',
      'pow',
      'acos',
      'asin',
      'atan',
      'atan2',
      'sin',
      'cos',
      'tan',
      'exp',
      'log',
      'sqrt',
      'cbrt',
  
      # memory functions
      'malloc',
      'realloc',
      'calloc',
      'free',
      'memcpy',
      'memmove',
      'memcmp',
      'memset',
  
      # string functions
      'strlen',
      'strcmp',
      'strncmp',
      'printf',
      'fprintf',
      'sprintf',
      '_snprintf',
      'snprintf',
      'vsprintf',
      '_vsnprintf',
      'vsnprintf',
      'sscanf',
      'vsscanf',
  
      # streams
      'stdout',
      'stderr',
      'stdin',
  
      # file ops
      'fopen',
      'fclose',
      'fread',
      'fwrite',
      'fseek',
      'ftell',
      'fflush',
      'fputc',
  
      # misc
      'abort',
      'exit',
      'setjmp',
      'longjmp',
  
      # variable/argument names which have shadowing issues with platform headers
      # see e.g. https://github.com/svaarala/duktape/pull/810
      'index',
      'rindex',
  
      # for consistency avoid these too, use obj_idx rather than obj_index, etc
      'obj_index',
      'from_index',
      'to_index',
      'arr_index',
      'uindex',
  
      # avoid in internals, use duk_hthread exclusively (except in duktape.h.in)
      'duk_context',
  
      # avoid in internal, except for duk_util_memory.c
      'DUK_MEMCPY',
      'DUK_MEMMOVE',
      'DUK_MEMCMP',
      'DUK_MEMSET',
      'DUK_MEMZERO'
  ]
  rejected_plain_identifiers = {}
  for id in rejected_plain_identifiers_list:
      rejected_plain_identifiers[id] = True
  
  debuglog_wrappers = {
      'DUK_DPRINT': 'DUK_D',
      'DUK_DDPRINT': 'DUK_DD',
      'DUK_DDDPRINT': 'DUK_DDD'
  }
  
  allowed_visibility_macros = [
      'DUK_EXTERNAL_DECL',
      'DUK_EXTERNAL',
      'DUK_INTERNAL_DECL',
      'DUK_INTERNAL',
      'DUK_LOCAL_DECL',
      'DUK_LOCAL'
  ]
  
  problems = []
  
  re_repl_c_comments = re.compile(r'/\*.*?\*/', re.DOTALL)
  re_repl_cpp_comments = re.compile(r'//.*?\n', re.DOTALL)
  re_repl_string_literals_dquot = re.compile(r'''\"(?:\\\"|[^\"])*\"''')
  re_repl_string_literals_squot = re.compile(r'''\'(?:\\\'|[^\'])*\'''')
  re_repl_expect_strings = re.compile(r'/\*===.*?===*?\*/', re.DOTALL)
  re_not_newline = re.compile(r'[^\n]+', re.DOTALL)
  
  def repl_c(m):
      tmp = re.sub(re_not_newline, '', m.group(0))
      if tmp == '':
          tmp = ' '  # avoid /**/
      return '/*' + tmp + '*/'
  def repl_cpp(m):
      return '// removed\n'
  def repl_dquot(m):
      return '"' + ('.' * (len(m.group(0)) - 2)) + '"'
  def repl_squot(m):
      return "'" + ('.' * (len(m.group(0)) - 2)) + "'"
  
  def removeLiterals(data):
      data = re.sub(re_repl_string_literals_dquot, repl_dquot, data)
      data = re.sub(re_repl_string_literals_squot, repl_squot, data)
      return data
  
  def removeCCommentsAndLiterals(data):
      data = re.sub(re_repl_c_comments, repl_c, data)
      data = re.sub(re_repl_string_literals_dquot, repl_dquot, data)
      data = re.sub(re_repl_string_literals_squot, repl_squot, data)
      return data
  
  def removeAnyCommentsAndLiterals(data):
      data = re.sub(re_repl_c_comments, repl_c, data)
      data = re.sub(re_repl_cpp_comments, repl_cpp, data)
      data = re.sub(re_repl_string_literals_dquot, repl_dquot, data)
      data = re.sub(re_repl_string_literals_squot, repl_squot, data)
      return data
  
  def removeExpectStrings(data):
      def repl(m):
          tmp = re.sub(re_not_newline, '', m.group(0))
          if tmp == '':
              tmp = ' '  # avoid /*======*/
          return '/*===' + tmp + '===*/'
  
      data = re.sub(re_repl_expect_strings, repl, data)
      return data
  
  def checkDebugLogCalls(lines, idx, filename):
      # Allowed debug log forms:
      #
      #     DUK_D(DUK_DPRINT(...))
      #     DUK_DD(DUK_DDPRINT(...))
      #     DUK_DDD(DUK_DDDPRINT(...))
      #
      # The calls may span multiple lines, but the wrapper (DUK_D)
      # and the log macro (DUK_DPRINT) must be on the same line.
  
      line = lines[idx]
      if 'DPRINT' not in line:
          return
  
      m = re_debuglog_callsite.match(line)
      if m is None:
          return
  
      log_macro = m.group(1)
      log_wrapper = debuglog_wrappers[log_macro]
      if log_wrapper + '(' in line:
          return
  
      # exclude '#define DUK_DPRINT...' macros in duk_debug.h
      if len(line) >= 1 and line[0] == '#':
          return
  
      # exclude a few comment lines in duk_debug.h
      if len(line) >= 3 and line[0:3] == ' * ':
          return
  
      raise Exception('invalid debug log call form')
  
  def checkTrailingWhitespace(lines, idx, filename):
      line = lines[idx]
      if len(line) > 0 and line[-1] == '\n':
          line = line[:-1]
  
      m = re_trailing_ws.match(line)
      if m is None:
          return
  
      raise Exception('trailing whitespace')
  
  def checkCarriageReturns(lines, idx, filename):
      line = lines[idx]
      if not '\x0d' in line:
          return
  
      raise Exception('carriage return')
  
  def checkMixedIndent(lines, idx, filename):
      line = lines[idx]
      if not '\x20\x09' in line:
          return
  
      # Mixed tab/space are only allowed after non-whitespace characters
      idx = line.index('\x20\x09')
      tmp = line[0:idx]
      m = re_only_ws.match(tmp)
      if m is None:
          return
  
      raise Exception('mixed space/tab indent (idx %d)' % idx)
  
  def checkTabIndent(lines, idx, filename):
      line = lines[idx]
      if not '\x09' in line:
          return
  
      # Now just checks for presence of TAB characters which is fine for Python
      # code (which this check is used for).
  
      raise Exception('tab indent (idx %d)' % idx)
  
  def checkNonLeadingTab(lines, idx, filename):
      line = lines[idx]
      m = re_nonleading_tab.match(line)
      if m is None:
          return
  
      raise Exception('non-leading tab (idx %d)' % idx)
  
  def checkFixme(lines, idx, filename):
      line = lines[idx]
      if not fixmeString in line:
          return
  
      raise Exception(fixmeString + ' on line')
  
  def checkIdentifiers(lines, idx, filename):
      line = lines[idx]
      # XXX: this now executes for every line which is pointless
      bn = os.path.basename(filename)
      excludePlain = (bn[0:5] == 'test-')
  
      for m in re.finditer(re_identifier, line):
          if rejected_plain_identifiers.has_key(m.group(0)):
              if m.group(0) in [ 'duk_context' ] and bn == 'duktape.h.in':
                  continue  # duk_context allowed in public API header
              if m.group(0) in [ 'DUK_MEMCPY', 'DUK_MEMMOVE', 'DUK_MEMCMP', 'DUK_MEMSET', 'DUK_MEMZERO' ] and \
                 bn in [ 'duk_util_memory.c', 'duk_util.h' ]:
                  continue
              if not excludePlain:
                  raise Exception('invalid identifier %r (perhaps plain)' % m.group(0))
  
  def checkNonAscii(lines, idx, filename):
      line = lines[idx]
      m = re_nonascii.match(line)
      if m is None:
          return
  
      bn = os.path.basename(filename)
      if bn == 'test-lex-utf8.js':
          # this specific file is intentionally exempt
          pass
      else:
          raise Exception('non-ascii character')
  
  def checkNoSymbolVisibility(lines, idx, filename):
      line = lines[idx]
  
      # Workaround for DUK_ALWAYS_INLINE preceding a declaration
      # (e.g. "DUK_ALWAYS_INLINE DUK_LOCAL ...")
      if line.startswith('DUK_ALWAYS_INLINE '):
          line = line[18:]
  
      m = re_func_decl_or_def.match(line)
      if m is None:
          return
  
      bn = os.path.basename(filename)
      if not ((bn[-2:] == '.c' or bn[-2:] == '.h' or bn[-5:] == '.h.in') and bn[0:5] != 'test-'):
          # Apply to only specific files in src-input/
          return
  
      if m.group(1) in allowed_visibility_macros and \
         not ((m.group(1) != 'DUK_LOCAL' and m.group(1) != 'DUK_LOCAL_DECL') and 'duk__' in m.group(2)) and \
         not ((m.group(1) == 'DUK_LOCAL' or m.group(1) == 'DUK_LOCAL_DECL') and 'duk__' not in m.group(2)):
          return
  
      # Previous line may contain the declaration (alone)
      if idx > 0 and lines[idx - 1].strip() in allowed_visibility_macros:
          return
  
      # Special exceptions
      # (None now)
  
      raise Exception('missing symbol visibility macro')
  
  def checkIfdefIfndef(lines, idx, filename):
      line = lines[idx]
  
      if not line.startswith('#if'):
          return
      if line.startswith('#ifdef'):
          raise Exception('#ifdef found, #if defined is preferred')
      if line.startswith('#ifndef'):
          raise Exception('#ifndef found, #if !defined is preferred')
  
  def checkLongLongConstants(lines, idx, filename):
      line = lines[idx]
  
      if not 'LL' in line:
          return
      for m in re.finditer(re_longlong_constant_hex, line):
          raise Exception('plain longlong constant, use DUK_U64_CONSTANT or DUK_I64_CONSTANT: ' + str(m.group(0)))
      for m in re.finditer(re_longlong_constant_dec, line):
          raise Exception('plain longlong constant, use DUK_U64_CONSTANT or DUK_I64_CONSTANT: ' + str(m.group(0)))
  
  def checkCppComment(lines, idx, filename):
      line = lines[idx]
      m = re_cpp_comment.match(line)
      if m is None:
          return
  
      raise Exception('c++ comment')
  
  def checkFloatCompare(lines, idx, filename):
      line = lines[idx]
      for m in re.finditer(re_float_compare, line):
          raise Exception('float/double compare, use duk_{double,float}_equals()')
  
  def processFile(filename, checkersRaw, checkersNoCommentsOrLiterals, checkersNoCCommentsOrLiterals, checkersNoExpectStrings):
      f = open(filename, 'rb')
      dataRaw = f.read()
      f.close()
  
      dataNoCommentsOrLiterals = removeAnyCommentsAndLiterals(dataRaw)   # no C/javascript comments, literals removed
      dataNoCCommentsOrLiterals = removeCCommentsAndLiterals(dataRaw)    # no C comments, literals removed
      dataNoExpectStrings = removeExpectStrings(dataRaw)                 # no testcase expect strings
  
      linesRaw = dataRaw.split('\n')
      linesNoCommentsOrLiterals = dataNoCommentsOrLiterals.split('\n')
      linesNoCCommentsOrLiterals = dataNoCCommentsOrLiterals.split('\n')
      linesNoExpectStrings = dataNoExpectStrings.split('\n')
  
      def f(lines, checkers):
          for linenumber in xrange(len(lines)):
              for fun in checkers:
                  try:
                      fun(lines, linenumber, filename)  # linenumber is zero-based here
                  except Exception as e:
                      problems.append(Problem(filename, linenumber + 1, lines[linenumber], str(e)))
  
      f(linesRaw, checkersRaw)
      f(linesNoCommentsOrLiterals, checkersNoCommentsOrLiterals)
      f(linesNoCCommentsOrLiterals, checkersNoCCommentsOrLiterals)
      f(linesNoExpectStrings, checkersNoExpectStrings)
  
      # Last line should have a newline, and there should not be an empty line.
      # The 'split' result will have one empty string as its last item in the
      # expected case.  For a single line file there will be two split results
      # (the line itself, and an empty string).
  
      if len(linesRaw) == 0 or \
         len(linesRaw) == 1 and linesRaw[-1] != '' or \
         len(linesRaw) >= 2 and linesRaw[-1] != '' or \
         len(linesRaw) >= 2 and linesRaw[-1] == '' and linesRaw[-2] == '':
          problems.append(Problem(filename, len(linesRaw), '(no line)', 'No newline on last line or empty line at end of file'))
  
      # First line should not be empty (unless it's the only line, len(linesRaw)==2)
      if len(linesRaw) > 2 and linesRaw[0] == '':
          problems.append(Problem(filename, 1, '(no line)', 'First line is empty'))
  
  def asciiOnly(x):
      return re.sub(r'[\x80-\xff]', '#', x)
  
  def main():
      parser = optparse.OptionParser()
      parser.add_option('--dump-vim-commands', dest='dump_vim_commands', default=False, help='Dump oneline vim command')
      parser.add_option('--check-debug-log-calls', dest='check_debug_log_calls', action='store_true', default=False, help='Check debug log call consistency')
      parser.add_option('--check-carriage-returns', dest='check_carriage_returns', action='store_true', default=False, help='Check carriage returns')
      parser.add_option('--check-fixme', dest='check_fixme', action='store_true', default=False, help='Check ' + fixmeString + ' tags')
      parser.add_option('--check-non-ascii', dest='check_non_ascii', action='store_true', default=False, help='Check non-ASCII characters')
      parser.add_option('--check-no-symbol-visibility', dest='check_no_symbol_visibility', action='store_true', default=False, help='Check for missing symbol visibility macros')
      parser.add_option('--check-rejected-identifiers', dest='check_rejected_identifiers', action='store_true', default=False, help='Check for rejected identifiers like plain "printf()" calls')
      parser.add_option('--check-trailing-whitespace', dest='check_trailing_whitespace', action='store_true', default=False, help='Check for trailing whitespace')
      parser.add_option('--check-mixed-indent', dest='check_mixed_indent', action='store_true', default=False, help='Check for mixed indent (space and tabs)')
      parser.add_option('--check-tab-indent', dest='check_tab_indent', action='store_true', default=False, help='Check for tab indent')
      parser.add_option('--check-nonleading-tab', dest='check_nonleading_tab', action='store_true', default=False, help='Check for non-leading tab characters')
      parser.add_option('--check-cpp-comment', dest='check_cpp_comment', action='store_true', default=False, help='Check for c++ comments ("// ...")')
      parser.add_option('--check-float-compare', dest='check_float_compare', action='store_true', default=False, help='Check for floating point comparisons (== 1.23)')
      parser.add_option('--check-ifdef-ifndef', dest='check_ifdef_ifndef', action='store_true', default=False, help='Check for #ifdef and #ifndef (prefer #if defined and #if !defined)')
      parser.add_option('--check-longlong-constants', dest='check_longlong_constants', action='store_true', default=False, help='Check for plain 123LL or 123ULL constants')
      parser.add_option('--fail-on-errors', dest='fail_on_errors', action='store_true', default=False, help='Fail on errors (exit code != 0)')
  
      (opts, args) = parser.parse_args()
  
      checkersRaw = []
      if opts.check_debug_log_calls:
          checkersRaw.append(checkDebugLogCalls)
      if opts.check_carriage_returns:
          checkersRaw.append(checkCarriageReturns)
      if opts.check_fixme:
          checkersRaw.append(checkFixme)
      if opts.check_non_ascii:
          checkersRaw.append(checkNonAscii)
      if opts.check_no_symbol_visibility:
          checkersRaw.append(checkNoSymbolVisibility)
      if opts.check_ifdef_ifndef:
          checkersRaw.append(checkIfdefIfndef)
      if opts.check_longlong_constants:
          checkersRaw.append(checkLongLongConstants)
  
      checkersNoCCommentsOrLiterals = []
      if opts.check_cpp_comment:
          checkersNoCCommentsOrLiterals.append(checkCppComment)
      if opts.check_float_compare:
          checkersNoCCommentsOrLiterals.append(checkFloatCompare)
  
      checkersNoCommentsOrLiterals = []
      if opts.check_rejected_identifiers:
          checkersNoCommentsOrLiterals.append(checkIdentifiers)
  
      checkersNoExpectStrings = []
      if opts.check_trailing_whitespace:
          checkersNoExpectStrings.append(checkTrailingWhitespace)
      if opts.check_mixed_indent:
          checkersNoExpectStrings.append(checkMixedIndent)
      if opts.check_tab_indent:
          checkersNoExpectStrings.append(checkTabIndent)
      if opts.check_nonleading_tab:
          checkersNoExpectStrings.append(checkNonLeadingTab)
  
      for filename in args:
          processFile(filename, checkersRaw, checkersNoCommentsOrLiterals, checkersNoCCommentsOrLiterals, checkersNoExpectStrings)
  
      if len(problems) > 0:
          for i in problems:
              tmp = 'vim +' + str(i.linenumber)
              while len(tmp) < 10:
                  tmp = tmp + ' '
              tmp += ' ' + str(i.filename) + ' : ' + str(i.reason)
              while len(tmp) < 80:
                  tmp = tmp + ' '
              tmp += ' - ' + asciiOnly(i.line.strip())
              print(tmp)
  
          print '*** Total: %d problems' % len(problems)
  
          if opts.dump_vim_commands:
              cmds = []
              for i in problems:
                  cmds.append('vim +' + str(i.linenumber) + ' "' + i.filename + '"')
              print ''
              print('; '.join(cmds))
  
          if opts.fail_on_errors:
              sys.exit(1)
  
      sys.exit(0)
  
  if __name__ == '__main__':
      main()
  