text: #
  # Copyright (c) 2018 nexB Inc. and others. All rights reserved.
  # http://nexb.com and https://github.com/nexB/scancode-toolkit/
  # The ScanCode software is licensed under the Apache License version 2.0.
  # Data generated with ScanCode require an acknowledgment.
  # ScanCode is a trademark of nexB Inc.
  #
  # You may not use this software except in compliance with the License.
  # You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
  # Unless required by applicable law or agreed to in writing, software distributed
  # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
  # CONDITIONS OF ANY KIND, either express or implied. See the License for the
  # specific language governing permissions and limitations under the License.
  #
  # When you publish or redistribute any data created with ScanCode or any ScanCode
  # derivative work, you must accompany this data with the following acknowledgment:
  #
  #  Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
  #  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
  #  ScanCode should be considered or used as legal advice. Consult an Attorney
  #  for any legal advice.
  #  ScanCode is a free software code scanning tool from nexB Inc. and others.
  #  Visit https://github.com/nexB/scancode-toolkit/ for support and download.
  
  from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  from __future__ import unicode_literals
  
  from collections import Counter
  from collections import defaultdict
  from collections import OrderedDict
  from functools import partial
  from itertools import chain
  import io
  from operator import itemgetter
  from os.path import abspath
  from os.path import dirname
  from os.path import exists
  from os.path import join
  import traceback
  
  import attr
  from license_expression import Licensing
  
  from commoncode.fileutils import copyfile
  from commoncode.fileutils import file_base_name
  from commoncode.fileutils import file_name
  from commoncode.fileutils import resource_iter
  from commoncode import saneyaml
  from licensedcode import MIN_MATCH_HIGH_LENGTH
  from licensedcode import MIN_MATCH_LENGTH
  from licensedcode import SMALL_RULE
  from licensedcode.tokenize import query_tokenizer
  from textcode.analysis import numbered_text_lines
  
  
  """
  Reference License and license Rule structures persisted as a combo of a YAML
  data file and one or more text files containing license or notice texts.
  """
  
  # Set to True to print more detailed representations of objects when tracing
  TRACE_REPR = False
  
  # these are globals but always side-by-side with the code so do no not move them around
  data_dir = join(abspath(dirname(__file__)), 'data')
  licenses_data_dir = join(data_dir, 'licenses')
  rules_data_dir = join(data_dir, 'rules')
  
  
  FOSS_CATEGORIES = set([
      'Copyleft',
      'Copyleft Limited',
      'Patent License',
      'Permissive',
      'Public Domain',
      'Source-available',
      'Free Restricted',
  ])
  
  
  OTHER_CATEGORIES = set([
      'Commercial',
      'Free Restricted',
      'Proprietary Free',
      'Unstated License',
  ])
  
  
  CATEGORIES = FOSS_CATEGORIES | OTHER_CATEGORIES
  
  
  @attr.s(slots=True)
  class License(object):
      """
      A license consists of these files, where <key> is the license key:
          - <key>.yml : the license data in YAML
          - <key>.LICENSE: the license text
  
      A License object is identified by a unique `key` and its data stored in the
      `src_dir` directory. Key is a lower-case unique ascii string.
      """
  
      __attrib = partial(attr.ib, repr=False)
  
      # unique key: lower case ASCII characters, digits, underscore and dots.
      key = attr.ib(default=None, repr=True)
  
      src_dir = __attrib(default=licenses_data_dir)
  
      # if this is a deprecated license, add also notes explaining why
      is_deprecated = __attrib(default=False)
  
      # if this license text is not in English, set this field to a two letter
      # ISO 639-1 language code https://en.wikipedia.org/wiki/ISO_639-1
      # NOTE: this is not yet supported.
      # NOTE: each translation of a license text MUST have a different license key
      language = __attrib(default='en')
  
      # commonly used short name, often abbreviated.
      short_name = __attrib(default=None)
      # full name.
      name = __attrib(default=None)
  
      # Permissive, Copyleft, etc
      category = __attrib(default=None)
  
      owner = __attrib(default=None)
      homepage_url = __attrib(default=None)
      notes = __attrib(default=None)
  
      # if this is a license exception, the license key this exception applies to
      is_exception = __attrib(default=False)
  
      # SPDX key for SPDX licenses
      spdx_license_key = __attrib(default=None)
      # list of other keys, such as deprecated ones
      other_spdx_license_keys = __attrib(default=attr.Factory(list))
  
      # Various URLs for info
      text_urls = __attrib(default=attr.Factory(list))
      osi_url = __attrib(default=None)
      faq_url = __attrib(default=None)
      other_urls = __attrib(default=attr.Factory(list))
  
      # various alternate keys for this license
      key_aliases = __attrib(default=attr.Factory(list))
  
      minimum_coverage = __attrib(default=0)
      standard_notice = __attrib(default=None)
  
      # lists of copuyrights, emails and URLs that can be ignored when detected
      # in this license as they are part of the license or rule text itself
      ignorable_copyrights = __attrib(default=attr.Factory(list))
      ignorable_authors = __attrib(default=attr.Factory(list))
      ignorable_holders = __attrib(default=attr.Factory(list))
      ignorable_urls = __attrib(default=attr.Factory(list))
      ignorable_emails = __attrib(default=attr.Factory(list))
  
      # data file paths and known extensions
      data_file = __attrib(default=None)
      text_file = __attrib(default=None)
  
      def __attrs_post_init__(self, *args, **kwargs):
  
          if self.src_dir:
              self.set_file_paths()
  
              if exists(self.data_file):
                  self.load()
  
      def set_file_paths(self):
          self.data_file = join(self.src_dir, self.key + '.yml')
          self.text_file = join(self.src_dir, self.key + '.LICENSE')
  
      def relocate(self, target_dir, new_key=None):
          """
          Return f copy of this license object relocated to f new `src_dir`.
          The data and license text files are persisted in the new `src_dir`.
          """
          if not target_dir or target_dir == self.src_dir:
              raise ValueError(
                  'Cannot relocate {} License to empty directory or same directory.'.format(self.key))
  
          if new_key:
              key = new_key
          else:
              key = self.key
  
          newl = License(key, target_dir)
  
          # copy fields
          excluded_fields = ('key', 'src_dir', 'data_file', 'text_file',)
          all_fields = attr.fields(self.__class__)
          attrs = [f.name for f in all_fields if f.name not in excluded_fields]
          for name in attrs:
              setattr(newl, name, getattr(self, name))
  
          # save it all to files
          if self.text:
              copyfile(self.text_file, newl.text_file)
          newl.dump()
          return newl
  
      def update(self, mapping):
          for k, v in mapping.items():
              setattr(self, k, v)
  
      def __copy__(self):
          oldl = self.to_dict()
          newl = License(key=self.key)
          newl.update(oldl)
          return newl
  
      @property
      def text(self):
          """
          License text, re-loaded on demand.
          """
          return self._read_text(self.text_file)
  
      def to_dict(self):
          """
          Return an OrderedDict of license data (excluding texts).
          Fields with empty values are not included.
          """
  
          # do not dump false, empties and paths
          def dict_fields(attr, value):
              if not value:
                  return False
  
              if attr.name in ('data_file', 'text_file', 'src_dir',):
                  return False
  
              # default to English
              if attr.name == 'language' and value == 'en':
                  return False
  
              if attr.name == 'minimum_coverage' and value == 100:
                  return False
              return True
  
          data = attr.asdict(self, filter=dict_fields, dict_factory=OrderedDict)
          cv = data.get('minimum_coverage')
          if cv and isinstance(cv, float) and int(cv) == cv:
              cv = int(cv)
              data['minimum_coverage'] = cv
          return data
  
      def dump(self):
          """
          Dump a representation of this license as two files:
           - <key>.yml : the license data in YAML
           - <key>.LICENSE: the license text
          """
  
          def write(location, byte_string):
              # we write as binary because rules and licenses texts and data are UTF-8-encoded bytes
              with io.open(location, 'wb') as of:
                  of.write(byte_string)
  
          as_yaml = saneyaml.dump(self.to_dict(), indent=4, encoding='utf-8')
          write(self.data_file, as_yaml)
          if self.text:
              write(self.text_file, self.text.encode('utf-8'))
  
      def load(self):
          """
          Populate license data from a YAML file stored in of self.src_dir.
          Does not load text files.
          Unknown fields are ignored and not bound to the License object.
          """
          try:
              with io.open(self.data_file, encoding='utf-8') as f:
                  data = saneyaml.load(f.read())
  
              numeric_keys = ('minimum_coverage', 'relevance')
              for k, v in data.items():
                  if k in numeric_keys:
                      v = int(v)
  
                  if k == 'key':
                      assert self.key == v, 'Inconsistent YAML key and file names for %r' % self.key
  
                  setattr(self, k, v)
  
          except Exception as e:
              # this is a rare case: fail loudly
              print()
              print('#############################')
              print('INVALID LICENSE YAML FILE:', 'file://' + self.data_file)
              print('#############################')
              print(e)
              print('#############################')
              raise
  
      def _read_text(self, location):
          if not exists(location):
              text = ''
          else:
              with io.open(location, encoding='utf-8') as f:
                  text = f.read()
          return text
  
      def spdx_keys(self):
          """
          Yield SPDX keys for this license.
          """
          if self.spdx_license_key:
              yield self.spdx_license_key
          for key in self.other_spdx_license_keys:
              yield key
  
      @staticmethod
      def validate(licenses, verbose=False, no_dupe_urls=False):
          """
          Check that licenses are valid. `licenses` is a mapping of key ->
          License. Return dictionaries of infos, errors and warnings mapping a
          license key to validation issue messages. Print messages if verbose is
          True.
  
          NOTE: we DO NOT run this validation as part of the loading or
          construction of License objects. Instead this is invoked ONLY as part of
          the test suite.
          """
          infos = defaultdict(list)
          warnings = defaultdict(list)
          errors = defaultdict(list)
  
          # used for global dedupe of texts
          by_spdx_key = defaultdict(list)
          by_text = defaultdict(list)
          by_short_name = defaultdict(list)
          by_name = defaultdict(list)
  
          for key, lic in licenses.items():
              warn = warnings[key].append
              info = infos[key].append
              error = errors[key].append
              by_name[lic.name].append(lic)
              by_short_name[lic.short_name].append(lic)
  
              if not lic.short_name:
                  error('No short name')
              if not lic.name:
                  error('No name')
              if not lic.category:
                  error('No category')
              if lic.category and lic.category not in CATEGORIES:
                  cats = '\n'.join(sorted(CATEGORIES))
                  error('Unknown license category: {}.\nUse one of these valid categories:\n{}'.format(lic.category, cats))
              if not lic.owner:
                  error('No owner')
  
              # URLS dedupe and consistency
              if no_dupe_urls:
                  if lic.text_urls and not all(lic.text_urls):
                      warn('Some empty text_urls values')
  
                  if lic.other_urls and not all(lic.other_urls):
                      warn('Some empty other_urls values')
  
                  # redundant URLs used multiple times
                  if lic.homepage_url:
                      if lic.homepage_url in lic.text_urls:
                          warn('Homepage URL also in text_urls')
                      if lic.homepage_url in lic.other_urls:
                          warn('Homepage URL also in other_urls')
                      if lic.homepage_url == lic.faq_url:
                          warn('Homepage URL same as faq_url')
                      if lic.homepage_url == lic.osi_url:
                          warn('Homepage URL same as osi_url')
  
                  if lic.osi_url or lic.faq_url:
                      if lic.osi_url == lic.faq_url:
                          warn('osi_url same as faq_url')
  
                  all_licenses = lic.text_urls + lic.other_urls
                  for url in lic.osi_url, lic.faq_url, lic.homepage_url:
                      if url:
                          all_licenses.append(url)
  
                  if not len(all_licenses) == len(set(all_licenses)):
                      warn('Some duplicated URLs')
  
              # local text consistency
              text = lic.text
  
              license_qtokens = tuple(query_tokenizer(text))
              if not license_qtokens:
                  info('No license text')
              else:
                  # for global dedupe
                  by_text[license_qtokens].append(key + ': TEXT')
  
              # SPDX consistency
              if lic.spdx_license_key:
                  by_spdx_key[lic.spdx_license_key].append(key)
              for oslk in lic.other_spdx_license_keys:
                  by_spdx_key[oslk].append(key)
  
          # global SPDX consistency
          multiple_spdx_keys_used = {k: v for k, v in by_spdx_key.items() if len(v) > 1}
          if multiple_spdx_keys_used:
              for k, lkeys in multiple_spdx_keys_used.items():
                  errors['GLOBAL'].append('SPDX key: ' + k + ' used in multiple licenses: ' + ', '.join(sorted(lkeys)))
  
          # global text dedupe
          multiple_texts = {k: v for k, v in by_text.items() if len(v) > 1}
          if multiple_texts:
              for k, msgs in multiple_texts.items():
                  errors['GLOBAL'].append('Duplicate texts in multiple licenses:' + ', '.join(sorted(msgs)))
  
          # global short_name dedupe
          for short_name, licenses in by_short_name.items():
              if len(licenses) == 1:
                  continue
              errors['GLOBAL'].append('Duplicate short name:' + short_name + ' in licenses:' + ', '.join(l.key for l in licenses))
  
          # global name dedupe
          for name, licenses in by_name.items():
              if len(licenses) == 1:
                  continue
              errors['GLOBAL'].append('Duplicate name:' + name + ' in licenses:' + ', '.join(l.key for l in licenses))
  
          errors = {k: v for k, v in errors.items() if v}
          warnings = {k: v for k, v in warnings.items() if v}
          infos = {k: v for k, v in infos.items() if v}
  
          if verbose:
              print('Licenses validation errors:')
              for key, msgs in sorted(errors.items()):
                  print('ERRORS for:', key, ':', '\n'.join(msgs))
  
              print('Licenses validation warnings:')
              for key, msgs in sorted(warnings.items()):
                  print('WARNINGS for:', key, ':', '\n'.join(msgs))
  
              print('Licenses validation infos:')
              for key, msgs in sorted(infos.items()):
                  print('INFOS for:', key, ':', '\n'.join(msgs))
  
          return errors, warnings, infos
  
  
  def load_licenses(licenses_data_dir=licenses_data_dir , with_deprecated=False):
      """
      Return a mapping of key -> license objects, loaded from license files.
      Raise Exceptions if there are dangling orphaned files.
      """
      licenses = {}
      used_files = set()
      all_files = set(resource_iter(licenses_data_dir, with_dirs=False))
      for data_file in sorted(all_files):
          if data_file.endswith('.yml'):
              key = file_base_name(data_file)
              lic = License(key, licenses_data_dir)
              used_files.add(data_file)
              if exists(lic.text_file):
                  used_files.add(lic.text_file)
              if not with_deprecated and lic.is_deprecated:
                  continue
              licenses[key] = lic
  
      dangling = all_files.difference(used_files)
      if dangling:
          msg = 'Some License data or text files are orphaned in "{}".\n'.format(licenses_data_dir)
          msg += '\n'.join('file://{}'.format(f) for f in sorted(dangling))
          raise Exception(msg)
      return licenses
  
  
  def get_rules(licenses_data_dir=licenses_data_dir, rules_data_dir=rules_data_dir):
      """
      Yield Rule objects loaded from license files found in `licenses_data_dir`
      and rule files fourn in `rules_data_dir`. Raise a Exceptions if a rule is
      inconsistent or incorrect.
      """
      from licensedcode.cache import get_licenses_db
      licenses = get_licenses_db(licenses_data_dir=licenses_data_dir)
      rules = list(load_rules(rules_data_dir=rules_data_dir))
      check_rules_integrity(rules, licenses)
      licenses_as_rules = build_rules_from_licenses(licenses)
      return chain(licenses_as_rules, rules)
  
  
  class MissingLicenses(Exception):
      pass
  
  class MissingFlags(Exception):
      pass
  
  
  def check_rules_integrity(rules, licenses_by_key):
      """
      Given a lists of `rules`, check that all the rule license keys reference a
      known license from a mapping of `licenses_by_key `(key->license). Raise a
      MissingLicense exception with a message containing the list of rule files
      without a corresponding license.
      """
      invalid_rules = defaultdict(set)
      rules_without_flags = set()
      for rule in rules:
          unknown_keys = [key for key in rule.license_keys()
                          if key not in licenses_by_key]
          if unknown_keys:
              invalid_rules[rule.data_file].update(unknown_keys)
  
          if not rule.has_flags and not (rule.is_negative or rule.is_false_positive):
              rules_without_flags.add(rule.data_file)
  
      if invalid_rules:
          invalid_rules = (
              ' '.join(keys) + '\n' +
              'file://' + data_file + '\n' +
              'file://' + data_file.replace('.yml', '.RULE') + '\n'
          for data_file, keys in invalid_rules.items() if keys)
          msg = 'Rules referencing missing licenses:\n' + '\n'.join(sorted(invalid_rules))
          raise MissingLicenses(msg)
  
      if rules_without_flags:
          invalid_rules = (
              'file://' + data_file + '\n' +
              'file://' + data_file.replace('.yml', '.RULE') + '\n'
          for data_file in sorted(rules_without_flags))
          msg = 'Rules without is_license_xxx flags:\n' + '\n'.join(sorted(invalid_rules))
          raise MissingFlags(msg)
  
  
  def build_rules_from_licenses(licenses):
      """
      Return an iterable of rules built from each license text from a `licenses`
      iterable of license objects.
      """
      for license_key, license_obj in licenses.items():
          text_file = join(license_obj.src_dir, license_obj.text_file)
          if exists(text_file):
              minimum_coverage = license_obj.minimum_coverage or 0
              yield Rule(
                  text_file=text_file,
                  license_expression=license_key,
  
                  has_stored_relevance=False,
                  relevance=100,
  
                  has_stored_minimum_coverage=bool(minimum_coverage),
                  minimum_coverage=minimum_coverage,
  
                  is_license=True,
                  is_license_text=True,
  
                  ignorable_copyrights=license_obj.ignorable_copyrights,
                  ignorable_holders=license_obj.ignorable_holders,
                  ignorable_authors=license_obj.ignorable_authors,
                  ignorable_urls=license_obj.ignorable_urls,
                  ignorable_emails=license_obj.ignorable_emails,
              )
  
  
  def get_all_spdx_keys(licenses):
      """
      Return an iterable of SPDX license keys collected from a `licenses` iterable
      of license objects.
      """
      for lic in licenses.values():
          for spdx_key in lic.spdx_keys():
              yield spdx_key
  
  
  def get_essential_spdx_tokens():
      """
      Yield essential SPDX tokens.
      """
      yield 'spdx'
      yield 'license'
      yield 'identifier'
      yield 'licenseref'
  
  
  def get_all_spdx_key_tokens(licenses):
      """
      Yield token strings collected from a `licenses` iterable of license objects'
      SPDX license keys.
      """
      for tok in get_essential_spdx_tokens():
          yield tok
  
      for spdx_key in get_all_spdx_keys(licenses):
          for token in query_tokenizer(spdx_key):
              yield token
  
  
  def load_rules(rules_data_dir=rules_data_dir):
      """
      Return an iterable of rules loaded from rule files.
      """
      # TODO: OPTIMIZE: create a graph of rules to account for containment and
      # similarity clusters?
      seen_files = set()
      processed_files = set()
      lower_case_files = set()
      case_problems = set()
      space_problems = []
      model_errors = []
      for data_file in resource_iter(rules_data_dir, with_dirs=False):
          if data_file.endswith('.yml'):
              base_name = file_base_name(data_file)
              if ' ' in base_name:
                  space_problems.append(data_file)
              rule_file = join(rules_data_dir, base_name + '.RULE')
              try:
                  rule = Rule(data_file=data_file, text_file=rule_file)
                  yield rule
              except Exception as re:
                  model_errors.append(str(re))
              # accumulate sets to ensures we do not have illegal names or extra
              # orphaned files
              data_lower = data_file.lower()
              if data_lower in lower_case_files:
                  case_problems.add(data_lower)
              else:
                  lower_case_files.add(data_lower)
  
              rule_lower = rule_file.lower()
              if rule_lower in lower_case_files:
                  case_problems.add(rule_lower)
              else:
                  lower_case_files.add(rule_lower)
  
              processed_files.update([data_file, rule_file])
  
          if not data_file.endswith('~'):
              seen_files.add(data_file)
  
  
      unknown_files = seen_files - processed_files
      if unknown_files or case_problems or model_errors or space_problems:
          msg = ''
  
          if model_errors:
              errors = '\n'.join(model_errors)
              msg += '\nInvalid rule YAML in directory: %(rules_data_dir)r\n%(errors)s' % locals()
  
          if unknown_files:
              files = '\n'.join(sorted('file://' + f for f in unknown_files))
              msg += '\nOrphaned files in rule directory: %(rules_data_dir)r\n%(files)s' % locals()
  
          if case_problems:
              files = '\n'.join(sorted('file://' + f for f in case_problems))
              msg += '\nRule files with non-unique name ignoring casein rule directory: %(rules_data_dir)r\n%(files)s' % locals()
  
          if space_problems:
              files = '\n'.join(sorted('"file://' + f + '"' for f in space_problems))
              msg += '\nRule files name cannot contain spaces: %(rules_data_dir)r\n%(files)s' % locals()
  
          raise Exception(msg)
  
  
  @attr.s(slots=True)
  class Rule(object):
      """
      A detection rule object is a text to use for detection and corresponding
      detected licenses and metadata.
      """
      licensing = Licensing()
  
      ###########
      # FIXME: !!! TWO RULES MAY DIFFER BECAUSE THEY ARE UPDATED BY INDEXING
      ###########
  
      # optional rule id int typically assigned at indexing time
      rid = attr.ib(default=None, repr=TRACE_REPR)
  
      # unique identifier
      identifier = attr.ib(default=None)
  
      # License expression string
      license_expression = attr.ib(default=None)
  
      # License expression object, created at build time
      license_expression_object = attr.ib(default=None, repr=False)
  
      # an indication of what this rule importance is (e.g. how important is its
      # text when detected as a licensing clue) as one of several flags:
  
      # for a license full text: this provides the highest level of confidence wrt
      # detection
      is_license_text = attr.ib(default=False, repr=False)
  
      # for a license notice: this provides a strong confidence wrt detection
      is_license_notice = attr.ib(default=False, repr=False)
  
      # reference for a mere short license reference such as its bare name or a URL
      # this provides a weak confidence wrt detection
      is_license_reference = attr.ib(default=False, repr=False)
  
      # tag for a structured licensing tag such as a package manifest metadata or
      # an SPDX license identifier or similar package manifest tag
      # this provides a strong confidence wrt detection
      is_license_tag = attr.ib(default=False, repr=False)
  
      # is this rule text a false positive when matched? it will filtered out at
      # the end if matched
      is_false_positive = attr.ib(default=False, repr=False)
  
      # is this rule text a negative rule? it will be removed from the matchable
      # text the start if matched
      is_negative = attr.ib(default=False, repr=False)
  
      # is this rule text only to be matched with a minimum coverage e.g. a
      # minimum proportion of tokens as a float between 0 and 100 where 100 means
      # all tokens must be matched and a smaller value means a smaller propertion
      # of matched tokens is acceptable. this is computed unless this is provided
      # here.
      minimum_coverage = attr.ib(default=0)
      has_stored_minimum_coverage = attr.ib(default=False, repr=False)
      # same as minimum_coverage but divided/100
      _minimum_containment = attr.ib(default=0, repr=False)
  
      # Can this rule be matched if there are unknown words in its matched range?
      # The default is to allow known and unknown words. Unknown words are words
      # that do not exist in the text of any indexed license or license detection
      # rule.
      only_known_words = attr.ib(default=False)
  
      # what is the relevance of a match to this rule text? a float between 0 and
      # 100 where 100 means highly relevant and 0 menas not relevant at all.
      # For instance a match to the "gpl" or the "cpol" words have a fairly low
      # relevance as they are a weak indication of an actual license and could be
      # a false positive. In somce cases, this may even be used to discard obvious
      # false positive matches automatically.
      relevance = attr.ib(default=100)
      has_stored_relevance = attr.ib(default=False, repr=False)
  
      # The rule contains a reference to some file name that comtains the text
      referenced_filenames = attr.ib(default=attr.Factory(list), repr=False)
  
      # optional, free text
      notes = attr.ib(default=None, repr=False)
  
      # set to True if the rule is built from a .LICENSE full text
      is_license = attr.ib(default=False, repr=False)
  
      # lists of copuyrights, emails and URLs that can be ignored when detected
      # in this license as they are part of the license or rule text itself
      ignorable_copyrights = attr.ib(default=attr.Factory(list), repr=False)
      ignorable_holders = attr.ib(default=attr.Factory(list), repr=False)
      ignorable_authors = attr.ib(default=attr.Factory(list), repr=False)
      ignorable_urls = attr.ib(default=attr.Factory(list), repr=False)
      ignorable_emails = attr.ib(default=attr.Factory(list), repr=False)
  
      ###########################################################################
  
      # path to the YAML data file for this rule
      data_file = attr.ib(default=None, repr=False)
  
      # path to the rule text file
      text_file = attr.ib(default=None, repr=False)
  
      # text of this rule for special cases where the rule is not backed by a file:
      # for SPDX license expression dynamic rules or testing
      stored_text = attr.ib(default=None, repr=False)
  
      # These attributes are computed upon text loading or setting the thresholds
      ###########################################################################
  
      # lengths in tokens
      length = attr.ib(default=0)
      min_matched_length = attr.ib(default=0, repr=TRACE_REPR)
  
      high_length = attr.ib(default=0, repr=TRACE_REPR)
      min_high_matched_length = attr.ib(default=0, repr=TRACE_REPR)
  
      # lengths in unique token.
      length_unique = attr.ib(default=0, repr=TRACE_REPR)
      min_matched_length_unique = attr.ib(default=0, repr=TRACE_REPR)
  
      high_length_unique = attr.ib(default=0, repr=TRACE_REPR)
      min_high_matched_length_unique = attr.ib(default=0, repr=TRACE_REPR)
  
      is_small = attr.ib(default=False, repr=TRACE_REPR)
  
      has_computed_thresholds = attr.ib(default=False, repr=False)
  
      def get_length(self, unique=False):
          return self.length_unique if unique else self.length
  
      def get_min_matched_length(self, unique=False):
          return (self.min_matched_length_unique if unique
                  else self.min_matched_length)
  
      def get_high_length(self, unique=False):
          return self.high_length_unique if unique else self.high_length
  
      def get_min_high_matched_length(self, unique=False):
          return (self.min_high_matched_length_unique if unique
                  else self.min_high_matched_length)
  
      def __attrs_post_init__(self, *args, **kwargs):
          if not self.text_file:
              # for SPDX or tests only
              if not self.stored_text :
                  raise Exception('Invalid rule without its corresponding text file: {}'.format(self))
              self.identifier = '_tst_' + str(len(self.stored_text))
          else:
              self.identifier = file_name(self.text_file)
  
          if self.data_file:
              try:
                  self.load()
              except Exception as e:
                  data_file = self.data_file
                  trace = traceback.format_exc()
                  message = 'While loading: file://{data_file}\n{trace}'.format(**locals())
                  raise Exception(message)
  
          if self.relevance and self.relevance != 100:
              self.has_stored_relevance = True
  
          if self.minimum_coverage:
              self.has_stored_minimum_coverage = True
  
          if self.license_expression:
              try:
                  expression = self.licensing.parse(self.license_expression)
              except:
                  raise Exception(
                      'Unable to parse License rule expression: '
                      +repr(self.license_expression) + ' for: file://' + self.data_file +
                      '\n' + traceback.format_exc()
                  )
              if expression is None:
                  raise Exception(
                      'Unable to parse License rule expression: '
                      +repr(self.license_expression) + ' for: file://' + self.data_file)
  
              self.license_expression = expression.render()
              self.license_expression_object = expression
  
      def tokens(self):
          """
          Return an iterable of token strings for this rule. Length, relevance and
          minimum_coverage may be recomputed as a side effect.
          """
          length = 0
          text = self.text()
          text = text.strip()
  
          # FIXME: this is weird:
  
          # We tag this rule as being a bare URL if it starts with a scheme and is
          # on one line: this is used to determine a matching approach
  
          # FIXME: this does not lower the text first??
          if text.startswith(('http://', 'https://', 'ftp://')) and '\n' not in text[:1000].lower():
              self.minimum_coverage = 100
  
          for token in query_tokenizer(self.text()):
              length += 1
              yield token
  
          self.length = length
          self.compute_relevance()
  
      def text(self):
          """
          Return the rule text loaded from its file.
          """
          if self.text_file and exists(self.text_file):
              # IMPORTANT: use the same process as query text loading for symmetry
              numbered_lines = numbered_text_lines(self.text_file, demarkup=False, plain_text=True)
              return ''.join(l for _, l in numbered_lines)
  
          # used for non-file backed rules
          elif self.stored_text:
              return self.stored_text
  
          else:
              raise Exception('Inconsistent rule text for: ' +
                              self.identifier + '\nfile://' + self.text_file)
  
      def license_keys(self, unique=True):
          """
          Return a list of license keys for this rule.
          """
          if not self.license_expression:
              return []
          return self.licensing.license_keys(self.license_expression_object, unique=unique)
  
      def same_licensing(self, other):
          """
          Return True if the other rule has the same licensing as this rule.
          """
          if self.license_expression and other.license_expression:
              return self.licensing.is_equivalent(
                  self.license_expression_object, other.license_expression_object)
  
      def licensing_contains(self, other):
          """
          Return True if this rule licensing contains the other rule licensing.
          """
          if self.license_expression and other.license_expression:
              return self.licensing.contains(
                  self.license_expression_object, other.license_expression_object)
  
      def compute_thresholds(self, small_rule=SMALL_RULE):
          """
          Compute and set thresholds either considering the occurrence of all
          tokens or the occurance of unique tokens.
          """
          minimum_coverage, self.min_matched_length, self.min_high_matched_length = (
              compute_thresholds_occurences(
                  self.minimum_coverage,
                  self.length,
                  self.high_length))
          if not self.has_stored_minimum_coverage:
              self.minimum_coverage = minimum_coverage
  
          self._minimum_containment = self.minimum_coverage / 100
  
          self.min_matched_length_unique, self.min_high_matched_length_unique = (
          compute_thresholds_unique(
              self.minimum_coverage,
              self.length,
              self.length_unique, self.high_length_unique))
  
          self.is_small = self.length < small_rule
  
      def to_dict(self):
          """
          Return an ordered mapping of self, excluding texts. Used for
          serialization. Empty values are not included.
          """
          data = OrderedDict()
          if self.license_expression:
              data['license_expression'] = self.license_expression
  
          flags = (
              'is_false_positive',
              'is_negative',
              'is_license_text',
              'is_license_notice',
              'is_license_reference',
              'is_license_tag',
              'only_known_words',
          )
  
          for flag in flags:
              tag_value = getattr(self, flag, False)
              if tag_value:
                  data[flag] = tag_value
  
          if self.has_stored_relevance and self.relevance:
              rl = self.relevance
              if isinstance(rl, float) and int(rl) == rl:
                  rl = int(rl)
              data['relevance'] = rl
  
          if self.has_stored_minimum_coverage and self.minimum_coverage > 0:
              cv = self.minimum_coverage
              if isinstance(cv, float) and int(cv) == cv:
                  cv = int(cv)
              data['minimum_coverage'] = cv
  
          if self.referenced_filenames:
              data['referenced_filenames'] = self.referenced_filenames
  
          if self.notes:
              data['notes'] = self.notes
  
          if self.ignorable_copyrights:
              data['ignorable_copyrights'] = self.ignorable_copyrights
          if self.ignorable_holders:
              data['ignorable_holders'] = self.ignorable_holders
          if self.ignorable_authors:
              data['ignorable_authors'] = self.ignorable_authors
          if self.ignorable_urls:
              data['ignorable_urls'] = self.ignorable_urls
          if self.ignorable_emails:
              data['ignorable_emails'] = self.ignorable_emails
  
          return data
  
      def dump(self):
          """
          Dump a representation of this rule as two files:
           - a .yml for the rule data in YAML (self.data_file)
           - a .RULE: the rule text as a UTF-8 file (self.text_file)
          Does nothing if this rule was created from a License (e.g.
          `is_license` is True)
          """
          if self.is_license:
              return
  
          def write(location, byte_string):
              # we write as binary because rules and licenses texts and data are UTF-8-encoded bytes
              with io.open(location, 'wb') as of:
                  of.write(byte_string)
  
          if self.data_file:
              as_yaml = saneyaml.dump(self.to_dict(), indent=4, encoding='utf-8')
              write(self.data_file, as_yaml)
              write(self.text_file, self.text().encode('utf-8'))
  
      def load(self):
          """
          Load self from a .RULE YAML file stored in self.data_file.
          Does not load the rule text file.
          Unknown fields are ignored and not bound to the Rule object.
          """
          try:
              with io.open(self.data_file, encoding='utf-8') as f:
                  data = saneyaml.load(f.read())
          except Exception as e:
              print('#############################')
              print('INVALID LICENSE RULE FILE:', 'file://' + self.data_file)
              print('#############################')
              print(e)
              print('#############################')
              # this is a rare case, but yes we abruptly stop.
              raise e
          known_attributes = set(attr.fields_dict(self.__class__))
          data_file_attributes = set(data)
          unknown_attributes = data_file_attributes.difference(known_attributes)
          if unknown_attributes:
              unknown_attributes = ', '.join(sorted(unknown_attributes))
              msg = 'License rule {} data file has unknown attributes: {}'
              raise Exception(msg.format(self, unknown_attributes))
  
          self.license_expression = data.get('license_expression')
          self.is_negative = data.get('is_negative', False)
          self.is_false_positive = data.get('is_false_positive', False)
  
          if not self.license_expression and not (self.is_negative or self.is_false_positive):
              msg = 'License rule {} is missing a license_expression.'
              raise Exception(msg.format(self))
  
          relevance = float(data.get('relevance', 0))
          if relevance:
              if relevance <= 0 or relevance > 100:
                  msg = ('License rule {} data file has an invalid relevance. '
                         'Should be above 0 and 100 or less: {}')
                  raise Exception(msg.format(self, repr(relevance)))
              # Keep track if we have a stored relevance of not.
              self.relevance = relevance
              self.has_stored_relevance = True
  
          self.minimum_coverage = float(data.get('minimum_coverage', 0))
          self._minimum_containment = self.minimum_coverage / 100
  
          if not (0 <= self.minimum_coverage <= 100):
              msg = (
                  'License rule {} data file has an invalid minimum_coverage. '
                  'Should be between 0 and 100: {}')
              raise Exception(msg.format(self, self.minimum_coverage))
  
          self.is_license_text = data.get('is_license_text', False)
          self.is_license_notice = data.get('is_license_notice', False)
          self.is_license_tag = data.get('is_license_tag', False)
          self.is_license_reference = data.get('is_license_reference', False)
          self.only_known_words = data.get('only_known_words', False)
          self.referenced_filenames = data.get('referenced_filenames', []) or []
          if not isinstance(self.referenced_filenames, list):
              msg = (
                  'License rule {} data file has an invalid referenced_filenames. '
                  'Should be a list: {}')
              raise Exception(msg.format(self, self.referenced_filenames))
  
          # these are purely informational and not used at run time
          notes = data.get('notes')
          if notes:
              self.notes = notes.strip()
  
          if not self.notes and (self.is_negative or self.is_false_positive):
              msg = 'Special License rule {} is missing explanatory notes.'
              raise Exception(msg.format(self))
  
          self.ignorable_copyrights = data.get('ignorable_copyrights', [])
          self.ignorable_holders = data.get('ignorable_holders', [])
          self.ignorable_authors = data.get('ignorable_authors', [])
          self.ignorable_urls = data.get('ignorable_urls', [])
          self.ignorable_emails = data.get('ignorable_emails', [])
  
          return self
  
      def compute_relevance(self):
          """
          Compute and set the `relevance` attribute for this rule. The
          relevance is a float between 0 and 100 where 100 means highly
          relevant and 0 means not relevant at all.
  
          For instance a match to the "gpl" or the "cpol" words have a fairly low
          relevance as they are a weak indication of an actual license and could be a
          false positive and should therefore be assigned a low relevance. In contrast
          a match to most or all of the apache-2.0 license text is highly relevant. The
          Rule relevance is used as the basis to compute a match score.
  
          The relevance is either pre-defined in the rule YAML data file with the
          "relevance" attribute or computed base on the rule length here using
          this approach:
  
          - a false positive or a negative rule has a relevance of zero.
          - a rule of length equal to or larger than a threshold has a 100 relevance
          - a rule of length smaller than a threshold has a relevance of
            100/threshold, rounded down.
  
          The current threshold is 18 words.
          """
  
          if isinstance(self, SpdxRule):
              self.relevance = 100
              return
  
          if self.has_stored_relevance:
              return
  
          # case for false positive
          if self.is_false_positive:
              self.relevance = 100
              return
  
          # case for negative rules with no license (and are not an FP)
          # they do not have licenses and their matches are never returned
          if self.is_negative:
              self.relevance = 100
              return
  
          threshold = 18.0
          relevance_of_one_word = round((1 / threshold) * 100, 2)
          length = self.length
          if length >= threshold:
              # general case
              self.relevance = 100
          else:
              computed = int(length * relevance_of_one_word)
              self.relevance = min([100, computed])
  
      @property
      def has_flags(self):
          """
          Return True if this Rule has at least one flag set.
          """
          return (self.is_license_text or self.is_license_notice
              or self.is_license_reference or self.is_license_tag)
  
  
  def compute_thresholds_occurences(minimum_coverage, length, high_length,
          _MIN_MATCH_HIGH_LENGTH=MIN_MATCH_HIGH_LENGTH,
          _MIN_MATCH_LENGTH=MIN_MATCH_LENGTH):
      """
      Compute and return thresholds considering the occurrence of all tokens.
      """
      if minimum_coverage == 100:
          min_matched_length = length
          min_high_matched_length = high_length
          return minimum_coverage, min_matched_length, min_high_matched_length
  
      if length < 3:
          min_high_matched_length = high_length
          min_matched_length = length
          minimum_coverage = 100
  
      elif length < 10:
          min_matched_length = length
          min_high_matched_length = high_length
          minimum_coverage = 80
  
      elif length < 30:
          min_matched_length = length // 2
          min_high_matched_length = min(high_length, _MIN_MATCH_HIGH_LENGTH)
          minimum_coverage = 50
  
      elif length < 200:
          min_matched_length = _MIN_MATCH_LENGTH
          min_high_matched_length = min(high_length, _MIN_MATCH_HIGH_LENGTH)
          # minimum_coverage = max(15, int(length//10))
  
      else:  # if length >= 200:
          min_matched_length = length // 10
          min_high_matched_length = high_length // 10
          # minimum_coverage = int(length//10)
  
      return minimum_coverage, min_matched_length, min_high_matched_length
  
  
  def compute_thresholds_unique(minimum_coverage, length, length_unique, high_length_unique,
          _MIN_MATCH_HIGH_LENGTH=MIN_MATCH_HIGH_LENGTH,
          _MIN_MATCH_LENGTH=MIN_MATCH_LENGTH):
      """
      Compute and set thresholds considering the occurrence of only unique tokens.
      """
      if minimum_coverage == 100:
          min_matched_length_unique = length_unique
          min_high_matched_length_unique = high_length_unique
          return min_matched_length_unique, min_high_matched_length_unique
  
      if length > 200:
          min_matched_length_unique = length // 10
          min_high_matched_length_unique = high_length_unique // 10
  
      elif length < 5:
          min_matched_length_unique = length_unique
          min_high_matched_length_unique = high_length_unique
  
      elif length < 10:
          if length_unique < 2:
              min_matched_length_unique = length_unique
          else:
              min_matched_length_unique = length_unique - 1
          min_high_matched_length_unique = high_length_unique
  
      elif length < 20:
          min_matched_length_unique = high_length_unique
          min_high_matched_length_unique = high_length_unique
  
      else:
          min_matched_length_unique = _MIN_MATCH_LENGTH
          highu = (int(high_length_unique // 2)) or high_length_unique
          min_high_matched_length_unique = min(highu, _MIN_MATCH_HIGH_LENGTH)
  
      return min_matched_length_unique, min_high_matched_length_unique
  
  
  @attr.s(slots=True, repr=False)
  class SpdxRule(Rule):
      """
      A specialized rule object that is used for the special case of SPDX license
      expressions.
  
      Since we may have an infinite possible number of SPDX expressions and these
      are not backed by a traditional rule text file, we use this class to handle
      the specifics of these how rules that are built at matching time: one rule
      is created for each detected SPDX license expression.
      """
  
      def __attrs_post_init__(self, *args, **kwargs):
          self.identifier = 'spdx-license-identifier: ' + self.license_expression
          expression = None
          try:
              expression = self.licensing.parse(self.license_expression)
          except:
              raise Exception(
                  'Unable to parse License rule expression: ' +
                  repr(self.license_expression) + ' for: SPDX rule:' +
                  self.stored_text +
                  '\n' + traceback.format_exc())
          if expression is None:
              raise Exception(
                  'Unable to parse License rule expression: '
                  +repr(self.license_expression) + ' for:' + repr(self.data_file))
  
          self.license_expression = expression.render()
          self.license_expression_object = expression
          self.is_license_tag = True
          self.is_small = False
  
      def load(self):
          raise NotImplementedError
  
      def dump(self):
          raise NotImplementedError
  
  
  def _print_rule_stats():
      """
      Print rules statistics.
      """
      from licensedcode.cache import get_index
      idx = get_index()
      rules = idx.rules_by_rid
      sizes = Counter(r.length for r in rules)
      print('Top 15 lengths: ', sizes.most_common(15))
      print('15 smallest lengths: ', sorted(sizes.items(),
                                            key=itemgetter(0))[:15])
  
      high_sizes = Counter(r.high_length for r in rules)
      print('Top 15 high lengths: ', high_sizes.most_common(15))
      print('15 smallest high lengths: ', sorted(high_sizes.items(),
                                                 key=itemgetter(0))[:15])
  
  
  def update_ignorables(licensish, verbose=False, dump=True):
      """
      Collect, update and save the ignorable_* attributes of a `licensish` Rule or
      License object.
      """
      location = licensish.text_file
  
      if verbose:
          print('Processing:', 'file://' + location)
  
      if not exists(location):
          return licensish
  
      # collect and set ignorable copyrights, holders and authors
      from cluecode.copyrights import detect_copyrights
      copyrights = set()
      holders = set()
      authors = set()
  
      for dtype, value, _start, _end in detect_copyrights(location):
          if dtype == 'copyrights':
              copyrights.add(value)
          elif dtype == 'holders':
              holders.add(value)
          elif dtype == 'authors':
              authors.add(value)
  
      licensish.ignorable_copyrights = sorted(copyrights)
      licensish.ignorable_holders = sorted(holders)
      licensish.ignorable_authors = sorted(authors)
  
      # collect and set ignrable emails and urls
      from cluecode.finder import find_urls
      from cluecode.finder import find_emails
  
      urls = set(u for (u, _ln) in find_urls(location) if u)
      licensish.ignorable_urls = sorted(urls)
  
      emails = set(u for (u, _ln) in find_emails(location) if u)
      licensish.ignorable_emails = sorted(emails)
      if dump:
          licensish.dump()
      return licensish
  