files: import hashlib
  import json
  
  import datetime
  import dpath.util
  from dpath.exceptions import PathNotFound
  from copy import deepcopy
  
  from security_monkey import datastore, app
  from cloudaux.orchestration.aws.arn import ARN
  from security_monkey.datastore import Item, ItemRevision, ItemAudit
  
  prims = [int, str, unicode, bool, float, type(None)]
  
  
  def persist_item(item, db_item, technology, account, complete_hash, durable_hash, durable):
      if not db_item:
          if account.account_type.name != "AWS":
              db_item = create_item(item, technology, account)
          else:
              db_item = create_item_aws(item, technology, account)
  
      if db_item.latest_revision_complete_hash == complete_hash:
          app.logger.debug("Change persister doesn't see any change. Ignoring...")
          return
  
      # Create the new revision
      if durable:
          revision = create_revision(item.config, db_item)
          db_item.revisions.append(revision)
  
      # Ephemeral -- update the existing revision:
      else:
          revision = db_item.revisions.first()
          revision.date_last_ephemeral_change = datetime.datetime.utcnow()
          revision.config = item.config
          app.logger.debug("Persisting EPHEMERAL change to item: {technology}/{account}/{item}".format(
              technology=technology.name, account=account.name, item=db_item.name
          ))
  
      db_item.latest_revision_complete_hash = complete_hash
      db_item.latest_revision_durable_hash = durable_hash
  
      datastore.db.session.add(db_item)
      datastore.db.session.add(revision)
      datastore.db.session.commit()
  
      if durable:
          app.logger.debug("Persisting DURABLE change to item: {technology}/{account}/{item}".format(
              technology=technology.name, account=account.name, item=db_item.name
          ))
          datastore.db.session.refresh(revision)
          db_item.latest_revision_id = revision.id
          datastore.db.session.add(revision)
          datastore.db.session.add(db_item)
          datastore.db.session.commit()
  
  
  def is_active(config):
      if config.keys() == ['Arn']:
          return False
  
      if set(config.keys()) == {'account_number', 'technology', 'region', 'name'}:
          return False
  
      return True
  
  
  def create_revision(config, db_item):
      return ItemRevision(
          active=is_active(config),
          config=config,
          item_id=db_item.id,
      )
  
  
  def create_item_aws(item, technology, account):
      arn = ARN(item.config.get('Arn'))
      return Item(
          region=arn.region or 'universal',
          name=arn.parsed_name or arn.name,
          arn=item.config.get('Arn'),
          tech_id=technology.id,
          account_id=account.id
      )
  
  
  def create_item(item, technology, account):
      return Item(
          region=item.region or 'universal',
          name=item.name,
          arn=item.arn,
          tech_id=technology.id,
          account_id=account.id
      )
  
  
  def detect_change(item, account, technology, complete_hash, durable_hash):
      """
      Checks the database to see if the latest revision of the specified
      item matches what Security Monkey has pulled from AWS.
  
      Note: this method makes no distinction between a changed item, a new item,
      a deleted item, or one that only has changes in the ephemeral section.
  
      :param item: dict describing an item tracked by Security Monkey
      :param hash: hash of the item dict for quicker change detection
      :return: bool. True if the database differs from our copy of item
      """
      result = result_from_item(item, account, technology)
  
      # new item doesn't yet exist in DB
      if not result:
          app.logger.debug("Couldn't find item: {tech}/{account}/{region}/{item} in DB.".format(
              tech=item.index, account=item.account, region=item.region, item=item.name
          ))
          return True, 'durable', result, 'created'
  
      if result.latest_revision_durable_hash != durable_hash:
          app.logger.debug("Item: {tech}/{account}/{region}/{item} in DB has a DURABLE CHANGE.".format(
              tech=item.index, account=item.account, region=item.region, item=item.name
          ))
          return True, 'durable', result, 'changed'
  
      elif result.latest_revision_complete_hash != complete_hash:
          app.logger.debug("Item: {tech}/{account}/{region}/{item} in DB has an EPHEMERAL CHANGE.".format(
              tech=item.index, account=item.account, region=item.region, item=item.name
          ))
          return True, 'ephemeral', result, None
  
      else:
          app.logger.debug("Item: {tech}/{account}/{region}/{item} in DB has NO CHANGE.".format(
              tech=item.index, account=item.account, region=item.region, item=item.name
          ))
          return False, None, result, None
  
  
  def result_from_item(item, account, technology):
      # Construct the query to obtain the specific item from the database:
      return datastore.Item.query.filter(Item.name == item.name, Item.region == item.region,
                                         Item.account_id == account.id, Item.tech_id == technology.id).scalar()
  
  
  def inactivate_old_revisions(watcher, arns, account, technology):
      result = Item.query.filter(
          Item.account_id == account.id,
          Item.tech_id == technology.id,
          Item.arn.notin_(arns)
      ).join((ItemRevision, Item.latest_revision_id == ItemRevision.id)) \
          .filter(ItemRevision.active == True).all()  # noqa
  
      for db_item in result:
          app.logger.debug("Deleting {technology}/{account}/{name}".format(
              technology=technology.name, account=account.name, name=db_item.name
          ))
  
          # Create the new revision
          config = {"Arn": db_item.arn}
          revision = create_revision(config, db_item)
          db_item.revisions.append(revision)
  
          complete_hash, durable_hash = hash_item(config, watcher.ephemeral_paths)
  
          db_item.latest_revision_complete_hash = complete_hash
          db_item.latest_revision_durable_hash = durable_hash
  
          # Add the revision:
          datastore.db.session.add(db_item)
          datastore.db.session.add(revision)
          datastore.db.session.commit()
  
          # Do it again to update the latest revision ID:
          datastore.db.session.refresh(revision)
          db_item.latest_revision_id = revision.id
          datastore.db.session.add(db_item)
  
          datastore.db.session.commit()
  
      return result
  
  
  def hash_item(config, ephemeral_paths):
      """
      Finds the hash of a dict.
  
      :param ephemeral_paths:
      :param config:
      :param item: dictionary, typically representing an item tracked in SM
                   such as an IAM role
      :return: hash of the json dump of the item
      """
      complete = hash_config(config)
      durable = durable_hash(config, ephemeral_paths)
      return complete, durable
  
  
  def durable_hash(config, ephemeral_paths):
      durable_item = deepcopy(config)
      for path in ephemeral_paths:
          try:
              dpath.util.delete(durable_item, path, separator='$')
          except PathNotFound:
              pass
      return hash_config(durable_item)
  
  
  def hash_config(config):
      item = sub_dict(config)
      item_str = json.dumps(item, sort_keys=True)
      item_hash = hashlib.md5(item_str) # nosec: not used for security
      return item_hash.hexdigest()
  
  
  def sub_list(l):
      """
      Recursively walk a data-structure sorting any lists along the way.
  
      :param l: list
      :return: sorted list, where any child lists are also sorted.
      """
      r = []
  
      for i in l:
          if type(i) in prims:
              r.append(i)
          elif type(i) is list:
              r.append(sub_list(i))
          elif type(i) is dict:
              r.append(sub_dict(i))
          else:
              print "Unknown Type: {}".format(type(i))
      r = sorted(r)
      return r
  
  
  def sub_dict(d):
      """
      Recursively walk a data-structure sorting any lists along the way.
  
      :param d: dict
      :return: dict where any lists, even those buried deep in the structure, have been sorted.
      """
      r = {}
      for k in d:
          if type(d[k]) in prims:
              r[k] = d[k]
          elif type(d[k]) is list:
              r[k] = sub_list(d[k])
          elif type(d[k]) is dict:
              r[k] = sub_dict(d[k])
          else:
              print "Unknown Type: {}".format(type(d[k]))
      return r
  
files: security_monkey/datastore_utils.py
files: import hashlib
  import json
  
  import datetime
  import dpath.util
  from dpath.exceptions import PathNotFound
  from copy import deepcopy
  
  from security_monkey import datastore, app
  from cloudaux.orchestration.aws.arn import ARN
  from security_monkey.datastore import Item, ItemRevision, ItemAudit
  
  prims = [int, str, unicode, bool, float, type(None)]
  
  
  def persist_item(item, db_item, technology, account, complete_hash, durable_hash, durable):
      if not db_item:
          if account.account_type.name != "AWS":
              db_item = create_item(item, technology, account)
          else:
              db_item = create_item_aws(item, technology, account)
  
      if db_item.latest_revision_complete_hash == complete_hash:
          app.logger.debug("Change persister doesn't see any change. Ignoring...")
          return
  
      # Create the new revision
      if durable:
          revision = create_revision(item.config, db_item)
          db_item.revisions.append(revision)
  
      # Ephemeral -- update the existing revision:
      else:
          revision = db_item.revisions.first()
          revision.date_last_ephemeral_change = datetime.datetime.utcnow()
          revision.config = item.config
          app.logger.debug("Persisting EPHEMERAL change to item: {technology}/{account}/{item}".format(
              technology=technology.name, account=account.name, item=db_item.name
          ))
  
      db_item.latest_revision_complete_hash = complete_hash
      db_item.latest_revision_durable_hash = durable_hash
  
      datastore.db.session.add(db_item)
      datastore.db.session.add(revision)
      datastore.db.session.commit()
  
      if durable:
          app.logger.debug("Persisting DURABLE change to item: {technology}/{account}/{item}".format(
              technology=technology.name, account=account.name, item=db_item.name
          ))
          datastore.db.session.refresh(revision)
          db_item.latest_revision_id = revision.id
          datastore.db.session.add(revision)
          datastore.db.session.add(db_item)
          datastore.db.session.commit()
  
  
  def is_active(config):
      if config.keys() == ['Arn']:
          return False
  
      if set(config.keys()) == {'account_number', 'technology', 'region', 'name'}:
          return False
  
      return True
  
  
  def create_revision(config, db_item):
      return ItemRevision(
          active=is_active(config),
          config=config,
          item_id=db_item.id,
      )
  
  
  def create_item_aws(item, technology, account):
      arn = ARN(item.config.get('Arn'))
      return Item(
          region=arn.region or 'universal',
          name=arn.parsed_name or arn.name,
          arn=item.config.get('Arn'),
          tech_id=technology.id,
          account_id=account.id
      )
  
  
  def create_item(item, technology, account):
      return Item(
          region=item.region or 'universal',
          name=item.name,
          arn=item.arn,
          tech_id=technology.id,
          account_id=account.id
      )
  
  
  
  def detect_change(item, account, technology, complete_hash, durable_hash):
      """
      Checks the database to see if the latest revision of the specified
      item matches what Security Monkey has pulled from AWS.
  
      Note: this method makes no distinction between a changed item, a new item,
      a deleted item, or one that only has changes in the ephemeral section.
  
      :param item: dict describing an item tracked by Security Monkey
      :param hash: hash of the item dict for quicker change detection
      :return: bool. True if the database differs from our copy of item
      """
      result = result_from_item(item, account, technology)
  
      # new item doesn't yet exist in DB
      if not result:
          app.logger.debug("Couldn't find item: {tech}/{account}/{region}/{item} in DB.".format(
              tech=item.index, account=item.account, region=item.region, item=item.name
          ))
          return True, 'durable', result, 'created'
  
      if result.latest_revision_durable_hash != durable_hash:
          app.logger.debug("Item: {tech}/{account}/{region}/{item} in DB has a DURABLE CHANGE.".format(
              tech=item.index, account=item.account, region=item.region, item=item.name
          ))
          return True, 'durable', result, 'changed'
  
      elif result.latest_revision_complete_hash != complete_hash:
          app.logger.debug("Item: {tech}/{account}/{region}/{item} in DB has an EPHEMERAL CHANGE.".format(
              tech=item.index, account=item.account, region=item.region, item=item.name
          ))
          return True, 'ephemeral', result, None
  
      else:
          app.logger.debug("Item: {tech}/{account}/{region}/{item} in DB has NO CHANGE.".format(
              tech=item.index, account=item.account, region=item.region, item=item.name
          ))
          return False, None, result, None
  
  
  def result_from_item(item, account, technology):
      # Construct the query to obtain the specific item from the database:
      return datastore.Item.query.filter(Item.name == item.name, Item.region == item.region,
                                         Item.account_id == account.id, Item.tech_id == technology.id).scalar()
  
  
  def inactivate_old_revisions(watcher, arns, account, technology):
      result = Item.query.filter(
          Item.account_id == account.id,
          Item.tech_id == technology.id,
          Item.arn.notin_(arns)
      ).join((ItemRevision, Item.latest_revision_id == ItemRevision.id)) \
          .filter(ItemRevision.active == True).all()  # noqa
  
      for db_item in result:
          app.logger.debug("Deleting {technology}/{account}/{name}".format(
              technology=technology.name, account=account.name, name=db_item.name
          ))
  
          # Create the new revision
          config = {"Arn": db_item.arn}
          revision = create_revision(config, db_item)
          db_item.revisions.append(revision)
  
          complete_hash, durable_hash = hash_item(config, watcher.ephemeral_paths)
  
          db_item.latest_revision_complete_hash = complete_hash
          db_item.latest_revision_durable_hash = durable_hash
  
          # Add the revision:
          datastore.db.session.add(db_item)
          datastore.db.session.add(revision)
          datastore.db.session.commit()
  
          # Do it again to update the latest revision ID:
          datastore.db.session.refresh(revision)
          db_item.latest_revision_id = revision.id
          datastore.db.session.add(db_item)
  
          datastore.db.session.commit()
  
      return result
  
  
  def hash_item(config, ephemeral_paths):
      """
      Finds the hash of a dict.
  
      :param ephemeral_paths:
      :param config:
      :param item: dictionary, typically representing an item tracked in SM
                   such as an IAM role
      :return: hash of the json dump of the item
      """
      complete = hash_config(config)
      durable = durable_hash(config, ephemeral_paths)
      return complete, durable
  
  
  def durable_hash(config, ephemeral_paths):
      durable_item = deepcopy(config)
      for path in ephemeral_paths:
          try:
              dpath.util.delete(durable_item, path, separator='$')
          except PathNotFound:
              pass
      return hash_config(durable_item)
  
  
  def hash_config(config):
      item = sub_dict(config)
      item_str = json.dumps(item, sort_keys=True)
      item_hash = hashlib.md5(item_str) # nosec: not used for security
      return item_hash.hexdigest()
  
  
  def sub_list(l):
      """
      Recursively walk a data-structure sorting any lists along the way.
  
      :param l: list
      :return: sorted list, where any child lists are also sorted.
      """
      r = []
  
      for i in l:
          if type(i) in prims:
              r.append(i)
          elif type(i) is list:
              r.append(sub_list(i))
          elif type(i) is dict:
              r.append(sub_dict(i))
          else:
              print "Unknown Type: {}".format(type(i))
      r = sorted(r)
      return r
  
  
  def sub_dict(d):
      """
      Recursively walk a data-structure sorting any lists along the way.
  
      :param d: dict
      :return: dict where any lists, even those buried deep in the structure, have been sorted.
      """
      r = {}
      for k in d:
          if type(d[k]) in prims:
              r[k] = d[k]
          elif type(d[k]) is list:
              r[k] = sub_list(d[k])
          elif type(d[k]) is dict:
              r[k] = sub_dict(d[k])
          else:
              print "Unknown Type: {}".format(type(d[k]))
      return r
  
files: security_monkey/datastore_utils.py
files: """
  .. module: security_monkey.task_scheduler.tasks
      :platform: Unix
      :synopsis: Sets up the Celery task scheduling for watching, auditing, and reporting changes in the environment.
  
  .. version:: $$VERSION$$
  .. moduleauthor:: Mike Grima <mgrima@netflix.com>
  
  """
  import time
  import traceback
  
  from security_monkey import app, db, jirasync, sentry
  from security_monkey.alerter import Alerter
  from security_monkey.datastore import store_exception, clear_old_exceptions, Technology, Account, Item, ItemRevision
  from security_monkey.monitors import get_monitors, get_monitors_and_dependencies
  from security_monkey.reporter import Reporter
  from security_monkey.task_scheduler.util import CELERY, setup
  from sqlalchemy.exc import OperationalError, InvalidRequestError, StatementError
  
  
  @CELERY.task(bind=True, max_retries=3)
  def task_account_tech(self, account_name, technology_name):
      setup()
      app.logger.info("[ ] Executing Celery task for account: {}, technology: {}".format(account_name, technology_name))
      time1 = time.time()
  
      try:
          reporter_logic(account_name, technology_name)
  
          time2 = time.time()
          app.logger.info('[@] Run Account for Technology (%s/%s) took %0.1f s' % (account_name,
                                                                                   technology_name, (time2 - time1)))
          app.logger.info(
              "[+] Completed Celery task for account: {}, technology: {}".format(account_name, technology_name))
      except Exception as e:
          if sentry:
              sentry.captureException()
          app.logger.error("[X] Task Account Scheduler Exception ({}/{}): {}".format(account_name, technology_name, e))
          app.logger.error(traceback.format_exc())
          store_exception("scheduler-exception-on-watch", None, e)
          raise self.retry(exc=e)
  
  
  @CELERY.task(bind=True, max_retries=3)
  def task_audit(self, account_name, technology_name):
      setup()
  
      app.logger.info("[ ] Executing Celery task to audit changes for Account: {} Technology: {}".format(account_name,
                                                                                                         technology_name))
      try:
          audit_changes([account_name], [technology_name], True)
  
          app.logger.info("[+] Completed Celery task for account: {}, technology: {}".format(account_name,
                                                                                             technology_name))
  
      except Exception as e:
          if sentry:
              sentry.captureException()
          app.logger.error("[X] Task Audit Scheduler Exception ({}/{}): {}".format(account_name, technology_name, e))
          app.logger.error(traceback.format_exc())
          store_exception("scheduler-exception-on-audit", None, e)
          self.retry(exc=e)
  
  
  @CELERY.task()
  def clear_expired_exceptions():
      app.logger.info("[ ] Clearing out exceptions that have an expired TTL...")
      clear_old_exceptions()
      app.logger.info("[-] Completed clearing out exceptions that have an expired TTL.")
  
  
  def fix_orphaned_deletions(account_name, technology_name):
      """
      Possible issue with orphaned items. This will check if there are any, and will assume that the item
      was deleted. This will create a deletion change record to it.
  
      :param account_name:
      :param technology_name:
      :return:
      """
      # If technology doesn't exist, then create it:
      technology = Technology.query.filter(Technology.name == technology_name).first()
      if not technology:
          technology = Technology(name=technology_name)
          db.session.add(technology)
          db.session.commit()
          app.logger.info("Technology: {} did not exist... created it...".format(technology_name))
  
      account = Account.query.filter(Account.name == account_name).one()
  
      # Query for orphaned items of the given technology/account pair:
      orphaned_items = Item.query.filter(Item.account_id == account.id, Item.tech_id == technology.id,
                                         Item.latest_revision_id == None).all()  # noqa
  
      if not orphaned_items:
          app.logger.info("[@] No orphaned items have been found. (This is good)")
          return
  
      # Fix the orphaned items:
      for oi in orphaned_items:
          app.logger.error("[?] Found an orphaned item: {}. Creating a deletion record for it".format(oi.name))
          revision = ItemRevision(active=False, config={})
          oi.revisions.append(revision)
          db.session.add(revision)
          db.session.add(oi)
          db.session.commit()
  
          # Update the latest revision id:
          db.session.refresh(revision)
          oi.latest_revision_id = revision.id
          db.session.add(oi)
  
          db.session.commit()
          app.logger.info("[-] Created deletion record for item: {}.".format(oi.name))
  
  
  def reporter_logic(account_name, technology_name):
      """Logic for the run change reporter"""
      try:
          # Before doing anything... Look for orphaned items for this given technology. If they exist, then delete them:
          fix_orphaned_deletions(account_name, technology_name)
  
          # Watch and Audit:
          monitors = find_changes(account_name, technology_name)
  
          # Alert:
          app.logger.info("[ ] Sending alerts (if applicable) for account: {}, technology: {}".format(account_name,
                                                                                                      technology_name))
          Alerter(monitors, account=account_name).report()
      except (OperationalError, InvalidRequestError, StatementError) as e:
          app.logger.exception("[X] Database error processing account %s - technology %s cleaning up session.",
                               account_name, technology_name)
          db.session.remove()
          store_exception("scheduler-task-account-tech", None, e)
          raise e
  
  
  def manual_run_change_reporter(accounts):
      """Manual change reporting from the command line"""
      app.logger.info("[ ] Executing manual change reporter task...")
  
      try:
          for account in accounts:
              time1 = time.time()
              rep = Reporter(account=account)
  
              for monitor in rep.all_monitors:
                  if monitor.watcher:
                      app.logger.info("[ ] Running change finder for "
                                      "account: {} technology: {}".format(account, monitor.watcher.index))
                      reporter_logic(account, monitor.watcher.index)
  
              time2 = time.time()
              app.logger.info('[@] Run Account %s took %0.1f s' % (account, (time2 - time1)))
  
          app.logger.info("[+] Completed manual change reporting.")
      except (OperationalError, InvalidRequestError, StatementError) as e:
          app.logger.exception("[X] Database error processing cleaning up session.")
          db.session.remove()
          store_exception("scheduler-run-change-reporter", None, e)
          raise e
  
  
  def manual_run_change_finder(accounts, technologies):
      """Manual change finder"""
      app.logger.info("[ ] Executing manual find changes task...")
  
      try:
          for account in accounts:
              time1 = time.time()
  
              for tech in technologies:
                  find_changes(account, tech)
  
              time2 = time.time()
              app.logger.info('[@] Run Account %s took %0.1f s' % (account, (time2 - time1)))
          app.logger.info("[+] Completed manual change finder.")
      except (OperationalError, InvalidRequestError, StatementError) as e:
          app.logger.exception("[X] Database error processing cleaning up session.")
          db.session.remove()
          store_exception("scheduler-run-change-reporter", None, e)
          raise e
  
  
  def find_changes(account_name, monitor_name, debug=True):
      """
          Runs the watcher and stores the result, re-audits all types to account
          for downstream dependencies.
      """
      # Before doing anything... Look for orphaned items for this given technology. If they exist, then delete them:
      fix_orphaned_deletions(account_name, monitor_name)
  
      monitors = get_monitors(account_name, [monitor_name], debug)
      for mon in monitors:
          cw = mon.watcher
          app.logger.info("[-->] Looking for changes in account: {}, technology: {}".format(account_name, cw.index))
          if mon.batch_support:
              batch_logic(mon, cw, account_name, debug)
          else:
              # Just fetch normally...
              (items, exception_map) = cw.slurp()
              cw.find_changes(current=items, exception_map=exception_map)
              cw.save()
  
      # Batched monitors have already been monitored, and they will be skipped over.
      audit_changes([account_name], [monitor_name], False, debug)
      db.session.close()
  
      return monitors
  
  
  def audit_changes(accounts, monitor_names, send_report, debug=True, skip_batch=True):
      """
      Audits changes in the accounts
      :param accounts:
      :param monitor_names:
      :param send_report:
      :param debug:
      :param skip_batch:
      :return:
      """
      for account in accounts:
          monitors = get_monitors_and_dependencies(account, monitor_names, debug)
          for monitor in monitors:
              # Skip batch support monitors... They have already been monitored.
              if monitor.batch_support and skip_batch:
                  continue
  
              app.logger.debug("[-->] Auditing account: {}, technology: {}".format(account, monitor.watcher.index))
              _audit_changes(account, monitor.auditors, send_report, debug)
  
  
  def batch_logic(monitor, current_watcher, account_name, debug):
      """
      Performs the batch watcher finding and auditing.
  
      TODO: Investigate how this could, in the future, be set to parallelize the batches.
      :param monitor:
      :param current_watcher:
      :param account_name:
      :param debug:
      :return:
      """
      # Fetch the full list of items that we need to obtain:
      _, exception_map = current_watcher.slurp_list()
      if len(exception_map) > 0:
          # Get the location tuple to collect the region:
          location = exception_map.keys()[0]
          if len(location) > 2:
              region = location[2]
          else:
              region = "unknown"
  
          app.logger.error("[X] Exceptions have caused nothing to be fetched for {technology}"
                           "/{account}/{region}..."
                           " CANNOT CONTINUE FOR THIS WATCHER!".format(technology=current_watcher.i_am_plural,
                                                                       account=account_name,
                                                                       region=region))
          return
  
      while not current_watcher.done_slurping:
          app.logger.debug("[-->] Fetching a batch of {batch} items for {technology}/{account}.".format(
              batch=current_watcher.batched_size, technology=current_watcher.i_am_plural, account=account_name
          ))
          (items, exception_map) = current_watcher.slurp()
  
          audit_items = current_watcher.find_changes(current=items, exception_map=exception_map)
          _audit_specific_changes(monitor, audit_items, False, debug)
  
      # Delete the items that no longer exist:
      app.logger.debug("[-->] Deleting all items for {technology}/{account} that no longer exist.".format(
          technology=current_watcher.i_am_plural, account=account_name
      ))
      current_watcher.find_deleted_batch(account_name)
  
  
  def _audit_changes(account, auditors, send_report, debug=True):
      """ Runs auditors on all items """
      try:
          for au in auditors:
              au.items = au.read_previous_items()
              au.audit_objects()
              au.save_issues()
              if send_report:
                  report = au.create_report()
                  au.email_report(report)
  
              if jirasync:
                  app.logger.info('[-->] Syncing {} issues on {} with Jira'.format(au.index, account))
                  jirasync.sync_issues([account], au.index)
      except (OperationalError, InvalidRequestError, StatementError) as e:
          app.logger.exception("[X] Database error processing accounts %s, cleaning up session.", account)
          db.session.remove()
          store_exception("scheduler-audit-changes", None, e)
  
  
  def _audit_specific_changes(monitor, audit_items, send_report, debug=True):
      """
      Runs the auditor on specific items that are passed in.
      :param monitor:
      :param audit_items:
      :param send_report:
      :param debug:
      :return:
      """
      try:
          for au in monitor.auditors:
              au.items = audit_items
              au.audit_objects()
              au.save_issues()
              if send_report:
                  report = au.create_report()
                  au.email_report(report)
  
              if jirasync:
                  app.logger.info('[-->] Syncing {} issues on {} with Jira'.format(au.index, monitor.watcher.accounts[0]))
                  jirasync.sync_issues(monitor.watcher.accounts, au.index)
      except (OperationalError, InvalidRequestError, StatementError) as e:
          app.logger.exception("[X] Database error processing accounts %s, cleaning up session.",
                               monitor.watcher.accounts[0])
          db.session.remove()
          store_exception("scheduler-audit-changes", None, e)
  
files: security_monkey/task_scheduler/tasks.py
files: """
  .. module: security_monkey.task_scheduler.tasks
      :platform: Unix
      :synopsis: Sets up the Celery task scheduling for watching, auditing, and reporting changes in the environment.
  
  .. version:: $$VERSION$$
  .. moduleauthor:: Mike Grima <mgrima@netflix.com>
  
  """
  import time
  import traceback
  
  from security_monkey import app, db, jirasync, sentry
  from security_monkey.alerter import Alerter
  from security_monkey.datastore import store_exception, clear_old_exceptions
  from security_monkey.monitors import get_monitors, get_monitors_and_dependencies
  from security_monkey.reporter import Reporter
  from security_monkey.task_scheduler.util import CELERY, setup
  from sqlalchemy.exc import OperationalError, InvalidRequestError, StatementError
  
  
  @CELERY.task(bind=True, max_retries=3)
  def task_account_tech(self, account_name, technology_name):
      setup()
      app.logger.info("[ ] Executing Celery task for account: {}, technology: {}".format(account_name, technology_name))
      time1 = time.time()
  
      try:
          reporter_logic(account_name, technology_name)
  
          time2 = time.time()
          app.logger.info('[@] Run Account for Technology (%s/%s) took %0.1f s' % (account_name,
                                                                                   technology_name, (time2 - time1)))
          app.logger.info(
              "[+] Completed Celery task for account: {}, technology: {}".format(account_name, technology_name))
      except Exception as e:
          if sentry:
              sentry.captureException()
          app.logger.error("[X] Task Account Scheduler Exception ({}/{}): {}".format(account_name, technology_name, e))
          app.logger.error(traceback.format_exc())
          store_exception("scheduler-exception-on-watch", None, e)
          raise self.retry(exc=e)
  
  
  @CELERY.task(bind=True, max_retries=3)
  def task_audit(self, account_name, technology_name):
      setup()
  
      app.logger.info("[ ] Executing Celery task to audit changes for Account: {} Technology: {}".format(account_name,
                                                                                                         technology_name))
      try:
          audit_changes([account_name], [technology_name], True)
  
          app.logger.info("[+] Completed Celery task for account: {}, technology: {}".format(account_name,
                                                                                             technology_name))
  
      except Exception as e:
          if sentry:
              sentry.captureException()
          app.logger.error("[X] Task Audit Scheduler Exception ({}/{}): {}".format(account_name, technology_name, e))
          app.logger.error(traceback.format_exc())
          store_exception("scheduler-exception-on-audit", None, e)
          self.retry(exc=e)
  
  
  @CELERY.task()
  def clear_expired_exceptions():
      app.logger.info("[ ] Clearing out exceptions that have an expired TTL...")
      clear_old_exceptions()
      app.logger.info("[-] Completed clearing out exceptions that have an expired TTL.")
  
  
  def reporter_logic(account_name, technology_name):
      """Logic for the run change reporter"""
      try:
          # Watch and Audit:
          monitors = find_changes(account_name, technology_name)
  
          # Alert:
          app.logger.info("[ ] Sending alerts (if applicable) for account: {}, technology: {}".format(account_name,
                                                                                                      technology_name))
          Alerter(monitors, account=account_name).report()
      except (OperationalError, InvalidRequestError, StatementError) as e:
          app.logger.exception("[X] Database error processing account %s - technology %s cleaning up session.",
                               account_name, technology_name)
          db.session.remove()
          store_exception("scheduler-task-account-tech", None, e)
          raise e
  
  
  def manual_run_change_reporter(accounts):
      """Manual change reporting from the command line"""
      app.logger.info("[ ] Executing manual change reporter task...")
  
      try:
          for account in accounts:
              time1 = time.time()
              rep = Reporter(account=account)
  
              for monitor in rep.all_monitors:
                  if monitor.watcher:
                      app.logger.info("[ ] Running change finder for "
                                      "account: {} technology: {}".format(account, monitor.watcher.index))
                      reporter_logic(account, monitor.watcher.index)
  
              time2 = time.time()
              app.logger.info('[@] Run Account %s took %0.1f s' % (account, (time2 - time1)))
  
          app.logger.info("[+] Completed manual change reporting.")
      except (OperationalError, InvalidRequestError, StatementError) as e:
          app.logger.exception("[X] Database error processing cleaning up session.")
          db.session.remove()
          store_exception("scheduler-run-change-reporter", None, e)
          raise e
  
  
  def manual_run_change_finder(accounts, technologies):
      """Manual change finder"""
      app.logger.info("[ ] Executing manual find changes task...")
  
      try:
          for account in accounts:
              time1 = time.time()
  
              for tech in technologies:
                  find_changes(account, tech)
  
              time2 = time.time()
              app.logger.info('[@] Run Account %s took %0.1f s' % (account, (time2 - time1)))
          app.logger.info("[+] Completed manual change finder.")
      except (OperationalError, InvalidRequestError, StatementError) as e:
          app.logger.exception("[X] Database error processing cleaning up session.")
          db.session.remove()
          store_exception("scheduler-run-change-reporter", None, e)
          raise e
  
  
  def find_changes(account_name, monitor_name, debug=True):
      """
          Runs the watcher and stores the result, re-audits all types to account
          for downstream dependencies.
      """
      monitors = get_monitors(account_name, [monitor_name], debug)
      for mon in monitors:
          cw = mon.watcher
          app.logger.info("[-->] Looking for changes in account: {}, technology: {}".format(account_name, cw.index))
          if mon.batch_support:
              batch_logic(mon, cw, account_name, debug)
          else:
              # Just fetch normally...
              (items, exception_map) = cw.slurp()
              cw.find_changes(current=items, exception_map=exception_map)
              cw.save()
  
      # Batched monitors have already been monitored, and they will be skipped over.
      audit_changes([account_name], [monitor_name], False, debug)
      db.session.close()
  
      return monitors
  
  
  def audit_changes(accounts, monitor_names, send_report, debug=True, skip_batch=True):
      """
      Audits changes in the accounts
      :param accounts:
      :param monitor_names:
      :param send_report:
      :param debug:
      :param skip_batch:
      :return:
      """
      for account in accounts:
          monitors = get_monitors_and_dependencies(account, monitor_names, debug)
          for monitor in monitors:
              # Skip batch support monitors... They have already been monitored.
              if monitor.batch_support and skip_batch:
                  continue
  
              app.logger.debug("[-->] Auditing account: {}, technology: {}".format(account, monitor.watcher.index))
              _audit_changes(account, monitor.auditors, send_report, debug)
  
  
  def batch_logic(monitor, current_watcher, account_name, debug):
      """
      Performs the batch watcher finding and auditing.
  
      TODO: Investigate how this could, in the future, be set to parallelize the batches.
      :param monitor:
      :param current_watcher:
      :param account_name:
      :param debug:
      :return:
      """
      # Fetch the full list of items that we need to obtain:
      _, exception_map = current_watcher.slurp_list()
      if len(exception_map) > 0:
          # Get the location tuple to collect the region:
          location = exception_map.keys()[0]
          if len(location) > 2:
              region = location[2]
          else:
              region = "unknown"
  
          app.logger.error("[X] Exceptions have caused nothing to be fetched for {technology}"
                           "/{account}/{region}..."
                           " CANNOT CONTINUE FOR THIS WATCHER!".format(technology=current_watcher.i_am_plural,
                                                                       account=account_name,
                                                                       region=region))
          return
  
      while not current_watcher.done_slurping:
          app.logger.debug("[-->] Fetching a batch of {batch} items for {technology}/{account}.".format(
              batch=current_watcher.batched_size, technology=current_watcher.i_am_plural, account=account_name
          ))
          (items, exception_map) = current_watcher.slurp()
  
          audit_items = current_watcher.find_changes(current=items, exception_map=exception_map)
          _audit_specific_changes(monitor, audit_items, False, debug)
  
      # Delete the items that no longer exist:
      app.logger.debug("[-->] Deleting all items for {technology}/{account} that no longer exist.".format(
          technology=current_watcher.i_am_plural, account=account_name
      ))
      current_watcher.find_deleted_batch(account_name)
  
  
  def _audit_changes(account, auditors, send_report, debug=True):
      """ Runs auditors on all items """
      try:
          for au in auditors:
              au.items = au.read_previous_items()
              au.audit_objects()
              au.save_issues()
              if send_report:
                  report = au.create_report()
                  au.email_report(report)
  
              if jirasync:
                  app.logger.info('[-->] Syncing {} issues on {} with Jira'.format(au.index, account))
                  jirasync.sync_issues([account], au.index)
      except (OperationalError, InvalidRequestError, StatementError) as e:
          app.logger.exception("[X] Database error processing accounts %s, cleaning up session.", account)
          db.session.remove()
          store_exception("scheduler-audit-changes", None, e)
  
  
  def _audit_specific_changes(monitor, audit_items, send_report, debug=True):
      """
      Runs the auditor on specific items that are passed in.
      :param monitor:
      :param audit_items:
      :param send_report:
      :param debug:
      :return:
      """
      try:
          for au in monitor.auditors:
              au.items = audit_items
              au.audit_objects()
              au.save_issues()
              if send_report:
                  report = au.create_report()
                  au.email_report(report)
  
              if jirasync:
                  app.logger.info('[-->] Syncing {} issues on {} with Jira'.format(au.index, monitor.watcher.accounts[0]))
                  jirasync.sync_issues(monitor.watcher.accounts, au.index)
      except (OperationalError, InvalidRequestError, StatementError) as e:
          app.logger.exception("[X] Database error processing accounts %s, cleaning up session.",
                               monitor.watcher.accounts[0])
          db.session.remove()
          store_exception("scheduler-audit-changes", None, e)
  
files: security_monkey/task_scheduler/tasks.py
files: #     Copyright 2018 Netflix, Inc.
  #
  #     Licensed under the Apache License, Version 2.0 (the "License");
  #     you may not use this file except in compliance with the License.
  #     You may obtain a copy of the License at
  #
  #         http://www.apache.org/licenses/LICENSE-2.0
  #
  #     Unless required by applicable law or agreed to in writing, software
  #     distributed under the License is distributed on an "AS IS" BASIS,
  #     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #     See the License for the specific language governing permissions and
  #     limitations under the License.
  """
  .. module: security_monkey.tests.scheduling.test_celery_scheduler
      :platform: Unix
  .. version:: $$VERSION$$
  .. moduleauthor::  Mike Grima <mgrima@netflix.com>
  """
  import json
  
  import boto3
  import mock
  from mock import patch
  from moto import mock_iam, mock_sts
  from pytest import raises
  
  from security_monkey import db, ARN_PREFIX
  from security_monkey.datastore import Account, AccountType, Technology, Item, ItemAudit, ItemRevision
  from security_monkey.monitors import Monitor
  from security_monkey.tests import SecurityMonkeyTestCase
  from security_monkey.watcher import Watcher
  
  OPEN_POLICY = {
      "Statement": [
          {
              "Effect": "Allow",
              "Action": "*",
              "Resource": "*"
          }
      ]
  }
  
  ROLE_CONF = {
      "account_number": "012345678910",
      "technology": "iamrole",
      "region": "universal",
      "name": "roleNumber",
      "InlinePolicies": {"ThePolicy": OPEN_POLICY},
      "Arn": ARN_PREFIX + ":iam::012345678910:role/roleNumber"
  }
  
  
  class CelerySchedulerTestCase(SecurityMonkeyTestCase):
      test_account1 = None
      test_account2 = None
      test_account3 = None
      test_account4 = None
  
      def pre_test_setup(self):
          account_type_result = AccountType(name='AWS')
          db.session.add(account_type_result)
          db.session.commit()
  
          account = Account(identifier="012345678910", name="TEST_ACCOUNT1",
                            account_type_id=account_type_result.id, notes="TEST_ACCOUNT1",
                            third_party=False, active=True)
          db.session.add(account)
  
          account = Account(identifier="123123123123", name="TEST_ACCOUNT2",
                            account_type_id=account_type_result.id, notes="TEST_ACCOUNT2",
                            third_party=False, active=True)
          db.session.add(account)
  
          account = Account(identifier="109876543210", name="TEST_ACCOUNT3",
                            account_type_id=account_type_result.id, notes="TEST_ACCOUNT3",
                            third_party=False, active=False)
          db.session.add(account)
  
          account = Account(identifier="456456456456", name="TEST_ACCOUNT4",
                            account_type_id=account_type_result.id, notes="TEST_ACCOUNT4",
                            third_party=False, active=False)
          db.session.add(account)
  
          db.session.commit()
  
      @patch("security_monkey.task_scheduler.tasks.fix_orphaned_deletions")
      def test_find_batch_changes(self, mock_fix_orphaned):
          """
          Runs through a full find job via the IAMRole watcher, as that supports batching.
  
          However, this is mostly testing the logic through each function call -- this is
          not going to do any boto work and that will instead be mocked out.
          :return:
          """
          from security_monkey.task_scheduler.tasks import manual_run_change_finder
          from security_monkey.monitors import Monitor
          from security_monkey.watchers.iam.iam_role import IAMRole
          from security_monkey.auditors.iam.iam_role import IAMRoleAuditor
          test_account = Account(name="TEST_ACCOUNT1")
          watcher = IAMRole(accounts=[test_account.name])
  
          technology = Technology(name="iamrole")
          db.session.add(technology)
          db.session.commit()
  
          watcher.batched_size = 3  # should loop 4 times
  
          self.add_roles()
  
          # Set up the monitor:
          batched_monitor = Monitor(IAMRole, test_account)
          batched_monitor.watcher = watcher
          batched_monitor.auditors = [IAMRoleAuditor(accounts=[test_account.name])]
  
          import security_monkey.task_scheduler.tasks
          old_get_monitors = security_monkey.task_scheduler.tasks.get_monitors
          security_monkey.task_scheduler.tasks.get_monitors = lambda x, y, z: [batched_monitor]
  
          # Moto screws up the IAM Role ARN -- so we need to fix it:
          original_slurp_list = watcher.slurp_list
          original_slurp = watcher.slurp
  
          def mock_slurp_list():
              items, exception_map = original_slurp_list()
  
              for item in watcher.total_list:
                  item["Arn"] = ARN_PREFIX + ":iam::012345678910:role/{}".format(item["RoleName"])
  
              return items, exception_map
  
          def mock_slurp():
              batched_items, exception_map = original_slurp()
  
              for item in batched_items:
                  item.arn = ARN_PREFIX + ":iam::012345678910:role/{}".format(item.name)
                  item.config["Arn"] = item.arn
                  item.config["RoleId"] = item.name  # Need this to stay the same
  
              return batched_items, exception_map
  
          watcher.slurp_list = mock_slurp_list
          watcher.slurp = mock_slurp
  
          manual_run_change_finder([test_account.name], [watcher.index])
          assert mock_fix_orphaned.called
  
          # Check that all items were added to the DB:
          assert len(Item.query.all()) == 11
  
          # Check that we have exactly 11 item revisions:
          assert len(ItemRevision.query.all()) == 11
  
          # Check that there are audit issues for all 11 items:
          assert len(ItemAudit.query.all()) == 11
  
          # Delete one of the items:
          # Moto lacks implementation for "delete_role" (and I'm too lazy to submit a PR :D) -- so need to create again...
          mock_iam().stop()
          mock_sts().stop()
          self.add_roles(initial=False)
  
          # Run the it again:
          watcher.current_account = None  # Need to reset the watcher
          manual_run_change_finder([test_account.name], [watcher.index])
  
          # Check that nothing new was added:
          assert len(Item.query.all()) == 11
  
          # There should be the same number of issues and 2 more revisions:
          assert len(ItemAudit.query.all()) == 11
          assert len(ItemRevision.query.all()) == 13
  
          # Check that the deleted roles show as being inactive:
          ir = ItemRevision.query.join((Item, ItemRevision.id == Item.latest_revision_id)) \
              .filter(Item.arn.in_(
                  [ARN_PREFIX + ":iam::012345678910:role/roleNumber9",
                   ARN_PREFIX + ":iam::012345678910:role/roleNumber10"])).all()
  
          assert len(ir) == 2
          assert not ir[0].active
          assert not ir[1].active
  
          # Finally -- test with a slurp list exception (just checking that things don't blow up):
          import security_monkey.watchers.iam.iam_role
          old_list_roles = security_monkey.watchers.iam.iam_role.list_roles
  
          def mock_slurp_list_with_exception():
              security_monkey.watchers.iam.iam_role.list_roles = lambda **kwargs: 1 / 0
  
              items, exception_map = original_slurp_list()
  
              assert len(exception_map) > 0
              return items, exception_map
  
          watcher.slurp_list = mock_slurp_list_with_exception
          watcher.current_account = None  # Need to reset the watcher
          manual_run_change_finder([test_account.name], [watcher.index])
  
          security_monkey.task_scheduler.tasks.get_monitors = old_get_monitors
          security_monkey.watchers.iam.iam_role.list_roles = old_list_roles
  
          mock_iam().stop()
          mock_sts().stop()
  
      def test_audit_specific_changes(self):
          from security_monkey.task_scheduler.tasks import _audit_specific_changes
          from security_monkey.monitors import Monitor
          from security_monkey.watchers.iam.iam_role import IAMRole
          from security_monkey.cloudaux_watcher import CloudAuxChangeItem
          from security_monkey.auditors.iam.iam_role import IAMRoleAuditor
  
          # Set up the monitor:
          test_account = Account.query.filter(Account.name == "TEST_ACCOUNT1").one()
          batched_monitor = Monitor(IAMRole, test_account)
          batched_monitor.auditors = [IAMRoleAuditor(accounts=[test_account.name])]
  
          technology = Technology(name="iamrole")
          db.session.add(technology)
          db.session.commit()
  
          watcher = Watcher(accounts=[test_account.name])
          watcher.current_account = (test_account, 0)
          watcher.technology = technology
  
          # Create some IAM roles for testing:
          items = []
          for x in range(0, 3):
              role_policy = dict(ROLE_CONF)
              role_policy["Arn"] = ARN_PREFIX + ":iam::012345678910:role/roleNumber{}".format(x)
              role_policy["RoleName"] = "roleNumber{}".format(x)
              role = CloudAuxChangeItem.from_item(name=role_policy['RoleName'], item=role_policy,
                                                  record_region='universal', account_name=test_account.name,
                                                  index='iamrole', source_watcher=watcher)
              items.append(role)
  
          audit_items = watcher.find_changes_batch(items, {})
          assert len(audit_items) == 3
  
          # Perform the audit:
          _audit_specific_changes(batched_monitor, audit_items, False)
  
          # Check all the issues are there:
          assert len(ItemAudit.query.all()) == 3
  
      def add_roles(self, initial=True):
          mock_sts().start()
          mock_iam().start()
          client = boto3.client("iam")
  
          aspd = {
              "Statement": [
                  {
                      "Effect": "Allow",
                      "Action": "sts:AssumeRole",
                      "Principal": {
                          "Service": "ec2.amazonaws.com"
                      }
                  }
              ]
          }
  
          if initial:
              last = 11
          else:
              last = 9  # Simulates 2 deleted roles...
  
          for x in range(0, last):
              # Create the IAM Role via Moto:
              aspd["Statement"][0]["Resource"] = ARN_PREFIX + ":iam:012345678910:role/roleNumber{}".format(x)
              client.create_role(Path="/", RoleName="roleNumber{}".format(x),
                                 AssumeRolePolicyDocument=json.dumps(aspd, indent=4))
              client.put_role_policy(RoleName="roleNumber{}".format(x), PolicyName="testpolicy",
                                     PolicyDocument=json.dumps(OPEN_POLICY, indent=4))
  
      @patch("security_monkey.task_scheduler.tasks.fix_orphaned_deletions")
      def test_report_batch_changes(self, mock_fix_orphaned):
          from security_monkey.task_scheduler.tasks import manual_run_change_reporter
          from security_monkey.datastore import Item, ItemRevision, ItemAudit
          from security_monkey.monitors import Monitor
          from security_monkey.watchers.iam.iam_role import IAMRole
          from security_monkey.auditors.iam.iam_role import IAMRoleAuditor
  
          test_account = Account.query.filter(Account.name == "TEST_ACCOUNT1").one()
  
          watcher = IAMRole(accounts=[test_account.name])
  
          watcher.batched_size = 3  # should loop 4 times
  
          self.add_roles()
  
          # Set up the monitor:
          batched_monitor = Monitor(IAMRole, test_account)
          batched_monitor.watcher = watcher
          batched_monitor.auditors = [IAMRoleAuditor(accounts=[test_account.name])]
  
          # Set up the Reporter:
          import security_monkey.reporter
          old_all_monitors = security_monkey.reporter.all_monitors
          security_monkey.reporter.all_monitors = lambda x, y: [batched_monitor]
  
          import security_monkey.task_scheduler.tasks
          old_get_monitors = security_monkey.task_scheduler.tasks.get_monitors
          security_monkey.task_scheduler.tasks.get_monitors = lambda x, y, z: [batched_monitor]
  
          # Moto screws up the IAM Role ARN -- so we need to fix it:
          original_slurp_list = watcher.slurp_list
          original_slurp = watcher.slurp
  
          def mock_slurp_list():
              items, exception_map = original_slurp_list()
  
              for item in watcher.total_list:
                  item["Arn"] = ARN_PREFIX + ":iam::012345678910:role/{}".format(item["RoleName"])
  
              return items, exception_map
  
          def mock_slurp():
              batched_items, exception_map = original_slurp()
  
              for item in batched_items:
                  item.arn = ARN_PREFIX + ":iam::012345678910:role/{}".format(item.name)
                  item.config["Arn"] = item.arn
                  item.config["RoleId"] = item.name  # Need this to stay the same
  
              return batched_items, exception_map
  
          watcher.slurp_list = mock_slurp_list
          watcher.slurp = mock_slurp
  
          manual_run_change_reporter([test_account.name])
  
          assert mock_fix_orphaned.called
  
          # Check that all items were added to the DB:
          assert len(Item.query.all()) == 11
  
          # Check that we have exactly 11 item revisions:
          assert len(ItemRevision.query.all()) == 11
  
          # Check that there are audit issues for all 11 items:
          assert len(ItemAudit.query.all()) == 11
  
          mock_iam().stop()
          mock_sts().stop()
  
          security_monkey.reporter.all_monitors = old_all_monitors
          security_monkey.task_scheduler.tasks.get_monitors = old_get_monitors
  
      def test_celery_purge(self):
          from security_monkey.task_scheduler.beat import purge_it
          with patch("security_monkey.task_scheduler.beat.CELERY") as mock:
              purge_it()
              assert mock.control.purge.called
  
      def test_fix_orphaned_deletions(self):
          test_account = Account.query.filter(Account.name == "TEST_ACCOUNT1").one()
          technology = Technology(name="orphaned")
  
          db.session.add(technology)
          db.session.commit()
  
          orphaned_item = Item(name="orphaned", region="us-east-1", tech_id=technology.id, account_id=test_account.id)
          db.session.add(orphaned_item)
          db.session.commit()
  
          assert not orphaned_item.latest_revision_id
          assert not orphaned_item.revisions.count()
          assert len(Item.query.filter(Item.account_id == test_account.id, Item.tech_id == technology.id,
                                       Item.latest_revision_id == None).all()) == 1  # noqa
  
          from security_monkey.task_scheduler.tasks import fix_orphaned_deletions
          fix_orphaned_deletions(test_account.name, technology.name)
  
          assert not Item.query.filter(Item.account_id == test_account.id, Item.tech_id == technology.id,
                                       Item.latest_revision_id == None).all()  # noqa
  
          assert orphaned_item.latest_revision_id
          assert orphaned_item.revisions.count() == 1
          assert orphaned_item.latest_config == {}
  
      @patch("security_monkey.task_scheduler.beat.setup")
      @patch("security_monkey.task_scheduler.beat.purge_it")
      @patch("security_monkey.task_scheduler.tasks.task_account_tech")
      @patch("security_monkey.task_scheduler.tasks.task_audit")
      @patch("security_monkey.task_scheduler.tasks.clear_expired_exceptions")
      def test_celery_beat(self, mock_expired_exceptions, mock_task_audit, mock_account_tech, mock_purge, mock_setup):
          from security_monkey.task_scheduler.beat import setup_the_tasks
          from security_monkey.watchers.iam.iam_role import IAMRole
          from security_monkey.auditors.iam.iam_role import IAMRoleAuditor
  
          # Set up the monitor:
          test_account = Account.query.filter(Account.name == "TEST_ACCOUNT1").one()
          watcher = IAMRole(accounts=[test_account.name])
          batched_monitor = Monitor(IAMRole, test_account)
          batched_monitor.watcher = watcher
          batched_monitor.auditors = [IAMRoleAuditor(accounts=[test_account.name])]
  
          import security_monkey.task_scheduler.tasks
          old_get_monitors = security_monkey.task_scheduler.tasks.get_monitors
          security_monkey.task_scheduler.tasks.get_monitors = lambda x, y, z: [batched_monitor]
  
          setup_the_tasks(mock.Mock())
  
          assert mock_setup.called
          assert mock_purge.called
  
          # "apply_async" where the immediately scheduled tasks
          assert mock_account_tech.apply_async.called
  
          # The ".s" are the scheduled tasks. Too lazy to grab the intervals out.
          assert mock_account_tech.s.called
          assert mock_expired_exceptions.s.called
          assert mock_task_audit.s.called
  
          # Build the expected mock results:
          scheduled_tech_result_list = []
          async_result_list = []
          audit_result_list = []
  
          import security_monkey.watcher
          import security_monkey.auditor
  
          for account in Account.query.filter(Account.third_party == False).filter(Account.active == True).all():  # noqa
              for w in security_monkey.watcher.watcher_registry.iterkeys():
                  scheduled_tech_result_list.append(((account.name, w),))
                  async_result_list.append((((account.name, w),),))
  
              # It's just policy for IAM:
              audit_result_list.append(((account.name, "policy"),))
  
          assert mock_account_tech.s.call_args_list == scheduled_tech_result_list
          assert async_result_list == mock_account_tech.apply_async.call_args_list
          assert audit_result_list == mock_task_audit.s.call_args_list
  
          security_monkey.task_scheduler.tasks.get_monitors = old_get_monitors
  
      @patch("security_monkey.task_scheduler.tasks.clear_old_exceptions")
      def test_celery_exception_task(self, mock_exception_clear):
          from security_monkey.task_scheduler.tasks import clear_expired_exceptions
          clear_expired_exceptions()
          assert mock_exception_clear.assert_called
  
      @patch("security_monkey.task_scheduler.tasks.setup")
      @patch("security_monkey.task_scheduler.tasks.audit_changes")
      @patch("security_monkey.task_scheduler.tasks.store_exception")
      def test_audit_task(self, mock_store_exception, mock_audit_changes, mock_setup):
          from security_monkey.task_scheduler.tasks import task_audit
  
          account_name = "TEST_ACCOUNT1"
          technology_name = "iamrole"
  
          task_audit(account_name, technology_name)  # noqa
          mock_audit_changes.assert_called_with([account_name], [technology_name], True)
  
          assert mock_setup.called
  
          exception = Exception("Testing")
          mock_audit_changes.side_effect = exception
          with raises(Exception):
              task_audit(account_name, technology_name)  # noqa
  
          mock_store_exception.assert_called_with("scheduler-exception-on-audit", None, exception)
  
      @patch("security_monkey.task_scheduler.tasks.setup")
      @patch("security_monkey.task_scheduler.tasks.reporter_logic")
      @patch("security_monkey.task_scheduler.tasks.store_exception")
      def test_account_tech_task(self, mock_store_exception, mock_reporter, mock_setup):
          from security_monkey.task_scheduler.tasks import task_account_tech
  
          account_name = "TEST_ACCOUNT1"
          technology_name = "iamrole"
  
          task_account_tech(account_name, technology_name)  # noqa
          mock_reporter.assert_called_with(account_name, technology_name)
  
          assert mock_setup.called
  
          exception = Exception("Testing")
          mock_reporter.side_effect = exception
          with raises(Exception):
              task_account_tech(account_name, technology_name)  # noqa
  
          mock_store_exception.assert_called_with("scheduler-exception-on-watch", None, exception)
  
files: security_monkey/tests/scheduling/test_celery_scheduler.py
files: #     Copyright 2018 Netflix, Inc.
  #
  #     Licensed under the Apache License, Version 2.0 (the "License");
  #     you may not use this file except in compliance with the License.
  #     You may obtain a copy of the License at
  #
  #         http://www.apache.org/licenses/LICENSE-2.0
  #
  #     Unless required by applicable law or agreed to in writing, software
  #     distributed under the License is distributed on an "AS IS" BASIS,
  #     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #     See the License for the specific language governing permissions and
  #     limitations under the License.
  """
  .. module: security_monkey.tests.scheduling.test_celery_scheduler
      :platform: Unix
  .. version:: $$VERSION$$
  .. moduleauthor::  Mike Grima <mgrima@netflix.com>
  """
  import json
  
  import boto3
  import mock
  from mock import patch
  from moto import mock_iam, mock_sts
  from pytest import raises
  
  from security_monkey import db, ARN_PREFIX
  from security_monkey.datastore import Account, AccountType, Technology, Item, ItemAudit, ItemRevision
  from security_monkey.monitors import Monitor
  from security_monkey.tests import SecurityMonkeyTestCase
  from security_monkey.watcher import Watcher
  
  OPEN_POLICY = {
      "Statement": [
          {
              "Effect": "Allow",
              "Action": "*",
              "Resource": "*"
          }
      ]
  }
  
  ROLE_CONF = {
      "account_number": "012345678910",
      "technology": "iamrole",
      "region": "universal",
      "name": "roleNumber",
      "InlinePolicies": {"ThePolicy": OPEN_POLICY},
      "Arn": ARN_PREFIX + ":iam::012345678910:role/roleNumber"
  }
  
  
  class CelerySchedulerTestCase(SecurityMonkeyTestCase):
      test_account1 = None
      test_account2 = None
      test_account3 = None
      test_account4 = None
  
      def pre_test_setup(self):
          account_type_result = AccountType(name='AWS')
          db.session.add(account_type_result)
          db.session.commit()
  
          account = Account(identifier="012345678910", name="TEST_ACCOUNT1",
                            account_type_id=account_type_result.id, notes="TEST_ACCOUNT1",
                            third_party=False, active=True)
          db.session.add(account)
  
          account = Account(identifier="123123123123", name="TEST_ACCOUNT2",
                            account_type_id=account_type_result.id, notes="TEST_ACCOUNT2",
                            third_party=False, active=True)
          db.session.add(account)
  
          account = Account(identifier="109876543210", name="TEST_ACCOUNT3",
                            account_type_id=account_type_result.id, notes="TEST_ACCOUNT3",
                            third_party=False, active=False)
          db.session.add(account)
  
          account = Account(identifier="456456456456", name="TEST_ACCOUNT4",
                            account_type_id=account_type_result.id, notes="TEST_ACCOUNT4",
                            third_party=False, active=False)
          db.session.add(account)
  
          db.session.commit()
  
      def test_find_batch_changes(self):
          """
          Runs through a full find job via the IAMRole watcher, as that supports batching.
  
          However, this is mostly testing the logic through each function call -- this is
          not going to do any boto work and that will instead be mocked out.
          :return:
          """
          from security_monkey.task_scheduler.tasks import manual_run_change_finder, setup
          from security_monkey.monitors import Monitor
          from security_monkey.watchers.iam.iam_role import IAMRole
          from security_monkey.auditors.iam.iam_role import IAMRoleAuditor
          test_account = Account(name="TEST_ACCOUNT1")
          watcher = IAMRole(accounts=[test_account.name])
  
          technology = Technology(name="iamrole")
          db.session.add(technology)
          db.session.commit()
  
          watcher.batched_size = 3  # should loop 4 times
  
          self.add_roles()
  
          # Set up the monitor:
          batched_monitor = Monitor(IAMRole, test_account)
          batched_monitor.watcher = watcher
          batched_monitor.auditors = [IAMRoleAuditor(accounts=[test_account.name])]
  
          import security_monkey.task_scheduler.tasks
          old_get_monitors = security_monkey.task_scheduler.tasks.get_monitors
          security_monkey.task_scheduler.tasks.get_monitors = lambda x, y, z: [batched_monitor]
  
          # Moto screws up the IAM Role ARN -- so we need to fix it:
          original_slurp_list = watcher.slurp_list
          original_slurp = watcher.slurp
  
          def mock_slurp_list():
              items, exception_map = original_slurp_list()
  
              for item in watcher.total_list:
                  item["Arn"] = ARN_PREFIX + ":iam::012345678910:role/{}".format(item["RoleName"])
  
              return items, exception_map
  
          def mock_slurp():
              batched_items, exception_map = original_slurp()
  
              for item in batched_items:
                  item.arn = ARN_PREFIX + ":iam::012345678910:role/{}".format(item.name)
                  item.config["Arn"] = item.arn
                  item.config["RoleId"] = item.name  # Need this to stay the same
  
              return batched_items, exception_map
  
          watcher.slurp_list = mock_slurp_list
          watcher.slurp = mock_slurp
  
          manual_run_change_finder([test_account.name], [watcher.index])
  
          # Check that all items were added to the DB:
          assert len(Item.query.all()) == 11
  
          # Check that we have exactly 11 item revisions:
          assert len(ItemRevision.query.all()) == 11
  
          # Check that there are audit issues for all 11 items:
          assert len(ItemAudit.query.all()) == 11
  
          # Delete one of the items:
          # Moto lacks implementation for "delete_role" (and I'm too lazy to submit a PR :D) -- so need to create again...
          mock_iam().stop()
          mock_sts().stop()
          self.add_roles(initial=False)
  
          # Run the it again:
          watcher.current_account = None  # Need to reset the watcher
          manual_run_change_finder([test_account.name], [watcher.index])
  
          # Check that nothing new was added:
          assert len(Item.query.all()) == 11
  
          # There should be the same number of issues and 2 more revisions:
          assert len(ItemAudit.query.all()) == 11
          assert len(ItemRevision.query.all()) == 13
  
          # Check that the deleted roles show as being inactive:
          ir = ItemRevision.query.join((Item, ItemRevision.id == Item.latest_revision_id)) \
              .filter(Item.arn.in_(
                  [ARN_PREFIX + ":iam::012345678910:role/roleNumber9",
                   ARN_PREFIX + ":iam::012345678910:role/roleNumber10"])).all()
  
          assert len(ir) == 2
          assert not ir[0].active
          assert not ir[1].active
  
          # Finally -- test with a slurp list exception (just checking that things don't blow up):
          import security_monkey.watchers.iam.iam_role
          old_list_roles = security_monkey.watchers.iam.iam_role.list_roles
  
          def mock_slurp_list_with_exception():
              security_monkey.watchers.iam.iam_role.list_roles = lambda **kwargs: 1 / 0
  
              items, exception_map = original_slurp_list()
  
              assert len(exception_map) > 0
              return items, exception_map
  
          watcher.slurp_list = mock_slurp_list_with_exception
          watcher.current_account = None  # Need to reset the watcher
          manual_run_change_finder([test_account.name], [watcher.index])
  
          security_monkey.task_scheduler.tasks.get_monitors = old_get_monitors
          security_monkey.watchers.iam.iam_role.list_roles = old_list_roles
  
          mock_iam().stop()
          mock_sts().stop()
  
      def test_audit_specific_changes(self):
          from security_monkey.task_scheduler.tasks import _audit_specific_changes
          from security_monkey.monitors import Monitor
          from security_monkey.watchers.iam.iam_role import IAMRole
          from security_monkey.cloudaux_watcher import CloudAuxChangeItem
          from security_monkey.auditors.iam.iam_role import IAMRoleAuditor
  
          # Set up the monitor:
          test_account = Account.query.filter(Account.name == "TEST_ACCOUNT1").one()
          batched_monitor = Monitor(IAMRole, test_account)
          batched_monitor.auditors = [IAMRoleAuditor(accounts=[test_account.name])]
  
          technology = Technology(name="iamrole")
          db.session.add(technology)
          db.session.commit()
  
          watcher = Watcher(accounts=[test_account.name])
          watcher.current_account = (test_account, 0)
          watcher.technology = technology
  
          # Create some IAM roles for testing:
          items = []
          for x in range(0, 3):
              role_policy = dict(ROLE_CONF)
              role_policy["Arn"] = ARN_PREFIX + ":iam::012345678910:role/roleNumber{}".format(x)
              role_policy["RoleName"] = "roleNumber{}".format(x)
              role = CloudAuxChangeItem.from_item(name=role_policy['RoleName'], item=role_policy,
                                                  record_region='universal', account_name=test_account.name,
                                                  index='iamrole', source_watcher=watcher)
              items.append(role)
  
          audit_items = watcher.find_changes_batch(items, {})
          assert len(audit_items) == 3
  
          # Perform the audit:
          _audit_specific_changes(batched_monitor, audit_items, False)
  
          # Check all the issues are there:
          assert len(ItemAudit.query.all()) == 3
  
      def add_roles(self, initial=True):
          mock_sts().start()
          mock_iam().start()
          client = boto3.client("iam")
  
          aspd = {
              "Statement": [
                  {
                      "Effect": "Allow",
                      "Action": "sts:AssumeRole",
                      "Principal": {
                          "Service": "ec2.amazonaws.com"
                      }
                  }
              ]
          }
  
          if initial:
              last = 11
          else:
              last = 9  # Simulates 2 deleted roles...
  
          for x in range(0, last):
              # Create the IAM Role via Moto:
              aspd["Statement"][0]["Resource"] = ARN_PREFIX + ":iam:012345678910:role/roleNumber{}".format(x)
              client.create_role(Path="/", RoleName="roleNumber{}".format(x),
                                 AssumeRolePolicyDocument=json.dumps(aspd, indent=4))
              client.put_role_policy(RoleName="roleNumber{}".format(x), PolicyName="testpolicy",
                                     PolicyDocument=json.dumps(OPEN_POLICY, indent=4))
  
      def test_report_batch_changes(self):
          from security_monkey.task_scheduler.tasks import manual_run_change_reporter, setup
          from security_monkey.datastore import Item, ItemRevision, ItemAudit
          from security_monkey.monitors import Monitor
          from security_monkey.watchers.iam.iam_role import IAMRole
          from security_monkey.auditors.iam.iam_role import IAMRoleAuditor
  
          test_account = Account.query.filter(Account.name == "TEST_ACCOUNT1").one()
  
          watcher = IAMRole(accounts=[test_account.name])
  
          watcher.batched_size = 3  # should loop 4 times
  
          self.add_roles()
  
          # Set up the monitor:
          batched_monitor = Monitor(IAMRole, test_account)
          batched_monitor.watcher = watcher
          batched_monitor.auditors = [IAMRoleAuditor(accounts=[test_account.name])]
  
          # Set up the Reporter:
          import security_monkey.reporter
          old_all_monitors = security_monkey.reporter.all_monitors
          security_monkey.reporter.all_monitors = lambda x, y: [batched_monitor]
  
          import security_monkey.task_scheduler.tasks
          old_get_monitors = security_monkey.task_scheduler.tasks.get_monitors
          security_monkey.task_scheduler.tasks.get_monitors = lambda x, y, z: [batched_monitor]
  
          # Moto screws up the IAM Role ARN -- so we need to fix it:
          original_slurp_list = watcher.slurp_list
          original_slurp = watcher.slurp
  
          def mock_slurp_list():
              items, exception_map = original_slurp_list()
  
              for item in watcher.total_list:
                  item["Arn"] = ARN_PREFIX + ":iam::012345678910:role/{}".format(item["RoleName"])
  
              return items, exception_map
  
          def mock_slurp():
              batched_items, exception_map = original_slurp()
  
              for item in batched_items:
                  item.arn = ARN_PREFIX + ":iam::012345678910:role/{}".format(item.name)
                  item.config["Arn"] = item.arn
                  item.config["RoleId"] = item.name  # Need this to stay the same
  
              return batched_items, exception_map
  
          watcher.slurp_list = mock_slurp_list
          watcher.slurp = mock_slurp
  
          manual_run_change_reporter([test_account.name])
  
          # Check that all items were added to the DB:
          assert len(Item.query.all()) == 11
  
          # Check that we have exactly 11 item revisions:
          assert len(ItemRevision.query.all()) == 11
  
          # Check that there are audit issues for all 11 items:
          assert len(ItemAudit.query.all()) == 11
  
          mock_iam().stop()
          mock_sts().stop()
  
          security_monkey.reporter.all_monitors = old_all_monitors
          security_monkey.task_scheduler.tasks.get_monitors = old_get_monitors
  
      def test_celery_purge(self):
          from security_monkey.task_scheduler.beat import purge_it
          with patch("security_monkey.task_scheduler.beat.CELERY") as mock:
              purge_it()
              assert mock.control.purge.called
  
      @patch("security_monkey.task_scheduler.beat.setup")
      @patch("security_monkey.task_scheduler.beat.purge_it")
      @patch("security_monkey.task_scheduler.tasks.task_account_tech")
      @patch("security_monkey.task_scheduler.tasks.task_audit")
      @patch("security_monkey.task_scheduler.tasks.clear_expired_exceptions")
      def test_celery_beat(self, mock_expired_exceptions, mock_task_audit, mock_account_tech, mock_purge, mock_setup):
          from security_monkey.task_scheduler.beat import setup_the_tasks
          from security_monkey.watchers.iam.iam_role import IAMRole
          from security_monkey.auditors.iam.iam_role import IAMRoleAuditor
  
          # Set up the monitor:
          test_account = Account.query.filter(Account.name == "TEST_ACCOUNT1").one()
          watcher = IAMRole(accounts=[test_account.name])
          batched_monitor = Monitor(IAMRole, test_account)
          batched_monitor.watcher = watcher
          batched_monitor.auditors = [IAMRoleAuditor(accounts=[test_account.name])]
  
          import security_monkey.task_scheduler.tasks
          old_get_monitors = security_monkey.task_scheduler.tasks.get_monitors
          security_monkey.task_scheduler.tasks.get_monitors = lambda x, y, z: [batched_monitor]
  
          setup_the_tasks(mock.Mock())
  
          assert mock_setup.called
          assert mock_purge.called
  
          # "apply_async" where the immediately scheduled tasks
          assert mock_account_tech.apply_async.called
  
          # The ".s" are the scheduled tasks. Too lazy to grab the intervals out.
          assert mock_account_tech.s.called
          assert mock_expired_exceptions.s.called
          assert mock_task_audit.s.called
  
          # Build the expected mock results:
          scheduled_tech_result_list = []
          async_result_list = []
          audit_result_list = []
  
          import security_monkey.watcher
          import security_monkey.auditor
  
          for account in Account.query.filter(Account.third_party == False).filter(Account.active == True).all():  # noqa
              for w in security_monkey.watcher.watcher_registry.iterkeys():
                  scheduled_tech_result_list.append(((account.name, w),))
                  async_result_list.append((((account.name, w),),))
  
              # It's just policy for IAM:
              audit_result_list.append(((account.name, "policy"),))
  
          assert mock_account_tech.s.call_args_list == scheduled_tech_result_list
          assert async_result_list == mock_account_tech.apply_async.call_args_list
          assert audit_result_list == mock_task_audit.s.call_args_list
  
          security_monkey.task_scheduler.tasks.get_monitors = old_get_monitors
  
      @patch("security_monkey.task_scheduler.tasks.clear_old_exceptions")
      def test_celery_exception_task(self, mock_exception_clear):
          from security_monkey.task_scheduler.tasks import clear_expired_exceptions
          clear_expired_exceptions()
          assert mock_exception_clear.assert_called
  
      @patch("security_monkey.task_scheduler.tasks.setup")
      @patch("security_monkey.task_scheduler.tasks.audit_changes")
      @patch("security_monkey.task_scheduler.tasks.store_exception")
      def test_audit_task(self, mock_store_exception, mock_audit_changes, mock_setup):
          from security_monkey.task_scheduler.tasks import task_audit
  
          account_name = "TEST_ACCOUNT1"
          technology_name = "iamrole"
  
          task_audit(account_name, technology_name)  # noqa
          mock_audit_changes.assert_called_with([account_name], [technology_name], True)
  
          assert mock_setup.called
  
          exception = Exception("Testing")
          mock_audit_changes.side_effect = exception
          with raises(Exception):
              task_audit(account_name, technology_name)  # noqa
  
          mock_store_exception.assert_called_with("scheduler-exception-on-audit", None, exception)
  
      @patch("security_monkey.task_scheduler.tasks.setup")
      @patch("security_monkey.task_scheduler.tasks.reporter_logic")
      @patch("security_monkey.task_scheduler.tasks.store_exception")
      def test_account_tech_task(self, mock_store_exception, mock_reporter, mock_setup):
          from security_monkey.task_scheduler.tasks import task_account_tech
  
          account_name = "TEST_ACCOUNT1"
          technology_name = "iamrole"
  
          task_account_tech(account_name, technology_name)  # noqa
          mock_reporter.assert_called_with(account_name, technology_name)
  
          assert mock_setup.called
  
          exception = Exception("Testing")
          mock_reporter.side_effect = exception
          with raises(Exception):
              task_account_tech(account_name, technology_name)  # noqa
  
          mock_store_exception.assert_called_with("scheduler-exception-on-watch", None, exception)
  
files: security_monkey/tests/scheduling/test_celery_scheduler.py
files: """
  .. module: security_monkey.watcher
      :platform: Unix
      :synopsis: Slurps the current config from AWS and compares it to what has previously
      been recorded in the database to find any changes.
  
  .. version:: $$VERSION$$
  .. moduleauthor:: Patrick Kelley <pkelley@netflix.com> @monkeysecurity
  
  """
  from botocore.exceptions import ClientError
  
  from common.PolicyDiff import PolicyDiff
  from common.utils import sub_dict
  from security_monkey import app
  from security_monkey.datastore import Account, IgnoreListEntry, db
  from security_monkey.datastore import Technology, WatcherConfig, store_exception
  from security_monkey.common.jinja import get_jinja_env
  from security_monkey.alerters.custom_alerter import report_watcher_changes
  
  from boto.exception import BotoServerError
  import time
  
  import datastore
  from copy import deepcopy
  import dpath.util
  from dpath.exceptions import PathNotFound
  
  import logging
  
  watcher_registry = {}
  abstract_classes = set(['Watcher', 'CloudAuxWatcher', 'CloudAuxBatchedWatcher'])
  
  
  if not app.config.get("DONT_IGNORE_BOTO_VERBOSE_LOGGERS"):
      logging.getLogger('botocore.vendored.requests.packages.urllib3').setLevel(logging.WARNING)
      logging.getLogger('botocore.credentials').setLevel(logging.WARNING)
  
  
  class WatcherType(type):
      def __init__(cls, name, bases, attrs):
          super(WatcherType, cls).__init__(name, bases, attrs)
          if cls.__name__ not in abstract_classes and cls.index:
              app.logger.debug("Registering watcher {} {}.{}".format(cls.index, cls.__module__, cls.__name__))
              watcher_registry[cls.index] = cls
  
  
  class Watcher(object):
      """Slurps the current config from AWS and compares it to what has previously
        been recorded in the database to find any changes."""
      index = 'abstract'
      i_am_singular = 'Abstract'
      i_am_plural = 'Abstracts'
      rate_limit_delay = 0
      ignore_list = []
      interval = 60    #in minutes
      active = True
      account_type = 'AWS'
      __metaclass__ = WatcherType
  
      def __init__(self, accounts=None, debug=False):
          """Initializes the Watcher"""
          self.datastore = datastore.Datastore()
          if not accounts:
              accounts = Account.query.filter(Account.third_party==False).filter(Account.active==True).all()
          else:
              accounts = Account.query.filter(Account.third_party==False).filter(Account.active==True).filter(Account.name.in_(accounts)).all()
          if not accounts:
              raise ValueError('Watcher needs a valid account')
          self.accounts = [account.name for account in accounts]
          self.account_identifiers = [account.identifier for account in accounts]
          self.debug = debug
          self.created_items = []
          self.deleted_items = []
          self.changed_items = []
          self.ephemeral_items = []
          # TODO: grab these from DB, keyed on account
          self.rate_limit_delay = 0
          self.honor_ephemerals = False
          self.ephemeral_paths = []
  
          # Batching attributes:
          self.batched_size = 0   # Don't batch anything by default
          self.done_slurping = True   # Don't batch anything by default
          self.total_list = []    # This will hold the full list of items to batch over
          self.batch_counter = 0  # Keeps track of the batch we are on -- can be used for retry logic
          self.current_account = None  # Tuple that holds the current account and account index we are on.
          self.technology = None
          # Region is probably not needed if we are using CloudAux's iter_account_region -- will test this in
          # the future as we add more items with batching support.
  
      def prep_for_slurp(self):
          """
          Should be run before slurp is run to grab the IgnoreList.
          """
          query = IgnoreListEntry.query
          query = query.join((Technology, Technology.id == IgnoreListEntry.tech_id))
          self.ignore_list = query.filter(Technology.name == self.index).all()
  
      def prep_for_batch_slurp(self):
          """
          Should be run before batching slurps to set the current account (and region).
  
          This will load the DB objects for account and technology for where we are currently at in the process.
          :return:
          """
          self.prep_for_slurp()
  
          # Which account are we currently on?
          if not self.current_account:
              index = 0
  
              # Get the Technology
              # If technology doesn't exist, then create it:
              technology = Technology.query.filter(Technology.name == self.index).first()
              if not technology:
                  technology = Technology(name=self.index)
                  db.session.add(technology)
                  db.session.commit()
                  app.logger.info("Technology: {} did not exist... created it...".format(self.index))
  
              self.technology = technology
          else:
              index = self.current_account[1] + 1
  
          self.current_account = (Account.query.filter(Account.name == self.accounts[index]).one(), index)
  
          # We will not be using CloudAux's iter_account_region for multi-account -- we want
          # to have per-account level of batching
          self.total_list = []    # Reset the total list for a new account to run against.
          self.done_slurping = False
          self.batch_counter = 0
  
      def check_ignore_list(self, name):
          """
          See if the given item has a name flagging it to be ignored by security_monkey.
          """
          for result in self.ignore_list:
              # Empty prefix comes back as None instead of an empty string ...
              prefix = result.prefix or ""
              if name.lower().startswith(prefix.lower()):
                  app.logger.info("Ignoring {}/{} because of IGNORELIST prefix {}".format(self.index, name, result.prefix))
                  return True
  
          return False
  
      def wrap_aws_rate_limited_call(self, awsfunc, *args, **nargs):
          attempts = 0
  
          def increase_delay():
              if self.rate_limit_delay == 0:
                  self.rate_limit_delay = 1
                  app.logger.warn(('Being rate-limited by AWS. Increasing delay on tech {} ' +
                                  'in account {} from 0 to 1 second. Attempt {}')
                                  .format(self.index, self.accounts, attempts))
              elif self.rate_limit_delay < 4:
                  self.rate_limit_delay = self.rate_limit_delay * 2
                  app.logger.warn(('Still being rate-limited by AWS. Increasing delay on tech {} ' +
                                  'in account {} to {} seconds. Attempt {}')
                                  .format(self.index, self.accounts, self.rate_limit_delay, attempts))
              else:
                  app.logger.warn(('Still being rate-limited by AWS. Keeping delay on tech {} ' +
                                  'in account {} at {} seconds. Attempt {}')
                                  .format(self.index, self.accounts, self.rate_limit_delay, attempts))
  
          while True:
              attempts = attempts + 1
              try:
                  if self.rate_limit_delay > 0:
                      time.sleep(self.rate_limit_delay)
  
                  retval = awsfunc(*args, **nargs)
  
                  if self.rate_limit_delay > 0:
                      app.logger.warn("Successfully Executed Rate-Limited Function. "
                                      "Tech: {} Account: {}. Removing sleep period."
                                      .format(self.index, self.accounts))
                      self.rate_limit_delay = 0
  
                  return retval
              except BotoServerError as e:  # Boto
                  if not e.error_code == 'Throttling':
                      raise e
                  increase_delay()
              except ClientError as e:  # Botocore
                  if not e.response["Error"]["Code"] == "Throttling":
                      raise e
                  increase_delay()
  
      def created(self):
          """
          Used by the Jinja templates
          :returns: True if created_items is not empty
          :returns: False otherwise.
          """
          return len(self.created_items) > 0
  
      def deleted(self):
          """
          Used by the Jinja templates
          :returns: True if deleted_items is not empty
          :returns: False otherwise.
          """
          return len(self.deleted_items) > 0
  
      def changed(self):
          """
          Used by the Jinja templates
          :returns: True if changed_items is not empty
          :returns: False otherwise.
          """
          return len(self.changed_items) > 0
  
      def slurp_list(self):
          """
          This will fetch all the items in question that will need to get slurped.
          This is used to know what we are going to have to batch up.
          :return:
          """
          raise NotImplementedError()
  
      def slurp(self):
          """
          method to slurp configuration from AWS for whatever it is that I'm
          interested in. This will be overridden for each technology.
          """
          raise NotImplementedError()
  
      def slurp_exception(self, location=None, exception=None, exception_map={}, source="watcher"):
          """
          Logs any exceptions that happen in slurp and adds them to the exception_map
          using their location as the key.  The location is a tuple in the form:
          (technology, account, region, item_name) that describes the object where the exception occurred.
          Location can also exclude an item_name if the exception is region wide.
          """
          if location in exception_map:
              app.logger.debug("Exception map already has location {}. This should not happen.".format(location))
          exception_map[location] = exception
          app.logger.debug("Adding {} to the exceptions list. Exception was: {}".format(location, str(exception)))
  
          # Store it to the database:
          store_exception(source, location, exception)
  
      def location_in_exception_map(self, item_location, exception_map={}):
          """
          Determines whether a given location is covered by an exception already in the
          exception map.
  
              Item location: (self.index, self.account, self.region, self.name)
            exception Maps: (index, account, region, name)
                            (index, account, region)
                            (index, account)
  
              :returns: True if location is covered by an entry in the exception map.
              :returns: False if location is not covered by an entry in the exception map.
          """
          # Exact Match
          if item_location in exception_map:
              app.logger.debug("Skipping {} due to an item-level exception {}.".format(item_location, exception_map[item_location]))
              return True
  
          # (index, account, region)
          if item_location[0:3] in exception_map:
              app.logger.debug("Skipping {} due to a region-level exception {}.".format(item_location, exception_map[item_location[0:3]]))
              return True
  
          # (index, account)
          if item_location[0:2] in exception_map:
              app.logger.debug("Skipping {} due to an account-level exception {}.".format(item_location, exception_map[item_location[0:2]]))
              return True
  
          # (index)
          if item_location[0:1] in exception_map:
              app.logger.debug("Skipping {} due to a technology-level exception {}.".format(item_location, exception_map[item_location[0:1]]))
              return True
  
          return False
  
      def find_deleted(self, previous=[], current=[], exception_map={}):
          """
          Find any items that have been deleted since the last run of the watcher.
          Add these items to the deleted_items list.
          """
          prev_map = {item.location(): item for item in previous}
          curr_map = {item.location(): item for item in current}
  
          item_locations = list(set(prev_map).difference(set(curr_map)))
          item_locations = [item_location for item_location in item_locations if not self.location_in_exception_map(item_location, exception_map)]
          list_deleted_items = [prev_map[item] for item in item_locations]
  
          for item in list_deleted_items:
              deleted_change_item = ChangeItem.from_items(old_item=item, new_item=None, source_watcher=self)
              app.logger.debug("%s: %s/%s/%s deleted" % (self.i_am_singular, item.account, item.region, item.name))
              self.deleted_items.append(deleted_change_item)
  
      def find_new(self, previous=[], current=[]):
          """
          Find any new objects that have been created since the last run of the watcher.
          Add these items to the created_items list.
          """
          prev_map = {item.location(): item for item in previous}
          curr_map = {item.location(): item for item in current}
  
          item_locations = list(set(curr_map).difference(set(prev_map)))
          list_new_items = [curr_map[item] for item in item_locations]
  
          for item in list_new_items:
              new_change_item = ChangeItem.from_items(old_item=None, new_item=item, source_watcher=self)
              self.created_items.append(new_change_item)
              app.logger.debug("%s: %s/%s/%s created" % (self.i_am_singular, item.account, item.region, item.name))
  
      def find_modified(self, previous=[], current=[], exception_map={}):
          """
          Find any objects that have been changed since the last run of the watcher.
          Add these items to the changed_items list.
          """
          prev_map = {item.location(): item for item in previous}
          curr_map = {item.location(): item for item in current}
  
          item_locations = list(set(curr_map).intersection(set(prev_map)))
          item_locations = [item_location for item_location in item_locations if not self.location_in_exception_map(item_location, exception_map)]
  
          for location in item_locations:
              prev_item = prev_map[location]
              curr_item = curr_map[location]
              # ChangeItem with and without ephemeral changes
              eph_change_item = None
              dur_change_item = None
  
              if not sub_dict(prev_item.config) == sub_dict(curr_item.config):
                  eph_change_item = ChangeItem.from_items(old_item=prev_item, new_item=curr_item, source_watcher=self)
  
              if self.ephemerals_skipped():
                  # deepcopy configs before filtering
                  dur_prev_item = deepcopy(prev_item)
                  dur_curr_item = deepcopy(curr_item)
                  # filter-out ephemeral paths in both old and new config dicts
                  for path in self.ephemeral_paths:
                      for cfg in [dur_prev_item.config, dur_curr_item.config]:
                          try:
                              dpath.util.delete(cfg, path, separator='$')
                          except PathNotFound:
                              pass
  
                  # now, compare only non-ephemeral paths
                  if not sub_dict(dur_prev_item.config) == sub_dict(dur_curr_item.config):
                      dur_change_item = ChangeItem.from_items(old_item=dur_prev_item, new_item=dur_curr_item,
                                                              source_watcher=self)
  
                  # store all changes, divided in specific categories
                  if eph_change_item:
                      self.ephemeral_items.append(eph_change_item)
                      app.logger.debug("%s: ephemeral changes in item %s/%s/%s" % (self.i_am_singular, eph_change_item.account, eph_change_item.region, eph_change_item.name))
                  if dur_change_item:
                      self.changed_items.append(dur_change_item)
                      app.logger.debug("%s: durable changes in item %s/%s/%s" % (self.i_am_singular, dur_change_item.account, dur_change_item.region, dur_change_item.name))
  
              elif eph_change_item is not None:
                  # store all changes, handle them all equally
                  self.changed_items.append(eph_change_item)
                  app.logger.debug("%s: changes in item %s/%s/%s" % (self.i_am_singular, eph_change_item.account, eph_change_item.region, eph_change_item.name))
  
      def find_changes(self, current=None, exception_map=None):
          """
          Identify changes between the configuration I have and what I had
          last time the watcher ran.
          This ignores any account/region which caused an exception during slurp.
          """
          current = current or []
          exception_map = exception_map or {}
  
          # Batching only logic here:
          if self.batched_size > 0:
              # Return the items that should be audited:
              return self.find_changes_batch(current, exception_map)
  
          else:
              prev = self.read_previous_items()
              self.find_deleted(previous=prev, current=current, exception_map=exception_map)
              self.find_new(previous=prev, current=current)
              self.find_modified(previous=prev, current=current, exception_map=exception_map)
  
      def find_changes_batch(self, items, exception_map):
          # Given the list of items, find new items that don't yet exist:
          durable_items = []
  
          from security_monkey.datastore_utils import hash_item, detect_change, persist_item
          for item in items:
              complete_hash, durable_hash = hash_item(item.config, self.ephemeral_paths)
  
              # Detect if a change occurred:
              is_change, change_type, db_item, created_changed = detect_change(
                  item, self.current_account[0], self.technology, complete_hash, durable_hash)
  
              if not is_change:
                  continue
  
              is_durable = (change_type == "durable")
  
              if is_durable:
                  durable_items.append(item)
  
              if created_changed == 'created':
                  self.created_items.append(ChangeItem.from_items(old_item=None, new_item=item, source_watcher=self))
  
              if created_changed == 'changed':
                  db_item.audit_issues = db_item.issues
                  db_item.config = db_item.revisions.first().config
                  self.changed_items.append(ChangeItem.from_items(old_item=db_item, new_item=item, source_watcher=self))
  
              persist_item(item, db_item, self.technology, self.current_account[0], complete_hash,
                           durable_hash, is_durable)
  
          return durable_items
  
      def find_deleted_batch(self, exception_map):
          from datastore_utils import inactivate_old_revisions
          existing_arns = [item["Arn"] for item in self.total_list if item.get("Arn")]
          deleted_items = inactivate_old_revisions(self, existing_arns, self.current_account[0], self.technology)
  
          for item in deleted_items:
              # An inactive revision has already been commited to the DB.
              # So here, we need to pull the last two revisions to build out our
              # ChangeItem.
              recent_revisions=item.revisions.limit(2).all()
              old_config=recent_revisions[1].config
              new_config=recent_revisions[0].config
              change_item = ChangeItem(
                  index=item.technology.name, region=item.region,
                  account=item.account.name, name=item.name, arn=item.arn,
                  old_config=old_config, new_config=new_config, active=False,
                  audit_issues=item.issues)
              self.deleted_items.append(change_item)
  
      def read_previous_items(self):
          """
          Pulls the last-recorded configuration from the database.
          :return: List of all items for the given technology and the given account.
          """
          prev_list = []
          for account in self.accounts:
              prev = self.datastore.get_all_ctype_filtered(tech=self.index, account=account, include_inactive=False)
              # Returns a map of {Item: ItemRevision}
              for item in prev:
                  item_revision = prev[item]
                  new_item = ChangeItem(index=self.index,
                                        region=item.region,
                                        account=item.account.name,
                                        name=item.name,
                                        new_config=item_revision.config)
                  prev_list.append(new_item)
  
          return prev_list
  
      def is_changed(self):
          """
          Note: It is intentional that self.ephemeral_items is not included here
          so that emails will not go out about those changes.
          Those changes will still be recorded in the database and visible in the UI.
          :return: boolean whether or not we've found any changes
          """
          return self.deleted_items or self.created_items or self.changed_items
  
      def issues_found(self):
          """
          Runs through any changed items to see if any have issues.
          :return: boolean whether any changed items have issues
          """
          has_issues = False
          has_new_issue = False
          has_unjustified_issue = False
          for item in self.created_items + self.changed_items:
              if item.audit_issues:
                  has_issues = True
                  if item.found_new_issue:
                      has_new_issue = True
                      has_unjustified_issue = True
                      break
                  for issue in item.confirmed_existing_issues:
                      if not issue.justified:
                          has_unjustified_issue = True
                          break
  
          return has_issues, has_new_issue, has_unjustified_issue
  
      def save(self):
          """
          save new configs, if necessary
          """
          app.logger.info("{} deleted {} in {}".format(len(self.deleted_items), self.i_am_plural, self.accounts))
          app.logger.info("{} created {} in {}".format(len(self.created_items), self.i_am_plural, self.accounts))
          for item in self.created_items + self.deleted_items:
              item.save(self.datastore)
  
          if self.ephemerals_skipped():
              changed_locations = [item.location() for item in self.changed_items]
  
              new_item_revisions = [item for item in self.ephemeral_items if item.location() in changed_locations]
              app.logger.info("{} changed {} in {}".format(len(new_item_revisions), self.i_am_plural, self.accounts))
              for item in new_item_revisions:
                  item.save(self.datastore)
  
              edit_item_revisions = [item for item in self.ephemeral_items if item.location() not in changed_locations]
              app.logger.info("{} ephemerally changed {} in {}".format(len(edit_item_revisions), self.i_am_plural, self.accounts))
              for item in edit_item_revisions:
                  item.save(self.datastore, ephemeral=True)
          else:
              app.logger.info("{} changed {} in {}".format(len(self.changed_items), self.i_am_plural, self.accounts))
              for item in self.changed_items:
                  item.save(self.datastore)
          report_watcher_changes(self)
  
      def plural_name(self):
          """
          Used for Jinja Template
          :return: i_am_plural
          """
          return self.i_am_plural
  
      def singular_name(self):
          """
          Used for Jinja Template
          :return: i_am_singular
          """
          return self.i_am_singular
  
      def get_interval(self):
          """ Returns interval time (in minutes) """
          config = WatcherConfig.query.filter(WatcherConfig.index == self.index).first()
          if config:
              return config.interval
  
          return self.interval
  
      def is_active(self):
          """ Returns active """
          config = WatcherConfig.query.filter(WatcherConfig.index == self.index).first()
          if config:
              return config.active
  
          return self.active
  
      def ephemerals_skipped(self):
          """ Returns whether ephemerals locations are ignored """
          return self.honor_ephemerals
  
  
  class ChangeItem(object):
      """
      Object tracks two different revisions of a given item.
      """
  
      def __init__(self, index=None, region=None, account=None, name=None, arn=None, old_config=None, new_config=None,
                   active=False, audit_issues=None, source_watcher=None):
          self.index = index
          self.region = region
          self.account = account
          self.name = name
          self.arn = arn
          self.old_config = old_config if old_config else {}
          self.new_config = new_config if new_config else {}
          self.active = active
          self.audit_issues = audit_issues or []
          self.confirmed_new_issues = []
          self.confirmed_fixed_issues = []
          self.confirmed_existing_issues = []
          self.found_new_issue = False
          self.watcher = source_watcher
  
      @classmethod
      def from_items(cls, old_item=None, new_item=None, source_watcher=None):
          """
          Create ChangeItem from two separate items.
          :return: An instance of ChangeItem
          """
          if not old_item and not new_item:
              return
          valid_item = new_item if new_item else old_item
          audit_issues = old_item.audit_issues if old_item else []
          active = True if new_item else False
          old_config = old_item.config if old_item else {}
          new_config = new_item.config if new_item else {}
          return cls(index=valid_item.index,
                     region=valid_item.region,
                     account=valid_item.account,
                     name=valid_item.name,
                     arn=valid_item.arn,
                     old_config=old_config,
                     new_config=new_config,
                     active=active,
                     audit_issues=audit_issues,
                     source_watcher=source_watcher)
  
      @property
      def config(self):
          return self.new_config
  
      def location(self):
          """
          Construct a location from the object.
          :return: tuple containing index, account, region, and name.
          """
          return (self.index, self.account, self.region, self.name)
  
      def get_pdiff_html(self):
          pdiff = PolicyDiff(self.new_config, self.old_config)
          return pdiff.produceDiffHTML()
  
      def _dict_for_template(self):
          return {
              'account': self.account,
              'region': self.region,
              'name': self.name,
              'confirmed_new_issues': self.confirmed_new_issues,
              'confirmed_fixed_issues': self.confirmed_fixed_issues,
              'confirmed_existing_issues': self.confirmed_existing_issues,
              'pdiff_html': self.get_pdiff_html()
          }
  
      def description(self):
          """
          Provide an HTML description of the object for change emails and the Jinja templates.
          :return: string of HTML describing the object.
          """
          jenv = get_jinja_env()
          template = jenv.get_template('jinja_change_item.html')
          body = template.render(self._dict_for_template())
          # app.logger.info(body)
          return body
  
      def save(self, datastore, ephemeral=False):
          """
          Save the item
          """
          app.logger.debug("Saving {}/{}/{}/{}\n\t{}".format(self.index, self.account, self.region, self.name, self.new_config))
          self.db_item = datastore.store(
              self.index,
              self.region,
              self.account,
              self.name,
              self.active,
              self.new_config,
              arn=self.arn,
              new_issues=self.audit_issues,
              ephemeral=ephemeral,
              source_watcher=self.watcher)
  
files: security_monkey/watcher.py
files: """
  .. module: security_monkey.watcher
      :platform: Unix
      :synopsis: Slurps the current config from AWS and compares it to what has previously
      been recorded in the database to find any changes.
  
  .. version:: $$VERSION$$
  .. moduleauthor:: Patrick Kelley <pkelley@netflix.com> @monkeysecurity
  
  """
  from botocore.exceptions import ClientError
  
  from common.PolicyDiff import PolicyDiff
  from common.utils import sub_dict
  from security_monkey import app
  from security_monkey.datastore import Account, IgnoreListEntry, db
  from security_monkey.datastore import Technology, WatcherConfig, store_exception
  from security_monkey.common.jinja import get_jinja_env
  from security_monkey.alerters.custom_alerter import report_watcher_changes
  
  from boto.exception import BotoServerError
  import time
  
  import datastore
  from copy import deepcopy
  import dpath.util
  from dpath.exceptions import PathNotFound
  
  watcher_registry = {}
  abstract_classes = set(['Watcher', 'CloudAuxWatcher', 'CloudAuxBatchedWatcher'])
  
  
  class WatcherType(type):
      def __init__(cls, name, bases, attrs):
          super(WatcherType, cls).__init__(name, bases, attrs)
          if cls.__name__ not in abstract_classes and cls.index:
              app.logger.debug("Registering watcher {} {}.{}".format(cls.index, cls.__module__, cls.__name__))
              watcher_registry[cls.index] = cls
  
  
  class Watcher(object):
      """Slurps the current config from AWS and compares it to what has previously
        been recorded in the database to find any changes."""
      index = 'abstract'
      i_am_singular = 'Abstract'
      i_am_plural = 'Abstracts'
      rate_limit_delay = 0
      ignore_list = []
      interval = 60    #in minutes
      active = True
      account_type = 'AWS'
      __metaclass__ = WatcherType
  
      def __init__(self, accounts=None, debug=False):
          """Initializes the Watcher"""
          self.datastore = datastore.Datastore()
          if not accounts:
              accounts = Account.query.filter(Account.third_party==False).filter(Account.active==True).all()
          else:
              accounts = Account.query.filter(Account.third_party==False).filter(Account.active==True).filter(Account.name.in_(accounts)).all()
          if not accounts:
              raise ValueError('Watcher needs a valid account')
          self.accounts = [account.name for account in accounts]
          self.account_identifiers = [account.identifier for account in accounts]
          self.debug = debug
          self.created_items = []
          self.deleted_items = []
          self.changed_items = []
          self.ephemeral_items = []
          # TODO: grab these from DB, keyed on account
          self.rate_limit_delay = 0
          self.honor_ephemerals = False
          self.ephemeral_paths = []
  
          # Batching attributes:
          self.batched_size = 0   # Don't batch anything by default
          self.done_slurping = True   # Don't batch anything by default
          self.total_list = []    # This will hold the full list of items to batch over
          self.batch_counter = 0  # Keeps track of the batch we are on -- can be used for retry logic
          self.current_account = None  # Tuple that holds the current account and account index we are on.
          self.technology = None
          # Region is probably not needed if we are using CloudAux's iter_account_region -- will test this in
          # the future as we add more items with batching support.
  
      def prep_for_slurp(self):
          """
          Should be run before slurp is run to grab the IgnoreList.
          """
          query = IgnoreListEntry.query
          query = query.join((Technology, Technology.id == IgnoreListEntry.tech_id))
          self.ignore_list = query.filter(Technology.name == self.index).all()
  
      def prep_for_batch_slurp(self):
          """
          Should be run before batching slurps to set the current account (and region).
  
          This will load the DB objects for account and technology for where we are currently at in the process.
          :return:
          """
          self.prep_for_slurp()
  
          # Which account are we currently on?
          if not self.current_account:
              index = 0
  
              # Get the Technology
              # If technology doesn't exist, then create it:
              technology = Technology.query.filter(Technology.name == self.index).first()
              if not technology:
                  technology = Technology(name=self.index)
                  db.session.add(technology)
                  db.session.commit()
                  app.logger.info("Technology: {} did not exist... created it...".format(self.index))
  
              self.technology = technology
          else:
              index = self.current_account[1] + 1
  
          self.current_account = (Account.query.filter(Account.name == self.accounts[index]).one(), index)
  
          # We will not be using CloudAux's iter_account_region for multi-account -- we want
          # to have per-account level of batching
          self.total_list = []    # Reset the total list for a new account to run against.
          self.done_slurping = False
          self.batch_counter = 0
  
      def check_ignore_list(self, name):
          """
          See if the given item has a name flagging it to be ignored by security_monkey.
          """
          for result in self.ignore_list:
              # Empty prefix comes back as None instead of an empty string ...
              prefix = result.prefix or ""
              if name.lower().startswith(prefix.lower()):
                  app.logger.info("Ignoring {}/{} because of IGNORELIST prefix {}".format(self.index, name, result.prefix))
                  return True
  
          return False
  
      def wrap_aws_rate_limited_call(self, awsfunc, *args, **nargs):
          attempts = 0
  
          def increase_delay():
              if self.rate_limit_delay == 0:
                  self.rate_limit_delay = 1
                  app.logger.warn(('Being rate-limited by AWS. Increasing delay on tech {} ' +
                                  'in account {} from 0 to 1 second. Attempt {}')
                                  .format(self.index, self.accounts, attempts))
              elif self.rate_limit_delay < 4:
                  self.rate_limit_delay = self.rate_limit_delay * 2
                  app.logger.warn(('Still being rate-limited by AWS. Increasing delay on tech {} ' +
                                  'in account {} to {} seconds. Attempt {}')
                                  .format(self.index, self.accounts, self.rate_limit_delay, attempts))
              else:
                  app.logger.warn(('Still being rate-limited by AWS. Keeping delay on tech {} ' +
                                  'in account {} at {} seconds. Attempt {}')
                                  .format(self.index, self.accounts, self.rate_limit_delay, attempts))
  
          while True:
              attempts = attempts + 1
              try:
                  if self.rate_limit_delay > 0:
                      time.sleep(self.rate_limit_delay)
  
                  retval = awsfunc(*args, **nargs)
  
                  if self.rate_limit_delay > 0:
                      app.logger.warn("Successfully Executed Rate-Limited Function. "
                                      "Tech: {} Account: {}. Removing sleep period."
                                      .format(self.index, self.accounts))
                      self.rate_limit_delay = 0
  
                  return retval
              except BotoServerError as e:  # Boto
                  if not e.error_code == 'Throttling':
                      raise e
                  increase_delay()
              except ClientError as e:  # Botocore
                  if not e.response["Error"]["Code"] == "Throttling":
                      raise e
                  increase_delay()
  
      def created(self):
          """
          Used by the Jinja templates
          :returns: True if created_items is not empty
          :returns: False otherwise.
          """
          return len(self.created_items) > 0
  
      def deleted(self):
          """
          Used by the Jinja templates
          :returns: True if deleted_items is not empty
          :returns: False otherwise.
          """
          return len(self.deleted_items) > 0
  
      def changed(self):
          """
          Used by the Jinja templates
          :returns: True if changed_items is not empty
          :returns: False otherwise.
          """
          return len(self.changed_items) > 0
  
      def slurp_list(self):
          """
          This will fetch all the items in question that will need to get slurped.
          This is used to know what we are going to have to batch up.
          :return:
          """
          raise NotImplementedError()
  
      def slurp(self):
          """
          method to slurp configuration from AWS for whatever it is that I'm
          interested in. This will be overridden for each technology.
          """
          raise NotImplementedError()
  
      def slurp_exception(self, location=None, exception=None, exception_map={}, source="watcher"):
          """
          Logs any exceptions that happen in slurp and adds them to the exception_map
          using their location as the key.  The location is a tuple in the form:
          (technology, account, region, item_name) that describes the object where the exception occurred.
          Location can also exclude an item_name if the exception is region wide.
          """
          if location in exception_map:
              app.logger.debug("Exception map already has location {}. This should not happen.".format(location))
          exception_map[location] = exception
          app.logger.debug("Adding {} to the exceptions list. Exception was: {}".format(location, str(exception)))
  
          # Store it to the database:
          store_exception(source, location, exception)
  
      def location_in_exception_map(self, item_location, exception_map={}):
          """
          Determines whether a given location is covered by an exception already in the
          exception map.
  
              Item location: (self.index, self.account, self.region, self.name)
            exception Maps: (index, account, region, name)
                            (index, account, region)
                            (index, account)
  
              :returns: True if location is covered by an entry in the exception map.
              :returns: False if location is not covered by an entry in the exception map.
          """
          # Exact Match
          if item_location in exception_map:
              app.logger.debug("Skipping {} due to an item-level exception {}.".format(item_location, exception_map[item_location]))
              return True
  
          # (index, account, region)
          if item_location[0:3] in exception_map:
              app.logger.debug("Skipping {} due to a region-level exception {}.".format(item_location, exception_map[item_location[0:3]]))
              return True
  
          # (index, account)
          if item_location[0:2] in exception_map:
              app.logger.debug("Skipping {} due to an account-level exception {}.".format(item_location, exception_map[item_location[0:2]]))
              return True
  
          # (index)
          if item_location[0:1] in exception_map:
              app.logger.debug("Skipping {} due to a technology-level exception {}.".format(item_location, exception_map[item_location[0:1]]))
              return True
  
          return False
  
      def find_deleted(self, previous=[], current=[], exception_map={}):
          """
          Find any items that have been deleted since the last run of the watcher.
          Add these items to the deleted_items list.
          """
          prev_map = {item.location(): item for item in previous}
          curr_map = {item.location(): item for item in current}
  
          item_locations = list(set(prev_map).difference(set(curr_map)))
          item_locations = [item_location for item_location in item_locations if not self.location_in_exception_map(item_location, exception_map)]
          list_deleted_items = [prev_map[item] for item in item_locations]
  
          for item in list_deleted_items:
              deleted_change_item = ChangeItem.from_items(old_item=item, new_item=None, source_watcher=self)
              app.logger.debug("%s: %s/%s/%s deleted" % (self.i_am_singular, item.account, item.region, item.name))
              self.deleted_items.append(deleted_change_item)
  
      def find_new(self, previous=[], current=[]):
          """
          Find any new objects that have been created since the last run of the watcher.
          Add these items to the created_items list.
          """
          prev_map = {item.location(): item for item in previous}
          curr_map = {item.location(): item for item in current}
  
          item_locations = list(set(curr_map).difference(set(prev_map)))
          list_new_items = [curr_map[item] for item in item_locations]
  
          for item in list_new_items:
              new_change_item = ChangeItem.from_items(old_item=None, new_item=item, source_watcher=self)
              self.created_items.append(new_change_item)
              app.logger.debug("%s: %s/%s/%s created" % (self.i_am_singular, item.account, item.region, item.name))
  
      def find_modified(self, previous=[], current=[], exception_map={}):
          """
          Find any objects that have been changed since the last run of the watcher.
          Add these items to the changed_items list.
          """
          prev_map = {item.location(): item for item in previous}
          curr_map = {item.location(): item for item in current}
  
          item_locations = list(set(curr_map).intersection(set(prev_map)))
          item_locations = [item_location for item_location in item_locations if not self.location_in_exception_map(item_location, exception_map)]
  
          for location in item_locations:
              prev_item = prev_map[location]
              curr_item = curr_map[location]
              # ChangeItem with and without ephemeral changes
              eph_change_item = None
              dur_change_item = None
  
              if not sub_dict(prev_item.config) == sub_dict(curr_item.config):
                  eph_change_item = ChangeItem.from_items(old_item=prev_item, new_item=curr_item, source_watcher=self)
  
              if self.ephemerals_skipped():
                  # deepcopy configs before filtering
                  dur_prev_item = deepcopy(prev_item)
                  dur_curr_item = deepcopy(curr_item)
                  # filter-out ephemeral paths in both old and new config dicts
                  for path in self.ephemeral_paths:
                      for cfg in [dur_prev_item.config, dur_curr_item.config]:
                          try:
                              dpath.util.delete(cfg, path, separator='$')
                          except PathNotFound:
                              pass
  
                  # now, compare only non-ephemeral paths
                  if not sub_dict(dur_prev_item.config) == sub_dict(dur_curr_item.config):
                      dur_change_item = ChangeItem.from_items(old_item=dur_prev_item, new_item=dur_curr_item,
                                                              source_watcher=self)
  
                  # store all changes, divided in specific categories
                  if eph_change_item:
                      self.ephemeral_items.append(eph_change_item)
                      app.logger.debug("%s: ephemeral changes in item %s/%s/%s" % (self.i_am_singular, eph_change_item.account, eph_change_item.region, eph_change_item.name))
                  if dur_change_item:
                      self.changed_items.append(dur_change_item)
                      app.logger.debug("%s: durable changes in item %s/%s/%s" % (self.i_am_singular, dur_change_item.account, dur_change_item.region, dur_change_item.name))
  
              elif eph_change_item is not None:
                  # store all changes, handle them all equally
                  self.changed_items.append(eph_change_item)
                  app.logger.debug("%s: changes in item %s/%s/%s" % (self.i_am_singular, eph_change_item.account, eph_change_item.region, eph_change_item.name))
  
      def find_changes(self, current=None, exception_map=None):
          """
          Identify changes between the configuration I have and what I had
          last time the watcher ran.
          This ignores any account/region which caused an exception during slurp.
          """
          current = current or []
          exception_map = exception_map or {}
  
          # Batching only logic here:
          if self.batched_size > 0:
              # Return the items that should be audited:
              return self.find_changes_batch(current, exception_map)
  
          else:
              prev = self.read_previous_items()
              self.find_deleted(previous=prev, current=current, exception_map=exception_map)
              self.find_new(previous=prev, current=current)
              self.find_modified(previous=prev, current=current, exception_map=exception_map)
  
      def find_changes_batch(self, items, exception_map):
          # Given the list of items, find new items that don't yet exist:
          durable_items = []
  
          from security_monkey.datastore_utils import hash_item, detect_change, persist_item
          for item in items:
              complete_hash, durable_hash = hash_item(item.config, self.ephemeral_paths)
  
              # Detect if a change occurred:
              is_change, change_type, db_item, created_changed = detect_change(
                  item, self.current_account[0], self.technology, complete_hash, durable_hash)
  
              if not is_change:
                  continue
  
              is_durable = (change_type == "durable")
  
              if is_durable:
                  durable_items.append(item)
  
              if created_changed == 'created':
                  self.created_items.append(ChangeItem.from_items(old_item=None, new_item=item, source_watcher=self))
  
              if created_changed == 'changed':
                  db_item.audit_issues = db_item.issues
                  db_item.config = db_item.revisions.first().config
                  self.changed_items.append(ChangeItem.from_items(old_item=db_item, new_item=item, source_watcher=self))
  
              persist_item(item, db_item, self.technology, self.current_account[0], complete_hash,
                           durable_hash, is_durable)
  
          return durable_items
  
      def find_deleted_batch(self, exception_map):
          from datastore_utils import inactivate_old_revisions
          existing_arns = [item["Arn"] for item in self.total_list if item.get("Arn")]
          deleted_items = inactivate_old_revisions(self, existing_arns, self.current_account[0], self.technology)
  
          for item in deleted_items:
              # An inactive revision has already been commited to the DB.
              # So here, we need to pull the last two revisions to build out our
              # ChangeItem.
              recent_revisions=item.revisions.limit(2).all()
              old_config=recent_revisions[1].config
              new_config=recent_revisions[0].config
              change_item = ChangeItem(
                  index=item.technology.name, region=item.region,
                  account=item.account.name, name=item.name, arn=item.arn,
                  old_config=old_config, new_config=new_config, active=False,
                  audit_issues=item.issues)
              self.deleted_items.append(change_item)
  
      def read_previous_items(self):
          """
          Pulls the last-recorded configuration from the database.
          :return: List of all items for the given technology and the given account.
          """
          prev_list = []
          for account in self.accounts:
              prev = self.datastore.get_all_ctype_filtered(tech=self.index, account=account, include_inactive=False)
              # Returns a map of {Item: ItemRevision}
              for item in prev:
                  item_revision = prev[item]
                  new_item = ChangeItem(index=self.index,
                                        region=item.region,
                                        account=item.account.name,
                                        name=item.name,
                                        new_config=item_revision.config)
                  prev_list.append(new_item)
  
          return prev_list
  
      def is_changed(self):
          """
          Note: It is intentional that self.ephemeral_items is not included here
          so that emails will not go out about those changes.
          Those changes will still be recorded in the database and visible in the UI.
          :return: boolean whether or not we've found any changes
          """
          return self.deleted_items or self.created_items or self.changed_items
  
      def issues_found(self):
          """
          Runs through any changed items to see if any have issues.
          :return: boolean whether any changed items have issues
          """
          has_issues = False
          has_new_issue = False
          has_unjustified_issue = False
          for item in self.created_items + self.changed_items:
              if item.audit_issues:
                  has_issues = True
                  if item.found_new_issue:
                      has_new_issue = True
                      has_unjustified_issue = True
                      break
                  for issue in item.confirmed_existing_issues:
                      if not issue.justified:
                          has_unjustified_issue = True
                          break
  
          return has_issues, has_new_issue, has_unjustified_issue
  
      def save(self):
          """
          save new configs, if necessary
          """
          app.logger.info("{} deleted {} in {}".format(len(self.deleted_items), self.i_am_plural, self.accounts))
          app.logger.info("{} created {} in {}".format(len(self.created_items), self.i_am_plural, self.accounts))
          for item in self.created_items + self.deleted_items:
              item.save(self.datastore)
  
          if self.ephemerals_skipped():
              changed_locations = [item.location() for item in self.changed_items]
  
              new_item_revisions = [item for item in self.ephemeral_items if item.location() in changed_locations]
              app.logger.info("{} changed {} in {}".format(len(new_item_revisions), self.i_am_plural, self.accounts))
              for item in new_item_revisions:
                  item.save(self.datastore)
  
              edit_item_revisions = [item for item in self.ephemeral_items if item.location() not in changed_locations]
              app.logger.info("{} ephemerally changed {} in {}".format(len(edit_item_revisions), self.i_am_plural, self.accounts))
              for item in edit_item_revisions:
                  item.save(self.datastore, ephemeral=True)
          else:
              app.logger.info("{} changed {} in {}".format(len(self.changed_items), self.i_am_plural, self.accounts))
              for item in self.changed_items:
                  item.save(self.datastore)
          report_watcher_changes(self)
  
      def plural_name(self):
          """
          Used for Jinja Template
          :return: i_am_plural
          """
          return self.i_am_plural
  
      def singular_name(self):
          """
          Used for Jinja Template
          :return: i_am_singular
          """
          return self.i_am_singular
  
      def get_interval(self):
          """ Returns interval time (in minutes) """
          config = WatcherConfig.query.filter(WatcherConfig.index == self.index).first()
          if config:
              return config.interval
  
          return self.interval
  
      def is_active(self):
          """ Returns active """
          config = WatcherConfig.query.filter(WatcherConfig.index == self.index).first()
          if config:
              return config.active
  
          return self.active
  
      def ephemerals_skipped(self):
          """ Returns whether ephemerals locations are ignored """
          return self.honor_ephemerals
  
  
  class ChangeItem(object):
      """
      Object tracks two different revisions of a given item.
      """
  
      def __init__(self, index=None, region=None, account=None, name=None, arn=None, old_config=None, new_config=None,
                   active=False, audit_issues=None, source_watcher=None):
          self.index = index
          self.region = region
          self.account = account
          self.name = name
          self.arn = arn
          self.old_config = old_config if old_config else {}
          self.new_config = new_config if new_config else {}
          self.active = active
          self.audit_issues = audit_issues or []
          self.confirmed_new_issues = []
          self.confirmed_fixed_issues = []
          self.confirmed_existing_issues = []
          self.found_new_issue = False
          self.watcher = source_watcher
  
      @classmethod
      def from_items(cls, old_item=None, new_item=None, source_watcher=None):
          """
          Create ChangeItem from two separate items.
          :return: An instance of ChangeItem
          """
          if not old_item and not new_item:
              return
          valid_item = new_item if new_item else old_item
          audit_issues = old_item.audit_issues if old_item else []
          active = True if new_item else False
          old_config = old_item.config if old_item else {}
          new_config = new_item.config if new_item else {}
          return cls(index=valid_item.index,
                     region=valid_item.region,
                     account=valid_item.account,
                     name=valid_item.name,
                     arn=valid_item.arn,
                     old_config=old_config,
                     new_config=new_config,
                     active=active,
                     audit_issues=audit_issues,
                     source_watcher=source_watcher)
  
      @property
      def config(self):
          return self.new_config
  
      def location(self):
          """
          Construct a location from the object.
          :return: tuple containing index, account, region, and name.
          """
          return (self.index, self.account, self.region, self.name)
  
      def get_pdiff_html(self):
          pdiff = PolicyDiff(self.new_config, self.old_config)
          return pdiff.produceDiffHTML()
  
      def _dict_for_template(self):
          return {
              'account': self.account,
              'region': self.region,
              'name': self.name,
              'confirmed_new_issues': self.confirmed_new_issues,
              'confirmed_fixed_issues': self.confirmed_fixed_issues,
              'confirmed_existing_issues': self.confirmed_existing_issues,
              'pdiff_html': self.get_pdiff_html()
          }
  
      def description(self):
          """
          Provide an HTML description of the object for change emails and the Jinja templates.
          :return: string of HTML describing the object.
          """
          jenv = get_jinja_env()
          template = jenv.get_template('jinja_change_item.html')
          body = template.render(self._dict_for_template())
          # app.logger.info(body)
          return body
  
      def save(self, datastore, ephemeral=False):
          """
          Save the item
          """
          app.logger.debug("Saving {}/{}/{}/{}\n\t{}".format(self.index, self.account, self.region, self.name, self.new_config))
          self.db_item = datastore.store(
              self.index,
              self.region,
              self.account,
              self.name,
              self.active,
              self.new_config,
              arn=self.arn,
              new_issues=self.audit_issues,
              ephemeral=ephemeral,
              source_watcher=self.watcher)
  
files: security_monkey/watcher.py
files: #     Copyright 2018 Netflix, Inc.
  #
  #     Licensed under the Apache License, Version 2.0 (the "License");
  #     you may not use this file except in compliance with the License.
  #     You may obtain a copy of the License at
  #
  #         http://www.apache.org/licenses/LICENSE-2.0
  #
  #     Unless required by applicable law or agreed to in writing, software
  #     distributed under the License is distributed on an "AS IS" BASIS,
  #     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #     See the License for the specific language governing permissions and
  #     limitations under the License.
  """
  .. module: security_monkey.watchers.sqs
      :platform: Unix
  
  .. version:: $$VERSION$$
  .. moduleauthor:: Mike Grima <mgrima@netflix.com>
  """
  from botocore.exceptions import ClientError
  from cloudaux.aws.sqs import list_queues
  from cloudaux.orchestration.aws.sqs import get_queue
  
  from security_monkey import app
  from security_monkey.cloudaux_batched_watcher import CloudAuxBatchedWatcher
  
  
  class SQS(CloudAuxBatchedWatcher):
      index = 'sqs'
      i_am_singular = 'SQS Policy'
      i_am_plural = 'SQS Policies'
  
      def __init__(self, **kwargs):
          super(SQS, self).__init__(**kwargs)
          self.service_name = "sqs"
          self.honor_ephemerals = True
          self.ephemeral_paths = [
              '_version',
              'Attributes$LastModifiedTimestamp',
              'Attributes$ApproximateNumberOfMessagesNotVisible',
              'Attributes$ApproximateNumberOfMessages',
              'Attributes$ApproximateNumberOfMessagesDelayed'
          ]
          self.batched_size = 200
  
          # SQS returns a list of URLs. The DB wants the total list of items to be a dict that contains
          # an "Arn" field. Since that is not present, this is used to help update the list with the ARNs
          # when they are discovered.
          self.corresponding_items = {}
  
      def get_name_from_list_output(self, item):
          # SQS returns URLs. Need to deconstruct the URL to pull out the name :/
          name = item["Url"].split("{}/".format(self.account_identifiers[0]))[1]
  
          return name
  
      def list_method(self, **kwargs):
          """
          Get the list of SQS queues. Also, create the corresponding lookup table with URL -> position
          in the total list.
          :param kwargs:
          :return:
          """
          items = []
          queues = list_queues(**kwargs)
  
          # Offset by the existing items in the list (from other regions)
          offset = len(self.corresponding_items)
          queue_count = -1
  
          for item_count in range(0, len(queues)):
              if self.corresponding_items.get(queues[item_count]):
                  app.logger.error("[?] Received a duplicate item in the SQS list: {}. Skipping it.".format(queues[item_count]))
                  continue
              queue_count += 1
              items.append({"Url": queues[item_count], "Region": kwargs["region"]})
              self.corresponding_items[queues[item_count]] = queue_count + offset
  
          return items
  
      def get_method(self, item, **kwargs):
          try:
              queue = get_queue(item["Url"], **kwargs)
  
              # Update the current position in the total list by replacing the SQS queue URL with
              # the ARN of the queue
              self.total_list[self.corresponding_items[item["Url"]]] = {"Arn": queue["Arn"]}
          except ClientError as ce:
              # In case the queue was created and then deleted really quickly:
              if "NonExistentQueue" in ce.response["Error"]["Code"]:
                  app.logger.debug("Queue with URL: {} - was listed, but is no longer present".format(item["Url"]))
                  return
              else:
                  raise ce
  
          return queue
  
files: security_monkey/watchers/sqs.py
files: #     Copyright 2018 Netflix, Inc.
  #
  #     Licensed under the Apache License, Version 2.0 (the "License");
  #     you may not use this file except in compliance with the License.
  #     You may obtain a copy of the License at
  #
  #         http://www.apache.org/licenses/LICENSE-2.0
  #
  #     Unless required by applicable law or agreed to in writing, software
  #     distributed under the License is distributed on an "AS IS" BASIS,
  #     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #     See the License for the specific language governing permissions and
  #     limitations under the License.
  """
  .. module: security_monkey.watchers.sqs
      :platform: Unix
  
  .. version:: $$VERSION$$
  .. moduleauthor:: Mike Grima <mgrima@netflix.com>
  """
  from botocore.exceptions import ClientError
  from cloudaux.aws.sqs import list_queues
  from cloudaux.orchestration.aws.sqs import get_queue
  
  from security_monkey import app
  from security_monkey.cloudaux_batched_watcher import CloudAuxBatchedWatcher
  
  
  class SQS(CloudAuxBatchedWatcher):
      index = 'sqs'
      i_am_singular = 'SQS Policy'
      i_am_plural = 'SQS Policies'
  
      def __init__(self, **kwargs):
          super(SQS, self).__init__(**kwargs)
          self.service_name = "sqs"
          self.honor_ephemerals = True
          self.ephemeral_paths = [
              '_version',
              'Attributes$LastModifiedTimestamp',
              'Attributes$ApproximateNumberOfMessagesNotVisible',
              'Attributes$ApproximateNumberOfMessages',
              'Attributes$ApproximateNumberOfMessagesDelayed'
          ]
          self.batched_size = 200
  
          # SQS returns a list of URLs. The DB wants the total list of items to be a dict that contains
          # an "Arn" field. Since that is not present, this is used to help update the list with the ARNs
          # when they are discovered.
          self.corresponding_items = {}
  
      def get_name_from_list_output(self, item):
          # SQS returns URLs. Need to deconstruct the URL to pull out the name :/
          name = item["Url"].split("{}/".format(self.account_identifiers[0]))[1]
  
          return name
  
      def list_method(self, **kwargs):
          """
          Get the list of SQS queues. Also, create the corresponding lookup table with URL -> position
          in the total list.
          :param kwargs:
          :return:
          """
          items = []
          queues = list_queues(**kwargs)
  
          # Offset by the existing items in the list (from other regions)
          offset = len(self.corresponding_items)
  
          for i in range(0, len(queues)):
              items.append({"Url": queues[i], "Region": kwargs["region"]})
              self.corresponding_items[queues[i]] = i + offset
  
          return items
  
      def get_method(self, item, **kwargs):
          try:
              queue = get_queue(item["Url"], **kwargs)
  
              # Update the current position in the total list by replacing the SQS queue URL with
              # the ARN of the queue
              self.total_list[self.corresponding_items[item["Url"]]] = {"Arn": queue["Arn"]}
          except ClientError as ce:
              # In case the queue was created and then deleted really quickly:
              if "NonExistentQueue" in ce.response["Error"]["Code"]:
                  app.logger.debug("Queue with URL: {} - was listed, but is no longer present".format(item["Url"]))
                  return
              else:
                  raise ce
  
          return queue
  
files: security_monkey/watchers/sqs.py