files: import hashlib
  import json
  
  import datetime
  import dpath.util
  from dpath.exceptions import PathNotFound
  from copy import deepcopy
  
  from security_monkey import datastore, app
  from cloudaux.orchestration.aws.arn import ARN
  from security_monkey.datastore import Item, ItemRevision, ItemAudit
  
  prims = [int, str, unicode, bool, float, type(None)]
  
  
  def persist_item(item, db_item, technology, account, complete_hash, durable_hash, durable):
      if not db_item:
          if account.account_type.name != "AWS":
              db_item = create_item(item, technology, account)
          else:
              db_item = create_item_aws(item, technology, account)
  
      if db_item.latest_revision_complete_hash == complete_hash:
          app.logger.debug("Change persister doesn't see any change. Ignoring...")
          return
  
      # Create the new revision
      if durable:
          revision = create_revision(item.config, db_item)
          db_item.revisions.append(revision)
  
      # Ephemeral -- update the existing revision:
      else:
          revision = db_item.revisions.first()
          revision.date_last_ephemeral_change = datetime.datetime.utcnow()
          revision.config = item.config
          app.logger.debug("Persisting EPHEMERAL change to item: {technology}/{account}/{item}".format(
              technology=technology.name, account=account.name, item=db_item.name
          ))
  
      db_item.latest_revision_complete_hash = complete_hash
      db_item.latest_revision_durable_hash = durable_hash
  
      datastore.db.session.add(db_item)
      datastore.db.session.add(revision)
      datastore.db.session.commit()
  
      if durable:
          app.logger.debug("Persisting DURABLE change to item: {technology}/{account}/{item}".format(
              technology=technology.name, account=account.name, item=db_item.name
          ))
          datastore.db.session.refresh(revision)
          db_item.latest_revision_id = revision.id
          datastore.db.session.add(revision)
          datastore.db.session.add(db_item)
          datastore.db.session.commit()
  
  
  def is_active(config):
      if config.keys() == ['Arn']:
          return False
  
      if set(config.keys()) == {'account_number', 'technology', 'region', 'name'}:
          return False
  
      return True
  
  
  def create_revision(config, db_item):
      return ItemRevision(
          active=is_active(config),
          config=config,
          item_id=db_item.id,
      )
  
  
  def create_item_aws(item, technology, account):
      arn = ARN(item.config.get('Arn'))
      return Item(
          region=arn.region or 'universal',
          name=arn.parsed_name or arn.name,
          arn=item.config.get('Arn'),
          tech_id=technology.id,
          account_id=account.id
      )
  
  
  def create_item(item, technology, account):
      return Item(
          region=item.region or 'universal',
          name=item.name,
          arn=item.arn,
          tech_id=technology.id,
          account_id=account.id
      )
  
  
  def detect_change(item, account, technology, complete_hash, durable_hash):
      """
      Checks the database to see if the latest revision of the specified
      item matches what Security Monkey has pulled from AWS.
  
      Note: this method makes no distinction between a changed item, a new item,
      a deleted item, or one that only has changes in the ephemeral section.
  
      :param item: dict describing an item tracked by Security Monkey
      :param hash: hash of the item dict for quicker change detection
      :
  
  [... the rest is omitted ...]
files: security_monkey/datastore_utils.py
files: import hashlib
  import json
  
  import datetime
  import dpath.util
  from dpath.exceptions import PathNotFound
  from copy import deepcopy
  
  from security_monkey import datastore, app
  from cloudaux.orchestration.aws.arn import ARN
  from security_monkey.datastore import Item, ItemRevision, ItemAudit
  
  prims = [int, str, unicode, bool, float, type(None)]
  
  
  def persist_item(item, db_item, technology, account, complete_hash, durable_hash, durable):
      if not db_item:
          if account.account_type.name != "AWS":
              db_item = create_item(item, technology, account)
          else:
              db_item = create_item_aws(item, technology, account)
  
      if db_item.latest_revision_complete_hash == complete_hash:
          app.logger.debug("Change persister doesn't see any change. Ignoring...")
          return
  
      # Create the new revision
      if durable:
          revision = create_revision(item.config, db_item)
          db_item.revisions.append(revision)
  
      # Ephemeral -- update the existing revision:
      else:
          revision = db_item.revisions.first()
          revision.date_last_ephemeral_change = datetime.datetime.utcnow()
          revision.config = item.config
          app.logger.debug("Persisting EPHEMERAL change to item: {technology}/{account}/{item}".format(
              technology=technology.name, account=account.name, item=db_item.name
          ))
  
      db_item.latest_revision_complete_hash = complete_hash
      db_item.latest_revision_durable_hash = durable_hash
  
      datastore.db.session.add(db_item)
      datastore.db.session.add(revision)
      datastore.db.session.commit()
  
      if durable:
          app.logger.debug("Persisting DURABLE change to item: {technology}/{account}/{item}".format(
              technology=technology.name, account=account.name, item=db_item.name
          ))
          datastore.db.session.refresh(revision)
          db_item.latest_revision_id = revision.id
          datastore.db.session.add(revision)
          datastore.db.session.add(db_item)
          datastore.db.session.commit()
  
  
  def is_active(config):
      if config.keys() == ['Arn']:
          return False
  
      if set(config.keys()) == {'account_number', 'technology', 'region', 'name'}:
          return False
  
      return True
  
  
  def create_revision(config, db_item):
      return ItemRevision(
          active=is_active(config),
          config=config,
          item_id=db_item.id,
      )
  
  
  def create_item_aws(item, technology, account):
      arn = ARN(item.config.get('Arn'))
      return Item(
          region=arn.region or 'universal',
          name=arn.parsed_name or arn.name,
          arn=item.config.get('Arn'),
          tech_id=technology.id,
          account_id=account.id
      )
  
  
  def create_item(item, technology, account):
      return Item(
          region=item.region or 'universal',
          name=item.name,
          arn=item.arn,
          tech_id=technology.id,
          account_id=account.id
      )
  
  
  
  def detect_change(item, account, technology, complete_hash, durable_hash):
      """
      Checks the database to see if the latest revision of the specified
      item matches what Security Monkey has pulled from AWS.
  
      Note: this method makes no distinction between a changed item, a new item,
      a deleted item, or one that only has changes in the ephemeral section.
  
      :param item: dict describing an item tracked by Security Monkey
      :param hash: hash of the item dict for quicker change detection
      :
  
  [... the rest is omitted ...]
files: security_monkey/datastore_utils.py
files: """
  .. module: security_monkey.task_scheduler.tasks
      :platform: Unix
      :synopsis: Sets up the Celery task scheduling for watching, auditing, and reporting changes in the environment.
  
  .. version:: $$VERSION$$
  .. moduleauthor:: Mike Grima <mgrima@netflix.com>
  
  """
  import time
  import traceback
  
  from security_monkey import app, db, jirasync, sentry
  from security_monkey.alerter import Alerter
  from security_monkey.datastore import store_exception, clear_old_exceptions, Technology, Account, Item, ItemRevision
  from security_monkey.monitors import get_monitors, get_monitors_and_dependencies
  from security_monkey.reporter import Reporter
  from security_monkey.task_scheduler.util import CELERY, setup
  from sqlalchemy.exc import OperationalError, InvalidRequestError, StatementError
  
  
  @CELERY.task(bind=True, max_retries=3)
  def task_account_tech(self, account_name, technology_name):
      setup()
      app.logger.info("[ ] Executing Celery task for account: {}, technology: {}".format(account_name, technology_name))
      time1 = time.time()
  
      try:
          reporter_logic(account_name, technology_name)
  
          time2 = time.time()
          app.logger.info('[@] Run Account for Technology (%s/%s) took %0.1f s' % (account_name,
                                                                                   technology_name, (time2 - time1)))
          app.logger.info(
              "[+] Completed Celery task for account: {}, technology: {}".format(account_name, technology_name))
      except Exception as e:
          if sentry:
              sentry.captureException()
          app.logger.error("[X] Task Account Scheduler Exception ({}/{}): {}".format(account_name, technology_name, e))
          app.logger.error(traceback.format_exc())
          store_exception("scheduler-exception-on-watch", None, e)
          raise self.retry(exc=e)
  
  
  @CELERY.task(bind=True, max_retries=3)
  def task_audit(self, account_name, technology_name):
      setup()
  
      app.logger.info("[ ] Executing Celery task to audit changes for Account: {} Technology: {}".format(account_name,
                                                                                                         technology_name))
      try:
          audit_changes([account_name], [technology_name], True)
  
          app.logger.info("[+] Completed Celery task for account: {}, technology: {}".format(account_name,
                                                                                             technology_name))
  
      except Exception as e:
          if sentry:
              sentry.captureException()
          app.logger.error("[X] Task Audit Scheduler Exception ({}/{}): {}".format(account_name, technology_name, e))
          app.logger.error(traceback.format_exc())
          store_exception("scheduler-exception-on-audit", None, e)
          self.retry(exc=e)
  
  
  @CELERY.task()
  def clear_expired_exceptions():
      app.logger.info("[ ] Clearing out exceptions that have an expired TTL...")
      clear_old_exceptions()
      app.logger.info("[-] Completed clearing out exceptions that have an expired TTL.")
  
  
  def fix_orphaned_deletions(account_name, technology_name):
      """
      Possible issue with orphaned items. This will check if there are any, and will assume that the item
      was deleted. This will create a deletion change record to it.
  
      :param account_name:
      :param technology_name:
      :return:
      """
      # If technology doesn't exist, then create it:
      technology = Technology.query.filter(Technology.name == technology_name).first()
      if not technology:
          technology = Technology(name=technology_name)
          db.session.add(technology)
         
  
  [... the rest is omitted ...]
files: security_monkey/task_scheduler/tasks.py
files: """
  .. module: security_monkey.task_scheduler.tasks
      :platform: Unix
      :synopsis: Sets up the Celery task scheduling for watching, auditing, and reporting changes in the environment.
  
  .. version:: $$VERSION$$
  .. moduleauthor:: Mike Grima <mgrima@netflix.com>
  
  """
  import time
  import traceback
  
  from security_monkey import app, db, jirasync, sentry
  from security_monkey.alerter import Alerter
  from security_monkey.datastore import store_exception, clear_old_exceptions
  from security_monkey.monitors import get_monitors, get_monitors_and_dependencies
  from security_monkey.reporter import Reporter
  from security_monkey.task_scheduler.util import CELERY, setup
  from sqlalchemy.exc import OperationalError, InvalidRequestError, StatementError
  
  
  @CELERY.task(bind=True, max_retries=3)
  def task_account_tech(self, account_name, technology_name):
      setup()
      app.logger.info("[ ] Executing Celery task for account: {}, technology: {}".format(account_name, technology_name))
      time1 = time.time()
  
      try:
          reporter_logic(account_name, technology_name)
  
          time2 = time.time()
          app.logger.info('[@] Run Account for Technology (%s/%s) took %0.1f s' % (account_name,
                                                                                   technology_name, (time2 - time1)))
          app.logger.info(
              "[+] Completed Celery task for account: {}, technology: {}".format(account_name, technology_name))
      except Exception as e:
          if sentry:
              sentry.captureException()
          app.logger.error("[X] Task Account Scheduler Exception ({}/{}): {}".format(account_name, technology_name, e))
          app.logger.error(traceback.format_exc())
          store_exception("scheduler-exception-on-watch", None, e)
          raise self.retry(exc=e)
  
  
  @CELERY.task(bind=True, max_retries=3)
  def task_audit(self, account_name, technology_name):
      setup()
  
      app.logger.info("[ ] Executing Celery task to audit changes for Account: {} Technology: {}".format(account_name,
                                                                                                         technology_name))
      try:
          audit_changes([account_name], [technology_name], True)
  
          app.logger.info("[+] Completed Celery task for account: {}, technology: {}".format(account_name,
                                                                                             technology_name))
  
      except Exception as e:
          if sentry:
              sentry.captureException()
          app.logger.error("[X] Task Audit Scheduler Exception ({}/{}): {}".format(account_name, technology_name, e))
          app.logger.error(traceback.format_exc())
          store_exception("scheduler-exception-on-audit", None, e)
          self.retry(exc=e)
  
  
  @CELERY.task()
  def clear_expired_exceptions():
      app.logger.info("[ ] Clearing out exceptions that have an expired TTL...")
      clear_old_exceptions()
      app.logger.info("[-] Completed clearing out exceptions that have an expired TTL.")
  
  
  def reporter_logic(account_name, technology_name):
      """Logic for the run change reporter"""
      try:
          # Watch and Audit:
          monitors = find_changes(account_name, technology_name)
  
          # Alert:
          app.logger.info("[ ] Sending alerts (if applicable) for account: {}, technology: {}".format(account_name,
                                                                                                      technology_name))
          Alerter(monitors, account=account_name).report()
      except (OperationalError, InvalidRequestError, StatementError) as e:
          app.logger.exception("[X] Database error processing account %s - technology %s cleaning up session.",
  
  
  [... the rest is omitted ...]
files: security_monkey/task_scheduler/tasks.py
files: #     Copyright 2018 Netflix, Inc.
  #
  #     Licensed under the Apache License, Version 2.0 (the "License");
  #     you may not use this file except in compliance with the License.
  #     You may obtain a copy of the License at
  #
  #         http://www.apache.org/licenses/LICENSE-2.0
  #
  #     Unless required by applicable law or agreed to in writing, software
  #     distributed under the License is distributed on an "AS IS" BASIS,
  #     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #     See the License for the specific language governing permissions and
  #     limitations under the License.
  """
  .. module: security_monkey.tests.scheduling.test_celery_scheduler
      :platform: Unix
  .. version:: $$VERSION$$
  .. moduleauthor::  Mike Grima <mgrima@netflix.com>
  """
  import json
  
  import boto3
  import mock
  from mock import patch
  from moto import mock_iam, mock_sts
  from pytest import raises
  
  from security_monkey import db, ARN_PREFIX
  from security_monkey.datastore import Account, AccountType, Technology, Item, ItemAudit, ItemRevision
  from security_monkey.monitors import Monitor
  from security_monkey.tests import SecurityMonkeyTestCase
  from security_monkey.watcher import Watcher
  
  OPEN_POLICY = {
      "Statement": [
          {
              "Effect": "Allow",
              "Action": "*",
              "Resource": "*"
          }
      ]
  }
  
  ROLE_CONF = {
      "account_number": "012345678910",
      "technology": "iamrole",
      "region": "universal",
      "name": "roleNumber",
      "InlinePolicies": {"ThePolicy": OPEN_POLICY},
      "Arn": ARN_PREFIX + ":iam::012345678910:role/roleNumber"
  }
  
  
  class CelerySchedulerTestCase(SecurityMonkeyTestCase):
      test_account1 = None
      test_account2 = None
      test_account3 = None
      test_account4 = None
  
      def pre_test_setup(self):
          account_type_result = AccountType(name='AWS')
          db.session.add(account_type_result)
          db.session.commit()
  
          account = Account(identifier="012345678910", name="TEST_ACCOUNT1",
                            account_type_id=account_type_result.id, notes="TEST_ACCOUNT1",
                            third_party=False, active=True)
          db.session.add(account)
  
          account = Account(identifier="123123123123", name="TEST_ACCOUNT2",
                            account_type_id=account_type_result.id, notes="TEST_ACCOUNT2",
                            third_party=False, active=True)
          db.session.add(account)
  
          account = Account(identifier="109876543210", name="TEST_ACCOUNT3",
                            account_type_id=account_type_result.id, notes="TEST_ACCOUNT3",
                            third_party=False, active=False)
          db.session.add(account)
  
          account = Account(identifier="456456456456", name="TEST_ACCOUNT4",
                            account_type_id=account_type_result.id, notes="TEST_ACCOUNT4",
                            third_party=False, active=False)
          db.session.add(account)
  
          db.session.commit()
  
      @patch("security_monkey.task_scheduler.tasks.fix_orphaned_deletions")
      def test_find_batch_changes(self, mock_fix_orphaned):
          """
          Runs through a full find job via the IAMRole watcher, as that supports batching.
  
          However, this is mostly testing the logic through each function call -- this is
          not going to do any boto work and that will instead be mocked out.
          :return:
         
  
  [... the rest is omitted ...]
files: security_monkey/tests/scheduling/test_celery_scheduler.py
files: #     Copyright 2018 Netflix, Inc.
  #
  #     Licensed under the Apache License, Version 2.0 (the "License");
  #     you may not use this file except in compliance with the License.
  #     You may obtain a copy of the License at
  #
  #         http://www.apache.org/licenses/LICENSE-2.0
  #
  #     Unless required by applicable law or agreed to in writing, software
  #     distributed under the License is distributed on an "AS IS" BASIS,
  #     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #     See the License for the specific language governing permissions and
  #     limitations under the License.
  """
  .. module: security_monkey.tests.scheduling.test_celery_scheduler
      :platform: Unix
  .. version:: $$VERSION$$
  .. moduleauthor::  Mike Grima <mgrima@netflix.com>
  """
  import json
  
  import boto3
  import mock
  from mock import patch
  from moto import mock_iam, mock_sts
  from pytest import raises
  
  from security_monkey import db, ARN_PREFIX
  from security_monkey.datastore import Account, AccountType, Technology, Item, ItemAudit, ItemRevision
  from security_monkey.monitors import Monitor
  from security_monkey.tests import SecurityMonkeyTestCase
  from security_monkey.watcher import Watcher
  
  OPEN_POLICY = {
      "Statement": [
          {
              "Effect": "Allow",
              "Action": "*",
              "Resource": "*"
          }
      ]
  }
  
  ROLE_CONF = {
      "account_number": "012345678910",
      "technology": "iamrole",
      "region": "universal",
      "name": "roleNumber",
      "InlinePolicies": {"ThePolicy": OPEN_POLICY},
      "Arn": ARN_PREFIX + ":iam::012345678910:role/roleNumber"
  }
  
  
  class CelerySchedulerTestCase(SecurityMonkeyTestCase):
      test_account1 = None
      test_account2 = None
      test_account3 = None
      test_account4 = None
  
      def pre_test_setup(self):
          account_type_result = AccountType(name='AWS')
          db.session.add(account_type_result)
          db.session.commit()
  
          account = Account(identifier="012345678910", name="TEST_ACCOUNT1",
                            account_type_id=account_type_result.id, notes="TEST_ACCOUNT1",
                            third_party=False, active=True)
          db.session.add(account)
  
          account = Account(identifier="123123123123", name="TEST_ACCOUNT2",
                            account_type_id=account_type_result.id, notes="TEST_ACCOUNT2",
                            third_party=False, active=True)
          db.session.add(account)
  
          account = Account(identifier="109876543210", name="TEST_ACCOUNT3",
                            account_type_id=account_type_result.id, notes="TEST_ACCOUNT3",
                            third_party=False, active=False)
          db.session.add(account)
  
          account = Account(identifier="456456456456", name="TEST_ACCOUNT4",
                            account_type_id=account_type_result.id, notes="TEST_ACCOUNT4",
                            third_party=False, active=False)
          db.session.add(account)
  
          db.session.commit()
  
      def test_find_batch_changes(self):
          """
          Runs through a full find job via the IAMRole watcher, as that supports batching.
  
          However, this is mostly testing the logic through each function call -- this is
          not going to do any boto work and that will instead be mocked out.
          :return:
          """
          from security_monkey.task_scheduler.tasks import manual_run_change_finder, setup
          from security_monkey.monitors
  
  [... the rest is omitted ...]
files: security_monkey/tests/scheduling/test_celery_scheduler.py
files: """
  .. module: security_monkey.watcher
      :platform: Unix
      :synopsis: Slurps the current config from AWS and compares it to what has previously
      been recorded in the database to find any changes.
  
  .. version:: $$VERSION$$
  .. moduleauthor:: Patrick Kelley <pkelley@netflix.com> @monkeysecurity
  
  """
  from botocore.exceptions import ClientError
  
  from common.PolicyDiff import PolicyDiff
  from common.utils import sub_dict
  from security_monkey import app
  from security_monkey.datastore import Account, IgnoreListEntry, db
  from security_monkey.datastore import Technology, WatcherConfig, store_exception
  from security_monkey.common.jinja import get_jinja_env
  from security_monkey.alerters.custom_alerter import report_watcher_changes
  
  from boto.exception import BotoServerError
  import time
  
  import datastore
  from copy import deepcopy
  import dpath.util
  from dpath.exceptions import PathNotFound
  
  import logging
  
  watcher_registry = {}
  abstract_classes = set(['Watcher', 'CloudAuxWatcher', 'CloudAuxBatchedWatcher'])
  
  
  if not app.config.get("DONT_IGNORE_BOTO_VERBOSE_LOGGERS"):
      logging.getLogger('botocore.vendored.requests.packages.urllib3').setLevel(logging.WARNING)
      logging.getLogger('botocore.credentials').setLevel(logging.WARNING)
  
  
  class WatcherType(type):
      def __init__(cls, name, bases, attrs):
          super(WatcherType, cls).__init__(name, bases, attrs)
          if cls.__name__ not in abstract_classes and cls.index:
              app.logger.debug("Registering watcher {} {}.{}".format(cls.index, cls.__module__, cls.__name__))
              watcher_registry[cls.index] = cls
  
  
  class Watcher(object):
      """Slurps the current config from AWS and compares it to what has previously
        been recorded in the database to find any changes."""
      index = 'abstract'
      i_am_singular = 'Abstract'
      i_am_plural = 'Abstracts'
      rate_limit_delay = 0
      ignore_list = []
      interval = 60    #in minutes
      active = True
      account_type = 'AWS'
      __metaclass__ = WatcherType
  
      def __init__(self, accounts=None, debug=False):
          """Initializes the Watcher"""
          self.datastore = datastore.Datastore()
          if not accounts:
              accounts = Account.query.filter(Account.third_party==False).filter(Account.active==True).all()
          else:
              accounts = Account.query.filter(Account.third_party==False).filter(Account.active==True).filter(Account.name.in_(accounts)).all()
          if not accounts:
              raise ValueError('Watcher needs a valid account')
          self.accounts = [account.name for account in accounts]
          self.account_identifiers = [account.identifier for account in accounts]
          self.debug = debug
          self.created_items = []
          self.deleted_items = []
          self.changed_items = []
          self.ephemeral_items = []
          # TODO: grab these from DB, keyed on account
          self.rate_limit_delay = 0
          self.honor_ephemerals = False
          self.ephemeral_paths = []
  
          # Batching attributes:
          self.batched_size = 0   # Don't batch anything by default
          self.done_slurping = True   # Don't batch anything by default
          self.total_list = []    # This will hold the full list
  
  [... the rest is omitted ...]
files: security_monkey/watcher.py
files: """
  .. module: security_monkey.watcher
      :platform: Unix
      :synopsis: Slurps the current config from AWS and compares it to what has previously
      been recorded in the database to find any changes.
  
  .. version:: $$VERSION$$
  .. moduleauthor:: Patrick Kelley <pkelley@netflix.com> @monkeysecurity
  
  """
  from botocore.exceptions import ClientError
  
  from common.PolicyDiff import PolicyDiff
  from common.utils import sub_dict
  from security_monkey import app
  from security_monkey.datastore import Account, IgnoreListEntry, db
  from security_monkey.datastore import Technology, WatcherConfig, store_exception
  from security_monkey.common.jinja import get_jinja_env
  from security_monkey.alerters.custom_alerter import report_watcher_changes
  
  from boto.exception import BotoServerError
  import time
  
  import datastore
  from copy import deepcopy
  import dpath.util
  from dpath.exceptions import PathNotFound
  
  watcher_registry = {}
  abstract_classes = set(['Watcher', 'CloudAuxWatcher', 'CloudAuxBatchedWatcher'])
  
  
  class WatcherType(type):
      def __init__(cls, name, bases, attrs):
          super(WatcherType, cls).__init__(name, bases, attrs)
          if cls.__name__ not in abstract_classes and cls.index:
              app.logger.debug("Registering watcher {} {}.{}".format(cls.index, cls.__module__, cls.__name__))
              watcher_registry[cls.index] = cls
  
  
  class Watcher(object):
      """Slurps the current config from AWS and compares it to what has previously
        been recorded in the database to find any changes."""
      index = 'abstract'
      i_am_singular = 'Abstract'
      i_am_plural = 'Abstracts'
      rate_limit_delay = 0
      ignore_list = []
      interval = 60    #in minutes
      active = True
      account_type = 'AWS'
      __metaclass__ = WatcherType
  
      def __init__(self, accounts=None, debug=False):
          """Initializes the Watcher"""
          self.datastore = datastore.Datastore()
          if not accounts:
              accounts = Account.query.filter(Account.third_party==False).filter(Account.active==True).all()
          else:
              accounts = Account.query.filter(Account.third_party==False).filter(Account.active==True).filter(Account.name.in_(accounts)).all()
          if not accounts:
              raise ValueError('Watcher needs a valid account')
          self.accounts = [account.name for account in accounts]
          self.account_identifiers = [account.identifier for account in accounts]
          self.debug = debug
          self.created_items = []
          self.deleted_items = []
          self.changed_items = []
          self.ephemeral_items = []
          # TODO: grab these from DB, keyed on account
          self.rate_limit_delay = 0
          self.honor_ephemerals = False
          self.ephemeral_paths = []
  
          # Batching attributes:
          self.batched_size = 0   # Don't batch anything by default
          self.done_slurping = True   # Don't batch anything by default
          self.total_list = []    # This will hold the full list of items to batch over
          self.batch_counter = 0  # Keeps track of the batch we are on -- can be used for retry logic
          self.current_account = None  # Tuple that holds the current account and account index we are on.
          self.technology = None
          # Region is probably not
  
  [... the rest is omitted ...]
files: security_monkey/watcher.py
files: #     Copyright 2018 Netflix, Inc.
  #
  #     Licensed under the Apache License, Version 2.0 (the "License");
  #     you may not use this file except in compliance with the License.
  #     You may obtain a copy of the License at
  #
  #         http://www.apache.org/licenses/LICENSE-2.0
  #
  #     Unless required by applicable law or agreed to in writing, software
  #     distributed under the License is distributed on an "AS IS" BASIS,
  #     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #     See the License for the specific language governing permissions and
  #     limitations under the License.
  """
  .. module: security_monkey.watchers.sqs
      :platform: Unix
  
  .. version:: $$VERSION$$
  .. moduleauthor:: Mike Grima <mgrima@netflix.com>
  """
  from botocore.exceptions import ClientError
  from cloudaux.aws.sqs import list_queues
  from cloudaux.orchestration.aws.sqs import get_queue
  
  from security_monkey import app
  from security_monkey.cloudaux_batched_watcher import CloudAuxBatchedWatcher
  
  
  class SQS(CloudAuxBatchedWatcher):
      index = 'sqs'
      i_am_singular = 'SQS Policy'
      i_am_plural = 'SQS Policies'
  
      def __init__(self, **kwargs):
          super(SQS, self).__init__(**kwargs)
          self.service_name = "sqs"
          self.honor_ephemerals = True
          self.ephemeral_paths = [
              '_version',
              'Attributes$LastModifiedTimestamp',
              'Attributes$ApproximateNumberOfMessagesNotVisible',
              'Attributes$ApproximateNumberOfMessages',
              'Attributes$ApproximateNumberOfMessagesDelayed'
          ]
          self.batched_size = 200
  
          # SQS returns a list of URLs. The DB wants the total list of items to be a dict that contains
          # an "Arn" field. Since that is not present, this is used to help update the list with the ARNs
          # when they are discovered.
          self.corresponding_items = {}
  
      def get_name_from_list_output(self, item):
          # SQS returns URLs. Need to deconstruct the URL to pull out the name :/
          name = item["Url"].split("{}/".format(self.account_identifiers[0]))[1]
  
          return name
  
      def list_method(self, **kwargs):
          """
          Get the list of SQS queues. Also, create the corresponding lookup table with URL -> position
          in the total list.
          :param kwargs:
          :return:
          """
          items = []
          queues = list_queues(**kwargs)
  
          # Offset by the existing items in the list (from other regions)
          offset = len(self.corresponding_items)
          queue_count = -1
  
          for item_count in range(0, len(queues)):
              if self.corresponding_items.get(queues[item_count]):
                  app.logger.error("[?] Received a duplicate item in the SQS list: {}. Skipping it.".format(queues[item_count]))
                  continue
              queue_count += 1
              items.append({"Url": queues[item_count], "Region": kwargs["region"]})
              self.corresponding_items[queues[item_count]] = queue_count + offset
  
          return items
  
      def get_method(self, item, **kwargs):
          try:
              queue = get_queue(item["Url"], **kwargs)
  
              # Update the current position in the total list by replacing the SQS queue URL with
              # the ARN of the queue
              self.total_list[self.corresponding_items[item["Url"]]]
  
  [... the rest is omitted ...]
files: security_monkey/watchers/sqs.py
files: #     Copyright 2018 Netflix, Inc.
  #
  #     Licensed under the Apache License, Version 2.0 (the "License");
  #     you may not use this file except in compliance with the License.
  #     You may obtain a copy of the License at
  #
  #         http://www.apache.org/licenses/LICENSE-2.0
  #
  #     Unless required by applicable law or agreed to in writing, software
  #     distributed under the License is distributed on an "AS IS" BASIS,
  #     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #     See the License for the specific language governing permissions and
  #     limitations under the License.
  """
  .. module: security_monkey.watchers.sqs
      :platform: Unix
  
  .. version:: $$VERSION$$
  .. moduleauthor:: Mike Grima <mgrima@netflix.com>
  """
  from botocore.exceptions import ClientError
  from cloudaux.aws.sqs import list_queues
  from cloudaux.orchestration.aws.sqs import get_queue
  
  from security_monkey import app
  from security_monkey.cloudaux_batched_watcher import CloudAuxBatchedWatcher
  
  
  class SQS(CloudAuxBatchedWatcher):
      index = 'sqs'
      i_am_singular = 'SQS Policy'
      i_am_plural = 'SQS Policies'
  
      def __init__(self, **kwargs):
          super(SQS, self).__init__(**kwargs)
          self.service_name = "sqs"
          self.honor_ephemerals = True
          self.ephemeral_paths = [
              '_version',
              'Attributes$LastModifiedTimestamp',
              'Attributes$ApproximateNumberOfMessagesNotVisible',
              'Attributes$ApproximateNumberOfMessages',
              'Attributes$ApproximateNumberOfMessagesDelayed'
          ]
          self.batched_size = 200
  
          # SQS returns a list of URLs. The DB wants the total list of items to be a dict that contains
          # an "Arn" field. Since that is not present, this is used to help update the list with the ARNs
          # when they are discovered.
          self.corresponding_items = {}
  
      def get_name_from_list_output(self, item):
          # SQS returns URLs. Need to deconstruct the URL to pull out the name :/
          name = item["Url"].split("{}/".format(self.account_identifiers[0]))[1]
  
          return name
  
      def list_method(self, **kwargs):
          """
          Get the list of SQS queues. Also, create the corresponding lookup table with URL -> position
          in the total list.
          :param kwargs:
          :return:
          """
          items = []
          queues = list_queues(**kwargs)
  
          # Offset by the existing items in the list (from other regions)
          offset = len(self.corresponding_items)
  
          for i in range(0, len(queues)):
              items.append({"Url": queues[i], "Region": kwargs["region"]})
              self.corresponding_items[queues[i]] = i + offset
  
          return items
  
      def get_method(self, item, **kwargs):
          try:
              queue = get_queue(item["Url"], **kwargs)
  
              # Update the current position in the total list by replacing the SQS queue URL with
              # the ARN of the queue
              self.total_list[self.corresponding_items[item["Url"]]] = {"Arn": queue["Arn"]}
          except ClientError as ce:
              # In case the queue was created and then deleted really quickly:
              if "NonExistentQueue" in ce.response["Error"]["Code"]:
                  app.logger.debug("Queue with URL: {} - was listed, but is no longer present".format(item
  
  [... the rest is omitted ...]
files: security_monkey/watchers/sqs.py