files.new_contents: import hashlib
  import json
  
  import datetime
  import dpath.util
  from dpath.exceptions import PathNotFound
  from copy import deepcopy
  
  from security_monkey import datastore, app
  from cloudaux.orchestration.aws.arn import ARN
  from security_monkey.datastore import Item, ItemRevision, ItemAudit
  
  prims = [int, str, unicode, bool, float, type(None)]
  
  
  def persist_item(item, db_item, technology, account, complete_hash, durable_hash, durable):
      if not db_item:
          if account.account_type.name != "AWS":
              db_item = create_item(item, technology, account)
          else:
              db_item = create_item_aws(item, technology, account)
  
      if db_item.latest_revision_complete_hash == complete_hash:
          app.logger.debug("Change persister doesn't see any change. Ignoring...")
          return
  
      # Create the new revision
      if durable:
          revision = create_revision(item.config, db_item)
          db_item.revisions.append(revision)
  
      # Ephemeral -- update the existing revision:
      else:
          revision = db_item.revisions.first()
          revision.date_last_ephemeral_change = datetime.datetime.utcnow()
          revision.config = item.config
          app.logger.debug("Persisting EPHEMERAL change to item: {technology}/{account}/{item}".format(
              technology=technology.name, account=account.name, item=db_item.name
          ))
  
      db_item.latest_revision_complete_hash = complete_hash
      db_item.latest_revision_durable_hash = durable_hash
  
      datastore.db.session.add(db_item)
      datastore.db.session.add(revision)
      datastore.db.session.commit()
  
      if durable:
          app.logger.debug("Persisting DURABLE change to item: {technology}/{account
  
  [... the rest is omitted ...]
files.new_path: security_monkey/datastore_utils.py
files.old_contents: import hashlib
  import json
  
  import datetime
  import dpath.util
  from dpath.exceptions import PathNotFound
  from copy import deepcopy
  
  from security_monkey import datastore, app
  from cloudaux.orchestration.aws.arn import ARN
  from security_monkey.datastore import Item, ItemRevision, ItemAudit
  
  prims = [int, str, unicode, bool, float, type(None)]
  
  
  def persist_item(item, db_item, technology, account, complete_hash, durable_hash, durable):
      if not db_item:
          if account.account_type.name != "AWS":
              db_item = create_item(item, technology, account)
          else:
              db_item = create_item_aws(item, technology, account)
  
      if db_item.latest_revision_complete_hash == complete_hash:
          app.logger.debug("Change persister doesn't see any change. Ignoring...")
          return
  
      # Create the new revision
      if durable:
          revision = create_revision(item.config, db_item)
          db_item.revisions.append(revision)
  
      # Ephemeral -- update the existing revision:
      else:
          revision = db_item.revisions.first()
          revision.date_last_ephemeral_change = datetime.datetime.utcnow()
          revision.config = item.config
          app.logger.debug("Persisting EPHEMERAL change to item: {technology}/{account}/{item}".format(
              technology=technology.name, account=account.name, item=db_item.name
          ))
  
      db_item.latest_revision_complete_hash = complete_hash
      db_item.latest_revision_durable_hash = durable_hash
  
      datastore.db.session.add(db_item)
      datastore.db.session.add(revision)
      datastore.db.session.commit()
  
      if durable:
          app.logger.debug("Persisting DURABLE change to item: {technology}/{account
  
  [... the rest is omitted ...]
files.old_path: security_monkey/datastore_utils.py
files.new_contents: """
  .. module: security_monkey.task_scheduler.tasks
      :platform: Unix
      :synopsis: Sets up the Celery task scheduling for watching, auditing, and reporting changes in the environment.
  
  .. version:: $$VERSION$$
  .. moduleauthor:: Mike Grima <mgrima@netflix.com>
  
  """
  import time
  import traceback
  
  from security_monkey import app, db, jirasync, sentry
  from security_monkey.alerter import Alerter
  from security_monkey.datastore import store_exception, clear_old_exceptions, Technology, Account, Item, ItemRevision
  from security_monkey.monitors import get_monitors, get_monitors_and_dependencies
  from security_monkey.reporter import Reporter
  from security_monkey.task_scheduler.util import CELERY, setup
  from sqlalchemy.exc import OperationalError, InvalidRequestError, StatementError
  
  
  @CELERY.task(bind=True, max_retries=3)
  def task_account_tech(self, account_name, technology_name):
      setup()
      app.logger.info("[ ] Executing Celery task for account: {}, technology: {}".format(account_name, technology_name))
      time1 = time.time()
  
      try:
          reporter_logic(account_name, technology_name)
  
          time2 = time.time()
          app.logger.info('[@] Run Account for Technology (%s/%s) took %0.1f s' % (account_name,
                                                                                   technology_name, (time2 - time1)))
          app.logger.info(
              "[+] Completed Celery task for account: {}, technology: {}".format(account_name, technology_name))
      except Exception as e:
          if sentry:
              sentry.captureException()
          app.logger.error("[X] Task Account Scheduler Exception ({}/{}): {}".format(account_name, technology_name, e))
          app.logger.error(traceback.format_ex
  
  [... the rest is omitted ...]
files.new_path: security_monkey/task_scheduler/tasks.py
files.old_contents: """
  .. module: security_monkey.task_scheduler.tasks
      :platform: Unix
      :synopsis: Sets up the Celery task scheduling for watching, auditing, and reporting changes in the environment.
  
  .. version:: $$VERSION$$
  .. moduleauthor:: Mike Grima <mgrima@netflix.com>
  
  """
  import time
  import traceback
  
  from security_monkey import app, db, jirasync, sentry
  from security_monkey.alerter import Alerter
  from security_monkey.datastore import store_exception, clear_old_exceptions
  from security_monkey.monitors import get_monitors, get_monitors_and_dependencies
  from security_monkey.reporter import Reporter
  from security_monkey.task_scheduler.util import CELERY, setup
  from sqlalchemy.exc import OperationalError, InvalidRequestError, StatementError
  
  
  @CELERY.task(bind=True, max_retries=3)
  def task_account_tech(self, account_name, technology_name):
      setup()
      app.logger.info("[ ] Executing Celery task for account: {}, technology: {}".format(account_name, technology_name))
      time1 = time.time()
  
      try:
          reporter_logic(account_name, technology_name)
  
          time2 = time.time()
          app.logger.info('[@] Run Account for Technology (%s/%s) took %0.1f s' % (account_name,
                                                                                   technology_name, (time2 - time1)))
          app.logger.info(
              "[+] Completed Celery task for account: {}, technology: {}".format(account_name, technology_name))
      except Exception as e:
          if sentry:
              sentry.captureException()
          app.logger.error("[X] Task Account Scheduler Exception ({}/{}): {}".format(account_name, technology_name, e))
          app.logger.error(traceback.format_exc())
          store_exception("scheduler
  
  [... the rest is omitted ...]
files.old_path: security_monkey/task_scheduler/tasks.py
files.new_contents: #     Copyright 2018 Netflix, Inc.
  #
  #     Licensed under the Apache License, Version 2.0 (the "License");
  #     you may not use this file except in compliance with the License.
  #     You may obtain a copy of the License at
  #
  #         http://www.apache.org/licenses/LICENSE-2.0
  #
  #     Unless required by applicable law or agreed to in writing, software
  #     distributed under the License is distributed on an "AS IS" BASIS,
  #     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #     See the License for the specific language governing permissions and
  #     limitations under the License.
  """
  .. module: security_monkey.tests.scheduling.test_celery_scheduler
      :platform: Unix
  .. version:: $$VERSION$$
  .. moduleauthor::  Mike Grima <mgrima@netflix.com>
  """
  import json
  
  import boto3
  import mock
  from mock import patch
  from moto import mock_iam, mock_sts
  from pytest import raises
  
  from security_monkey import db, ARN_PREFIX
  from security_monkey.datastore import Account, AccountType, Technology, Item, ItemAudit, ItemRevision
  from security_monkey.monitors import Monitor
  from security_monkey.tests import SecurityMonkeyTestCase
  from security_monkey.watcher import Watcher
  
  OPEN_POLICY = {
      "Statement": [
          {
              "Effect": "Allow",
              "Action": "*",
              "Resource": "*"
          }
      ]
  }
  
  ROLE_CONF = {
      "account_number": "012345678910",
      "technology": "iamrole",
      "region": "universal",
      "name": "roleNumber",
      "InlinePolicies": {"ThePolicy": OPEN_POLICY},
      "Arn": ARN_PREFIX + ":iam::012345678910:role
  
  [... the rest is omitted ...]
files.new_path: security_monkey/tests/scheduling/test_celery_scheduler.py
files.old_contents: #     Copyright 2018 Netflix, Inc.
  #
  #     Licensed under the Apache License, Version 2.0 (the "License");
  #     you may not use this file except in compliance with the License.
  #     You may obtain a copy of the License at
  #
  #         http://www.apache.org/licenses/LICENSE-2.0
  #
  #     Unless required by applicable law or agreed to in writing, software
  #     distributed under the License is distributed on an "AS IS" BASIS,
  #     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #     See the License for the specific language governing permissions and
  #     limitations under the License.
  """
  .. module: security_monkey.tests.scheduling.test_celery_scheduler
      :platform: Unix
  .. version:: $$VERSION$$
  .. moduleauthor::  Mike Grima <mgrima@netflix.com>
  """
  import json
  
  import boto3
  import mock
  from mock import patch
  from moto import mock_iam, mock_sts
  from pytest import raises
  
  from security_monkey import db, ARN_PREFIX
  from security_monkey.datastore import Account, AccountType, Technology, Item, ItemAudit, ItemRevision
  from security_monkey.monitors import Monitor
  from security_monkey.tests import SecurityMonkeyTestCase
  from security_monkey.watcher import Watcher
  
  OPEN_POLICY = {
      "Statement": [
          {
              "Effect": "Allow",
              "Action": "*",
              "Resource": "*"
          }
      ]
  }
  
  ROLE_CONF = {
      "account_number": "012345678910",
      "technology": "iamrole",
      "region": "universal",
      "name": "roleNumber",
      "InlinePolicies": {"ThePolicy": OPEN_POLICY},
      "Arn": ARN_PREFIX + ":iam::012345678910:role
  
  [... the rest is omitted ...]
files.old_path: security_monkey/tests/scheduling/test_celery_scheduler.py
files.new_contents: """
  .. module: security_monkey.watcher
      :platform: Unix
      :synopsis: Slurps the current config from AWS and compares it to what has previously
      been recorded in the database to find any changes.
  
  .. version:: $$VERSION$$
  .. moduleauthor:: Patrick Kelley <pkelley@netflix.com> @monkeysecurity
  
  """
  from botocore.exceptions import ClientError
  
  from common.PolicyDiff import PolicyDiff
  from common.utils import sub_dict
  from security_monkey import app
  from security_monkey.datastore import Account, IgnoreListEntry, db
  from security_monkey.datastore import Technology, WatcherConfig, store_exception
  from security_monkey.common.jinja import get_jinja_env
  from security_monkey.alerters.custom_alerter import report_watcher_changes
  
  from boto.exception import BotoServerError
  import time
  
  import datastore
  from copy import deepcopy
  import dpath.util
  from dpath.exceptions import PathNotFound
  
  import logging
  
  watcher_registry = {}
  abstract_classes = set(['Watcher', 'CloudAuxWatcher', 'CloudAuxBatchedWatcher'])
  
  
  if not app.config.get("DONT_IGNORE_BOTO_VERBOSE_LOGGERS"):
      logging.getLogger('botocore.vendored.requests.packages.urllib3').setLevel(logging.WARNING)
      logging.getLogger('botocore.credentials').setLevel(logging.WARNING)
  
  
  class WatcherType(type):
      def __init__(cls, name, bases, attrs):
          super(WatcherType, cls).__init__(name, bases, attrs)
          if cls.__name__ not in abstract_classes and cls.index:
              app.logger.debug("Registering watcher {} {}.{}".format(cls.index, cls.__module__, cls.__name__))
              wat
  
  [... the rest is omitted ...]
files.new_path: security_monkey/watcher.py
files.old_contents: """
  .. module: security_monkey.watcher
      :platform: Unix
      :synopsis: Slurps the current config from AWS and compares it to what has previously
      been recorded in the database to find any changes.
  
  .. version:: $$VERSION$$
  .. moduleauthor:: Patrick Kelley <pkelley@netflix.com> @monkeysecurity
  
  """
  from botocore.exceptions import ClientError
  
  from common.PolicyDiff import PolicyDiff
  from common.utils import sub_dict
  from security_monkey import app
  from security_monkey.datastore import Account, IgnoreListEntry, db
  from security_monkey.datastore import Technology, WatcherConfig, store_exception
  from security_monkey.common.jinja import get_jinja_env
  from security_monkey.alerters.custom_alerter import report_watcher_changes
  
  from boto.exception import BotoServerError
  import time
  
  import datastore
  from copy import deepcopy
  import dpath.util
  from dpath.exceptions import PathNotFound
  
  watcher_registry = {}
  abstract_classes = set(['Watcher', 'CloudAuxWatcher', 'CloudAuxBatchedWatcher'])
  
  
  class WatcherType(type):
      def __init__(cls, name, bases, attrs):
          super(WatcherType, cls).__init__(name, bases, attrs)
          if cls.__name__ not in abstract_classes and cls.index:
              app.logger.debug("Registering watcher {} {}.{}".format(cls.index, cls.__module__, cls.__name__))
              watcher_registry[cls.index] = cls
  
  
  class Watcher(object):
      """Slurps the current config from AWS and compares it to what has previously
        been recorded in the database to find any changes."""
      index = 'abstract'
      i_am_singular = 'Abstract'
      i_am_plural =
  
  [... the rest is omitted ...]
files.old_path: security_monkey/watcher.py
files.new_contents: #     Copyright 2018 Netflix, Inc.
  #
  #     Licensed under the Apache License, Version 2.0 (the "License");
  #     you may not use this file except in compliance with the License.
  #     You may obtain a copy of the License at
  #
  #         http://www.apache.org/licenses/LICENSE-2.0
  #
  #     Unless required by applicable law or agreed to in writing, software
  #     distributed under the License is distributed on an "AS IS" BASIS,
  #     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #     See the License for the specific language governing permissions and
  #     limitations under the License.
  """
  .. module: security_monkey.watchers.sqs
      :platform: Unix
  
  .. version:: $$VERSION$$
  .. moduleauthor:: Mike Grima <mgrima@netflix.com>
  """
  from botocore.exceptions import ClientError
  from cloudaux.aws.sqs import list_queues
  from cloudaux.orchestration.aws.sqs import get_queue
  
  from security_monkey import app
  from security_monkey.cloudaux_batched_watcher import CloudAuxBatchedWatcher
  
  
  class SQS(CloudAuxBatchedWatcher):
      index = 'sqs'
      i_am_singular = 'SQS Policy'
      i_am_plural = 'SQS Policies'
  
      def __init__(self, **kwargs):
          super(SQS, self).__init__(**kwargs)
          self.service_name = "sqs"
          self.honor_ephemerals = True
          self.ephemeral_paths = [
              '_version',
              'Attributes$LastModifiedTimestamp',
              'Attributes$ApproximateNumberOfMessagesNotVisible',
              'Attributes$ApproximateNumberOfMessages',
              'Attributes$ApproximateNumberOfMessagesDelayed'
          ]
          self.batched_size = 200
  
          # S
  
  [... the rest is omitted ...]
files.new_path: security_monkey/watchers/sqs.py
files.old_contents: #     Copyright 2018 Netflix, Inc.
  #
  #     Licensed under the Apache License, Version 2.0 (the "License");
  #     you may not use this file except in compliance with the License.
  #     You may obtain a copy of the License at
  #
  #         http://www.apache.org/licenses/LICENSE-2.0
  #
  #     Unless required by applicable law or agreed to in writing, software
  #     distributed under the License is distributed on an "AS IS" BASIS,
  #     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  #     See the License for the specific language governing permissions and
  #     limitations under the License.
  """
  .. module: security_monkey.watchers.sqs
      :platform: Unix
  
  .. version:: $$VERSION$$
  .. moduleauthor:: Mike Grima <mgrima@netflix.com>
  """
  from botocore.exceptions import ClientError
  from cloudaux.aws.sqs import list_queues
  from cloudaux.orchestration.aws.sqs import get_queue
  
  from security_monkey import app
  from security_monkey.cloudaux_batched_watcher import CloudAuxBatchedWatcher
  
  
  class SQS(CloudAuxBatchedWatcher):
      index = 'sqs'
      i_am_singular = 'SQS Policy'
      i_am_plural = 'SQS Policies'
  
      def __init__(self, **kwargs):
          super(SQS, self).__init__(**kwargs)
          self.service_name = "sqs"
          self.honor_ephemerals = True
          self.ephemeral_paths = [
              '_version',
              'Attributes$LastModifiedTimestamp',
              'Attributes$ApproximateNumberOfMessagesNotVisible',
              'Attributes$ApproximateNumberOfMessages',
              'Attributes$ApproximateNumberOfMessagesDelayed'
          ]
          self.batched_size = 200
  
          # S
  
  [... the rest is omitted ...]
files.old_path: security_monkey/watchers/sqs.py