<NME> redshift.py
<BEF> #     Copyright 2014 Yelp, Inc.
#
#     Licensed under the Apache License, Version 2.0 (the "License");
#     you may not use this file except in compliance with the License.
#     You may obtain a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#     Unless required by applicable law or agreed to in writing, software
#     distributed under the License is distributed on an "AS IS" BASIS,
#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#     See the License for the specific language governing permissions and
#     limitations under the License.
"""
.. module: security_monkey.watchers.redshift
    :platform: Unix

.. version:: $$VERSION$$
.. moduleauthor:: Ivan Leichtling <ivanlei@yelp.com> @c0wl

"""

from security_monkey.watcher import Watcher
from security_monkey.watcher import ChangeItem
from security_monkey.constants import TROUBLE_REGIONS
from security_monkey.exceptions import BotoConnectionIssue
from security_monkey.datastore import Account
from security_monkey import app, ARN_PREFIX

from boto.redshift import regions


class Redshift(Watcher):
    index = 'redshift'
    i_am_singular = 'Redshift Cluster'
    i_am_plural = 'Redshift Clusters'

    def __init__(self, accounts=None, debug=False):
        super(Redshift, self).__init__(accounts=accounts, debug=debug)
        self.honor_ephemerals = True
        self.ephemeral_paths = [
            "RestoreStatus",
            "ClusterStatus",
            "ClusterParameterGroups$ParameterApplyStatus",
            "ClusterParameterGroups$ClusterParameterStatusList$ParameterApplyErrorDescription",
            "ClusterParameterGroups$ClusterParameterStatusList$ParameterApplyStatus",
            "ClusterRevisionNumber"
        ]

    def slurp(self):
        """
        :returns: item_list - list of Redshift Policies.
                app.logger.debug("Checking {}/{}/{}".format(self.index, account, region.name))
                try:
                    redshift = connect(account, 'redshift', region=region)
                    response = self.wrap_aws_rate_limited_call(
                        redshift.describe_clusters
                    )
                    all_clusters = response['DescribeClustersResponse']['DescribeClustersResult']['Clusters']
                except Exception as e:
                    if region.name not in TROUBLE_REGIONS:
                        exc = BotoConnectionIssue(str(e), 'redshift', account, region.name)
                        response = self.wrap_aws_rate_limited_call(
                            redshift.describe_clusters,
                            marker=marker
                        )
                        all_clusters.extend(response['DescribeClustersResponse']['DescribeClustersResult']['Clusters'])
                        if response['DescribeClustersResponse']['DescribeClustersResult']['Marker'] is not None:
                            marker = response['DescribeClustersResponse']['DescribeClustersResult']['Marker']
                        else:
                            break

                except Exception as e:
                    if region.name not in TROUBLE_REGIONS:
                        exc = BotoConnectionIssue(str(e), 'redshift', account, region.name)
                        self.slurp_exception((self.index, account, region.name), exc, exception_map,
                                             source="{}-watcher".format(self.index))
                    continue
                app.logger.debug("Found {} {}".format(len(all_clusters), Redshift.i_am_plural))
                for cluster in all_clusters:
                    cluster_id = cluster['ClusterIdentifier']
                    if self.check_ignore_list(cluster_id):
                        continue

                    arn = ARN_PREFIX + ':redshift:{region}:{account_number}:cluster:{name}'.format(
                        region=region.name,
                        account_number=account_number,
                        name=cluster_id)

                    cluster['arn'] = arn

                    item = RedshiftCluster(region=region.name, account=account, name=cluster_id, arn=arn,
                                           config=dict(cluster), source_watcher=self)
                    item_list.append(item)

        return item_list, exception_map


class RedshiftCluster(ChangeItem):
    def __init__(self, region=None, account=None, name=None, arn=None, config=None, source_watcher=None):
        super(RedshiftCluster, self).__init__(
            index=Redshift.index,
            region=region,
            account=account,
            name=name,
            arn=arn,
            new_config=config if config else {},
            source_watcher=source_watcher)

<MSG> Adding marker/pagination code to redshift.describe_clusters()

<DFF> @@ -53,10 +53,20 @@ class Redshift(Watcher):
                 app.logger.debug("Checking {}/{}/{}".format(self.index, account, region.name))
                 try:
                     redshift = connect(account, 'redshift', region=region)
-                    response = self.wrap_aws_rate_limited_call(
-                        redshift.describe_clusters
-                    )
-                    all_clusters = response['DescribeClustersResponse']['DescribeClustersResult']['Clusters']
+
+                    all_clusters = []
+                    marker = None
+                    while True:
+                        response = self.wrap_aws_rate_limited_call(
+                            redshift.describe_clusters,
+                            marker=marker
+                        )
+                        all_clusters.extend(response['DescribeClustersResponse']['DescribeClustersResult']['Clusters'])
+                        if response['DescribeClustersResponse']['DescribeClustersResult']['Marker'] is not None:
+                            marker = response['DescribeClustersResponse']['DescribeClustersResult']['Marker']
+                        else:
+                            break
+
                 except Exception as e:
                     if region.name not in TROUBLE_REGIONS:
                         exc = BotoConnectionIssue(str(e), 'redshift', account, region.name)

{'commit': 'diff --git a/security_monkey/watchers/redshift.py b/security_monkey/watchers/redshift.py\nindex 56c24ec..6825396 100644\n--- a/security_monkey/watchers/redshift.py\n+++ b/security_monkey/watchers/redshift.py\n@@ -53,10 +53,20 @@ class Redshift(Watcher):\n                 app.logger.debug("Checking {}/{}/{}".format(self.index, account, region.name))\n                 try:\n                     redshift = connect(account, \'redshift\', region=region)\n-                    response = self.wrap_aws_rate_limited_call(\n-                        redshift.describe_clusters\n-                    )\n-                    all_clusters = response[\'DescribeClustersResponse\'][\'DescribeClustersResult\'][\'Clusters\']\n+\n+                    all_clusters = []\n+                    marker = None\n+                    while True:\n+                        response = self.wrap_aws_rate_limited_call(\n+                            redshift.describe_clusters,\n+                            marker=marker\n+                        )\n+                        all_clusters.extend(response[\'DescribeClustersResponse\'][\'DescribeClustersResult\'][\'Clusters\'])\n+                        if response[\'DescribeClustersResponse\'][\'DescribeClustersResult\'][\'Marker\'] is not None:\n+                            marker = response[\'DescribeClustersResponse\'][\'DescribeClustersResult\'][\'Marker\']\n+                        else:\n+                            break\n+\n                 except Exception as e:\n                     if region.name not in TROUBLE_REGIONS:\n                         exc = BotoConnectionIssue(str(e), \'redshift\', account, region.name)\n', 'commit_subject': 'Adding marker/pagination code to redshift.describe_clusters()', 'repo_name': 'Netflix/security_monkey', 'committer': {'name': 'Patrick Kelley', 'email': 'e0a41d54504520e8ef87d3e2884cb9b788db50ff@netflix.com', 'time_sec': '1427555292', 'tz_offset': '-420', 'date': {'seconds': '1427555292'}}, 'commit_tree': '6011f3924b064afcea0a36eedbf16749978c72be', 'commit_message': 'Adding marker/pagination code to redshift.describe_clusters()\n', 'language_name': 'Dart', 'license': 'apache-2.0', 'hunks': '@@ -53,10 +53,20 @@ class Redshift(Watcher):\n                 app.logger.debug("Checking {}/{}/{}".format(self.index, account, region.name))\n                 try:\n                     redshift = connect(account, \'redshift\', region=region)\n-                    response = self.wrap_aws_rate_limited_call(\n-                        redshift.describe_clusters\n-                    )\n-                    all_clusters = response[\'DescribeClustersResponse\'][\'DescribeClustersResult\'][\'Clusters\']\n+\n+                    all_clusters = []\n+                    marker = None\n+                    while True:\n+                        response = self.wrap_aws_rate_limited_call(\n+                            redshift.describe_clusters,\n+                            marker=marker\n+                        )\n+                        all_clusters.extend(response[\'DescribeClustersResponse\'][\'DescribeClustersResult\'][\'Clusters\'])\n+                        if response[\'DescribeClustersResponse\'][\'DescribeClustersResult\'][\'Marker\'] is not None:\n+                            marker = response[\'DescribeClustersResponse\'][\'DescribeClustersResult\'][\'Marker\']\n+                        else:\n+                            break\n+\n                 except Exception as e:\n                     if region.name not in TROUBLE_REGIONS:\n                         exc = BotoConnectionIssue(str(e), \'redshift\', account, region.name)\n', 'addition_count': 14, 'deletion_count': 4, 'src_file': 'a/security_monkey/watchers/redshift.py', 'tgt_file': 'b/security_monkey/watchers/redshift.py', 'file_extension': '.py', 'before_file': '#     Copyright 2014 Yelp, Inc.\n#\n#     Licensed under the Apache License, Version 2.0 (the "License");\n#     you may not use this file except in compliance with the License.\n#     You may obtain a copy of the License at\n#\n#         http://www.apache.org/licenses/LICENSE-2.0\n#\n#     Unless required by applicable law or agreed to in writing, software\n#     distributed under the License is distributed on an "AS IS" BASIS,\n#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n#     See the License for the specific language governing permissions and\n#     limitations under the License.\n"""\n.. module: security_monkey.watchers.redshift\n    :platform: Unix\n\n.. version:: $$VERSION$$\n.. moduleauthor:: Ivan Leichtling <ivanlei@yelp.com> @c0wl\n\n"""\n\nfrom security_monkey.watcher import Watcher\nfrom security_monkey.watcher import ChangeItem\nfrom security_monkey.constants import TROUBLE_REGIONS\nfrom security_monkey.exceptions import BotoConnectionIssue\nfrom security_monkey.datastore import Account\nfrom security_monkey import app, ARN_PREFIX\n\nfrom boto.redshift import regions\n\n\nclass Redshift(Watcher):\n    index = \'redshift\'\n    i_am_singular = \'Redshift Cluster\'\n    i_am_plural = \'Redshift Clusters\'\n\n    def __init__(self, accounts=None, debug=False):\n        super(Redshift, self).__init__(accounts=accounts, debug=debug)\n        self.honor_ephemerals = True\n        self.ephemeral_paths = [\n            "RestoreStatus",\n            "ClusterStatus",\n            "ClusterParameterGroups$ParameterApplyStatus",\n            "ClusterParameterGroups$ClusterParameterStatusList$ParameterApplyErrorDescription",\n            "ClusterParameterGroups$ClusterParameterStatusList$ParameterApplyStatus",\n            "ClusterRevisionNumber"\n        ]\n\n    def slurp(self):\n        """\n        :returns: item_list - list of Redshift Policies.\n                app.logger.debug("Checking {}/{}/{}".format(self.index, account, region.name))\n                try:\n                    redshift = connect(account, \'redshift\', region=region)\n                    response = self.wrap_aws_rate_limited_call(\n                        redshift.describe_clusters\n                    )\n                    all_clusters = response[\'DescribeClustersResponse\'][\'DescribeClustersResult\'][\'Clusters\']\n                except Exception as e:\n                    if region.name not in TROUBLE_REGIONS:\n                        exc = BotoConnectionIssue(str(e), \'redshift\', account, region.name)\n                        response = self.wrap_aws_rate_limited_call(\n                            redshift.describe_clusters,\n                            marker=marker\n                        )\n                        all_clusters.extend(response[\'DescribeClustersResponse\'][\'DescribeClustersResult\'][\'Clusters\'])\n                        if response[\'DescribeClustersResponse\'][\'DescribeClustersResult\'][\'Marker\'] is not None:\n                            marker = response[\'DescribeClustersResponse\'][\'DescribeClustersResult\'][\'Marker\']\n                        else:\n                            break\n\n                except Exception as e:\n                    if region.name not in TROUBLE_REGIONS:\n                        exc = BotoConnectionIssue(str(e), \'redshift\', account, region.name)\n                        self.slurp_exception((self.index, account, region.name), exc, exception_map,\n                                             source="{}-watcher".format(self.index))\n                    continue\n                app.logger.debug("Found {} {}".format(len(all_clusters), Redshift.i_am_plural))\n                for cluster in all_clusters:\n                    cluster_id = cluster[\'ClusterIdentifier\']\n                    if self.check_ignore_list(cluster_id):\n                        continue\n\n                    arn = ARN_PREFIX + \':redshift:{region}:{account_number}:cluster:{name}\'.format(\n                        region=region.name,\n                        account_number=account_number,\n                        name=cluster_id)\n\n                    cluster[\'arn\'] = arn\n\n                    item = RedshiftCluster(region=region.name, account=account, name=cluster_id, arn=arn,\n                                           config=dict(cluster), source_watcher=self)\n                    item_list.append(item)\n\n        return item_list, exception_map\n\n\nclass RedshiftCluster(ChangeItem):\n    def __init__(self, region=None, account=None, name=None, arn=None, config=None, source_watcher=None):\n        super(RedshiftCluster, self).__init__(\n            index=Redshift.index,\n            region=region,\n            account=account,\n            name=name,\n            arn=arn,\n            new_config=config if config else {},\n            source_watcher=source_watcher)\n', 'message': 'Adding marker/pagination code to redshift.describe_clusters()\n', 'author_name': 'Patrick Kelley', 'lang': 'py'}