We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
[nginx] No debug logging when doing Pokémon exception handling
The [nginx code](https://github.com/DataDog/dd-agent/blob/49952b4429b125619bc8d9f51bb6564e7c0d2e12/checks.d/nginx.py#L45-L50) follows a [pattern of catching all exceptions](http://blog.codinghorror.com/new-programming-jargon/) when attempting to submit a metric, which makes sense due to wanting to continue without error. Unfortunately, it additionally eats the real error message and displays a super generic message in the logs. This makes it very difficult to actually debug issues.

I ended up manually modifying the code on my local install to figure out that my `nginx.yaml` had a syntax error in it. It would have been more obvious if the true exception (`unhashable type: 'dict'`) had been printed out. (Additionally, there should probably be error checking when tags are loaded instead of reported.)

I'm willing to fix either or both of these issues, but would like advice on how you would like to see them done (e.g. log at a different level? Log to stderr?). Thanks!


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `checks.d/nginx.py`
Content:
```
1 # stdlib
2 import re
3 import urlparse
4 
5 # 3rd party
6 import requests
7 import simplejson as json
8 
9 # project
10 from checks import AgentCheck
11 from util import headers
12 
13 
14 class Nginx(AgentCheck):
15     """Tracks basic nginx metrics via the status module
16     * number of connections
17     * number of requets per second
18 
19     Requires nginx to have the status option compiled.
20     See http://wiki.nginx.org/HttpStubStatusModule for more details
21 
22     $ curl http://localhost:81/nginx_status/
23     Active connections: 8
24     server accepts handled requests
25      1156958 1156958 4491319
26     Reading: 0 Writing: 2 Waiting: 6
27 
28     """
29     def check(self, instance):
30         if 'nginx_status_url' not in instance:
31             raise Exception('NginX instance missing "nginx_status_url" value.')
32         tags = instance.get('tags', [])
33 
34         response, content_type = self._get_data(instance)
35         self.log.debug(u"Nginx status `response`: {0}".format(response))
36         self.log.debug(u"Nginx status `content_type`: {0}".format(content_type))
37 
38         if content_type.startswith('application/json'):
39             metrics = self.parse_json(response, tags)
40         else:
41             metrics = self.parse_text(response, tags)
42 
43         funcs = {
44             'gauge': self.gauge,
45             'rate': self.rate
46         }
47         for row in metrics:
48             try:
49                 name, value, tags, metric_type = row
50                 func = funcs[metric_type]
51                 func(name, value, tags)
52             except Exception:
53                 self.log.error(u'Could not submit metric: %s' % repr(row))
54 
55     def _get_data(self, instance):
56         url = instance.get('nginx_status_url')
57         ssl_validation = instance.get('ssl_validation', True)
58 
59         auth = None
60         if 'user' in instance and 'password' in instance:
61             auth = (instance['user'], instance['password'])
62 
63         # Submit a service check for status page availability.
64         parsed_url = urlparse.urlparse(url)
65         nginx_host = parsed_url.hostname
66         nginx_port = parsed_url.port or 80
67         service_check_name = 'nginx.can_connect'
68         service_check_tags = ['host:%s' % nginx_host, 'port:%s' % nginx_port]
69         try:
70             self.log.debug(u"Querying URL: {0}".format(url))
71             r = requests.get(url, auth=auth, headers=headers(self.agentConfig),
72                              verify=ssl_validation)
73             r.raise_for_status()
74         except Exception:
75             self.service_check(service_check_name, AgentCheck.CRITICAL,
76                                tags=service_check_tags)
77             raise
78         else:
79             self.service_check(service_check_name, AgentCheck.OK,
80                                tags=service_check_tags)
81 
82         body = r.content
83         resp_headers = r.headers
84         return body, resp_headers.get('content-type', 'text/plain')
85 
86     @classmethod
87     def parse_text(cls, raw, tags):
88         # Thanks to http://hostingfu.com/files/nginx/nginxstats.py for this code
89         # Connections
90         output = []
91         parsed = re.search(r'Active connections:\s+(\d+)', raw)
92         if parsed:
93             connections = int(parsed.group(1))
94             output.append(('nginx.net.connections', connections, tags, 'gauge'))
95 
96         # Requests per second
97         parsed = re.search(r'\s*(\d+)\s+(\d+)\s+(\d+)', raw)
98         if parsed:
99             conn = int(parsed.group(1))
100             handled = int(parsed.group(2))
101             requests = int(parsed.group(3))
102             output.extend([('nginx.net.conn_opened_per_s', conn, tags, 'rate'),
103                            ('nginx.net.conn_dropped_per_s', conn - handled, tags, 'rate'),
104                            ('nginx.net.request_per_s', requests, tags, 'rate')])
105 
106         # Connection states, reading, writing or waiting for clients
107         parsed = re.search(r'Reading: (\d+)\s+Writing: (\d+)\s+Waiting: (\d+)', raw)
108         if parsed:
109             reading, writing, waiting = parsed.groups()
110             output.extend([
111                 ("nginx.net.reading", int(reading), tags, 'gauge'),
112                 ("nginx.net.writing", int(writing), tags, 'gauge'),
113                 ("nginx.net.waiting", int(waiting), tags, 'gauge'),
114             ])
115         return output
116 
117     @classmethod
118     def parse_json(cls, raw, tags=None):
119         if tags is None:
120             tags = []
121         parsed = json.loads(raw)
122         metric_base = 'nginx'
123         output = []
124         all_keys = parsed.keys()
125 
126         tagged_keys = [('caches', 'cache'), ('server_zones', 'server_zone'),
127                        ('upstreams', 'upstream')]
128 
129         # Process the special keys that should turn into tags instead of
130         # getting concatenated to the metric name
131         for key, tag_name in tagged_keys:
132             metric_name = '%s.%s' % (metric_base, tag_name)
133             for tag_val, data in parsed.get(key, {}).iteritems():
134                 tag = '%s:%s' % (tag_name, tag_val)
135                 output.extend(cls._flatten_json(metric_name, data, tags + [tag]))
136 
137         # Process the rest of the keys
138         rest = set(all_keys) - set([k for k, _ in tagged_keys])
139         for key in rest:
140             metric_name = '%s.%s' % (metric_base, key)
141             output.extend(cls._flatten_json(metric_name, parsed[key], tags))
142 
143         return output
144 
145     @classmethod
146     def _flatten_json(cls, metric_base, val, tags):
147         ''' Recursively flattens the nginx json object. Returns the following:
148             [(metric_name, value, tags)]
149         '''
150         output = []
151         if isinstance(val, dict):
152             # Pull out the server as a tag instead of trying to read as a metric
153             if 'server' in val and val['server']:
154                 server = 'server:%s' % val.pop('server')
155                 if tags is None:
156                     tags = [server]
157                 else:
158                     tags = tags + [server]
159             for key, val2 in val.iteritems():
160                 metric_name = '%s.%s' % (metric_base, key)
161                 output.extend(cls._flatten_json(metric_name, val2, tags))
162 
163         elif isinstance(val, list):
164             for val2 in val:
165                 output.extend(cls._flatten_json(metric_base, val2, tags))
166 
167         elif isinstance(val, bool):
168             # Turn bools into 0/1 values
169             if val:
170                 val = 1
171             else:
172                 val = 0
173             output.append((metric_base, val, tags, 'gauge'))
174 
175         elif isinstance(val, (int, float)):
176             output.append((metric_base, val, tags, 'gauge'))
177 
178         return output
179
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/checks.d/nginx.py b/checks.d/nginx.py
--- a/checks.d/nginx.py
+++ b/checks.d/nginx.py
@@ -49,8 +49,8 @@
                 name, value, tags, metric_type = row
                 func = funcs[metric_type]
                 func(name, value, tags)
-            except Exception:
-                self.log.error(u'Could not submit metric: %s' % repr(row))
+            except Exception, e:
+                self.log.error(u'Could not submit metric: %s: %s' % (repr(row), str(e)))
 
     def _get_data(self, instance):
         url = instance.get('nginx_status_url')

{"golden_diff": "diff --git a/checks.d/nginx.py b/checks.d/nginx.py\n--- a/checks.d/nginx.py\n+++ b/checks.d/nginx.py\n@@ -49,8 +49,8 @@\n                 name, value, tags, metric_type = row\n                 func = funcs[metric_type]\n                 func(name, value, tags)\n-            except Exception:\n-                self.log.error(u'Could not submit metric: %s' % repr(row))\n+            except Exception, e:\n+                self.log.error(u'Could not submit metric: %s: %s' % (repr(row), str(e)))\n \n     def _get_data(self, instance):\n         url = instance.get('nginx_status_url')\n", "issue": "[nginx] No debug logging when doing Pok\u00e9mon exception handling\nThe [nginx code](https://github.com/DataDog/dd-agent/blob/49952b4429b125619bc8d9f51bb6564e7c0d2e12/checks.d/nginx.py#L45-L50) follows a [pattern of catching all exceptions](http://blog.codinghorror.com/new-programming-jargon/) when attempting to submit a metric, which makes sense due to wanting to continue without error. Unfortunately, it additionally eats the real error message and displays a super generic message in the logs. This makes it very difficult to actually debug issues.\n\nI ended up manually modifying the code on my local install to figure out that my `nginx.yaml` had a syntax error in it. It would have been more obvious if the true exception (`unhashable type: 'dict'`) had been printed out. (Additionally, there should probably be error checking when tags are loaded instead of reported.)\n\nI'm willing to fix either or both of these issues, but would like advice on how you would like to see them done (e.g. log at a different level? Log to stderr?). Thanks!\n\n", "before_files": [{"content": "# stdlib\nimport re\nimport urlparse\n\n# 3rd party\nimport requests\nimport simplejson as json\n\n# project\nfrom checks import AgentCheck\nfrom util import headers\n\n\nclass Nginx(AgentCheck):\n    \"\"\"Tracks basic nginx metrics via the status module\n    * number of connections\n    * number of requets per second\n\n    Requires nginx to have the status option compiled.\n    See http://wiki.nginx.org/HttpStubStatusModule for more details\n\n    $ curl http://localhost:81/nginx_status/\n    Active connections: 8\n    server accepts handled requests\n     1156958 1156958 4491319\n    Reading: 0 Writing: 2 Waiting: 6\n\n    \"\"\"\n    def check(self, instance):\n        if 'nginx_status_url' not in instance:\n            raise Exception('NginX instance missing \"nginx_status_url\" value.')\n        tags = instance.get('tags', [])\n\n        response, content_type = self._get_data(instance)\n        self.log.debug(u\"Nginx status `response`: {0}\".format(response))\n        self.log.debug(u\"Nginx status `content_type`: {0}\".format(content_type))\n\n        if content_type.startswith('application/json'):\n            metrics = self.parse_json(response, tags)\n        else:\n            metrics = self.parse_text(response, tags)\n\n        funcs = {\n            'gauge': self.gauge,\n            'rate': self.rate\n        }\n        for row in metrics:\n            try:\n                name, value, tags, metric_type = row\n                func = funcs[metric_type]\n                func(name, value, tags)\n            except Exception:\n                self.log.error(u'Could not submit metric: %s' % repr(row))\n\n    def _get_data(self, instance):\n        url = instance.get('nginx_status_url')\n        ssl_validation = instance.get('ssl_validation', True)\n\n        auth = None\n        if 'user' in instance and 'password' in instance:\n            auth = (instance['user'], instance['password'])\n\n        # Submit a service check for status page availability.\n        parsed_url = urlparse.urlparse(url)\n        nginx_host = parsed_url.hostname\n        nginx_port = parsed_url.port or 80\n        service_check_name = 'nginx.can_connect'\n        service_check_tags = ['host:%s' % nginx_host, 'port:%s' % nginx_port]\n        try:\n            self.log.debug(u\"Querying URL: {0}\".format(url))\n            r = requests.get(url, auth=auth, headers=headers(self.agentConfig),\n                             verify=ssl_validation)\n            r.raise_for_status()\n        except Exception:\n            self.service_check(service_check_name, AgentCheck.CRITICAL,\n                               tags=service_check_tags)\n            raise\n        else:\n            self.service_check(service_check_name, AgentCheck.OK,\n                               tags=service_check_tags)\n\n        body = r.content\n        resp_headers = r.headers\n        return body, resp_headers.get('content-type', 'text/plain')\n\n    @classmethod\n    def parse_text(cls, raw, tags):\n        # Thanks to http://hostingfu.com/files/nginx/nginxstats.py for this code\n        # Connections\n        output = []\n        parsed = re.search(r'Active connections:\\s+(\\d+)', raw)\n        if parsed:\n            connections = int(parsed.group(1))\n            output.append(('nginx.net.connections', connections, tags, 'gauge'))\n\n        # Requests per second\n        parsed = re.search(r'\\s*(\\d+)\\s+(\\d+)\\s+(\\d+)', raw)\n        if parsed:\n            conn = int(parsed.group(1))\n            handled = int(parsed.group(2))\n            requests = int(parsed.group(3))\n            output.extend([('nginx.net.conn_opened_per_s', conn, tags, 'rate'),\n                           ('nginx.net.conn_dropped_per_s', conn - handled, tags, 'rate'),\n                           ('nginx.net.request_per_s', requests, tags, 'rate')])\n\n        # Connection states, reading, writing or waiting for clients\n        parsed = re.search(r'Reading: (\\d+)\\s+Writing: (\\d+)\\s+Waiting: (\\d+)', raw)\n        if parsed:\n            reading, writing, waiting = parsed.groups()\n            output.extend([\n                (\"nginx.net.reading\", int(reading), tags, 'gauge'),\n                (\"nginx.net.writing\", int(writing), tags, 'gauge'),\n                (\"nginx.net.waiting\", int(waiting), tags, 'gauge'),\n            ])\n        return output\n\n    @classmethod\n    def parse_json(cls, raw, tags=None):\n        if tags is None:\n            tags = []\n        parsed = json.loads(raw)\n        metric_base = 'nginx'\n        output = []\n        all_keys = parsed.keys()\n\n        tagged_keys = [('caches', 'cache'), ('server_zones', 'server_zone'),\n                       ('upstreams', 'upstream')]\n\n        # Process the special keys that should turn into tags instead of\n        # getting concatenated to the metric name\n        for key, tag_name in tagged_keys:\n            metric_name = '%s.%s' % (metric_base, tag_name)\n            for tag_val, data in parsed.get(key, {}).iteritems():\n                tag = '%s:%s' % (tag_name, tag_val)\n                output.extend(cls._flatten_json(metric_name, data, tags + [tag]))\n\n        # Process the rest of the keys\n        rest = set(all_keys) - set([k for k, _ in tagged_keys])\n        for key in rest:\n            metric_name = '%s.%s' % (metric_base, key)\n            output.extend(cls._flatten_json(metric_name, parsed[key], tags))\n\n        return output\n\n    @classmethod\n    def _flatten_json(cls, metric_base, val, tags):\n        ''' Recursively flattens the nginx json object. Returns the following:\n            [(metric_name, value, tags)]\n        '''\n        output = []\n        if isinstance(val, dict):\n            # Pull out the server as a tag instead of trying to read as a metric\n            if 'server' in val and val['server']:\n                server = 'server:%s' % val.pop('server')\n                if tags is None:\n                    tags = [server]\n                else:\n                    tags = tags + [server]\n            for key, val2 in val.iteritems():\n                metric_name = '%s.%s' % (metric_base, key)\n                output.extend(cls._flatten_json(metric_name, val2, tags))\n\n        elif isinstance(val, list):\n            for val2 in val:\n                output.extend(cls._flatten_json(metric_base, val2, tags))\n\n        elif isinstance(val, bool):\n            # Turn bools into 0/1 values\n            if val:\n                val = 1\n            else:\n                val = 0\n            output.append((metric_base, val, tags, 'gauge'))\n\n        elif isinstance(val, (int, float)):\n            output.append((metric_base, val, tags, 'gauge'))\n\n        return output\n", "path": "checks.d/nginx.py"}], "after_files": [{"content": "# stdlib\nimport re\nimport urlparse\n\n# 3rd party\nimport requests\nimport simplejson as json\n\n# project\nfrom checks import AgentCheck\nfrom util import headers\n\n\nclass Nginx(AgentCheck):\n    \"\"\"Tracks basic nginx metrics via the status module\n    * number of connections\n    * number of requets per second\n\n    Requires nginx to have the status option compiled.\n    See http://wiki.nginx.org/HttpStubStatusModule for more details\n\n    $ curl http://localhost:81/nginx_status/\n    Active connections: 8\n    server accepts handled requests\n     1156958 1156958 4491319\n    Reading: 0 Writing: 2 Waiting: 6\n\n    \"\"\"\n    def check(self, instance):\n        if 'nginx_status_url' not in instance:\n            raise Exception('NginX instance missing \"nginx_status_url\" value.')\n        tags = instance.get('tags', [])\n\n        response, content_type = self._get_data(instance)\n        self.log.debug(u\"Nginx status `response`: {0}\".format(response))\n        self.log.debug(u\"Nginx status `content_type`: {0}\".format(content_type))\n\n        if content_type.startswith('application/json'):\n            metrics = self.parse_json(response, tags)\n        else:\n            metrics = self.parse_text(response, tags)\n\n        funcs = {\n            'gauge': self.gauge,\n            'rate': self.rate\n        }\n        for row in metrics:\n            try:\n                name, value, tags, metric_type = row\n                func = funcs[metric_type]\n                func(name, value, tags)\n            except Exception, e:\n                self.log.error(u'Could not submit metric: %s: %s' % (repr(row), str(e)))\n\n    def _get_data(self, instance):\n        url = instance.get('nginx_status_url')\n        ssl_validation = instance.get('ssl_validation', True)\n\n        auth = None\n        if 'user' in instance and 'password' in instance:\n            auth = (instance['user'], instance['password'])\n\n        # Submit a service check for status page availability.\n        parsed_url = urlparse.urlparse(url)\n        nginx_host = parsed_url.hostname\n        nginx_port = parsed_url.port or 80\n        service_check_name = 'nginx.can_connect'\n        service_check_tags = ['host:%s' % nginx_host, 'port:%s' % nginx_port]\n        try:\n            self.log.debug(u\"Querying URL: {0}\".format(url))\n            r = requests.get(url, auth=auth, headers=headers(self.agentConfig),\n                             verify=ssl_validation)\n            r.raise_for_status()\n        except Exception:\n            self.service_check(service_check_name, AgentCheck.CRITICAL,\n                               tags=service_check_tags)\n            raise\n        else:\n            self.service_check(service_check_name, AgentCheck.OK,\n                               tags=service_check_tags)\n\n        body = r.content\n        resp_headers = r.headers\n        return body, resp_headers.get('content-type', 'text/plain')\n\n    @classmethod\n    def parse_text(cls, raw, tags):\n        # Thanks to http://hostingfu.com/files/nginx/nginxstats.py for this code\n        # Connections\n        output = []\n        parsed = re.search(r'Active connections:\\s+(\\d+)', raw)\n        if parsed:\n            connections = int(parsed.group(1))\n            output.append(('nginx.net.connections', connections, tags, 'gauge'))\n\n        # Requests per second\n        parsed = re.search(r'\\s*(\\d+)\\s+(\\d+)\\s+(\\d+)', raw)\n        if parsed:\n            conn = int(parsed.group(1))\n            handled = int(parsed.group(2))\n            requests = int(parsed.group(3))\n            output.extend([('nginx.net.conn_opened_per_s', conn, tags, 'rate'),\n                           ('nginx.net.conn_dropped_per_s', conn - handled, tags, 'rate'),\n                           ('nginx.net.request_per_s', requests, tags, 'rate')])\n\n        # Connection states, reading, writing or waiting for clients\n        parsed = re.search(r'Reading: (\\d+)\\s+Writing: (\\d+)\\s+Waiting: (\\d+)', raw)\n        if parsed:\n            reading, writing, waiting = parsed.groups()\n            output.extend([\n                (\"nginx.net.reading\", int(reading), tags, 'gauge'),\n                (\"nginx.net.writing\", int(writing), tags, 'gauge'),\n                (\"nginx.net.waiting\", int(waiting), tags, 'gauge'),\n            ])\n        return output\n\n    @classmethod\n    def parse_json(cls, raw, tags=None):\n        if tags is None:\n            tags = []\n        parsed = json.loads(raw)\n        metric_base = 'nginx'\n        output = []\n        all_keys = parsed.keys()\n\n        tagged_keys = [('caches', 'cache'), ('server_zones', 'server_zone'),\n                       ('upstreams', 'upstream')]\n\n        # Process the special keys that should turn into tags instead of\n        # getting concatenated to the metric name\n        for key, tag_name in tagged_keys:\n            metric_name = '%s.%s' % (metric_base, tag_name)\n            for tag_val, data in parsed.get(key, {}).iteritems():\n                tag = '%s:%s' % (tag_name, tag_val)\n                output.extend(cls._flatten_json(metric_name, data, tags + [tag]))\n\n        # Process the rest of the keys\n        rest = set(all_keys) - set([k for k, _ in tagged_keys])\n        for key in rest:\n            metric_name = '%s.%s' % (metric_base, key)\n            output.extend(cls._flatten_json(metric_name, parsed[key], tags))\n\n        return output\n\n    @classmethod\n    def _flatten_json(cls, metric_base, val, tags):\n        ''' Recursively flattens the nginx json object. Returns the following:\n            [(metric_name, value, tags)]\n        '''\n        output = []\n        if isinstance(val, dict):\n            # Pull out the server as a tag instead of trying to read as a metric\n            if 'server' in val and val['server']:\n                server = 'server:%s' % val.pop('server')\n                if tags is None:\n                    tags = [server]\n                else:\n                    tags = tags + [server]\n            for key, val2 in val.iteritems():\n                metric_name = '%s.%s' % (metric_base, key)\n                output.extend(cls._flatten_json(metric_name, val2, tags))\n\n        elif isinstance(val, list):\n            for val2 in val:\n                output.extend(cls._flatten_json(metric_base, val2, tags))\n\n        elif isinstance(val, bool):\n            # Turn bools into 0/1 values\n            if val:\n                val = 1\n            else:\n                val = 0\n            output.append((metric_base, val, tags, 'gauge'))\n\n        elif isinstance(val, (int, float)):\n            output.append((metric_base, val, tags, 'gauge'))\n\n        return output\n", "path": "checks.d/nginx.py"}]}