We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
ci - intermittent sqlite-related failures
I've had a couple CI builds run today associated with #4101 - tests succeed locally but fail intermittently in CI for one or more ubuntu-based python environments. I see similar failures for #4035  and #4090 . The failures are all related to sqlite, which appears to be missing from the build image sometimes. That causes test results to vary across builds.

I don't know if this is a transient issue or not. Is it worth having a script step for ubuntu-based builds to ensure that sqlite is available?

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `tools/c7n_mailer/c7n_mailer/ldap_lookup.py`
Content:
```
1 # Copyright 2017 Capital One Services, LLC
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 import json
15 
16 import re
17 import redis
18 
19 try:
20     import sqlite3
21 except ImportError:
22     have_sqlite = False
23 else:
24     have_sqlite = True
25 from ldap3 import Connection
26 from ldap3.core.exceptions import LDAPSocketOpenError
27 
28 
29 class LdapLookup(object):
30 
31     def __init__(self, config, logger):
32         self.log = logger
33         self.connection = self.get_connection(
34             config.get('ldap_uri'),
35             config.get('ldap_bind_user', None),
36             config.get('ldap_bind_password', None)
37         )
38         self.base_dn = config.get('ldap_bind_dn')
39         self.email_key = config.get('ldap_email_key', 'mail')
40         self.manager_attr = config.get('ldap_manager_attribute', 'manager')
41         self.uid_key = config.get('ldap_uid_attribute', 'sAMAccountName')
42         self.attributes = ['displayName', self.uid_key, self.email_key, self.manager_attr]
43         self.uid_regex = config.get('ldap_uid_regex', None)
44         self.cache_engine = config.get('cache_engine', None)
45         if self.cache_engine == 'redis':
46             redis_host = config.get('redis_host')
47             redis_port = int(config.get('redis_port', 6379))
48             self.caching = self.get_redis_connection(redis_host, redis_port)
49         elif self.cache_engine == 'sqlite':
50             if not have_sqlite:
51                 raise RuntimeError('No sqlite available: stackoverflow.com/q/44058239')
52             self.caching = LocalSqlite(config.get('ldap_cache_file', '/var/tmp/ldap.cache'), logger)
53 
54     def get_redis_connection(self, redis_host, redis_port):
55         return Redis(redis_host=redis_host, redis_port=redis_port, db=0)
56 
57     def get_connection(self, ldap_uri, ldap_bind_user, ldap_bind_password):
58         # note, if ldap_bind_user and ldap_bind_password are None
59         # an anonymous bind will be attempted.
60         try:
61             return Connection(
62                 ldap_uri, user=ldap_bind_user, password=ldap_bind_password,
63                 auto_bind=True,
64                 receive_timeout=30,
65                 auto_referrals=False,
66             )
67         except LDAPSocketOpenError:
68             self.log.error('Not able to establish a connection with LDAP.')
69 
70     def search_ldap(self, base_dn, ldap_filter, attributes):
71         self.connection.search(base_dn, ldap_filter, attributes=self.attributes)
72         if len(self.connection.entries) == 0:
73             self.log.warning("user not found. base_dn: %s filter: %s", base_dn, ldap_filter)
74             return {}
75         if len(self.connection.entries) > 1:
76             self.log.warning("too many results for search %s", ldap_filter)
77             return {}
78         return self.connection.entries[0]
79 
80     def get_email_to_addrs_from_uid(self, uid, manager=False):
81         to_addrs = []
82         uid_metadata = self.get_metadata_from_uid(uid)
83         uid_email = uid_metadata.get(self.email_key, None)
84         if uid_email:
85             to_addrs.append(uid_email)
86         if manager:
87             uid_manager_dn = uid_metadata.get(self.manager_attr, None)
88             uid_manager_email = None
89             if uid_manager_dn:
90                 uid_manager = self.get_metadata_from_dn(uid_manager_dn)
91                 uid_manager_email = uid_manager.get('mail')
92             if uid_manager_email:
93                 to_addrs.append(uid_manager_email)
94         return to_addrs
95 
96     # eg, dn = uid=bill_lumbergh,cn=users,dc=initech,dc=com
97     def get_metadata_from_dn(self, user_dn):
98         if self.cache_engine:
99             cache_result = self.caching.get(user_dn)
100             if cache_result:
101                 cache_msg = 'Got ldap metadata from local cache for: %s' % user_dn
102                 self.log.debug(cache_msg)
103                 return cache_result
104         ldap_filter = '(%s=*)' % self.uid_key
105         ldap_results = self.search_ldap(user_dn, ldap_filter, attributes=self.attributes)
106         if ldap_results:
107             ldap_user_metadata = self.get_dict_from_ldap_object(self.connection.entries[0])
108         else:
109             self.caching.set(user_dn, {})
110             return {}
111         if self.cache_engine:
112             self.log.debug('Writing user: %s metadata to cache engine.' % user_dn)
113             self.caching.set(user_dn, ldap_user_metadata)
114             self.caching.set(ldap_user_metadata[self.uid_key], ldap_user_metadata)
115         return ldap_user_metadata
116 
117     def get_dict_from_ldap_object(self, ldap_user_object):
118         ldap_user_metadata = {attr.key: attr.value for attr in ldap_user_object}
119         ldap_user_metadata['dn'] = ldap_user_object.entry_dn
120 
121         email_key = ldap_user_metadata.get(self.email_key, None)
122         uid_key = ldap_user_metadata.get(self.uid_key, None)
123 
124         if not email_key or not uid_key:
125             return {}
126         else:
127             ldap_user_metadata['self.email_key'] = email_key.lower()
128             ldap_user_metadata['self.uid_key'] = uid_key.lower()
129 
130         return ldap_user_metadata
131 
132     # eg, uid = bill_lumbergh
133     def get_metadata_from_uid(self, uid):
134         uid = uid.lower()
135         if self.uid_regex:
136             # for example if you set ldap_uid_regex in your mailer.yml to "^[0-9]{6}$" then it
137             # would only query LDAP if your string length is 6 characters long and only digits.
138             # re.search("^[0-9]{6}$", "123456")
139             # Out[41]: <_sre.SRE_Match at 0x1109ab440>
140             # re.search("^[0-9]{6}$", "1234567") returns None, or "12345a' also returns None
141             if not re.search(self.uid_regex, uid):
142                 regex_msg = 'uid does not match regex: %s %s' % (self.uid_regex, uid)
143                 self.log.debug(regex_msg)
144                 return {}
145         if self.cache_engine:
146             cache_result = self.caching.get(uid)
147             if cache_result or cache_result == {}:
148                 cache_msg = 'Got ldap metadata from local cache for: %s' % uid
149                 self.log.debug(cache_msg)
150                 return cache_result
151         ldap_filter = '(%s=%s)' % (self.uid_key, uid)
152         ldap_results = self.search_ldap(self.base_dn, ldap_filter, attributes=self.attributes)
153         if ldap_results:
154             ldap_user_metadata = self.get_dict_from_ldap_object(self.connection.entries[0])
155             if self.cache_engine:
156                 self.log.debug('Writing user: %s metadata to cache engine.' % uid)
157                 if ldap_user_metadata.get('dn'):
158                     self.caching.set(ldap_user_metadata['dn'], ldap_user_metadata)
159                     self.caching.set(uid, ldap_user_metadata)
160                 else:
161                     self.caching.set(uid, {})
162         else:
163             if self.cache_engine:
164                 self.caching.set(uid, {})
165             return {}
166         return ldap_user_metadata
167 
168 
169 # Use sqlite as a local cache for folks not running the mailer in lambda, avoids extra daemons
170 # as dependencies. This normalizes the methods to set/get functions, so you can interchangeable
171 # decide which caching system to use, a local file, or memcache, redis, etc
172 # If you don't want a redis dependency and aren't running the mailer in lambda this works well
173 class LocalSqlite(object):
174     def __init__(self, local_filename, logger):
175         self.log = logger
176         self.sqlite = sqlite3.connect(local_filename)
177         self.sqlite.execute('''CREATE TABLE IF NOT EXISTS ldap_cache(key text, value text)''')
178 
179     def get(self, key):
180         sqlite_result = self.sqlite.execute("select * FROM ldap_cache WHERE key=?", (key,))
181         result = sqlite_result.fetchall()
182         if len(result) != 1:
183             error_msg = 'Did not get 1 result from sqlite, something went wrong with key: %s' % key
184             self.log.error(error_msg)
185             return None
186         return json.loads(result[0][1])
187 
188     def set(self, key, value):
189         # note, the ? marks are required to ensure escaping into the database.
190         self.sqlite.execute("INSERT INTO ldap_cache VALUES (?, ?)", (key, json.dumps(value)))
191         self.sqlite.commit()
192 
193 
194 # redis can't write complex python objects like dictionaries as values (the way memcache can)
195 # so we turn our dict into a json string when setting, and json.loads when getting
196 class Redis(object):
197     def __init__(self, redis_host=None, redis_port=6379, db=0):
198         self.connection = redis.StrictRedis(host=redis_host, port=redis_port, db=db)
199 
200     def get(self, key):
201         cache_value = self.connection.get(key)
202         if cache_value:
203             return json.loads(cache_value)
204 
205     def set(self, key, value):
206         return self.connection.set(key, json.dumps(value))
207
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/tools/c7n_mailer/c7n_mailer/ldap_lookup.py b/tools/c7n_mailer/c7n_mailer/ldap_lookup.py
--- a/tools/c7n_mailer/c7n_mailer/ldap_lookup.py
+++ b/tools/c7n_mailer/c7n_mailer/ldap_lookup.py
@@ -16,12 +16,7 @@
 import re
 import redis
 
-try:
-    import sqlite3
-except ImportError:
-    have_sqlite = False
-else:
-    have_sqlite = True
+import sqlite3
 from ldap3 import Connection
 from ldap3.core.exceptions import LDAPSocketOpenError
 
@@ -47,8 +42,6 @@
             redis_port = int(config.get('redis_port', 6379))
             self.caching = self.get_redis_connection(redis_host, redis_port)
         elif self.cache_engine == 'sqlite':
-            if not have_sqlite:
-                raise RuntimeError('No sqlite available: stackoverflow.com/q/44058239')
             self.caching = LocalSqlite(config.get('ldap_cache_file', '/var/tmp/ldap.cache'), logger)
 
     def get_redis_connection(self, redis_host, redis_port):

{"golden_diff": "diff --git a/tools/c7n_mailer/c7n_mailer/ldap_lookup.py b/tools/c7n_mailer/c7n_mailer/ldap_lookup.py\n--- a/tools/c7n_mailer/c7n_mailer/ldap_lookup.py\n+++ b/tools/c7n_mailer/c7n_mailer/ldap_lookup.py\n@@ -16,12 +16,7 @@\n import re\n import redis\n \n-try:\n-    import sqlite3\n-except ImportError:\n-    have_sqlite = False\n-else:\n-    have_sqlite = True\n+import sqlite3\n from ldap3 import Connection\n from ldap3.core.exceptions import LDAPSocketOpenError\n \n@@ -47,8 +42,6 @@\n             redis_port = int(config.get('redis_port', 6379))\n             self.caching = self.get_redis_connection(redis_host, redis_port)\n         elif self.cache_engine == 'sqlite':\n-            if not have_sqlite:\n-                raise RuntimeError('No sqlite available: stackoverflow.com/q/44058239')\n             self.caching = LocalSqlite(config.get('ldap_cache_file', '/var/tmp/ldap.cache'), logger)\n \n     def get_redis_connection(self, redis_host, redis_port):\n", "issue": "ci - intermittent sqlite-related failures\nI've had a couple CI builds run today associated with #4101 - tests succeed locally but fail intermittently in CI for one or more ubuntu-based python environments. I see similar failures for #4035  and #4090 . The failures are all related to sqlite, which appears to be missing from the build image sometimes. That causes test results to vary across builds.\r\n\r\nI don't know if this is a transient issue or not. Is it worth having a script step for ubuntu-based builds to ensure that sqlite is available?\n", "before_files": [{"content": "# Copyright 2017 Capital One Services, LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nimport json\n\nimport re\nimport redis\n\ntry:\n    import sqlite3\nexcept ImportError:\n    have_sqlite = False\nelse:\n    have_sqlite = True\nfrom ldap3 import Connection\nfrom ldap3.core.exceptions import LDAPSocketOpenError\n\n\nclass LdapLookup(object):\n\n    def __init__(self, config, logger):\n        self.log = logger\n        self.connection = self.get_connection(\n            config.get('ldap_uri'),\n            config.get('ldap_bind_user', None),\n            config.get('ldap_bind_password', None)\n        )\n        self.base_dn = config.get('ldap_bind_dn')\n        self.email_key = config.get('ldap_email_key', 'mail')\n        self.manager_attr = config.get('ldap_manager_attribute', 'manager')\n        self.uid_key = config.get('ldap_uid_attribute', 'sAMAccountName')\n        self.attributes = ['displayName', self.uid_key, self.email_key, self.manager_attr]\n        self.uid_regex = config.get('ldap_uid_regex', None)\n        self.cache_engine = config.get('cache_engine', None)\n        if self.cache_engine == 'redis':\n            redis_host = config.get('redis_host')\n            redis_port = int(config.get('redis_port', 6379))\n            self.caching = self.get_redis_connection(redis_host, redis_port)\n        elif self.cache_engine == 'sqlite':\n            if not have_sqlite:\n                raise RuntimeError('No sqlite available: stackoverflow.com/q/44058239')\n            self.caching = LocalSqlite(config.get('ldap_cache_file', '/var/tmp/ldap.cache'), logger)\n\n    def get_redis_connection(self, redis_host, redis_port):\n        return Redis(redis_host=redis_host, redis_port=redis_port, db=0)\n\n    def get_connection(self, ldap_uri, ldap_bind_user, ldap_bind_password):\n        # note, if ldap_bind_user and ldap_bind_password are None\n        # an anonymous bind will be attempted.\n        try:\n            return Connection(\n                ldap_uri, user=ldap_bind_user, password=ldap_bind_password,\n                auto_bind=True,\n                receive_timeout=30,\n                auto_referrals=False,\n            )\n        except LDAPSocketOpenError:\n            self.log.error('Not able to establish a connection with LDAP.')\n\n    def search_ldap(self, base_dn, ldap_filter, attributes):\n        self.connection.search(base_dn, ldap_filter, attributes=self.attributes)\n        if len(self.connection.entries) == 0:\n            self.log.warning(\"user not found. base_dn: %s filter: %s\", base_dn, ldap_filter)\n            return {}\n        if len(self.connection.entries) > 1:\n            self.log.warning(\"too many results for search %s\", ldap_filter)\n            return {}\n        return self.connection.entries[0]\n\n    def get_email_to_addrs_from_uid(self, uid, manager=False):\n        to_addrs = []\n        uid_metadata = self.get_metadata_from_uid(uid)\n        uid_email = uid_metadata.get(self.email_key, None)\n        if uid_email:\n            to_addrs.append(uid_email)\n        if manager:\n            uid_manager_dn = uid_metadata.get(self.manager_attr, None)\n            uid_manager_email = None\n            if uid_manager_dn:\n                uid_manager = self.get_metadata_from_dn(uid_manager_dn)\n                uid_manager_email = uid_manager.get('mail')\n            if uid_manager_email:\n                to_addrs.append(uid_manager_email)\n        return to_addrs\n\n    # eg, dn = uid=bill_lumbergh,cn=users,dc=initech,dc=com\n    def get_metadata_from_dn(self, user_dn):\n        if self.cache_engine:\n            cache_result = self.caching.get(user_dn)\n            if cache_result:\n                cache_msg = 'Got ldap metadata from local cache for: %s' % user_dn\n                self.log.debug(cache_msg)\n                return cache_result\n        ldap_filter = '(%s=*)' % self.uid_key\n        ldap_results = self.search_ldap(user_dn, ldap_filter, attributes=self.attributes)\n        if ldap_results:\n            ldap_user_metadata = self.get_dict_from_ldap_object(self.connection.entries[0])\n        else:\n            self.caching.set(user_dn, {})\n            return {}\n        if self.cache_engine:\n            self.log.debug('Writing user: %s metadata to cache engine.' % user_dn)\n            self.caching.set(user_dn, ldap_user_metadata)\n            self.caching.set(ldap_user_metadata[self.uid_key], ldap_user_metadata)\n        return ldap_user_metadata\n\n    def get_dict_from_ldap_object(self, ldap_user_object):\n        ldap_user_metadata = {attr.key: attr.value for attr in ldap_user_object}\n        ldap_user_metadata['dn'] = ldap_user_object.entry_dn\n\n        email_key = ldap_user_metadata.get(self.email_key, None)\n        uid_key = ldap_user_metadata.get(self.uid_key, None)\n\n        if not email_key or not uid_key:\n            return {}\n        else:\n            ldap_user_metadata['self.email_key'] = email_key.lower()\n            ldap_user_metadata['self.uid_key'] = uid_key.lower()\n\n        return ldap_user_metadata\n\n    # eg, uid = bill_lumbergh\n    def get_metadata_from_uid(self, uid):\n        uid = uid.lower()\n        if self.uid_regex:\n            # for example if you set ldap_uid_regex in your mailer.yml to \"^[0-9]{6}$\" then it\n            # would only query LDAP if your string length is 6 characters long and only digits.\n            # re.search(\"^[0-9]{6}$\", \"123456\")\n            # Out[41]: <_sre.SRE_Match at 0x1109ab440>\n            # re.search(\"^[0-9]{6}$\", \"1234567\") returns None, or \"12345a' also returns None\n            if not re.search(self.uid_regex, uid):\n                regex_msg = 'uid does not match regex: %s %s' % (self.uid_regex, uid)\n                self.log.debug(regex_msg)\n                return {}\n        if self.cache_engine:\n            cache_result = self.caching.get(uid)\n            if cache_result or cache_result == {}:\n                cache_msg = 'Got ldap metadata from local cache for: %s' % uid\n                self.log.debug(cache_msg)\n                return cache_result\n        ldap_filter = '(%s=%s)' % (self.uid_key, uid)\n        ldap_results = self.search_ldap(self.base_dn, ldap_filter, attributes=self.attributes)\n        if ldap_results:\n            ldap_user_metadata = self.get_dict_from_ldap_object(self.connection.entries[0])\n            if self.cache_engine:\n                self.log.debug('Writing user: %s metadata to cache engine.' % uid)\n                if ldap_user_metadata.get('dn'):\n                    self.caching.set(ldap_user_metadata['dn'], ldap_user_metadata)\n                    self.caching.set(uid, ldap_user_metadata)\n                else:\n                    self.caching.set(uid, {})\n        else:\n            if self.cache_engine:\n                self.caching.set(uid, {})\n            return {}\n        return ldap_user_metadata\n\n\n# Use sqlite as a local cache for folks not running the mailer in lambda, avoids extra daemons\n# as dependencies. This normalizes the methods to set/get functions, so you can interchangeable\n# decide which caching system to use, a local file, or memcache, redis, etc\n# If you don't want a redis dependency and aren't running the mailer in lambda this works well\nclass LocalSqlite(object):\n    def __init__(self, local_filename, logger):\n        self.log = logger\n        self.sqlite = sqlite3.connect(local_filename)\n        self.sqlite.execute('''CREATE TABLE IF NOT EXISTS ldap_cache(key text, value text)''')\n\n    def get(self, key):\n        sqlite_result = self.sqlite.execute(\"select * FROM ldap_cache WHERE key=?\", (key,))\n        result = sqlite_result.fetchall()\n        if len(result) != 1:\n            error_msg = 'Did not get 1 result from sqlite, something went wrong with key: %s' % key\n            self.log.error(error_msg)\n            return None\n        return json.loads(result[0][1])\n\n    def set(self, key, value):\n        # note, the ? marks are required to ensure escaping into the database.\n        self.sqlite.execute(\"INSERT INTO ldap_cache VALUES (?, ?)\", (key, json.dumps(value)))\n        self.sqlite.commit()\n\n\n# redis can't write complex python objects like dictionaries as values (the way memcache can)\n# so we turn our dict into a json string when setting, and json.loads when getting\nclass Redis(object):\n    def __init__(self, redis_host=None, redis_port=6379, db=0):\n        self.connection = redis.StrictRedis(host=redis_host, port=redis_port, db=db)\n\n    def get(self, key):\n        cache_value = self.connection.get(key)\n        if cache_value:\n            return json.loads(cache_value)\n\n    def set(self, key, value):\n        return self.connection.set(key, json.dumps(value))\n", "path": "tools/c7n_mailer/c7n_mailer/ldap_lookup.py"}], "after_files": [{"content": "# Copyright 2017 Capital One Services, LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\nimport json\n\nimport re\nimport redis\n\nimport sqlite3\nfrom ldap3 import Connection\nfrom ldap3.core.exceptions import LDAPSocketOpenError\n\n\nclass LdapLookup(object):\n\n    def __init__(self, config, logger):\n        self.log = logger\n        self.connection = self.get_connection(\n            config.get('ldap_uri'),\n            config.get('ldap_bind_user', None),\n            config.get('ldap_bind_password', None)\n        )\n        self.base_dn = config.get('ldap_bind_dn')\n        self.email_key = config.get('ldap_email_key', 'mail')\n        self.manager_attr = config.get('ldap_manager_attribute', 'manager')\n        self.uid_key = config.get('ldap_uid_attribute', 'sAMAccountName')\n        self.attributes = ['displayName', self.uid_key, self.email_key, self.manager_attr]\n        self.uid_regex = config.get('ldap_uid_regex', None)\n        self.cache_engine = config.get('cache_engine', None)\n        if self.cache_engine == 'redis':\n            redis_host = config.get('redis_host')\n            redis_port = int(config.get('redis_port', 6379))\n            self.caching = self.get_redis_connection(redis_host, redis_port)\n        elif self.cache_engine == 'sqlite':\n            self.caching = LocalSqlite(config.get('ldap_cache_file', '/var/tmp/ldap.cache'), logger)\n\n    def get_redis_connection(self, redis_host, redis_port):\n        return Redis(redis_host=redis_host, redis_port=redis_port, db=0)\n\n    def get_connection(self, ldap_uri, ldap_bind_user, ldap_bind_password):\n        # note, if ldap_bind_user and ldap_bind_password are None\n        # an anonymous bind will be attempted.\n        try:\n            return Connection(\n                ldap_uri, user=ldap_bind_user, password=ldap_bind_password,\n                auto_bind=True,\n                receive_timeout=30,\n                auto_referrals=False,\n            )\n        except LDAPSocketOpenError:\n            self.log.error('Not able to establish a connection with LDAP.')\n\n    def search_ldap(self, base_dn, ldap_filter, attributes):\n        self.connection.search(base_dn, ldap_filter, attributes=self.attributes)\n        if len(self.connection.entries) == 0:\n            self.log.warning(\"user not found. base_dn: %s filter: %s\", base_dn, ldap_filter)\n            return {}\n        if len(self.connection.entries) > 1:\n            self.log.warning(\"too many results for search %s\", ldap_filter)\n            return {}\n        return self.connection.entries[0]\n\n    def get_email_to_addrs_from_uid(self, uid, manager=False):\n        to_addrs = []\n        uid_metadata = self.get_metadata_from_uid(uid)\n        uid_email = uid_metadata.get(self.email_key, None)\n        if uid_email:\n            to_addrs.append(uid_email)\n        if manager:\n            uid_manager_dn = uid_metadata.get(self.manager_attr, None)\n            uid_manager_email = None\n            if uid_manager_dn:\n                uid_manager = self.get_metadata_from_dn(uid_manager_dn)\n                uid_manager_email = uid_manager.get('mail')\n            if uid_manager_email:\n                to_addrs.append(uid_manager_email)\n        return to_addrs\n\n    # eg, dn = uid=bill_lumbergh,cn=users,dc=initech,dc=com\n    def get_metadata_from_dn(self, user_dn):\n        if self.cache_engine:\n            cache_result = self.caching.get(user_dn)\n            if cache_result:\n                cache_msg = 'Got ldap metadata from local cache for: %s' % user_dn\n                self.log.debug(cache_msg)\n                return cache_result\n        ldap_filter = '(%s=*)' % self.uid_key\n        ldap_results = self.search_ldap(user_dn, ldap_filter, attributes=self.attributes)\n        if ldap_results:\n            ldap_user_metadata = self.get_dict_from_ldap_object(self.connection.entries[0])\n        else:\n            self.caching.set(user_dn, {})\n            return {}\n        if self.cache_engine:\n            self.log.debug('Writing user: %s metadata to cache engine.' % user_dn)\n            self.caching.set(user_dn, ldap_user_metadata)\n            self.caching.set(ldap_user_metadata[self.uid_key], ldap_user_metadata)\n        return ldap_user_metadata\n\n    def get_dict_from_ldap_object(self, ldap_user_object):\n        ldap_user_metadata = {attr.key: attr.value for attr in ldap_user_object}\n        ldap_user_metadata['dn'] = ldap_user_object.entry_dn\n\n        email_key = ldap_user_metadata.get(self.email_key, None)\n        uid_key = ldap_user_metadata.get(self.uid_key, None)\n\n        if not email_key or not uid_key:\n            return {}\n        else:\n            ldap_user_metadata['self.email_key'] = email_key.lower()\n            ldap_user_metadata['self.uid_key'] = uid_key.lower()\n\n        return ldap_user_metadata\n\n    # eg, uid = bill_lumbergh\n    def get_metadata_from_uid(self, uid):\n        uid = uid.lower()\n        if self.uid_regex:\n            # for example if you set ldap_uid_regex in your mailer.yml to \"^[0-9]{6}$\" then it\n            # would only query LDAP if your string length is 6 characters long and only digits.\n            # re.search(\"^[0-9]{6}$\", \"123456\")\n            # Out[41]: <_sre.SRE_Match at 0x1109ab440>\n            # re.search(\"^[0-9]{6}$\", \"1234567\") returns None, or \"12345a' also returns None\n            if not re.search(self.uid_regex, uid):\n                regex_msg = 'uid does not match regex: %s %s' % (self.uid_regex, uid)\n                self.log.debug(regex_msg)\n                return {}\n        if self.cache_engine:\n            cache_result = self.caching.get(uid)\n            if cache_result or cache_result == {}:\n                cache_msg = 'Got ldap metadata from local cache for: %s' % uid\n                self.log.debug(cache_msg)\n                return cache_result\n        ldap_filter = '(%s=%s)' % (self.uid_key, uid)\n        ldap_results = self.search_ldap(self.base_dn, ldap_filter, attributes=self.attributes)\n        if ldap_results:\n            ldap_user_metadata = self.get_dict_from_ldap_object(self.connection.entries[0])\n            if self.cache_engine:\n                self.log.debug('Writing user: %s metadata to cache engine.' % uid)\n                if ldap_user_metadata.get('dn'):\n                    self.caching.set(ldap_user_metadata['dn'], ldap_user_metadata)\n                    self.caching.set(uid, ldap_user_metadata)\n                else:\n                    self.caching.set(uid, {})\n        else:\n            if self.cache_engine:\n                self.caching.set(uid, {})\n            return {}\n        return ldap_user_metadata\n\n\n# Use sqlite as a local cache for folks not running the mailer in lambda, avoids extra daemons\n# as dependencies. This normalizes the methods to set/get functions, so you can interchangeable\n# decide which caching system to use, a local file, or memcache, redis, etc\n# If you don't want a redis dependency and aren't running the mailer in lambda this works well\nclass LocalSqlite(object):\n    def __init__(self, local_filename, logger):\n        self.log = logger\n        self.sqlite = sqlite3.connect(local_filename)\n        self.sqlite.execute('''CREATE TABLE IF NOT EXISTS ldap_cache(key text, value text)''')\n\n    def get(self, key):\n        sqlite_result = self.sqlite.execute(\"select * FROM ldap_cache WHERE key=?\", (key,))\n        result = sqlite_result.fetchall()\n        if len(result) != 1:\n            error_msg = 'Did not get 1 result from sqlite, something went wrong with key: %s' % key\n            self.log.error(error_msg)\n            return None\n        return json.loads(result[0][1])\n\n    def set(self, key, value):\n        # note, the ? marks are required to ensure escaping into the database.\n        self.sqlite.execute(\"INSERT INTO ldap_cache VALUES (?, ?)\", (key, json.dumps(value)))\n        self.sqlite.commit()\n\n\n# redis can't write complex python objects like dictionaries as values (the way memcache can)\n# so we turn our dict into a json string when setting, and json.loads when getting\nclass Redis(object):\n    def __init__(self, redis_host=None, redis_port=6379, db=0):\n        self.connection = redis.StrictRedis(host=redis_host, port=redis_port, db=db)\n\n    def get(self, key):\n        cache_value = self.connection.get(key)\n        if cache_value:\n            return json.loads(cache_value)\n\n    def set(self, key, value):\n        return self.connection.set(key, json.dumps(value))\n", "path": "tools/c7n_mailer/c7n_mailer/ldap_lookup.py"}]}