We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Add limit for cache size (especially memory)
original: https://github.com/mozilla-services/cliquet/issues/406

That's an improvement so we control the cache size. Otherwise it's a free wheel that eventually crashes the server. 

Which makes me think: if I do a load test on stage today, is the kinto process growing indefinitely in memory ?


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `kinto/core/__init__.py`
Content:
```
1 """Main entry point
2 """
3 import pkg_resources
4 
5 from cornice import Service as CorniceService
6 from pyramid.settings import aslist
7 
8 from kinto.core import errors
9 from kinto.core import events
10 from kinto.core.initialization import (  # NOQA
11     initialize, install_middlewares,
12     load_default_settings)
13 from kinto.core.utils import (
14     follow_subrequest, current_service, current_resource_name,
15     prefixed_userid, prefixed_principals)
16 from kinto.core.logs import logger
17 
18 
19 # Module version, as defined in PEP-0396.
20 __version__ = pkg_resources.get_distribution('kinto').version  # FIXME?
21 
22 
23 DEFAULT_SETTINGS = {
24     'backoff': None,
25     'batch_max_requests': 25,
26     'cache_backend': '',
27     'cache_url': '',
28     'cache_pool_size': 25,
29     'cache_prefix': '',
30     'cors_origins': '*',
31     'cors_max_age_seconds': 3600,
32     'eos': None,
33     'eos_message': None,
34     'eos_url': None,
35     'error_info_link': 'https://github.com/Kinto/kinto/issues/',
36     'http_host': None,
37     'http_scheme': None,
38     'id_generator': 'kinto.core.storage.generators.UUID4',
39     'includes': '',
40     'initialization_sequence': (
41         'kinto.core.initialization.setup_request_bound_data',
42         'kinto.core.initialization.setup_json_serializer',
43         'kinto.core.initialization.setup_logging',
44         'kinto.core.initialization.setup_storage',
45         'kinto.core.initialization.setup_permission',
46         'kinto.core.initialization.setup_cache',
47         'kinto.core.initialization.setup_requests_scheme',
48         'kinto.core.initialization.setup_version_redirection',
49         'kinto.core.initialization.setup_deprecation',
50         'kinto.core.initialization.setup_authentication',
51         'kinto.core.initialization.setup_backoff',
52         'kinto.core.initialization.setup_statsd',
53         'kinto.core.initialization.setup_listeners',
54         'kinto.core.events.setup_transaction_hook',
55     ),
56     'event_listeners': '',
57     'heartbeat_timeout_seconds': 10,
58     'logging_renderer': 'kinto.core.logs.ClassicLogRenderer',
59     'newrelic_config': None,
60     'newrelic_env': 'dev',
61     'paginate_by': None,
62     'permission_backend': '',
63     'permission_url': '',
64     'permission_pool_size': 25,
65     'profiler_dir': '/tmp',
66     'profiler_enabled': False,
67     'project_docs': '',
68     'project_name': '',
69     'project_version': '',
70     'readonly': False,
71     'retry_after_seconds': 30,
72     'statsd_backend': 'kinto.core.statsd',
73     'statsd_prefix': 'kinto.core',
74     'statsd_url': None,
75     'storage_backend': '',
76     'storage_url': '',
77     'storage_max_fetch_size': 10000,
78     'storage_pool_size': 25,
79     'tm.annotate_user': False,  # Do annotate transactions with the user-id.
80     'transaction_per_request': True,
81     'userid_hmac_secret': '',
82     'version_json_path': 'version.json',
83     'version_prefix_redirect_enabled': True,
84     'trailing_slash_redirect_enabled': True,
85     'multiauth.groupfinder': 'kinto.core.authorization.groupfinder',
86     'multiauth.policies': 'basicauth',
87     'multiauth.policy.basicauth.use': ('kinto.core.authentication.'
88                                        'BasicAuthAuthenticationPolicy'),
89     'multiauth.authorization_policy': ('kinto.core.authorization.'
90                                        'AuthorizationPolicy')
91 }
92 
93 
94 class Service(CorniceService):
95     """Subclass of the default cornice service.
96 
97     This is useful in order to attach specific behaviours without monkey
98     patching the default cornice service (which would impact other uses of it)
99     """
100     default_cors_headers = ('Backoff', 'Retry-After', 'Alert',
101                             'Content-Length')
102 
103     def error_handler(self, request):
104         return errors.json_error_handler(request)
105 
106     @classmethod
107     def init_from_settings(cls, settings):
108         cls.cors_origins = tuple(aslist(settings['cors_origins']))
109         cors_max_age = settings['cors_max_age_seconds']
110         cls.cors_max_age = int(cors_max_age) if cors_max_age else None
111 
112 
113 def includeme(config):
114     settings = config.get_settings()
115 
116     # Heartbeat registry.
117     config.registry.heartbeats = {}
118 
119     # Public settings registry.
120     config.registry.public_settings = {'batch_max_requests', 'readonly'}
121 
122     # Directive to declare arbitrary API capabilities.
123     def add_api_capability(config, identifier, description="", url="", **kw):
124         existing = config.registry.api_capabilities.get(identifier)
125         if existing:
126             error_msg = "The '%s' API capability was already registered (%s)."
127             raise ValueError(error_msg % (identifier, existing))
128 
129         capability = dict(description=description, url=url, **kw)
130         config.registry.api_capabilities[identifier] = capability
131 
132     config.add_directive('add_api_capability', add_api_capability)
133     config.registry.api_capabilities = {}
134 
135     # Resource events helpers.
136     config.add_request_method(events.get_resource_events,
137                               name='get_resource_events')
138     config.add_request_method(events.notify_resource_event,
139                               name='notify_resource_event')
140 
141     # Setup cornice.
142     config.include("cornice")
143 
144     # Per-request transaction.
145     config.include("pyramid_tm")
146 
147     # Add CORS settings to the base kinto.core Service class.
148     Service.init_from_settings(settings)
149 
150     # Setup components.
151     for step in aslist(settings['initialization_sequence']):
152         step_func = config.maybe_dotted(step)
153         step_func(config)
154 
155     # Custom helpers.
156     config.add_request_method(follow_subrequest)
157     config.add_request_method(prefixed_userid, property=True)
158     config.add_request_method(prefixed_principals, reify=True)
159     config.add_request_method(lambda r: {
160         'id': r.prefixed_userid,
161         'principals': r.prefixed_principals},
162         name='get_user_info')
163     config.add_request_method(current_resource_name, reify=True)
164     config.add_request_method(current_service, reify=True)
165     config.commit()
166 
167     # Include plugins after init, unlike pyramid includes.
168     includes = aslist(settings['includes'])
169     for app in includes:
170         config.include(app)
171 
172     # # Show settings to output.
173     # for key, value in settings.items():
174     #     logger.info('Using %s = %s' % (key, value))
175 
176     # Scan views.
177     config.scan("kinto.core.views")
178 
179     # Give sign of life.
180     msg = "Running %(project_name)s %(project_version)s."
181     logger.info(msg % settings)
182
```

Path: `kinto/core/cache/memory.py`
Content:
```
1 from kinto.core.cache import CacheBase
2 from kinto.core.utils import msec_time, synchronized
3 
4 
5 class Cache(CacheBase):
6     """Cache backend implementation in local process memory.
7 
8     Enable in configuration::
9 
10         kinto.cache_backend = kinto.core.cache.memory
11 
12     :noindex:
13     """
14 
15     def __init__(self, *args, **kwargs):
16         super(Cache, self).__init__(*args, **kwargs)
17         self.flush()
18 
19     def initialize_schema(self, dry_run=False):
20         # Nothing to do.
21         pass
22 
23     def flush(self):
24         self._ttl = {}
25         self._store = {}
26 
27     @synchronized
28     def ttl(self, key):
29         ttl = self._ttl.get(self.prefix + key)
30         if ttl is not None:
31             return (ttl - msec_time()) / 1000.0
32         return -1
33 
34     @synchronized
35     def expire(self, key, ttl):
36         self._ttl[self.prefix + key] = msec_time() + int(ttl * 1000.0)
37 
38     @synchronized
39     def set(self, key, value, ttl=None):
40         if ttl is not None:
41             self.expire(key, ttl)
42         self._store[self.prefix + key] = value
43 
44     @synchronized
45     def get(self, key):
46         current = msec_time()
47         expired = [k for k, v in self._ttl.items() if current >= v]
48         for expired_item_key in expired:
49             self.delete(expired_item_key[len(self.prefix):])
50         return self._store.get(self.prefix + key)
51 
52     @synchronized
53     def delete(self, key):
54         key = self.prefix + key
55         self._ttl.pop(key, None)
56         self._store.pop(key, None)
57 
58 
59 def load_from_config(config):
60     settings = config.get_settings()
61     return Cache(cache_prefix=settings['cache_prefix'])
62
```

Path: `kinto/core/cache/__init__.py`
Content:
```
1 import random
2 
3 from kinto.core.logs import logger
4 
5 
6 _HEARTBEAT_DELETE_RATE = 0.5
7 _HEARTBEAT_KEY = '__heartbeat__'
8 _HEARTBEAT_TTL_SECONDS = 3600
9 
10 
11 class CacheBase(object):
12 
13     def __init__(self, *args, **kwargs):
14         self.prefix = kwargs['cache_prefix']
15         pass
16 
17     def initialize_schema(self, dry_run=False):
18         """Create every necessary objects (like tables or indices) in the
19         backend.
20 
21         This is executed when the ``kinto migrate`` command is run.
22 
23         :param bool dry_run: simulate instead of executing the operations.
24         """
25         raise NotImplementedError
26 
27     def flush(self):
28         """Delete every values."""
29         raise NotImplementedError
30 
31     def ttl(self, key):
32         """Obtain the expiration value of the specified `key`.
33 
34         :param str key: key
35         :returns: number of seconds or negative if no TTL.
36         :rtype: float
37         """
38         raise NotImplementedError
39 
40     def expire(self, key, ttl):
41         """Set the expiration value `ttl` for the specified `key`.
42 
43         :param str key: key
44         :param float ttl: number of seconds
45         """
46         raise NotImplementedError
47 
48     def set(self, key, value, ttl=None):
49         """Store a value with the specified `key`. If `ttl` is provided,
50         set an expiration value.
51 
52         :param str key: key
53         :param str value: value to store
54         :param float ttl: expire after number of seconds
55         """
56         raise NotImplementedError
57 
58     def get(self, key):
59         """Obtain the value of the specified `key`.
60 
61         :param str key: key
62         :returns: the stored value or None if missing.
63         :rtype: str
64         """
65         raise NotImplementedError
66 
67     def delete(self, key):
68         """Delete the value of the specified `key`.
69 
70         :param str key: key
71         """
72         raise NotImplementedError
73 
74 
75 def heartbeat(backend):
76     def ping(request):
77         """Test that cache backend is operationnal.
78 
79         :param request: current request object
80         :type request: :class:`~pyramid:pyramid.request.Request`
81         :returns: ``True`` is everything is ok, ``False`` otherwise.
82         :rtype: bool
83         """
84         # No specific case for readonly mode because the cache should
85         # continue to work in that mode.
86         try:
87             if random.random() < _HEARTBEAT_DELETE_RATE:
88                 backend.delete(_HEARTBEAT_KEY)
89             else:
90                 backend.set(_HEARTBEAT_KEY, 'alive', _HEARTBEAT_TTL_SECONDS)
91             return True
92         except:
93             logger.exception("Heartbeat Failure")
94             return False
95 
96     return ping
97
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/kinto/core/__init__.py b/kinto/core/__init__.py
--- a/kinto/core/__init__.py
+++ b/kinto/core/__init__.py
@@ -27,6 +27,7 @@
     'cache_url': '',
     'cache_pool_size': 25,
     'cache_prefix': '',
+    'cache_max_size_bytes': 524288,
     'cors_origins': '*',
     'cors_max_age_seconds': 3600,
     'eos': None,
diff --git a/kinto/core/cache/__init__.py b/kinto/core/cache/__init__.py
--- a/kinto/core/cache/__init__.py
+++ b/kinto/core/cache/__init__.py
@@ -12,7 +12,7 @@
 
     def __init__(self, *args, **kwargs):
         self.prefix = kwargs['cache_prefix']
-        pass
+        self.max_size_bytes = kwargs.get('cache_max_size_bytes')
 
     def initialize_schema(self, dry_run=False):
         """Create every necessary objects (like tables or indices) in the
diff --git a/kinto/core/cache/memory.py b/kinto/core/cache/memory.py
--- a/kinto/core/cache/memory.py
+++ b/kinto/core/cache/memory.py
@@ -21,8 +21,25 @@
         pass
 
     def flush(self):
+        self._created_at = {}
         self._ttl = {}
         self._store = {}
+        self._quota = 0
+
+    def _clean_expired(self):
+        current = msec_time()
+        expired = [k for k, v in self._ttl.items() if current >= v]
+        for expired_item_key in expired:
+            self.delete(expired_item_key[len(self.prefix):])
+
+    def _clean_oversized(self):
+        if self._quota < self.max_size_bytes:
+            return
+
+        for key, value in sorted(self._created_at.items(), key=lambda k: k[1]):
+            if self._quota < (self.max_size_bytes * 0.8):
+                break
+            self.delete(key[len(self.prefix):])
 
     @synchronized
     def ttl(self, key):
@@ -37,25 +54,36 @@
 
     @synchronized
     def set(self, key, value, ttl=None):
+        self._clean_expired()
+        self._clean_oversized()
         if ttl is not None:
             self.expire(key, ttl)
-        self._store[self.prefix + key] = value
+        item_key = self.prefix + key
+        self._store[item_key] = value
+        self._created_at[item_key] = msec_time()
+        self._quota += size_of(item_key, value)
 
     @synchronized
     def get(self, key):
-        current = msec_time()
-        expired = [k for k, v in self._ttl.items() if current >= v]
-        for expired_item_key in expired:
-            self.delete(expired_item_key[len(self.prefix):])
+        self._clean_expired()
         return self._store.get(self.prefix + key)
 
     @synchronized
     def delete(self, key):
         key = self.prefix + key
         self._ttl.pop(key, None)
-        self._store.pop(key, None)
+        self._created_at.pop(key, None)
+        value = self._store.pop(key, None)
+        self._quota -= size_of(key, value)
 
 
 def load_from_config(config):
     settings = config.get_settings()
-    return Cache(cache_prefix=settings['cache_prefix'])
+    return Cache(cache_prefix=settings['cache_prefix'],
+                 cache_max_size_bytes=settings['cache_max_size_bytes'])
+
+
+def size_of(key, value):
+    # Key used for ttl, created_at and store.
+    # Int size is 24 bytes one for ttl and one for created_at values
+    return len(key) * 3 + len(str(value)) + 24 * 2

{"golden_diff": "diff --git a/kinto/core/__init__.py b/kinto/core/__init__.py\n--- a/kinto/core/__init__.py\n+++ b/kinto/core/__init__.py\n@@ -27,6 +27,7 @@\n     'cache_url': '',\n     'cache_pool_size': 25,\n     'cache_prefix': '',\n+    'cache_max_size_bytes': 524288,\n     'cors_origins': '*',\n     'cors_max_age_seconds': 3600,\n     'eos': None,\ndiff --git a/kinto/core/cache/__init__.py b/kinto/core/cache/__init__.py\n--- a/kinto/core/cache/__init__.py\n+++ b/kinto/core/cache/__init__.py\n@@ -12,7 +12,7 @@\n \n     def __init__(self, *args, **kwargs):\n         self.prefix = kwargs['cache_prefix']\n-        pass\n+        self.max_size_bytes = kwargs.get('cache_max_size_bytes')\n \n     def initialize_schema(self, dry_run=False):\n         \"\"\"Create every necessary objects (like tables or indices) in the\ndiff --git a/kinto/core/cache/memory.py b/kinto/core/cache/memory.py\n--- a/kinto/core/cache/memory.py\n+++ b/kinto/core/cache/memory.py\n@@ -21,8 +21,25 @@\n         pass\n \n     def flush(self):\n+        self._created_at = {}\n         self._ttl = {}\n         self._store = {}\n+        self._quota = 0\n+\n+    def _clean_expired(self):\n+        current = msec_time()\n+        expired = [k for k, v in self._ttl.items() if current >= v]\n+        for expired_item_key in expired:\n+            self.delete(expired_item_key[len(self.prefix):])\n+\n+    def _clean_oversized(self):\n+        if self._quota < self.max_size_bytes:\n+            return\n+\n+        for key, value in sorted(self._created_at.items(), key=lambda k: k[1]):\n+            if self._quota < (self.max_size_bytes * 0.8):\n+                break\n+            self.delete(key[len(self.prefix):])\n \n     @synchronized\n     def ttl(self, key):\n@@ -37,25 +54,36 @@\n \n     @synchronized\n     def set(self, key, value, ttl=None):\n+        self._clean_expired()\n+        self._clean_oversized()\n         if ttl is not None:\n             self.expire(key, ttl)\n-        self._store[self.prefix + key] = value\n+        item_key = self.prefix + key\n+        self._store[item_key] = value\n+        self._created_at[item_key] = msec_time()\n+        self._quota += size_of(item_key, value)\n \n     @synchronized\n     def get(self, key):\n-        current = msec_time()\n-        expired = [k for k, v in self._ttl.items() if current >= v]\n-        for expired_item_key in expired:\n-            self.delete(expired_item_key[len(self.prefix):])\n+        self._clean_expired()\n         return self._store.get(self.prefix + key)\n \n     @synchronized\n     def delete(self, key):\n         key = self.prefix + key\n         self._ttl.pop(key, None)\n-        self._store.pop(key, None)\n+        self._created_at.pop(key, None)\n+        value = self._store.pop(key, None)\n+        self._quota -= size_of(key, value)\n \n \n def load_from_config(config):\n     settings = config.get_settings()\n-    return Cache(cache_prefix=settings['cache_prefix'])\n+    return Cache(cache_prefix=settings['cache_prefix'],\n+                 cache_max_size_bytes=settings['cache_max_size_bytes'])\n+\n+\n+def size_of(key, value):\n+    # Key used for ttl, created_at and store.\n+    # Int size is 24 bytes one for ttl and one for created_at values\n+    return len(key) * 3 + len(str(value)) + 24 * 2\n", "issue": "Add limit for cache size (especially memory)\noriginal: https://github.com/mozilla-services/cliquet/issues/406\n\nThat's an improvement so we control the cache size. Otherwise it's a free wheel that eventually crashes the server. \n\nWhich makes me think: if I do a load test on stage today, is the kinto process growing indefinitely in memory ?\n\n", "before_files": [{"content": "\"\"\"Main entry point\n\"\"\"\nimport pkg_resources\n\nfrom cornice import Service as CorniceService\nfrom pyramid.settings import aslist\n\nfrom kinto.core import errors\nfrom kinto.core import events\nfrom kinto.core.initialization import (  # NOQA\n    initialize, install_middlewares,\n    load_default_settings)\nfrom kinto.core.utils import (\n    follow_subrequest, current_service, current_resource_name,\n    prefixed_userid, prefixed_principals)\nfrom kinto.core.logs import logger\n\n\n# Module version, as defined in PEP-0396.\n__version__ = pkg_resources.get_distribution('kinto').version  # FIXME?\n\n\nDEFAULT_SETTINGS = {\n    'backoff': None,\n    'batch_max_requests': 25,\n    'cache_backend': '',\n    'cache_url': '',\n    'cache_pool_size': 25,\n    'cache_prefix': '',\n    'cors_origins': '*',\n    'cors_max_age_seconds': 3600,\n    'eos': None,\n    'eos_message': None,\n    'eos_url': None,\n    'error_info_link': 'https://github.com/Kinto/kinto/issues/',\n    'http_host': None,\n    'http_scheme': None,\n    'id_generator': 'kinto.core.storage.generators.UUID4',\n    'includes': '',\n    'initialization_sequence': (\n        'kinto.core.initialization.setup_request_bound_data',\n        'kinto.core.initialization.setup_json_serializer',\n        'kinto.core.initialization.setup_logging',\n        'kinto.core.initialization.setup_storage',\n        'kinto.core.initialization.setup_permission',\n        'kinto.core.initialization.setup_cache',\n        'kinto.core.initialization.setup_requests_scheme',\n        'kinto.core.initialization.setup_version_redirection',\n        'kinto.core.initialization.setup_deprecation',\n        'kinto.core.initialization.setup_authentication',\n        'kinto.core.initialization.setup_backoff',\n        'kinto.core.initialization.setup_statsd',\n        'kinto.core.initialization.setup_listeners',\n        'kinto.core.events.setup_transaction_hook',\n    ),\n    'event_listeners': '',\n    'heartbeat_timeout_seconds': 10,\n    'logging_renderer': 'kinto.core.logs.ClassicLogRenderer',\n    'newrelic_config': None,\n    'newrelic_env': 'dev',\n    'paginate_by': None,\n    'permission_backend': '',\n    'permission_url': '',\n    'permission_pool_size': 25,\n    'profiler_dir': '/tmp',\n    'profiler_enabled': False,\n    'project_docs': '',\n    'project_name': '',\n    'project_version': '',\n    'readonly': False,\n    'retry_after_seconds': 30,\n    'statsd_backend': 'kinto.core.statsd',\n    'statsd_prefix': 'kinto.core',\n    'statsd_url': None,\n    'storage_backend': '',\n    'storage_url': '',\n    'storage_max_fetch_size': 10000,\n    'storage_pool_size': 25,\n    'tm.annotate_user': False,  # Do annotate transactions with the user-id.\n    'transaction_per_request': True,\n    'userid_hmac_secret': '',\n    'version_json_path': 'version.json',\n    'version_prefix_redirect_enabled': True,\n    'trailing_slash_redirect_enabled': True,\n    'multiauth.groupfinder': 'kinto.core.authorization.groupfinder',\n    'multiauth.policies': 'basicauth',\n    'multiauth.policy.basicauth.use': ('kinto.core.authentication.'\n                                       'BasicAuthAuthenticationPolicy'),\n    'multiauth.authorization_policy': ('kinto.core.authorization.'\n                                       'AuthorizationPolicy')\n}\n\n\nclass Service(CorniceService):\n    \"\"\"Subclass of the default cornice service.\n\n    This is useful in order to attach specific behaviours without monkey\n    patching the default cornice service (which would impact other uses of it)\n    \"\"\"\n    default_cors_headers = ('Backoff', 'Retry-After', 'Alert',\n                            'Content-Length')\n\n    def error_handler(self, request):\n        return errors.json_error_handler(request)\n\n    @classmethod\n    def init_from_settings(cls, settings):\n        cls.cors_origins = tuple(aslist(settings['cors_origins']))\n        cors_max_age = settings['cors_max_age_seconds']\n        cls.cors_max_age = int(cors_max_age) if cors_max_age else None\n\n\ndef includeme(config):\n    settings = config.get_settings()\n\n    # Heartbeat registry.\n    config.registry.heartbeats = {}\n\n    # Public settings registry.\n    config.registry.public_settings = {'batch_max_requests', 'readonly'}\n\n    # Directive to declare arbitrary API capabilities.\n    def add_api_capability(config, identifier, description=\"\", url=\"\", **kw):\n        existing = config.registry.api_capabilities.get(identifier)\n        if existing:\n            error_msg = \"The '%s' API capability was already registered (%s).\"\n            raise ValueError(error_msg % (identifier, existing))\n\n        capability = dict(description=description, url=url, **kw)\n        config.registry.api_capabilities[identifier] = capability\n\n    config.add_directive('add_api_capability', add_api_capability)\n    config.registry.api_capabilities = {}\n\n    # Resource events helpers.\n    config.add_request_method(events.get_resource_events,\n                              name='get_resource_events')\n    config.add_request_method(events.notify_resource_event,\n                              name='notify_resource_event')\n\n    # Setup cornice.\n    config.include(\"cornice\")\n\n    # Per-request transaction.\n    config.include(\"pyramid_tm\")\n\n    # Add CORS settings to the base kinto.core Service class.\n    Service.init_from_settings(settings)\n\n    # Setup components.\n    for step in aslist(settings['initialization_sequence']):\n        step_func = config.maybe_dotted(step)\n        step_func(config)\n\n    # Custom helpers.\n    config.add_request_method(follow_subrequest)\n    config.add_request_method(prefixed_userid, property=True)\n    config.add_request_method(prefixed_principals, reify=True)\n    config.add_request_method(lambda r: {\n        'id': r.prefixed_userid,\n        'principals': r.prefixed_principals},\n        name='get_user_info')\n    config.add_request_method(current_resource_name, reify=True)\n    config.add_request_method(current_service, reify=True)\n    config.commit()\n\n    # Include plugins after init, unlike pyramid includes.\n    includes = aslist(settings['includes'])\n    for app in includes:\n        config.include(app)\n\n    # # Show settings to output.\n    # for key, value in settings.items():\n    #     logger.info('Using %s = %s' % (key, value))\n\n    # Scan views.\n    config.scan(\"kinto.core.views\")\n\n    # Give sign of life.\n    msg = \"Running %(project_name)s %(project_version)s.\"\n    logger.info(msg % settings)\n", "path": "kinto/core/__init__.py"}, {"content": "from kinto.core.cache import CacheBase\nfrom kinto.core.utils import msec_time, synchronized\n\n\nclass Cache(CacheBase):\n    \"\"\"Cache backend implementation in local process memory.\n\n    Enable in configuration::\n\n        kinto.cache_backend = kinto.core.cache.memory\n\n    :noindex:\n    \"\"\"\n\n    def __init__(self, *args, **kwargs):\n        super(Cache, self).__init__(*args, **kwargs)\n        self.flush()\n\n    def initialize_schema(self, dry_run=False):\n        # Nothing to do.\n        pass\n\n    def flush(self):\n        self._ttl = {}\n        self._store = {}\n\n    @synchronized\n    def ttl(self, key):\n        ttl = self._ttl.get(self.prefix + key)\n        if ttl is not None:\n            return (ttl - msec_time()) / 1000.0\n        return -1\n\n    @synchronized\n    def expire(self, key, ttl):\n        self._ttl[self.prefix + key] = msec_time() + int(ttl * 1000.0)\n\n    @synchronized\n    def set(self, key, value, ttl=None):\n        if ttl is not None:\n            self.expire(key, ttl)\n        self._store[self.prefix + key] = value\n\n    @synchronized\n    def get(self, key):\n        current = msec_time()\n        expired = [k for k, v in self._ttl.items() if current >= v]\n        for expired_item_key in expired:\n            self.delete(expired_item_key[len(self.prefix):])\n        return self._store.get(self.prefix + key)\n\n    @synchronized\n    def delete(self, key):\n        key = self.prefix + key\n        self._ttl.pop(key, None)\n        self._store.pop(key, None)\n\n\ndef load_from_config(config):\n    settings = config.get_settings()\n    return Cache(cache_prefix=settings['cache_prefix'])\n", "path": "kinto/core/cache/memory.py"}, {"content": "import random\n\nfrom kinto.core.logs import logger\n\n\n_HEARTBEAT_DELETE_RATE = 0.5\n_HEARTBEAT_KEY = '__heartbeat__'\n_HEARTBEAT_TTL_SECONDS = 3600\n\n\nclass CacheBase(object):\n\n    def __init__(self, *args, **kwargs):\n        self.prefix = kwargs['cache_prefix']\n        pass\n\n    def initialize_schema(self, dry_run=False):\n        \"\"\"Create every necessary objects (like tables or indices) in the\n        backend.\n\n        This is executed when the ``kinto migrate`` command is run.\n\n        :param bool dry_run: simulate instead of executing the operations.\n        \"\"\"\n        raise NotImplementedError\n\n    def flush(self):\n        \"\"\"Delete every values.\"\"\"\n        raise NotImplementedError\n\n    def ttl(self, key):\n        \"\"\"Obtain the expiration value of the specified `key`.\n\n        :param str key: key\n        :returns: number of seconds or negative if no TTL.\n        :rtype: float\n        \"\"\"\n        raise NotImplementedError\n\n    def expire(self, key, ttl):\n        \"\"\"Set the expiration value `ttl` for the specified `key`.\n\n        :param str key: key\n        :param float ttl: number of seconds\n        \"\"\"\n        raise NotImplementedError\n\n    def set(self, key, value, ttl=None):\n        \"\"\"Store a value with the specified `key`. If `ttl` is provided,\n        set an expiration value.\n\n        :param str key: key\n        :param str value: value to store\n        :param float ttl: expire after number of seconds\n        \"\"\"\n        raise NotImplementedError\n\n    def get(self, key):\n        \"\"\"Obtain the value of the specified `key`.\n\n        :param str key: key\n        :returns: the stored value or None if missing.\n        :rtype: str\n        \"\"\"\n        raise NotImplementedError\n\n    def delete(self, key):\n        \"\"\"Delete the value of the specified `key`.\n\n        :param str key: key\n        \"\"\"\n        raise NotImplementedError\n\n\ndef heartbeat(backend):\n    def ping(request):\n        \"\"\"Test that cache backend is operationnal.\n\n        :param request: current request object\n        :type request: :class:`~pyramid:pyramid.request.Request`\n        :returns: ``True`` is everything is ok, ``False`` otherwise.\n        :rtype: bool\n        \"\"\"\n        # No specific case for readonly mode because the cache should\n        # continue to work in that mode.\n        try:\n            if random.random() < _HEARTBEAT_DELETE_RATE:\n                backend.delete(_HEARTBEAT_KEY)\n            else:\n                backend.set(_HEARTBEAT_KEY, 'alive', _HEARTBEAT_TTL_SECONDS)\n            return True\n        except:\n            logger.exception(\"Heartbeat Failure\")\n            return False\n\n    return ping\n", "path": "kinto/core/cache/__init__.py"}], "after_files": [{"content": "\"\"\"Main entry point\n\"\"\"\nimport pkg_resources\n\nfrom cornice import Service as CorniceService\nfrom pyramid.settings import aslist\n\nfrom kinto.core import errors\nfrom kinto.core import events\nfrom kinto.core.initialization import (  # NOQA\n    initialize, install_middlewares,\n    load_default_settings)\nfrom kinto.core.utils import (\n    follow_subrequest, current_service, current_resource_name,\n    prefixed_userid, prefixed_principals)\nfrom kinto.core.logs import logger\n\n\n# Module version, as defined in PEP-0396.\n__version__ = pkg_resources.get_distribution('kinto').version  # FIXME?\n\n\nDEFAULT_SETTINGS = {\n    'backoff': None,\n    'batch_max_requests': 25,\n    'cache_backend': '',\n    'cache_url': '',\n    'cache_pool_size': 25,\n    'cache_prefix': '',\n    'cache_max_size_bytes': 524288,\n    'cors_origins': '*',\n    'cors_max_age_seconds': 3600,\n    'eos': None,\n    'eos_message': None,\n    'eos_url': None,\n    'error_info_link': 'https://github.com/Kinto/kinto/issues/',\n    'http_host': None,\n    'http_scheme': None,\n    'id_generator': 'kinto.core.storage.generators.UUID4',\n    'includes': '',\n    'initialization_sequence': (\n        'kinto.core.initialization.setup_request_bound_data',\n        'kinto.core.initialization.setup_json_serializer',\n        'kinto.core.initialization.setup_logging',\n        'kinto.core.initialization.setup_storage',\n        'kinto.core.initialization.setup_permission',\n        'kinto.core.initialization.setup_cache',\n        'kinto.core.initialization.setup_requests_scheme',\n        'kinto.core.initialization.setup_version_redirection',\n        'kinto.core.initialization.setup_deprecation',\n        'kinto.core.initialization.setup_authentication',\n        'kinto.core.initialization.setup_backoff',\n        'kinto.core.initialization.setup_statsd',\n        'kinto.core.initialization.setup_listeners',\n        'kinto.core.events.setup_transaction_hook',\n    ),\n    'event_listeners': '',\n    'heartbeat_timeout_seconds': 10,\n    'logging_renderer': 'kinto.core.logs.ClassicLogRenderer',\n    'newrelic_config': None,\n    'newrelic_env': 'dev',\n    'paginate_by': None,\n    'permission_backend': '',\n    'permission_url': '',\n    'permission_pool_size': 25,\n    'profiler_dir': '/tmp',\n    'profiler_enabled': False,\n    'project_docs': '',\n    'project_name': '',\n    'project_version': '',\n    'readonly': False,\n    'retry_after_seconds': 30,\n    'statsd_backend': 'kinto.core.statsd',\n    'statsd_prefix': 'kinto.core',\n    'statsd_url': None,\n    'storage_backend': '',\n    'storage_url': '',\n    'storage_max_fetch_size': 10000,\n    'storage_pool_size': 25,\n    'tm.annotate_user': False,  # Do annotate transactions with the user-id.\n    'transaction_per_request': True,\n    'userid_hmac_secret': '',\n    'version_json_path': 'version.json',\n    'version_prefix_redirect_enabled': True,\n    'trailing_slash_redirect_enabled': True,\n    'multiauth.groupfinder': 'kinto.core.authorization.groupfinder',\n    'multiauth.policies': 'basicauth',\n    'multiauth.policy.basicauth.use': ('kinto.core.authentication.'\n                                       'BasicAuthAuthenticationPolicy'),\n    'multiauth.authorization_policy': ('kinto.core.authorization.'\n                                       'AuthorizationPolicy')\n}\n\n\nclass Service(CorniceService):\n    \"\"\"Subclass of the default cornice service.\n\n    This is useful in order to attach specific behaviours without monkey\n    patching the default cornice service (which would impact other uses of it)\n    \"\"\"\n    default_cors_headers = ('Backoff', 'Retry-After', 'Alert',\n                            'Content-Length')\n\n    def error_handler(self, request):\n        return errors.json_error_handler(request)\n\n    @classmethod\n    def init_from_settings(cls, settings):\n        cls.cors_origins = tuple(aslist(settings['cors_origins']))\n        cors_max_age = settings['cors_max_age_seconds']\n        cls.cors_max_age = int(cors_max_age) if cors_max_age else None\n\n\ndef includeme(config):\n    settings = config.get_settings()\n\n    # Heartbeat registry.\n    config.registry.heartbeats = {}\n\n    # Public settings registry.\n    config.registry.public_settings = {'batch_max_requests', 'readonly'}\n\n    # Directive to declare arbitrary API capabilities.\n    def add_api_capability(config, identifier, description=\"\", url=\"\", **kw):\n        existing = config.registry.api_capabilities.get(identifier)\n        if existing:\n            error_msg = \"The '%s' API capability was already registered (%s).\"\n            raise ValueError(error_msg % (identifier, existing))\n\n        capability = dict(description=description, url=url, **kw)\n        config.registry.api_capabilities[identifier] = capability\n\n    config.add_directive('add_api_capability', add_api_capability)\n    config.registry.api_capabilities = {}\n\n    # Resource events helpers.\n    config.add_request_method(events.get_resource_events,\n                              name='get_resource_events')\n    config.add_request_method(events.notify_resource_event,\n                              name='notify_resource_event')\n\n    # Setup cornice.\n    config.include(\"cornice\")\n\n    # Per-request transaction.\n    config.include(\"pyramid_tm\")\n\n    # Add CORS settings to the base kinto.core Service class.\n    Service.init_from_settings(settings)\n\n    # Setup components.\n    for step in aslist(settings['initialization_sequence']):\n        step_func = config.maybe_dotted(step)\n        step_func(config)\n\n    # Custom helpers.\n    config.add_request_method(follow_subrequest)\n    config.add_request_method(prefixed_userid, property=True)\n    config.add_request_method(prefixed_principals, reify=True)\n    config.add_request_method(lambda r: {\n        'id': r.prefixed_userid,\n        'principals': r.prefixed_principals},\n        name='get_user_info')\n    config.add_request_method(current_resource_name, reify=True)\n    config.add_request_method(current_service, reify=True)\n    config.commit()\n\n    # Include plugins after init, unlike pyramid includes.\n    includes = aslist(settings['includes'])\n    for app in includes:\n        config.include(app)\n\n    # # Show settings to output.\n    # for key, value in settings.items():\n    #     logger.info('Using %s = %s' % (key, value))\n\n    # Scan views.\n    config.scan(\"kinto.core.views\")\n\n    # Give sign of life.\n    msg = \"Running %(project_name)s %(project_version)s.\"\n    logger.info(msg % settings)\n", "path": "kinto/core/__init__.py"}, {"content": "from kinto.core.cache import CacheBase\nfrom kinto.core.utils import msec_time, synchronized\n\n\nclass Cache(CacheBase):\n    \"\"\"Cache backend implementation in local process memory.\n\n    Enable in configuration::\n\n        kinto.cache_backend = kinto.core.cache.memory\n\n    :noindex:\n    \"\"\"\n\n    def __init__(self, *args, **kwargs):\n        super(Cache, self).__init__(*args, **kwargs)\n        self.flush()\n\n    def initialize_schema(self, dry_run=False):\n        # Nothing to do.\n        pass\n\n    def flush(self):\n        self._created_at = {}\n        self._ttl = {}\n        self._store = {}\n        self._quota = 0\n\n    def _clean_expired(self):\n        current = msec_time()\n        expired = [k for k, v in self._ttl.items() if current >= v]\n        for expired_item_key in expired:\n            self.delete(expired_item_key[len(self.prefix):])\n\n    def _clean_oversized(self):\n        if self._quota < self.max_size_bytes:\n            return\n\n        for key, value in sorted(self._created_at.items(), key=lambda k: k[1]):\n            if self._quota < (self.max_size_bytes * 0.8):\n                break\n            self.delete(key[len(self.prefix):])\n\n    @synchronized\n    def ttl(self, key):\n        ttl = self._ttl.get(self.prefix + key)\n        if ttl is not None:\n            return (ttl - msec_time()) / 1000.0\n        return -1\n\n    @synchronized\n    def expire(self, key, ttl):\n        self._ttl[self.prefix + key] = msec_time() + int(ttl * 1000.0)\n\n    @synchronized\n    def set(self, key, value, ttl=None):\n        self._clean_expired()\n        self._clean_oversized()\n        if ttl is not None:\n            self.expire(key, ttl)\n        item_key = self.prefix + key\n        self._store[item_key] = value\n        self._created_at[item_key] = msec_time()\n        self._quota += size_of(item_key, value)\n\n    @synchronized\n    def get(self, key):\n        self._clean_expired()\n        return self._store.get(self.prefix + key)\n\n    @synchronized\n    def delete(self, key):\n        key = self.prefix + key\n        self._ttl.pop(key, None)\n        self._created_at.pop(key, None)\n        value = self._store.pop(key, None)\n        self._quota -= size_of(key, value)\n\n\ndef load_from_config(config):\n    settings = config.get_settings()\n    return Cache(cache_prefix=settings['cache_prefix'],\n                 cache_max_size_bytes=settings['cache_max_size_bytes'])\n\n\ndef size_of(key, value):\n    # Key used for ttl, created_at and store.\n    # Int size is 24 bytes one for ttl and one for created_at values\n    return len(key) * 3 + len(str(value)) + 24 * 2\n", "path": "kinto/core/cache/memory.py"}, {"content": "import random\n\nfrom kinto.core.logs import logger\n\n\n_HEARTBEAT_DELETE_RATE = 0.5\n_HEARTBEAT_KEY = '__heartbeat__'\n_HEARTBEAT_TTL_SECONDS = 3600\n\n\nclass CacheBase(object):\n\n    def __init__(self, *args, **kwargs):\n        self.prefix = kwargs['cache_prefix']\n        self.max_size_bytes = kwargs.get('cache_max_size_bytes')\n\n    def initialize_schema(self, dry_run=False):\n        \"\"\"Create every necessary objects (like tables or indices) in the\n        backend.\n\n        This is executed when the ``kinto migrate`` command is run.\n\n        :param bool dry_run: simulate instead of executing the operations.\n        \"\"\"\n        raise NotImplementedError\n\n    def flush(self):\n        \"\"\"Delete every values.\"\"\"\n        raise NotImplementedError\n\n    def ttl(self, key):\n        \"\"\"Obtain the expiration value of the specified `key`.\n\n        :param str key: key\n        :returns: number of seconds or negative if no TTL.\n        :rtype: float\n        \"\"\"\n        raise NotImplementedError\n\n    def expire(self, key, ttl):\n        \"\"\"Set the expiration value `ttl` for the specified `key`.\n\n        :param str key: key\n        :param float ttl: number of seconds\n        \"\"\"\n        raise NotImplementedError\n\n    def set(self, key, value, ttl=None):\n        \"\"\"Store a value with the specified `key`. If `ttl` is provided,\n        set an expiration value.\n\n        :param str key: key\n        :param str value: value to store\n        :param float ttl: expire after number of seconds\n        \"\"\"\n        raise NotImplementedError\n\n    def get(self, key):\n        \"\"\"Obtain the value of the specified `key`.\n\n        :param str key: key\n        :returns: the stored value or None if missing.\n        :rtype: str\n        \"\"\"\n        raise NotImplementedError\n\n    def delete(self, key):\n        \"\"\"Delete the value of the specified `key`.\n\n        :param str key: key\n        \"\"\"\n        raise NotImplementedError\n\n\ndef heartbeat(backend):\n    def ping(request):\n        \"\"\"Test that cache backend is operationnal.\n\n        :param request: current request object\n        :type request: :class:`~pyramid:pyramid.request.Request`\n        :returns: ``True`` is everything is ok, ``False`` otherwise.\n        :rtype: bool\n        \"\"\"\n        # No specific case for readonly mode because the cache should\n        # continue to work in that mode.\n        try:\n            if random.random() < _HEARTBEAT_DELETE_RATE:\n                backend.delete(_HEARTBEAT_KEY)\n            else:\n                backend.set(_HEARTBEAT_KEY, 'alive', _HEARTBEAT_TTL_SECONDS)\n            return True\n        except:\n            logger.exception(\"Heartbeat Failure\")\n            return False\n\n    return ping\n", "path": "kinto/core/cache/__init__.py"}]}