# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Caches processed query results in memcache and datastore.

Memcache is not very reliable for the perf dashboard. Prometheus team explained
that memcache is LRU and shared between multiple applications, so their activity
may result in our data being evicted. To prevent this, we cache processed
query results in the data store. Using NDB, the values are also cached in
memcache if possible. This improves performance because doing a get()
for a key which has a single BlobProperty is much quicker than a complex query
over a large dataset.
(Background: http://g/prometheus-discuss/othVtufGIyM/wjAS5djyG8kJ)

When an item is cached, layered_cache does the following:
1) Namespaces the key based on whether datastore_hooks says the request is
internal_only.
2) Pickles the value (memcache does this internally), and adds a data store
entity with the key and a BlobProperty with the pickled value.

Retrieving values checks memcache via NDB first, and if datastore is used it
unpickles.

When an item is removed from the the cache, it is removed from both internal and
external caches, since removals are usually caused by large changes that affect
both caches.

Although this module contains ndb.Model classes, these are not intended
to be used directly by other modules.
"""
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import

import six.moves.cPickle as cPickle
import datetime
import logging

from google.appengine.api import datastore_errors
from google.appengine.runtime import apiproxy_errors
from google.appengine.ext import ndb

from dashboard.common import datastore_hooks
from dashboard.common import namespaced_stored_object
from dashboard.common import stored_object


class CachedPickledString(ndb.Model):
  value = ndb.BlobProperty()
  expire_time = ndb.DateTimeProperty()

  @classmethod
  def NamespacedKey(cls, key, namespace):
    return ndb.Key(cls.__name__,
                   namespaced_stored_object.NamespaceKey(key, namespace))

  @classmethod
  def GetExpiredKeys(cls):
    """Gets keys of expired entities.

    Returns:
      List of keys for items which are expired.
    """
    current_time = datetime.datetime.now()
    query = cls.query(cls.expire_time < current_time)
    query = query.filter(cls.expire_time != None)
    return query.fetch(keys_only=True)


def Get(key):
  """Gets the value from the datastore."""
  if key is None:
    return None
  namespaced_key = namespaced_stored_object.NamespaceKey(key)
  entity = ndb.Key('CachedPickledString',
                   namespaced_key).get(read_policy=ndb.EVENTUAL_CONSISTENCY)
  if entity:
    return cPickle.loads(entity.value)
  return stored_object.Get(key)


def GetExternal(key):
  """Gets the value from the datastore for the externally namespaced key."""
  if key is None:
    return None
  namespaced_key = namespaced_stored_object.NamespaceKey(
      key, datastore_hooks.EXTERNAL)
  entity = ndb.Key('CachedPickledString',
                   namespaced_key).get(read_policy=ndb.EVENTUAL_CONSISTENCY)
  if entity:
    return cPickle.loads(entity.value)
  return stored_object.Get(key)


def Set(key, value, days_to_keep=None, namespace=None):
  """Sets the value in the datastore.

  Args:
    key: The key name, which will be namespaced.
    value: The value to set.
    days_to_keep: Number of days to keep entity in datastore, default is None.
    Entity will not expire when this value is 0 or None.
    namespace: Optional namespace, otherwise namespace will be retrieved
        using datastore_hooks.GetNamespace().
  """
  # When number of days to keep is given, calculate expiration time for
  # the entity and store it in datastore.
  # Once the entity expires, it will be deleted from the datastore.
  expire_time = None
  if days_to_keep:
    expire_time = datetime.datetime.now() + datetime.timedelta(
        days=days_to_keep)
  namespaced_key = namespaced_stored_object.NamespaceKey(key, namespace)

  try:
    CachedPickledString(
        id=namespaced_key, value=cPickle.dumps(value),
        expire_time=expire_time).put()
  except datastore_errors.BadRequestError as e:
    logging.warning('BadRequestError for key %s: %s', key, e)
  except apiproxy_errors.RequestTooLargeError as e:
    stored_object.Set(key, value)


def SetExternal(key, value, days_to_keep=None):
  """Sets the value in the datastore for the externally namespaced key.

  Needed for things like /add_point that update internal/external data at the
  same time.

  Args:
    key: The key name, which will be namespaced as externally_visible.
    value: The value to set.
    days_to_keep: Number of days to keep entity in datastore, default is None.
        Entity will not expire when this value is 0 or None.
  """
  Set(key, value, days_to_keep, datastore_hooks.EXTERNAL)


@ndb.synctasklet
def Delete(key):
  """Clears the value from the datastore."""
  yield DeleteAsync(key)


@ndb.tasklet
def DeleteAsync(key):
  unnamespaced_future = stored_object.DeleteAsync(key)
  # See the comment in stored_object.DeleteAsync() about this get().
  entities = yield ndb.get_multi_async([
      CachedPickledString.NamespacedKey(key, datastore_hooks.INTERNAL),
      CachedPickledString.NamespacedKey(key, datastore_hooks.EXTERNAL),
  ])
  keys = [entity.key for entity in entities if entity]
  yield (unnamespaced_future, ndb.delete_multi_async(keys))


def DeleteAllExpiredEntities():
  """Deletes all expired entities from the datastore."""
  ndb.delete_multi(CachedPickledString.GetExpiredKeys())

[((49, 10, 49, 28), 'google.appengine.ext.ndb.BlobProperty', 'ndb.BlobProperty', ({}, {}), '()', False, 'from google.appengine.ext import ndb\n'), ((50, 16, 50, 38), 'google.appengine.ext.ndb.DateTimeProperty', 'ndb.DateTimeProperty', ({}, {}), '()', False, 'from google.appengine.ext import ndb\n'), ((74, 19, 74, 61), 'dashboard.common.namespaced_stored_object.NamespaceKey', 'namespaced_stored_object.NamespaceKey', ({(74, 57, 74, 60): 'key'}, {}), '(key)', False, 'from dashboard.common import namespaced_stored_object\n'), ((79, 9, 79, 31), 'dashboard.common.stored_object.Get', 'stored_object.Get', ({(79, 27, 79, 30): 'key'}, {}), '(key)', False, 'from dashboard.common import stored_object\n'), ((86, 19, 87, 36), 'dashboard.common.namespaced_stored_object.NamespaceKey', 'namespaced_stored_object.NamespaceKey', ({(87, 6, 87, 9): 'key', (87, 11, 87, 35): 'datastore_hooks.EXTERNAL'}, {}), '(key, datastore_hooks.EXTERNAL)', False, 'from dashboard.common import namespaced_stored_object\n'), ((92, 9, 92, 31), 'dashboard.common.stored_object.Get', 'stored_object.Get', ({(92, 27, 92, 30): 'key'}, {}), '(key)', False, 'from dashboard.common import stored_object\n'), ((113, 19, 113, 72), 'dashboard.common.namespaced_stored_object.NamespaceKey', 'namespaced_stored_object.NamespaceKey', ({(113, 57, 113, 60): 'key', (113, 62, 113, 71): 'namespace'}, {}), '(key, namespace)', False, 'from dashboard.common import namespaced_stored_object\n'), ((148, 24, 148, 54), 'dashboard.common.stored_object.DeleteAsync', 'stored_object.DeleteAsync', ({(148, 50, 148, 53): 'key'}, {}), '(key)', False, 'from dashboard.common import stored_object\n'), ((64, 19, 64, 42), 'datetime.datetime.now', 'datetime.datetime.now', ({}, {}), '()', False, 'import datetime\n'), ((78, 11, 78, 38), 'six.moves.cPickle.loads', 'cPickle.loads', ({(78, 25, 78, 37): 'entity.value'}, {}), '(entity.value)', True, 'import six.moves.cPickle as cPickle\n'), ((91, 11, 91, 38), 'six.moves.cPickle.loads', 'cPickle.loads', ({(91, 25, 91, 37): 'entity.value'}, {}), '(entity.value)', True, 'import six.moves.cPickle as cPickle\n'), ((55, 19, 55, 72), 'dashboard.common.namespaced_stored_object.NamespaceKey', 'namespaced_stored_object.NamespaceKey', ({(55, 57, 55, 60): 'key', (55, 62, 55, 71): 'namespace'}, {}), '(key, namespace)', False, 'from dashboard.common import namespaced_stored_object\n'), ((75, 11, 76, 34), 'google.appengine.ext.ndb.Key', 'ndb.Key', ({(75, 19, 75, 40): '"""CachedPickledString"""', (76, 19, 76, 33): 'namespaced_key'}, {}), "('CachedPickledString', namespaced_key)", False, 'from google.appengine.ext import ndb\n'), ((88, 11, 89, 34), 'google.appengine.ext.ndb.Key', 'ndb.Key', ({(88, 19, 88, 40): '"""CachedPickledString"""', (89, 19, 89, 33): 'namespaced_key'}, {}), "('CachedPickledString', namespaced_key)", False, 'from google.appengine.ext import ndb\n'), ((111, 18, 111, 41), 'datetime.datetime.now', 'datetime.datetime.now', ({}, {}), '()', False, 'import datetime\n'), ((111, 44, 112, 26), 'datetime.timedelta', 'datetime.timedelta', (), '', False, 'import datetime\n'), ((120, 4, 120, 61), 'logging.warning', 'logging.warning', ({(120, 20, 120, 52): '"""BadRequestError for key %s: %s"""', (120, 54, 120, 57): 'key', (120, 59, 120, 60): 'e'}, {}), "('BadRequestError for key %s: %s', key, e)", False, 'import logging\n'), ((122, 4, 122, 33), 'dashboard.common.stored_object.Set', 'stored_object.Set', ({(122, 22, 122, 25): 'key', (122, 27, 122, 32): 'value'}, {}), '(key, value)', False, 'from dashboard.common import stored_object\n'), ((155, 30, 155, 58), 'google.appengine.ext.ndb.delete_multi_async', 'ndb.delete_multi_async', ({(155, 53, 155, 57): 'keys'}, {}), '(keys)', False, 'from google.appengine.ext import ndb\n'), ((117, 33, 117, 53), 'six.moves.cPickle.dumps', 'cPickle.dumps', ({(117, 47, 117, 52): 'value'}, {}), '(value)', True, 'import six.moves.cPickle as cPickle\n')]