'Make the bigtable RPC with retries.
Args:
callback: a function to call, typically a bigtable row mutation.commit
mode: A string to indicate what kind of db operation this is "read",
"write", "delete".
*args: args to pass to the callback
**kwargs: keyword args to pass to the callback
Returns:
Callback result.
Raises:
AccessError: if we hit our RPC retry limit, or the RPC error isn\'t
retryable.
ValueError: if you pass an unknown operation in mode.'

def CallWithRetry(self, callback, mode, *args, **kwargs):

    if (mode not in set(['read', 'write', 'delete'])):
        raise ValueError("Mode    must    be    'read',    'write',    'delete'")
    retry_count = 0
    sleep_interval = config.CONFIG['CloudBigtable.retry_interval']
    while (retry_count < config.CONFIG['CloudBigtable.retry_max_attempts']):
        try:
            response = callback(*args, **kwargs)
            return response
        except (face.ExpirationError, face.AbortionError) as e:
            last_error = e
            last_traceback = traceback.format_exc()
            print ('Retrying    %s' % last_traceback)
        time.sleep(sleep_interval.seconds)
        logging.info('Retrying    callback:    %s', callback)
        retry_count += 1
        stats.STATS.IncrementCounter(('grr_cloud_bigtable_%s_retries' % mode))
        sleep_interval *= config.CONFIG['CloudBigtable.retry_multiplier']
    stats.STATS.IncrementCounter(('grr_cloud_bigtable_%s_failures' % mode))
    logging.error('Gave    up    on    %s    %s    after    %s    retries.    %s', mode, callback, retry_count, last_traceback)
    raise AccessError(('Giving    up    on    %s    callback:%s    after    %s    retries.    Last    error:    %s.' % (mode, callback, retry_count, last_error)))
