text: # -*- coding: utf-8 -*- #
  # Copyright 2013 Google Inc. All Rights Reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #    http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  """Module with logging related functionality for calliope."""
  
  from __future__ import absolute_import
  from __future__ import division
  from __future__ import unicode_literals
  
  from collections import OrderedDict
  import contextlib
  import copy
  import datetime
  import errno
  import json
  import logging
  import os
  import sys
  import time
  
  from googlecloudsdk.core import properties
  from googlecloudsdk.core.console import console_attr
  from googlecloudsdk.core.console.style import parser as style_parser
  from googlecloudsdk.core.console.style import text
  from googlecloudsdk.core.util import files
  from googlecloudsdk.core.util import platforms
  from googlecloudsdk.core.util import times
  
  import six
  
  
  LOG_FILE_ENCODING = 'utf-8'
  
  DEFAULT_VERBOSITY = logging.WARNING
  DEFAULT_VERBOSITY_STRING = 'warning'
  DEFAULT_USER_OUTPUT_ENABLED = True
  
  _VERBOSITY_LEVELS = [
      ('debug', logging.DEBUG),
      ('info', logging.INFO),
      ('warning', logging.WARNING),
      ('error', logging.ERROR),
      ('critical', logging.CRITICAL),
      ('none', logging.CRITICAL + 10)]
  VALID_VERBOSITY_STRINGS = dict(_VERBOSITY_LEVELS)
  LOG_FILE_EXTENSION = '.log'
  # datastore upload and download creates temporary sql3 files in the log dir.
  _KNOWN_LOG_FILE_EXTENSIONS = [LOG_FILE_EXTENSION, '.sql3']
  
  # This is a regular expression pattern that matches the format of the date
  # marker that marks the beginning of a new log line in a log file. It can be
  # used in parsing log files.
  LOG_PREFIX_PATTERN = r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}'
  
  
  # These are the formats for the log directories and files.
  # For example, `logs/1970.01.01/12.00.00.000000.log`.
  DAY_DIR_FORMAT = '%Y.%m.%d'
  FILENAME_FORMAT = '%H.%M.%S.%f'
  
  # These are for Structured (JSON) Log Records
  STRUCTURED_RECORD_VERSION = '0.0.1'
  STRUCTURED_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%3f%Ez'
  
  # These fields are ordered by how they will appear in the log file
  # for consistency. All values are strings.
  # (Output Field, LogRecord Field, Description)
  STRUCTURED_RECORD_FIELDS = [
      ('version', 'version',
       'Semantic version of the message format. E.g. v0.0.1'),
      ('verbosity', 'levelname',
       'Logging Level: e.g. debug, info, warn, error, critical, exception.'),
      ('timestamp', 'asctime', 'UTC time event logged'),
      ('message', 'message', 'Log/Error message.'),
      ('error', 'error',
       'Actual exception or error raised, if message contains exception data.'),
  ]
  REQUIRED_STRUCTURED_RECORD_FIELDS = OrderedDict((x[:2] for x in
                                                   STRUCTURED_RECORD_FIELDS))
  
  
  class _NullHandler(logging.Handler, object):
    """A replication of python2.7's logging.NullHandler.
  
    We recreate this class here to ease python2.6 compatibility.
    """
  
    def handle(self, record):
      pass
  
    def emit(self, record):
      pass
  
    def createLock(self):
      self.lock = None
  
  
  class _UserOutputFilter(object):
    """A filter to turn on and off user output.
  
    This filter is used by the ConsoleWriter to determine if output messages
    should be printed or not.
    """
  
    def __init__(self, enabled):
      """Creates the filter.
  
      Args:
        enabled: bool, True to enable output, false to suppress.
      """
      self.enabled = enabled
  
  
  class _StreamWrapper(object):
    """A class to hold an output stream that we can manipulate."""
  
    def __init__(self, stream):
      """Creates the stream wrapper.
  
      Args:
        stream: The stream to hold on to.
      """
      self.stream = stream
  
  
  class _ConsoleWriter(object):
    """A class that wraps stdout or stderr so we can control how it gets logged.
  
    This class is a stripped down file-like object that provides the basic
    writing methods.  When you write to this stream, if it is enabled, it will be
    written to stdout.  All strings will also be logged at DEBUG level so they
    can be captured by the log file.
    """
  
    def __init__(self, logger, output_filter, stream_wrapper, always_flush=False):
      """Creates a new _ConsoleWriter wrapper.
  
      Args:
        logger: logging.Logger, The logger to log to.
        output_filter: _UserOutputFilter, Used to determine whether to write
          output or not.
        stream_wrapper: _StreamWrapper, The wrapper for the output stream,
          stdout or stderr.
        always_flush: bool, always flush stream_wrapper, default to False.
      """
      self.__logger = logger
      self.__filter = output_filter
      self.__stream_wrapper = stream_wrapper
      self.__always_flush = always_flush
  
    def ParseMsg(self, msg):
      """Converts msg to a console safe pair of plain and ANSI-annotated strings.
  
      Args:
        msg: str or text.TypedText, the message to parse into plain and
          ANSI-annotated strings.
      Returns:
        str, str: A plain text string and a string that may also contain ANSI
          constrol sequences. If ANSI is not supported or color is disabled,
          then the second string will be identical to the first.
      """
      plain_text, styled_text = msg, msg
      if isinstance(msg, text.TypedText):
        typed_text_parser = style_parser.GetTypedTextParser()
        plain_text = typed_text_parser.ParseTypedTextToString(msg, stylize=False)
        styled_text = typed_text_parser.ParseTypedTextToString(
            msg, stylize=self.isatty())
      plain_text = console_attr.SafeText(
          plain_text, encoding=LOG_FILE_ENCODING, escape=False)
      styled_text = console_attr.SafeText(
          styled_text, encoding=LOG_FILE_ENCODING, escape=False)
      return plain_text, styled_text
  
    def Print(self, *tokens):
      """Writes the given tokens to the output stream, and adds a newline.
  
      This method has the same output behavior as the builtin print method but
      respects the configured verbosity.
  
      Args:
        *tokens: str or text.TypedTextor any object with a str() or unicode()
          method, The messages to print, which are joined with ' '.
      """
      plain_tokens, styled_tokens = [], []
      for token in tokens:
        plain_text, styled_text = self.ParseMsg(token)
        plain_tokens.append(plain_text)
        styled_tokens.append(styled_text)
  
      plain_text = ' '.join(plain_tokens) + '\n'
      styled_text = ' '.join(styled_tokens) + '\n'
      self._Write(plain_text, styled_text)
  
    def GetConsoleWriterStream(self):
      """Returns the console writer output stream."""
      return self.__stream_wrapper.stream
  
    def _Write(self, msg, styled_msg):
      """Just a helper so we don't have to double encode from Print and write.
  
      Args:
        msg: A text string that only has characters that are safe to encode with
          utf-8.
        styled_msg: A text string with the same properties as msg but also
          contains ANSI control sequences.
      """
      # The log file always uses utf-8 encoding, just give it the string.
      self.__logger.info(msg)
  
      if self.__filter.enabled:
        # Make sure the string is safe to print in the console. The console might
        # have a more restrictive encoding than utf-8.
        stream_encoding = console_attr.GetConsoleAttr().GetEncoding()
        stream_msg = console_attr.SafeText(
            styled_msg, encoding=stream_encoding, escape=False)
        if six.PY2:
          # Encode to byte strings for output only on Python 2.
          stream_msg = styled_msg.encode(stream_encoding or 'utf8', 'replace')
        self.__stream_wrapper.stream.write(stream_msg)
        if self.__always_flush:
          self.flush()
  
    # pylint: disable=g-bad-name, This must match file-like objects
    def write(self, msg):
      plain_text, styled_text = self.ParseMsg(msg)
      self._Write(plain_text, styled_text)
  
    # pylint: disable=g-bad-name, This must match file-like objects
    def writelines(self, lines):
      for line in lines:
        self.write(line)
  
    # pylint: disable=g-bad-name, This must match file-like objects
    def flush(self):
      if self.__filter.enabled:
        self.__stream_wrapper.stream.flush()
  
    def isatty(self):
      isatty = getattr(self.__stream_wrapper.stream, 'isatty', None)
      return isatty() if isatty else False
  
  
  def _FmtString(fmt):
    """Gets the correct format string to use based on the Python version.
  
    Args:
      fmt: text string, The format string to convert.
  
    Returns:
      A byte string on Python 2 or the original string on Python 3.
    """
    # On Py2, log messages come in as both text and byte strings so the format
    # strings needs to be bytes so it doesn't coerce byte messages to unicode.
    # On Py3, only text strings come in and if the format is bytes it can't
    # combine with the log message.
    if six.PY2:
      return fmt.encode('utf8')
    return fmt
  
  
  @contextlib.contextmanager
  def _SafeDecodedLogRecord(record, encoding):
    """Temporarily modifies a log record to make the message safe to print.
  
    Python logging creates a single log record for each log event. Each handler
    is given that record and asked format it. To avoid unicode issues, we decode
    all the messages in case they are byte strings. Doing this we also want to
    ensure the resulting string is able to be printed to the given output target.
  
    Some handlers target the console (which can have many different encodings) and
    some target the log file (which we always write as utf-8. If we modify the
    record, depending on the order of handlers, the log message could lose
    information along the way.
  
    For example, if the user has an ascii console, we replace non-ascii characters
    in the string with '?' to print. Then if the log file handler is called, the
    original unicode data is gone, even though it could successfully be printed
    to the log file. This context manager changes the log record briefly so it can
    be formatted without changing it for later handlers.
  
    Args:
      record: The log record.
      encoding: The name of the encoding to SafeDecode with.
    Yields:
      None, yield is necessary as this is a context manager.
    """
    original_msg = record.msg
    try:
      record.msg = console_attr.SafeText(
          record.msg, encoding=encoding, escape=False)
      yield
    finally:
      record.msg = original_msg
  
  
  class _LogFileFormatter(logging.Formatter):
    """A formatter for log file contents."""
    # TODO(b/116495229): Add a test to ensure consitency.
    # Note: if this ever changes, please update LOG_PREFIX_PATTERN
    FORMAT = _FmtString('%(asctime)s %(levelname)-8s %(name)-15s %(message)s')
  
    def __init__(self):
      super(_LogFileFormatter, self).__init__(fmt=_LogFileFormatter.FORMAT)
  
    def format(self, record):
      record = copy.copy(record)
  
      if isinstance(record.msg, text.TypedText):
        record.msg = style_parser.GetTypedTextParser().ParseTypedTextToString(
            record.msg, stylize=False)
  
      # There are some cases where record.args ends up being a dict.
      if isinstance(record.args, tuple):
        new_args = []
        for arg in record.args:
          if isinstance(arg, text.TypedText):
            arg = style_parser.GetTypedTextParser().ParseTypedTextToString(
                arg, stylize=False)
          new_args.append(arg)
        record.args = tuple(new_args)
      # The log file handler expects text strings always, and encodes them to
      # utf-8 before writing to the file.
      with _SafeDecodedLogRecord(record, LOG_FILE_ENCODING):
        msg = super(_LogFileFormatter, self).format(record)
      return msg
  
  
  class _ConsoleFormatter(logging.Formatter):
    """A formatter for the console logger, handles colorizing messages."""
  
    LEVEL = _FmtString('%(levelname)s:')
    MESSAGE = _FmtString(' %(message)s')
    DEFAULT_FORMAT = LEVEL + MESSAGE
  
    RED = _FmtString('\033[1;31m')
    YELLOW = _FmtString('\033[1;33m')
    END = _FmtString('\033[0m')
  
    FORMATS = {}
    COLOR_FORMATS = {
        logging.WARNING: YELLOW + LEVEL + END + MESSAGE,
        logging.ERROR: RED + LEVEL + END + MESSAGE,
        logging.FATAL: RED + LEVEL + MESSAGE + END,
    }
  
    def __init__(self, out_stream):
      super(_ConsoleFormatter, self).__init__()
      # TODO(b/113585509): Remove this coloring code.
      use_color = not properties.VALUES.core.disable_color.GetBool(validate=False)
      use_color &= out_stream.isatty()
      use_color &= (platforms.OperatingSystem.Current() !=
                    platforms.OperatingSystem.WINDOWS)
      self._formats = (_ConsoleFormatter.COLOR_FORMATS
                       if use_color else _ConsoleFormatter.FORMATS)
  
    def format(self, record):
      fmt = self._formats.get(record.levelno, _ConsoleFormatter.DEFAULT_FORMAT)
      # We are doing some hackery here to change the log format on the fly. In
      # Python 3, they changed the internal workings of the formatter class so we
      # need to do something a little different to maintain the behavior.
      self._fmt = fmt
      if six.PY3:
        # pylint: disable=protected-access
        self._style._fmt = fmt
      # Convert either bytes or text into a text string that is safe for printing.
      # This is the first time we are able to intercept messages that come
      # directly from the log methods (as opposed to the out.write() methods
      # above).
      stream_encoding = console_attr.GetConsoleAttr().GetEncoding()
      with _SafeDecodedLogRecord(record, stream_encoding):
        msg = super(_ConsoleFormatter, self).format(record)
      if six.PY2:
        msg = msg.encode(stream_encoding or 'utf8', 'replace')
      return msg
  
  
  class _JsonFormatter(logging.Formatter):
    """A formatter that handles formatting log messages as JSON."""
  
    def __init__(self,
                 required_fields,
                 json_serializer=None,
                 json_encoder=None):
  
      super(_JsonFormatter, self).__init__()
      self.required_fields = required_fields
      self.json_encoder = json_encoder
      self.json_serializer = json_serializer or json.dumps
      self.default_time_format = STRUCTURED_TIME_FORMAT
  
    def GetErrorDict(self, log_record):
      """Extract exception info from a logging.LogRecord as an OrderedDict."""
      error_dict = OrderedDict()
      if log_record.exc_info:
        if not log_record.exc_text:
          log_record.exc_text = self.formatException(log_record.exc_info)
  
        if issubclass(type(log_record.msg), BaseException):
          error_dict['type'] = type(log_record.msg).__name__
          error_dict['details'] = six.text_type(log_record.msg)
          error_dict['stacktrace'] = getattr(log_record.msg,
                                             '__traceback__', None)
        elif issubclass(type(log_record.exc_info[0]), BaseException):
          error_dict['type'] = log_record.exc_info[0]
          error_dict['details'] = log_record.exc_text
          error_dict['stacktrace'] = log_record.exc_info[2]
        else:
          error_dict['type'] = log_record.exc_text
          error_dict['details'] = log_record.exc_text
          error_dict['stacktrace'] = log_record.exc_text
        return error_dict
      return None
  
    def BuildLogMsg(self, log_record):
      """Converts a logging.LogRecord object to a JSON serializable OrderedDict.
  
      Utilizes supplied set of required_fields to determine output fields.
  
      Args:
        log_record: logging.LogRecord, log record to be converted
  
      Returns:
        OrderedDict of required_field values.
      """
      message_dict = OrderedDict()
      # This perserves the order in the output for each JSON message
      for outfield, logfield in six.iteritems(self.required_fields):
        if outfield == 'version':
          message_dict[outfield] = STRUCTURED_RECORD_VERSION
        else:
          message_dict[outfield] = log_record.__dict__.get(logfield)
      return message_dict
  
    def LogRecordToJson(self, log_record):
      """Returns a json string of the log message."""
      log_message = self.BuildLogMsg(log_record)
      if not log_message.get('error'):
        log_message.pop('error')
  
      return self.json_serializer(log_message,
                                  cls=self.json_encoder)
  
    def formatTime(self, record, datefmt=None):
      return times.FormatDateTime(
          times.GetDateTimeFromTimeStamp(record.created),
          fmt=datefmt,
          tzinfo=times.UTC)
  
    def format(self, record):
      """Formats a log record and serializes to json."""
      record.__dict__['error'] = self.GetErrorDict(record)
      record.message = record.getMessage()
      record.asctime = self.formatTime(record, self.default_time_format)
      return self.LogRecordToJson(record)
  
  
  class _ConsoleLoggingFormatterMuxer(logging.Formatter):
    """Logging Formatter Composed of other formatters."""
  
    def __init__(self,
                 structured_formatter,
                 stream_writter,
                 default_formatter=None):
      logging.Formatter.__init__(self)
      self.default_formatter = default_formatter or logging.Formatter
      self.structured_formatter = structured_formatter
      self.terminal = stream_writter.isatty()
  
    def ShowStructuredOutput(self):
      """Returns True if output should be Structured, False otherwise."""
      show_messages = properties.VALUES.core.show_structured_logs.Get()
      if any([show_messages == 'terminal' and self.terminal,
              show_messages == 'log' and not self.terminal,
              show_messages == 'always']):
        return True
  
      return False
  
    def format(self, record):
      """Formats the record using the proper formatter."""
      show_structured_output = self.ShowStructuredOutput()
  
      # The logged msg was a TypedText so convert msg to a normal str.
      stylize = self.terminal and not show_structured_output
      record = copy.copy(record)
      if isinstance(record.msg, text.TypedText):
        record.msg = style_parser.GetTypedTextParser().ParseTypedTextToString(
            record.msg, stylize=stylize)
  
      # There are some cases where record.args ends up being a dict.
      if isinstance(record.args, tuple):
        new_args = []
        for arg in record.args:
          if isinstance(arg, text.TypedText):
            arg = style_parser.GetTypedTextParser().ParseTypedTextToString(
                arg, stylize=stylize)
          new_args.append(arg)
        record.args = tuple(new_args)
  
      if show_structured_output:
        return self.structured_formatter.format(record)
      return self.default_formatter.format(record)
  
  
  class _LogManager(object):
    """A class to manage the logging handlers based on how calliope is being used.
  
    We want to always log to a file, in addition to logging to stdout if in CLI
    mode.  This sets up the required handlers to do this.
    """
    FILE_ONLY_LOGGER_NAME = '___FILE_ONLY___'
  
    def __init__(self):
      self._file_formatter = _LogFileFormatter()
  
      # Set up the root logger, it accepts all levels.
      self._root_logger = logging.getLogger()
      self._root_logger.setLevel(logging.NOTSET)
  
      # This logger will get handlers for each output file, but will not propagate
      # to the root logger.  This allows us to log exceptions and errors to the
      # files without it showing up in the terminal.
      self.file_only_logger = logging.getLogger(_LogManager.FILE_ONLY_LOGGER_NAME)
      # Accept all log levels for files.
      self.file_only_logger.setLevel(logging.NOTSET)
      self.file_only_logger.propagate = False
  
      self._logs_dirs = []
  
      self._console_formatter = None
      self._user_output_filter = _UserOutputFilter(DEFAULT_USER_OUTPUT_ENABLED)
      self.stdout_stream_wrapper = _StreamWrapper(None)
      self.stderr_stream_wrapper = _StreamWrapper(None)
  
      self.stdout_writer = _ConsoleWriter(self.file_only_logger,
                                          self._user_output_filter,
                                          self.stdout_stream_wrapper)
      self.stderr_writer = _ConsoleWriter(self.file_only_logger,
                                          self._user_output_filter,
                                          self.stderr_stream_wrapper,
                                          always_flush=True)
  
      self.verbosity = None
      self.user_output_enabled = None
      self.current_log_file = None
      self.Reset(sys.stdout, sys.stderr)
  
    def Reset(self, stdout, stderr):
      """Resets all logging functionality to its default state."""
      # Clears any existing logging handlers.
      self._root_logger.handlers[:] = []
  
      # Refresh the streams for the console writers.
      self.stdout_stream_wrapper.stream = stdout
      self.stderr_stream_wrapper.stream = stderr
  
      # Configure Formatters
      json_formatter = _JsonFormatter(REQUIRED_STRUCTURED_RECORD_FIELDS)
      std_console_formatter = _ConsoleFormatter(stderr)
      console_formatter = _ConsoleLoggingFormatterMuxer(
          json_formatter,
          self.stderr_writer,
          default_formatter=std_console_formatter)
      # Reset the color and structured output handling.
      self._console_formatter = console_formatter
      # A handler to redirect logs to stderr, this one is standard.
      self.stderr_handler = logging.StreamHandler(stderr)
      self.stderr_handler.setFormatter(self._console_formatter)
      self.stderr_handler.setLevel(DEFAULT_VERBOSITY)
      self._root_logger.addHandler(self.stderr_handler)
  
      # Reset all the log file handlers.
      for f in self.file_only_logger.handlers:
        f.close()
      self.file_only_logger.handlers[:] = []
      self.file_only_logger.addHandler(_NullHandler())
  
      # Reset verbosity and output settings.
      self.SetVerbosity(None)
      self.SetUserOutputEnabled(None)
      self.current_log_file = None
  
    def SetVerbosity(self, verbosity):
      """Sets the active verbosity for the logger.
  
      Args:
        verbosity: int, A verbosity constant from the logging module that
          determines what level of logs will show in the console. If None, the
          value from properties or the default will be used.
  
      Returns:
        int, The current verbosity.
      """
      if verbosity is None:
        # Try to load from properties if set.
        verbosity_string = properties.VALUES.core.verbosity.Get()
        if verbosity_string is not None:
          verbosity = VALID_VERBOSITY_STRINGS.get(verbosity_string.lower())
      if verbosity is None:
        # Final fall back to default verbosity.
        verbosity = DEFAULT_VERBOSITY
  
      if self.verbosity == verbosity:
        return self.verbosity
  
      self.stderr_handler.setLevel(verbosity)
  
      old_verbosity = self.verbosity
      self.verbosity = verbosity
      return old_verbosity
  
    def SetUserOutputEnabled(self, enabled):
      """Sets whether user output should go to the console.
  
      Args:
        enabled: bool, True to enable output, False to suppress.  If None, the
          value from properties or the default will be used.
  
      Returns:
        bool, The old value of enabled.
      """
      if enabled is None:
        enabled = properties.VALUES.core.user_output_enabled.GetBool(
            validate=False)
      if enabled is None:
        enabled = DEFAULT_USER_OUTPUT_ENABLED
  
      self._user_output_filter.enabled = enabled
  
      old_enabled = self.user_output_enabled
      self.user_output_enabled = enabled
      return old_enabled
  
    def _GetMaxLogDays(self):
      """Gets the max log days for the logger.
  
      Returns:
        max_log_days: int, the maximum days for log file retention
      """
      # Fetch from properties. Defaults to 30 if unset.
      return properties.VALUES.core.max_log_days.GetInt()
  
    def _GetMaxAge(self):
      """Gets max_log_day's worth of seconds."""
      return 60 * 60 * 24 * self._GetMaxLogDays()
  
    def _GetMaxAgeTimeDelta(self):
      return datetime.timedelta(days=self._GetMaxLogDays())
  
    def _GetFileDatetime(self, path):
      return datetime.datetime.strptime(os.path.basename(path),
                                        DAY_DIR_FORMAT)
  
    def AddLogsDir(self, logs_dir):
      """Adds a new logging directory to the logging config.
  
      Args:
        logs_dir: str, Path to a directory to store log files under.  This method
          has no effect if this is None, or if this directory has already been
          registered.
      """
      if not logs_dir or logs_dir in self._logs_dirs:
        return
      self._logs_dirs.append(logs_dir)
  
      # If logs cleanup has been enabled, try to delete old log files
      # in the given directory. Continue normally if we try to delete log files
      # that do not exist. This can happen when two gcloud instances are cleaning
      # up logs in parallel.
      self._CleanUpLogs(logs_dir)
  
      # A handler to write DEBUG and above to log files in the given directory
      try:
        log_file = self._SetupLogsDir(logs_dir)
        file_handler = logging.FileHandler(
            log_file, encoding=LOG_FILE_ENCODING)  # pytype: disable=wrong-arg-types
      except (OSError, IOError, files.Error) as exp:
        warning('Could not setup log file in {0}, ({1}: {2})'
                .format(logs_dir, type(exp).__name__, exp))
        return
  
      self.current_log_file = log_file
      file_handler.setLevel(logging.NOTSET)
      file_handler.setFormatter(self._file_formatter)
      self._root_logger.addHandler(file_handler)
      self.file_only_logger.addHandler(file_handler)
  
    def _CleanUpLogs(self, logs_dir):
      """Clean up old log files if log cleanup has been enabled."""
      if self._GetMaxLogDays():
        try:
          self._CleanLogsDir(logs_dir)
        except OSError as exp:
          if exp.errno != errno.ENOENT:
            raise
  
    def _CleanLogsDir(self, logs_dir):
      """Cleans up old log files form the given logs directory.
  
      Args:
        logs_dir: str, The path to the logs directory.
      """
      now = datetime.datetime.now()
      now_seconds = time.time()
  
      try:
        dirnames = os.listdir(logs_dir)
      except (OSError, UnicodeError):
        # In event of a non-existing or non-readable directory, we don't want to
        # cause an error
        return
      for dirname in dirnames:
        dir_path = os.path.join(logs_dir, dirname)
        if self._ShouldDeleteDir(now, dir_path):
          for filename in os.listdir(dir_path):
            log_file_path = os.path.join(dir_path, filename)
            if self._ShouldDeleteFile(now_seconds, log_file_path):
              os.remove(log_file_path)
          try:
            os.rmdir(dir_path)
          except OSError:
            # If the directory isn't empty, or isn't removable for some other
            # reason. This is okay.
            pass
  
    def _ShouldDeleteDir(self, now, path):
      """Determines if the directory should be deleted.
  
      True iff:
      * path is a directory
      * path name is formatted according to DAY_DIR_FORMAT
      * age of path (according to DAY_DIR_FORMAT) is slightly older than the
        MAX_AGE of a log file
  
      Args:
        now: datetime.datetime object indicating the current date/time.
        path: the full path to the directory in question.
  
      Returns:
        bool, whether the path is a valid directory that should be deleted
      """
      if not os.path.isdir(path):
        return False
      try:
        dir_date = self._GetFileDatetime(path)
      except ValueError:
        # Not in a format we're expecting; we probably shouldn't mess with it
        return False
      dir_age = now - dir_date
      # Add an additional day to this. It's better to delete a whole directory at
      # once and leave some extra files on disk than to loop through on each run
      # (some customers have pathologically large numbers of log files).
      return dir_age > self._GetMaxAgeTimeDelta() + datetime.timedelta(1)
  
    def _ShouldDeleteFile(self, now_seconds, path):
      """Determines if the file is old enough to be deleted.
  
      If the file is not a file that we recognize, return False.
  
      Args:
        now_seconds: int, The current time in seconds.
        path: str, The file or directory path to check.
  
      Returns:
        bool, True if it should be deleted, False otherwise.
      """
      if os.path.splitext(path)[1] not in _KNOWN_LOG_FILE_EXTENSIONS:
        # If we don't recognize this file, don't delete it
        return False
      stat_info = os.stat(path)
      return now_seconds - stat_info.st_mtime > self._GetMaxAge()
  
    def _SetupLogsDir(self, logs_dir):
      """Creates the necessary log directories and get the file name to log to.
  
      Logs are created under the given directory.  There is a sub-directory for
      each day, and logs for individual invocations are created under that.
  
      Deletes files in this directory that are older than MAX_AGE.
  
      Args:
        logs_dir: str, Path to a directory to store log files under
  
      Returns:
        str, The path to the file to log to
      """
      now = datetime.datetime.now()
      day_dir_name = now.strftime(DAY_DIR_FORMAT)
      day_dir_path = os.path.join(logs_dir, day_dir_name)
      files.MakeDir(day_dir_path)
  
      filename = '{timestamp}{ext}'.format(
          timestamp=now.strftime(FILENAME_FORMAT), ext=LOG_FILE_EXTENSION)
      log_file = os.path.join(day_dir_path, filename)
      return log_file
  
  
  _log_manager = _LogManager()
  
  # The configured stdout writer.  This writer is a stripped down file-like
  # object that provides the basic writing methods.  When you write to this
  # stream, it will be written to stdout only if user output is enabled.  All
  # strings will also be logged at INFO level to any registered log files.
  out = _log_manager.stdout_writer
  
  
  # The configured stderr writer.  This writer is a stripped down file-like
  # object that provides the basic writing methods.  When you write to this
  # stream, it will be written to stderr only if user output is enabled.  All
  # strings will also be logged at INFO level to any registered log files.
  err = _log_manager.stderr_writer
  
  # Status output writer. For things that are useful to know for someone watching
  # a command run, but aren't normally scraped.
  status = err
  
  
  # Gets a logger object that logs only to a file and never to the console.
  # You usually don't want to use this logger directly.  All normal logging will
  # also go to files.  This logger specifically prevents the messages from going
  # to the console under any verbosity setting.
  file_only_logger = _log_manager.file_only_logger
  
  
  def Print(*msg):
    """Writes the given message to the output stream, and adds a newline.
  
    This method has the same output behavior as the builtin print method but
    respects the configured user output setting.
  
    Args:
      *msg: str, The messages to print.
    """
    out.Print(*msg)
  
  
  def WriteToFileOrStdout(path, content, overwrite=True, binary=False,
                          private=False):
    """Writes content to the specified file or stdout if path is '-'.
  
    Args:
      path: str, The path of the file to write.
      content: str, The content to write to the file.
      overwrite: bool, Whether or not to overwrite the file if it exists.
      binary: bool, True to open the file in binary mode.
      private: bool, Whether to write the file in private mode.
  
    Raises:
      Error: If the file cannot be written.
    """
    if path == '-':
      if binary:
        files.WriteStreamBytes(sys.stdout, content)
      else:
        out.write(content)
    elif binary:
      files.WriteBinaryFileContents(path, content, overwrite=overwrite,
                                    private=private)
    else:
      files.WriteFileContents(path, content, overwrite=overwrite, private=private)
  
  
  def Reset(stdout=None, stderr=None):
    """Reinitialize the logging system.
  
    This clears all loggers registered in the logging module, and reinitializes
    it with the specific loggers we want for calliope.
  
    This will set the initial values for verbosity or user_output_enabled to their
    values saved in the properties.
  
    Since we are using the python logging module, and that is all statically
    initialized, this method does not actually turn off all the loggers.  If you
    hold references to loggers or writers after calling this method, it is
    possible they will continue to work, but their behavior might change when the
    logging framework is reinitialized.  This is useful mainly for clearing the
    loggers between tests so stubs can get reset.
  
    Args:
      stdout: the file-like object to restore to stdout. If not given, sys.stdout
        is used
      stderr: the file-like object to restore to stderr. If not given, sys.stderr
        is used
    """
    _log_manager.Reset(stdout or sys.stdout, stderr or sys.stderr)
  
  
  def SetVerbosity(verbosity):
    """Sets the active verbosity for the logger.
  
    Args:
      verbosity: int, A verbosity constant from the logging module that
        determines what level of logs will show in the console. If None, the
        value from properties or the default will be used.
  
    Returns:
      int, The current verbosity.
    """
    return _log_manager.SetVerbosity(verbosity)
  
  
  def GetVerbosity():
    """Gets the current verbosity setting.
  
    Returns:
      int, The current verbosity.
    """
    return _log_manager.verbosity
  
  
  def GetVerbosityName(verbosity=None):
    """Gets the name for the current verbosity setting or verbosity if not None.
  
    Args:
      verbosity: int, Returns the name for this verbosity if not None.
  
    Returns:
      str, The verbosity name or None if the verbosity is unknown.
    """
    if verbosity is None:
      verbosity = GetVerbosity()
    for name, num in six.iteritems(VALID_VERBOSITY_STRINGS):
      if verbosity == num:
        return name
    return None
  
  
  def OrderedVerbosityNames():
    """Gets all the valid verbosity names from most verbose to least verbose."""
    return [name for name, _ in _VERBOSITY_LEVELS]
  
  
  def _GetEffectiveVerbosity(verbosity):
    """Returns the effective verbosity for verbosity. Handles None => NOTSET."""
    return verbosity or logging.NOTSET
  
  
  def SetLogFileVerbosity(verbosity):
    """Sets the log file verbosity.
  
    Args:
      verbosity: int, A verbosity constant from the logging module that
        determines what level of logs will be written to the log file. If None,
        the default will be used.
  
    Returns:
      int, The current verbosity.
    """
    return _GetEffectiveVerbosity(
        _log_manager.file_only_logger.setLevel(verbosity))
  
  
  def GetLogFileVerbosity():
    """Gets the current log file verbosity setting.
  
    Returns:
      int, The log file current verbosity.
    """
    return _GetEffectiveVerbosity(
        _log_manager.file_only_logger.getEffectiveLevel())
  
  
  class LogFileVerbosity(object):
    """A log file verbosity context manager.
  
    Attributes:
      _context_verbosity: int, The log file verbosity during the context.
      _original_verbosity: int, The original log file verbosity before the
        context was entered.
  
    Returns:
      The original verbosity is returned in the "as" clause.
    """
  
    def __init__(self, verbosity):
      self._context_verbosity = verbosity
  
    def __enter__(self):
      self._original_verbosity = SetLogFileVerbosity(self._context_verbosity)
      return self._original_verbosity
  
    def __exit__(self, exc_type, exc_value, traceback):
      SetLogFileVerbosity(self._original_verbosity)
      return False
  
  
  def SetUserOutputEnabled(enabled):
    """Sets whether user output should go to the console.
  
    Args:
      enabled: bool, True to enable output, false to suppress.
  
    Returns:
      bool, The old value of enabled.
    """
    return _log_manager.SetUserOutputEnabled(enabled)
  
  
  def IsUserOutputEnabled():
    """Gets whether user output is enabled or not.
  
    Returns:
      bool, True if user output is enabled, False otherwise.
    """
    return _log_manager.user_output_enabled
  
  
  def AddFileLogging(logs_dir):
    """Adds a new logging file handler to the root logger.
  
    Args:
      logs_dir: str, The root directory to store logs in.
    """
    _log_manager.AddLogsDir(logs_dir=logs_dir)
  
  
  def GetLogDir():
    """Gets the path to the currently in use log directory.
  
    Returns:
      str, The logging directory path.
    """
    log_file = _log_manager.current_log_file
    if not log_file:
      return None
    return os.path.dirname(log_file)
  
  
  def GetLogFileName(suffix):
    """Returns a new log file name based on the currently active log file.
  
    Args:
      suffix: str, A suffix to add to the current log file name.
  
    Returns:
      str, The name of a log file, or None if file logging is not on.
    """
    log_file = _log_manager.current_log_file
    if not log_file:
      return None
    log_filename = os.path.basename(log_file)
    log_file_root_name = log_filename[:-len(LOG_FILE_EXTENSION)]
    return log_file_root_name + suffix
  
  
  def GetLogFilePath():
    """Return the path to the currently active log file.
  
    Returns:
      str, The name of a log file, or None if file logging is not on.
    """
    return _log_manager.current_log_file
  
  
  # TODO(b/117488015): Remove this in favor of console.style.log
  def _PrintResourceChange(operation,
                           resource,
                           kind,
                           is_async,
                           details,
                           failed,
                           operation_past_tense=None):
    """Prints a status message for operation on resource.
  
    The non-failure status messages are disabled when user output is disabled.
  
    Args:
      operation: str, The completed operation name.
      resource: str, The resource name.
      kind: str, The resource kind (instance, cluster, project, etc.).
      is_async: bool, True if the operation is in progress.
      details: str, Extra details appended to the message. Keep it succinct.
      failed: str, Failure message. For commands that operate on multiple
        resources and report all successes and failures before exiting. Failure
        messages use log.error. This will display the message on the standard
        error even when user output is disabled.
      operation_past_tense: str, The past tense version of the operation verb.
        If None assumes operation + 'd'
    """
    msg = []
    if failed:
      msg.append('Failed to')
      msg.append(operation)
    elif is_async:
      msg.append(operation.capitalize())
      msg.append('in progress for')
    else:
      verb = operation_past_tense or '{0}d'.format(operation)
      msg.append('{0}'.format(verb.capitalize()))
  
    if kind:
      msg.append(kind)
    if resource:
      msg.append('[{0}]'.format(str(resource)))
    if details:
      msg.append(details)
    if failed:
      msg[-1] = '{0}:'.format(msg[-1])
      msg.append(failed)
    period = '' if msg[-1].endswith('.') else '.'
    writer = error if failed else status.Print
    writer('{0}{1}'.format(' '.join(msg), period))
  
  
  def CreatedResource(resource, kind=None, is_async=False, details=None,
                      failed=None):
    """Prints a status message indicating that a resource was created.
  
    Args:
      resource: str, The resource name.
      kind: str, The resource kind (instance, cluster, project, etc.).
      is_async: bool, True if the operation is in progress.
      details: str, Extra details appended to the message. Keep it succinct.
      failed: str, Failure message.
    """
    _PrintResourceChange('create', resource, kind, is_async, details, failed)
  
  
  def DeletedResource(resource, kind=None, is_async=False, details=None,
                      failed=None):
    """Prints a status message indicating that a resource was deleted.
  
    Args:
      resource: str, The resource name.
      kind: str, The resource kind (instance, cluster, project, etc.).
      is_async: bool, True if the operation is in progress.
      details: str, Extra details appended to the message. Keep it succinct.
      failed: str, Failure message.
    """
    _PrintResourceChange('delete', resource, kind, is_async, details, failed)
  
  
  def RestoredResource(resource, kind=None, is_async=False, details=None,
                       failed=None):
    """Prints a status message indicating that a resource was restored.
  
    Args:
      resource: str, The resource name.
      kind: str, The resource kind (instance, cluster, project, etc.).
      is_async: bool, True if the operation is in progress.
      details: str, Extra details appended to the message. Keep it succinct.
      failed: str, Failure message.
    """
    _PrintResourceChange('restore', resource, kind, is_async, details, failed)
  
  
  def UpdatedResource(resource, kind=None, is_async=False, details=None,
                      failed=None):
    """Prints a status message indicating that a resource was updated.
  
    Args:
      resource: str, The resource name.
      kind: str, The resource kind (instance, cluster, project, etc.).
      is_async: bool, True if the operation is in progress.
      details: str, Extra details appended to the message. Keep it succinct.
      failed: str, Failure message.
    """
    _PrintResourceChange('update', resource, kind, is_async, details, failed)
  
  
  def ResetResource(resource, kind=None, is_async=False, details=None,
                    failed=None):
    """Prints a status message indicating that a resource was reset.
  
    Args:
      resource: str, The resource name.
      kind: str, The resource kind (instance, cluster, project, etc.).
      is_async: bool, True if the operation is in progress.
      details: str, Extra details appended to the message. Keep it succinct.
      failed: str, Failure message.
    """
    _PrintResourceChange('reset', resource, kind, is_async, details, failed,
                         operation_past_tense='reset')
  
  
  def ExportResource(resource,
                     kind=None,
                     is_async=False,
                     details=None,
                     failed=None):
    """Prints a status message indicating that a resource was exported.
  
    Args:
      resource: str, The resource name.
      kind: str, The resource kind (instance, cluster, project, etc.).
      is_async: bool, True if the operation is in progress.
      details: str, Extra details appended to the message. Keep it succinct.
      failed: str, Failure message.
    """
    _PrintResourceChange('export', resource, kind, is_async, details, failed)
  
  
  # pylint: disable=invalid-name
  # There are simple redirects to the logging module as a convenience.
  getLogger = logging.getLogger
  log = logging.log
  debug = logging.debug
  info = logging.info
  warning = logging.warning
  error = logging.error
  critical = logging.critical
  fatal = logging.fatal
  exception = logging.exception
  