files: # Copyright 2017 Mycroft AI Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #    http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  import imp
  import operator
  import sys
  import time
  import csv
  import inspect
  from inspect import signature
  from datetime import datetime, timedelta
  
  import abc
  import re
  from adapt.intent import Intent, IntentBuilder
  from os.path import join, abspath, dirname, basename, exists
  from threading import Event
  
  from mycroft import dialog
  from mycroft.api import DeviceApi
  from mycroft.audio import wait_while_speaking
  from mycroft.client.enclosure.api import EnclosureAPI
  from mycroft.configuration import Configuration
  from mycroft.dialog import DialogLoader
  from mycroft.filesystem import FileSystemAccess
  from mycroft.messagebus.message import Message
  from mycroft.metrics import report_metric, report_timing, Stopwatch
  from mycroft.skills.settings import SkillSettings
  from mycroft.skills.skill_data import (load_vocabulary, load_regex, to_alnum,
                                         munge_regex, munge_intent_parser)
  from mycroft.util import resolve_resource_file
  from mycroft.util.log import LOG
  
  MainModule = '__init__'
  
  
  def dig_for_message():
      """
          Dig Through the stack for message.
      """
      stack = inspect.stack()
      # Limit search to 10 frames back
      stack = stack if len(stack) < 10 else stack[:10]
      local_vars = [frame[0].f_locals for frame in stack]
      for l in local_vars:
          if 'message' in l and isinstance(l['message'], Message):
              return l['message']
  
  
  def unmunge_message(message, skill_id):
      """Restore message keywords by removing the Letterified skill ID.
  
      Args:
          message (Message): Intent result message
          skill_id (str): skill identifier
  
      Returns:
          Message without clear keywords
      """
      if isinstance(message, Message) and isinstance(message.data, dict):
          skill_id = to_alnum(skill_id)
          for key in message.data:
              if key[:len(skill_id)] == skill_id:
                  new_key = key[len(skill_id):]
                  message.data[new_key] = message.data.pop(key)
  
      return message
  
  
  def open_intent_envelope(message):
      """ Convert dictionary received over messagebus to Intent. """
      intent_dict = message.data
      return Intent(intent_dict.get('name'),
                    intent_dict.get('requires'),
                    intent_dict.get('at_least_one'),
                    intent_dict.get('optional'))
  
  
  def load_skill(skill_descriptor, emitter, skill_id, BLACKLISTED_SKILLS=None):
      """
          load skill from skill descriptor.
  
          Args:
              skill_descriptor: descriptor of skill to load
              emitter:          messagebus emitter
              skill_id:         id number for skill
          Returns:
              MycroftSkill: the loaded skill or None on failure
      """
      BLACKLISTED_SKILLS = BLACKLISTED_SKILLS or []
      path = skill_descriptor["path"]
      name = basename(path)
      LOG.info("ATTEMPTING TO LOAD SKILL: {} with ID {}".format(
          name, skill_id
      ))
      if name in BLACKLISTED_SKILLS:
          LOG.info("SKILL IS BLACKLISTED " + name)
          return None
      main_file = join(path, MainModule + '.py')
      try:
          with open(main_file, 'rb') as fp:
              skill_module = imp.load_module(
                  name.replace('.', '_'), fp, main_file,
                  ('.py', 'rb', imp.PY_SOURCE)
              )
          if (hasattr(skill_module, 'create_skill') and
                  callable(skill_module.create_skill)):
              # v2 skills framework
              skill = skill_module.create_skill()
              skill.settings.allow_overwrite = True
              skill.settings.load_skill_settings_from_file()
              skill.bind(emitter)
              skill.skill_id = skill_id
              skill.load_data_files(path)
              # Set up intent handlers
              skill.initialize()
              skill._register_decorated()
              LOG.info("Loaded " + name)
  
              # The very first time a skill is run, speak the intro
              first_run = skill.settings.get("__mycroft_skill_firstrun", True)
              if first_run:
                  LOG.info("First run of " + name)
                  skill.settings["__mycroft_skill_firstrun"] = False
                  skill.settings.store()
                  intro = skill.get_intro_message()
                  if intro:
                      skill.speak(intro)
              return skill
          else:
              LOG.warning("Module {} does not appear to be skill".format(name))
      except Exception:
          LOG.exception("Failed to load skill: " + name)
      return None
  
  
  def create_skill_descriptor(skill_path):
      return {"path": skill_path}
  
  
  def get_handler_name(handler):
      """
          Return name (including class if available) of handler
          function.
  
          Args:
              handler (function): Function to be named
  
          Returns: handler name as string
      """
      name = ''
      if '__self__' in dir(handler) and 'name' in dir(handler.__self__):
          name += handler.__self__.name + '.'
      name += handler.__name__
      return name
  
  
  def intent_handler(intent_parser):
      """ Decorator for adding a method as an intent handler. """
  
      def real_decorator(func):
          # Store the intent_parser inside the function
          # This will be used later to call register_intent
          if not hasattr(func, 'intents'):
              func.intents = []
          func.intents.append(intent_parser)
          return func
  
      return real_decorator
  
  
  def intent_file_handler(intent_file):
      """ Decorator for adding a method as an intent file handler. """
  
      def real_decorator(func):
          # Store the intent_file inside the function
          # This will be used later to call register_intent_file
          if not hasattr(func, 'intent_files'):
              func.intent_files = []
          func.intent_files.append(intent_file)
          return func
  
      return real_decorator
  
  
  #######################################################################
  # MycroftSkill base class
  #######################################################################
  class MycroftSkill(object):
      """
      Abstract base class which provides common behaviour and parameters to all
      Skills implementation.
      """
  
      def __init__(self, name=None, emitter=None):
          self.name = name or self.__class__.__name__
          # Get directory of skill
          self._dir = dirname(abspath(sys.modules[self.__module__].__file__))
          self.settings = SkillSettings(self._dir, self.name)
  
          self.bind(emitter)
          self.config_core = Configuration.get()
          self.config = self.config_core.get(self.name) or {}
          self.dialog_renderer = None
          self.vocab_dir = None
          self.root_dir = None
          self.file_system = FileSystemAccess(join('skills', self.name))
          self.registered_intents = []
          self.log = LOG.create_logger(self.name)
          self.reload_skill = True  # allow reloading
          self.events = []
          self.scheduled_repeats = []
          self.skill_id = 0
  
      @property
      def location(self):
          """ Get the JSON data struction holding location information. """
          # TODO: Allow Enclosure to override this for devices that
          # contain a GPS.
          return self.config_core.get('location')
  
      @property
      def location_pretty(self):
          """ Get a more 'human' version of the location as a string. """
          loc = self.location
          if type(loc) is dict and loc["city"]:
              return loc["city"]["name"]
          return None
  
      @property
      def location_timezone(self):
          """ Get the timezone code, such as 'America/Los_Angeles' """
          loc = self.location
          if type(loc) is dict and loc["timezone"]:
              return loc["timezone"]["code"]
          return None
  
      @property
      def lang(self):
          return self.config_core.get('lang')
  
      def bind(self, emitter):
          """ Register emitter with skill. """
          if emitter:
              self.emitter = emitter
              self.enclosure = EnclosureAPI(emitter, self.name)
              self.__register_stop()
              self.add_event('mycroft.skill.enable_intent',
                             self.handle_enable_intent)
              self.add_event('mycroft.skill.disable_intent',
                             self.handle_disable_intent)
  
              name = 'mycroft.skills.settings.update'
              func = self.settings.run_poll
              emitter.on(name, func)
              self.events.append((name, func))
  
      def __register_stop(self):
          self.stop_time = time.time()
          self.stop_threshold = self.config_core.get("skills").get(
              'stop_threshold')
          self.add_event('mycroft.stop', self.__handle_stop)
  
      def detach(self):
          for (name, intent) in self.registered_intents:
              name = str(self.skill_id) + ':' + name
              self.emitter.emit(Message("detach_intent", {"intent_name": name}))
  
      def initialize(self):
          """
          Invoked after the skill is fully constructed and registered with the
          system.  Use to perform any final setup needed for the skill.
          """
          pass
  
      def get_intro_message(self):
          """
          Get a message to speak on first load of the skill.  Useful
          for post-install setup instructions.
  
          Returns:
              str: message that will be spoken to the user
          """
          return None
  
      def converse(self, utterances, lang="en-us"):
          """
              Handle conversation. This method can be used to override the normal
              intent handler after the skill has been invoked once.
  
              To enable this override thise converse method and return True to
              indicate that the utterance has been handled.
  
              Args:
                  utterances (list): The utterances from the user
                  lang:       language the utterance is in
  
              Returns:    True if an utterance was handled, otherwise False
          """
          return False
  
      def __get_response(self):
          """
          Helper to get a reponse from the user
  
          Returns:
              str: user's response or None on a timeout
          """
          event = Event()
  
          def converse(utterances, lang="en-us"):
              converse.response = utterances[0] if utterances else None
              event.set()
              return True
  
          # install a temporary conversation handler
          self.make_active()
          converse.response = None
          default_converse = self.converse
          self.converse = converse
          event.wait(15)  # 10 for listener, 5 for SST, then timeout
          self.converse = default_converse
          return converse.response
  
      def get_response(self, dialog='', data=None, announcement='',
                       validator=None, on_fail=None, num_retries=-1):
          """
          Prompt user and wait for response
  
          The given dialog or announcement will be spoken, the immediately
          listen and return user response.  The response can optionally be
          validated.
  
          Example:
              color = self.get_response('ask.favorite.color')
  
          Args:
              dialog (str): Announcement dialog to read to the user
              data (dict): Data used to render the dialog
              announcement (str): Literal string (overrides dialog)
              validator (any): Function with following signature
                  def validator(utterance):
                      return utterance != "red"
              on_fail (any): Dialog or function returning literal string
                             to speak on invalid input.  For example:
                  def on_fail(utterance):
                      return "nobody likes the color red, pick another"
              num_retries (int): Times to ask user for input, -1 for infinite
                  NOTE: User can not respond and timeout or say "cancel" to stop
  
          Returns:
              str: User's reply or None if timed out or canceled
          """
          data = data or {}
  
          def get_announcement():
              return announcement or self.dialog_renderer.render(dialog, data)
  
          if not get_announcement():
              raise ValueError('announcement or dialog message required')
  
          def on_fail_default(utterance):
              fail_data = data.copy()
              fail_data['utterance'] = utterance
              if on_fail:
                  return self.dialog_renderer.render(on_fail, fail_data)
              else:
                  return get_announcement()
  
          # TODO: Load with something like mycroft.dialog.get_all()
          cancel_voc = 'text/' + self.lang + '/cancel.voc'
          with open(resolve_resource_file(cancel_voc)) as f:
              cancel_words = list(filter(bool, f.read().split('\n')))
  
          def is_cancel(utterance):
              return utterance in cancel_words
  
          def validator_default(utterance):
              # accept anything except 'cancel'
              return not is_cancel(utterance)
  
          validator = validator or validator_default
          on_fail_fn = on_fail if callable(on_fail) else on_fail_default
  
          self.speak(get_announcement(), expect_response=True)
          wait_while_speaking()
          num_fails = 0
          while True:
              response = self.__get_response()
  
              if response is None:
                  # if nothing said, prompt one more time
                  num_none_fails = 1 if num_retries < 0 else num_retries
                  if num_fails >= num_none_fails:
                      return None
              else:
                  if validator(response):
                      return response
  
                  # catch user saying 'cancel'
                  if is_cancel(response):
                      return None
  
              num_fails += 1
              if 0 < num_retries < num_fails:
                  return None
  
              line = on_fail_fn(response)
              self.speak(line, expect_response=True)
  
      def report_metric(self, name, data):
          """
          Report a skill metric to the Mycroft servers
  
          Args:
              name (str): Name of metric. Must use only letters and hyphens
              data (
  
  [... the rest is omitted ...]
files: mycroft/skills/core.py
files: # Copyright 2017 Mycroft AI Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #    http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  import imp
  import operator
  import sys
  import time
  import csv
  import inspect
  from inspect import signature
  from datetime import datetime, timedelta
  
  import abc
  import re
  from adapt.intent import Intent, IntentBuilder
  from os.path import join, abspath, dirname, basename, exists
  from threading import Event
  
  from mycroft import dialog
  from mycroft.api import DeviceApi
  from mycroft.audio import wait_while_speaking
  from mycroft.client.enclosure.api import EnclosureAPI
  from mycroft.configuration import Configuration
  from mycroft.dialog import DialogLoader
  from mycroft.filesystem import FileSystemAccess
  from mycroft.messagebus.message import Message
  from mycroft.metrics import report_metric, report_timing, Stopwatch
  from mycroft.skills.settings import SkillSettings
  from mycroft.skills.skill_data import (load_vocabulary, load_regex, to_letters,
                                         munge_regex, munge_intent_parser)
  from mycroft.util import resolve_resource_file
  from mycroft.util.log import LOG
  
  MainModule = '__init__'
  
  
  def dig_for_message():
      """
          Dig Through the stack for message.
      """
      stack = inspect.stack()
      # Limit search to 10 frames back
      stack = stack if len(stack) < 10 else stack[:10]
      local_vars = [frame[0].f_locals for frame in stack]
      for l in local_vars:
          if 'message' in l and isinstance(l['message'], Message):
              return l['message']
  
  
  def unmunge_message(message, skill_id):
      """Restore message keywords by removing the Letterified skill ID.
  
      Args:
          message (Message): Intent result message
          skill_id (int): skill identifier
  
      Returns:
          Message without clear keywords
      """
      if isinstance(message, Message) and isinstance(message.data, dict):
          skill_id = to_letters(skill_id)
          for key in message.data:
              if key[:len(skill_id)] == skill_id:
                  new_key = key[len(skill_id):]
                  message.data[new_key] = message.data.pop(key)
  
      return message
  
  
  def open_intent_envelope(message):
      """ Convert dictionary received over messagebus to Intent. """
      intent_dict = message.data
      return Intent(intent_dict.get('name'),
                    intent_dict.get('requires'),
                    intent_dict.get('at_least_one'),
                    intent_dict.get('optional'))
  
  
  def load_skill(skill_descriptor, emitter, skill_id, BLACKLISTED_SKILLS=None):
      """
          load skill from skill descriptor.
  
          Args:
              skill_descriptor: descriptor of skill to load
              emitter:          messagebus emitter
              skill_id:         id number for skill
          Returns:
              MycroftSkill: the loaded skill or None on failure
      """
      BLACKLISTED_SKILLS = BLACKLISTED_SKILLS or []
      path = skill_descriptor["path"]
      name = basename(path)
      LOG.info("ATTEMPTING TO LOAD SKILL: {} with ID {}".format(
          name, skill_id
      ))
      if name in BLACKLISTED_SKILLS:
          LOG.info("SKILL IS BLACKLISTED " + name)
          return None
      main_file = join(path, MainModule + '.py')
      try:
          with open(main_file, 'rb') as fp:
              skill_module = imp.load_module(
                  name.replace('.', '_'), fp, main_file,
                  ('.py', 'rb', imp.PY_SOURCE)
              )
          if (hasattr(skill_module, 'create_skill') and
                  callable(skill_module.create_skill)):
              # v2 skills framework
              skill = skill_module.create_skill()
              skill.settings.allow_overwrite = True
              skill.settings.load_skill_settings_from_file()
              skill.bind(emitter)
              skill.skill_id = skill_id
              skill.load_data_files(path)
              # Set up intent handlers
              skill.initialize()
              skill._register_decorated()
              LOG.info("Loaded " + name)
  
              # The very first time a skill is run, speak the intro
              first_run = skill.settings.get("__mycroft_skill_firstrun", True)
              if first_run:
                  LOG.info("First run of " + name)
                  skill.settings["__mycroft_skill_firstrun"] = False
                  skill.settings.store()
                  intro = skill.get_intro_message()
                  if intro:
                      skill.speak(intro)
              return skill
          else:
              LOG.warning("Module {} does not appear to be skill".format(name))
      except Exception:
          LOG.exception("Failed to load skill: " + name)
      return None
  
  
  def create_skill_descriptor(skill_path):
      return {"path": skill_path}
  
  
  def get_handler_name(handler):
      """
          Return name (including class if available) of handler
          function.
  
          Args:
              handler (function): Function to be named
  
          Returns: handler name as string
      """
      name = ''
      if '__self__' in dir(handler) and 'name' in dir(handler.__self__):
          name += handler.__self__.name + '.'
      name += handler.__name__
      return name
  
  
  def intent_handler(intent_parser):
      """ Decorator for adding a method as an intent handler. """
  
      def real_decorator(func):
          # Store the intent_parser inside the function
          # This will be used later to call register_intent
          if not hasattr(func, 'intents'):
              func.intents = []
          func.intents.append(intent_parser)
          return func
  
      return real_decorator
  
  
  def intent_file_handler(intent_file):
      """ Decorator for adding a method as an intent file handler. """
  
      def real_decorator(func):
          # Store the intent_file inside the function
          # This will be used later to call register_intent_file
          if not hasattr(func, 'intent_files'):
              func.intent_files = []
          func.intent_files.append(intent_file)
          return func
  
      return real_decorator
  
  
  #######################################################################
  # MycroftSkill base class
  #######################################################################
  class MycroftSkill(object):
      """
      Abstract base class which provides common behaviour and parameters to all
      Skills implementation.
      """
  
      def __init__(self, name=None, emitter=None):
          self.name = name or self.__class__.__name__
          # Get directory of skill
          self._dir = dirname(abspath(sys.modules[self.__module__].__file__))
          self.settings = SkillSettings(self._dir, self.name)
  
          self.bind(emitter)
          self.config_core = Configuration.get()
          self.config = self.config_core.get(self.name) or {}
          self.dialog_renderer = None
          self.vocab_dir = None
          self.root_dir = None
          self.file_system = FileSystemAccess(join('skills', self.name))
          self.registered_intents = []
          self.log = LOG.create_logger(self.name)
          self.reload_skill = True  # allow reloading
          self.events = []
          self.scheduled_repeats = []
          self.skill_id = 0
  
      @property
      def location(self):
          """ Get the JSON data struction holding location information. """
          # TODO: Allow Enclosure to override this for devices that
          # contain a GPS.
          return self.config_core.get('location')
  
      @property
      def location_pretty(self):
          """ Get a more 'human' version of the location as a string. """
          loc = self.location
          if type(loc) is dict and loc["city"]:
              return loc["city"]["name"]
          return None
  
      @property
      def location_timezone(self):
          """ Get the timezone code, such as 'America/Los_Angeles' """
          loc = self.location
          if type(loc) is dict and loc["timezone"]:
              return loc["timezone"]["code"]
          return None
  
      @property
      def lang(self):
          return self.config_core.get('lang')
  
      def bind(self, emitter):
          """ Register emitter with skill. """
          if emitter:
              self.emitter = emitter
              self.enclosure = EnclosureAPI(emitter, self.name)
              self.__register_stop()
              self.add_event('mycroft.skill.enable_intent',
                             self.handle_enable_intent)
              self.add_event('mycroft.skill.disable_intent',
                             self.handle_disable_intent)
  
              name = 'mycroft.skills.settings.update'
              func = self.settings.run_poll
              emitter.on(name, func)
              self.events.append((name, func))
  
      def __register_stop(self):
          self.stop_time = time.time()
          self.stop_threshold = self.config_core.get("skills").get(
              'stop_threshold')
          self.add_event('mycroft.stop', self.__handle_stop)
  
      def detach(self):
          for (name, intent) in self.registered_intents:
              name = str(self.skill_id) + ':' + name
              self.emitter.emit(Message("detach_intent", {"intent_name": name}))
  
      def initialize(self):
          """
          Invoked after the skill is fully constructed and registered with the
          system.  Use to perform any final setup needed for the skill.
          """
          pass
  
      def get_intro_message(self):
          """
          Get a message to speak on first load of the skill.  Useful
          for post-install setup instructions.
  
          Returns:
              str: message that will be spoken to the user
          """
          return None
  
      def converse(self, utterances, lang="en-us"):
          """
              Handle conversation. This method can be used to override the normal
              intent handler after the skill has been invoked once.
  
              To enable this override thise converse method and return True to
              indicate that the utterance has been handled.
  
              Args:
                  utterances (list): The utterances from the user
                  lang:       language the utterance is in
  
              Returns:    True if an utterance was handled, otherwise False
          """
          return False
  
      def __get_response(self):
          """
          Helper to get a reponse from the user
  
          Returns:
              str: user's response or None on a timeout
          """
          event = Event()
  
          def converse(utterances, lang="en-us"):
              converse.response = utterances[0] if utterances else None
              event.set()
              return True
  
          # install a temporary conversation handler
          self.make_active()
          converse.response = None
          default_converse = self.converse
          self.converse = converse
          event.wait(15)  # 10 for listener, 5 for SST, then timeout
          self.converse = default_converse
          return converse.response
  
      def get_response(self, dialog='', data=None, announcement='',
                       validator=None, on_fail=None, num_retries=-1):
          """
          Prompt user and wait for response
  
          The given dialog or announcement will be spoken, the immediately
          listen and return user response.  The response can optionally be
          validated.
  
          Example:
              color = self.get_response('ask.favorite.color')
  
          Args:
              dialog (str): Announcement dialog to read to the user
              data (dict): Data used to render the dialog
              announcement (str): Literal string (overrides dialog)
              validator (any): Function with following signature
                  def validator(utterance):
                      return utterance != "red"
              on_fail (any): Dialog or function returning literal string
                             to speak on invalid input.  For example:
                  def on_fail(utterance):
                      return "nobody likes the color red, pick another"
              num_retries (int): Times to ask user for input, -1 for infinite
                  NOTE: User can not respond and timeout or say "cancel" to stop
  
          Returns:
              str: User's reply or None if timed out or canceled
          """
          data = data or {}
  
          def get_announcement():
              return announcement or self.dialog_renderer.render(dialog, data)
  
          if not get_announcement():
              raise ValueError('announcement or dialog message required')
  
          def on_fail_default(utterance):
              fail_data = data.copy()
              fail_data['utterance'] = utterance
              if on_fail:
                  return self.dialog_renderer.render(on_fail, fail_data)
              else:
                  return get_announcement()
  
          # TODO: Load with something like mycroft.dialog.get_all()
          cancel_voc = 'text/' + self.lang + '/cancel.voc'
          with open(resolve_resource_file(cancel_voc)) as f:
              cancel_words = list(filter(bool, f.read().split('\n')))
  
          def is_cancel(utterance):
              return utterance in cancel_words
  
          def validator_default(utterance):
              # accept anything except 'cancel'
              return not is_cancel(utterance)
  
          validator = validator or validator_default
          on_fail_fn = on_fail if callable(on_fail) else on_fail_default
  
          self.speak(get_announcement(), expect_response=True)
          wait_while_speaking()
          num_fails = 0
          while True:
              response = self.__get_response()
  
              if response is None:
                  # if nothing said, prompt one more time
                  num_none_fails = 1 if num_retries < 0 else num_retries
                  if num_fails >= num_none_fails:
                      return None
              else:
                  if validator(response):
                      return response
  
                  # catch user saying 'cancel'
                  if is_cancel(response):
                      return None
  
              num_fails += 1
              if 0 < num_retries < num_fails:
                  return None
  
              line = on_fail_fn(response)
              self.speak(line, expect_response=True)
  
      def report_metric(self, name, data):
          """
          Report a skill metric to the Mycroft servers
  
          Args:
              name (str): Name of metric. Must use only letters and hyphens
              data (
  
  [... the rest is omitted ...]
files: mycroft/skills/core.py
files: # Copyright 2017 Mycroft AI Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #    http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  import time
  from adapt.context import ContextManagerFrame
  from adapt.engine import IntentDeterminationEngine
  
  from mycroft.configuration import Configuration
  from mycroft.messagebus.message import Message
  from mycroft.skills.core import open_intent_envelope
  from mycroft.util.log import LOG
  from mycroft.util.parse import normalize
  from mycroft.metrics import report_timing, Stopwatch
  
  
  class ContextManager(object):
      """
      ContextManager
      Use to track context throughout the course of a conversational session.
      How to manage a session's lifecycle is not captured here.
      """
  
      def __init__(self, timeout):
          self.frame_stack = []
          self.timeout = timeout * 60  # minutes to seconds
  
      def clear_context(self):
          self.frame_stack = []
  
      def remove_context(self, context_id):
          self.frame_stack = [(f, t) for (f, t) in self.frame_stack
                              if context_id in f.entities[0].get('data', [])]
  
      def inject_context(self, entity, metadata=None):
          """
          Args:
              entity(object): Format example...
                                 {'data': 'Entity tag as <str>',
                                  'key': 'entity proper name as <str>',
                                  'confidence': <float>'
                                 }
              metadata(object): dict, arbitrary metadata about entity injected
          """
          metadata = metadata or {}
          try:
              if len(self.frame_stack) > 0:
                  top_frame = self.frame_stack[0]
              else:
                  top_frame = None
              if top_frame and top_frame[0].metadata_matches(metadata):
                  top_frame[0].merge_context(entity, metadata)
              else:
                  frame = ContextManagerFrame(entities=[entity],
                                              metadata=metadata.copy())
                  self.frame_stack.insert(0, (frame, time.time()))
          except (IndexError, KeyError):
              pass
  
      def get_context(self, max_frames=None, missing_entities=None):
          """ Constructs a list of entities from the context.
  
          Args:
              max_frames(int): maximum number of frames to look back
              missing_entities(list of str): a list or set of tag names,
              as strings
  
          Returns:
              list: a list of entities
          """
          missing_entities = missing_entities or []
  
          relevant_frames = [frame[0] for frame in self.frame_stack if
                             time.time() - frame[1] < self.timeout]
          if not max_frames or max_frames > len(relevant_frames):
              max_frames = len(relevant_frames)
  
          missing_entities = list(missing_entities)
          context = []
          for i in range(max_frames):
              frame_entities = [entity.copy() for entity in
                                relevant_frames[i].entities]
              for entity in frame_entities:
                  entity['confidence'] = entity.get('confidence', 1.0) \
                      / (2.0 + i)
              context += frame_entities
  
          result = []
          if len(missing_entities) > 0:
              for entity in context:
                  if entity.get('data') in missing_entities:
                      result.append(entity)
                      # NOTE: this implies that we will only ever get one
                      # of an entity kind from context, unless specified
                      # multiple times in missing_entities. Cannot get
                      # an arbitrary number of an entity kind.
                      missing_entities.remove(entity.get('data'))
          else:
              result = context
  
          # Only use the latest instance of each keyword
          stripped = []
          processed = []
          for f in result:
              keyword = f['data'][0][1]
              if keyword not in processed:
                  stripped.append(f)
                  processed.append(keyword)
          result = stripped
          return result
  
  
  class IntentService(object):
      def __init__(self, emitter):
          self.config = Configuration.get().get('context', {})
          self.engine = IntentDeterminationEngine()
  
          # Dictionary for translating a skill id to a name
          self.skill_names = {}
          # Context related intializations
          self.context_keywords = self.config.get('keywords', [])
          self.context_max_frames = self.config.get('max_frames', 3)
          self.context_timeout = self.config.get('timeout', 2)
          self.context_greedy = self.config.get('greedy', False)
          self.context_manager = ContextManager(self.context_timeout)
          self.emitter = emitter
          self.emitter.on('register_vocab', self.handle_register_vocab)
          self.emitter.on('register_intent', self.handle_register_intent)
          self.emitter.on('recognizer_loop:utterance', self.handle_utterance)
          self.emitter.on('detach_intent', self.handle_detach_intent)
          self.emitter.on('detach_skill', self.handle_detach_skill)
          # Context related handlers
          self.emitter.on('add_context', self.handle_add_context)
          self.emitter.on('remove_context', self.handle_remove_context)
          self.emitter.on('clear_context', self.handle_clear_context)
          # Converse method
          self.emitter.on('skill.converse.response',
                          self.handle_converse_response)
          self.emitter.on('mycroft.speech.recognition.unknown',
                          self.reset_converse)
          self.emitter.on('mycroft.skills.loaded', self.update_skill_name_dict)
  
          def add_active_skill_handler(message):
              self.add_active_skill(message.data['skill_id'])
          self.emitter.on('active_skill_request', add_active_skill_handler)
          self.active_skills = []  # [skill_id , timestamp]
          self.converse_timeout = 5  # minutes to prune active_skills
  
      def update_skill_name_dict(self, message):
          """
              Messagebus handler, updates dictionary of if to skill name
              conversions.
          """
          self.skill_names[message.data['id']] = message.data['name']
  
      def get_skill_name(self, skill_id):
          """ Get skill name from skill ID.
  
          Args:
              skill_id: a skill id as encoded in Intent handlers.
  
          Returns:
              (str) Skill name or the skill id if the skill wasn't found
          """
          return self.skill_names.get(skill_id, skill_id)
  
      def reset_converse(self, message):
          """Let skills know there was a problem with speech recognition"""
          lang = message.data.get('lang', "en-us")
          for skill in self.active_skills:
              self.do_converse(None, skill[0], lang)
  
      def do_converse(self, utterances, skill_id, lang):
          self.emitter.emit(Message("skill.converse.request", {
              "skill_id": skill_id, "utterances": utterances, "lang": lang}))
          self.waiting = True
          self.result = False
          start_time = time.time()
          t = 0
          while self.waiting and t < 5:
              t = time.time() - start_time
              time.sleep(0.1)
          self.waiting = False
          return self.result
  
      def handle_converse_response(self, message):
          # id = message.data["skill_id"]
          # no need to crosscheck id because waiting before new request is made
          # no other skill will make this request is safe assumption
          result = message.data["result"]
          self.result = result
          self.waiting = False
  
      def remove_active_skill(self, skill_id):
          for skill in self.active_skills:
              if skill[0] == skill_id:
                  self.active_skills.remove(skill)
  
      def add_active_skill(self, skill_id):
          # search the list for an existing entry that already contains it
          # and remove that reference
          self.remove_active_skill(skill_id)
          # add skill with timestamp to start of skill_list
          self.active_skills.insert(0, [skill_id, time.time()])
  
      def update_context(self, intent):
          """ Updates context with keyword from the intent.
  
          NOTE: This method currently won't handle one_of intent keywords
                since it's not using quite the same format as other intent
                keywords. This is under investigation in adapt, PR pending.
  
          Args:
              intent: Intent to scan for keywords
          """
          for tag in intent['__tags__']:
              if 'entities' not in tag:
                  continue
              context_entity = tag['entities'][0]
              if self.context_greedy:
                  self.context_manager.inject_context(context_entity)
              elif context_entity['data'][0][1] in self.context_keywords:
                  self.context_manager.inject_context(context_entity)
  
      def send_metrics(self, intent, context, stopwatch):
          """
          Send timing metrics to the backend.
  
          NOTE: This only applies to those with Opt In.
          """
          LOG.debug('Sending metric if opt_in is enabled')
          ident = context['ident'] if context else None
          if intent:
              # Recreate skill name from skill id
              parts = intent.get('intent_type', '').split(':')
              intent_type = self.get_skill_name(parts[0])
              if len(parts) > 1:
                  intent_type = ':'.join([intent_type] + parts[1:])
              report_timing(ident, 'intent_service', stopwatch,
                            {'intent_type': intent_type})
          else:
              report_timing(ident, 'intent_service', stopwatch,
                            {'intent_type': 'intent_failure'})
  
      def handle_utterance(self, message):
          """ Main entrypoint for handling user utterances with Mycroft skills
  
          Monitor the messagebus for 'recognizer_loop:utterance', typically
          generated by a spoken interaction but potentially also from a CLI
          or other method of injecting a 'user utterance' into the system.
  
          Utterances then work through this sequence to be handled:
          1) Active skills attempt to handle using converse()
          2) Adapt intent handlers
          3) Padatious intent handlers
          4) Other fallbacks
  
          Args:
              message (Message): The messagebus data
          """
          try:
              # Get language of the utterance
              lang = message.data.get('lang', "en-us")
              utterances = message.data.get('utterances', '')
  
              stopwatch = Stopwatch()
              with stopwatch:
                  # Give active skills an opportunity to handle the utterance
                  converse = self._converse(utterances, lang)
  
                  if not converse:
                      # No conversation, use intent system to handle utterance
                      intent = self._adapt_intent_match(utterances, lang)
  
              if converse:
                  # Report that converse handled the intent and return
                  ident = message.context['ident'] if message.context else None
                  report_timing(ident, 'intent_service', stopwatch,
                                {'intent_type': 'converse'})
                  return
              elif intent:
                  # Send the message to the intent handler
                  reply = message.reply(intent.get('intent_type'), intent)
              else:
                  # Allow fallback system to handle utterance
                  # NOTE: Padatious intents are handled this way, too
                  reply = message.reply('intent_failure',
                                        {'utterance': utterances[0],
                                         'lang': lang})
              self.emitter.emit(reply)
              self.send_metrics(intent, message.context, stopwatch)
          except Exception as e:
              LOG.exception(e)
  
      def _converse(self, utterances, lang):
          """ Give active skills a chance at the utterance
  
          Args:
              utterances (list):  list of utterances
              lang (string):      4 letter ISO language code
  
          Returns:
              bool: True if converse handled it, False if  no skill processes it
          """
  
          # check for conversation time-out
          self.active_skills = [skill for skill in self.active_skills
                                if time.time() - skill[
                                    1] <= self.converse_timeout * 60]
  
          # check if any skill wants to handle utterance
          for skill in self.active_skills:
              if self.do_converse(utterances, skill[0], lang):
                  # update timestamp, or there will be a timeout where
                  # intent stops conversing whether its being used or not
                  self.add_active_skill(skill[0])
                  return True
          return False
  
      def _adapt_intent_match(self, utterances, lang):
          """ Run the Adapt engine to search for an matching intent
  
          Args:
              utterances (list):  list of utterances
              lang (string):      4 letter ISO language code
  
          Returns:
              Intent structure, or None if no match was found.
          """
          best_intent = None
          for utterance in utterances:
              try:
                  # normalize() changes "it's a boy" to "it is boy", etc.
                  best_intent = next(self.engine.determine_intent(
                      normalize(utterance, lang), 100,
                      include_tags=True,
                      context_manager=self.context_manager))
                  # TODO - Should Adapt handle this?
                  best_intent['utterance'] = utterance
              except StopIteration:
                  # don't show error in log
                  continue
              except Exception as e:
                  LOG.exception(e)
                  continue
  
          if best_intent and best_intent.get('confidence', 0.0) > 0.0:
              self.update_context(best_intent)
              # update active skills
              skill_id = best_intent['intent_type'].split(":")[0]
              self.add_active_skill(skill_id)
              return best_intent
  
      def handle_register_vocab(self, message):
          start_concept = message.data.get('start')
          end_concept = message.data.get('end')
          regex_str = message.data.get('regex')
          alias_of = message.data.get('alias_of')
          if regex_str:
              self.engine.register_regex_entity(regex_str)
          else:
              self.engine.register_entity(
                  start_concept, end_concept, alias_of=alias_of)
  
      def handle_register_intent(self, message):
          intent = open_intent_envelope(message)
          self.engine.register_intent_parser(intent)
  
      def handle_detach_intent(self, message):
          intent
  
  [... the rest is omitted ...]
files: mycroft/skills/intent_service.py
files: # Copyright 2017 Mycroft AI Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #    http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  import time
  from adapt.context import ContextManagerFrame
  from adapt.engine import IntentDeterminationEngine
  
  from mycroft.configuration import Configuration
  from mycroft.messagebus.message import Message
  from mycroft.skills.core import open_intent_envelope
  from mycroft.util.log import LOG
  from mycroft.util.parse import normalize
  from mycroft.metrics import report_timing, Stopwatch
  
  
  class ContextManager(object):
      """
      ContextManager
      Use to track context throughout the course of a conversational session.
      How to manage a session's lifecycle is not captured here.
      """
  
      def __init__(self, timeout):
          self.frame_stack = []
          self.timeout = timeout * 60  # minutes to seconds
  
      def clear_context(self):
          self.frame_stack = []
  
      def remove_context(self, context_id):
          self.frame_stack = [(f, t) for (f, t) in self.frame_stack
                              if context_id in f.entities[0].get('data', [])]
  
      def inject_context(self, entity, metadata=None):
          """
          Args:
              entity(object): Format example...
                                 {'data': 'Entity tag as <str>',
                                  'key': 'entity proper name as <str>',
                                  'confidence': <float>'
                                 }
              metadata(object): dict, arbitrary metadata about entity injected
          """
          metadata = metadata or {}
          try:
              if len(self.frame_stack) > 0:
                  top_frame = self.frame_stack[0]
              else:
                  top_frame = None
              if top_frame and top_frame[0].metadata_matches(metadata):
                  top_frame[0].merge_context(entity, metadata)
              else:
                  frame = ContextManagerFrame(entities=[entity],
                                              metadata=metadata.copy())
                  self.frame_stack.insert(0, (frame, time.time()))
          except (IndexError, KeyError):
              pass
  
      def get_context(self, max_frames=None, missing_entities=None):
          """ Constructs a list of entities from the context.
  
          Args:
              max_frames(int): maximum number of frames to look back
              missing_entities(list of str): a list or set of tag names,
              as strings
  
          Returns:
              list: a list of entities
          """
          missing_entities = missing_entities or []
  
          relevant_frames = [frame[0] for frame in self.frame_stack if
                             time.time() - frame[1] < self.timeout]
          if not max_frames or max_frames > len(relevant_frames):
              max_frames = len(relevant_frames)
  
          missing_entities = list(missing_entities)
          context = []
          for i in range(max_frames):
              frame_entities = [entity.copy() for entity in
                                relevant_frames[i].entities]
              for entity in frame_entities:
                  entity['confidence'] = entity.get('confidence', 1.0) \
                      / (2.0 + i)
              context += frame_entities
  
          result = []
          if len(missing_entities) > 0:
              for entity in context:
                  if entity.get('data') in missing_entities:
                      result.append(entity)
                      # NOTE: this implies that we will only ever get one
                      # of an entity kind from context, unless specified
                      # multiple times in missing_entities. Cannot get
                      # an arbitrary number of an entity kind.
                      missing_entities.remove(entity.get('data'))
          else:
              result = context
  
          # Only use the latest instance of each keyword
          stripped = []
          processed = []
          for f in result:
              keyword = f['data'][0][1]
              if keyword not in processed:
                  stripped.append(f)
                  processed.append(keyword)
          result = stripped
          return result
  
  
  class IntentService(object):
      def __init__(self, emitter):
          self.config = Configuration.get().get('context', {})
          self.engine = IntentDeterminationEngine()
  
          # Dictionary for translating a skill id to a name
          self.skill_names = {}
          # Context related intializations
          self.context_keywords = self.config.get('keywords', [])
          self.context_max_frames = self.config.get('max_frames', 3)
          self.context_timeout = self.config.get('timeout', 2)
          self.context_greedy = self.config.get('greedy', False)
          self.context_manager = ContextManager(self.context_timeout)
          self.emitter = emitter
          self.emitter.on('register_vocab', self.handle_register_vocab)
          self.emitter.on('register_intent', self.handle_register_intent)
          self.emitter.on('recognizer_loop:utterance', self.handle_utterance)
          self.emitter.on('detach_intent', self.handle_detach_intent)
          self.emitter.on('detach_skill', self.handle_detach_skill)
          # Context related handlers
          self.emitter.on('add_context', self.handle_add_context)
          self.emitter.on('remove_context', self.handle_remove_context)
          self.emitter.on('clear_context', self.handle_clear_context)
          # Converse method
          self.emitter.on('skill.converse.response',
                          self.handle_converse_response)
          self.emitter.on('mycroft.speech.recognition.unknown',
                          self.reset_converse)
          self.emitter.on('mycroft.skills.loaded', self.update_skill_name_dict)
  
          def add_active_skill_handler(message):
              self.add_active_skill(message.data['skill_id'])
          self.emitter.on('active_skill_request', add_active_skill_handler)
          self.active_skills = []  # [skill_id , timestamp]
          self.converse_timeout = 5  # minutes to prune active_skills
  
      def update_skill_name_dict(self, message):
          """
              Messagebus handler, updates dictionary of if to skill name
              conversions.
          """
          self.skill_names[message.data['id']] = message.data['name']
  
      def get_skill_name(self, skill_id):
          """ Get skill name from skill ID.
  
          Args:
              skill_id: a skill id as encoded in Intent handlers.
  
          Returns:
              (str) Skill name or the skill id if the skill wasn't found
          """
          return self.skill_names.get(int(skill_id), skill_id)
  
      def reset_converse(self, message):
          """Let skills know there was a problem with speech recognition"""
          lang = message.data.get('lang', "en-us")
          for skill in self.active_skills:
              self.do_converse(None, skill[0], lang)
  
      def do_converse(self, utterances, skill_id, lang):
          self.emitter.emit(Message("skill.converse.request", {
              "skill_id": skill_id, "utterances": utterances, "lang": lang}))
          self.waiting = True
          self.result = False
          start_time = time.time()
          t = 0
          while self.waiting and t < 5:
              t = time.time() - start_time
              time.sleep(0.1)
          self.waiting = False
          return self.result
  
      def handle_converse_response(self, message):
          # id = message.data["skill_id"]
          # no need to crosscheck id because waiting before new request is made
          # no other skill will make this request is safe assumption
          result = message.data["result"]
          self.result = result
          self.waiting = False
  
      def remove_active_skill(self, skill_id):
          for skill in self.active_skills:
              if skill[0] == skill_id:
                  self.active_skills.remove(skill)
  
      def add_active_skill(self, skill_id):
          # search the list for an existing entry that already contains it
          # and remove that reference
          self.remove_active_skill(skill_id)
          # add skill with timestamp to start of skill_list
          self.active_skills.insert(0, [skill_id, time.time()])
  
      def update_context(self, intent):
          """ Updates context with keyword from the intent.
  
          NOTE: This method currently won't handle one_of intent keywords
                since it's not using quite the same format as other intent
                keywords. This is under investigation in adapt, PR pending.
  
          Args:
              intent: Intent to scan for keywords
          """
          for tag in intent['__tags__']:
              if 'entities' not in tag:
                  continue
              context_entity = tag['entities'][0]
              if self.context_greedy:
                  self.context_manager.inject_context(context_entity)
              elif context_entity['data'][0][1] in self.context_keywords:
                  self.context_manager.inject_context(context_entity)
  
      def send_metrics(self, intent, context, stopwatch):
          """
          Send timing metrics to the backend.
  
          NOTE: This only applies to those with Opt In.
          """
          LOG.debug('Sending metric if opt_in is enabled')
          ident = context['ident'] if context else None
          if intent:
              # Recreate skill name from skill id
              parts = intent.get('intent_type', '').split(':')
              intent_type = self.get_skill_name(parts[0])
              if len(parts) > 1:
                  intent_type = ':'.join([intent_type] + parts[1:])
              report_timing(ident, 'intent_service', stopwatch,
                            {'intent_type': intent_type})
          else:
              report_timing(ident, 'intent_service', stopwatch,
                            {'intent_type': 'intent_failure'})
  
      def handle_utterance(self, message):
          """ Main entrypoint for handling user utterances with Mycroft skills
  
          Monitor the messagebus for 'recognizer_loop:utterance', typically
          generated by a spoken interaction but potentially also from a CLI
          or other method of injecting a 'user utterance' into the system.
  
          Utterances then work through this sequence to be handled:
          1) Active skills attempt to handle using converse()
          2) Adapt intent handlers
          3) Padatious intent handlers
          4) Other fallbacks
  
          Args:
              message (Message): The messagebus data
          """
          try:
              # Get language of the utterance
              lang = message.data.get('lang', "en-us")
              utterances = message.data.get('utterances', '')
  
              stopwatch = Stopwatch()
              with stopwatch:
                  # Give active skills an opportunity to handle the utterance
                  converse = self._converse(utterances, lang)
  
                  if not converse:
                      # No conversation, use intent system to handle utterance
                      intent = self._adapt_intent_match(utterances, lang)
  
              if converse:
                  # Report that converse handled the intent and return
                  ident = message.context['ident'] if message.context else None
                  report_timing(ident, 'intent_service', stopwatch,
                                {'intent_type': 'converse'})
                  return
              elif intent:
                  # Send the message to the intent handler
                  reply = message.reply(intent.get('intent_type'), intent)
              else:
                  # Allow fallback system to handle utterance
                  # NOTE: Padatious intents are handled this way, too
                  reply = message.reply('intent_failure',
                                        {'utterance': utterances[0],
                                         'lang': lang})
              self.emitter.emit(reply)
              self.send_metrics(intent, message.context, stopwatch)
          except Exception as e:
              LOG.exception(e)
  
      def _converse(self, utterances, lang):
          """ Give active skills a chance at the utterance
  
          Args:
              utterances (list):  list of utterances
              lang (string):      4 letter ISO language code
  
          Returns:
              bool: True if converse handled it, False if  no skill processes it
          """
  
          # check for conversation time-out
          self.active_skills = [skill for skill in self.active_skills
                                if time.time() - skill[
                                    1] <= self.converse_timeout * 60]
  
          # check if any skill wants to handle utterance
          for skill in self.active_skills:
              if self.do_converse(utterances, skill[0], lang):
                  # update timestamp, or there will be a timeout where
                  # intent stops conversing whether its being used or not
                  self.add_active_skill(skill[0])
                  return True
          return False
  
      def _adapt_intent_match(self, utterances, lang):
          """ Run the Adapt engine to search for an matching intent
  
          Args:
              utterances (list):  list of utterances
              lang (string):      4 letter ISO language code
  
          Returns:
              Intent structure, or None if no match was found.
          """
          best_intent = None
          for utterance in utterances:
              try:
                  # normalize() changes "it's a boy" to "it is boy", etc.
                  best_intent = next(self.engine.determine_intent(
                      normalize(utterance, lang), 100,
                      include_tags=True,
                      context_manager=self.context_manager))
                  # TODO - Should Adapt handle this?
                  best_intent['utterance'] = utterance
              except StopIteration:
                  # don't show error in log
                  continue
              except Exception as e:
                  LOG.exception(e)
                  continue
  
          if best_intent and best_intent.get('confidence', 0.0) > 0.0:
              self.update_context(best_intent)
              # update active skills
              skill_id = int(best_intent['intent_type'].split(":")[0])
              self.add_active_skill(skill_id)
              return best_intent
  
      def handle_register_vocab(self, message):
          start_concept = message.data.get('start')
          end_concept = message.data.get('end')
          regex_str = message.data.get('regex')
          alias_of = message.data.get('alias_of')
          if regex_str:
              self.engine.register_regex_entity(regex_str)
          else:
              self.engine.register_entity(
                  start_concept, end_concept, alias_of=alias_of)
  
      def handle_register_intent(self, message):
          intent = open_intent_envelope(message)
          self.engine.register_intent_parser(intent)
  
      def handle_detach_intent(self, message
  
  [... the rest is omitted ...]
files: mycroft/skills/intent_service.py
files: # Copyright 2017 Mycroft AI Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #    http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  import gc
  import os
  import sys
  import time
  from itertools import chain
  from threading import Timer, Thread, Event
  
  import monotonic
  from glob import glob
  from os.path import exists, join, basename, dirname, expanduser, isfile
  from msm import MycroftSkillsManager, SkillRepo
  
  import mycroft.lock
  from mycroft import dialog
  from mycroft.api import is_paired, BackendDown
  from mycroft.client.enclosure.api import EnclosureAPI
  from mycroft.configuration import Configuration
  from mycroft.messagebus.client.ws import WebsocketClient
  from mycroft.messagebus.message import Message
  from mycroft.skills.core import load_skill, create_skill_descriptor, \
      MainModule, FallbackSkill
  from mycroft.skills.event_scheduler import EventScheduler
  from mycroft.skills.intent_service import IntentService
  from mycroft.skills.padatious_service import PadatiousService
  from mycroft.util import (
      connected, wait_while_speaking, reset_sigint_handler,
      create_echo_function, create_daemon, wait_for_exit_signal
  )
  from mycroft.util.log import LOG
  
  ws = None
  event_scheduler = None
  skill_manager = None
  
  # Remember "now" at startup.  Used to detect clock changes.
  start_ticks = monotonic.monotonic()
  start_clock = time.time()
  
  DEBUG = Configuration.get().get("debug", False)
  skills_config = Configuration.get().get("skills")
  BLACKLISTED_SKILLS = skills_config.get("blacklisted_skills", [])
  PRIORITY_SKILLS = skills_config.get("priority_skills", [])
  SKILLS_DIR = '/opt/mycroft/skills'
  
  installer_config = Configuration.get().get("SkillInstallerSkill")
  
  MINUTES = 60  # number of seconds in a minute (syntatic sugar)
  
  
  def connect():
      global ws
      ws.run_forever()
  
  
  def _starting_up():
      """
          Start loading skills.
  
          Starts
          - SkillManager to load/reloading of skills when needed
          - a timer to check for internet connection
          - adapt intent service
          - padatious intent service
      """
      global ws, skill_manager, event_scheduler
  
      ws.on('intent_failure', FallbackSkill.make_intent_failure_handler(ws))
  
      # Create the Intent manager, which converts utterances to intents
      # This is the heart of the voice invoked skill system
  
      service = IntentService(ws)
      PadatiousService(ws, service)
      event_scheduler = EventScheduler(ws)
  
      # Create a thread that monitors the loaded skills, looking for updates
      skill_manager = SkillManager(ws)
      skill_manager.daemon = True
      # Wait until skills have been loaded once before starting to check
      # network connection
      skill_manager.load_priority()
      skill_manager.start()
      check_connection()
  
  
  def check_connection():
      """
          Check for network connection. If not paired trigger pairing.
          Runs as a Timer every second until connection is detected.
      """
      if connected():
          enclosure = EnclosureAPI(ws)
  
          if is_paired():
              # Skip the sync message when unpaired because the prompt to go to
              # home.mycrof.ai will be displayed by the pairing skill
              enclosure.mouth_text(dialog.get("message_synching.clock"))
  
          # Force a sync of the local clock with the internet
          config = Configuration.get()
          platform = config['enclosure'].get("platform", "unknown")
          if platform in ['mycroft_mark_1', 'picroft']:
              ws.emit(Message("system.ntp.sync"))
              time.sleep(15)  # TODO: Generate/listen for a message response...
  
          # Check if the time skewed significantly.  If so, reboot
          skew = abs((monotonic.monotonic() - start_ticks) -
                     (time.time() - start_clock))
          if skew > 60 * 60:
              # Time moved by over an hour in the NTP sync. Force a reboot to
              # prevent weird things from occcurring due to the 'time warp'.
              #
              data = {'utterance': dialog.get("time.changed.reboot")}
              ws.emit(Message("speak", data))
              wait_while_speaking()
  
              # provide visual indicators of the reboot
              enclosure.mouth_text(dialog.get("message_rebooting"))
              enclosure.eyes_color(70, 65, 69)  # soft gray
              enclosure.eyes_spin()
  
              # give the system time to finish processing enclosure messages
              time.sleep(1.0)
  
              # reboot
              ws.emit(Message("system.reboot"))
              return
          else:
              ws.emit(Message("enclosure.mouth.reset"))
              time.sleep(0.5)
  
          ws.emit(Message('mycroft.internet.connected'))
          # check for pairing, if not automatically start pairing
          try:
              if not is_paired(ignore_errors=False):
                  payload = {
                      'utterances': ["pair my device"],
                      'lang': "en-us"
                  }
                  ws.emit(Message("recognizer_loop:utterance", payload))
              else:
                  from mycroft.api import DeviceApi
                  api = DeviceApi()
                  api.update_version()
          except BackendDown:
              data = {'utterance': dialog.get("backend.down")}
              ws.emit(Message("speak", data))
              ws.emit(Message("backend.down"))
  
      else:
          thread = Timer(1, check_connection)
          thread.daemon = True
          thread.start()
  
  
  def _get_last_modified_date(path):
      """
          Get last modified date excluding compiled python files, hidden
          directories and the settings.json file.
  
          Args:
              path:   skill directory to check
  
          Returns:
              int: time of last change
      """
      last_date = 0
      root_dir, subdirs, files = next(os.walk(path))
      # check files of interest in the skill root directory
      files = [f for f in files
               if not f.endswith('.pyc') and f != 'settings.json' and
               not f.startswith('.')]
      for f in files:
          last_date = max(last_date, os.path.getmtime(os.path.join(path, f)))
      return last_date
  
  
  class SkillManager(Thread):
      """ Load, update and manage instances of Skill on this system. """
  
      def __init__(self, ws):
          super(SkillManager, self).__init__()
          self._stop_event = Event()
          self._connected_event = Event()
  
          self.loaded_skills = {}
          self.msm_blocked = False
          self.ws = ws
          self.enclosure = EnclosureAPI(ws)
  
          # Schedule install/update of default skill
          self.dot_msm = join(SKILLS_DIR, '.msm')
          if exists(self.dot_msm):
              self.next_download = os.path.getmtime(self.dot_msm) + 60 * MINUTES
          else:
              self.next_download = time.time() - 1
  
          # Conversation management
          ws.on('skill.converse.request', self.handle_converse_request)
  
          # Update on initial connection
          ws.on('mycroft.internet.connected',
                lambda x: self._connected_event.set())
  
          # Update upon request
          ws.on('skillmanager.update', self.schedule_now)
          ws.on('skillmanager.list', self.send_skill_list)
  
          self.msm = self.create_msm()
  
      @staticmethod
      def create_msm():
          config = Configuration.get()
          msm_config = config['skills']['msm']
          repo_config = msm_config['repo']
          platform = config['enclosure'].get('platform', 'default')
          return MycroftSkillsManager(
              platform=platform, skills_dir=msm_config['directory'],
              repo=SkillRepo(
                  repo_config['cache'], repo_config['url'], repo_config['branch']
              ), versioned=msm_config['versioned']
          )
  
      def schedule_now(self, message=None):
          self.next_download = time.time() - 1
  
      @property
      def installed_skills_file(self):
          venv = dirname(dirname(sys.executable))
          if os.access(venv, os.W_OK | os.R_OK | os.X_OK):
              return join(venv, '.mycroft-skills')
          return expanduser('~/.mycroft/.mycroft-skills')
  
      def load_installed_skills(self) -> set:
          skills_file = self.installed_skills_file
          if not isfile(skills_file):
              return set()
          with open(skills_file) as f:
              return {
                  i.strip() for i in f.read().split('\n') if i.strip()
              }
  
      def save_installed_skills(self, skill_names):
          with open(self.installed_skills_file, 'w') as f:
              f.write('\n'.join(skill_names))
  
      def download_skills(self, speak=False):
          """ Invoke MSM to install default skills and/or update installed skills
  
              Args:
                  speak (bool, optional): Speak the result? Defaults to False
          """
          if not connected():
              LOG.error('msm failed, network connection not available')
              if speak:
                  self.ws.emit(Message("speak", {
                      'utterance': dialog.get(
                          "not connected to the internet")}))
              self.next_download = time.time() + 5 * MINUTES
              return False
  
          installed_skills = self.load_installed_skills()
          default_groups = dict(self.msm.repo.get_default_skill_names())
          default_names = set(chain(default_groups['default'],
                                    default_groups[self.msm.platform]))
  
          def install_or_update(skill):
              """Install missing defaults and update existing skills"""
              if skill.is_local:
                  skill.update()
                  if skill.name not in installed_skills:
                      skill.update_deps()
                      installed_skills.add(skill.name)
              elif skill.name in default_names:
                  skill.install()
                  installed_skills.add(skill.name)
  
          self.msm.apply(install_or_update, self.msm.list())
          self.save_installed_skills(installed_skills)
  
          with open(self.dot_msm, 'a'):
              os.utime(self.dot_msm, None)
          self.next_download = time.time() + 60 * MINUTES
  
          if speak:
              data = {'utterance': dialog.get("skills updated")}
              self.ws.emit(Message("speak", data))
          return True
  
      def _load_or_reload_skill(self, skill_path):
          """
              Check if unloaded skill or changed skill needs reloading
              and perform loading if necessary.
  
              Returns True if the skill was loaded/reloaded
          """
          skill_path = skill_path.rstrip('/')
          skill = self.loaded_skills.setdefault(skill_path, {})
          skill.update({
              "id": basename(skill_path),
              "path": skill_path
          })
  
          # check if folder is a skill (must have __init__.py)
          if not MainModule + ".py" in os.listdir(skill_path):
              return False
  
          # getting the newest modified date of skill
          modified = _get_last_modified_date(skill_path)
          last_mod = skill.get("last_modified", 0)
  
          # checking if skill is loaded and hasn't been modified on disk
          if skill.get("loaded") and modified <= last_mod:
              return False  # Nothing to do!
  
          # check if skill was modified
          elif skill.get("instance") and modified > last_mod:
              # check if skill has been blocked from reloading
              if not skill["instance"].reload_skill:
                  return False
  
              LOG.debug("Reloading Skill: " + basename(skill_path))
              # removing listeners and stopping threads
              try:
                  skill["instance"]._shutdown()
              except Exception:
                  LOG.exception("An error occured while shutting down {}"
                                .format(skill["instance"].name))
  
              if DEBUG:
                  gc.collect()  # Collect garbage to remove false references
                  # Remove two local references that are known
                  refs = sys.getrefcount(skill["instance"]) - 2
                  if refs > 0:
                      msg = ("After shutdown of {} there are still "
                             "{} references remaining. The skill "
                             "won't be cleaned from memory.")
                      LOG.warning(msg.format(skill['instance'].name, refs))
              del skill["instance"]
              self.ws.emit(Message("mycroft.skills.shutdown",
                                   {"path": skill_path,
                                    "id": skill["id"]}))
  
          skill["loaded"] = True
          desc = create_skill_descriptor(skill_path)
          skill["instance"] = load_skill(desc,
                                         self.ws, skill["id"],
                                         BLACKLISTED_SKILLS)
          skill["last_modified"] = modified
          if skill['instance'] is not None:
              self.ws.emit(Message('mycroft.skills.loaded',
                                   {'path': skill_path,
                                    'id': skill['id'],
                                    'name': skill['instance'].name,
                                    'modified': modified}))
              return True
          else:
              self.ws.emit(Message('mycroft.skills.loading_failure',
                                   {'path': skill_path,
                                    'id': skill['id']}))
          return False
  
      def load_priority(self):
          skills = {skill.name: skill for skill in self.msm.list()}
          for skill_name in PRIORITY_SKILLS:
              skill = skills[skill_name]
              if not skill.is_local:
                  try:
                      skill.install()
                  except Exception:
                      LOG.exception('Downloading priority skill:' + skill.name)
                      if not skill.is_local:
                          continue
              self._load_or_reload_skill(skill.path)
  
      def run(self):
          """ Load skills and update periodically from disk and internet """
  
         
  
  [... the rest is omitted ...]
files: mycroft/skills/main.py
files: # Copyright 2017 Mycroft AI Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #    http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  import gc
  import os
  import sys
  import time
  from itertools import chain
  from threading import Timer, Thread, Event
  
  import monotonic
  from glob import glob
  from os.path import exists, join, basename, dirname, expanduser, isfile
  from msm import MycroftSkillsManager, SkillRepo
  
  import mycroft.lock
  from mycroft import dialog
  from mycroft.api import is_paired, BackendDown
  from mycroft.client.enclosure.api import EnclosureAPI
  from mycroft.configuration import Configuration
  from mycroft.messagebus.client.ws import WebsocketClient
  from mycroft.messagebus.message import Message
  from mycroft.skills.core import load_skill, create_skill_descriptor, \
      MainModule, FallbackSkill
  from mycroft.skills.event_scheduler import EventScheduler
  from mycroft.skills.intent_service import IntentService
  from mycroft.skills.padatious_service import PadatiousService
  from mycroft.util import (
      connected, wait_while_speaking, reset_sigint_handler,
      create_echo_function, create_daemon, wait_for_exit_signal
  )
  from mycroft.util.log import LOG
  
  ws = None
  event_scheduler = None
  skill_manager = None
  
  # Remember "now" at startup.  Used to detect clock changes.
  start_ticks = monotonic.monotonic()
  start_clock = time.time()
  
  DEBUG = Configuration.get().get("debug", False)
  skills_config = Configuration.get().get("skills")
  BLACKLISTED_SKILLS = skills_config.get("blacklisted_skills", [])
  PRIORITY_SKILLS = skills_config.get("priority_skills", [])
  SKILLS_DIR = '/opt/mycroft/skills'
  
  installer_config = Configuration.get().get("SkillInstallerSkill")
  
  MINUTES = 60  # number of seconds in a minute (syntatic sugar)
  
  
  def connect():
      global ws
      ws.run_forever()
  
  
  def _starting_up():
      """
          Start loading skills.
  
          Starts
          - SkillManager to load/reloading of skills when needed
          - a timer to check for internet connection
          - adapt intent service
          - padatious intent service
      """
      global ws, skill_manager, event_scheduler
  
      ws.on('intent_failure', FallbackSkill.make_intent_failure_handler(ws))
  
      # Create the Intent manager, which converts utterances to intents
      # This is the heart of the voice invoked skill system
  
      service = IntentService(ws)
      PadatiousService(ws, service)
      event_scheduler = EventScheduler(ws)
  
      # Create a thread that monitors the loaded skills, looking for updates
      skill_manager = SkillManager(ws)
      skill_manager.daemon = True
      # Wait until skills have been loaded once before starting to check
      # network connection
      skill_manager.load_priority()
      skill_manager.start()
      check_connection()
  
  
  def check_connection():
      """
          Check for network connection. If not paired trigger pairing.
          Runs as a Timer every second until connection is detected.
      """
      if connected():
          enclosure = EnclosureAPI(ws)
  
          if is_paired():
              # Skip the sync message when unpaired because the prompt to go to
              # home.mycrof.ai will be displayed by the pairing skill
              enclosure.mouth_text(dialog.get("message_synching.clock"))
  
          # Force a sync of the local clock with the internet
          config = Configuration.get()
          platform = config['enclosure'].get("platform", "unknown")
          if platform in ['mycroft_mark_1', 'picroft']:
              ws.emit(Message("system.ntp.sync"))
              time.sleep(15)  # TODO: Generate/listen for a message response...
  
          # Check if the time skewed significantly.  If so, reboot
          skew = abs((monotonic.monotonic() - start_ticks) -
                     (time.time() - start_clock))
          if skew > 60 * 60:
              # Time moved by over an hour in the NTP sync. Force a reboot to
              # prevent weird things from occcurring due to the 'time warp'.
              #
              data = {'utterance': dialog.get("time.changed.reboot")}
              ws.emit(Message("speak", data))
              wait_while_speaking()
  
              # provide visual indicators of the reboot
              enclosure.mouth_text(dialog.get("message_rebooting"))
              enclosure.eyes_color(70, 65, 69)  # soft gray
              enclosure.eyes_spin()
  
              # give the system time to finish processing enclosure messages
              time.sleep(1.0)
  
              # reboot
              ws.emit(Message("system.reboot"))
              return
          else:
              ws.emit(Message("enclosure.mouth.reset"))
              time.sleep(0.5)
  
          ws.emit(Message('mycroft.internet.connected'))
          # check for pairing, if not automatically start pairing
          try:
              if not is_paired(ignore_errors=False):
                  payload = {
                      'utterances': ["pair my device"],
                      'lang': "en-us"
                  }
                  ws.emit(Message("recognizer_loop:utterance", payload))
              else:
                  from mycroft.api import DeviceApi
                  api = DeviceApi()
                  api.update_version()
          except BackendDown:
              data = {'utterance': dialog.get("backend.down")}
              ws.emit(Message("speak", data))
              ws.emit(Message("backend.down"))
  
      else:
          thread = Timer(1, check_connection)
          thread.daemon = True
          thread.start()
  
  
  def _get_last_modified_date(path):
      """
          Get last modified date excluding compiled python files, hidden
          directories and the settings.json file.
  
          Args:
              path:   skill directory to check
  
          Returns:
              int: time of last change
      """
      last_date = 0
      root_dir, subdirs, files = next(os.walk(path))
      # check files of interest in the skill root directory
      files = [f for f in files
               if not f.endswith('.pyc') and f != 'settings.json' and
               not f.startswith('.')]
      for f in files:
          last_date = max(last_date, os.path.getmtime(os.path.join(path, f)))
      return last_date
  
  
  class SkillManager(Thread):
      """ Load, update and manage instances of Skill on this system. """
  
      def __init__(self, ws):
          super(SkillManager, self).__init__()
          self._stop_event = Event()
          self._connected_event = Event()
  
          self.loaded_skills = {}
          self.msm_blocked = False
          self.ws = ws
          self.enclosure = EnclosureAPI(ws)
  
          # Schedule install/update of default skill
          self.dot_msm = join(SKILLS_DIR, '.msm')
          if exists(self.dot_msm):
              self.next_download = os.path.getmtime(self.dot_msm) + 60 * MINUTES
          else:
              self.next_download = time.time() - 1
  
          # Conversation management
          ws.on('skill.converse.request', self.handle_converse_request)
  
          # Update on initial connection
          ws.on('mycroft.internet.connected',
                lambda x: self._connected_event.set())
  
          # Update upon request
          ws.on('skillmanager.update', self.schedule_now)
          ws.on('skillmanager.list', self.send_skill_list)
  
          self.msm = self.create_msm()
  
      @staticmethod
      def create_msm():
          config = Configuration.get()
          msm_config = config['skills']['msm']
          repo_config = msm_config['repo']
          platform = config['enclosure'].get('platform', 'default')
          return MycroftSkillsManager(
              platform=platform, skills_dir=msm_config['directory'],
              repo=SkillRepo(
                  repo_config['cache'], repo_config['url'], repo_config['branch']
              ), versioned=msm_config['versioned']
          )
  
      def schedule_now(self, message=None):
          self.next_download = time.time() - 1
  
      @property
      def installed_skills_file(self):
          venv = dirname(dirname(sys.executable))
          if os.access(venv, os.W_OK | os.R_OK | os.X_OK):
              return join(venv, '.mycroft-skills')
          return expanduser('~/.mycroft/.mycroft-skills')
  
      def load_installed_skills(self) -> set:
          skills_file = self.installed_skills_file
          if not isfile(skills_file):
              return set()
          with open(skills_file) as f:
              return {
                  i.strip() for i in f.read().split('\n') if i.strip()
              }
  
      def save_installed_skills(self, skill_names):
          with open(self.installed_skills_file, 'w') as f:
              f.write('\n'.join(skill_names))
  
      def download_skills(self, speak=False):
          """ Invoke MSM to install default skills and/or update installed skills
  
              Args:
                  speak (bool, optional): Speak the result? Defaults to False
          """
          if not connected():
              LOG.error('msm failed, network connection not available')
              if speak:
                  self.ws.emit(Message("speak", {
                      'utterance': dialog.get(
                          "not connected to the internet")}))
              self.next_download = time.time() + 5 * MINUTES
              return False
  
          installed_skills = self.load_installed_skills()
          default_groups = dict(self.msm.repo.get_default_skill_names())
          default_names = set(chain(default_groups['default'],
                                    default_groups[self.msm.platform]))
  
          def install_or_update(skill):
              """Install missing defaults and update existing skills"""
              if skill.is_local:
                  skill.update()
                  if skill.name not in installed_skills:
                      skill.update_deps()
                      installed_skills.add(skill.name)
              elif skill.name in default_names:
                  skill.install()
                  installed_skills.add(skill.name)
  
          self.msm.apply(install_or_update, self.msm.list())
          self.save_installed_skills(installed_skills)
  
          with open(self.dot_msm, 'a'):
              os.utime(self.dot_msm, None)
          self.next_download = time.time() + 60 * MINUTES
  
          if speak:
              data = {'utterance': dialog.get("skills updated")}
              self.ws.emit(Message("speak", data))
          return True
  
      def _load_or_reload_skill(self, skill_path):
          """
              Check if unloaded skill or changed skill needs reloading
              and perform loading if necessary.
  
              Returns True if the skill was loaded/reloaded
          """
          skill_path = skill_path.rstrip('/')
          skill = self.loaded_skills.setdefault(skill_path, {})
          skill.update({
              "id": hash(skill_path),
              "path": skill_path
          })
  
          # check if folder is a skill (must have __init__.py)
          if not MainModule + ".py" in os.listdir(skill_path):
              return False
  
          # getting the newest modified date of skill
          modified = _get_last_modified_date(skill_path)
          last_mod = skill.get("last_modified", 0)
  
          # checking if skill is loaded and hasn't been modified on disk
          if skill.get("loaded") and modified <= last_mod:
              return False  # Nothing to do!
  
          # check if skill was modified
          elif skill.get("instance") and modified > last_mod:
              # check if skill has been blocked from reloading
              if not skill["instance"].reload_skill:
                  return False
  
              LOG.debug("Reloading Skill: " + basename(skill_path))
              # removing listeners and stopping threads
              try:
                  skill["instance"]._shutdown()
              except Exception:
                  LOG.exception("An error occured while shutting down {}"
                                .format(skill["instance"].name))
  
              if DEBUG:
                  gc.collect()  # Collect garbage to remove false references
                  # Remove two local references that are known
                  refs = sys.getrefcount(skill["instance"]) - 2
                  if refs > 0:
                      msg = ("After shutdown of {} there are still "
                             "{} references remaining. The skill "
                             "won't be cleaned from memory.")
                      LOG.warning(msg.format(skill['instance'].name, refs))
              del skill["instance"]
              self.ws.emit(Message("mycroft.skills.shutdown",
                                   {"path": skill_path,
                                    "id": skill["id"]}))
  
          skill["loaded"] = True
          desc = create_skill_descriptor(skill_path)
          skill["instance"] = load_skill(desc,
                                         self.ws, skill["id"],
                                         BLACKLISTED_SKILLS)
          skill["last_modified"] = modified
          if skill['instance'] is not None:
              self.ws.emit(Message('mycroft.skills.loaded',
                                   {'path': skill_path,
                                    'id': skill['id'],
                                    'name': skill['instance'].name,
                                    'modified': modified}))
              return True
          else:
              self.ws.emit(Message('mycroft.skills.loading_failure',
                                   {'path': skill_path,
                                    'id': skill['id']}))
          return False
  
      def load_priority(self):
          skills = {skill.name: skill for skill in self.msm.list()}
          for skill_name in PRIORITY_SKILLS:
              skill = skills[skill_name]
              if not skill.is_local:
                  try:
                      skill.install()
                  except Exception:
                      LOG.exception('Downloading priority skill:' + skill.name)
                      if not skill.is_local:
                          continue
              self._load_or_reload_skill(skill.path)
  
      def run(self):
          """ Load skills and update periodically from disk and internet """
  
          #
  
  [... the rest is omitted ...]
files: mycroft/skills/main.py
files: # Copyright 2018 Mycroft AI Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #    http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  
  """Module containing methods needed to load skill
  data such as dialogs, intents and regular expressions.
  """
  
  from os import listdir
  from os.path import splitext, join
  import re
  
  from mycroft.messagebus.message import Message
  
  
  def load_vocab_from_file(path, vocab_type, emitter):
      """Load Mycroft vocabulary from file
      The vocab is sent to the intent handler using the message bus
  
      Args:
          path:           path to vocabulary file (*.voc)
          vocab_type:     keyword name
          emitter:        emitter to access the message bus
          skill_id(str):  skill id
      """
      if path.endswith('.voc'):
          with open(path, 'r') as voc_file:
              for line in voc_file.readlines():
                  if line.startswith("#"):
                      continue
                  parts = line.strip().split("|")
                  entity = parts[0]
                  emitter.emit(Message("register_vocab", {
                      'start': entity, 'end': vocab_type
                  }))
                  for alias in parts[1:]:
                      emitter.emit(Message("register_vocab", {
                          'start': alias, 'end': vocab_type, 'alias_of': entity
                      }))
  
  
  def load_regex_from_file(path, emitter, skill_id):
      """Load regex from file
      The regex is sent to the intent handler using the message bus
  
      Args:
          path:       path to vocabulary file (*.voc)
          emitter:    emitter to access the message bus
      """
      if path.endswith('.rx'):
          with open(path, 'r') as reg_file:
              for line in reg_file.readlines():
                  if line.startswith("#"):
                      continue
                  re.compile(munge_regex(line.strip(), skill_id))
                  emitter.emit(
                      Message("register_vocab",
                              {'regex': munge_regex(line.strip(), skill_id)}))
  
  
  def load_vocabulary(basedir, emitter, skill_id):
      """Load vocabulary from all files in the specified directory.
  
      Args:
          basedir (str): path of directory to load from
          emitter (messagebus emitter): websocket used to send the vocab to
                                        the intent service
          skill_id: skill the data belongs to
      """
      for vocab_file in listdir(basedir):
          if vocab_file.endswith(".voc"):
              vocab_type = to_alnum(skill_id) + splitext(vocab_file)[0]
              load_vocab_from_file(
                  join(basedir, vocab_file), vocab_type, emitter)
  
  
  def load_regex(basedir, emitter, skill_id):
      """Load regex from all files in the specified directory.
  
      Args:
          basedir (str): path of directory to load from
          emitter (messagebus emitter): websocket used to send the vocab to
                                        the intent service
          skill_id (str): skill identifier
      """
      for regex_type in listdir(basedir):
          if regex_type.endswith(".rx"):
              load_regex_from_file(join(basedir, regex_type), emitter, skill_id)
  
  
  def to_alnum(skill_id):
      """Convert a skill id to only alphanumeric characters
  
       Non alpha-numeric characters are converted to "_"
  
      Args:
          skill_id (str): identifier to be converted
      Returns:
          (str) String of letters
      """
      return ''.join(c if c.isalnum() else '_' for c in str(skill_id))
  
  
  def munge_regex(regex, skill_id):
      """Insert skill id as letters into match groups.
  
      Args:
          regex (str): regex string
          skill_id (str): skill identifier
      Returns:
          (str) munged regex
      """
      base = '(?P<' + to_alnum(skill_id)
      return base.join(regex.split('(?P<'))
  
  
  def munge_intent_parser(intent_parser, name, skill_id):
      """Rename intent keywords to make them skill exclusive
      This gives the intent parser an exclusive name in the
      format <skill_id>:<name>.  The keywords are given unique
      names in the format <Skill id as letters><Intent name>.
  
      The function will not munge instances that's already been
      munged
  
      Args:
          intent_parser: (IntentParser) object to update
          name: (str) Skill name
          skill_id: (int) skill identifier
      """
      # Munge parser name
      if str(skill_id) + ':' not in name:
          intent_parser.name = str(skill_id) + ':' + name
      else:
          intent_parser.name = name
  
      # Munge keywords
      skill_id = to_alnum(skill_id)
      # Munge required keyword
      reqs = []
      for i in intent_parser.requires:
          if skill_id not in i[0]:
              kw = (skill_id + i[0], skill_id + i[0])
              reqs.append(kw)
          else:
              reqs.append(i)
      intent_parser.requires = reqs
  
      # Munge optional keywords
      opts = []
      for i in intent_parser.optional:
          if skill_id not in i[0]:
              kw = (skill_id + i[0], skill_id + i[0])
              opts.append(kw)
          else:
              opts.append(i)
      intent_parser.optional = opts
  
      # Munge at_least_one keywords
      at_least_one = []
      for i in intent_parser.at_least_one:
          element = [skill_id + e.replace(skill_id, '') for e in i]
          at_least_one.append(tuple(element))
      intent_parser.at_least_one = at_least_one
  
  
  [... the rest is omitted ...]
files: mycroft/skills/skill_data.py
files: # Copyright 2018 Mycroft AI Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #    http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  
  """Module containing methods needed to load skill
  data such as dialogs, intents and regular expressions.
  """
  
  from os import listdir
  from os.path import splitext, join
  import re
  
  from mycroft.messagebus.message import Message
  
  
  def load_vocab_from_file(path, vocab_type, emitter):
      """Load Mycroft vocabulary from file
      The vocab is sent to the intent handler using the message bus
  
      Args:
          path:           path to vocabulary file (*.voc)
          vocab_type:     keyword name
          emitter:        emitter to access the message bus
          skill_id(str):  skill id
      """
      if path.endswith('.voc'):
          with open(path, 'r') as voc_file:
              for line in voc_file.readlines():
                  if line.startswith("#"):
                      continue
                  parts = line.strip().split("|")
                  entity = parts[0]
                  emitter.emit(Message("register_vocab", {
                      'start': entity, 'end': vocab_type
                  }))
                  for alias in parts[1:]:
                      emitter.emit(Message("register_vocab", {
                          'start': alias, 'end': vocab_type, 'alias_of': entity
                      }))
  
  
  def load_regex_from_file(path, emitter, skill_id):
      """Load regex from file
      The regex is sent to the intent handler using the message bus
  
      Args:
          path:       path to vocabulary file (*.voc)
          emitter:    emitter to access the message bus
      """
      if path.endswith('.rx'):
          with open(path, 'r') as reg_file:
              for line in reg_file.readlines():
                  if line.startswith("#"):
                      continue
                  re.compile(munge_regex(line.strip(), skill_id))
                  emitter.emit(
                      Message("register_vocab",
                              {'regex': munge_regex(line.strip(), skill_id)}))
  
  
  def load_vocabulary(basedir, emitter, skill_id):
      """Load vocabulary from all files in the specified directory.
  
      Args:
          basedir (str): path of directory to load from
          emitter (messagebus emitter): websocket used to send the vocab to
                                        the intent service
          skill_id: skill the data belongs to
      """
      for vocab_file in listdir(basedir):
          if vocab_file.endswith(".voc"):
              vocab_type = to_letters(skill_id) + splitext(vocab_file)[0]
              load_vocab_from_file(
                  join(basedir, vocab_file), vocab_type, emitter)
  
  
  def load_regex(basedir, emitter, skill_id):
      """Load regex from all files in the specified directory.
  
      Args:
          basedir (str): path of directory to load from
          emitter (messagebus emitter): websocket used to send the vocab to
                                        the intent service
          skill_id (int): skill identifier
      """
      for regex_type in listdir(basedir):
          if regex_type.endswith(".rx"):
              load_regex_from_file(
                  join(basedir, regex_type), emitter, skill_id)
  
  
  def to_letters(number):
      """Convert number to string of letters.
  
      0 -> A, 1 -> B, etc.
  
      Args:
          number (int): number to be converted
      Returns:
          (str) String of letters
      """
      ret = ''
      for n in str(number).strip('-'):
          ret += chr(65 + int(n))
      return ret
  
  
  def munge_regex(regex, skill_id):
      """Insert skill id as letters into match groups.
  
      Args:
          regex (str): regex string
          skill_id (int): skill identifier
      Returns:
          (str) munged regex
      """
      base = '(?P<' + to_letters(skill_id)
      return base.join(regex.split('(?P<'))
  
  
  def munge_intent_parser(intent_parser, name, skill_id):
      """Rename intent keywords to make them skill exclusive
      This gives the intent parser an exclusive name in the
      format <skill_id>:<name>.  The keywords are given unique
      names in the format <Skill id as letters><Intent name>.
  
      The function will not munge instances that's already been
      munged
  
      Args:
          intent_parser: (IntentParser) object to update
          name: (str) Skill name
          skill_id: (int) skill identifier
      """
      # Munge parser name
      if str(skill_id) + ':' not in name:
          intent_parser.name = str(skill_id) + ':' + name
      else:
          intent_parser.name = name
  
      # Munge keywords
      skill_id = to_letters(skill_id)
      # Munge required keyword
      reqs = []
      for i in intent_parser.requires:
          if skill_id not in i[0]:
              kw = (skill_id + i[0], skill_id + i[0])
              reqs.append(kw)
          else:
              reqs.append(i)
      intent_parser.requires = reqs
  
      # Munge optional keywords
      opts = []
      for i in intent_parser.optional:
          if skill_id not in i[0]:
              kw = (skill_id + i[0], skill_id + i[0])
              opts.append(kw)
          else:
              opts.append(i)
      intent_parser.optional = opts
  
      # Munge at_least_one keywords
      at_least_one = []
      for i in intent_parser.at_least_one:
          element = [skill_id + e.replace(skill_id, '') for e in i]
          at_least_one.append(tuple(element))
      intent_parser.at_least_one = at_least_one
  
  
  [... the rest is omitted ...]
files: mycroft/skills/skill_data.py
files: # -*- coding: utf-8 -*-
  # Copyright 2017 Mycroft AI Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #    http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  import sys
  import unittest
  
  import mock
  from adapt.intent import IntentBuilder
  from os.path import join, dirname, abspath
  from re import error
  from datetime import datetime
  
  from mycroft.configuration import Configuration
  from mycroft.messagebus.message import Message
  from mycroft.skills.skill_data import load_regex_from_file, load_regex, \
      load_vocab_from_file, load_vocabulary
  from mycroft.skills.core import MycroftSkill, load_skill, \
      create_skill_descriptor, open_intent_envelope
  
  from mycroft.configuration.config import LocalConf, DEFAULT_CONFIG
  
  BASE_CONF = LocalConf(DEFAULT_CONFIG)
  
  
  class MockEmitter(object):
      def __init__(self):
          self.reset()
  
      def emit(self, message):
          self.types.append(message.type)
          self.results.append(message.data)
  
      def get_types(self):
          return self.types
  
      def get_results(self):
          return self.results
  
      def on(self, event, f):
          pass
  
      def reset(self):
          self.types = []
          self.results = []
  
  
  class MycroftSkillTest(unittest.TestCase):
      emitter = MockEmitter()
      regex_path = abspath(join(dirname(__file__), '../regex_test'))
      vocab_path = abspath(join(dirname(__file__), '../vocab_test'))
  
      def setUp(self):
          self.emitter.reset()
  
      def check_vocab_from_file(self, filename, vocab_type=None,
                                result_list=None):
          result_list = result_list or []
          load_vocab_from_file(join(self.vocab_path, filename), vocab_type,
                               self.emitter)
          self.check_emitter(result_list)
  
      def check_regex_from_file(self, filename, result_list=None):
          result_list = result_list or []
          regex_file = join(self.regex_path, filename)
          load_regex_from_file(regex_file, self.emitter, 'A')
          self.check_emitter(result_list)
  
      def check_vocab(self, path, result_list=None):
          result_list = result_list or []
          load_vocabulary(path, self.emitter, 'A')
          self.check_emitter(result_list)
  
      def check_regex(self, path, result_list=None):
          result_list = result_list or []
          load_regex(path, self.emitter, 'A')
          self.check_emitter(result_list)
  
      def check_emitter(self, result_list):
          for type in self.emitter.get_types():
              self.assertEquals(type, 'register_vocab')
          self.assertEquals(sorted(self.emitter.get_results(),
                                   key=lambda d: sorted(d.items())),
                            sorted(result_list, key=lambda d: sorted(d.items())))
          self.emitter.reset()
  
      def test_load_regex_from_file_single(self):
          self.check_regex_from_file('valid/single.rx',
                                     [{'regex': '(?P<ASingleTest>.*)'}])
  
      def test_load_regex_from_file_multiple(self):
          self.check_regex_from_file('valid/multiple.rx',
                                     [{'regex': '(?P<AMultipleTest1>.*)'},
                                      {'regex': '(?P<AMultipleTest2>.*)'}])
  
      def test_load_regex_from_file_none(self):
          self.check_regex_from_file('invalid/none.rx')
  
      def test_load_regex_from_file_invalid(self):
          with self.assertRaises(error):
              self.check_regex_from_file('invalid/invalid.rx')
  
      def test_load_regex_from_file_does_not_exist(self):
          with self.assertRaises(IOError):
              self.check_regex_from_file('does_not_exist.rx')
  
      def test_load_regex_full(self):
          self.check_regex(join(self.regex_path, 'valid'),
                           [{'regex': '(?P<AMultipleTest1>.*)'},
                            {'regex': '(?P<AMultipleTest2>.*)'},
                            {'regex': '(?P<ASingleTest>.*)'}])
  
      def test_load_regex_empty(self):
          self.check_regex(join(dirname(__file__),
                                'empty_dir'))
  
      def test_load_regex_fail(self):
          try:
              self.check_regex(join(dirname(__file__),
                                    'regex_test_fail'))
          except OSError as e:
              self.assertEquals(e.strerror, 'No such file or directory')
  
      def test_load_vocab_from_file_single(self):
          self.check_vocab_from_file('valid/single.voc', 'test_type',
                                     [{'start': 'test', 'end': 'test_type'}])
  
      def test_load_vocab_from_file_single_alias(self):
          self.check_vocab_from_file('valid/singlealias.voc', 'test_type',
                                     [{'start': 'water', 'end': 'test_type'},
                                      {'start': 'watering', 'end': 'test_type',
                                       'alias_of': 'water'}])
  
      def test_load_vocab_from_file_multiple(self):
          self.check_vocab_from_file('valid/multiple.voc', 'test_type',
                                     [{'start': 'animal', 'end': 'test_type'},
                                      {'start': 'animals', 'end': 'test_type'}])
  
      def test_load_vocab_from_file_multiple_alias(self):
          self.check_vocab_from_file('valid/multiplealias.voc', 'test_type',
                                     [{'start': 'chair', 'end': 'test_type'},
                                      {'start': 'chairs', 'end': 'test_type',
                                       'alias_of': 'chair'},
                                      {'start': 'table', 'end': 'test_type'},
                                      {'start': 'tables', 'end': 'test_type',
                                       'alias_of': 'table'}])
  
      def test_load_vocab_from_file_none(self):
          self.check_vocab_from_file('none.voc')
  
      def test_load_vocab_from_file_does_not_exist(self):
          try:
              self.check_vocab_from_file('does_not_exist.voc')
          except IOError as e:
              self.assertEquals(e.strerror, 'No such file or directory')
  
      def test_load_vocab_full(self):
          self.check_vocab(join(self.vocab_path, 'valid'),
                           [{'start': 'test', 'end': 'Asingle'},
                            {'start': 'water', 'end': 'Asinglealias'},
                            {'start': 'watering', 'end': 'Asinglealias',
                             'alias_of': 'water'},
                            {'start': 'animal', 'end': 'Amultiple'},
                            {'start': 'animals', 'end': 'Amultiple'},
                            {'start': 'chair', 'end': 'Amultiplealias'},
                            {'start': 'chairs', 'end': 'Amultiplealias',
                             'alias_of': 'chair'},
                            {'start': 'table', 'end': 'Amultiplealias'},
                            {'start': 'tables', 'end': 'Amultiplealias',
                             'alias_of': 'table'}])
  
      def test_load_vocab_empty(self):
          self.check_vocab(join(dirname(__file__), 'empty_dir'))
  
      def test_load_vocab_fail(self):
          try:
              self.check_regex(join(dirname(__file__),
                                    'vocab_test_fail'))
          except OSError as e:
              self.assertEquals(e.strerror, 'No such file or directory')
  
      def test_open_envelope(self):
          name = 'Jerome'
          intent = IntentBuilder(name).require('Keyword')
          intent.name = name
          m = Message("register_intent", intent.__dict__)
          unpacked_intent = open_intent_envelope(m)
          self.assertEqual(intent.__dict__, unpacked_intent.__dict__)
  
      def test_load_skill(self):
          """ Verify skill load function. """
          e_path = join(dirname(__file__), 'test_skill')
          s = load_skill(create_skill_descriptor(e_path), MockEmitter(), 847)
          self.assertEquals(s._dir, e_path)
          self.assertEquals(s.skill_id, 847)
          self.assertEquals(s.name, 'LoadTestSkill')
  
      def check_detach_intent(self):
          self.assertTrue(len(self.emitter.get_types()) > 0)
          for msg_type in self.emitter.get_types():
              self.assertEquals(msg_type, 'detach_intent')
          self.emitter.reset()
  
      def check_register_intent(self, result_list):
          for type in self.emitter.get_types():
              self.assertEquals(type, 'register_intent')
          self.assertEquals(sorted(self.emitter.get_results()),
                            sorted(result_list))
          self.emitter.reset()
  
      def check_register_vocabulary(self, result_list):
          for type in self.emitter.get_types():
              self.assertEquals(type, 'register_vocab')
          self.assertEquals(sorted(self.emitter.get_results()),
                            sorted(result_list))
          self.emitter.reset()
  
      def test_register_intent(self):
          # Test register Intent object
          s = SimpleSkill1()
          s.bind(self.emitter)
          s.initialize()
          expected = [{'at_least_one': [],
                       'name': 'A:a',
                       'optional': [],
                       'requires': [('AKeyword', 'AKeyword')]}]
          self.check_register_intent(expected)
  
          # Test register IntentBuilder object
          s = SimpleSkill2()
          s.bind(self.emitter)
          s.initialize()
          expected = [{'at_least_one': [],
                       'name': 'A:a',
                       'optional': [],
                       'requires': [('AKeyword', 'AKeyword')]}]
  
          self.check_register_intent(expected)
  
          # Test register IntentBuilder object
          with self.assertRaises(ValueError):
              s = SimpleSkill3()
              s.bind(self.emitter)
              s.initialize()
  
      def test_enable_disable_intent(self):
          """Test disable/enable intent."""
          # Setup basic test
          s = SimpleSkill1()
          s.bind(self.emitter)
          s.initialize()
          expected = [{'at_least_one': [],
                       'name': 'A:a',
                       'optional': [],
                       'requires': [('AKeyword', 'AKeyword')]}]
          self.check_register_intent(expected)
  
          # Test disable/enable cycle
          s.disable_intent('a')
          self.check_detach_intent()
          s.enable_intent('a')
          self.check_register_intent(expected)
  
      def test_register_vocab(self):
          """Test disable/enable intent."""
          # Setup basic test
          s = SimpleSkill1()
          s.bind(self.emitter)
          s.initialize()
  
          # Normal vocaubulary
          self.emitter.reset()
          expected = [{'start': 'hello', 'end': 'AHelloKeyword'}]
          s.register_vocabulary('hello', 'HelloKeyword')
          self.check_register_vocabulary(expected)
          # Regex
          s.register_regex('weird (?P<Weird>.+) stuff')
          expected = [{'regex': 'weird (?P<AWeird>.+) stuff'}]
          self.check_register_vocabulary(expected)
  
      def check_register_object_file(self, types_list, result_list):
          self.assertEquals(sorted(self.emitter.get_types()),
                            sorted(types_list))
          self.assertEquals(sorted(self.emitter.get_results(),
                                   key=lambda d: sorted(d.items())),
                            sorted(result_list, key=lambda d: sorted(d.items())))
          self.emitter.reset()
  
      def test_register_intent_file(self):
          s = SimpleSkill4()
          s.bind(self.emitter)
          s.vocab_dir = join(dirname(__file__), 'intent_file')
          s.initialize()
  
          expected_types = [
              'padatious:register_intent',
              'padatious:register_entity'
          ]
  
          expected_results = [
              {
                  'file_name': join(dirname(__file__),
                                    'intent_file', 'test.intent'),
                  'name': str(s.skill_id) + ':test.intent'
              },
              {
                  'file_name': join(dirname(__file__),
                                    'intent_file', 'test_ent.entity'),
                  'name': str(s.skill_id) + ':test_ent'
              }
          ]
  
          self.check_register_object_file(expected_types, expected_results)
  
      def check_register_decorators(self, result_list):
          self.assertEquals(sorted(self.emitter.get_results(),
                                   key=lambda d: sorted(d.items())),
                            sorted(result_list, key=lambda d: sorted(d.items())))
          self.emitter.reset()
  
      def test_register_decorators(self):
          """ Test decorated intents """
          path_orig = sys.path
          sys.path.append(abspath(dirname(__file__)))
          SimpleSkill5 = __import__('decorator_test_skill').TestSkill
          s = SimpleSkill5()
          s.skill_id = 'A'
          s.vocab_dir = join(dirname(__file__), 'intent_file')
          s.bind(self.emitter)
          s.initialize()
          s._register_decorated()
          expected = [{'at_least_one': [],
                       'name': 'A:a',
                       'optional': [],
                       'requires': [('AKeyword', 'AKeyword')]},
                      {
                       'file_name': join(dirname(__file__), 'intent_file',
                                         'test.intent'),
                       'name': str(s.skill_id) + ':test.intent'}]
  
          self.check_register_decorators(expected)
  
      def test_failing_set_context(self):
          s = SimpleSkill1()
          s.bind(self.emitter)
          with self.assertRaises(ValueError):
              s.set_context(1)
          with self.assertRaises(ValueError):
              s.set_context(1, 1)
          with self.assertRaises(ValueError):
              s.set_context('Kowabunga', 1)
  
      def test_set_context(self):
          def check_set_context(result_list):
              for type in self.emitter.get_types():
                  self.assertEquals(type, 'add_context')
              self.assertEquals(sorted(self.emitter.get_results()),
                                sorted(result_list))
              self.emitter.reset()
  
          s = SimpleSkill1()
          s.bind(self.em
  
  [... the rest is omitted ...]
files: test/unittests/skills/core.py
files: # -*- coding: utf-8 -*-
  # Copyright 2017 Mycroft AI Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #    http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  import sys
  import unittest
  
  import mock
  from adapt.intent import IntentBuilder
  from os.path import join, dirname, abspath
  from re import error
  from datetime import datetime
  
  from mycroft.configuration import Configuration
  from mycroft.messagebus.message import Message
  from mycroft.skills.skill_data import load_regex_from_file, load_regex, \
      load_vocab_from_file, load_vocabulary
  from mycroft.skills.core import MycroftSkill, load_skill, \
      create_skill_descriptor, open_intent_envelope
  
  from mycroft.configuration.config import LocalConf, DEFAULT_CONFIG
  
  BASE_CONF = LocalConf(DEFAULT_CONFIG)
  
  
  class MockEmitter(object):
      def __init__(self):
          self.reset()
  
      def emit(self, message):
          self.types.append(message.type)
          self.results.append(message.data)
  
      def get_types(self):
          return self.types
  
      def get_results(self):
          return self.results
  
      def on(self, event, f):
          pass
  
      def reset(self):
          self.types = []
          self.results = []
  
  
  class MycroftSkillTest(unittest.TestCase):
      emitter = MockEmitter()
      regex_path = abspath(join(dirname(__file__), '../regex_test'))
      vocab_path = abspath(join(dirname(__file__), '../vocab_test'))
  
      def setUp(self):
          self.emitter.reset()
  
      def check_vocab_from_file(self, filename, vocab_type=None,
                                result_list=None):
          result_list = result_list or []
          load_vocab_from_file(join(self.vocab_path, filename), vocab_type,
                               self.emitter)
          self.check_emitter(result_list)
  
      def check_regex_from_file(self, filename, result_list=None):
          result_list = result_list or []
          load_regex_from_file(join(self.regex_path, filename), self.emitter, 0)
          self.check_emitter(result_list)
  
      def check_vocab(self, path, result_list=None):
          result_list = result_list or []
          load_vocabulary(path, self.emitter, 0)
          self.check_emitter(result_list)
  
      def check_regex(self, path, result_list=None):
          result_list = result_list or []
          load_regex(path, self.emitter, 0)
          self.check_emitter(result_list)
  
      def check_emitter(self, result_list):
          for type in self.emitter.get_types():
              self.assertEquals(type, 'register_vocab')
          self.assertEquals(sorted(self.emitter.get_results(),
                                   key=lambda d: sorted(d.items())),
                            sorted(result_list, key=lambda d: sorted(d.items())))
          self.emitter.reset()
  
      def test_load_regex_from_file_single(self):
          self.check_regex_from_file('valid/single.rx',
                                     [{'regex': '(?P<ASingleTest>.*)'}])
  
      def test_load_regex_from_file_multiple(self):
          self.check_regex_from_file('valid/multiple.rx',
                                     [{'regex': '(?P<AMultipleTest1>.*)'},
                                      {'regex': '(?P<AMultipleTest2>.*)'}])
  
      def test_load_regex_from_file_none(self):
          self.check_regex_from_file('invalid/none.rx')
  
      def test_load_regex_from_file_invalid(self):
          with self.assertRaises(error):
              self.check_regex_from_file('invalid/invalid.rx')
  
      def test_load_regex_from_file_does_not_exist(self):
          with self.assertRaises(IOError):
              self.check_regex_from_file('does_not_exist.rx')
  
      def test_load_regex_full(self):
          self.check_regex(join(self.regex_path, 'valid'),
                           [{'regex': '(?P<AMultipleTest1>.*)'},
                            {'regex': '(?P<AMultipleTest2>.*)'},
                            {'regex': '(?P<ASingleTest>.*)'}])
  
      def test_load_regex_empty(self):
          self.check_regex(join(dirname(__file__),
                                'empty_dir'))
  
      def test_load_regex_fail(self):
          try:
              self.check_regex(join(dirname(__file__),
                                    'regex_test_fail'))
          except OSError as e:
              self.assertEquals(e.strerror, 'No such file or directory')
  
      def test_load_vocab_from_file_single(self):
          self.check_vocab_from_file('valid/single.voc', 'test_type',
                                     [{'start': 'test', 'end': 'test_type'}])
  
      def test_load_vocab_from_file_single_alias(self):
          self.check_vocab_from_file('valid/singlealias.voc', 'test_type',
                                     [{'start': 'water', 'end': 'test_type'},
                                      {'start': 'watering', 'end': 'test_type',
                                       'alias_of': 'water'}])
  
      def test_load_vocab_from_file_multiple(self):
          self.check_vocab_from_file('valid/multiple.voc', 'test_type',
                                     [{'start': 'animal', 'end': 'test_type'},
                                      {'start': 'animals', 'end': 'test_type'}])
  
      def test_load_vocab_from_file_multiple_alias(self):
          self.check_vocab_from_file('valid/multiplealias.voc', 'test_type',
                                     [{'start': 'chair', 'end': 'test_type'},
                                      {'start': 'chairs', 'end': 'test_type',
                                       'alias_of': 'chair'},
                                      {'start': 'table', 'end': 'test_type'},
                                      {'start': 'tables', 'end': 'test_type',
                                       'alias_of': 'table'}])
  
      def test_load_vocab_from_file_none(self):
          self.check_vocab_from_file('none.voc')
  
      def test_load_vocab_from_file_does_not_exist(self):
          try:
              self.check_vocab_from_file('does_not_exist.voc')
          except IOError as e:
              self.assertEquals(e.strerror, 'No such file or directory')
  
      def test_load_vocab_full(self):
          self.check_vocab(join(self.vocab_path, 'valid'),
                           [{'start': 'test', 'end': 'Asingle'},
                            {'start': 'water', 'end': 'Asinglealias'},
                            {'start': 'watering', 'end': 'Asinglealias',
                             'alias_of': 'water'},
                            {'start': 'animal', 'end': 'Amultiple'},
                            {'start': 'animals', 'end': 'Amultiple'},
                            {'start': 'chair', 'end': 'Amultiplealias'},
                            {'start': 'chairs', 'end': 'Amultiplealias',
                             'alias_of': 'chair'},
                            {'start': 'table', 'end': 'Amultiplealias'},
                            {'start': 'tables', 'end': 'Amultiplealias',
                             'alias_of': 'table'}])
  
      def test_load_vocab_empty(self):
          self.check_vocab(join(dirname(__file__), 'empty_dir'))
  
      def test_load_vocab_fail(self):
          try:
              self.check_regex(join(dirname(__file__),
                                    'vocab_test_fail'))
          except OSError as e:
              self.assertEquals(e.strerror, 'No such file or directory')
  
      def test_open_envelope(self):
          name = 'Jerome'
          intent = IntentBuilder(name).require('Keyword')
          intent.name = name
          m = Message("register_intent", intent.__dict__)
          unpacked_intent = open_intent_envelope(m)
          self.assertEqual(intent.__dict__, unpacked_intent.__dict__)
  
      def test_load_skill(self):
          """ Verify skill load function. """
          e_path = join(dirname(__file__), 'test_skill')
          s = load_skill(create_skill_descriptor(e_path), MockEmitter(), 847)
          self.assertEquals(s._dir, e_path)
          self.assertEquals(s.skill_id, 847)
          self.assertEquals(s.name, 'LoadTestSkill')
  
      def check_detach_intent(self):
          self.assertTrue(len(self.emitter.get_types()) > 0)
          for msg_type in self.emitter.get_types():
              self.assertEquals(msg_type, 'detach_intent')
          self.emitter.reset()
  
      def check_register_intent(self, result_list):
          for type in self.emitter.get_types():
              self.assertEquals(type, 'register_intent')
          self.assertEquals(sorted(self.emitter.get_results()),
                            sorted(result_list))
          self.emitter.reset()
  
      def check_register_vocabulary(self, result_list):
          for type in self.emitter.get_types():
              self.assertEquals(type, 'register_vocab')
          self.assertEquals(sorted(self.emitter.get_results()),
                            sorted(result_list))
          self.emitter.reset()
  
      def test_register_intent(self):
          # Test register Intent object
          s = SimpleSkill1()
          s.bind(self.emitter)
          s.initialize()
          expected = [{'at_least_one': [],
                       'name': '0:a',
                       'optional': [],
                       'requires': [('AKeyword', 'AKeyword')]}]
          self.check_register_intent(expected)
  
          # Test register IntentBuilder object
          s = SimpleSkill2()
          s.bind(self.emitter)
          s.initialize()
          expected = [{'at_least_one': [],
                       'name': '0:a',
                       'optional': [],
                       'requires': [('AKeyword', 'AKeyword')]}]
  
          self.check_register_intent(expected)
  
          # Test register IntentBuilder object
          with self.assertRaises(ValueError):
              s = SimpleSkill3()
              s.bind(self.emitter)
              s.initialize()
  
      def test_enable_disable_intent(self):
          """Test disable/enable intent."""
          # Setup basic test
          s = SimpleSkill1()
          s.bind(self.emitter)
          s.initialize()
          expected = [{'at_least_one': [],
                       'name': '0:a',
                       'optional': [],
                       'requires': [('AKeyword', 'AKeyword')]}]
          self.check_register_intent(expected)
  
          # Test disable/enable cycle
          s.disable_intent('a')
          self.check_detach_intent()
          s.enable_intent('a')
          self.check_register_intent(expected)
  
      def test_register_vocab(self):
          """Test disable/enable intent."""
          # Setup basic test
          s = SimpleSkill1()
          s.bind(self.emitter)
          s.initialize()
  
          # Normal vocaubulary
          self.emitter.reset()
          expected = [{'start': 'hello', 'end': 'AHelloKeyword'}]
          s.register_vocabulary('hello', 'HelloKeyword')
          self.check_register_vocabulary(expected)
          # Regex
          s.register_regex('weird (?P<Weird>.+) stuff')
          expected = [{'regex': 'weird (?P<AWeird>.+) stuff'}]
          self.check_register_vocabulary(expected)
  
      def check_register_object_file(self, types_list, result_list):
          self.assertEquals(sorted(self.emitter.get_types()),
                            sorted(types_list))
          self.assertEquals(sorted(self.emitter.get_results(),
                                   key=lambda d: sorted(d.items())),
                            sorted(result_list, key=lambda d: sorted(d.items())))
          self.emitter.reset()
  
      def test_register_intent_file(self):
          s = SimpleSkill4()
          s.bind(self.emitter)
          s.vocab_dir = join(dirname(__file__), 'intent_file')
          s.initialize()
  
          expected_types = [
              'padatious:register_intent',
              'padatious:register_entity'
          ]
  
          expected_results = [
              {
                  'file_name': join(dirname(__file__),
                                    'intent_file', 'test.intent'),
                  'name': str(s.skill_id) + ':test.intent'
              },
              {
                  'file_name': join(dirname(__file__),
                                    'intent_file', 'test_ent.entity'),
                  'name': str(s.skill_id) + ':test_ent'
              }
          ]
  
          self.check_register_object_file(expected_types, expected_results)
  
      def check_register_decorators(self, result_list):
          self.assertEquals(sorted(self.emitter.get_results(),
                                   key=lambda d: sorted(d.items())),
                            sorted(result_list, key=lambda d: sorted(d.items())))
          self.emitter.reset()
  
      def test_register_decorators(self):
          """ Test decorated intents """
          path_orig = sys.path
          sys.path.append(abspath(dirname(__file__)))
          SimpleSkill5 = __import__('decorator_test_skill').TestSkill
          s = SimpleSkill5()
          s.vocab_dir = join(dirname(__file__), 'intent_file')
          s.bind(self.emitter)
          s.initialize()
          s._register_decorated()
          expected = [{'at_least_one': [],
                       'name': '0:a',
                       'optional': [],
                       'requires': [('AKeyword', 'AKeyword')]},
                      {
                       'file_name': join(dirname(__file__), 'intent_file',
                                         'test.intent'),
                       'name': str(s.skill_id) + ':test.intent'}]
  
          self.check_register_decorators(expected)
  
      def test_failing_set_context(self):
          s = SimpleSkill1()
          s.bind(self.emitter)
          with self.assertRaises(ValueError):
              s.set_context(1)
          with self.assertRaises(ValueError):
              s.set_context(1, 1)
          with self.assertRaises(ValueError):
              s.set_context('Kowabunga', 1)
  
      def test_set_context(self):
          def check_set_context(result_list):
              for type in self.emitter.get_types():
                  self.assertEquals(type, 'add_context')
              self.assertEquals(sorted(self.emitter.get_results()),
                                sorted(result_list))
              self.emitter.reset()
  
          s = SimpleSkill1()
          s.bind(self.emitter)
          # No context content
          s.set_context('TurtlePower
  
  [... the rest is omitted ...]
files: test/unittests/skills/core.py