'Cleanup any instances which are erroneously still running after
having been deleted.
Valid actions to take are:
1. noop - do nothing
2. log - log which instances are erroneously running
3. reap - shutdown and cleanup any erroneously running instances
4. shutdown - power off *and disable* any erroneously running
instances
The use-case for this cleanup task is: for various reasons, it may be
possible for the database to show an instance as deleted but for that
instance to still be running on a host machine (see bug
https://bugs.launchpad.net/nova/+bug/911366).
This cleanup task is a cross-hypervisor utility for finding these
zombied instances and either logging the discrepancy (likely what you
should do in production), or automatically reaping the instances (more
appropriate for dev environments).'

@periodic_task.periodic_task(spacing=CONF.running_deleted_instance_poll_interval)
def _cleanup_running_deleted_instances(self, context):

    action = CONF.running_deleted_instance_action
    if (action == 'noop'):
        return
    with utils.temporary_mutation(context, read_deleted='yes'):
        for instance in self._running_deleted_instances(context):
            if (action == 'log'):
                LOG.warning("Detected    instance    with    name    label    '%s'    which    is    marked    as    DELETED    but    still    present    on    host.", instance.name, instance=instance)
            elif (action == 'shutdown'):
                LOG.info("Powering    off    instance    with    name    label    '%s'    which    is    marked    as    DELETED    but    still    present    on    host.", instance.name, instance=instance)
                try:
                    try:
                        self.driver.set_bootable(instance, False)
                    except NotImplementedError:
                        LOG.debug('set_bootable    is    not    implemented    for    the    current    driver')
                    self.driver.power_off(instance)
                except Exception:
                    LOG.warning('Failed    to    power    off    instance', instance=instance, exc_info=True)
            elif (action == 'reap'):
                LOG.info("Destroying    instance    with    name    label    '%s'    which    is    marked    as    DELETED    but    still    present    on    host.", instance.name, instance=instance)
                bdms = objects.BlockDeviceMappingList.get_by_instance_uuid(context, instance.uuid, use_slave=True)
                self.instance_events.clear_events_for_instance(instance)
                try:
                    self._shutdown_instance(context, instance, bdms, notify=False)
                    self._cleanup_volumes(context, instance.uuid, bdms)
                except Exception as e:
                    LOG.warning('Periodic    cleanup    failed    to    delete    instance:    %s', e, instance=instance)
            else:
                raise Exception((_("Unrecognized    value    '%s'    for    CONF.running_deleted_instance_action") % action))
