text: import hashlib
  import logging
  import multiprocessing
  
  import os
  import shutil
  
  import sys
  
  import asyncio
  
  import vanir
  import vanir.backup
  import vanir.exc
  import vanir.storage.lvm
  import vanir.tests
  import vanir.tests.storage_lvm
  import vanir.vm
  import vanir.vm.appvm
  import vanir.vm.templatevm
  import vanir.vm.vanirvm
  
  try:
      import vaniradmin.backup.restore
      import vaniradmin.exc
      restore_available = True
  except ImportError:
      restore_available = False
  
  
  # noinspection PyAttributeOutsideInit
  class BackupTestsMixin(object):
      class BackupErrorHandler(logging.Handler):
          def __init__(self, errors_queue, level=logging.NOTSET):
              super(BackupTestsMixin.BackupErrorHandler, self).__init__(level)
              self.errors_queue = errors_queue
  
          def emit(self, record):
              self.errors_queue.put(record.getMessage())
  
      def setUp(self):
          super(BackupTestsMixin, self).setUp()
          try:
              self.init_default_template(self.template)
          except AttributeError:
              self.init_default_template()
          self.error_detected = multiprocessing.Queue()
  
          self.log.debug("Creating backupvm")
  
          self.backupdir = os.path.join(os.environ["HOME"], "test-backup")
          if os.path.exists(self.backupdir):
              shutil.rmtree(self.backupdir)
          os.mkdir(self.backupdir)
  
          self.error_handler = self.BackupErrorHandler(self.error_detected,
              level=logging.WARNING)
          backup_log = logging.getLogger('vaniradmin.backup')
          backup_log.addHandler(self.error_handler)
  
      def tearDown(self):
          shutil.rmtree(self.backupdir)
  
          backup_log = logging.getLogger('vanir.backup')
          backup_log.removeHandler(self.error_handler)
          super(BackupTestsMixin, self).tearDown()
  
      def fill_image(self, path, size=None, sparse=False):
          block_size = 4096
  
          self.log.debug("Filling %s" % path)
          try:
              f = open(path, 'rb+')
          except FileNotFoundError:
              f = open(path, 'wb+')
          if size is None:
              f.seek(0, 2)
              size = f.tell()
          f.seek(0)
  
          for block_num in range(int(size/block_size)):
              if sparse:
                  f.seek(block_size, 1)
              f.write(b'a' * block_size)
  
          f.close()
  
      # NOTE: this was create_basic_vms
      def create_backup_vms(self, pool=None):
          template = self.app.default_template
  
          vms = []
          vmname = self.make_vm_name('test-net')
          self.log.debug("Creating %s" % vmname)
          testnet = self.app.add_new_vm(vanir.vm.appvm.AppVM,
              name=vmname, template=template, provides_network=True,
              label='red')
          self.loop.run_until_complete(
              testnet.create_on_disk(pool=pool))
          testnet.features['service.ntpd'] = True
          vms.append(testnet)
          self.fill_image(testnet.storage.export('private'), 20*1024*1024)
  
          vmname = self.make_vm_name('test1')
          self.log.debug("Creating %s" % vmname)
          testvm1 = self.app.add_new_vm(vanir.vm.appvm.AppVM,
              name=vmname, template=template, label='red')
          testvm1.netvm = testnet
          self.loop.run_until_complete(
              testvm1.create_on_disk(pool=pool))
          vms.append(testvm1)
          self.fill_image(testvm1.storage.export('private'), 100 * 1024 * 1024)
  
          vmname = self.make_vm_name('testhvm1')
          self.log.debug("Creating %s" % vmname)
          testvm2 = self.app.add_new_vm(vanir.vm.standalonevm.StandaloneVM,
                                        name=vmname,
                                        virt_mode='hvm',
                                        label='red')
          self.loop.run_until_complete(
              testvm2.create_on_disk(pool=pool))
          self.fill_image(testvm2.storage.export('root'), 1024 * 1024 * 1024, \
              True)
          vms.append(testvm2)
  
          vmname = self.make_vm_name('template')
          self.log.debug("Creating %s" % vmname)
          testvm3 = self.app.add_new_vm(vanir.vm.templatevm.TemplateVM,
              name=vmname, label='red')
          self.loop.run_until_complete(
              testvm3.create_on_disk(pool=pool))
          self.fill_image(testvm3.storage.export('root'), 100 * 1024 * 1024, True)
          vms.append(testvm3)
  
          vmname = self.make_vm_name('custom')
          self.log.debug("Creating %s" % vmname)
          testvm4 = self.app.add_new_vm(vanir.vm.appvm.AppVM,
              name=vmname, template=testvm3, label='red')
          self.loop.run_until_complete(
              testvm4.create_on_disk(pool=pool))
          vms.append(testvm4)
  
          self.app.save()
  
          return vms
  
      def make_backup(self, vms, target=None, expect_failure=False, **kwargs):
          if target is None:
              target = self.backupdir
          try:
              backup = vanir.backup.Backup(self.app, vms, **kwargs)
          except vanir.exc.VanirException as e:
              if not expect_failure:
                  self.fail("VanirException during backup_prepare: %s" % str(e))
              else:
                  raise
  
          if 'passphrase' not in kwargs:
              backup.passphrase = 'vanir'
          backup.target_dir = target
  
          try:
              self.loop.run_until_complete(backup.backup_do())
          except vanir.exc.VanirException as e:
              if not expect_failure:
                  self.fail("VanirException during backup_do: %s" % str(e))
              else:
                  raise
  
      def remove_vms(self, vms):
          vms = list(vms)
          for vm in vms:
              vm.netvm = None
              vm.default_dispvm = None
          super(BackupTestsMixin, self).remove_vms(vms)
  
      def restore_backup(self, source=None, appvm=None, options=None,
                         expect_errors=None, manipulate_restore_info=None,
                         passphrase='vanir'):
          if not restore_available:
              self.skipTest('vaniradmin module not available')
  
          if source is None:
              backupfile = os.path.join(self.backupdir,
                                        sorted(os.listdir(self.backupdir))[-1])
          else:
              backupfile = source
  
          client_app = vaniradmin.Vanir()
          if appvm:
              appvm = self.loop.run_until_complete(
                  self.loop.run_in_executor(None,
                      client_app.domains.__getitem__, appvm.name))
          with self.assertNotRaises(vaniradmin.exc.VanirException):
              restore_op = self.loop.run_until_complete(
                  self.loop.run_in_executor(None,
                      vaniradmin.backup.restore.BackupRestore,
                      client_app, backupfile, appvm, passphrase))
              if options:
                  for key, value in options.items():
                      setattr(restore_op.options, key, value)
              restore_info = self.loop.run_until_complete(
                  self.loop.run_in_executor(None,
                      restore_op.get_restore_info))
          if callable(manipulate_restore_info):
              restore_info = manipulate_restore_info(restore_info)
          self.log.debug(restore_op.get_restore_summary(restore_info))
  
          with self.assertNotRaises(vaniradmin.exc.VanirException):
              self.loop.run_until_complete(
                  self.loop.run_in_executor(None,
                      restore_op.restore_do, restore_info))
  
          errors = []
          if expect_errors is None:
              expect_errors = []
          else:
              self.assertFalse(self.error_detected.empty(),
                  "Restore errors expected, but none detected")
          while not self.error_detected.empty():
              current_error = self.error_detected.get()
              if any(map(current_error.startswith, expect_errors)):
                  continue
              errors.append(current_error)
          self.assertTrue(len(errors) == 0,
                           "Error(s) detected during backup_restore_do: %s" %
                           '\n'.join(errors))
          if not appvm and not os.path.isdir(backupfile):
              os.unlink(backupfile)
  
      def create_sparse(self, path, size):
          f = open(path, "w")
          f.truncate(size)
          f.close()
  
      def vm_checksum(self, vms):
          hashes = {}
          for vm in vms:
              assert isinstance(vm, vanir.vm.vanirvm.VanirVM)
              hashes[vm.name] = {}
              for name, volume in vm.volumes.items():
                  if not volume.rw or not volume.save_on_stop:
                      continue
                  vol_path = volume.export()
                  hasher = hashlib.sha1()
                  with open(vol_path, 'rb') as afile:
                      for buf in iter(lambda: afile.read(4096000), b''):
                          hasher.update(buf)
                  hashes[vm.name][name] = hasher.hexdigest()
          return hashes
  
      def get_vms_info(self, vms):
          ''' Get VM metadata, for comparing VM later without holding actual
          reference to the old object.'''
  
          vms_info = {}
          for vm in vms:
              vm_info = {
                  'properties': {},
                  'default': {},
                  'devices': {},
              }
              for prop in ('name', 'kernel',
                      'memory', 'maxmem', 'kernelopts',
                      'services', 'vcpus', 'features'
                      'include_in_backups', 'default_user', 'qrexec_timeout',
                      'autostart', 'pci_strictreset', 'debug',
                      'internal', 'netvm', 'template', 'label'):
                  if not hasattr(vm, prop):
                      continue
                  vm_info['properties'][prop] = str(getattr(vm, prop))
                  vm_info['default'][prop] = vm.property_is_default(prop)
              for dev_class in vm.devices.keys():
                  vm_info['devices'][dev_class] = {}
                  for dev_ass in vm.devices[dev_class].assignments():
                      vm_info['devices'][dev_class][str(dev_ass.device)] = \
                          dev_ass.options
              vms_info[vm.name] = vm_info
  
          return vms_info
  
      def assertCorrectlyRestored(self, vms_info, orig_hashes):
          ''' Verify if restored VMs are identical to those before backup.
  
          :param orig_vms: collection of original VanirVM objects
          :param orig_hashes: result of :py:meth:`vm_checksum` on original VMs,
              before backup
          :return:
          '''
          for vm_name in vms_info:
              vm_info = vms_info[vm_name]
              self.assertIn(vm_name, self.app.domains)
              restored_vm = self.app.domains[vm_name]
              for prop in vm_info['properties']:
                  self.assertEqual(
                      vm_info['properties'][prop],
                      str(getattr(restored_vm, prop)),
                      "VM {} - property {} not properly restored".format(
                          vm_name, prop))
                  self.assertEqual(
                      vm_info['default'][prop],
                      restored_vm.property_is_default(prop),
                      "VM {} - property {} differs in being default".format(
                          vm_name, prop))
              for dev_class in vm_info['devices']:
                  for dev in vm_info['devices'][dev_class]:
                      found = False
                      for restored_dev_ass in restored_vm.devices[
                              dev_class].assignments():
                          if str(restored_dev_ass.device) == dev:
                              found = True
                              self.assertEqual(vm_info['devices'][dev_class][dev],
                                  restored_dev_ass.options,
                                  'VM {} - {} device {} options mismatch'.format(
                                      vm_name, dev_class, str(dev)))
                      self.assertTrue(found,
                          'VM {} - {} device {} not restored'.format(
                              vm_name, dev_class, dev))
  
              if orig_hashes:
                  hashes = self.vm_checksum([restored_vm])[restored_vm.name]
                  self.assertEqual(orig_hashes[vm_name], hashes,
                      "VM {} - disk images are not properly restored".format(
                          vm_name))
  
  
  class TC_00_Backup(BackupTestsMixin, vanir.tests.SystemTestCase):
      def test_000_basic_backup(self):
          vms = self.create_backup_vms()
          try:
              orig_hashes = self.vm_checksum(vms)
              vms_info = self.get_vms_info(vms)
              self.make_backup(vms)
              self.remove_vms(reversed(vms))
          finally:
              del vms
          self.restore_backup()
          self.assertCorrectlyRestored(vms_info, orig_hashes)
  
      def test_001_compressed_backup(self):
          vms = self.create_backup_vms()
          try:
              orig_hashes = self.vm_checksum(vms)
              vms_info = self.get_vms_info(vms)
              self.make_backup(vms, compressed=True)
              self.remove_vms(reversed(vms))
          finally:
              del vms
          self.restore_backup()
          self.assertCorrectlyRestored(vms_info, orig_hashes)
  
      def test_004_sparse_multipart(self):
          vms = []
          try:
              vmname = self.make_vm_name('testhvm2')
              self.log.debug("Creating %s" % vmname)
  
              self.hvmtemplate = self.app.add_new_vm(
                  vanir.vm.templatevm.TemplateVM, name=vmname, virt_mode='hvm', label='red')
              self.loop.run_until_complete(self.hvmtemplate.create_on_disk())
              self.fill_image(
                  os.path.join(self.hvmtemplate.dir_path, '00file'),
                  195 * 1024 * 1024 - 4096 * 3)
              self.fill_image(self.hvmtemplate.storage.export('private'),
                              195 * 1024 * 1024 - 4096 * 3)
              self.fill_image(self.hvmtemplate.storage.export('root'), 1024 * 1024 * 1024,
                              sparse=True)
              vms.append(self.hvmtemplate)
              self.app.save()
              orig_hashes = self.vm_checksum(vms)
              vms_info = self.get_vms_info(vms)
  
              self.make_backup(vms)
              self.remove_vms(reversed(vms))
              self.restore_backup()
              self.assertCorrectlyRestored(vms_info, orig_hashes)
              # TODO check vm.backup_timestamp
          finally:
              del vms
  
      def test_005_compressed_custom(self):
          vms = self.create_backup_vms()
          try:
              orig_hashes = self.vm_checksum(vms)
              vms_info = self.get_vms_info(vms)
              self.make_backup(vms, compression_filter="bzip2")
              self.remove_vms(reversed(vms))
              self.restore_backup()
              self.assertCorrectlyRestored(vms_info, orig_hashes)
          finally:
              del vms
  
      def test_010_selective_restore(self):
          # create backup with internal dependencies (template, netvm etc)
          # try restoring only AppVMs (but not templates, netvms) - should
          # handle according to options set
          exclude = [
              self.make_vm_name('test-net'),
              self.make_vm_name('template')
          ]
          def exclude_some(restore_info):
              for name in exclude:
                  restore_info.pop(name)
              return restore_info
          vms = self.create_backup_vms()
          try:
              orig_hashes = self.vm_checksum(vms)
              vms_info = self.get_vms_info(vms)
              self.make_backup(vms, compression_filter="bzip2")
              self.remove_vms(reversed(vms))
              self.restore_backup(manipulate_restore_info=exclude_some)
              for vm_name in vms_info:
                  if vm_name == self.make_vm_name('test1'):
                      # netvm was set to 'test-inst-test-net' - excluded
                      vms_info[vm_name]['properties']['netvm'] = \
                          str(self.app.default_netvm)
                      vms_info[vm_name]['default']['netvm'] = True
                  elif vm_name == self.make_vm_name('custom'):
                      # template was set to 'test-inst-template' - excluded
                      vms_info[vm_name]['properties']['template'] = \
                          str(self.app.default_template)
              for excluded in exclude:
                  vms_info.pop(excluded, None)
              self.assertCorrectlyRestored(vms_info, orig_hashes)
          finally:
              del vms
  
      def test_020_encrypted_backup_non_ascii(self):
          vms = self.create_backup_vms()
          try:
              orig_hashes = self.vm_checksum(vms)
              vms_info = self.get_vms_info(vms)
              self.make_backup(vms, passphrase=u'zażółć gęślą jaźń')
              self.remove_vms(reversed(vms))
              self.restore_backup(passphrase=u'zażółć gęślą jaźń')
              self.assertCorrectlyRestored(vms_info, orig_hashes)
          finally:
              del vms
  
      def test_100_backup_dom0_no_restore(self):
          # do not write it into dom0 home itself...
          os.mkdir('/var/tmp/test-backup')
          self.backupdir = '/var/tmp/test-backup'
          self.make_backup([self.app.domains[0]])
          # TODO: think of some safe way to test restore...
  
      def test_200_restore_over_existing_directory(self):
          """
          Regression test for #1386
          :return:
          """
          vms = self.create_backup_vms()
          try:
              orig_hashes = self.vm_checksum(vms)
              vms_info = self.get_vms_info(vms)
              self.make_backup(vms)
              test_dir = vms[0].dir_path
              self.remove_vms(reversed(vms))
              os.mkdir(test_dir)
              with open(os.path.join(test_dir, 'some-file.txt'), 'w') as f:
                  f.write('test file\n')
              self.restore_backup()
              self.assertCorrectlyRestored(vms_info, orig_hashes)
          finally:
              del vms
  
      def test_210_auto_rename(self):
          """
          Test for #869
          :return:
          """
          vms = self.create_backup_vms()
          vms_info = self.get_vms_info(vms)
          try:
              self.make_backup(vms)
              self.restore_backup(options={
                  'rename_conflicting': True
              })
              for vm_name in vms_info:
                  with self.assertNotRaises(
                          (vanir.exc.VanirVMNotFoundError, KeyError)):
                      restored_vm = self.app.domains[vm_name + '1']
                  if vms_info[vm_name]['properties']['netvm'] and \
                          not vms_info[vm_name]['default']['netvm']:
                      self.assertEqual(restored_vm.netvm.name,
                          vms_info[vm_name]['properties']['netvm'] + '1')
          finally:
              del vms
  
      def _find_pool(self, volume_group, thin_pool):
          ''' Returns the pool matching the specified ``volume_group`` &
              ``thin_pool``, or None.
          '''
          pools = [p for p in self.app.pools
                   if issubclass(p.__class__, vanir.storage.lvm.ThinPool)]
          for pool in pools:
              if pool.volume_group == volume_group \
                      and pool.thin_pool == thin_pool:
                  return pool
          return None
  
      @vanir.tests.storage_lvm.skipUnlessLvmPoolExists
      def test_300_backup_lvm(self):
          volume_group, thin_pool = \
              vanir.tests.storage_lvm.DEFAULT_LVM_POOL.split('/', 1)
          self.pool = self._find_pool(volume_group, thin_pool)
          if not self.pool:
              self.pool = self.loop.run_until_complete(
                  self.app.add_pool(
                      **vanir.tests.storage_lvm.POOL_CONF))
              self.created_pool = True
          vms = self.create_backup_vms(pool=self.pool)
          try:
              orig_hashes = self.vm_checksum(vms)
              vms_info = self.get_vms_info(vms)
              self.make_backup(vms)
              self.remove_vms(reversed(vms))
              self.restore_backup()
              self.assertCorrectlyRestored(vms_info, orig_hashes)
          finally:
              del vms
  
      @vanir.tests.storage_lvm.skipUnlessLvmPoolExists
      def test_301_restore_to_lvm(self):
          volume_group, thin_pool = \
              vanir.tests.storage_lvm.DEFAULT_LVM_POOL.split('/', 1)
          self.pool = self._find_pool(volume_group, thin_pool)
          if not self.pool:
              self.pool = self.loop.run_until_complete(
                  self.app.add_pool(
                      **vanir.tests.storage_lvm.POOL_CONF))
              self.created_pool = True
          vms = self.create_backup_vms()
          try:
              orig_hashes = self.vm_checksum(vms)
              vms_info = self.get_vms_info(vms)
              self.make_backup(vms)
              self.remove_vms(reversed(vms))
              self.restore_backup(options={'override_pool': self.pool.name})
              self.assertCorrectlyRestored(vms_info, orig_hashes)
              for vm_name in vms_info:
                  vm = self.app.domains[vm_name]
                  for volume in vm.volumes.values():
                      if volume.save_on_stop:
                          self.assertEqual(volume.pool, self.pool.name)
          finally:
              del vms
  
  
  class TC_10_BackupVMMixin(BackupTestsMixin):
      def setUp(self):
          super(TC_10_BackupVMMixin, self).setUp()
          self.backupvm = self.app.add_new_vm(
              vanir.vm.appvm.AppVM,
              label='red',
              name=self.make_vm_name('backupvm'),
              template=self.template
          )
          self.loop.run_until_complete(self.backupvm.create_on_disk())
  
      def test_100_send_to_vm_file_with_spaces(self):
          vms = self.create_backup_vms()
          orig_hashes = self.vm_checksum(vms)
          vms_info = self.get_vms_info(vms)
          try:
              self.loop.run_until_complete(self.backupvm.start())
              self.loop.run_until_complete(self.backupvm.run_for_stdio(
                  "mkdir '/var/tmp/backup directory'"))
              self.make_backup(vms, target_vm=self.backupvm,
                  compressed=True,
                  target='/var/tmp/backup directory')
              self.remove_vms(reversed(vms))
              (backup_path, _) = self.loop.run_until_complete(
                  self.backupvm.run_for_stdio("ls /var/tmp/backup*/vanir-backup*"))
              backup_path = backup_path.decode().strip()
              self.restore_backup(source=backup_path,
                                  appvm=self.backupvm)
              self.assertCorrectlyRestored(vms_info, orig_hashes)
          finally:
              del vms
  
      def test_110_send_to_vm_command(self):
          vms = self.create_backup_vms()
          orig_hashes = self.vm_checksum(vms)
          vms_info = self.get_vms_info(vms)
          try:
              self.loop.run_until_complete(self.backupvm.start())
              self.make_backup(vms, target_vm=self.backupvm,
                  compressed=True,
                  target='dd of=/var/tmp/backup-test')
              self.remove_vms(reversed(vms))
              self.restore_backup(source='dd if=/var/tmp/backup-test',
                                  appvm=self.backupvm)
              self.assertCorrectlyRestored(vms_info, orig_hashes)
          finally:
              del vms
  
      def test_110_send_to_vm_no_space(self):
          """
          Check whether backup properly report failure when no enough space is
          available
          :return:
          """
          vms = self.create_backup_vms()
          try:
              self.loop.run_until_complete(self.backupvm.start())
              self.loop.run_until_complete(self.backupvm.run_for_stdio(
                  # Debian 7 has too old losetup to handle loop-control device
                  "mknod /dev/loop0 b 7 0;"
                  "truncate -s 50M /home/user/backup.img && "
                  "mkfs.ext4 -F /home/user/backup.img && "
                  "mkdir /home/user/backup && "
                  "mount /home/user/backup.img /home/user/backup -o loop &&"
                  "chmod 777 /home/user/backup",
                  user="root"))
              with self.assertRaises(vanir.exc.VanirException):
                  self.make_backup(vms, target_vm=self.backupvm,
                      compressed=False,
                      target='/home/user/backup',
                      expect_failure=True)
          finally:
              del vms
  
  
  def create_testcases_for_templates():
      return vanir.tests.create_testcases_for_templates('TC_10_BackupVM',
          TC_10_BackupVMMixin, vanir.tests.SystemTestCase,
          module=sys.modules[__name__])
  
  def load_tests(loader, tests, pattern):
      tests.addTests(loader.loadTestsFromNames(
          create_testcases_for_templates()))
      return tests
  
  vanir.tests.maybe_create_testcases_on_import(create_testcases_for_templates)
  