text: # -*- coding: utf-8 -*-
  
  from __future__ import (unicode_literals, division, absolute_import,
                          print_function)
  import six
  from six.moves import range
  
  __license__   = 'GPL v3'
  __copyright__ = '2019, Jim Miller'
  __docformat__ = 'restructuredtext en'
  
  from .fanficfare.six import string_types, text_type as unicode
  
  # import cProfile
  
  # def do_cprofile(func):
  #     def profiled_func(*args, **kwargs):
  #         profile = cProfile.Profile()
  #         try:
  #             profile.enable()
  #             result = func(*args, **kwargs)
  #             profile.disable()
  #             return result
  #         finally:
  #             profile.print_stats()
  #     return profiled_func
  
  
  import logging
  logger = logging.getLogger(__name__)
  
  import os, copy, threading, re, platform, sys
  from io import BytesIO
  from functools import partial
  from datetime import datetime, time, date
  from string import Template
  import email
  import traceback
  
  try:
      from PyQt5.Qt import (QApplication, QMenu, QTimer, Qt, QToolButton)
      from PyQt5.QtCore import QBuffer
  except ImportError as e:
      from PyQt4.Qt import (QApplication, QMenu, QTimer, Qt, QToolButton)
      from PyQt4.QtCore import QBuffer
  
  from calibre.constants import numeric_version as calibre_version
  
  from calibre.ptempfile import PersistentTemporaryFile, PersistentTemporaryDirectory, remove_dir
  from calibre.ebooks.metadata import MetaInformation
  from calibre.ebooks.metadata.meta import get_metadata as calibre_get_metadata
  from calibre.gui2 import error_dialog, warning_dialog, question_dialog, info_dialog
  from calibre.gui2.dialogs.message_box import ViewLog
  from calibre.gui2.dialogs.confirm_delete import confirm
  from calibre.utils.config import prefs as calibre_prefs
  from calibre.utils.date import local_tz
  from calibre.constants import config_dir as calibre_config_dir
  
  # The class that all interface action plugins must inherit from
  from calibre.gui2.actions import InterfaceAction
  
  # pulls in translation files for _() strings
  try:
      load_translations()
  except NameError:
      pass # load_translations() added in calibre 1.9
  
  try:
      # should be present from cal2.3.0 on.
      from calibre.ebooks.covers import generate_cover as cal_generate_cover
      HAS_CALGC=True
  except:
      HAS_CALGC=False
  
  from calibre.library.field_metadata import FieldMetadata
  field_metadata = FieldMetadata()
  
  from calibre_plugins.fanficfare_plugin.common_utils import (
      set_plugin_icon_resources, get_icon, create_menu_action_unique,
      get_library_uuid, busy_cursor)
  
  from calibre_plugins.fanficfare_plugin.fanficfare import (
      adapters, exceptions)
  
  from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import (
      get_dcsource, get_dcsource_chaptercount, get_story_url_from_epub_html,
      get_story_url_from_zip_html, reset_orig_chapters_epub, get_cover_data)
  
  from calibre_plugins.fanficfare_plugin.fanficfare.geturls import (
      get_urls_from_page, get_urls_from_html,get_urls_from_text,
      get_urls_from_imap)
  
  from calibre_plugins.fanficfare_plugin.fff_util import (
      get_fff_adapter, get_fff_config, get_fff_personalini)
  
  from calibre_plugins.fanficfare_plugin.config import (
      permitted_values, rejecturllist, STD_COLS_SKIP)
  
  from calibre_plugins.fanficfare_plugin.prefs import (
      prefs, SAVE_YES, SAVE_NO, SAVE_YES_IF_IMG, SAVE_YES_UNLESS_IMG)
  
  from calibre_plugins.fanficfare_plugin.dialogs import (
      AddNewDialog, UpdateExistingDialog,
      LoopProgressDialog, UserPassDialog, AboutDialog, CollectURLDialog,
      RejectListDialog, EmailPassDialog,
      OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY,
      CALIBREONLYSAVECOL,
      NotGoingToDownload, RejectUrlEntry )
  
  # because calibre immediately transforms html into zip and don't want
  # to have an 'if html'.  db.has_format is cool with the case mismatch,
  # but if I'm doing it anyway...
  formmapping = {
      'epub':'EPUB',
      'mobi':'MOBI',
      'html':'ZIP',
      'txt':'TXT'
      }
  
  imagetypes = {
      'image/jpeg':'jpg',
      'image/png':'png',
      'image/gif':'gif',
      'image/svg+xml':'svg',
      }
  
  PLUGIN_ICONS = ['images/icon.png']
  
  class FanFicFarePlugin(InterfaceAction):
  
      name = 'FanFicFare'
  
      # Declare the main action associated with this plugin
      # The keyboard shortcut can be None if you dont want to use a keyboard
      # shortcut. Remember that currently calibre has no central management for
      # keyboard shortcuts, so try to use an unusual/unused shortcut.
      # (text, icon_path, tooltip, keyboard shortcut)
      # icon_path isn't in the zip--icon loaded below.
      action_spec = (_('FanFicFare'), None,
                     _('Download FanFiction stories from various web sites'), ())
      # None for keyboard shortcut doesn't allow shortcut.  () does, there just isn't one yet
  
      action_type = 'global'
      # make button menu drop down only
      #popup_type = QToolButton.InstantPopup
  
      def genesis(self):
  
          # This method is called once per plugin, do initial setup here
  
          # Read the plugin icons and store for potential sharing with the config widget
          icon_resources = self.load_resources(PLUGIN_ICONS)
          set_plugin_icon_resources(self.name, icon_resources)
  
          base = self.interface_action_base_plugin
          self.version = base.name+" v%d.%d.%d"%base.version
  
          # Set the icon for this interface action
          # The get_icons function is a builtin function defined for all your
          # plugin code. It loads icons from the plugin zip file. It returns
          # QIcon objects, if you want the actual data, use the analogous
          # get_resources builtin function.
  
          # Note that if you are loading more than one icon, for performance, you
          # should pass a list of names to get_icons. In this case, get_icons
          # will return a dictionary mapping names to QIcons. Names that
          # are not found in the zip file will result in null QIcons.
          icon = get_icon('images/icon.png')
  
          self.qaction.setText(_('FanFicFare'))
  
          # The qaction is automatically created from the action_spec defined
          # above
          self.qaction.setIcon(icon)
  
          # Call function when plugin triggered.
          self.qaction.triggered.connect(self.plugin_button)
  
          # Assign our menu to this action
          self.menu = QMenu(self.gui)
          # menu_actions is to keep a live reference to the menu items
          # to prevent GC removing it and so rebuild_menus has a list
          self.menu_actions = []
          self.qaction.setMenu(self.menu)
          self.menus_lock = threading.RLock()
          self.menu.aboutToShow.connect(self.about_to_show_menu)
  
          self.imap_pass = None
  
      def initialization_complete(self):
          # otherwise configured hot keys won't work until the menu's
          # been displayed once.
          self.rebuild_menus()
          self.set_popup_mode()
          self.add_new_dialog = AddNewDialog(self.gui,
                                             prefs,
                                             self.qaction.icon())
  
      ## Kludgey, yes, but with the real configuration inside the
      ## library now, how else would a user be able to change this
      ## setting if it's crashing calibre?
      def check_macmenuhack(self):
          try:
              return self.macmenuhack
          except:
              file_path = os.path.join(calibre_config_dir,
                                       *("plugins/fanficfare_macmenuhack.txt".split('/')))
              file_path = os.path.abspath(file_path)
              logger.debug("Plugin %s macmenuhack file_path:%s"%(self.name,file_path))
              self.macmenuhack = os.access(file_path, os.F_OK)
              return self.macmenuhack
  
      accepts_drops = True
  
      def accept_enter_event(self, event, mime_data):
          if mime_data.hasFormat("application/calibre+from_library") or \
                  mime_data.hasFormat("text/plain") or \
                  mime_data.hasFormat("text/uri-list"):
              return True
  
          return False
  
      def accept_drag_move_event(self, event, mime_data):
          return self.accept_enter_event(event, mime_data)
  
      def drop_event(self, event, mime_data):
  
          dropped_ids=None
          urllist=[]
  
          mime = 'application/calibre+from_library'
          if mime_data.hasFormat(mime):
              dropped_ids = [ int(x) for x in str(mime_data.data(mime)).split() ]
  
          mimetype='text/uri-list'
          filelist="%s"%event.mimeData().data(mimetype)
          if filelist:
              for f in filelist.splitlines():
                  #print("filename:%s"%f)
                  if f.endswith(".eml"):
                      fhandle = six.moves.urllib.request.urlopen(f)
                      msg = email.message_from_file(fhandle)
                      if msg.is_multipart():
                          for part in msg.walk():
                              #print("part type:%s"%part.get_content_type())
                              if part.get_content_type() == "text/html":
                                  #print("URL list:%s"%get_urls_from_data(part.get_payload(decode=True)))
                                  urllist.extend(get_urls_from_html(part.get_payload(decode=True)))
                              if part.get_content_type() == "text/plain":
                                  #print("part content:text/plain")
                                  #print("part content:%s"%part.get_payload(decode=True))
                                  urllist.extend(get_urls_from_text(part.get_payload(decode=True)))
                      else:
                          urllist.extend(get_urls_from_text("%s"%msg))
                  else:
                      urllist.extend(get_urls_from_text(f))
          else:
              mimetype='text/plain'
              if mime_data.hasFormat(mimetype):
                  #print("text/plain:%s"%event.mimeData().data(mimetype))
                  urllist.extend(get_urls_from_text(event.mimeData().data(mimetype)))
  
          # print("urllist:%s\ndropped_ids:%s"%(urllist,dropped_ids))
          if urllist or dropped_ids:
              QTimer.singleShot(1, partial(self.do_drop,
                                           dropped_ids=dropped_ids,
                                           urllist=urllist))
              return True
  
          return False
  
      def do_drop(self,dropped_ids=None,urllist=None):
          # shouldn't ever be both.
          if dropped_ids:
              self.update_dialog(dropped_ids)
          elif urllist:
              self.add_dialog("\n".join(urllist))
  
      def about_to_show_menu(self):
          self.rebuild_menus()
  
      def library_changed(self, db):
          # We need to reset our menus after switching libraries
          self.rebuild_menus()
          self.set_popup_mode()
          rejecturllist.clear_cache()
          self.imap_pass = None
  
      def rebuild_menus(self):
          with self.menus_lock:
              do_user_config = self.interface_action_base_plugin.do_user_config
              self.menu.clear()
  
              for action in self.menu_actions:
                  self.gui.keyboard.unregister_shortcut(action.calibre_shortcut_unique_name)
                  # starting in calibre 2.10.0, actions are registers at
                  # the top gui level for OSX' benefit.
                  if calibre_version >= (2,10,0):
                      self.gui.removeAction(action)
              self.menu_actions = []
  
              self.add_action = self.create_menu_item_ex(self.menu, _('&Download from URLs'), image='plus.png',
                                                         unique_name='Download FanFiction Books from URLs',
                                                         shortcut_name=_('Download FanFiction Books from URLs'),
                                                         triggered=self.add_dialog )
  
              self.update_action = self.create_menu_item_ex(self.menu, _('&Update Existing FanFiction Books'), image='plusplus.png',
                                                            unique_name='&Update Existing FanFiction Books',
                                                            triggered=self.update_dialog)
  
              if prefs['imapserver'] and prefs['imapuser'] and prefs['imapfolder']:
                  self.get_list_imap_action = self.create_menu_item_ex(self.menu, _('Get Story URLs from &Email'), image='view.png',
                                                                       unique_name='Get Story URLs from IMAP',
                                                                       triggered=self.get_urls_from_imap_menu)
  
              self.get_list_url_action = self.create_menu_item_ex(self.menu, _('Get Story URLs from Web Page'), image='view.png',
                                                                  unique_name='Get Story URLs from Web Page',
                                                                  triggered=self.get_urls_from_page_menu)
  
              if self.get_epubmerge_plugin():
                  self.menu.addSeparator()
                  self.makeanth_action = self.create_menu_item_ex(self.menu, _('&Make Anthology Epub from URLs'), image='plusplus.png',
                                                                  unique_name='Make FanFiction Anthology Epub from URLs',
                                                                  shortcut_name=_('Make FanFiction Anthology Epub from URLs'),
                                                                  triggered=partial(self.add_dialog,merge=True) )
  
                  self.get_anthlist_url_action = self.create_menu_item_ex(self.menu, _('Make Anthology Epub from Web Page'), image='view.png',
                                                                          unique_name='Make FanFiction Anthology Epub from Web Page',
                                                                          shortcut_name=_('Make FanFiction Anthology Epub from Web Page'),
                                                                          triggered=partial(self.get_urls_from_page_menu,anthology=True))
  
                  self.updateanth_action = self.create_menu_item_ex(self.menu, _('Update Anthology Epub'), image='plusplus.png',
                                                                    unique_name='Update FanFiction Anthology Epub',
                                                                    shortcut_name=_('Update FanFiction Anthology Epub'),
                                                                    triggered=self.update_anthology)
  
              if 'Reading List' in self.gui.iactions and (prefs['addtolists'] or prefs['addtoreadlists']) :
                  self.menu.addSeparator()
                  addmenutxt, rmmenutxt = None, None
                  if prefs['addtolists'] and prefs['addtoreadlists'] :
                      addmenutxt = _('Mark Unread: Add to "To Read" and "Send to Device" Lists')
                      if prefs['addtolistsonread']:
                          rmmenutxt = _('Mark Read: Remove from "To Read" and add to "Send to Device" Lists')
                      else:
                          rmmenutxt = _('Mark Read: Remove from "To Read" Lists')
                  elif prefs['addtolists'] :
                      addmenutxt = _('Add to "Send to Device" Lists')
                  elif prefs['addtoreadlists']:
                      addmenutxt = _('Mark Unread: Add to "To Read" Lists')
                      rmmenutxt = _('Mark Read: Remove from "To Read" Lists')
  
                  if addmenutxt:
                      self.add_send_action = self.create_menu_item_ex(self.menu, addmenutxt,
                                                                      unique_name='Add to "To Read" and "Send to Device" Lists',
                                                                      image='plusplus.png',
                                                                      triggered=partial(self.update_lists,add=True))
  
                  if rmmenutxt:
                      self.add_remove_action = self.create_menu_item_ex(self.menu, rmmenutxt,
                                                                        unique_name='Remove from "To Read" and add to "Send to Device" Lists',
                                                                        image='minusminus.png',
                                                                        triggered=partial(self.update_lists,add=False))
  
              self.menu.addSeparator()
              self.get_list_action = self.create_menu_item_ex(self.menu, _('Remove "New" Chapter Marks from Selected books'),
                                                              unique_name='Remove "(new)" chapter marks created by personal.ini <i>mark_new_chapters</i> setting.',
                                                              image='edit-undo.png',
                                                              triggered=self.unnew_books)
  
              self.menu.addSeparator()
              self.get_list_action = self.create_menu_item_ex(self.menu, _('Get Story URLs from Selected Books'),
                                                              unique_name='Get URLs from Selected Books',
                                                              image='bookmarks.png',
                                                              triggered=self.list_story_urls)
  
              self.reject_list_action = self.create_menu_item_ex(self.menu, _('Reject Selected Books'),
                                                                 unique_name='Reject Selected Books', image='rotate-right.png',
                                                                 triggered=self.reject_list_urls)
  
              # print("platform.system():%s"%platform.system())
              # print("platform.mac_ver()[0]:%s"%platform.mac_ver()[0])
              if not self.check_macmenuhack(): # not platform.mac_ver()[0]: # Some macs crash on these menu items for unknown reasons.
                  self.menu.addSeparator()
                  self.config_action = self.create_menu_item_ex(self.menu, _('&Configure FanFicFare'),
                                                                image= 'config.png',
                                                                unique_name='Configure FanFicFare',
                                                                shortcut_name=_('Configure FanFicFare'),
                                                                triggered=partial(do_user_config,parent=self.gui))
  
                  self.about_action = self.create_menu_item_ex(self.menu, _('About FanFicFare'),
                                                               image= 'images/icon.png',
                                                               unique_name='About FanFicFare',
                                                               shortcut_name=_('About FanFicFare'),
                                                               triggered=self.about)
  
              self.gui.keyboard.finalize()
  
  
      def about(self):
          # Get the about text from a file inside the plugin zip file
          # The get_resources function is a builtin function defined for all your
          # plugin code. It loads files from the plugin zip file. It returns
          # the bytes from the specified file.
          #
          # Note that if you are loading more than one file, for performance, you
          # should pass a list of names to get_resources. In this case,
          # get_resources will return a dictionary mapping names to bytes. Names that
          # are not found in the zip file will not be in the returned dictionary.
  
          text = get_resources('about.html').decode('utf-8')
          AboutDialog(self.gui,self.qaction.icon(),self.version + text).exec_()
  
      def create_menu_item_ex(self, parent_menu, menu_text, image=None, tooltip=None,
                             shortcut=None, triggered=None, is_checked=None, shortcut_name=None,
                             unique_name=None):
          #print("create_menu_item_ex before %s"%menu_text)
          ac = create_menu_action_unique(self, parent_menu, menu_text, image, tooltip,
                                         shortcut, triggered, is_checked, shortcut_name, unique_name)
          self.menu_actions.append(ac)
          #print("create_menu_item_ex after %s"%menu_text)
          return ac
  
      def is_library_view(self):
          # 0 = library, 1 = main, 2 = card_a, 3 = card_b
          return self.gui.stack.currentIndex() == 0
  
      def plugin_button(self):
          if self.is_library_view() and \
                  len(self.gui.library_view.get_selected_ids()) > 0 and \
                  prefs['updatedefault']:
              self.update_dialog()
          else:
              self.add_dialog()
  
      def set_popup_mode(self):
          if prefs['button_instantpopup']:
              self.popup_type = QToolButton.InstantPopup
          else:
              self.popup_type = QToolButton.MenuButtonPopup
          for bar in self.gui.bars_manager.bars:
              w = bar.widgetForAction(self.qaction)
              if w is not None:
                  w.setPopupMode(self.popup_type)
                  w.update()
          return
  
      def get_epubmerge_plugin(self):
          if 'EpubMerge' in self.gui.iactions and self.gui.iactions['EpubMerge'].interface_action_base_plugin.version >= (1,3,1):
              return self.gui.iactions['EpubMerge']
  
      def update_lists(self,add=True):
          if prefs['addtolists'] or prefs['addtoreadlists']:
              if not self.is_library_view():
                  self.gui.status_bar.show_message(_('Cannot Update Reading Lists from Device View'), 3000)
                  return
  
              if len(self.gui.library_view.get_selected_ids()) == 0:
                  self.gui.status_bar.show_message(_('No Selected Books to Update Reading Lists'), 3000)
                  return
  
              self.update_reading_lists(self.gui.library_view.get_selected_ids(),add)
              if not add and prefs['autounnew']:
                  self.unnew_books()
  
      def get_urls_from_imap_menu(self):
  
          if not prefs['imapserver'] or not prefs['imapuser'] or not prefs['imapfolder']:
              s=_('FanFicFare Email Settings are not configured.')
              info_dialog(self.gui, s, s, show=True, show_copy_button=False)
              return
  
          imap_pass = None
          if prefs['imappass']:
              imap_pass = prefs['imappass']
          elif self.imap_pass is not None:
              imap_pass = self.imap_pass
  
          if not imap_pass:
              d = EmailPassDialog(self.gui,prefs['imapuser'])
              d.exec_()
              if not d.status:
                  return
              imap_pass = d.get_pass()
              if prefs['imapsessionpass']:
                  self.imap_pass = imap_pass
  
          with busy_cursor():
              self.gui.status_bar.show_message(_('Fetching Story URLs from Email...'))
              url_list = get_urls_from_imap(prefs['imapserver'],
                                            prefs['imapuser'],
                                            imap_pass,
                                            prefs['imapfolder'],
                                            prefs['imapmarkread'],)
  
              ## reject will now be redundant with reject check inside
              ## prep_downloads because of change-able story URLs.
              ## Keeping here to because it's the far more common case.
              reject_list=set()
              if prefs['auto_reject_from_email']:
                  # need to normalize for reject list.
                  reject_list = set([x for x in url_list if rejecturllist.check(adapters.getNormalStoryURL(x))])
              url_list = url_list - reject_list
  
              ## feature for update-only - check url_list with
              ## self.do_id_search(url)
              notupdate_list = set()
              if prefs['update_existing_only_from_email']:
                  notupdate_list = set([x for x in url_list if not self.do_id_search(adapters.getNormalStoryURL(x))])
              url_list = url_list - notupdate_list
  
              self.gui.status_bar.show_message(_('No Valid Story URLs Found in Unread Emails.'),3000)
  
          if prefs['download_from_email_immediately']:
              ## do imap fetch w/o GUI elements
              if url_list:
                  self.prep_downloads({
                          'fileform': prefs['fileform'],
                          'collision': prefs['collision'],
                          'updatemeta': prefs['updatemeta'],
                          'bgmeta': False,
                          'updateepubcover': prefs['updateepubcover'],
                          'smarten_punctuation':prefs['smarten_punctuation'],
                          'do_wordcount':prefs['do_wordcount'],
                          'add_tag':prefs['imaptags'],
                          },"\n".join(url_list))
              else:
                  self.gui.status_bar.show_message(_('Finished Fetching Story URLs from Email.'),3000)
  
          else:
              if url_list:
                  if prefs['imaptags']:
                      message="<p>"+_("Tag(s) <b><i>%s</i></b> will be added to all stories downloaded in the next dialog, including any story URLs you add manually.")%prefs['imaptags']+"</p>"
                      confirm(message,'fff_add_imaptags', self.gui, show_cancel_button=False)
                  self.add_dialog("\n".join(url_list),merge=False,add_tag=prefs['imaptags'])
              else:
                  msg = _('No Valid Story URLs Found in Unread Emails.')
                  if reject_list:
                      msg = msg + '<p>'+(_('(%d Story URLs Skipped, on Rejected URL List)')%len(reject_list))+'</p>'
                  if notupdate_list:
                      msg = msg + '<p>'+(_("(%d Story URLs Skipped, no Existing Book in Library)")%len(notupdate_list))+'</p>'
                  info_dialog(self.gui, _('Get Story URLs from Email'),
                              msg,
                              show=True,
                              show_copy_button=False)
  
      def get_urls_from_page_menu(self,anthology=False):
  
          urltxt = ""
          if prefs['urlsfromclip']:
              try:
                  urltxt = self.get_urls_clip(storyurls=False)[0]
              except:
                  urltxt = ""
  
          d = CollectURLDialog(self.gui,_("Get Story URLs from Web Page"),urltxt,
                               anthology=(anthology or self.get_epubmerge_plugin()),
                               indiv=not anthology)
          d.exec_()
          if not d.status:
              return
          url = u"%s"%d.url.text()
  
          with busy_cursor():
              self.gui.status_bar.show_message(_('Fetching Story URLs from Page...'))
  
              url_list = self.get_urls_from_page(url)
  
              self.gui.status_bar.show_message(_('Finished Fetching Story URLs from Page.'),3000)
  
          if url_list:
              self.add_dialog("\n".join(url_list),merge=d.anthology,anthology_url=url)
          else:
              info_dialog(self.gui, _('List of Story URLs'),
                          _('No Valid Story URLs found on given page.'),
                          show=True,
                          show_copy_button=False)
  
      def get_urls_from_page(self,url):
          logger.debug("get_urls_from_page URL:%s"%url)
          ## some sites hide mature links unless logged in.
          if 'archiveofourown.org' in url or 'fimfiction.net' in url:
              configuration = get_fff_config(url)
          else:
              configuration = None
          return get_urls_from_page(url,configuration)
  
      def list_story_urls(self):
          '''Get list of URLs from existing books.'''
          if not self.gui.current_view().selectionModel().selectedRows() :
              self.gui.status_bar.show_message(_('No Selected Books to Get URLs From'),
                                               3000)
              return
  
          if self.is_library_view():
              book_list = [ self.make_book_id_only(x) for x in
                            self.gui.library_view.get_selected_ids() ]
  
          else: # device view, get from epubs on device.
              book_list = [ self.make_book_from_device_row(x) for x in
                            self.gui.current_view().selectionModel().selectedRows() ]
  
          LoopProgressDialog(self.gui,
                             book_list,
                             partial(self.get_list_story_urls_loop, db=self.gui.current_db),
                             self.get_list_story_urls_finish,
                             init_label=_("Collecting URLs for stories..."),
                             win_title=_("Get URLs for stories"),
                             status_prefix=_("URL retrieved"))
  
      def get_list_story_urls_loop(self,book,db=None):
          if book['calibre_id']:
              book['url'] = self.get_story_url(db,book_id=book['calibre_id'])
          elif book['path']:
              book['url'] = self.get_story_url(db,path=book['path'])
  
          if book['url'] == None:
              book['good']=False
              book['status']=_('Bad')
          else:
              book['good']=True
  
      def get_list_story_urls_finish(self, book_list):
          url_list = [ x['url'] for x in book_list if x['good'] ]
          if url_list:
              d = ViewLog(_("List of Story URLs"),"\n".join(url_list),parent=self.gui)
              d.setWindowIcon(get_icon('bookmarks.png'))
              d.exec_()
          else:
              info_dialog(self.gui, _('List of URLs'),
                          _('No Story URLs found in selected books.'),
                          show=True,
                          show_copy_button=False)
  
      def unnew_books(self):
          '''Get list of URLs from existing books.'''
          if not self.is_library_view():
              self.gui.status_bar.show_message(_('Can only UnNew books in library'),
                                               3000)
              return
  
          if not self.gui.current_view().selectionModel().selectedRows() :
              self.gui.status_bar.show_message(_('No Selected Books to Get URLs From'),
                                               3000)
              return
  
          book_list = [ self.make_book_id_only(x) for x in
                        self.gui.library_view.get_selected_ids() ]
  
          tdir = PersistentTemporaryDirectory(prefix='fanficfare_')
          LoopProgressDialog(self.gui,
                             book_list,
                             partial(self.get_unnew_books_loop, db=self.gui.current_db, tdir=tdir),
                             partial(self.get_unnew_books_finish, tdir=tdir),
                             init_label=_("UnNewing books..."),
                             win_title=_("UnNew Books"),
                             status_prefix=_("Books UnNewed"))
  
      def get_unnew_books_loop(self,book,db=None,tdir=None):
  
          book['changed']=False
          if book['calibre_id'] and db.has_format(book['calibre_id'],'EPUB',index_is_id=True) and self.get_story_url(db,book['calibre_id']):
              tmp = PersistentTemporaryFile(prefix='%s-'%book['calibre_id'],
                                            suffix='.epub',
                                            dir=tdir)
              db.copy_format_to(book['calibre_id'],'EPUB',tmp,index_is_id=True)
  
              unnewtmp = PersistentTemporaryFile(prefix='unnew-%s-'%book['calibre_id'],
                                                 suffix='.epub',
                                                 dir=tdir)
              book['changed']=reset_orig_chapters_epub(tmp,unnewtmp)
              if book['changed']:
                  try:
                      # Remove before adding so the previous version is in the
                      # OS Trash (on Windows, at least)
                      db.remove_format(book['calibre_id'],'EPUB', index_is_id=True)
                  except:
                      pass
                  db.add_format_with_hooks(book['calibre_id'],
                                           'EPUB',
                                           unnewtmp,
                                           index_is_id=True)
                  if prefs['deleteotherforms']:
                      fmts = db.formats(book['calibre_id'], index_is_id=True).split(',')
                      for fmt in fmts:
                          if fmt.lower() != formmapping['epub'].lower():
                              logger.debug("deleteotherforms remove f:"+fmt)
                              db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
                  elif prefs['autoconvert']:
                      ## 'Convert Book'.auto_convert_auto_add doesn't convert if
                      ## the format is already there.
                      fmt = calibre_prefs['output_format']
                      # delete if there, but not if the format we just made.
                      if fmt.lower() != 'epub' and db.has_format(book['calibre_id'],fmt,index_is_id=True):
                          logger.debug("autoconvert remove f:"+fmt)
                          db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
  
      def get_unnew_books_finish(self, book_list, tdir=None):
          remove_dir(tdir)
          if prefs['autoconvert']:
              changed_ids = [ x['calibre_id'] for x in book_list if x['changed'] ]
              if changed_ids:
                  logger.debug(_('Starting auto conversion of %d books.')%(len(changed_ids)))
                  self.gui.status_bar.show_message(_('Starting auto conversion of %d books.')%(len(changed_ids)), 3000)
                  self.gui.iactions['Convert Books'].auto_convert_auto_add(changed_ids)
  
      def reject_list_urls(self):
          if self.is_library_view():
              book_list = [ self.make_book_id_only(x) for x in
                            self.gui.library_view.get_selected_ids() ]
  
          else: # device view, get from epubs on device.
              view = self.gui.current_view()
              rows = view.selectionModel().selectedRows()
              #paths = view.model().paths(rows)
              book_list = [ self.make_book_from_device_row(x) for x in rows ]
  
          if len(book_list) == 0 :
              self.gui.status_bar.show_message(_('No Selected Books have URLs to Reject'), 3000)
              return
  
          # Progbar because fetching urls from device epubs can be slow.
          LoopProgressDialog(self.gui,
                             book_list,
                             partial(self.reject_list_urls_loop, db=self.gui.current_db),
                             self.reject_list_urls_finish,
                             init_label=_("Collecting URLs for Reject List..."),
                             win_title=_("Get URLs for Reject List"),
                             status_prefix=_("URL retrieved"))
  
      def reject_list_urls_loop(self,book,db=None):
          self.get_list_story_urls_loop(book,db) # common with get_list_story_urls_loop
          if book['calibre_id']:
              # want title/author, too, for rejects.
              self.populate_book_from_calibre_id(book,db)
          if book['url']:
              # get existing note, if on rejected list.
              book['oldrejnote']=rejecturllist.get_note(book['url'])
  
      def reject_list_urls_finish(self, book_list):
  
          # construct reject list of objects
          reject_list = [ RejectUrlEntry(x['url'],
                                         x['oldrejnote'],
                                         x['title'],
                                         ', '.join(x['author']),
                                         book_id=x['calibre_id'])
                          for x in book_list if x['good'] ]
          if reject_list:
              d = RejectListDialog(self.gui,reject_list,
                                   rejectreasons=rejecturllist.get_reject_reasons())
              d.exec_()
  
              if d.result() != d.Accepted:
                  return
  
              rejecturllist.add(d.get_reject_list())
  
              if d.get_deletebooks():
                  self.gui.iactions['Remove Books'].do_library_delete(d.get_reject_list_ids())
  
          else:
              message="<p>"+_("Rejecting FanFicFare URLs: None of the books selected have FanFiction URLs.")+"</p><p>"+_("Proceed to Remove?")+"</p>"
              if confirm(message,'fff_reject_non_fanfiction', self.gui):
                  self.gui.iactions['Remove Books'].delete_books()
  
      def add_dialog(self,url_list_text=None,merge=False,anthology_url=None,add_tag=None):
          'Both new individual stories and new anthologies are created here.'
  
          if not url_list_text:
              url_list = self.get_urls_clip()
              url_list_text = "\n".join(url_list)
  
          # AddNewDialog collects URLs, format and presents buttons.
          # add_new_dialog is modeless and reused, both for new stories
          # and anthologies, and for updating existing anthologies.
          self.add_new_dialog.show_dialog(url_list_text,
                                          self.prep_downloads,
                                          merge=merge,
                                          newmerge=True,
                                          extraoptions={'anthology_url':anthology_url,'add_tag':add_tag})
  
      def update_anthology(self):
          if not self.get_epubmerge_plugin():
              self.gui.status_bar.show_message(_('Cannot Make Anthologys without %s')%'EpubMerge 1.3.1+', 3000)
              return
  
          if not self.is_library_view():
              self.gui.status_bar.show_message(_('Cannot Update Books from Device View'), 3000)
              return
  
          if len(self.gui.library_view.get_selected_ids()) != 1:
              self.gui.status_bar.show_message(_('Can only update 1 anthology at a time'), 3000)
              return
  
          db = self.gui.current_db
          book_id = self.gui.library_view.get_selected_ids()[0]
          mergebook = self.make_book_id_only(book_id)
          self.populate_book_from_calibre_id(mergebook, db)
  
          if not db.has_format(book_id,'EPUB',index_is_id=True):
              self.gui.status_bar.show_message(_('Can only Update Epub Anthologies'), 3000)
              return
  
          tdir = PersistentTemporaryDirectory(prefix='fff_anthology_')
          logger.debug("tdir:\n%s"%tdir)
  
          bookepubio = BytesIO(db.format(book_id,'EPUB',index_is_id=True))
  
          filenames = self.get_epubmerge_plugin().do_unmerge(bookepubio,tdir)
          urlmapfile = {}
          url_list = []
          for f in filenames:
              url = adapters.getNormalStoryURL(get_dcsource(f))
              if url:
                  urlmapfile[url]=f
                  url_list.append(url)
  
          if not filenames or len(filenames) != len (url_list):
              info_dialog(self.gui, _("Cannot Update Anthology"),
                          "<p>"+_("Cannot Update Anthology")+"</p><p>"+_("Book isn't an FanFicFare Anthology or contains book(s) without valid Story URLs."),
                          show=True,
                          show_copy_button=False)
              remove_dir(tdir)
              return
  
          with busy_cursor():
              self.gui.status_bar.show_message(_('Fetching Story URLs for Series...'))
  
              # get list from identifiers:url/uri if present, but only if
              # it's *not* a valid story URL.
              mergeurl = self.get_story_url(db,book_id)
              if mergeurl and not self.is_good_downloader_url(mergeurl):
                  url_list = [ adapters.getNormalStoryURL(url) for url in self.get_urls_from_page(mergeurl) ]
  
              url_list_text = "\n".join(url_list)
  
              self.gui.status_bar.show_message(_('Finished Fetching Story URLs for Series.'),3000)
  
          #print("urlmapfile:%s"%urlmapfile)
  
          # AddNewDialog collects URLs, format and presents buttons.
          # add_new_dialog is modeless and reused, both for new stories
          # and anthologies, and for updating existing anthologies.
          self.add_new_dialog.show_dialog(url_list_text,
                                          self.prep_anthology_downloads,
                                          show=False,
                                          merge=True,
                                          newmerge=False,
                                          extrapayload=urlmapfile,
                                          extraoptions={'tdir':tdir,
                                                        'mergebook':mergebook})
          # Need to use AddNewDialog modal here because it's an update
          # of an existing book.  Don't want the user deleting it or
          # switching libraries on us.
          self.add_new_dialog.exec_()
  
  
      def prep_anthology_downloads(self, options, update_books,
                                   merge=False, urlmapfile=None):
  
          if isinstance(update_books, string_types):
              url_list = split_text_to_urls(update_books)
              update_books = self.convert_urls_to_books(url_list)
  
          for j, book in enumerate(update_books):
              url = book['url']
              book['listorder'] = j
              if url in urlmapfile:
                  #print("found epub for %s"%url)
                  book['epub_for_update']=urlmapfile[url]
                  del urlmapfile[url]
              #else:
                  #print("didn't found epub for %s"%url)
  
          if urlmapfile:
              text = '''
                   <p>%s</p>
                   <p>%s</p>
                   <ul>
                   <li>%s</li>
                   </ul>
                   <p>%s</p>'''%(
                  _('There are %d stories in the current anthology that are <b>not</b> going to be kept if you go ahead.')%len(urlmapfile),
                  _('Story URLs that will be removed:'),
                  "</li><li>".join(urlmapfile.keys()),
                  _('Update anyway?'))
              if not question_dialog(self.gui, _('Stories Removed'),
                                 text, show_copy_button=False):
                  logger.debug("Canceling anthology update due to removed stories.")
                  return
  
          # Now that we've
          self.prep_downloads( options, update_books, merge=True )
  
      def update_dialog(self, id_list=None):
          if not self.is_library_view():
              self.gui.status_bar.show_message(_('Cannot Update Books from Device View'), 3000)
              return
  
          if not id_list:
              id_list = self.gui.library_view.get_selected_ids()
  
          if len(id_list) == 0:
              self.gui.status_bar.show_message(_('No Selected Books to Update'), 3000)
              return
          #print("update_dialog()")
  
          db = self.gui.current_db
          books = [ self.make_book_id_only(x) for x in id_list ]
  
          for j, book in enumerate(books):
              book['listorder'] = j
  
          LoopProgressDialog(self.gui,
                             books,
                             partial(self.populate_book_from_calibre_id, db=self.gui.current_db),
                             self.update_dialog_finish,
                             init_label=_("Collecting stories for update..."),
                             win_title=_("Get stories for updates"),
                             status_prefix=_("URL retrieved"))
  
          #books = self.convert_calibre_ids_to_books(db, book_ids)
          #print("update books:%s"%books)
  
      def update_dialog_finish(self,book_list):
          '''Present list to update and head to prep when done.'''
  
          d = UpdateExistingDialog(self.gui,
                                   _('Update Existing List'),
                                   prefs,
                                   self.qaction.icon(),
                                   book_list,
                                   )
          d.exec_()
          if d.result() != d.Accepted:
              return
  
          update_books = d.get_books()
  
          #print("update_books:%s"%update_books)
          #print("options:%s"%d.get_fff_options())
          # only if there's some good ones.
          if any(x['good'] for x in update_books):
              options = d.get_fff_options()
              self.prep_downloads( options, update_books )
  
      def get_urls_clip(self,storyurls=True):
          url_list = []
          if prefs['urlsfromclip']:
              for url in unicode(QApplication.instance().clipboard().text()).split():
                  if not storyurls or self.is_good_downloader_url(url):
                      url_list.append(url)
  
          return url_list
  
      def apply_settings(self):
          # No need to do anything with perfs here, but we could.
          prefs
  
      def do_id_search(self,url):
          # older idents can be uri vs url and have | instead of : after
          # http, plus many sites are now switching to https.
          return self.gui.current_db.search_getting_ids('identifiers:"~ur(i|l):~^%s$"'%re.sub(r'https?\\\:','https?(\:|\|)',re.escape(url)),None,use_virtual_library=False)
  
      def prep_downloads(self, options, books, merge=False, extrapayload=None):
          '''Fetch metadata for stories from servers, launch BG job when done.'''
  
          logger.debug("add_tag:%s"%options.get('add_tag',None))
  
          if isinstance(books, string_types):
              url_list = split_text_to_urls(books)
              books = self.convert_urls_to_books(url_list)
  
          ## for tweak_fg_sleep
          options['ffnetcount']=sum(1 for x in books if x['site']=='www.fanfiction.net')
  
          options['version'] = self.version
          logger.debug(self.version)
          options['personal.ini'] = get_fff_personalini()
          options['savemetacol'] = prefs['savemetacol']
  
          #print("prep_downloads:%s"%books)
  
          if 'tdir' not in options: # if merging an anthology, there's alread a tdir.
              # create and pass temp dir.
              tdir = PersistentTemporaryDirectory(prefix='fanficfare_')
              options['tdir']=tdir
  
          if any(x['good'] for x in books):
              if options['bgmeta']:
                  status_bar=_('Start queuing downloading for %s stories.')%len(books)
                  init_label=_("Queuing download for stories...")
                  win_title=_("Queuing download for stories")
                  status_prefix=_("Queued download for")
              else:
                  status_bar=_('Started fetching metadata for %s stories.')%len(books)
                  init_label=_("Fetching metadata for stories...")
                  win_title=_("Downloading metadata for stories")
                  status_prefix=_("Fetched metadata for")
  
              self.gui.status_bar.show_message(status_bar, 3000)
              LoopProgressDialog(self.gui,
                                 books,
                                 partial(self.prep_download_loop, options = options, merge=merge),
                                 partial(self.start_download_job, options = options, merge=merge),
                                 init_label=init_label,
                                 win_title=win_title,
                                 status_prefix=status_prefix)
          else:
              self.gui.status_bar.show_message(_('No valid story URLs entered.'), 3000)
          # LoopProgressDialog calls prep_download_loop for each 'good' story,
          # prep_download_loop updates book object for each with metadata from site,
          # LoopProgressDialog calls start_download_job at the end which goes
          # into the BG, or shows list if no 'good' books.
  
      def reject_url(self,merge,book):
          url = book['url']
          if not merge and rejecturllist.check(url): # skip reject list when merging.
              rejnote = rejecturllist.get_full_note(url)
              if prefs['reject_always'] or question_dialog(self.gui, _('Reject URL?'),'''
                        <h3>%s</h3>
                        <p>%s</p>
                        <p>"<b>%s</b>"</p>
                        <p>%s</p>
                        <p>%s</p>'''%(
                      _('Reject URL?'),
                      _('<b>%s</b> is on your Reject URL list:')%url,
                      rejnote,
                      _("Click '<b>Yes</b>' to Reject."),
                      _("Click '<b>No</b>' to download anyway.")),
                                 show_copy_button=False):
                  book['comment'] = _("Story on Reject URLs list (%s).")%rejnote
                  book['good']=False
                  book['icon']='rotate-right.png'
                  book['status'] = _('Rejected')
                  return True
              else:
                  if question_dialog(self.gui, _('Remove Reject URL?'),'''
                            <h3>%s</h3>
                            <p>%s</p>
                            <p>"<b>%s</b>"</p>
                            <p>%s</p>
                            <p>%s</p>'''%(
                          _("Remove URL from Reject List?"),
                          _('<b>%s</b> is on your Reject URL list:')%url,
                          rejnote,
                          _("Click '<b>Yes</b>' to remove it from the list,"),
                          _("Click '<b>No</b>' to leave it on the list.")),
                                     show_copy_button=False):
                      rejecturllist.remove(url)
          return False
  
      def get_story_metadata_only(self,adapter):
          url = adapter.url
          ## three tries, that's enough if both user/pass & is_adult needed,
          ## or a couple tries of one or the other
          for x in range(0,2):
              try:
                  adapter.getStoryMetadataOnly(get_cover=False)
              except exceptions.FailedToLogin as f:
                  logger.warn("Login Failed, Need Username/Password.")
                  userpass = UserPassDialog(self.gui,url,f)
                  userpass.exec_() # exec_ will make it act modal
                  if userpass.status:
                      adapter.username = userpass.user.text()
                      adapter.password = userpass.passwd.text()
  
              except exceptions.AdultCheckRequired:
                  if question_dialog(self.gui, _('Are You an Adult?'), '<p>'+
                                     _("%s requires that you be an adult.  Please confirm you are an adult in your locale:")%url,
                                     show_copy_button=False):
                      adapter.is_adult=True
  
          # let other exceptions percolate up.
          return adapter.getStoryMetadataOnly(get_cover=False)
  
      # @do_cprofile
      def prep_download_loop(self,book,
                             options={'fileform':'epub',
                                      'collision':ADDNEW,
                                      'updatemeta':True,
                                      'bgmeta':False,
                                      'updateepubcover':True},
                             merge=False):
          '''
          Update passed in book dict with metadata from website and
          necessary data.  To be called from LoopProgressDialog
          'loop'.  Also pops dialogs for is adult, user/pass.
          '''
  
          url = book['url']
          logger.debug("url:%s"%url)
          mi = None
  
          # The current database shown in the GUI
          # db is an instance of the class LibraryDatabase2 from database.py
          # This class has many, many methods that allow you to do a lot of
          # things.
          db = self.gui.current_db
  
          fileform  = options['fileform']
          collision = book['collision'] = options['collision']
          updatemeta= options['updatemeta']
          bgmeta= options['bgmeta']
          updateepubcover= options['updateepubcover']
  
          ## Check reject list.  Redundant with below for when story URL
          ## changes, but also kept here to avoid network hit in most
          ## common case where given url is story url.
          if self.reject_url(merge,book):
              return
  
          # Dialogs should prevent this case now.
          if collision in (UPDATE,UPDATEALWAYS) and fileform != 'epub':
              raise NotGoingToDownload(_("Cannot update non-epub format."))
  
          if not book['good']:
              # book has already been flagged bad for whatever reason.
              return
  
          adapter = get_fff_adapter(url,fileform)
          ## chapter range for title_chapter_range_pattern
          adapter.setChaptersRange(book['begin'],book['end'])
          ## save and share cookiejar and pagecache between all
          ## downloads.
          configuration = adapter.get_configuration()
          if 'pagecache' not in options:
              options['pagecache'] = configuration.get_empty_pagecache()
          configuration.set_pagecache(options['pagecache'])
          if 'cookiejar' not in options:
              options['cookiejar'] = configuration.get_empty_cookiejar()
          configuration.set_cookiejar(options['cookiejar'])
  
          if collision in (CALIBREONLY, CALIBREONLYSAVECOL):
              ## Getting metadata from configured column.
              custom_columns = self.gui.library_view.model().custom_columns
              if ( collision in (CALIBREONLYSAVECOL) and
                   prefs['savemetacol'] != '' and
                   prefs['savemetacol'] in custom_columns ):
  
                  savedmeta_book_id = book['calibre_id']
                  # won't have calibre_id if update by URL vs book.
                  if not savedmeta_book_id:
                      identicalbooks = self.do_id_search(url)
                      if len(identicalbooks) == 1:
                          savedmeta_book_id = identicalbooks.pop()
  
                  if savedmeta_book_id:
                      label = custom_columns[prefs['savemetacol']]['label']
                      savedmetadata = db.get_custom(savedmeta_book_id, label=label, index_is_id=True)
                  else:
                      savedmetadata = None
  
                  if savedmetadata:
                      # sets flag inside story so getStoryMetadataOnly won't hit server.
                      adapter.setStoryMetadata(savedmetadata)
  
              # let other exceptions percolate up.
              # bgmeta doesn't work with CALIBREONLY.
              story = self.get_story_metadata_only(adapter)
              bgmeta = False
          else:
              # reduce foreground sleep time for ffnet when few books.
              if 'ffnetcount' in options and \
                      adapter.getConfig('tweak_fg_sleep') and \
                      adapter.getSiteDomain() == 'www.fanfiction.net':
                  minslp = float(adapter.getConfig('min_fg_sleep'))
                  maxslp = float(adapter.getConfig('max_fg_sleep'))
                  dwnlds = float(adapter.getConfig('max_fg_sleep_at_downloads'))
                  m = (maxslp-minslp) / (dwnlds-1)
                  b = minslp - m
                  slp = min(maxslp,m*float(options['ffnetcount'])+b)
                  #print("m:%s b:%s = %s"%(m,b,slp))
                  configuration.set_sleep(slp)
  
              if not bgmeta:
                  story = self.get_story_metadata_only(adapter)
                  book['title'] = story.getMetadata('title')
                  book['author'] = [story.getMetadata('author')]
                  url = book['url'] = story.getMetadata('storyUrl', removeallentities=True)
  
              ## Check reject list.  Redundant with below for when story
              ## URL changes, but also kept here to avoid network hit in
              ## most common case where given url is story url.
              if self.reject_url(merge,book):
                  return
  
          # logger.debug("series:%s"%story.getMetadata('series'))
          # logger.debug("seriesUrl:%s"%story.getMetadata('seriesUrl'))
          # logger.debug("search seriesUrl:%s"%self.do_id_search(story.getMetadata('seriesUrl')))
          if not bgmeta:
              series = story.getMetadata('series')
              if not merge and series and prefs['checkforseriesurlid']:
                  # try to find *series anthology* by *seriesUrl* identifier url or uri first.
                  identicalbooks = self.do_id_search(story.getMetadata('seriesUrl'))
                  # print("identicalbooks:%s"%identicalbooks)
                  if len(identicalbooks) > 0 and (prefs['auto_reject_seriesurlid'] or \
                                                      question_dialog(self.gui, _('Skip Story?'),'''
                                                                    <h3>%s</h3>
                                                                    <p>%s</p>
                                                                    <p>%s</p>
                                                                    <p>%s</p>
                                                                 '''%(
                                                                 _('Skip Anthology Story?'),
                                                                 _('"<b>%s</b>" is in series "<b><a href="%s">%s</a></b>" that you have an anthology book for.')%(story.getMetadata('title'),story.getMetadata('seriesUrl'),series[:series.index(' [')]),
                                                                 _("Click '<b>Yes</b>' to Skip."),
                                                                 _("Click '<b>No</b>' to download anyway.")),
                                                                 show_copy_button=False)):
                      book['comment'] = _("Story in Series Anthology(%s).")%series
                      book['title'] = story.getMetadata('title')
                      book['author'] = [story.getMetadata('author')]
                      url = book['url'] = story.getMetadata('storyUrl', removeallentities=True)
                      book['good']=False
                      book['icon']='rotate-right.png'
                      book['status'] = _('Skipped')
                      return
  
          ################################################################################################################################################33
  
          book['is_adult'] = adapter.is_adult
          book['username'] = adapter.username
          book['password'] = adapter.password
  
          book['icon'] = 'plus.png'
          book['status'] = _('Add')
  
          if not bgmeta:
              # set PI version instead of default.
              if 'version' in options:
                  story.setMetadata('version',options['version'])
  
              # all_metadata duplicates some data, but also includes extra_entries, etc.
              book['all_metadata'] = story.getAllMetadata(removeallentities=True)
              if prefs['savemetacol'] != '':
                  # get metadata to save in configured column.
                  book['savemetacol'] = story.dump_html_metadata()
  
              book['title'] = story.getMetadata("title", removeallentities=True)
              book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
              book['publisher'] = story.getMetadata("publisher")
              url = book['url'] = story.getMetadata("storyUrl", removeallentities=True)
              book['tags'] = story.getSubjectTags(removeallentities=True)
              book['comments'] = story.get_sanitized_description()
              book['series'] = story.getMetadata("series", removeallentities=True)
  
              if story.getMetadataRaw('datePublished'):
                  book['pubdate'] = story.getMetadataRaw('datePublished').replace(tzinfo=local_tz)
              if story.getMetadataRaw('dateUpdated'):
                  book['updatedate'] = story.getMetadataRaw('dateUpdated').replace(tzinfo=local_tz)
              if story.getMetadataRaw('dateCreated'):
                  book['timestamp'] = story.getMetadataRaw('dateCreated').replace(tzinfo=local_tz)
              else:
                  book['timestamp'] = datetime.now().replace(tzinfo=local_tz) # need *something* there for calibre.
  
          if not merge:# skip all the collision code when d/ling for merging.
              if collision in (CALIBREONLY, CALIBREONLYSAVECOL):
                  book['icon'] = 'metadata.png'
                  book['status'] = _('Meta')
  
              book_id = None
  
              if book['calibre_id'] != None:
                  # updating an existing book.  Update mode applies.
                  logger.debug("update existing id:%s"%book['calibre_id'])
                  book_id = book['calibre_id']
                  # No handling needed: OVERWRITEALWAYS,CALIBREONLY
  
              # only care about collisions when not ADDNEW
              elif collision != ADDNEW:
                  # 'new' book from URL.  collision handling applies.
                  logger.debug("from URL(%s)"%url)
  
                  # try to find by identifier url or uri first.
                  identicalbooks = self.do_id_search(url)
                  # logger.debug("identicalbooks:%s"%identicalbooks)
                  if len(identicalbooks) < 1 and prefs['matchtitleauth']:
                      # find dups
                      mi = MetaInformation(book['title'],book['author'])
                      identicalbooks = db.find_identical_books(mi)
                      if len(identicalbooks) > 0:
                          logger.debug("existing found by title/author(s)")
  
                  else:
                      logger.debug("existing found by identifier URL")
  
                  if collision == SKIP and identicalbooks:
                      raise NotGoingToDownload(_("Skipping duplicate story."),"list_remove.png")
  
                  if len(identicalbooks) > 1:
                      raise NotGoingToDownload(_("More than one identical book by Identifier URL or title/author(s)--can't tell which book to update/overwrite."),"minusminus.png")
  
                  ## changed: add new book when CALIBREONLY if none found.
                  if collision in (CALIBREONLY, CALIBREONLYSAVECOL) and not identicalbooks:
                      logger.debug("No existing book for %s, changing collision to ADDNEW for this book only"%url)
                      collision = book['collision'] = ADDNEW
  
                  if len(identicalbooks)>0:
                      book_id = identicalbooks.pop()
                      book['calibre_id'] = book_id
                      book['icon'] = 'edit-redo.png'
                      book['status'] = _('Update')
  
                  if book_id and mi: # book_id and mi only set if matched by title/author.
                      liburl = self.get_story_url(db,book_id)
                      if book['url'] != liburl and \
                          not (book['url'].replace('https','http') == liburl): # several sites have been changing to
                                                                               # https now.  Don't flag when that's the only change.
                          tags = db.get_tags(book_id)
                          flag_tag = "FFF Frozen URL" # not translated so it works across languages.
                          if flag_tag in tags:
                              book['comment'] = _("Update declined due to differing story URL(%s)(%s tag present)")%(liburl,flag_tag)
                              book['good']=False
                              book['icon']='rotate-right.png'
                              book['status'] = _('Different URL')
                              return
                          if prefs['checkforurlchange'] and not question_dialog(self.gui, _('Change Story URL?'),'''
                                                    <h3>%s</h3>
                                                    <p>%s</p>
                                                    <p>%s</p>
                                                    <p>%s</p>
                                                    <p>%s</p>
                                                    <p>%s</p>'''%(
                                  _('Change Story URL?'),
                                  _('<b>%s</b> by <b>%s</b> is already in your library with a different source URL:')%(mi.title,', '.join(mi.author)),
                                  _('In library: <a href="%(liburl)s">%(liburl)s</a>')%{'liburl':liburl},
                                  _('New URL: <a href="%(newurl)s">%(newurl)s</a>')%{'newurl':book['url']},
                                  _("Click '<b>Yes</b>' to update/overwrite book with new URL."),
                                  _("Click '<b>No</b>' to skip updating/overwriting this book.")),
                                                 show_copy_button=False):
                              if question_dialog(self.gui, _('Download as New Book?'),'''
                                                    <h3>%s</h3>
                                                    <p>%s</p>
                                                    <p>%s</p>
                                                    <p>%s</p>
                                                    <p>%s</p>
                                                    <p>%s</p>'''%(
                                      _('Download as New Book?'),
                                      _('<b>%s</b> by <b>%s</b> is already in your library with a different source URL.')%(mi.title,', '.join(mi.author)),
                                      _('You chose not to update the existing book.  Do you want to add a new book for this URL?'),
                                      _('New URL: <a href="%(newurl)s">%(newurl)s</a>')%{'newurl':book['url']},
                                      _("Click '<b>Yes</b>' to a new book with new URL."),
                                      _("Click '<b>No</b>' to skip URL.")),
                                                 show_copy_button=False):
                                  book_id = None
                                  mi = None
                                  book['calibre_id'] = None
                              else:
                                  book['comment'] = _("Update declined by user due to differing story URL(%s)")%liburl
                                  book['good']=False
                                  book['icon']='rotate-right.png'
                                  book['status'] = _('Different URL')
                                  return
  
              if book_id != None and collision != ADDNEW:
                  if collision in (CALIBREONLY, CALIBREONLYSAVECOL):
                      book['comment'] = _('Metadata collected.')
                      # don't need temp file created below.
                      return
  
                  ## newer/chaptercount checks are the same for both:
                  # Update epub, but only if more chapters.
                  if not bgmeta and collision in (UPDATE,UPDATEALWAYS): # collision == UPDATE
                      # 'book' can exist without epub.  If there's no existing epub,
                      # let it go and it will download it.
                      if db.has_format(book_id,fileform,index_is_id=True):
                          (epuburl,chaptercount) = \
                              get_dcsource_chaptercount(BytesIO(db.format(book_id,'EPUB',
                                                                           index_is_id=True)))
                          #urlchaptercount = int(story.getMetadata('numChapters').replace(',',''))
                          # returns int adjusted for start-end range.
                          urlchaptercount = story.getChapterCount()
                          if chaptercount == urlchaptercount:
                              if collision == UPDATE:
                                  raise NotGoingToDownload(_("Already contains %d chapters.")%chaptercount,'edit-undo.png',showerror=False)
                          elif chaptercount > urlchaptercount:
                              raise NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update.") % (chaptercount,urlchaptercount),'dialog_error.png')
                          elif chaptercount == 0:
                              raise NotGoingToDownload(_("FanFicFare doesn't recognize chapters in existing epub, epub is probably from a different source. Use Overwrite to force update."),'dialog_error.png')
  
                  if collision == OVERWRITE and \
                          db.has_format(book_id,formmapping[fileform],index_is_id=True):
                      logger.debug("OVERWRITE file: "+db.format_abspath(book_id, formmapping[fileform], index_is_id=True))
                      fileupdated=datetime.fromtimestamp(os.stat(db.format_abspath(book_id, formmapping[fileform], index_is_id=True))[8])
                      logger.debug("OVERWRITE file updated: %s"%fileupdated)
                      book['fileupdated']=fileupdated
                      if not bgmeta:
                          # check make sure incoming is newer.
                          lastupdated=story.getMetadataRaw('dateUpdated')
                          logger.debug("OVERWRITE site updated: %s"%lastupdated)
  
                          # updated doesn't have time (or is midnight), use dates only.
                          # updated does have time, use full timestamps.
                          if (lastupdated.time() == time.min and fileupdated.date() > lastupdated.date()) or \
                                  (lastupdated.time() != time.min and fileupdated > lastupdated):
                              raise NotGoingToDownload(_("Not Overwriting, web site is not newer."),'edit-undo.png',showerror=False)
  
                  # For update, provide a tmp file copy of the existing epub so
                  # it can't change underneath us.  Now also overwrite for logpage preserve.
                  if collision in (UPDATE,UPDATEALWAYS,OVERWRITE,OVERWRITEALWAYS) and \
                          fileform == 'epub' and \
                          db.has_format(book['calibre_id'],'EPUB',index_is_id=True):
                      tmp = PersistentTemporaryFile(prefix='old-%s-'%book['calibre_id'],
                                                    suffix='.epub',
                                                    dir=options['tdir'])
                      db.copy_format_to(book_id,fileform,tmp,index_is_id=True)
                      logger.debug("existing epub tmp:"+tmp.name)
                      book['epub_for_update'] = tmp.name
  
              if book_id != None and prefs['injectseries']:
                  mi = db.get_metadata(book_id,index_is_id=True)
                  if not book['series'] and mi.series != None:
                      book['calibre_series'] = (mi.series,mi.series_index)
                      #print("calibre_series:%s [%s]"%book['calibre_series'])
  
          if book['good']: # there shouldn't be any !'good' books at this point.
  
              ## Filling calibre_std_* and calibre_cust_* metadata
              book['calibre_columns']={}
              if prefs['cal_cols_pass_in']:
                  # std columns
                  mi = db.get_metadata(book['calibre_id'],index_is_id=True)
                  # book['calibre_columns']['calibre_std_identifiers']=\
                  #     {'val':', '.join(["%s:%s"%(k,v) for (k,v) in mi.get_identifiers().iteritems()]),
                  #                      'label':_('Ids')}
                  for k in mi.standard_field_keys():
                  # for k in mi:
                      if k in STD_COLS_SKIP:
                          continue
                      (label,value,v,fmd) = mi.format_field_extended(k)
                      if not label and k in field_metadata:
                          label=field_metadata[k]['name']
                      key='calibre_std_'+k
  
                      # if k == 'user_categories':
                      #     value=u', '.join(mi.get(k))
                      #     label=_('User Categories')
  
                      if label: # only if it has a human readable name.
                          if value is None or not book['calibre_id']:
                              ## if existing book, populate existing calibre column
                              ## values in metadata, else '' to hide.
                              value=''
                          book['calibre_columns'][key]={'val':value,'label':label}
                          #logger.debug("%s(%s): %s"%(label,key,value))
  
                  # custom columns
                  for k, column in six.iteritems(self.gui.library_view.model().custom_columns):
                      if k != prefs['savemetacol']:
                          key='calibre_cust_'+k[1:]
                          label=column['name']
                          value=db.get_custom(book['calibre_id'],
                                              label=column['label'],
                                              index_is_id=True)
                          # custom always have name.
                          if value is None or not book['calibre_id']:
                              ## if existing book, populate existing calibre column
                              ## values in metadata, else '' to hide.
                              value=''
                          book['calibre_columns'][key]={'val':value,'label':label}
                          # logger.debug("%s(%s): %s"%(label,key,value))
  
              # if still 'good', make a temp file to write the output to.
              # For HTML format users, make the filename inside the zip something reasonable.
              # For crazy long titles/authors, limit it to 200chars.
              # For weird/OS-unsafe characters, use file safe only.
              try:
                  prefix = story.formatFileName("${title}-${author}-",allowunsafefilename=False)[:100]
              except NameError:
                  prefix = "bgmeta-"
              tmp = PersistentTemporaryFile(prefix=prefix,
                                            suffix='.'+options['fileform'],
                                            dir=options['tdir'])
              logger.debug("title:"+book['title'])
              logger.debug("outfile:"+tmp.name)
              book['outfile'] = tmp.name
  
          return
  
      def start_download_job(self,book_list,
                              options={'fileform':'epub',
                                       'collision':ADDNEW,
                                       'updatemeta':True,
                                       'bgmeta':False,
                                       'updateepubcover':True},
                              merge=False):
          '''
          Called by LoopProgressDialog to start story downloads BG processing.
          '''
          #print("start_download_job:book_list:%s"%book_list)
  
          ## No need to BG process when CALIBREONLY!  Fake it.  if
          ## CALIBREONLY, CALIBREONLYSAVECOL called on a url that isn't
          ## in the library, it's switched to ADDNEW for that one.  Only
          ## do NotJob version if not downloading any.
          calonly = True
          for book in book_list:
              if book['collision'] not in (CALIBREONLY, CALIBREONLYSAVECOL):
                  calonly = False
                  break
          if calonly:
              class NotJob(object):
                  def __init__(self,result):
                      self.failed=False
                      self.result=result
              notjob = NotJob(book_list)
              self.download_list_completed(notjob,options=options)
              return
  
          for book in book_list:
              if book['good']:
                  break
          else:
              ## No good stories to try to download, go straight to
              ## updating error col.
              msg = '''
                      <p>%s</p>
                      <p>%s</p>
                      <p>%s</p>'''%(
                  _('None of the <b>%d</b> URLs/stories given can be/need to be downloaded.')%len(book_list),
                  _('See log for details.'),
                  _('Proceed with updating your library(Error or Last Checked Columns, if configured)?'))
  
              htmllog='<html><body><table border="1"><tr><th>'+_('Status')+'</th><th>'+_('Title')+'</th><th>'+_('Author')+'</th><th>'+_('Comment')+'</th><th>URL</th></tr>'
              for book in book_list:
                  htmllog = htmllog + '<tr><td>' + '</td><td>'.join([escapehtml(book['status']),escapehtml(book['title']),escapehtml(", ".join(book['author'])),escapehtml(book['comment']),book['url']]) + '</td></tr>'
  
              htmllog = htmllog + '</table></body></html>'
  
              payload = ([], book_list, options)
  
              # log_viewer_unique_name implemented here: https://github.com/kovidgoyal/calibre/compare/v2.56.0...v2.57.0
              if calibre_version >= (2, 57, 0):
                  self.gui.proceed_question(self.update_error_column,
                                            payload, htmllog,
                                            _('FanFicFare log'), _('FanFicFare download ended'), msg,
                                            show_copy_button=False,
                                            log_viewer_unique_name="FanFicFare log viewer")
              else:
                  self.gui.proceed_question(self.update_error_column,
                                            payload, htmllog,
                                            _('FanFicFare log'), _('FanFicFare download ended'), msg,
                                            show_copy_button=False)
  
              return
  
          cookiejarfile = PersistentTemporaryFile(suffix='.cookiejar',
                                                  dir=options['tdir'])
          options['cookiejar'].save(cookiejarfile.name,
                                    ignore_discard=True,
                                    ignore_expires=True)
          options['cookiejarfile']=cookiejarfile.name
          del options['cookiejar'] ## can't be pickled.
  
          # pass the plugin path in for jobs.py to use for 'with:' to
          # get libs from plugin zip.
          options['plugin_path'] = self.interface_action_base_plugin.plugin_path
  
          func = 'arbitrary_n'
          cpus = self.gui.job_manager.server.pool_size
          args = ['calibre_plugins.fanficfare_plugin.jobs', 'do_download_worker',
                  (book_list, options, cpus, merge)]
          desc = _('Download %s FanFiction Book(s)') % sum(1 for x in book_list if x['good'])
          job = self.gui.job_manager.run_job(
                  self.Dispatcher(partial(self.download_list_completed,options=options,merge=merge)),
                  func, args=args,
                  description=desc)
  
          self.gui.jobs_pointer.start()
          self.gui.status_bar.show_message(_('Starting %d FanFicFare Downloads')%len(book_list),3000)
  
      def get_custom_col_label(self,col):
          custom_columns = self.gui.library_view.model().custom_columns
          if col and col in custom_columns:
              return custom_columns[col]['label']
          else:
              return None
  
      def update_books_loop(self,book,db=None,
                            options={'fileform':'epub',
                                     'collision':ADDNEW,
                                     'updatemeta':True,
                                     'bgmeta':False,
                                     'updateepubcover':True},
                            errorcol_label=None,
                            lastcheckedcol_label=None):
  
          logger.debug("add_tag:%s"%options.get('add_tag',None))
          if options.get('add_tag',False):
              book['tags'].extend(options.get('add_tag').split(','))
  
          self.update_error_column_loop(book,db,errorcol_label,lastcheckedcol_label)
  
          if not book['good']:
              return # only update errorcol on error.
  
          logger.debug("add/update %s %s"%(book['title'],book['url']))
          mi = self.make_mi_from_book(book)
  
          if book['collision'] not in (CALIBREONLY, CALIBREONLYSAVECOL):
              self.add_book_or_update_format(book,options,prefs,mi)
  
          if book['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL) or \
                  ( (options['updatemeta'] or book['added']) and book['good'] ):
              try:
                  self.update_metadata(db, book['calibre_id'], book, mi, options)
              except:
                  det_msg = "".join(traceback.format_exception(*sys.exc_info()))+"\n"+_("Story Details:")+pretty_book(book)
                  logger.error("Error Updating Metadata:\n%s"%det_msg)
                  error_dialog(self.gui,
                               _("Error Updating Metadata"),
                               "<p>"+_("An error has occurred while FanFicFare was updating calibre's metadata for <a href='%s'>%s</a>.")%(book['url'],book['title'])+"</p>"+
                               _("The ebook has been updated, but the metadata has not."),
                               det_msg=det_msg,
                               show=True)
  
      def update_books_finish(self, book_list, options={}, showlist=True):
          '''Notify calibre about updated rows, update external plugins
          (Reading Lists & Count Pages) as configured'''
  
          add_list = [ x for x in book_list if x['good'] and x['added'] ]
          add_ids = [ x['calibre_id'] for x in add_list ]
          update_list = [ x for x in book_list if x['good'] and not x['added'] ]
          update_ids = [ x['calibre_id'] for x in update_list ]
          all_ids = add_ids + update_ids
          all_not_calonly_list = [ x for x in add_list + update_list if x['collision'] not in (CALIBREONLY, CALIBREONLYSAVECOL) ]
          all_not_calonly_ids = [ x['calibre_id'] for x in all_not_calonly_list ]
  
          failed_list = [ x for x in book_list if not x['good'] ]
          failed_ids = [ x['calibre_id'] for x in failed_list ]
  
          if all_not_calonly_ids and \
                  (prefs['addtolists'] or prefs['addtoreadlists']):
              self.update_reading_lists(all_not_calonly_ids,add=True)
  
          if len(add_list):
              self.gui.library_view.model().books_added(len(add_list))
              self.gui.library_view.model().refresh_ids(add_ids)
  
          if len(update_list):
              self.gui.library_view.model().refresh_ids(update_ids)
  
          current = self.gui.library_view.currentIndex()
          self.gui.library_view.model().current_changed(current, self.previous)
          self.gui.tags_view.recount()
  
          if self.gui.cover_flow:
              self.gui.cover_flow.dataChanged()
  
          if showlist and prefs['mark']: # don't use with anthology
              db = self.gui.current_db
              marked_ids = dict()
              marked_text = "fff_success"
              for index, book_id in enumerate(all_ids):
                  marked_ids[book_id] = '%s_%04d' % (marked_text, index)
              for index, book_id in enumerate(failed_ids):
                  marked_ids[book_id] = 'fff_failed_%04d' % index
              # Mark the results in our database
              db.set_marked_ids(marked_ids)
  
              if prefs['showmarked']: # show add/update
                  # Search to display the list contents
                  self.gui.search.set_search_string('marked:' + marked_text)
                  # Sort by our marked column to display the books in order
                  self.gui.library_view.sort_by_named_field('marked', True)
  
          logger.debug(_('Finished Adding/Updating %d books.')%(len(update_list) + len(add_list)))
          self.gui.status_bar.show_message(_('Finished Adding/Updating %d books.')%(len(update_list) + len(add_list)), 3000)
          remove_dir(options['tdir'])
          logger.debug("removed tdir")
  
          if 'Count Pages' in self.gui.iactions and len(prefs['countpagesstats']) and len(all_ids):
              cp_plugin = self.gui.iactions['Count Pages']
              countpagesstats = list(prefs['countpagesstats']) # copy because we're changing it.
              # print("all_ids:%s"%all_ids)
              # print("countpagesstats:%s"%countpagesstats)
  
              ## If only some of the books need word counting, they'll
              ## have to be launched separately.
              if prefs['wordcountmissing'] and 'WordCount' in countpagesstats:
                  # print("numWords:%s"%[ y['all_metadata']['numWords'] for y in add_list + update_list ])
                  wc_ids = [ x['calibre_id'] for x in add_list + update_list if '' == x['all_metadata'].get('numWords','') ]
                  ## not all need word count
                  # print("wc_ids:%s"%wc_ids)
                  ## if lists don't match
                  if len(wc_ids)!=len(all_ids):
                      if wc_ids: # because often the lists don't match because 0
                          cp_plugin.count_statistics(wc_ids,['WordCount'])
                      ## don't do WordCount below.
                      while 'WordCount' in countpagesstats: countpagesstats.remove('WordCount')
  
              ## check that there's stuff to do in case wordcount was it.
              # print("countpagesstats:%s"%countpagesstats)
              if countpagesstats:
                  cp_plugin.count_statistics(all_ids,countpagesstats)
  
          if prefs['autoconvert'] and all_not_calonly_ids:
              logger.debug(_('Starting auto conversion of %d books.')%(len(all_ids)))
              self.gui.status_bar.show_message(_('Starting auto conversion of %d books.')%(len(all_ids)), 3000)
              self.gui.iactions['Convert Books'].auto_convert_auto_add(all_not_calonly_ids)
  
      def download_list_completed(self, job, options={},merge=False):
          if job.failed:
              self.gui.job_exception(job, dialog_title='Failed to Download Stories')
              return
  
          self.previous = self.gui.library_view.currentIndex()
          db = self.gui.current_db
  
          book_list = job.result
          good_list = [ x for x in book_list if x['good'] ]
          bad_list = [ x for x in book_list if not x['good'] ]
          try:
              good_list = sorted(good_list,key=lambda x : x['reportorder'])
              bad_list = sorted(bad_list,key=lambda x : x['reportorder'])
          except KeyError:
              good_list = sorted(good_list,key=lambda x : x['listorder'])
              bad_list = sorted(bad_list,key=lambda x : x['listorder'])
          #print("book_list:%s"%book_list)
          payload = (good_list, bad_list, options)
  
          if merge:
              if len(good_list) < 1:
                  info_dialog(self.gui, _('No Good Stories for Anthology'),
                              _('No good stories/updates where downloaded, Anthology creation/update aborted.'),
                              show=True,
                              show_copy_button=False)
                  return
  
              msg = '<p>'+_('FanFicFare found <b>%s</b> good and <b>%s</b> bad updates.')%(len(good_list),len(bad_list))+'</p>'
              if len(bad_list) > 0:
                  msg = msg + '''
                              <p>%s</p>
                              <p>%s</p>
                              <p>%s</p>
                              <p>%s</p>'''%(
                      _('Are you sure you want to continue with creating/updating this Anthology?'),
                      _('Any updates that failed will <b>not</b> be included in the Anthology.'),
                      _("However, if there's an older version, it will still be included."),
                      _('See log for details.'))
  
              msg = msg + '<p>'+_('Proceed with updating this anthology and your library?')+ '</p>'
  
              htmllog='<html><body><table border="1"><tr><th>'+_('Status')+'</th><th>'+_('Title')+'</th><th>'+_('Author')+'</th><th>'+_('Comment')+'</th><th>URL</th></tr>'
              for book in sorted(good_list+bad_list,key=lambda x : x['listorder']):
                  htmllog = htmllog + '<tr><td>' + '</td><td>'.join([escapehtml(book['status']),escapehtml(book['title']),escapehtml(", ".join(book['author'])),escapehtml(book['comment']),book['url']]) + '</td></tr>'
  
              htmllog = htmllog + '</table></body></html>'
  
              for book in bad_list:
                  if 'epub_for_update' in book:
                      book['good']=True
                      book['outfile'] = book['epub_for_update']
                      good_list.append(book)
  
              do_update_func = self.do_download_merge_update
          else:
              msg = '''
                    <p>%s</p>
                    <p>%s</p>
                    <p>%s</p>'''%(
                  _('FanFicFare found <b>%s</b> good and <b>%s</b> bad updates.')%(len(good_list),len(bad_list)),
                  _('See log for details.'),
                  _('Proceed with updating your library?')
                  )
  
              htmllog='<html><body><table border="1"><tr><th>'+_('Status')+'</th><th>'+_('Title')+'</th><th>'+_('Author')+'</th><th>'+_('Comment')+'</th><th>URL</th></tr>'
              for book in good_list:
                  htmllog = htmllog + '<tr><td>' + '</td><td>'.join([escapehtml(book['status']),escapehtml(book['title']),escapehtml(", ".join(book['author'])),escapehtml(book['comment']),book['url']]) + '</td></tr>'
  
              for book in bad_list:
                  htmllog = htmllog + '<tr><td>' + '</td><td>'.join([escapehtml(book['status']),escapehtml(book['title']),escapehtml(", ".join(book['author'])),escapehtml(book['comment']),book['url']]) + '</td></tr>'
  
              htmllog = htmllog + '</table></body></html>'
  
              do_update_func = self.do_download_list_update
  
          # log_viewer_unique_name implemented here: https://github.com/kovidgoyal/calibre/compare/v2.56.0...v2.57.0
          if calibre_version >= (2, 57, 0):
              self.gui.proceed_question(do_update_func,
                                        payload, htmllog,
                                        _('FanFicFare log'), _('FanFicFare download complete'), msg,
                                        show_copy_button=False,
                                        log_viewer_unique_name="FanFicFare log viewer")
          else:
              self.gui.proceed_question(do_update_func,
                                        payload, htmllog,
                                        _('FanFicFare log'), _('FanFicFare download complete'), msg,
                                        show_copy_button=False)
  
      def do_download_merge_update(self, payload):
          db = self.gui.current_db
  
          (good_list,bad_list,options) = payload
          total_good = len(good_list)
  
          logger.debug("merge titles:\n%s"%"\n".join([ "%s %s"%(x['title'],x['listorder']) for x in good_list ]))
  
          good_list = sorted(good_list,key=lambda x : x['listorder'])
          bad_list = sorted(bad_list,key=lambda x : x['listorder'])
  
          self.gui.status_bar.show_message(_('Merging %s books.')%total_good)
  
          existingbook = None
          if 'mergebook' in options:
              existingbook = options['mergebook']
          #print("existingbook:\n%s"%existingbook)
          mergebook = self.merge_meta_books(existingbook,good_list,options['fileform'])
  
          if 'mergebook' in options:
              mergebook['calibre_id'] = options['mergebook']['calibre_id']
  
          if 'anthology_url' in options:
              mergebook['url'] = options['anthology_url']
  
          #print("mergebook:\n%s"%mergebook)
  
          # make a temp file to write the output to.
          tmp = PersistentTemporaryFile(suffix='.'+options['fileform'],
                                        dir=options['tdir'])
          # logger.debug("title:"+mergebook['title'])
          logger.debug("outfile:"+tmp.name)
          mergebook['outfile'] = tmp.name
  
          ## Calibre's Polish heuristics for covers can cause problems
          ## if a merged anthology book has sub-book covers, but not a
          ## proper main cover.  So, now if there are any covers, we
          ## will force a main cover.
  
          ## start with None.  If no subbook covers, don't force one
          ## here.  User can configure FFF to always create/polish a
          ## cover if they want.  This is about when we force it.
          coverpath = None
          coverimgtype = None
  
          ## first, look for covers inside the subbooks.  Stop at the
          ## first one, which will be used if there isn't a pre-existing
          ## calibre cover.
          if not coverpath:
              for book in good_list:
                  coverdata = get_cover_data(book['outfile'])
                  if coverdata: # found a cover.
                      (coverimgtype,coverimgdata) = coverdata[4:6]
                      logger.debug('coverimgtype:%s [%s]'%(coverimgtype,imagetypes[coverimgtype]))
                      tmpcover = PersistentTemporaryFile(suffix='.'+imagetypes[coverimgtype],
                                                         dir=options['tdir'])
                      tmpcover.write(coverimgdata)
                      tmpcover.flush()
                      tmpcover.close()
                      coverpath = tmpcover.name
                      break
          # logger.debug('coverpath:%s'%coverpath)
  
          ## if updating an existing book and there is at least one
          ## subbook cover:
          if coverpath and mergebook['calibre_id']:
              # Couldn't find a better way to get the cover path.
              calcoverpath = os.path.join(db.library_path,
                                       db.path(mergebook['calibre_id'], index_is_id=True),
                                       'cover.jpg')
              ## if there's an existing cover, use it.  Calibre will set
              ## it for us during lots of different actions anyway.
              if os.path.exists(calcoverpath):
                  coverpath = calcoverpath
  
          # logger.debug('coverpath:%s'%coverpath)
          self.get_epubmerge_plugin().do_merge(tmp.name,
                                               [ x['outfile'] for x in good_list ],
                                               tags=mergebook['tags'],
                                               titleopt=mergebook['title'],
                                               keepmetadatafiles=True,
                                               source=mergebook['url'],
                                               coverjpgpath=coverpath)
  
          mergebook['collision'] = options['collision'] = OVERWRITEALWAYS
          errorcol_label = self.get_custom_col_label(prefs['errorcol'])
          lastcheckedcol_label = self.get_custom_col_label(prefs['lastcheckedcol'])
          self.update_books_loop(mergebook,
                                 self.gui.current_db,
                                 options,
                                 errorcol_label=errorcol_label,
                                 lastcheckedcol_label=lastcheckedcol_label)
          self.update_books_finish([mergebook], options=options, showlist=False)
  
      def do_download_list_update(self, payload):
  
          (good_list,bad_list,options) = payload
          good_list = sorted(good_list,key=lambda x : x['listorder'])
          bad_list = sorted(bad_list,key=lambda x : x['listorder'])
  
          self.gui.status_bar.show_message(_('FanFicFare Adding/Updating books.'))
          errorcol_label = self.get_custom_col_label(prefs['errorcol'])
          lastcheckedcol_label = self.get_custom_col_label(prefs['lastcheckedcol'])
  
          columns = self.gui.library_view.model().custom_columns
          if good_list or prefs['mark'] or (bad_list and errorcol_label) or lastcheckedcol_label:
              LoopProgressDialog(self.gui,
                                 good_list+bad_list,
                                 partial(self.update_books_loop,
                                         options=options,
                                         db=self.gui.current_db,
                                         errorcol_label=errorcol_label,
                                         lastcheckedcol_label=lastcheckedcol_label),
                                 partial(self.update_books_finish, options=options),
                                 init_label=_("Updating calibre for FanFiction stories..."),
                                 win_title=_("Update calibre for FanFiction stories"),
                                 status_prefix=_("Updated"))
  
      def update_error_column(self,payload):
          '''Update custom error column if configured.'''
          (empty_list,book_list,options)=payload
          errorcol_label = self.get_custom_col_label(prefs['errorcol'])
          lastcheckedcol_label = self.get_custom_col_label(prefs['lastcheckedcol'])
          if prefs['mark'] or errorcol_label or lastcheckedcol_label:
              self.previous = self.gui.library_view.currentIndex() # used by update_books_finish.
              self.gui.status_bar.show_message(_('Adding/Updating %s BAD books.')%len(book_list))
              LoopProgressDialog(self.gui,
                                 book_list,
                                 partial(self.update_error_column_loop, db=self.gui.current_db, errorcol_label=errorcol_label, lastcheckedcol_label=lastcheckedcol_label),
                                 partial(self.update_books_finish, options=options),
                                 init_label=_("Updating calibre for BAD FanFiction stories..."),
                                 win_title=_("Update calibre for BAD FanFiction stories"),
                                 status_prefix=_("Updated"))
  
      def update_error_column_loop(self,book,db=None,errorcol_label=None,lastcheckedcol_label=None):
          if book['calibre_id'] and errorcol_label:
              if not book['good'] and (book['showerror'] or prefs['save_all_errors']):
                  logger.debug("update_error_column_loop bad %s %s %s"%(book['title'],book['url'],book['comment']))
                  self.set_custom(db, book['calibre_id'], 'comment', book['comment'], label=errorcol_label, commit=True)
              else:
                  ## if not recording error, clear any existing error, but only if not empty.
                  prev_val = db.get_custom(book['calibre_id'],label=errorcol_label,index_is_id=True)
                  logger.debug("update_error_column_loop prev_val:%s"%prev_val)
                  if prev_val:
                      logger.debug("update_error_column_loop bad %s %s ''"%(book['title'],book['url']))
                      self.set_custom(db, book['calibre_id'], '(none)', '', label=errorcol_label, commit=True)
          if book['calibre_id'] and lastcheckedcol_label:
              #logger.debug("lastcheckedcol %s %s %s"%(book['title'],book['url'],book['timestamp']))
              self.set_custom(db, book['calibre_id'], 'timestamp',
                              book.get('timestamp',datetime.now().replace(tzinfo=local_tz)), # default to now if not in book.
                              label=lastcheckedcol_label, commit=True)
  
      def add_book_or_update_format(self,book,options,prefs,mi=None):
          db = self.gui.current_db
  
          if mi == None:
              mi = self.make_mi_from_book(book)
  
          book_id = book['calibre_id']
          if book_id == None:
              book_id = db.create_book_entry(mi,
                                             add_duplicates=True)
              book['calibre_id'] = book_id
              book['added'] = True
          else:
              book['added'] = False
  
          try:
              # Remove before adding so the previous version is in the
              # OS Trash (on Windows, at least)
              db.remove_format(book_id,options['fileform'], index_is_id=True)
          except:
              pass
          if not db.add_format_with_hooks(book_id,
                                          options['fileform'],
                                          book['outfile'], index_is_id=True):
              book['comment'] = _("Adding format to book failed for some reason...")
              book['good']=False
              book['icon']='dialog_error.png'
              book['status'] = _('Error')
  
          if prefs['deleteotherforms']:
              fmts = db.formats(book['calibre_id'], index_is_id=True).split(',')
              for fmt in fmts:
                  if fmt.lower() != formmapping[options['fileform']].lower():
                      logger.debug("deleteotherforms remove f:"+fmt)
                      db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
          elif prefs['autoconvert']:
              ## 'Convert Book'.auto_convert_auto_add doesn't convert if
              ## the format is already there.
              fmt = calibre_prefs['output_format']
              # delete if there, but not if the format we just made.
              if fmt.lower() != formmapping[options['fileform']].lower() and \
                      db.has_format(book_id,fmt,index_is_id=True):
                  logger.debug("autoconvert remove f:"+fmt)
                  db.remove_format(book['calibre_id'], fmt, index_is_id=True)#, notify=False
  
  
          return book_id
  
      def set_custom(self,db,book_id,meta,val,label,commit=True):
          try:
              db.set_custom(book_id, val, label=label, commit=commit)
          except Exception as e:
              errmsg="Trying to set entry (%s) value(%s) to column (#%s) failed (%s)"%(meta,val,label,e)
              logger.warn(errmsg)
              raise Exception(errmsg)
  
      def update_metadata(self, db, book_id, book, mi, options):
          oldmi = db.get_metadata(book_id,index_is_id=True)
          if prefs['keeptags']:
              old_tags = db.get_tags(book_id)
              #print("old_tags:%s"%old_tags)
              #print("mi.tags:%s"%mi.tags)
              # remove old Completed/In-Progress only if there's a new one.
              if 'Completed' in mi.tags or 'In-Progress' in mi.tags:
                  old_tags = [ x for x in old_tags if x not in ('Completed', 'In-Progress') ]
                  # remove old Last Update tags if there are new ones.
              if sum(1 for x in mi.tags if not x.startswith("Last Update")):
                  old_tags = [ x for x in old_tags if not x.startswith("Last Update") ]
  
              # mi.tags needs to be list, but set kills dups.
              # this way also removes case-mismatched dups, keeping old_tags version.
              foldedcase_tags = dict()
              for t in list(mi.tags) + list(old_tags):
                  foldedcase_tags[t.lower()] = t
  
              mi.tags = list(foldedcase_tags.values())
              #print("mi.tags:%s"%mi.tags)
  
          if book['all_metadata']['langcode']:
              # split due to anthologies.  Gives list of one for non-anth.
              mi.languages=book['all_metadata']['langcode'].split(', ')
          else:
              # Set language english, but only if not already set.
              if not oldmi.languages:
                  mi.languages=['en']
  
          # implement 'newonly' flags here by setting to the current
          # value again.
          if not book['added']:
              for (col,newonly) in six.iteritems(prefs['std_cols_newonly']):
                  if newonly:
                      if col == "identifiers":
                          mi.set_identifiers(oldmi.get_identifiers())
                      else:
                          try:
                              mi.__setattr__(col,oldmi.__getattribute__(col))
                          except AttributeError:
                              logger.warn("AttributeError? %s"%col)
                              pass
  
          ## fix for suppressauthorsort (Force Author into Author Sort)
          ## option overriding Author-New-Only setting.  not done where
          ## suppressauthorsort/suppresstitlesort done because that
          ## would need an additional check for new books.
          if prefs['std_cols_newonly'].get('authors',False):
              mi.author_sort = oldmi.author_sort
  
          ## Ditto for title sort.
          if prefs['std_cols_newonly'].get('title',False):
              mi.title_sort = oldmi.title_sort
  
          db.set_metadata(book_id,mi)
          if not prefs['std_cols_newonly'].get('authors',False):
              # mi.authors gets run through the string_to_authors and split on '&' ',' 'and' and 'with'
              db.set_authors(book_id,book['author'],
                             allow_case_change=True) # author is a list.
  
          # do configured column updates here.
          #print("all_metadata: %s"%book['all_metadata'])
          custom_columns = self.gui.library_view.model().custom_columns
  
          # save metadata to configured column
          if 'savemetacol' in book and prefs['savemetacol'] != '' and prefs['savemetacol'] in custom_columns:
              label = custom_columns[prefs['savemetacol']]['label']
              self.set_custom(db, book_id, 'comment', book['savemetacol'], label=label, commit=True)
  
          # save lastchecked for new books.  Otherwise new books don't get lastchecked
          if prefs['lastcheckedcol'] != '' and prefs['lastcheckedcol'] in custom_columns:
              label = custom_columns[prefs['lastcheckedcol']]['label']
              self.set_custom(db, book_id, 'lastcheckedcol', book['timestamp'], label=label, commit=True)
  
          #print("prefs['custom_cols'] %s"%prefs['custom_cols'])
          for col, meta in six.iteritems(prefs['custom_cols']):
              #print("setting %s to %s"%(col,meta))
              if col not in custom_columns:
                  logger.debug("%s not an existing column, skipping."%col)
                  continue
              coldef = custom_columns[col]
              if col in prefs['custom_cols_newonly'] and prefs['custom_cols_newonly'][col] and not book['added']:
                  logger.debug("Skipping custom column(%s) update, set to New Books Only"%coldef['name'])
                  continue
              if not meta.startswith('status-') and meta not in book['all_metadata'] or \
                      meta.startswith('status-') and 'status' not in book['all_metadata']:
                  logger.debug("No value for %s, skipping custom column(%s) update."%(meta,coldef['name']))
                  continue
              if meta not in permitted_values[coldef['datatype']]:
                  logger.debug("%s not a valid column type for %s, skipping."%(col,meta))
                  continue
              label = coldef['label']
              if coldef['datatype'] in ('enumeration','comments','datetime','series'):
                  self.set_custom(db, book_id, meta, book['all_metadata'][meta], label, commit=False)
              elif coldef['datatype'] == 'text':
                  joined_val = book['all_metadata'][meta]
                  # 'Contains names' custom columns need & separators.
                  # If user has changed join_string_X, it's on them to fix.
                  if coldef['display'].get('is_names',False):
                      joined_val = joined_val.replace(', ',' & ')
                  self.set_custom(db, book_id, meta, joined_val, label, commit=False)
              elif coldef['datatype'] in ('int','float'):
                  num = unicode(book['all_metadata'][meta]).replace(",","")
                  if num != '':
                      self.set_custom(db, book_id, meta, num, label=label, commit=False)
              elif coldef['datatype'] == 'bool' and meta.startswith('status-'):
                  if meta == 'status-C':
                      # Complete or Completed.
                      val = 'complete' in book['all_metadata']['status'].lower()
                  if meta == 'status-I':
                      # In-Progress, In Progress, or In progress.
                      val = 'progress' in book['all_metadata']['status'].lower()
                  self.set_custom(db, book_id, meta, val, label=label, commit=False)
  
          configuration = None
          if prefs['allow_custcol_from_ini']:
              configuration = get_fff_config(book['url'],options['fileform'])
              # meta => custcol[,a|n|r|n_anthaver,r_anthaver]
              # cliches=>\#acolumn,r
              for line in configuration.getConfig('custom_columns_settings').splitlines():
                  if "=>" in line:
                      (meta,custcol) = [ x.strip() for x in line.split("=>") ]
                      flag='r'
                      anthaver=False
                      if "," in custcol:
                          (custcol,flag) = [ x.strip() for x in custcol.split(",") ]
                          anthaver = 'anthaver' in flag
                          flag=flag[0] # first char only.
  
                      if meta not in book['all_metadata']:
                          # if double quoted, use as a literal value.
                          if meta[0] == '"' and meta[-1] == '"':
                              val = meta[1:-1]
                              logger.debug("No metadata value for %s, setting custom column(%s) literally to %s."%(meta,custcol,val))
                          else:
                              logger.debug("No value for %s, skipping custom column(%s) update."%(meta,custcol))
                              continue
                      else:
                          val = book['all_metadata'][meta]
  
                      if custcol not in custom_columns:
                          continue
                      else:
                          coldef = custom_columns[custcol]
                          label = coldef['label']
  
                      # 'Contains names' custom columns need & separators.
                      # If user has changed join_string_X, it's on them to fix.
                      if coldef['display'].get('is_names',False):
                          val = val.replace(', ',' & ')
  
                      if flag == 'r' or (flag == 'n' and book['added']):
                          if coldef['datatype'] in ('int','float'): # for favs, etc--site specific metadata.
                              if 'anthology_meta_list' in book and meta in book['anthology_meta_list']:
                                  # re-split list, strip commas, convert to floats
                                  items = [ float(x.replace(",","")) for x in val.split(", ") ]
                                  if anthaver:
                                      if items:
                                          val = sum(items) / float(len(items))
                                      else:
                                          val = 0
                                  else:
                                      val = sum(items)
                              else:
                                  val = unicode(val).replace(",","")
                          else:
                              val = val
                          if coldef['datatype'] == 'bool':
                              if val.lower() in ('t','true','1','yes','y'):
                                  val = True
                              elif val.lower() in ('f','false','0','no','n'):
                                  val = False
                              else:
                                  val = None # for tri-state 'booleans'. Yes/No/Null
                          # logger.debug("setting 'r' or 'added':meta:%s label:%s val:%s"%(meta,label,val))
                          if val != '':
                              self.set_custom(db, book_id, meta, val, label=label, commit=False)
  
                      if flag == 'a':
                          vallist = []
                          try:
                              existing=db.get_custom(book_id,label=label,index_is_id=True)
                              # logger.debug("existing:%s"%existing)
                              if isinstance(existing,list):
                                  vallist = existing
                              elif existing:
                                  vallist = [existing]
                          except:
                              pass
  
                          #print("vallist:%s"%vallist)
                          if val:
                              vallist.append(val)
  
                          if coldef['display'].get('is_names',False):
                              join_str=' & '
                          else:
                              join_str=', '
                          self.set_custom(db, book_id, meta, join_str.join(vallist), label=label, commit=False)
  
          # set author link if found.  All current adapters have authorUrl, except anonymous on AO3.
          # Moved down so author's already in the DB.
          if 'authorUrl' in book['all_metadata'] and prefs['set_author_url']:
              authurls = book['all_metadata']['authorUrl'].split(", ")
              authorlist = [ a.replace('&',';') for a in book['author'] ]
              authorids = db.new_api.get_item_ids('authors',authorlist)
              authordata = db.new_api.author_data(list(authorids.values()))
              # print("\n\nauthorids:%s"%authorids)
              # print("authordata:%s"%authordata)
  
              author_id_to_link_map = dict()
              for i, author in enumerate(authorlist):
                  if len(authurls) > i:
                      author_id_to_link_map[authorids[author]] = authurls[i]
  
              # print("author_id_to_link_map:%s\n\n"%author_id_to_link_map)
              db.new_api.set_link_for_authors(author_id_to_link_map)
  
          db.commit()
  
          # updating calibre cover from book.
          if options['fileform'] == 'epub' and (
              (prefs['updatecover'] and not prefs['updatecalcover']) ## backward compat
              or prefs['updatecalcover'] == SAVE_YES ## yes, always
              or (prefs['updatecalcover'] == SAVE_YES_IF_IMG ## yes, if image.
                  and book['all_metadata']['cover_image'] )): # in ('specific','first','default','old')
              existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
              epubmi = calibre_get_metadata(existingepub,'EPUB')
              if epubmi.cover_data[1] is not None:
                  try:
                      db.set_cover(book_id, epubmi.cover_data[1])
                  except:
                      logger.info("Failed to set_cover, skipping")
  
          # First, should cover generation happen at all?
          if (book['added'] or not prefs['gcnewonly']) and ( # skip if not new book and gcnewonly is True
              prefs['gencalcover'] == SAVE_YES ## yes, always
              or (prefs['gencalcover'] == SAVE_YES_UNLESS_IMG ## yes, unless image.
                  and book['all_metadata']['cover_image'] not in ('specific','first','default')) ):
  
              cover_generated = False # flag for polish below.
              # Yes, should do gencov.  Which?
              if prefs['calibre_gen_cover'] and HAS_CALGC:
                  ## calibre's builtin, if available.  fetch updated mi
                  ## object from database. Additional normalization of
                  ## series (at least) happens
                  realmi = db.get_metadata(book_id, index_is_id=True)
                  cdata = cal_generate_cover(realmi)
                  db.set_cover(book_id, cdata)
                  cover_generated = True
              elif prefs['plugin_gen_cover'] and 'Generate Cover' in self.gui.iactions:
                  # plugin, if available.
  
                  #logger.debug("Do Generate Cover added:%s gcnewonly:%s"%(book['added'],prefs['gcnewonly']))
  
                  # force a refresh if generating cover so complex composite
                  # custom columns are current and correct
                  db.refresh_ids([book_id])
  
                  gc_plugin = self.gui.iactions['Generate Cover']
                  setting_name = None
                  if prefs['allow_gc_from_ini']:
                      if not configuration: # might already have it from allow_custcol_from_ini
                          configuration = get_fff_config(book['url'],options['fileform'])
  
                      for (template,regexp,setting) in configuration.get_generate_cover_settings():
                          value = Template(template).safe_substitute(book['all_metadata'])
                          # print("%s(%s) => %s => %s"%(template,value,regexp,setting))
                          if re.search(regexp,value):
                              setting_name = setting
                              break
  
                      if setting_name:
                          logger.debug("Generate Cover Setting from generate_cover_settings(%s)"%setting_name)
                          if setting_name not in gc_plugin.get_saved_setting_names():
                              logger.info("GC Name %s not found, discarding! (check personal.ini for typos)"%setting_name)
                              setting_name = None
  
                  if not setting_name and book['all_metadata']['site'] in prefs['gc_site_settings']:
                      setting_name =  prefs['gc_site_settings'][book['all_metadata']['site']]
                      logger.debug("Generate Cover Setting from site(%s)"%setting_name)
  
                  if not setting_name and 'Default' in prefs['gc_site_settings']:
                      setting_name =  prefs['gc_site_settings']['Default']
                      logger.debug("Generate Cover Setting from Default(%s)"%setting_name)
  
                  if setting_name:
                      logger.debug("Running Generate Cover with settings %s."%setting_name)
                      ## fetch updated mi object from
                      ## database. Additional normalization of series
                      ## (at least) happens
                      realmi = db.get_metadata(book_id, index_is_id=True)
                      gc_plugin.generate_cover_for_book(realmi,saved_setting_name=setting_name)
                      cover_generated = True
  
              if cover_generated and prefs['gc_polish_cover'] and \
                      options['fileform'] == "epub":
                  # set cover inside epub from calibre's polish feature
                  from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
                  from calibre.utils.logging import Log
                  from collections import namedtuple
  
                  # Couldn't find a better way to get the cover path.
                  cover_path = os.path.join(db.library_path,
                                            db.path(book_id, index_is_id=True),
                                            'cover.jpg')
                  data = {'cover':cover_path}
                  #print("cover_path:%s"%cover_path)
                  opts = ALL_OPTS.copy()
                  opts.update(data)
                  O = namedtuple('Options', ' '.join(six.iterkeys(ALL_OPTS)))
                  opts = O(**opts)
  
                  log = Log(level=Log.DEBUG)
                  outfile = db.format_abspath(book_id,
                                              formmapping[options['fileform']],
                                              index_is_id=True)
                  #print("polish cover outfile:%s"%outfile)
                  polish({outfile:outfile}, opts, log, logger.info)
  
      def get_clean_reading_lists(self,lists):
          if lists == None or lists.strip() == "" :
              return []
          else:
              return [ x.strip() for x in lists.split(',') ]
  
      def update_reading_lists(self,book_ids,add=True):
          try:
              rl_plugin = self.gui.iactions['Reading List']
          except:
              if prefs['addtolists'] or prefs['addtoreadlists']:
                  message="<p>"+_("You configured FanFicFare to automatically update Reading Lists, but you don't have the %s plugin installed anymore?")%'Reading List'+"</p>"
                  confirm(message,'fff_no_reading_list_plugin', self.gui, show_cancel_button=False)
              return
  
          if prefs['addtoreadlists']:
              if add:
                  addremovefunc = rl_plugin.add_books_to_list
              else:
                  addremovefunc = rl_plugin.remove_books_from_list
  
              lists = self.get_clean_reading_lists(prefs['read_lists'])
              if len(lists) < 1 :
                  message="<p>"+_("You configured FanFicFare to automatically update \"To Read\" Reading Lists, but you don't have any lists set?")+"</p>"
                  confirm(message,'fff_no_read_lists', self.gui, show_cancel_button=False)
              for l in lists:
                  if l in rl_plugin.get_list_names():
                      #print("add good read l:(%s)"%l)
                      addremovefunc(l,
                                    book_ids,
                                    display_warnings=False)
                  else:
                      if l != '':
                          message="<p>"+_("You configured FanFicFare to automatically update Reading List '%s', but you don't have a list of that name?")%l+"</p>"
                          confirm(message,'fff_no_reading_list_%s'%l, self.gui, show_cancel_button=False)
  
          if prefs['addtolists'] and (add or (prefs['addtolistsonread'] and prefs['addtoreadlists']) ):
              lists = self.get_clean_reading_lists(prefs['send_lists'])
              if len(lists) < 1 :
                  message="<p>"+_("You configured FanFicFare to automatically update \"Send to Device\" Reading Lists, but you don't have any lists set?")+"</p>"
                  confirm(message,'fff_no_send_lists', self.gui, show_cancel_button=False)
  
              for l in lists:
                  if l in rl_plugin.get_list_names():
                      #print("good send l:(%s)"%l)
                      rl_plugin.add_books_to_list(l,
                                                  #add_book_ids,
                                                  book_ids,
                                                  display_warnings=False)
                  else:
                      if l != '':
                          message="<p>"+_("You configured FanFicFare to automatically update Reading List '%s', but you don't have a list of that name?")%l+"</p>"
                          confirm(message,'fff_no_reading_list_%s'%l, self.gui, show_cancel_button=False)
  
      def make_mi_from_book(self,book):
          if prefs['titlecase']:
              from calibre.ebooks.metadata.sources.base import fixcase
              book['title'] = fixcase(book['title'])
          if prefs['authorcase']:
              from calibre.ebooks.metadata.sources.base import fixauthors
              book['author'] = fixauthors(book['author'])
          mi = MetaInformation(book['title'],book['author']) # author is a list.
          if prefs['suppressauthorsort']:
              # otherwise author names will have calibre's sort algs
              # applied automatically.
              mi.author_sort = ' & '.join(book['author'])
          if prefs['suppresstitlesort']:
              # otherwise titles will have calibre's sort algs applied
              # automatically.
              mi.title_sort = book['title']
          mi.set_identifiers({'url':book['url']})
          mi.publisher = book['publisher']
          mi.tags = book['tags']
          #mi.languages = ['en'] # handled in update_metadata so it can check for existing lang.
          mi.pubdate = book['pubdate']
          mi.timestamp = book['timestamp']
          mi.comments = book['comments']
          mi.series = book['series']
          return mi
  
      # Can't make book a class because it needs to be passed into the
      # bg jobs and only serializable things can be.
      def make_book(self):
          book = {}
          book['title'] = 'Unknown'
          book['author_sort'] = book['author'] = ['Unknown'] # list
          book['comments'] = '' # note this is the book comments.
  
          book['good'] = True
          book['status'] = 'Bad'
          book['showerror'] = True # False when NotGoingToDownload is
                                   # not-overwrite / not-update / skip
                                   # -- what some would consider 'not an
                                   # error'
          book['calibre_id'] = None
          book['begin'] = None
          book['end'] = None
          book['comment'] = '' # note this is a comment on the d/l or update.
          book['url'] = ''
          book['site'] = ''
          book['series'] = ''
          book['added'] = False
          book['pubdate'] = None
          book['publisher'] = None
          return book
  
      def convert_urls_to_books(self, urls):
          books = []
          uniqueurls = set()
          for i, url in enumerate(urls):
              book = self.convert_url_to_book(url)
              if book['uniqueurl'] in uniqueurls:
                  book['good'] = False
                  book['comment'] = _("Same story already included.")
                  book['status']=_('Skipped')
              uniqueurls.add(book['uniqueurl'])
              book['listorder']=i # BG d/l jobs don't come back in order.
                                  # Didn't matter until anthologies & 'marked' successes
              books.append(book)
          return books
  
      def convert_url_to_book(self, url):
          book = self.make_book()
          # Allow chapter range with URL.
          # like test1.com?sid=5[4-6] or [4,6]
          url,book['begin'],book['end'] = adapters.get_url_chapter_range(url)
  
          self.set_book_url_and_comment(book,url) # normalizes book[url]
          # for case of trying to download book by sections. url[1-5], url[6-10], etc.
          book['uniqueurl']="%s[%s-%s]"%(book['url'],book['begin'],book['end'])
          return book
  
      # basic book, plus calibre_id.  Assumed bad until proven
      # otherwise.
      def make_book_id_only(self, idval):
          book = self.make_book()
          book['good'] = False
          book['calibre_id'] = idval
          return book
  
      def populate_book_from_mi(self,book,mi):
          book['title'] = mi.title
          book['author'] = mi.authors
          book['author_sort'] = mi.author_sort
          if hasattr(mi,'publisher'):
              book['publisher'] = mi.publisher
          if hasattr(mi,'path'):
              book['path'] = mi.path
          if hasattr(mi,'id'):
              book['calibre_id'] = mi.id
  
      # book data from device.  Assumed bad until proven otherwise.
      def make_book_from_device_row(self, row):
          book = self.make_book()
          mi = self.gui.current_view().model().get_book_display_info(row.row())
          self.populate_book_from_mi(book,mi)
          book['good'] = False
          return book
  
      def populate_book_from_calibre_id(self, book, db=None):
          mi = db.get_metadata(book['calibre_id'], index_is_id=True)
          #book = {}
          book['good'] = True
          self.populate_book_from_mi(book,mi)
  
          url = self.get_story_url(db,book['calibre_id'])
          self.set_book_url_and_comment(book,url)
          #return book - populated passed in book.
  
      def set_book_url_and_comment(self,book,url):
          if not url:
              book['comment'] = _("No story URL found.")
              book['good'] = False
              book['icon'] = 'search_delete_saved.png'
              book['status'] = _('Not Found')
          else:
              # get normalized url or None.
              urlsitetuple = adapters.getNormalStoryURLSite(url)
              if urlsitetuple == None:
                  book['url'] = url
                  book['comment'] = _("URL is not a valid story URL.")
                  book['good'] = False
                  book['icon']='dialog_error.png'
                  book['status'] = _('Bad URL')
              else:
                  (book['url'],book['site'])=urlsitetuple
  
      def get_story_url(self, db, book_id=None, path=None):
          if book_id == None:
              identifiers={}
          else:
              identifiers = db.get_identifiers(book_id,index_is_id=True)
          if 'url' in identifiers:
              # identifiers have :->| in url.
              # print("url from ident url:%s"%identifiers['url'].replace('|',':'))
              return identifiers['url'].replace('|',':')
          elif 'uri' in identifiers:
              # identifiers have :->| in uri.
              # print("uri from ident uri:%s"%identifiers['uri'].replace('|',':'))
              return identifiers['uri'].replace('|',':')
          else:
              existingepub = None
              if path == None and db.has_format(book_id,'EPUB',index_is_id=True):
                  existingepub = db.format(book_id,'EPUB',index_is_id=True, as_file=True)
                  mi = calibre_get_metadata(existingepub,'EPUB')
                  identifiers = mi.get_identifiers()
                  if 'url' in identifiers:
                      # print("url from get_metadata:%s"%identifiers['url'].replace('|',':'))
                      return identifiers['url'].replace('|',':')
                  elif 'uri' in identifiers:
                      # identifiers have :->| in uri.
                      # print("uri from ident uri:%s"%identifiers['uri'].replace('|',':'))
                      return identifiers['uri'].replace('|',':')
              elif path and path.lower().endswith('.epub'):
                  existingepub = path
  
              link = None
              ## only epub has URL in it--at least where I can easily find it.
              if existingepub:
                  # look for dc:source first, then scan HTML if lookforurlinhtml
                  link = get_dcsource(existingepub)
                  if link:
                      # print("url from get_dcsource:%s"%link)
                      return link
  
              ## now also can search html and txt formats.
              if prefs['lookforurlinhtml']:
                  # print(db.formats(book_id, index_is_id=True))
                  if existingepub:
                      link = get_story_url_from_epub_html(existingepub,self.is_good_downloader_url)
                      # print("url from get_story_url_from_epub_html:%s"%link)
                      if link:
                          return link
                  if db.has_format(book_id,'ZIP',index_is_id=True):
                      # print("has zip/html format")
                      existingziphtml = db.format(book_id,'ZIP',index_is_id=True, as_file=True)
                      link = get_story_url_from_zip_html(existingziphtml,self.is_good_downloader_url)
                      # print("url from get_story_url_from_zip_html:%s"%link)
                      if link:
                          return link
                  if db.has_format(book_id,'TXT',index_is_id=True):
                      # print("has txt format")
                      existingtxt = db.format(book_id,'TXT',index_is_id=True)
                      links = get_urls_from_text(existingtxt,normalize=True)
                      if links:
                          return links[0]
          return None
  
      def is_good_downloader_url(self,url):
          return adapters.getNormalStoryURL(url)
  
      def merge_meta_books(self,existingbook,book_list,fileform):
          book = self.make_book()
          book['author'] = []
          book['tags'] = []
          book['url'] = ''
          book['all_metadata'] = {}
          book['anthology_meta_list'] = {}
          book['comment'] = ''
          book['added'] = True
          book['good'] = True
          book['calibre_id'] = None
          book['series'] = None
  
          serieslist=[]
  
          # copy list top level
          for b in book_list:
              if b['series']:
                  try:
                      serieslist.append(b['series'][:b['series'].index(" [")])
                  except ValueError: # substring not found
                      serieslist.append(b['series'])
                  #print("book series:%s"%serieslist[-1])
  
              if b['publisher']:
                  if not book['publisher']:
                      ## not set in all_metadata because it's not one of
                      ## the permitted metadata--use site instead.
                      book['publisher']=b['publisher']
                  elif book['publisher']!=b['publisher']:
                      book['publisher']=None # if any are different, don't use.
  
              # copy authors & tags.
              for k in ('author','tags'):
                  for v in b[k]:
                      if v not in book[k]:
                          book[k].append(v)
  
              # fill from first of each if not already present:
              for k in ('pubdate', 'timestamp', 'updatedate'):
                  if k not in b or not b[k]: # not in this book?  Skip it.
                      continue
                  if k not in book or not book[k]: # first is good enough for publisher.
                      book[k]=b[k]
  
                  # Do these even on first to get the all_metadata settings.
                  # pubdate should be earliest date.
                  if k == 'pubdate' and book[k] >= b[k]:
                      book[k]=b[k]
                      book['all_metadata']['datePublished'] = b['all_metadata']['datePublished']
                  # timestamp should be latest date.
                  if k == 'timestamp' and book[k] <= b[k]:
                      book[k]=b[k]
                      book['all_metadata']['dateCreated'] = b['all_metadata']['dateCreated']
                  # updated should be latest date.
                  if k == 'updatedate' and book[k] <= b[k]:
                      book[k]=b[k]
                      book['all_metadata']['dateUpdated'] = b['all_metadata']['dateUpdated']
  
              # copy list all_metadata
              for (k,v) in six.iteritems(b['all_metadata']):
                  #print("merge_meta_books v:%s k:%s"%(v,k))
                  if k in ('numChapters','numWords'):
                      if k in b['all_metadata'] and b['all_metadata'][k]:
                          if k not in book['all_metadata']:
                              book['all_metadata'][k] = b['all_metadata'][k]
                          else:
                              # lot of work for a simple add.
                              book['all_metadata'][k] = unicode(int(book['all_metadata'][k].replace(',',''))+int(b['all_metadata'][k].replace(',','')))
                  elif k in ('dateUpdated','datePublished','dateCreated',
                             'series','status','title'):
                      pass # handled above, below or skip these for now, not going to do anything with them.
                  elif k not in book['all_metadata'] or not book['all_metadata'][k]:
                      book['all_metadata'][k]=v
                  elif v:
                      if k == 'description':
                          book['all_metadata'][k]=book['all_metadata'][k]+"\n\n"+v
                      else:
                          book['all_metadata'][k]=book['all_metadata'][k]+", "+v
                          # flag psuedo list element.  Used so numeric
                          # cust cols can convert back to numbers and
                          # add.
                          book['anthology_meta_list'][k]=True
  
          logger.debug("book['url']:%s"%book['url'])
  
          book['comments'] = _("Anthology containing:")+"\n\n"
          if len(book['author']) > 1:
              mkbooktitle = lambda x : _("%s by %s") % (x['title'],' & '.join(x['author']))
          else:
              mkbooktitle = lambda x : x['title']
  
          if prefs['includecomments']:
              def mkbookcomments(x):
                  if x['comments']:
                      return '<b>%s</b>\n\n%s'%(mkbooktitle(x),x['comments'])
                  else:
                      return '<b>%s</b>\n'%mkbooktitle(x)
  
              book['comments'] += ('<div class="mergedbook">' +
                              '<hr></div><div class="mergedbook">'.join([ mkbookcomments(x) for x in book_list]) +
                              '</div>')
          else:
              book['comments'] += '\n'.join( [ mkbooktitle(x) for x in book_list ] )
  
          configuration = get_fff_config(book['url'],fileform)
          if existingbook:
              book['title'] = deftitle = existingbook['title']
              if prefs['anth_comments_newonly']:
                  book['comments'] = existingbook['comments']
          else:
              book['title'] = deftitle = book_list[0]['title']
              # book['all_metadata']['description']
  
              # if all same series, use series for name.  But only if all and not previous named
              if len(serieslist) == len(book_list):
                  series = serieslist[0]
                  book['title'] = series
                  for sr in serieslist:
                      if series != sr:
                          book['title'] = deftitle
                          break
  
              logger.debug("anthology_title_pattern:%s"%configuration.getConfig('anthology_title_pattern'))
              if configuration.getConfig('anthology_title_pattern'):
                  tmplt = Template(configuration.getConfig('anthology_title_pattern'))
                  book['title'] = tmplt.safe_substitute({'title':book['title']})
              else:
                  # No setting, do fall back default.  Shouldn't happen,
                  # should always have a version in defaults.
                  book['title'] = book['title']+_(" Anthology")
  
          book['all_metadata']['title'] = book['title'] # because custom columns are set from all_metadata
          book['all_metadata']['author'] = ", ".join(book['author'])
          book['author_sort']=book['author']
          for v in ['Completed','In-Progress']:
              if v in book['tags']:
                  book['tags'].remove(v)
          book['tags'].extend(configuration.getConfigList('anthology_tags'))
          book['all_metadata']['anthology'] = "true"
  
          return book
  
  def split_text_to_urls(urls):
      # remove dups while preserving order.
      dups=set()
      def f(x):
          x=x.strip()
          if x and x not in dups:
              dups.add(x)
              return True
          else:
              return False
      return [ x for x in urls.strip().splitlines() if f(x)]
  
  def escapehtml(txt):
      return txt.replace("&","&amp;").replace(">","&gt;").replace("<","&lt;")
  
  def pretty_book(d, indent=0, spacer='     '):
      kindent = spacer * indent
  
      # if isinstance(d, list):
      #     return '\n'.join([(pretty_book(v, indent, spacer)) for v in d])
  
      if isinstance(d, dict):
          for k in ('password','username'):
              if k in d and d[k]:
                  d[k]=_('(was set, removed for security)')
          return '\n'.join(['%s%s:\n%s' % (kindent, k, pretty_book(v, indent + 1, spacer))
                            for k, v in d.items()])
      return "%s%s"%(kindent, d)
  