text: # -*- coding: utf-8 -*-
  
  # Copyright © 2012-2021 Roberto Alsina and others.
  
  # Permission is hereby granted, free of charge, to any
  # person obtaining a copy of this software and associated
  # documentation files (the "Software"), to deal in the
  # Software without restriction, including without limitation
  # the rights to use, copy, modify, merge, publish,
  # distribute, sublicense, and/or sell copies of the
  # Software, and to permit persons to whom the Software is
  # furnished to do so, subject to the following conditions:
  #
  # The above copyright notice and this permission notice
  # shall be included in all copies or substantial portions of
  # the Software.
  #
  # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  # PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
  # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  
  """The main Nikola site object."""
  
  import datetime
  import io
  import json
  import functools
  import logging
  import operator
  import os
  import sys
  import mimetypes
  from collections import defaultdict
  from copy import copy
  from urllib.parse import urlparse, urlsplit, urlunsplit, urljoin, unquote, parse_qs
  
  import dateutil.tz
  import lxml.etree
  import lxml.html
  import natsort
  import PyRSS2Gen as rss
  from pkg_resources import resource_filename
  from blinker import signal
  from yapsy.PluginManager import PluginManager
  
  from . import DEBUG, SHOW_TRACEBACKS, filters, utils, hierarchy_utils, shortcodes
  from . import metadata_extractors
  from .metadata_extractors import default_metadata_extractors_by
  from .post import Post  # NOQA
  from .plugin_categories import (
      Command,
      LateTask,
      PageCompiler,
      CompilerExtension,
      MarkdownExtension,
      RestExtension,
      MetadataExtractor,
      ShortcodePlugin,
      Task,
      TaskMultiplier,
      TemplateSystem,
      SignalHandler,
      ConfigPlugin,
      PostScanner,
      Taxonomy,
  )
  from .state import Persistor
  
  try:
      import pyphen
  except ImportError:
      pyphen = None
  
  if DEBUG:
      logging.basicConfig(level=logging.DEBUG)
  else:
      logging.basicConfig(level=logging.ERROR)
  
  # Default "Read more..." link
  DEFAULT_INDEX_READ_MORE_LINK = '<p class="more"><a href="{link}">{read_more}…</a></p>'
  DEFAULT_FEED_READ_MORE_LINK = '<p><a href="{link}">{read_more}…</a> ({min_remaining_read})</p>'
  
  
  config_changed = utils.config_changed
  
  __all__ = ('Nikola',)
  
  # We store legal values for some settings here.  For internal use.
  LEGAL_VALUES = {
      'DEFAULT_THEME': 'bootblog4',
      'COMMENT_SYSTEM': [
          'disqus',
          'facebook',
          'intensedebate',
          'isso',
          'muut',
          'commento',
          'utterances',
      ],
      'TRANSLATIONS': {
          'af': 'Afrikaans',
          'ar': 'Arabic',
          'az': 'Azerbaijani',
          'bg': 'Bulgarian',
          'bs': 'Bosnian',
          'ca': 'Catalan',
          ('cs', 'cz'): 'Czech',
          'da': 'Danish',
          'de': 'German',
          ('el', '!gr'): 'Greek',
          'en': 'English',
          'eo': 'Esperanto',
          'es': 'Spanish',
          'et': 'Estonian',
          'eu': 'Basque',
          'fa': 'Persian',
          'fi': 'Finnish',
          'fr': 'French',
          'fur': 'Friulian',
          'gl': 'Galician',
          'he': 'Hebrew',
          'hi': 'Hindi',
          'hr': 'Croatian',
          'hu': 'Hungarian',
          'ia': 'Interlingua',
          'id': 'Indonesian',
          'it': 'Italian',
          ('ja', '!jp'): 'Japanese',
          'ko': 'Korean',
          'lt': 'Lithuanian',
          'ml': 'Malayalam',
          'mr': 'Marathi',
          'nb': 'Norwegian (Bokmål)',
          'nl': 'Dutch',
          'pa': 'Punjabi',
          'pl': 'Polish',
          'pt': 'Portuguese',
          'pt_br': 'Portuguese (Brazil)',
          'ru': 'Russian',
          'sk': 'Slovak',
          'sl': 'Slovene',
          'sq': 'Albanian',
          'sr': 'Serbian (Cyrillic)',
          'sr_latin': 'Serbian (Latin)',
          'sv': 'Swedish',
          'te': 'Telugu',
          'th': 'Thai',
          ('tr', '!tr_TR'): 'Turkish',
          'uk': 'Ukrainian',
          'ur': 'Urdu',
          'vi': 'Vietnamese',
          'zh_cn': 'Chinese (Simplified)',
          'zh_tw': 'Chinese (Traditional)'
      },
      '_TRANSLATIONS_WITH_COUNTRY_SPECIFIERS': {
          # This dict is used in `init` in case of locales that exist with a
          # country specifier.  If there is no other locale that has the same
          # language with a different country, ``nikola init`` (but nobody else!)
          # will accept it, warning the user about it.
  
          # This dict is currently empty.
      },
      'LOCALES_BASE': {
          # A list of locale mappings to apply for every site. Can be overridden in the config.
          'sr_latin': 'sr_Latn',
      },
      'RTL_LANGUAGES': ('ar', 'fa', 'he', 'ur'),
      'LUXON_LOCALES': defaultdict(lambda: 'en', **{
          'af': 'af',
          'ar': 'ar',
          'az': 'az',
          'bg': 'bg',
          'bn': 'bn',
          'bs': 'bs',
          'ca': 'ca',
          'cs': 'cs',
          'cz': 'cs',
          'da': 'da',
          'de': 'de',
          'el': 'el',
          'en': 'en',
          'eo': 'eo',
          'es': 'es',
          'et': 'et',
          'eu': 'eu',
          'fa': 'fa',
          'fi': 'fi',
          'fr': 'fr',
          'fur': 'fur',
          'gl': 'gl',
          'hi': 'hi',
          'he': 'he',
          'hr': 'hr',
          'hu': 'hu',
          'ia': 'ia',
          'id': 'id',
          'it': 'it',
          'ja': 'ja',
          'ko': 'ko',
          'lt': 'lt',
          'ml': 'ml',
          'mr': 'mr',
          'nb': 'nb',
          'nl': 'nl',
          'pa': 'pa',
          'pl': 'pl',
          'pt': 'pt',
          'pt_br': 'pt-BR',
          'ru': 'ru',
          'sk': 'sk',
          'sl': 'sl',
          'sq': 'sq',
          'sr': 'sr-Cyrl',
          'sr_latin': 'sr-Latn',
          'sv': 'sv',
          'te': 'te',
          'tr': 'tr',
          'th': 'th',
          'uk': 'uk',
          'ur': 'ur',
          'vi': 'vi',
          'zh_cn': 'zh-CN',
          'zh_tw': 'zh-TW'
      }),
      # TODO: remove in v9
      'MOMENTJS_LOCALES': defaultdict(lambda: 'en', **{
          'af': 'af',
          'ar': 'ar',
          'az': 'az',
          'bg': 'bg',
          'bn': 'bn',
          'bs': 'bs',
          'ca': 'ca',
          'cs': 'cs',
          'cz': 'cs',
          'da': 'da',
          'de': 'de',
          'el': 'el',
          'en': 'en',
          'eo': 'eo',
          'es': 'es',
          'et': 'et',
          'eu': 'eu',
          'fa': 'fa',
          'fi': 'fi',
          'fr': 'fr',
          'gl': 'gl',
          'hi': 'hi',
          'he': 'he',
          'hr': 'hr',
          'hu': 'hu',
          'id': 'id',
          'it': 'it',
          'ja': 'ja',
          'ko': 'ko',
          'lt': 'lt',
          'ml': 'ml',
          'mr': 'mr',
          'nb': 'nb',
          'nl': 'nl',
          'pa': 'pa-in',
          'pl': 'pl',
          'pt': 'pt',
          'pt_br': 'pt-br',
          'ru': 'ru',
          'sk': 'sk',
          'sl': 'sl',
          'sq': 'sq',
          'sr': 'sr-cyrl',
          'sr_latin': 'sr',
          'sv': 'sv',
          'te': 'te',
          'tr': 'tr',
          'th': 'th',
          'uk': 'uk',
          'ur': 'ur',
          'vi': 'vi',
          'zh_cn': 'zh-cn',
          'zh_tw': 'zh-tw'
      }),
      'PYPHEN_LOCALES': {
          'af': 'af',
          'bg': 'bg',
          'ca': 'ca',
          'cs': 'cs',
          'cz': 'cs',
          'da': 'da',
          'de': 'de',
          'el': 'el',
          'en': 'en_US',
          'es': 'es',
          'et': 'et',
          'fr': 'fr',
          'hr': 'hr',
          'hu': 'hu',
          'it': 'it',
          'lt': 'lt',
          'nb': 'nb',
          'nl': 'nl',
          'pl': 'pl',
          'pt': 'pt',
          'pt_br': 'pt_BR',
          'ru': 'ru',
          'sk': 'sk',
          'sl': 'sl',
          'sr': 'sr',
          'sv': 'sv',
          'te': 'te',
          'uk': 'uk',
      },
      'DOCUTILS_LOCALES': {
          'af': 'af',
          'ca': 'ca',
          'da': 'da',
          'de': 'de',
          'en': 'en',
          'eo': 'eo',
          'es': 'es',
          'fa': 'fa',
          'fi': 'fi',
          'fr': 'fr',
          'gl': 'gl',
          'he': 'he',
          'it': 'it',
          'ja': 'ja',
          'lt': 'lt',
          'nl': 'nl',
          'pl': 'pl',
          'pt': 'pt_br',  # hope nobody will mind
          'pt_br': 'pt_br',
          'ru': 'ru',
          'sk': 'sk',
          'sv': 'sv',
          'zh_cn': 'zh_cn',
          'zh_tw': 'zh_tw'
      },
      "METADATA_MAPPING": ["yaml", "toml", "rest_docinfo", "markdown_metadata"],
  }
  
  # Mapping old pre-taxonomy plugin names to new post-taxonomy plugin names
  TAXONOMY_COMPATIBILITY_PLUGIN_NAME_MAP = {
      "render_archive": ["classify_archive"],
      "render_authors": ["classify_authors"],
      "render_indexes": ["classify_page_index", "classify_sections"],  # "classify_indexes" removed from list (see #2591 and special-case logic below)
      "render_tags": ["classify_categories", "classify_tags"],
  }
  
  # Default value for the pattern used to name translated files
  DEFAULT_TRANSLATIONS_PATTERN = '{path}.{lang}.{ext}'
  
  
  def _enclosure(post, lang):
      """Add an enclosure to RSS."""
      enclosure = post.meta('enclosure', lang)
      if enclosure:
          try:
              length = int(post.meta('enclosure_length', lang) or 0)
          except KeyError:
              length = 0
          except ValueError:
              utils.LOGGER.warning("Invalid enclosure length for post {0}".format(post.source_path))
              length = 0
          url = enclosure
          mime = mimetypes.guess_type(url)[0]
          return url, length, mime
  
  
  class Nikola(object):
      """Class that handles site generation.
  
      Takes a site config as argument on creation.
      """
  
      def __init__(self, **config):
          """Initialize proper environment for running tasks."""
          # Register our own path handlers
          self.path_handlers = {
              'slug': self.slug_path,
              'post_path': self.post_path,
              'root': self.root_path,
              'filename': self.filename_path,
          }
  
          self.strict = False
          self.posts = []
          self.all_posts = []
          self.posts_per_year = defaultdict(list)
          self.posts_per_month = defaultdict(list)
          self.posts_per_tag = defaultdict(list)
          self.posts_per_category = defaultdict(list)
          self.tags_per_language = defaultdict(list)
          self.post_per_file = {}
          self.timeline = []
          self.pages = []
          self._scanned = False
          self._template_system = None
          self._THEMES = None
          self._MESSAGES = None
          self.filters = {}
          self.debug = DEBUG
          self.show_tracebacks = SHOW_TRACEBACKS
          self.colorful = config.pop('__colorful__', False)
          self.invariant = config.pop('__invariant__', False)
          self.quiet = config.pop('__quiet__', False)
          self._doit_config = config.pop('DOIT_CONFIG', {})
          self.original_cwd = config.pop('__cwd__', False)
          self.configuration_filename = config.pop('__configuration_filename__', False)
          self.configured = bool(config)
          self.injected_deps = defaultdict(list)
          self.shortcode_registry = {}
          self.metadata_extractors_by = default_metadata_extractors_by()
          self.registered_auto_watched_folders = set()
  
          self.rst_transforms = []
          self.template_hooks = {
              'extra_head': utils.TemplateHookRegistry('extra_head', self),
              'body_end': utils.TemplateHookRegistry('body_end', self),
              'page_header': utils.TemplateHookRegistry('page_header', self),
              'menu': utils.TemplateHookRegistry('menu', self),
              'menu_alt': utils.TemplateHookRegistry('menu_alt', self),
              'page_footer': utils.TemplateHookRegistry('page_footer', self),
          }
  
          # Maintain API
          utils.generic_rss_renderer = self.generic_rss_renderer
  
          # This is the default config
          self.config = {
              'ARCHIVE_PATH': "",
              'ARCHIVE_FILENAME': "archive.html",
              'ARCHIVES_ARE_INDEXES': False,
              'AUTHOR_PATH': 'authors',
              'AUTHOR_PAGES_ARE_INDEXES': False,
              'AUTHOR_PAGES_DESCRIPTIONS': {},
              'AUTHORLIST_MINIMUM_POSTS': 1,
              'BLOG_AUTHOR': 'Default Author',
              'BLOG_TITLE': 'Default Title',
              'BLOG_EMAIL': '',
              'BLOG_DESCRIPTION': 'Default Description',
              'BODY_END': "",
              'CACHE_FOLDER': 'cache',
              'CATEGORIES_INDEX_PATH': '',
              'CATEGORY_PATH': None,  # None means: same as TAG_PATH
              'CATEGORY_PAGES_ARE_INDEXES': None,  # None means: same as TAG_PAGES_ARE_INDEXES
              'CATEGORY_DESCRIPTIONS': {},
              'CATEGORY_TITLES': {},
              'CATEGORY_PREFIX': 'cat_',
              'CATEGORY_ALLOW_HIERARCHIES': False,
              'CATEGORY_OUTPUT_FLAT_HIERARCHY': False,
              'CATEGORY_DESTPATH_AS_DEFAULT': False,
              'CATEGORY_DESTPATH_TRIM_PREFIX': False,
              'CATEGORY_DESTPATH_FIRST_DIRECTORY_ONLY': True,
              'CATEGORY_DESTPATH_NAMES': {},
              'CATEGORY_PAGES_FOLLOW_DESTPATH': False,
              'CATEGORY_TRANSLATIONS': [],
              'CATEGORY_TRANSLATIONS_ADD_DEFAULTS': False,
              'CODE_COLOR_SCHEME': 'default',
              'COMMENT_SYSTEM': 'disqus',
              'COMMENTS_IN_GALLERIES': False,
              'COMMENTS_IN_PAGES': False,
              'COMPILERS': {
                  "rest": ('.txt', '.rst'),
                  "markdown": ('.md', '.mdown', '.markdown'),
                  "textile": ('.textile',),
                  "txt2tags": ('.t2t',),
                  "bbcode": ('.bb',),
                  "wiki": ('.wiki',),
                  "ipynb": ('.ipynb',),
                  "html": ('.html', '.htm')
              },
              'CONTENT_FOOTER': '',
              'CONTENT_FOOTER_FORMATS': {},
              'RSS_COPYRIGHT': '',
              'RSS_COPYRIGHT_PLAIN': '',
              'RSS_COPYRIGHT_FORMATS': {},
              'COPY_SOURCES': True,
              'CREATE_ARCHIVE_NAVIGATION': False,
              'CREATE_MONTHLY_ARCHIVE': False,
              'CREATE_SINGLE_ARCHIVE': False,
              'CREATE_FULL_ARCHIVES': False,
              'CREATE_DAILY_ARCHIVE': False,
              'DATE_FORMAT': 'yyyy-MM-dd HH:mm',
              'DISABLE_INDEXES': False,
              'DISABLE_MAIN_ATOM_FEED': False,
              'DISABLE_MAIN_RSS_FEED': False,
              'MOMENTJS_DATE_FORMAT': 'YYYY-MM-DD HH:mm',
              'LUXON_DATE_FORMAT': {},
              'DATE_FANCINESS': 0,
              'DEFAULT_LANG': "en",
              'DEPLOY_COMMANDS': {'default': []},
              'DISABLED_PLUGINS': [],
              'EXTRA_PLUGINS_DIRS': [],
              'EXTRA_THEMES_DIRS': [],
              'COMMENT_SYSTEM_ID': 'nikolademo',
              'ENABLE_AUTHOR_PAGES': True,
              'EXIF_WHITELIST': {},
              'EXTRA_HEAD_DATA': '',
              'FAVICONS': (),
              'FEED_LENGTH': 10,
              'FILE_METADATA_REGEXP': None,
              'FILE_METADATA_UNSLUGIFY_TITLES': True,
              'ADDITIONAL_METADATA': {},
              'FILES_FOLDERS': {'files': ''},
              'FILTERS': {},
              'FORCE_ISO8601': False,
              'FRONT_INDEX_HEADER': '',
              'GALLERY_FOLDERS': {'galleries': 'galleries'},
              'GALLERY_SORT_BY_DATE': True,
              'GALLERIES_USE_THUMBNAIL': False,
              'GALLERIES_DEFAULT_THUMBNAIL': None,
              'GLOBAL_CONTEXT_FILLER': [],
              'GZIP_COMMAND': None,
              'GZIP_FILES': False,
              'GZIP_EXTENSIONS': ('.txt', '.htm', '.html', '.css', '.js', '.json', '.xml'),
              'HIDDEN_AUTHORS': [],
              'HIDDEN_TAGS': [],
              'HIDE_REST_DOCINFO': False,
              'HIDDEN_CATEGORIES': [],
              'HYPHENATE': False,
              'IMAGE_FOLDERS': {'images': ''},
              'INDEX_DISPLAY_POST_COUNT': 10,
              'INDEX_FILE': 'index.html',
              'INDEX_TEASERS': False,
              'IMAGE_THUMBNAIL_SIZE': 400,
              'IMAGE_THUMBNAIL_FORMAT': '{name}.thumbnail{ext}',
              'INDEXES_TITLE': "",
              'INDEXES_PAGES': "",
              'INDEXES_PAGES_MAIN': False,
              'INDEXES_PRETTY_PAGE_URL': False,
              'INDEXES_STATIC': True,
              'INDEX_PATH': '',
              'IPYNB_CONFIG': {},
              'KATEX_AUTO_RENDER': '',
              'LICENSE': '',
              'LINK_CHECK_WHITELIST': [],
              'LISTINGS_FOLDERS': {'listings': 'listings'},
              'LOGO_URL': '',
              'DEFAULT_PREVIEW_IMAGE': None,
              'NAVIGATION_LINKS': {},
              'NAVIGATION_ALT_LINKS': {},
              'MARKDOWN_EXTENSIONS': ['fenced_code', 'codehilite', 'extra'],
              'MARKDOWN_EXTENSION_CONFIGS': {},
              'MAX_IMAGE_SIZE': 1280,
              'MATHJAX_CONFIG': '',
              'METADATA_FORMAT': 'nikola',
              'METADATA_MAPPING': {},
              'MULTIPLE_AUTHORS_PER_POST': False,
              'NEW_POST_DATE_PATH': False,
              'NEW_POST_DATE_PATH_FORMAT': '%Y/%m/%d',
              'OLD_THEME_SUPPORT': True,
              'OUTPUT_FOLDER': 'output',
              'POSTS': (("posts/*.txt", "posts", "post.tmpl"),),
              'PRESERVE_EXIF_DATA': False,
              'PRESERVE_ICC_PROFILES': False,
              'PAGES': (("pages/*.txt", "pages", "page.tmpl"),),
              'PANDOC_OPTIONS': [],
              'PRETTY_URLS': True,
              'FUTURE_IS_NOW': False,
              'INDEX_READ_MORE_LINK': DEFAULT_INDEX_READ_MORE_LINK,
              'REDIRECTIONS': [],
              'ROBOTS_EXCLUSIONS': [],
              'GENERATE_ATOM': False,
              'ATOM_EXTENSION': '.atom',
              'ATOM_PATH': '',
              'ATOM_FILENAME_BASE': 'index',
              'FEED_TEASERS': True,
              'FEED_PLAIN': False,
              'FEED_READ_MORE_LINK': DEFAULT_FEED_READ_MORE_LINK,
              'FEED_LINKS_APPEND_QUERY': False,
              'GENERATE_RSS': True,
              'RSS_EXTENSION': '.xml',
              'RSS_LINK': None,
              'RSS_PATH': '',
              'RSS_FILENAME_BASE': 'rss',
              'SEARCH_FORM': '',
              'SHOW_BLOG_TITLE': True,
              'SHOW_INDEX_PAGE_NAVIGATION': False,
              'SHOW_SOURCELINK': True,
              'SHOW_UNTRANSLATED_POSTS': True,
              'SLUG_AUTHOR_PATH': True,
              'SLUG_TAG_PATH': True,
              'SOCIAL_BUTTONS_CODE': '',
              'SITE_URL': 'https://example.com/',
              'PAGE_INDEX': False,
              'SECTION_PATH': '',
              'STRIP_INDEXES': True,
              'TAG_PATH': 'categories',
              'TAG_PAGES_ARE_INDEXES': False,
              'TAG_DESCRIPTIONS': {},
              'TAG_TITLES': {},
              'TAG_TRANSLATIONS': [],
              'TAG_TRANSLATIONS_ADD_DEFAULTS': False,
              'TAGS_INDEX_PATH': '',
              'TAGLIST_MINIMUM_POSTS': 1,
              'TEMPLATE_FILTERS': {},
              'THEME': LEGAL_VALUES['DEFAULT_THEME'],
              'THEME_COLOR': '#5670d4',  # light "corporate blue"
              'THEME_CONFIG': {},
              'THUMBNAIL_SIZE': 180,
              'TRANSLATIONS_PATTERN': DEFAULT_TRANSLATIONS_PATTERN,
              'URL_TYPE': 'rel_path',
              'USE_BUNDLES': True,
              'USE_CDN': False,
              'USE_CDN_WARNING': True,
              'USE_REST_DOCINFO_METADATA': False,
              'USE_FILENAME_AS_TITLE': True,
              'USE_KATEX': False,
              'USE_SLUGIFY': True,
              'USE_TAG_METADATA': True,
              'TIMEZONE': 'UTC',
              'WARN_ABOUT_TAG_METADATA': True,
              'DEPLOY_DRAFTS': True,
              'DEPLOY_FUTURE': False,
              'SCHEDULE_ALL': False,
              'SCHEDULE_RULE': '',
              'DEMOTE_HEADERS': 1,
              'GITHUB_SOURCE_BRANCH': 'master',
              'GITHUB_DEPLOY_BRANCH': 'gh-pages',
              'GITHUB_REMOTE_NAME': 'origin',
              'GITHUB_COMMIT_SOURCE': False,  # WARNING: conf.py.in overrides this with True for backwards compatibility
              'META_GENERATOR_TAG': True,
              'REST_FILE_INSERTION_ENABLED': True,
              'TYPES_TO_HIDE_TITLE': [],
          }
  
          # set global_context for template rendering
          self._GLOBAL_CONTEXT = {}
  
          # dependencies for all pages, not included in global context
          self.ALL_PAGE_DEPS = {}
  
          self.config.update(config)
  
          # __builtins__ contains useless cruft
          if '__builtins__' in self.config:
              try:
                  del self.config['__builtins__']
              except KeyError:
                  del self.config[b'__builtins__']
  
          self.config['__colorful__'] = self.colorful
          self.config['__invariant__'] = self.invariant
          self.config['__quiet__'] = self.quiet
  
          # Use ATOM_PATH when set
          self.config['ATOM_PATH'] = self.config['ATOM_PATH'] or self.config['INDEX_PATH']
  
          # Make sure we have sane NAVIGATION_LINKS and NAVIGATION_ALT_LINKS.
          if not self.config['NAVIGATION_LINKS']:
              self.config['NAVIGATION_LINKS'] = {self.config['DEFAULT_LANG']: ()}
          if not self.config['NAVIGATION_ALT_LINKS']:
              self.config['NAVIGATION_ALT_LINKS'] = {self.config['DEFAULT_LANG']: ()}
  
          # Translatability configuration.
          self.config['TRANSLATIONS'] = self.config.get('TRANSLATIONS',
                                                        {self.config['DEFAULT_LANG']: ''})
          for k, v in self.config['TRANSLATIONS'].items():
              if os.path.isabs(v):
                  self.config['TRANSLATIONS'][k] = os.path.relpath(v, '/')
  
          utils.TranslatableSetting.default_lang = self.config['DEFAULT_LANG']
  
          self.TRANSLATABLE_SETTINGS = ('BLOG_AUTHOR',
                                        'BLOG_TITLE',
                                        'BLOG_DESCRIPTION',
                                        'LICENSE',
                                        'CONTENT_FOOTER',
                                        'SOCIAL_BUTTONS_CODE',
                                        'SEARCH_FORM',
                                        'BODY_END',
                                        'EXTRA_HEAD_DATA',
                                        'NAVIGATION_LINKS',
                                        'NAVIGATION_ALT_LINKS',
                                        'FRONT_INDEX_HEADER',
                                        'INDEX_READ_MORE_LINK',
                                        'FEED_READ_MORE_LINK',
                                        'INDEXES_TITLE',
                                        'CATEGORY_DESTPATH_NAMES',
                                        'INDEXES_PAGES',
                                        'INDEXES_PRETTY_PAGE_URL',
                                        'THEME_CONFIG',
                                        # PATH options (Issue #1914)
                                        'ARCHIVE_PATH',
                                        'ARCHIVE_FILENAME',
                                        'TAG_PATH',
                                        'TAGS_INDEX_PATH',
                                        'CATEGORY_PATH',
                                        'CATEGORIES_INDEX_PATH',
                                        'SECTION_PATH',
                                        'INDEX_PATH',
                                        'ATOM_PATH',
                                        'RSS_PATH',
                                        'RSS_FILENAME_BASE',
                                        'ATOM_FILENAME_BASE',
                                        'AUTHOR_PATH',
                                        'DATE_FORMAT',
                                        'LUXON_DATE_FORMAT',
                                        'MOMENTJS_DATE_FORMAT',  # TODO: remove in v9
                                        'RSS_COPYRIGHT',
                                        'RSS_COPYRIGHT_PLAIN',
                                        # Issue #2970
                                        'MARKDOWN_EXTENSION_CONFIGS',
                                        )
  
          self._GLOBAL_CONTEXT_TRANSLATABLE = ('blog_author',
                                               'blog_title',
                                               'blog_description',
                                               'license',
                                               'content_footer',
                                               'social_buttons_code',
                                               'search_form',
                                               'body_end',
                                               'extra_head_data',
                                               'date_format',
                                               'js_date_format',
                                               'luxon_date_format',
                                               'front_index_header',
                                               'theme_config',
                                               )
  
          self._ALL_PAGE_DEPS_TRANSLATABLE = ('atom_path',
                                              'rss_path',
                                              'rss_filename_base',
                                              'atom_filename_base',
                                              )
          # WARNING: navigation_(alt_)links SHOULD NOT be added to the list above.
          #          Themes ask for [lang] there and we should provide it.
  
          # Luxon setup is a dict of dicts, so we need to set up the default here.
          if not self.config['LUXON_DATE_FORMAT']:
              self.config['LUXON_DATE_FORMAT'] = {self.config['DEFAULT_LANG']: {'preset': False, 'format': 'yyyy-MM-dd HH:mm'}}
          # TODO: remove Moment.js stuff in v9
          if 'JS_DATE_FORMAT' in self.config:
              utils.LOGGER.warning("Moment.js was replaced by Luxon in the default themes, which uses different date formats.")
              utils.LOGGER.warning("If you’re using a built-in theme, set LUXON_DATE_FORMAT. If your theme uses Moment.js, you can silence this warning by renaming JS_DATE_FORMAT to MOMENTJS_DATE_FORMAT.")
              utils.LOGGER.warning("Sample Luxon config: LUXON_DATE_FORMAT = " + str(self.config['LUXON_DATE_FORMAT']))
              self.config['MOMENTJS_DATE_FORMAT'] = self.config['LUXON_DATE_FORMAT']
  
          # We first have to massage MOMENTJS_DATE_FORMAT and LUXON_DATE_FORMAT, otherwise we run into trouble
          if 'MOMENTJS_DATE_FORMAT' in self.config:
              if isinstance(self.config['MOMENTJS_DATE_FORMAT'], dict):
                  for k in self.config['MOMENTJS_DATE_FORMAT']:
                      self.config['MOMENTJS_DATE_FORMAT'][k] = json.dumps(self.config['MOMENTJS_DATE_FORMAT'][k])
              else:
                  self.config['MOMENTJS_DATE_FORMAT'] = json.dumps(self.config['MOMENTJS_DATE_FORMAT'])
  
          if 'LUXON_DATE_FORMAT' in self.config:
              for k in self.config['LUXON_DATE_FORMAT']:
                  self.config['LUXON_DATE_FORMAT'][k] = json.dumps(self.config['LUXON_DATE_FORMAT'][k])
  
          for i in self.TRANSLATABLE_SETTINGS:
              try:
                  self.config[i] = utils.TranslatableSetting(i, self.config[i], self.config['TRANSLATIONS'])
              except KeyError:
                  pass
  
          # A EXIF_WHITELIST implies you want to keep EXIF data
          if self.config['EXIF_WHITELIST'] and not self.config['PRESERVE_EXIF_DATA']:
              utils.LOGGER.warning('Setting EXIF_WHITELIST implies PRESERVE_EXIF_DATA is set to True')
              self.config['PRESERVE_EXIF_DATA'] = True
  
          # Setting PRESERVE_EXIF_DATA with an empty EXIF_WHITELIST implies 'keep everything'
          if self.config['PRESERVE_EXIF_DATA'] and not self.config['EXIF_WHITELIST']:
              utils.LOGGER.warning('You are setting PRESERVE_EXIF_DATA and not EXIF_WHITELIST so EXIF data is not really kept.')
  
          if 'UNSLUGIFY_TITLES' in self.config:
              utils.LOGGER.warning('The UNSLUGIFY_TITLES setting was renamed to FILE_METADATA_UNSLUGIFY_TITLES.')
              self.config['FILE_METADATA_UNSLUGIFY_TITLES'] = self.config['UNSLUGIFY_TITLES']
  
          if 'TAG_PAGES_TITLES' in self.config:
              utils.LOGGER.warning('The TAG_PAGES_TITLES setting was renamed to TAG_TITLES.')
              self.config['TAG_TITLES'] = self.config['TAG_PAGES_TITLES']
  
          if 'TAG_PAGES_DESCRIPTIONS' in self.config:
              utils.LOGGER.warning('The TAG_PAGES_DESCRIPTIONS setting was renamed to TAG_DESCRIPTIONS.')
              self.config['TAG_DESCRIPTIONS'] = self.config['TAG_PAGES_DESCRIPTIONS']
  
          if 'CATEGORY_PAGES_TITLES' in self.config:
              utils.LOGGER.warning('The CATEGORY_PAGES_TITLES setting was renamed to CATEGORY_TITLES.')
              self.config['CATEGORY_TITLES'] = self.config['CATEGORY_PAGES_TITLES']
  
          if 'CATEGORY_PAGES_DESCRIPTIONS' in self.config:
              utils.LOGGER.warning('The CATEGORY_PAGES_DESCRIPTIONS setting was renamed to CATEGORY_DESCRIPTIONS.')
              self.config['CATEGORY_DESCRIPTIONS'] = self.config['CATEGORY_PAGES_DESCRIPTIONS']
  
          if 'DISABLE_INDEXES_PLUGIN_INDEX_AND_ATOM_FEED' in self.config:
              utils.LOGGER.warning('The DISABLE_INDEXES_PLUGIN_INDEX_AND_ATOM_FEED setting was renamed and split to DISABLE_INDEXES and DISABLE_MAIN_ATOM_FEED.')
              self.config['DISABLE_INDEXES'] = self.config['DISABLE_INDEXES_PLUGIN_INDEX_AND_ATOM_FEED']
              self.config['DISABLE_MAIN_ATOM_FEED'] = self.config['DISABLE_INDEXES_PLUGIN_INDEX_AND_ATOM_FEED']
  
          if 'DISABLE_INDEXES_PLUGIN_RSS_FEED' in self.config:
              utils.LOGGER.warning('The DISABLE_INDEXES_PLUGIN_RSS_FEED setting was renamed to DISABLE_MAIN_RSS_FEED.')
              self.config['DISABLE_MAIN_RSS_FEED'] = self.config['DISABLE_INDEXES_PLUGIN_RSS_FEED']
  
          for val in self.config['DATE_FORMAT'].values.values():
              if '%' in val:
                  utils.LOGGER.error('The DATE_FORMAT setting needs to be upgraded.')
                  utils.LOGGER.warning("Nikola now uses CLDR-style date strings. http://cldr.unicode.org/translation/date-time-1/date-time")
                  utils.LOGGER.warning("Example: %Y-%m-%d %H:%M ==> yyyy-MM-dd HH:mm")
                  utils.LOGGER.warning("(note it’s different to what moment.js uses!)")
                  sys.exit(1)
  
          # Silently upgrade LOCALES (remove encoding)
          locales = LEGAL_VALUES['LOCALES_BASE']
          if 'LOCALES' in self.config:
              for k, v in self.config['LOCALES'].items():
                  self.config['LOCALES'][k] = v.split('.')[0]
              locales.update(self.config['LOCALES'])
          self.config['LOCALES'] = locales
  
          if self.config.get('POSTS_SECTIONS'):
              utils.LOGGER.warning("The sections feature has been removed and its functionality has been merged into categories.")
              utils.LOGGER.warning("For more information on how to migrate, please read: https://getnikola.com/blog/upgrading-to-nikola-v8.html#sections-were-replaced-by-categories")
  
              for section_config_suffix, cat_config_suffix in (
                  ('DESCRIPTIONS', 'DESCRIPTIONS'),
                  ('TITLE', 'TITLES'),
                  ('TRANSLATIONS', 'TRANSLATIONS')
              ):
                  section_config = 'POSTS_SECTION_' + section_config_suffix
                  cat_config = 'CATEGORY_' + cat_config_suffix
                  if section_config in self.config:
                      self.config[section_config].update(self.config[cat_config])
                      self.config[cat_config] = self.config[section_config]
  
              self.config['CATEGORY_DESTPATH_NAMES'] = self.config.get('POSTS_SECTION_NAME', {})
              # Need to mark this translatable manually.
              self.config['CATEGORY_DESTPATH_NAMES'] = utils.TranslatableSetting('CATEGORY_DESTPATH_NAMES', self.config['CATEGORY_DESTPATH_NAMES'], self.config['TRANSLATIONS'])
  
              self.config['CATEGORY_DESTPATH_AS_DEFAULT'] = not self.config.get('POSTS_SECTION_FROM_META')
              utils.LOGGER.info("Setting CATEGORY_DESTPATH_AS_DEFAULT = " + str(self.config['CATEGORY_DESTPATH_AS_DEFAULT']))
  
          if self.config.get('CATEGORY_PAGES_FOLLOW_DESTPATH') and (not self.config.get('CATEGORY_ALLOW_HIERARCHIES') or self.config.get('CATEGORY_OUTPUT_FLAT_HIERARCHY')):
              utils.LOGGER.error('CATEGORY_PAGES_FOLLOW_DESTPATH requires CATEGORY_ALLOW_HIERARCHIES = True, CATEGORY_OUTPUT_FLAT_HIERARCHY = False.')
              sys.exit(1)
  
          # The Utterances comment system has a required configuration value
          if self.config.get('COMMENT_SYSTEM') == 'utterances':
              utterances_config = self.config.get('GLOBAL_CONTEXT', {}).get('utterances_config', {})
              if not ('issue-term' in utterances_config or 'issue-number' in utterances_config):
                  utils.LOGGER.error("COMMENT_SYSTEM = 'utterances' must have either GLOBAL_CONTEXT['utterances_config']['issue-term'] or GLOBAL_CONTEXT['utterances_config']['issue-term'] defined.")
  
          # Handle CONTENT_FOOTER and RSS_COPYRIGHT* properly.
          # We provide the arguments to format in CONTENT_FOOTER_FORMATS and RSS_COPYRIGHT_FORMATS.
          self.config['CONTENT_FOOTER'].langformat(self.config['CONTENT_FOOTER_FORMATS'])
          self.config['RSS_COPYRIGHT'].langformat(self.config['RSS_COPYRIGHT_FORMATS'])
          self.config['RSS_COPYRIGHT_PLAIN'].langformat(self.config['RSS_COPYRIGHT_FORMATS'])
  
          # propagate USE_SLUGIFY
          utils.USE_SLUGIFY = self.config['USE_SLUGIFY']
  
          # Make sure we have pyphen installed if we are using it
          if self.config.get('HYPHENATE') and pyphen is None:
              utils.LOGGER.warning('To use the hyphenation, you have to install '
                                   'the "pyphen" package.')
              utils.LOGGER.warning('Setting HYPHENATE to False.')
              self.config['HYPHENATE'] = False
  
          # FIXME: Internally, we still use post_pages because it's a pain to change it
          self.config['post_pages'] = []
          for i1, i2, i3 in self.config['POSTS']:
              self.config['post_pages'].append([i1, i2, i3, True])
          for i1, i2, i3 in self.config['PAGES']:
              self.config['post_pages'].append([i1, i2, i3, False])
  
          # Handle old plugin names (from before merging the taxonomy PR #2535)
          for old_plugin_name, new_plugin_names in TAXONOMY_COMPATIBILITY_PLUGIN_NAME_MAP.items():
              if old_plugin_name in self.config['DISABLED_PLUGINS']:
                  missing_plugins = []
                  for plugin_name in new_plugin_names:
                      if plugin_name not in self.config['DISABLED_PLUGINS']:
                          missing_plugins.append(plugin_name)
                  if missing_plugins:
                      utils.LOGGER.warning('The "{}" plugin was replaced by several taxonomy plugins (see PR #2535): {}'.format(old_plugin_name, ', '.join(new_plugin_names)))
                      utils.LOGGER.warning('You are currently disabling "{}", but not the following new taxonomy plugins: {}'.format(old_plugin_name, ', '.join(missing_plugins)))
                      utils.LOGGER.warning('Please also disable these new plugins or remove "{}" from the DISABLED_PLUGINS list.'.format(old_plugin_name))
                      self.config['DISABLED_PLUGINS'].extend(missing_plugins)
          # Special-case logic for "render_indexes" to fix #2591
          if 'render_indexes' in self.config['DISABLED_PLUGINS']:
              if 'generate_rss' in self.config['DISABLED_PLUGINS'] or self.config['GENERATE_RSS'] is False:
                  if 'classify_indexes' not in self.config['DISABLED_PLUGINS']:
                      utils.LOGGER.warning('You are disabling the "render_indexes" plugin, as well as disabling the "generate_rss" plugin or setting GENERATE_RSS to False. To achieve the same effect, please disable the "classify_indexes" plugin in the future.')
                      self.config['DISABLED_PLUGINS'].append('classify_indexes')
              else:
                  if not self.config['DISABLE_INDEXES']:
                      utils.LOGGER.warning('You are disabling the "render_indexes" plugin, but not the generation of RSS feeds. Please put "DISABLE_INDEXES = True" into your configuration instead.')
                      self.config['DISABLE_INDEXES'] = True
  
          # Disable RSS.  For a successful disable, we must have both the option
          # false and the plugin disabled through the official means.
          if 'generate_rss' in self.config['DISABLED_PLUGINS'] and self.config['GENERATE_RSS'] is True:
              utils.LOGGER.warning('Please use GENERATE_RSS to disable RSS feed generation, instead of mentioning generate_rss in DISABLED_PLUGINS.')
              self.config['GENERATE_RSS'] = False
              self.config['DISABLE_MAIN_RSS_FEED'] = True
  
          # PRETTY_URLS defaults to enabling STRIP_INDEXES unless explicitly disabled
          if self.config.get('PRETTY_URLS') and 'STRIP_INDEXES' not in config:
              self.config['STRIP_INDEXES'] = True
  
          if not self.config.get('COPY_SOURCES'):
              self.config['SHOW_SOURCELINK'] = False
  
          if self.config['CATEGORY_PATH']._inp is None:
              self.config['CATEGORY_PATH'] = self.config['TAG_PATH']
          if self.config['CATEGORY_PAGES_ARE_INDEXES'] is None:
              self.config['CATEGORY_PAGES_ARE_INDEXES'] = self.config['TAG_PAGES_ARE_INDEXES']
  
          self.default_lang = self.config['DEFAULT_LANG']
          self.translations = self.config['TRANSLATIONS']
  
          utils.LocaleBorg.initialize(self.config.get('LOCALES', {}), self.default_lang)
  
          # BASE_URL defaults to SITE_URL
          if 'BASE_URL' not in self.config:
              self.config['BASE_URL'] = self.config.get('SITE_URL')
          # BASE_URL should *always* end in /
          if self.config['BASE_URL'] and self.config['BASE_URL'][-1] != '/':
              utils.LOGGER.warning("Your BASE_URL doesn't end in / -- adding it, but please fix it in your config file!")
              self.config['BASE_URL'] += '/'
  
          try:
              _bnl = urlsplit(self.config['BASE_URL']).netloc
              _bnl.encode('ascii')
              urlsplit(self.config['SITE_URL']).netloc.encode('ascii')
          except (UnicodeEncodeError, UnicodeDecodeError):
              utils.LOGGER.error("Your BASE_URL or SITE_URL contains an IDN expressed in Unicode.  Please convert it to Punycode.")
              utils.LOGGER.error("Punycode of {}: {}".format(_bnl, _bnl.encode('idna')))
              sys.exit(1)
  
          # Load built-in metadata extractors
          metadata_extractors.load_defaults(self, self.metadata_extractors_by)
          if metadata_extractors.DEFAULT_EXTRACTOR is None:
              utils.LOGGER.error("Could not find default meta extractor ({})".format(
                  metadata_extractors.DEFAULT_EXTRACTOR_NAME))
              sys.exit(1)
  
          # The Pelican metadata format requires a markdown extension
          if config.get('METADATA_FORMAT', 'nikola').lower() == 'pelican':
              if 'markdown.extensions.meta' not in config.get('MARKDOWN_EXTENSIONS', []) \
                      and 'markdown' in self.config['COMPILERS']:
                  utils.LOGGER.warning(
                      'To use the Pelican metadata format, you need to add '
                      '"markdown.extensions.meta" to your MARKDOWN_EXTENSIONS setting.')
  
          # We use one global tzinfo object all over Nikola.
          try:
              self.tzinfo = dateutil.tz.gettz(self.config['TIMEZONE'])
          except Exception as exc:
              utils.LOGGER.warning("Error getting TZ: {}", exc)
              self.tzinfo = dateutil.tz.gettz()
          self.config['__tzinfo__'] = self.tzinfo
  
          # Store raw compilers for internal use (need a copy for that)
          self.config['_COMPILERS_RAW'] = {}
          for k, v in self.config['COMPILERS'].items():
              self.config['_COMPILERS_RAW'][k] = list(v)
  
          # Get search path for themes
          self.themes_dirs = ['themes'] + self.config['EXTRA_THEMES_DIRS']
  
          # Register default filters
          filter_name_format = 'filters.{0}'
          for filter_name, filter_definition in filters.__dict__.items():
              # Ignore objects whose name starts with an underscore, or which are not callable
              if filter_name.startswith('_') or not callable(filter_definition):
                  continue
              # Register all other objects as filters
              self.register_filter(filter_name_format.format(filter_name), filter_definition)
  
          self._set_global_context_from_config()
          self._set_all_page_deps_from_config()
          # Read data files only if a site exists (Issue #2708)
          if self.configured:
              self._set_global_context_from_data()
  
          # Set persistent state facility
          self.state = Persistor('state_data.json')
  
          # Set cache facility
          self.cache = Persistor(os.path.join(self.config['CACHE_FOLDER'], 'cache_data.json'))
  
          # Create directories for persistors only if a site exists (Issue #2334)
          if self.configured:
              self.state._set_site(self)
              self.cache._set_site(self)
  
      def _filter_duplicate_plugins(self, plugin_list):
          """Find repeated plugins and discard the less local copy."""
          def plugin_position_in_places(plugin):
              # plugin here is a tuple:
              # (path to the .plugin file, path to plugin module w/o .py, plugin metadata)
              for i, place in enumerate(self._plugin_places):
                  if plugin[0].startswith(place):
                      return i
              utils.LOGGER.warn("Duplicate plugin found in unexpected location: {}".format(plugin[0]))
              return len(self._plugin_places)
  
          plugin_dict = defaultdict(list)
          for data in plugin_list:
              plugin_dict[data[2].name].append(data)
          result = []
          for _, plugins in plugin_dict.items():
              if len(plugins) > 1:
                  # Sort by locality
                  plugins.sort(key=plugin_position_in_places)
                  utils.LOGGER.debug("Plugin {} exists in multiple places, using {}".format(
                      plugins[-1][2].name, plugins[-1][0]))
              result.append(plugins[-1])
          return result
  
      def init_plugins(self, commands_only=False, load_all=False):
          """Load plugins as needed."""
          self.plugin_manager = PluginManager(categories_filter={
              "Command": Command,
              "Task": Task,
              "LateTask": LateTask,
              "TemplateSystem": TemplateSystem,
              "PageCompiler": PageCompiler,
              "TaskMultiplier": TaskMultiplier,
              "CompilerExtension": CompilerExtension,
              "MarkdownExtension": MarkdownExtension,
              "RestExtension": RestExtension,
              "MetadataExtractor": MetadataExtractor,
              "ShortcodePlugin": ShortcodePlugin,
              "SignalHandler": SignalHandler,
              "ConfigPlugin": ConfigPlugin,
              "PostScanner": PostScanner,
              "Taxonomy": Taxonomy,
          })
          self.plugin_manager.getPluginLocator().setPluginInfoExtension('plugin')
          extra_plugins_dirs = self.config['EXTRA_PLUGINS_DIRS']
          self._plugin_places = [
              resource_filename('nikola', 'plugins'),
              os.path.expanduser(os.path.join('~', '.nikola', 'plugins')),
              os.path.join(os.getcwd(), 'plugins'),
          ] + [path for path in extra_plugins_dirs if path]
  
          compilers = defaultdict(set)
          # Also add aliases for combinations with TRANSLATIONS_PATTERN
          for compiler, exts in self.config['COMPILERS'].items():
              for ext in exts:
                  compilers[compiler].add(ext)
                  for lang in self.config['TRANSLATIONS'].keys():
                      candidate = utils.get_translation_candidate(self.config, "f" + ext, lang)
                      compilers[compiler].add(candidate)
  
          # Avoid redundant compilers (if load_all is False):
          # Remove compilers (and corresponding compiler extensions) that are not marked as
          # needed by any PostScanner plugin and put them into self.disabled_compilers
          # (respectively self.disabled_compiler_extensions).
          self.config['COMPILERS'] = {}
          self.disabled_compilers = {}
          self.disabled_compiler_extensions = defaultdict(list)
  
          self.plugin_manager.getPluginLocator().setPluginPlaces(self._plugin_places)
          self.plugin_manager.locatePlugins()
          bad_candidates = set([])
          if not load_all:
              for p in self.plugin_manager._candidates:
                  if commands_only:
                      if p[-1].details.has_option('Nikola', 'PluginCategory'):
                          # FIXME TemplateSystem should not be needed
                          if p[-1].details.get('Nikola', 'PluginCategory') not in {'Command', 'Template'}:
                              bad_candidates.add(p)
                      else:
                          bad_candidates.add(p)
                  elif self.configured:  # Not commands-only, and configured
                      # Remove blacklisted plugins
                      if p[-1].name in self.config['DISABLED_PLUGINS']:
                          bad_candidates.add(p)
                          utils.LOGGER.debug('Not loading disabled plugin {}', p[-1].name)
                      # Remove compilers we don't use
                      if p[-1].details.has_option('Nikola', 'PluginCategory') and p[-1].details.get('Nikola', 'PluginCategory') in ('Compiler', 'PageCompiler'):
                          bad_candidates.add(p)
                          self.disabled_compilers[p[-1].name] = p
                      # Remove compiler extensions we don't need
                      if p[-1].details.has_option('Nikola', 'compiler') and p[-1].details.get('Nikola', 'compiler') in self.disabled_compilers:
                          bad_candidates.add(p)
                          self.disabled_compiler_extensions[p[-1].details.get('Nikola', 'compiler')].append(p)
              self.plugin_manager._candidates = list(set(self.plugin_manager._candidates) - bad_candidates)
  
          self.plugin_manager._candidates = self._filter_duplicate_plugins(self.plugin_manager._candidates)
          self.plugin_manager.loadPlugins()
  
          # Search for compiler plugins which we disabled but shouldn't have
          self._activate_plugins_of_category("PostScanner")
          if not load_all:
              file_extensions = set()
              for post_scanner in [p.plugin_object for p in self.plugin_manager.getPluginsOfCategory('PostScanner')]:
                  exts = post_scanner.supported_extensions()
                  if exts is not None:
                      file_extensions.update(exts)
                  else:
                      # Stop scanning for more: once we get None, we have to load all compilers anyway
                      utils.LOGGER.debug("Post scanner {0!r} does not implement `supported_extensions`, loading all compilers".format(post_scanner))
                      file_extensions = None
                      break
              to_add = []
              for k, v in compilers.items():
                  if file_extensions is None or file_extensions.intersection(v):
                      self.config['COMPILERS'][k] = sorted(list(v))
                      p = self.disabled_compilers.pop(k, None)
                      if p:
                          to_add.append(p)
                      for p in self.disabled_compiler_extensions.pop(k, []):
                          to_add.append(p)
              for _, p in self.disabled_compilers.items():
                  utils.LOGGER.debug('Not loading unneeded compiler {}', p[-1].name)
              for _, plugins in self.disabled_compiler_extensions.items():
                  for p in plugins:
                      utils.LOGGER.debug('Not loading compiler extension {}', p[-1].name)
              if to_add:
                  self.plugin_manager._candidates = self._filter_duplicate_plugins(to_add)
                  self.plugin_manager.loadPlugins()
  
          # Jupyter theme configuration.  If a website has ipynb enabled in post_pages
          # we should enable the Jupyter CSS (leaving that up to the theme itself).
          if 'needs_ipython_css' not in self._GLOBAL_CONTEXT:
              self._GLOBAL_CONTEXT['needs_ipython_css'] = 'ipynb' in self.config['COMPILERS']
  
          # Activate metadata extractors and prepare them for use
          for p in self._activate_plugins_of_category("MetadataExtractor"):
              metadata_extractors.classify_extractor(p.plugin_object, self.metadata_extractors_by)
  
          self._activate_plugins_of_category("Taxonomy")
          self.taxonomy_plugins = {}
          for taxonomy in [p.plugin_object for p in self.plugin_manager.getPluginsOfCategory('Taxonomy')]:
              if not taxonomy.is_enabled():
                  continue
              if taxonomy.classification_name in self.taxonomy_plugins:
                  utils.LOGGER.error("Found more than one taxonomy with classification name '{}'!".format(taxonomy.classification_name))
                  sys.exit(1)
              self.taxonomy_plugins[taxonomy.classification_name] = taxonomy
  
          self._activate_plugins_of_category("SignalHandler")
  
          # Emit signal for SignalHandlers which need to start running immediately.
          signal('sighandlers_loaded').send(self)
  
          self._commands = {}
  
          command_plugins = self._activate_plugins_of_category("Command")
          for plugin_info in command_plugins:
              plugin_info.plugin_object.short_help = plugin_info.description
              self._commands[plugin_info.name] = plugin_info.plugin_object
  
          self._activate_plugins_of_category("Task")
          self._activate_plugins_of_category("LateTask")
          self._activate_plugins_of_category("TaskMultiplier")
  
          # Activate all required compiler plugins
          self.compiler_extensions = self._activate_plugins_of_category("CompilerExtension")
          for plugin_info in self.plugin_manager.getPluginsOfCategory("PageCompiler"):
              if plugin_info.name in self.config["COMPILERS"].keys():
                  self.plugin_manager.activatePluginByName(plugin_info.name)
                  plugin_info.plugin_object.set_site(self)
  
          # Activate shortcode plugins
          self._activate_plugins_of_category("ShortcodePlugin")
  
          # Load compiler plugins
          self.compilers = {}
          self.inverse_compilers = {}
  
          for plugin_info in self.plugin_manager.getPluginsOfCategory(
                  "PageCompiler"):
              self.compilers[plugin_info.name] = \
                  plugin_info.plugin_object
  
          # Load config plugins and register templated shortcodes
          self._activate_plugins_of_category("ConfigPlugin")
          self._register_templated_shortcodes()
  
          # Check with registered filters and configure filters
          for actions in self.config['FILTERS'].values():
              for i, f in enumerate(actions):
                  if isinstance(f, str):
                      # Check whether this denotes a registered filter
                      _f = self.filters.get(f)
                      if _f is not None:
                          f = _f
                          actions[i] = f
                  if hasattr(f, 'configuration_variables'):
                      args = {}
                      for arg, config in f.configuration_variables.items():
                          if config in self.config:
                              args[arg] = self.config[config]
                      if args:
                          actions[i] = functools.partial(f, **args)
  
          # Signal that we are configured
          signal('configured').send(self)
  
      def _set_global_context_from_config(self):
          """Create global context from configuration.
  
          These are options that are used by templates, so they always need to be
          available.
          """
          self._GLOBAL_CONTEXT['url_type'] = self.config['URL_TYPE']
          self._GLOBAL_CONTEXT['timezone'] = self.tzinfo
          self._GLOBAL_CONTEXT['_link'] = self.link
          try:
              self._GLOBAL_CONTEXT['set_locale'] = utils.LocaleBorg().set_locale
          except utils.LocaleBorgUninitializedException:
              self._GLOBAL_CONTEXT['set_locale'] = None
          self._GLOBAL_CONTEXT['rel_link'] = self.rel_link
          self._GLOBAL_CONTEXT['abs_link'] = self.abs_link
          self._GLOBAL_CONTEXT['exists'] = self.file_exists
          self._GLOBAL_CONTEXT['index_display_post_count'] = self.config[
              'INDEX_DISPLAY_POST_COUNT']
          self._GLOBAL_CONTEXT['index_file'] = self.config['INDEX_FILE']
          self._GLOBAL_CONTEXT['use_bundles'] = self.config['USE_BUNDLES']
          self._GLOBAL_CONTEXT['use_cdn'] = self.config.get("USE_CDN")
          self._GLOBAL_CONTEXT['theme_color'] = self.config.get("THEME_COLOR")
          self._GLOBAL_CONTEXT['theme_config'] = self.config.get("THEME_CONFIG")
          self._GLOBAL_CONTEXT['favicons'] = self.config['FAVICONS']
          self._GLOBAL_CONTEXT['date_format'] = self.config.get('DATE_FORMAT')
          self._GLOBAL_CONTEXT['blog_author'] = self.config.get('BLOG_AUTHOR')
          self._GLOBAL_CONTEXT['blog_title'] = self.config.get('BLOG_TITLE')
          self._GLOBAL_CONTEXT['blog_email'] = self.config.get('BLOG_EMAIL')
          self._GLOBAL_CONTEXT['show_blog_title'] = self.config.get('SHOW_BLOG_TITLE')
          self._GLOBAL_CONTEXT['logo_url'] = self.config.get('LOGO_URL')
          self._GLOBAL_CONTEXT['blog_description'] = self.config.get('BLOG_DESCRIPTION')
          self._GLOBAL_CONTEXT['front_index_header'] = self.config.get('FRONT_INDEX_HEADER')
          self._GLOBAL_CONTEXT['color_hsl_adjust_hex'] = utils.color_hsl_adjust_hex
          self._GLOBAL_CONTEXT['colorize_str_from_base_color'] = utils.colorize_str_from_base_color
          self._GLOBAL_CONTEXT['blog_url'] = self.config.get('SITE_URL')
          self._GLOBAL_CONTEXT['template_hooks'] = self.template_hooks
          self._GLOBAL_CONTEXT['body_end'] = self.config.get('BODY_END')
          self._GLOBAL_CONTEXT['social_buttons_code'] = self.config.get('SOCIAL_BUTTONS_CODE')
          self._GLOBAL_CONTEXT['translations'] = self.config.get('TRANSLATIONS')
          self._GLOBAL_CONTEXT['license'] = self.config.get('LICENSE')
          self._GLOBAL_CONTEXT['search_form'] = self.config.get('SEARCH_FORM')
          self._GLOBAL_CONTEXT['comment_system'] = self.config.get('COMMENT_SYSTEM')
          self._GLOBAL_CONTEXT['comment_system_id'] = self.config.get('COMMENT_SYSTEM_ID')
          self._GLOBAL_CONTEXT['site_has_comments'] = bool(self.config.get('COMMENT_SYSTEM'))
          self._GLOBAL_CONTEXT['mathjax_config'] = self.config.get(
              'MATHJAX_CONFIG')
          self._GLOBAL_CONTEXT['use_katex'] = self.config.get('USE_KATEX')
          self._GLOBAL_CONTEXT['katex_auto_render'] = self.config.get('KATEX_AUTO_RENDER')
          self._GLOBAL_CONTEXT['content_footer'] = self.config.get(
              'CONTENT_FOOTER')
          self._GLOBAL_CONTEXT['generate_atom'] = self.config.get('GENERATE_ATOM')
          self._GLOBAL_CONTEXT['generate_rss'] = self.config.get('GENERATE_RSS')
          self._GLOBAL_CONTEXT['rss_link'] = self.config.get('RSS_LINK')
  
          self._GLOBAL_CONTEXT['navigation_links'] = self.config.get('NAVIGATION_LINKS')
          self._GLOBAL_CONTEXT['navigation_alt_links'] = self.config.get('NAVIGATION_ALT_LINKS')
  
          self._GLOBAL_CONTEXT['twitter_card'] = self.config.get(
              'TWITTER_CARD', {})
          self._GLOBAL_CONTEXT['hide_sourcelink'] = not self.config.get(
              'SHOW_SOURCELINK')
          self._GLOBAL_CONTEXT['show_sourcelink'] = self.config.get(
              'SHOW_SOURCELINK')
          self._GLOBAL_CONTEXT['extra_head_data'] = self.config.get('EXTRA_HEAD_DATA')
          self._GLOBAL_CONTEXT['date_fanciness'] = self.config.get('DATE_FANCINESS')
          self._GLOBAL_CONTEXT['luxon_locales'] = LEGAL_VALUES['LUXON_LOCALES']
          self._GLOBAL_CONTEXT['luxon_date_format'] = self.config.get('LUXON_DATE_FORMAT')
          # TODO: remove in v9
          self._GLOBAL_CONTEXT['js_date_format'] = self.config.get('MOMENTJS_DATE_FORMAT')
          self._GLOBAL_CONTEXT['momentjs_locales'] = LEGAL_VALUES['MOMENTJS_LOCALES']
          # Patch missing locales into momentjs defaulting to English (Issue #3216)
          for l in self._GLOBAL_CONTEXT['translations']:
              if l not in self._GLOBAL_CONTEXT['momentjs_locales']:
                  self._GLOBAL_CONTEXT['momentjs_locales'][l] = ""
          self._GLOBAL_CONTEXT['hidden_tags'] = self.config.get('HIDDEN_TAGS')
          self._GLOBAL_CONTEXT['hidden_categories'] = self.config.get('HIDDEN_CATEGORIES')
          self._GLOBAL_CONTEXT['hidden_authors'] = self.config.get('HIDDEN_AUTHORS')
          self._GLOBAL_CONTEXT['url_replacer'] = self.url_replacer
          self._GLOBAL_CONTEXT['sort_posts'] = utils.sort_posts
          self._GLOBAL_CONTEXT['smartjoin'] = utils.smartjoin
          self._GLOBAL_CONTEXT['colorize_str'] = utils.colorize_str
          self._GLOBAL_CONTEXT['meta_generator_tag'] = self.config.get('META_GENERATOR_TAG')
          self._GLOBAL_CONTEXT['multiple_authors_per_post'] = self.config.get('MULTIPLE_AUTHORS_PER_POST')
  
          self._GLOBAL_CONTEXT.update(self.config.get('GLOBAL_CONTEXT', {}))
  
      def _set_global_context_from_data(self):
          """Load files from data/ and put them in the global context."""
          self._GLOBAL_CONTEXT['data'] = {}
          for root, dirs, files in os.walk('data', followlinks=True):
              for fname in files:
                  fname = os.path.join(root, fname)
                  data = utils.load_data(fname)
                  key = os.path.splitext(fname.split(os.sep, 1)[1])[0]
                  self._GLOBAL_CONTEXT['data'][key] = data
          # Offer global_data as an alias for data (Issue #2488)
          self._GLOBAL_CONTEXT['global_data'] = self._GLOBAL_CONTEXT['data']
  
      def _set_all_page_deps_from_config(self):
          """Save dependencies for all pages from configuration.
  
          Changes of values in this dict will force a rebuild of all pages.
          Unlike global context, contents are NOT available to templates.
          """
          self.ALL_PAGE_DEPS['atom_extension'] = self.config.get('ATOM_EXTENSION')
          self.ALL_PAGE_DEPS['atom_path'] = self.config.get('ATOM_PATH')
          self.ALL_PAGE_DEPS['rss_extension'] = self.config.get('RSS_EXTENSION')
          self.ALL_PAGE_DEPS['rss_path'] = self.config.get('RSS_PATH')
          self.ALL_PAGE_DEPS['rss_filename_base'] = self.config.get('RSS_FILENAME_BASE')
          self.ALL_PAGE_DEPS['atom_filename_base'] = self.config.get('ATOM_FILENAME_BASE')
          self.ALL_PAGE_DEPS['slug_author_path'] = self.config.get('SLUG_AUTHOR_PATH')
          self.ALL_PAGE_DEPS['slug_tag_path'] = self.config.get('SLUG_TAG_PATH')
          self.ALL_PAGE_DEPS['locale'] = self.config.get('LOCALE')
  
      def _activate_plugins_of_category(self, category):
          """Activate all the plugins of a given category and return them."""
          # this code duplicated in tests/base.py
          plugins = []
          for plugin_info in self.plugin_manager.getPluginsOfCategory(category):
              self.plugin_manager.activatePluginByName(plugin_info.name)
              plugin_info.plugin_object.set_site(self)
              plugins.append(plugin_info)
          return plugins
  
      def _get_themes(self):
          if self._THEMES is None:
              try:
                  self._THEMES = utils.get_theme_chain(self.config['THEME'], self.themes_dirs)
              except Exception:
                  if self.config['THEME'] != LEGAL_VALUES['DEFAULT_THEME']:
                      utils.LOGGER.warning('''Cannot load theme "{0}", using '{1}' instead.'''.format(
                          self.config['THEME'], LEGAL_VALUES['DEFAULT_THEME']))
                      self.config['THEME'] = LEGAL_VALUES['DEFAULT_THEME']
                      return self._get_themes()
                  raise
              # Check consistency of USE_CDN and the current THEME (Issue #386)
              if self.config['USE_CDN'] and self.config['USE_CDN_WARNING']:
                  bootstrap_path = utils.get_asset_path(os.path.join(
                      'assets', 'css', 'bootstrap.min.css'), self._THEMES)
                  if bootstrap_path and bootstrap_path.split(os.sep)[-4] not in ['bootstrap', 'bootstrap3', 'bootstrap4']:
                      utils.LOGGER.warning('The USE_CDN option may be incompatible with your theme, because it uses a hosted version of bootstrap.')
  
          return self._THEMES
  
      THEMES = property(_get_themes)
  
      def _get_messages(self):
          try:
              if self._MESSAGES is None:
                  self._MESSAGES = utils.load_messages(self.THEMES,
                                                       self.translations,
                                                       self.default_lang,
                                                       themes_dirs=self.themes_dirs)
              return self._MESSAGES
          except utils.LanguageNotFoundError as e:
              utils.LOGGER.error('''Cannot load language "{0}".  Please make sure it is supported by Nikola itself, or that you have the appropriate messages files in your themes.'''.format(e.lang))
              sys.exit(1)
  
      MESSAGES = property(_get_messages)
  
      def _get_global_context(self):
          """Initialize some parts of GLOBAL_CONTEXT only when it's queried."""
          if 'messages' not in self._GLOBAL_CONTEXT:
              self._GLOBAL_CONTEXT['messages'] = self.MESSAGES
          if 'has_custom_css' not in self._GLOBAL_CONTEXT:
              # check if custom css exist and is not empty
              custom_css_path = utils.get_asset_path(
                  'assets/css/custom.css',
                  self.THEMES,
                  self.config['FILES_FOLDERS']
              )
              if custom_css_path and self.file_exists(custom_css_path, not_empty=True):
                  self._GLOBAL_CONTEXT['has_custom_css'] = True
              else:
                  self._GLOBAL_CONTEXT['has_custom_css'] = False
  
          return self._GLOBAL_CONTEXT
  
      GLOBAL_CONTEXT = property(_get_global_context)
  
      def _get_template_system(self):
          if self._template_system is None:
              # Load template plugin
              template_sys_name = utils.get_template_engine(self.THEMES)
              pi = self.plugin_manager.getPluginByName(
                  template_sys_name, "TemplateSystem")
              if pi is None:
                  sys.stderr.write("Error loading {0} template system "
                                   "plugin\n".format(template_sys_name))
                  sys.exit(1)
              self._template_system = pi.plugin_object
              lookup_dirs = ['templates'] + [os.path.join(utils.get_theme_path(name), "templates")
                                             for name in self.THEMES]
              self._template_system.set_directories(lookup_dirs,
                                                    self.config['CACHE_FOLDER'])
              self._template_system.set_site(self)
          return self._template_system
  
      template_system = property(_get_template_system)
  
      def get_compiler(self, source_name):
          """Get the correct compiler for a post from `conf.COMPILERS`.
  
          To make things easier for users, the mapping in conf.py is
          compiler->[extensions], although this is less convenient for us.
          The majority of this function is reversing that dictionary and error checking.
          """
          ext = os.path.splitext(source_name)[1]
          try:
              compiler = self.inverse_compilers[ext]
          except KeyError:
              # Find the correct compiler for this files extension
              lang_exts_tab = list(self.config['COMPILERS'].items())
              langs = [lang for lang, exts in lang_exts_tab if ext in exts or
                       len([ext_ for ext_ in exts if source_name.endswith(ext_)]) > 0]
              if len(langs) != 1:
                  if len(set(langs)) > 1:
                      sys.exit("Your file extension->compiler definition is "
                               "ambiguous.\nPlease remove one of the file "
                               "extensions from 'COMPILERS' in conf.py\n(The "
                               "error is in one of {0})".format(', '.join(langs)))
                  elif len(langs) > 1:
                      langs = langs[:1]
                  else:
                      sys.exit("COMPILERS in conf.py does not tell me how to "
                               "handle '{0}' extensions.".format(ext))
  
              lang = langs[0]
              try:
                  compiler = self.compilers[lang]
              except KeyError:
                  sys.exit("Cannot find '{0}' compiler; "
                           "it might require an extra plugin -- "
                           "do you have it installed?".format(lang))
              self.inverse_compilers[ext] = compiler
  
          return compiler
  
      def render_template(self, template_name, output_name, context, url_type=None, is_fragment=False):
          """Render a template with the global context.
  
          If ``output_name`` is None, will return a string and all URL
          normalization will be ignored (including the link:// scheme).
          If ``output_name`` is a string, URLs will be normalized and
          the resultant HTML will be saved to the named file (path must
          start with OUTPUT_FOLDER).
  
          The argument ``url_type`` allows to override the ``URL_TYPE``
          configuration.
  
          If ``is_fragment`` is set to ``True``, a HTML fragment will
          be rendered and not a whole HTML document.
          """
          local_context = {}
          local_context["template_name"] = template_name
          local_context.update(self.GLOBAL_CONTEXT)
          local_context.update(context)
          for k in self._GLOBAL_CONTEXT_TRANSLATABLE:
              local_context[k] = local_context[k](local_context['lang'])
          local_context['is_rtl'] = local_context['lang'] in LEGAL_VALUES['RTL_LANGUAGES']
          local_context['url_type'] = self.config['URL_TYPE'] if url_type is None else url_type
          local_context["translations_feedorder"] = sorted(
              local_context["translations"],
              key=lambda x: (int(x != local_context['lang']), x)
          )
          # string, arguments
          local_context["formatmsg"] = lambda s, *a: s % a
          for h in local_context['template_hooks'].values():
              h.context = context
  
          for func in self.config['GLOBAL_CONTEXT_FILLER']:
              func(local_context, template_name)
  
          data = self.template_system.render_template(
              template_name, None, local_context)
  
          if output_name is None:
              return data
  
          if not output_name.startswith(self.config["OUTPUT_FOLDER"]):
              raise ValueError("Output path for templates must start with OUTPUT_FOLDER")
          url_part = output_name[len(self.config["OUTPUT_FOLDER"]) + 1:]
  
          # Treat our site as if output/ is "/" and then make all URLs relative,
          # making the site "relocatable"
          src = os.sep + url_part
          src = os.path.normpath(src)
          # The os.sep is because normpath will change "/" to "\" on windows
          src = "/".join(src.split(os.sep))
  
          utils.makedirs(os.path.dirname(output_name))
          parser = lxml.html.HTMLParser(remove_blank_text=True)
          if is_fragment:
              doc = lxml.html.fragment_fromstring(data.strip(), parser)
          else:
              doc = lxml.html.document_fromstring(data.strip(), parser)
          self.rewrite_links(doc, src, context['lang'], url_type)
          if is_fragment:
              # doc.text contains text before the first HTML, or None if there was no text
              # The text after HTML elements is added by tostring() (because its implicit
              # argument with_tail has default value True).
              data = (doc.text or '').encode('utf-8') + b''.join([lxml.html.tostring(child, encoding='utf-8', method='html') for child in doc.iterchildren()])
          else:
              data = lxml.html.tostring(doc, encoding='utf8', method='html', pretty_print=True, doctype='<!DOCTYPE html>')
          with open(output_name, "wb+") as post_file:
              post_file.write(data)
  
      def rewrite_links(self, doc, src, lang, url_type=None):
          """Replace links in document to point to the right places."""
          # First let lxml replace most of them
          doc.rewrite_links(lambda dst: self.url_replacer(src, dst, lang, url_type), resolve_base_href=False)
  
          # lxml ignores srcset in img and source elements, so do that by hand
          objs = list(doc.xpath('(//img|//source)'))
          for obj in objs:
              if 'srcset' in obj.attrib:
                  urls = [u.strip() for u in obj.attrib['srcset'].split(',')]
                  urls = [self.url_replacer(src, dst, lang, url_type) for dst in urls]
                  obj.set('srcset', ', '.join(urls))
  
      def url_replacer(self, src, dst, lang=None, url_type=None):
          """Mangle URLs.
  
          * Replaces link:// URLs with real links
          * Makes dst relative to src
          * Leaves fragments unchanged
          * Leaves full URLs unchanged
          * Avoids empty links
  
          src is the URL where this link is used
          dst is the link to be mangled
          lang is used for language-sensitive URLs in link://
          url_type is used to determine final link appearance, defaulting to URL_TYPE from config
          """
          # Avoid mangling links within the page
          if dst.startswith('#'):
              return dst
  
          parsed_src = urlsplit(src)
          src_elems = parsed_src.path.split('/')[1:]
          dst_url = urlparse(dst)
          if lang is None:
              lang = self.default_lang
          if url_type is None:
              url_type = self.config.get('URL_TYPE')
  
          if dst_url.scheme and dst_url.scheme not in ['http', 'https', 'link']:
              return dst
  
          # Refuse to replace links that are full URLs.
          if dst_url.netloc:
              if dst_url.scheme == 'link':  # Magic link
                  if dst_url.query:
                      # If query strings are used in magic link, they will be
                      # passed to the path handler as keyword arguments (strings)
                      link_kwargs = {unquote(k): unquote(v[-1]) for k, v in parse_qs(dst_url.query).items()}
                  else:
                      link_kwargs = {}
  
                  # unquote from issue #2934
                  dst = self.link(dst_url.netloc, unquote(dst_url.path.lstrip('/')), lang, **link_kwargs)
                  if dst_url.fragment:
                      dst += '#' + dst_url.fragment
              # Assuming the site is served over one of these, and
              # since those are the only URLs we want to rewrite...
              else:
                  if '%' in dst_url.netloc:
                      # convert lxml percent-encoded garbage to punycode
                      nl = unquote(dst_url.netloc)
                      try:
                          nl = nl.decode('utf-8')
                      except AttributeError:
                          # python 3: already unicode
                          pass
                      nl = nl.encode('idna')
                      if isinstance(nl, bytes):
                          nl = nl.decode('latin-1')  # so idna stays unchanged
                      dst = urlunsplit((dst_url.scheme,
                                        nl,
                                        dst_url.path,
                                        dst_url.query,
                                        dst_url.fragment))
                  return dst
          elif dst_url.scheme == 'link':  # Magic absolute path link:
              dst = dst_url.path
              return dst
  
          # Refuse to replace links that consist of a fragment only
          if ((not dst_url.scheme) and (not dst_url.netloc) and
                  (not dst_url.path) and (not dst_url.params) and
                  (not dst_url.query) and dst_url.fragment):
              return dst
  
          # Normalize
          dst = urljoin(src, dst)
  
          # Avoid empty links.
          if src == dst:
              if url_type == 'absolute':
                  dst = urljoin(self.config['BASE_URL'], dst.lstrip('/'))
                  return dst
              elif url_type == 'full_path':
                  dst = urljoin(self.config['BASE_URL'], dst.lstrip('/'))
                  return utils.full_path_from_urlparse(urlparse(dst))
              else:
                  return "#"
  
          # Check that link can be made relative, otherwise return dest
          parsed_dst = urlsplit(dst)
          if parsed_src[:2] != parsed_dst[:2]:
              if url_type == 'absolute':
                  dst = urljoin(self.config['BASE_URL'], dst)
              return dst
  
          if url_type in ('full_path', 'absolute'):
              dst = urljoin(self.config['BASE_URL'], dst.lstrip('/'))
              if url_type == 'full_path':
                  parsed = urlparse(urljoin(self.config['BASE_URL'], dst.lstrip('/')))
                  dst = utils.full_path_from_urlparse(parsed)
              return dst
  
          # Now both paths are on the same site and absolute
          dst_elems = parsed_dst.path.split('/')[1:]
  
          i = 0
          for (i, s), d in zip(enumerate(src_elems), dst_elems):
              if s != d:
                  break
          # Now i is the longest common prefix
          result = '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:])
  
          if not result and not parsed_dst.fragment:
              result = "."
  
          # Don't forget the query part of the link
          if parsed_dst.query:
              result += "?" + parsed_dst.query
  
          # Don't forget the fragment (anchor) part of the link
          if parsed_dst.fragment:
              result += "#" + parsed_dst.fragment
  
          if not result:
              raise ValueError("Failed to parse link: {0}".format((src, dst, i, src_elems, dst_elems)))
  
          return result
  
      def _make_renderfunc(self, t_data, fname=None):
          """Return a function that can be registered as a template shortcode.
  
          The returned function has access to the passed template data and
          accepts any number of positional and keyword arguments. Positional
          arguments values are added as a tuple under the key ``_args`` to the
          keyword argument dict and then the latter provides the template
          context.
  
          Global context keys are made available as part of the context,
          respecting locale.
  
          As a special quirk, the "data" key from global_context is
          available only as "global_data" because of name clobbering.
  
          """
          def render_shortcode(*args, **kw):
              context = self.GLOBAL_CONTEXT.copy()
              context.update(kw)
              context['_args'] = args
              context['lang'] = utils.LocaleBorg().current_lang
              for k in self._GLOBAL_CONTEXT_TRANSLATABLE:
                  context[k] = context[k](context['lang'])
              output = self.template_system.render_template_to_string(t_data, context)
              if fname is not None:
                  dependencies = [fname] + self.template_system.get_deps(fname)
              else:
                  dependencies = []
              return output, dependencies
          return render_shortcode
  
      def _register_templated_shortcodes(self):
          """Register shortcodes based on templates.
  
          This will register a shortcode for any template found in shortcodes/
          folders and a generic "template" shortcode which will consider the
          content in the shortcode as a template in itself.
          """
          self.register_shortcode('template', self._template_shortcode_handler)
  
          builtin_sc_dir = resource_filename(
              'nikola',
              os.path.join('data', 'shortcodes', utils.get_template_engine(self.THEMES)))
  
          for sc_dir in [builtin_sc_dir, 'shortcodes']:
              if not os.path.isdir(sc_dir):
                  continue
  
              for fname in os.listdir(sc_dir):
                  name, ext = os.path.splitext(fname)
  
                  if ext != '.tmpl':
                      continue
                  with open(os.path.join(sc_dir, fname)) as fd:
                      self.register_shortcode(name, self._make_renderfunc(
                          fd.read(), os.path.join(sc_dir, fname)))
  
      def _template_shortcode_handler(self, *args, **kw):
          t_data = kw.pop('data', '')
          context = self.GLOBAL_CONTEXT.copy()
          context.update(kw)
          context['_args'] = args
          context['lang'] = utils.LocaleBorg().current_lang
          for k in self._GLOBAL_CONTEXT_TRANSLATABLE:
              context[k] = context[k](context['lang'])
          output = self.template_system.render_template_to_string(t_data, context)
          dependencies = self.template_system.get_string_deps(t_data)
          return output, dependencies
  
      def register_shortcode(self, name, f):
          """Register function f to handle shortcode "name"."""
          if name in self.shortcode_registry:
              utils.LOGGER.warning('Shortcode name conflict: {}', name)
              return
          self.shortcode_registry[name] = f
  
      def apply_shortcodes(self, data, filename=None, lang=None, extra_context=None):
          """Apply shortcodes from the registry on data."""
          if extra_context is None:
              extra_context = {}
          if lang is None:
              lang = utils.LocaleBorg().current_lang
          return shortcodes.apply_shortcodes(data, self.shortcode_registry, self, filename, lang=lang, extra_context=extra_context)
  
      def apply_shortcodes_uuid(self, data, _shortcodes, filename=None, lang=None, extra_context=None):
          """Apply shortcodes from the registry on data."""
          if lang is None:
              lang = utils.LocaleBorg().current_lang
          if extra_context is None:
              extra_context = {}
          deps = []
          for k, v in _shortcodes.items():
              replacement, _deps = shortcodes.apply_shortcodes(v, self.shortcode_registry, self, filename, lang=lang, extra_context=extra_context)
              data = data.replace(k, replacement)
              deps.extend(_deps)
          return data, deps
  
      def _get_rss_copyright(self, lang, rss_plain):
          if rss_plain:
              return (
                  self.config['RSS_COPYRIGHT_PLAIN'](lang) or
                  lxml.html.fromstring(self.config['RSS_COPYRIGHT'](lang)).text_content().strip())
          else:
              return self.config['RSS_COPYRIGHT'](lang)
  
      def generic_rss_feed(self, lang, title, link, description, timeline,
                           rss_teasers, rss_plain, feed_length=10, feed_url=None,
                           enclosure=_enclosure, rss_links_append_query=None, copyright_=None):
          """Generate an ExtendedRSS2 feed object for later use."""
          rss_obj = utils.ExtendedRSS2(
              title=title,
              link=utils.encodelink(link),
              description=description,
              lastBuildDate=datetime.datetime.utcnow(),
              generator='Nikola (getnikola.com)',
              language=lang
          )
  
          if copyright_ is None:
              copyright_ = self._get_rss_copyright(lang, rss_plain)
          # Use the configured or specified copyright string if present.
          if copyright_:
              rss_obj.copyright = copyright_
  
          if feed_url:
              absurl = '/' + feed_url[len(self.config['BASE_URL']):]
              rss_obj.xsl_stylesheet_href = self.url_replacer(absurl, "/assets/xml/rss.xsl")
  
          items = []
  
          feed_append_query = None
          if rss_links_append_query:
              if rss_links_append_query is True:
                  raise ValueError("RSS_LINKS_APPEND_QUERY (or FEED_LINKS_APPEND_QUERY) cannot be True. Valid values are False or a formattable string.")
              feed_append_query = rss_links_append_query.format(
                  feedRelUri='/' + feed_url[len(self.config['BASE_URL']):],
                  feedFormat="rss")
  
          for post in timeline[:feed_length]:
              data = post.text(lang, teaser_only=rss_teasers, strip_html=rss_plain,
                               feed_read_more_link=True, feed_links_append_query=feed_append_query)
              if feed_url is not None and data:
                  # Massage the post's HTML (unless plain)
                  if not rss_plain:
                      if 'previewimage' in post.meta[lang] and post.meta[lang]['previewimage'] not in data:
                          data = "<figure><img src=\"{}\"></figure> {}".format(post.meta[lang]['previewimage'], data)
                      # FIXME: this is duplicated with code in Post.text()
                      try:
                          doc = lxml.html.document_fromstring(data)
                          doc.rewrite_links(lambda dst: self.url_replacer(post.permalink(), dst, lang, 'absolute'))
                          try:
                              body = doc.body
                              data = (body.text or '') + ''.join(
                                  [lxml.html.tostring(child, encoding='unicode')
                                      for child in body.iterchildren()])
                          except IndexError:  # No body there, it happens sometimes
                              data = ''
                      except lxml.etree.ParserError as e:
                          if str(e) == "Document is empty":
                              data = ""
                          else:  # let other errors raise
                              raise
              args = {
                  'title': post.title(lang) if post.should_show_title() else None,
                  'link': post.permalink(lang, absolute=True, query=feed_append_query),
                  'description': data,
                  # PyRSS2Gen's pubDate is GMT time.
                  'pubDate': (post.date if post.date.tzinfo is None else
                              post.date.astimezone(dateutil.tz.tzutc())),
                  'categories': post._tags.get(lang, []),
                  'creator': post.author(lang),
                  'guid': post.guid(lang),
              }
  
              if post.author(lang):
                  rss_obj.rss_attrs["xmlns:dc"] = "http://purl.org/dc/elements/1.1/"
  
              if enclosure:
                  # enclosure callback returns None if post has no enclosure, or a
                  # 3-tuple of (url, length (0 is valid), mimetype)
                  enclosure_details = enclosure(post=post, lang=lang)
                  if enclosure_details is not None:
                      args['enclosure'] = rss.Enclosure(*enclosure_details)
  
              items.append(utils.ExtendedItem(**args))
  
          rss_obj.items = items
          rss_obj.self_url = feed_url
          rss_obj.rss_attrs["xmlns:atom"] = "http://www.w3.org/2005/Atom"
          return rss_obj
  
      def generic_rss_renderer(self, lang, title, link, description, timeline, output_path,
                               rss_teasers, rss_plain, feed_length=10, feed_url=None,
                               enclosure=_enclosure, rss_links_append_query=None, copyright_=None):
          """Take all necessary data, and render a RSS feed in output_path."""
          rss_obj = self.generic_rss_feed(lang, title, link, description, timeline,
                                          rss_teasers, rss_plain, feed_length=feed_length, feed_url=feed_url,
                                          enclosure=enclosure, rss_links_append_query=rss_links_append_query, copyright_=copyright_)
          utils.rss_writer(rss_obj, output_path)
  
      def path(self, kind, name, lang=None, is_link=False, **kwargs):
          r"""Build the path to a certain kind of page.
  
          These are mostly defined by plugins by registering via the
          register_path_handler method, except for slug, post_path, root
          and filename which are defined in this class' init method.
  
          Here's some of the others, for historical reasons:
  
          * root (name is ignored)
          * tag_index (name is ignored)
          * tag (and name is the tag name)
          * tag_rss (name is the tag name)
          * category (and name is the category name)
          * category_rss (and name is the category name)
          * archive (and name is the year, or None for the main archive index)
          * index (name is the number in index-number)
          * rss (name is ignored)
          * gallery (name is the gallery name)
          * listing (name is the source code file name)
          * post_path (name is 1st element in a POSTS/PAGES tuple)
          * slug (name is the slug of a post or page)
          * filename (name is the source filename of a post/page, in DEFAULT_LANG, relative to conf.py)
  
          The returned value is either a path relative to output, like "categories/whatever.html", or
          an absolute URL ("https://getnikola.com/"), if path handler returns a string.
  
          If is_link is True, the path is absolute and uses "/" as separator
          (ex: "/archive/index.html").
          If is_link is False, the path is relative to output and uses the
          platform's separator.
          (ex: "archive\index.html")
          If the registered path handler returns a string instead of path component list - it's
          considered to be an absolute URL and returned as is.
  
          """
          if lang is None:
              lang = utils.LocaleBorg().current_lang
  
          try:
              path = self.path_handlers[kind](name, lang, **kwargs)
          except KeyError:
              utils.LOGGER.warning("Unknown path request of kind: {0}".format(kind))
              return ""
  
          # If path handler returns a string we consider it to be an absolute URL not requiring any
          # further processing, i.e 'https://getnikola.com/'. See Issue #2876.
          if isinstance(path, str):
              return path
  
          if path is None:
              path = "#"
          else:
              path = [os.path.normpath(p) for p in path if p != '.']  # Fix Issue #1028
          if is_link:
              link = '/' + ('/'.join(path))
              index_len = len(self.config['INDEX_FILE'])
              if self.config['STRIP_INDEXES'] and \
                      link[-(1 + index_len):] == '/' + self.config['INDEX_FILE']:
                  return link[:-index_len]
              else:
                  return link
          else:
              return os.path.join(*path)
  
      def post_path(self, name, lang):
          """Link to the destination of an element in the POSTS/PAGES settings.
  
          Example:
  
          link://post_path/posts => /blog
          """
          return [_f for _f in [self.config['TRANSLATIONS'][lang],
                                os.path.dirname(name),
                                self.config['INDEX_FILE']] if _f]
  
      def root_path(self, name, lang):
          """Link to the current language's root.
  
          Example:
  
          link://root_path => /
  
          link://root_path => /translations/spanish/
          """
          d = self.config['TRANSLATIONS'][lang]
          if d:
              return [d, '']
          else:
              return []
  
      def slug_path(self, name, lang):
          """Return a link to a post with given slug, if not ambiguous.
  
          Example:
  
          link://slug/yellow-camaro => /posts/cars/awful/yellow-camaro/index.html
          """
          results = [p for p in self.timeline if p.meta('slug') == name]
          if not results:
              utils.LOGGER.warning("Cannot resolve path request for slug: {0}".format(name))
          else:
              if len(results) > 1:
                  utils.LOGGER.warning('Ambiguous path request for slug: {0}'.format(name))
              return [_f for _f in results[0].permalink(lang).split('/')]
  
      def filename_path(self, name, lang):
          """Link to post or page by source filename.
  
          Example:
  
          link://filename/manual.txt => /docs/handbook.html
          """
          results = [p for p in self.timeline if p.source_path == name]
          if not results:
              utils.LOGGER.warning("Cannot resolve path request for filename: {0}".format(name))
          else:
              if len(results) > 1:
                  utils.LOGGER.error("Ambiguous path request for filename: {0}".format(name))
              return [_f for _f in results[0].permalink(lang).split('/') if _f]
  
      def register_path_handler(self, kind, f):
          """Register a path handler."""
          if kind in self.path_handlers:
              utils.LOGGER.warning('Conflicting path handlers for kind: {0}'.format(kind))
          else:
              self.path_handlers[kind] = f
  
      def link(self, *args, **kwargs):
          """Create a link."""
          url = self.path(*args, is_link=True, **kwargs)
          url = utils.encodelink(url)
          return url
  
      def abs_link(self, dst, protocol_relative=False):
          """Get an absolute link."""
          # Normalize
          if dst:  # Mako templates and empty strings evaluate to False
              dst = urljoin(self.config['BASE_URL'], dst.lstrip('/'))
          else:
              dst = self.config['BASE_URL']
          url = urlparse(dst).geturl()
          if protocol_relative:
              url = url.split(":", 1)[1]
          url = utils.encodelink(url)
          return url
  
      def rel_link(self, src, dst):
          """Get a relative link."""
          # Normalize
          src = urljoin(self.config['BASE_URL'], src)
          dst = urljoin(src, dst)
          # Avoid empty links.
          if src == dst:
              return "#"
          # Check that link can be made relative, otherwise return dest
          parsed_src = urlsplit(src)
          parsed_dst = urlsplit(dst)
          if parsed_src[:2] != parsed_dst[:2]:
              return utils.encodelink(dst)
          # Now both paths are on the same site and absolute
          src_elems = parsed_src.path.split('/')[1:]
          dst_elems = parsed_dst.path.split('/')[1:]
          i = 0
          for (i, s), d in zip(enumerate(src_elems), dst_elems):
              if s != d:
                  break
          else:
              i += 1
          # Now i is the longest common prefix
          url = '/'.join(['..'] * (len(src_elems) - i - 1) + dst_elems[i:])
          url = utils.encodelink(url)
          return url
  
      def register_filter(self, filter_name, filter_definition):
          """Register a filter.
  
          filter_name should be a name not confusable with an actual
          executable. filter_definition should be a callable accepting
          one argument (the filename).
          """
          if filter_name in self.filters:
              utils.LOGGER.warning('''The filter "{0}" is defined more than once.'''.format(filter_name))
          self.filters[filter_name] = filter_definition
  
      def file_exists(self, path, not_empty=False):
          """Check if the file exists. If not_empty is True, it also must not be empty."""
          exists = os.path.exists(path)
          if exists and not_empty:
              exists = os.stat(path).st_size > 0
          return exists
  
      def clean_task_paths(self, task):
          """Normalize target paths in the task."""
          targets = task.get('targets', None)
          if targets is not None:
              task['targets'] = [os.path.normpath(t) for t in targets]
          return task
  
      def gen_tasks(self, name, plugin_category, doc=''):
          """Generate tasks."""
          def flatten(task):
              """Flatten lists of tasks."""
              if isinstance(task, dict):
                  yield task
              else:
                  for t in task:
                      for ft in flatten(t):
                          yield ft
  
          task_dep = []
          for pluginInfo in self.plugin_manager.getPluginsOfCategory(plugin_category):
              for task in flatten(pluginInfo.plugin_object.gen_tasks()):
                  if 'basename' not in task:
                      raise ValueError("Task {0} does not have a basename".format(task))
                  task = self.clean_task_paths(task)
                  if 'task_dep' not in task:
                      task['task_dep'] = []
                  task['task_dep'].extend(self.injected_deps[task['basename']])
                  yield task
                  for multi in self.plugin_manager.getPluginsOfCategory("TaskMultiplier"):
                      flag = False
                      for task in multi.plugin_object.process(task, name):
                          flag = True
                          yield self.clean_task_paths(task)
                      if flag:
                          task_dep.append('{0}_{1}'.format(name, multi.plugin_object.name))
              if pluginInfo.plugin_object.is_default:
                  task_dep.append(pluginInfo.plugin_object.name)
          yield {
              'basename': name,
              'doc': doc,
              'actions': None,
              'clean': True,
              'task_dep': task_dep
          }
  
      def parse_category_name(self, category_name):
          """Parse a category name into a hierarchy."""
          if self.config['CATEGORY_ALLOW_HIERARCHIES']:
              try:
                  return hierarchy_utils.parse_escaped_hierarchical_category_name(category_name)
              except Exception as e:
                  utils.LOGGER.error(str(e))
                  sys.exit(1)
          else:
              return [category_name] if len(category_name) > 0 else []
  
      def category_path_to_category_name(self, category_path):
          """Translate a category path to a category name."""
          if self.config['CATEGORY_ALLOW_HIERARCHIES']:
              return hierarchy_utils.join_hierarchical_category_path(category_path)
          else:
              return ''.join(category_path)
  
      def _add_post_to_category(self, post, category_name):
          """Add a post to a category."""
          category_path = self.parse_category_name(category_name)
          current_path = []
          current_subtree = self.category_hierarchy
          for current in category_path:
              current_path.append(current)
              if current not in current_subtree:
                  current_subtree[current] = {}
              current_subtree = current_subtree[current]
              self.posts_per_category[self.category_path_to_category_name(current_path)].append(post)
  
      def _sort_category_hierarchy(self):
          """Sort category hierarchy."""
          # First create a hierarchy of TreeNodes
          self.category_hierarchy_lookup = {}
  
          def create_hierarchy(cat_hierarchy, parent=None):
              """Create category hierarchy."""
              result = []
              for name, children in cat_hierarchy.items():
                  node = hierarchy_utils.TreeNode(name, parent)
                  node.children = create_hierarchy(children, node)
                  node.category_path = [pn.name for pn in node.get_path()]
                  node.category_name = self.category_path_to_category_name(node.category_path)
                  self.category_hierarchy_lookup[node.category_name] = node
                  if node.category_name not in self.config.get('HIDDEN_CATEGORIES'):
                      result.append(node)
              return natsort.natsorted(result, key=lambda e: e.name, alg=natsort.ns.F | natsort.ns.IC)
  
          root_list = create_hierarchy(self.category_hierarchy)
          # Next, flatten the hierarchy
          self.category_hierarchy = hierarchy_utils.flatten_tree_structure(root_list)
  
      @staticmethod
      def sort_posts_chronologically(posts, lang=None):
          """Sort a list of posts chronologically.
  
          This function also takes priority, title and source path into account.
          """
          # Last tie breaker: sort by source path (A-Z)
          posts = sorted(posts, key=lambda p: p.source_path)
          # Next tie breaker: sort by title if language is given (A-Z)
          if lang is not None:
              posts = natsort.natsorted(posts, key=lambda p: p.title(lang), alg=natsort.ns.F | natsort.ns.IC)
          # Next tie breaker: sort by date (reverse chronological order)
          posts = sorted(posts, key=lambda p: p.date, reverse=True)
          # Finally, sort by priority meta value (descending)
          posts = sorted(posts, key=lambda p: int(p.meta('priority')) if p.meta('priority') else 0, reverse=True)
          # Return result
          return posts
  
      def scan_posts(self, really=False, ignore_quit=False, quiet=False):
          """Scan all the posts.
  
          The `quiet` option is ignored.
          """
          if self._scanned and not really:
              return
  
          # Reset things
          self.posts = []
          self.all_posts = []
          self.posts_per_year = defaultdict(list)
          self.posts_per_month = defaultdict(list)
          self.posts_per_tag = defaultdict(list)
          self.posts_per_category = defaultdict(list)
          self.tags_per_language = defaultdict(list)
          self.category_hierarchy = {}
          self.post_per_file = {}
          self.post_per_input_file = {}
          self.timeline = []
          self.pages = []
  
          for p in sorted(self.plugin_manager.getPluginsOfCategory('PostScanner'), key=operator.attrgetter('name')):
              try:
                  timeline = p.plugin_object.scan()
              except Exception:
                  utils.LOGGER.error('Error reading timeline')
                  raise
              # FIXME: can there be conflicts here?
              self.timeline.extend(timeline)
  
          quit = False
          # Classify posts per year/tag/month/whatever
          slugged_tags = defaultdict(set)
          for post in self.timeline:
              if post.use_in_feeds:
                  self.posts.append(post)
                  self.posts_per_year[str(post.date.year)].append(post)
                  self.posts_per_month[
                      '{0}/{1:02d}'.format(post.date.year, post.date.month)].append(post)
                  for lang in self.config['TRANSLATIONS'].keys():
                      for tag in post.tags_for_language(lang):
                          _tag_slugified = utils.slugify(tag, lang)
                          slugged_tags[lang].add(_tag_slugified)
                          if post not in self.posts_per_tag[tag]:
                              self.posts_per_tag[tag].append(post)
                      self.tags_per_language[lang].extend(post.tags_for_language(lang))
                  self._add_post_to_category(post, post.meta('category'))
  
              if post.is_post:
                  # unpublished posts
                  self.all_posts.append(post)
              else:
                  self.pages.append(post)
  
              for lang in self.config['TRANSLATIONS'].keys():
                  dest = post.destination_path(lang=lang)
                  src_dest = post.destination_path(lang=lang, extension=post.source_ext())
                  src_file = post.translated_source_path(lang=lang)
                  if dest in self.post_per_file:
                      utils.LOGGER.error('Two posts are trying to generate {0}: {1} and {2}'.format(
                          dest,
                          self.post_per_file[dest].source_path,
                          post.source_path))
                      quit = True
                  if (src_dest in self.post_per_file) and self.config['COPY_SOURCES']:
                      utils.LOGGER.error('Two posts are trying to generate {0}: {1} and {2}'.format(
                          src_dest,
                          self.post_per_file[dest].source_path,
                          post.source_path))
                      quit = True
                  self.post_per_file[dest] = post
                  self.post_per_file[src_dest] = post
                  if src_file is not None:
                      self.post_per_input_file[src_file] = post
                  # deduplicate tags_per_language
                  self.tags_per_language[lang] = list(set(self.tags_per_language[lang]))
  
          # Sort everything.
  
          self.timeline = self.sort_posts_chronologically(self.timeline)
          self.posts = self.sort_posts_chronologically(self.posts)
          self.all_posts = self.sort_posts_chronologically(self.all_posts)
          self.pages = self.sort_posts_chronologically(self.pages)
          self._sort_category_hierarchy()
  
          for i, p in enumerate(self.posts[1:]):
              p.next_post = self.posts[i]
          for i, p in enumerate(self.posts[:-1]):
              p.prev_post = self.posts[i + 1]
          self._scanned = True
          if not self.quiet:
              print("done!", file=sys.stderr)
          if quit and not ignore_quit:
              sys.exit(1)
          signal('scanned').send(self)
  
      def generic_renderer(self, lang, output_name, template_name, filters, file_deps=None, uptodate_deps=None, context=None, context_deps_remove=None, post_deps_dict=None, url_type=None, is_fragment=False):
          """Create tasks for rendering pages and post lists and other related pages.
  
          lang is the current language.
          output_name is the destination file name.
          template_name is the template to be used.
          filters is the list of filters (usually site.config['FILTERS']) which will be used to post-process the result.
          file_deps (optional) is a list of additional file dependencies (next to template and its dependencies).
          uptodate_deps (optional) is a list of additional entries added to the task's uptodate list.
          context (optional) a dict used as a basis for the template context. The lang parameter will always be added.
          context_deps_remove (optional) is a list of keys to remove from the context after using it as an uptodate dependency. This should name all keys containing non-trivial Python objects; they can be replaced by adding JSON-style dicts in post_deps_dict.
          post_deps_dict (optional) is a dict merged into the copy of context which is used as an uptodate dependency.
          url_type (optional) allows to override the ``URL_TYPE`` configuration.
          is_fragment (optional) allows to write a HTML fragment instead of a HTML document.
          """
          utils.LocaleBorg().set_locale(lang)
  
          file_deps = copy(file_deps) if file_deps else []
          file_deps += self.template_system.template_deps(template_name)
          file_deps = sorted(list(filter(None, file_deps)))
  
          context = copy(context) if context else {}
          context["lang"] = lang
  
          deps_dict = copy(context)
          if context_deps_remove:
              for key in context_deps_remove:
                  deps_dict.pop(key)
          deps_dict['OUTPUT_FOLDER'] = self.config['OUTPUT_FOLDER']
          deps_dict['TRANSLATIONS'] = self.config['TRANSLATIONS']
          deps_dict['global'] = self.GLOBAL_CONTEXT
          deps_dict['all_page_deps'] = self.ALL_PAGE_DEPS
          if post_deps_dict:
              deps_dict.update(post_deps_dict)
  
          for k, v in self.GLOBAL_CONTEXT['template_hooks'].items():
              deps_dict['||template_hooks|{0}||'.format(k)] = v.calculate_deps()
  
          for k in self._GLOBAL_CONTEXT_TRANSLATABLE:
              deps_dict[k] = deps_dict['global'][k](lang)
          for k in self._ALL_PAGE_DEPS_TRANSLATABLE:
              deps_dict[k] = deps_dict['all_page_deps'][k](lang)
  
          deps_dict['navigation_links'] = deps_dict['global']['navigation_links'](lang)
          deps_dict['navigation_alt_links'] = deps_dict['global']['navigation_alt_links'](lang)
  
          task = {
              'name': os.path.normpath(output_name),
              'targets': [output_name],
              'file_dep': file_deps,
              'actions': [(self.render_template, [template_name, output_name,
                                                  context, url_type, is_fragment])],
              'clean': True,
              'uptodate': [config_changed(deps_dict, 'nikola.nikola.Nikola.generic_renderer')] + ([] if uptodate_deps is None else uptodate_deps)
          }
  
          return utils.apply_filters(task, filters)
  
      def generic_page_renderer(self, lang, post, filters, context=None):
          """Render post fragments to final HTML pages."""
          extension = post.compiler.extension()
          output_name = os.path.join(self.config['OUTPUT_FOLDER'],
                                     post.destination_path(lang, extension))
  
          deps = post.deps(lang)
          uptodate_deps = post.deps_uptodate(lang)
          deps.extend(utils.get_asset_path(x, self.THEMES) for x in ('bundles', 'parent', 'engine'))
          _theme_ini = utils.get_asset_path(self.config['THEME'] + '.theme', self.THEMES)
          if _theme_ini:
              deps.append(_theme_ini)
  
          context = copy(context) if context else {}
          context['post'] = post
          context['title'] = post.title(lang)
          context['description'] = post.description(lang)
          context['permalink'] = post.permalink(lang)
          if 'crumbs' not in context:
              crumb_path = post.permalink(lang).lstrip('/')
              if crumb_path.endswith(self.config['INDEX_FILE']):
                  crumb_path = crumb_path[:-len(self.config['INDEX_FILE'])]
              if crumb_path.endswith('/'):
                  context['crumbs'] = utils.get_crumbs(crumb_path.rstrip('/'), is_file=False)
              else:
                  context['crumbs'] = utils.get_crumbs(crumb_path, is_file=True)
          if 'pagekind' not in context:
              context['pagekind'] = ['generic_page']
          if post.use_in_feeds:
              context['enable_comments'] = True
          else:
              context['enable_comments'] = self.config['COMMENTS_IN_PAGES']
  
          deps_dict = {}
          if post.prev_post:
              deps_dict['PREV_LINK'] = [post.prev_post.permalink(lang)]
          if post.next_post:
              deps_dict['NEXT_LINK'] = [post.next_post.permalink(lang)]
          deps_dict['comments'] = context['enable_comments']
          if post:
              deps_dict['post_translations'] = post.translated_to
  
          signal('render_post').send({
              'site': self,
              'post': post,
              'lang': lang,
              'context': context,
              'deps_dict': deps_dict,
          })
  
          yield self.generic_renderer(lang, output_name, post.template_name, filters,
                                      file_deps=deps,
                                      uptodate_deps=uptodate_deps,
                                      context=context,
                                      context_deps_remove=['post'],
                                      post_deps_dict=deps_dict,
                                      url_type=post.url_type)
  
      def generic_post_list_renderer(self, lang, posts, output_name, template_name, filters, extra_context):
          """Render pages with lists of posts."""
          deps = []
          uptodate_deps = []
          for post in posts:
              deps += post.deps(lang)
              uptodate_deps += post.deps_uptodate(lang)
  
          context = {}
          context["posts"] = posts
          context["title"] = self.config['BLOG_TITLE'](lang)
          context["description"] = self.config['BLOG_DESCRIPTION'](lang)
          context["prevlink"] = None
          context["nextlink"] = None
          if extra_context:
              context.update(extra_context)
          if 'has_other_languages' not in context:
              context['has_other_languages'] = False
  
          post_deps_dict = {}
          post_deps_dict["posts"] = [(p.meta[lang]['title'], p.permalink(lang)) for p in posts]
  
          return self.generic_renderer(lang, output_name, template_name, filters,
                                       file_deps=deps,
                                       uptodate_deps=uptodate_deps,
                                       context=context,
                                       post_deps_dict=post_deps_dict)
  
      def atom_feed_renderer(self, lang, posts, output_path, filters,
                             extra_context):
          """Render Atom feeds and archives with lists of posts.
  
          Feeds are considered archives when no future updates to them are expected.
          """
          def atom_link(link_rel, link_type, link_href):
              link = lxml.etree.Element("link")
              link.set("rel", link_rel)
              link.set("type", link_type)
              link.set("href", utils.encodelink(link_href))
              return link
  
          utils.LocaleBorg().set_locale(lang)
          deps = []
          uptodate_deps = []
          for post in posts:
              deps += post.deps(lang)
              uptodate_deps += post.deps_uptodate(lang)
  
          context = {}
          blog_title = self.config['BLOG_TITLE'](lang)
          context["posts"] = posts
          context["title"] = blog_title
          context["description"] = self.config['BLOG_DESCRIPTION'](lang)
          context["lang"] = lang
          context.update(extra_context)
  
          context["title"] = "{0} ({1})".format(blog_title, context["title"]) if blog_title != context["title"] else blog_title
  
          deps_context = copy(context)
          deps_context["posts"] = [(p.meta[lang]['title'], p.permalink(lang)) for p in
                                   posts]
          deps_context["global"] = self.GLOBAL_CONTEXT
          deps_context["all_page_deps"] = self.ALL_PAGE_DEPS
  
          for k in self._GLOBAL_CONTEXT_TRANSLATABLE:
              deps_context[k] = deps_context['global'][k](lang)
          for k in self._ALL_PAGE_DEPS_TRANSLATABLE:
              deps_context[k] = deps_context['all_page_deps'][k](lang)
  
          feed_xsl_link = self.abs_link("/assets/xml/atom.xsl")
          feed_root = lxml.etree.Element("feed")
          feed_root.addprevious(lxml.etree.ProcessingInstruction(
              "xml-stylesheet",
              'href="' + utils.encodelink(feed_xsl_link) + '" type="text/xsl media="all"'))
          feed_root.set("{http://www.w3.org/XML/1998/namespace}lang", lang)
          feed_root.set("xmlns", "http://www.w3.org/2005/Atom")
          feed_title = lxml.etree.SubElement(feed_root, "title")
          feed_title.text = context["title"]
          feed_id = lxml.etree.SubElement(feed_root, "id")
          feed_id.text = self.abs_link(context["feedlink"])
          feed_updated = lxml.etree.SubElement(feed_root, "updated")
          feed_updated.text = utils.LocaleBorg().formatted_date('webiso', datetime.datetime.now(tz=dateutil.tz.tzutc()))
          feed_author = lxml.etree.SubElement(feed_root, "author")
          feed_author_name = lxml.etree.SubElement(feed_author, "name")
          feed_author_name.text = self.config["BLOG_AUTHOR"](lang)
          feed_root.append(atom_link("self", "application/atom+xml",
                                     self.abs_link(context["feedlink"])))
          feed_root.append(atom_link("alternate", "text/html",
                                     self.abs_link(context["permalink"])))
          feed_generator = lxml.etree.SubElement(feed_root, "generator")
          feed_generator.set("uri", "https://getnikola.com/")
          feed_generator.text = "Nikola"
  
          feed_append_query = None
          if self.config["FEED_LINKS_APPEND_QUERY"]:
              feed_append_query = self.config["FEED_LINKS_APPEND_QUERY"].format(
                  feedRelUri=context["feedlink"],
                  feedFormat="atom")
  
          def atom_post_text(post, text):
              if not self.config["FEED_PLAIN"]:
                  if 'previewimage' in post.meta[lang] and post.meta[lang]['previewimage'] not in text:
                      text = "<figure><img src=\"{}\"></figure> {}".format(post.meta[lang]['previewimage'], text)
  
                  # FIXME: this is duplicated with code in Post.text() and generic_rss_renderer
                  try:
                      doc = lxml.html.document_fromstring(text)
                      doc.rewrite_links(lambda dst: self.url_replacer(post.permalink(lang), dst, lang, 'absolute'))
                      try:
                          body = doc.body
                          text = (body.text or '') + ''.join(
                              [lxml.html.tostring(child, encoding='unicode')
                                  for child in body.iterchildren()])
                      except IndexError:  # No body there, it happens sometimes
                          text = ''
                  except lxml.etree.ParserError as e:
                      if str(e) == "Document is empty":
                          text = ""
                      else:  # let other errors raise
                          raise
              return text.strip()
  
          for post in posts:
              summary = atom_post_text(post, post.text(lang, teaser_only=True,
                                                       strip_html=self.config["FEED_PLAIN"],
                                                       feed_read_more_link=True,
                                                       feed_links_append_query=feed_append_query))
              content = None
              if not self.config["FEED_TEASERS"]:
                  content = atom_post_text(post, post.text(lang, teaser_only=self.config["FEED_TEASERS"],
                                                           strip_html=self.config["FEED_PLAIN"],
                                                           feed_read_more_link=True,
                                                           feed_links_append_query=feed_append_query))
  
              entry_root = lxml.etree.SubElement(feed_root, "entry")
              entry_title = lxml.etree.SubElement(entry_root, "title")
              entry_title.text = post.title(lang)
              entry_id = lxml.etree.SubElement(entry_root, "id")
              entry_id.text = post.permalink(lang, absolute=True)
              entry_updated = lxml.etree.SubElement(entry_root, "updated")
              entry_updated.text = post.formatted_updated('webiso')
              entry_published = lxml.etree.SubElement(entry_root, "published")
              entry_published.text = post.formatted_date('webiso')
              entry_author = lxml.etree.SubElement(entry_root, "author")
              entry_author_name = lxml.etree.SubElement(entry_author, "name")
              entry_author_name.text = post.author(lang)
              entry_root.append(atom_link("alternate", "text/html",
                                          post.permalink(lang, absolute=True,
                                                         query=feed_append_query)))
              entry_summary = lxml.etree.SubElement(entry_root, "summary")
              if not self.config["FEED_PLAIN"]:
                  entry_summary.set("type", "html")
              else:
                  entry_summary.set("type", "text")
              entry_summary.text = summary
              if content:
                  entry_content = lxml.etree.SubElement(entry_root, "content")
                  if not self.config["FEED_PLAIN"]:
                      entry_content.set("type", "html")
                  else:
                      entry_content.set("type", "text")
                  entry_content.text = content
              for category in post.tags_for_language(lang):
                  entry_category = lxml.etree.SubElement(entry_root, "category")
                  entry_category.set("term", utils.slugify(category, lang))
                  entry_category.set("label", category)
  
          dst_dir = os.path.dirname(output_path)
          utils.makedirs(dst_dir)
          with io.open(output_path, "w+", encoding="utf-8") as atom_file:
              data = lxml.etree.tostring(feed_root.getroottree(), encoding="UTF-8", pretty_print=True, xml_declaration=True)
              if isinstance(data, bytes):
                  data = data.decode('utf-8')
              atom_file.write(data)
  
      def generic_index_renderer(self, lang, posts, indexes_title, template_name, context_source, kw, basename, page_link, page_path, additional_dependencies=None):
          """Create an index page.
  
          lang: The language
          posts: A list of posts
          indexes_title: Title
          template_name: Name of template file
          context_source: This will be copied and extended and used as every
                          page's context
          kw: An extended version will be used for uptodate dependencies
          basename: Basename for task
          page_link: A function accepting an index i, the displayed page number,
                     the number of pages, and a boolean force_addition
                     which creates a link to the i-th page (where i ranges
                     between 0 and num_pages-1). The displayed page (between 1
                     and num_pages) is the number (optionally) displayed as
                     'page %d' on the rendered page. If force_addition is True,
                     the appendum (inserting '-%d' etc.) should be done also for
                     i == 0.
          page_path: A function accepting an index i, the displayed page number,
                     the number of pages, and a boolean force_addition,
                     which creates a path to the i-th page. All arguments are
                     as the ones for page_link.
          additional_dependencies: a list of dependencies which will be added
                                   to task['uptodate']
  
          Note: if context['featured'] is present, it must be a list of posts,
          whose dependencies will be taken added to task['uptodate'].
          """
          # Update kw
          kw = kw.copy()
          kw["tag_pages_are_indexes"] = self.config['TAG_PAGES_ARE_INDEXES']
          kw["index_display_post_count"] = self.config['INDEX_DISPLAY_POST_COUNT']
          kw["index_teasers"] = self.config['INDEX_TEASERS']
          kw["indexes_pages"] = self.config['INDEXES_PAGES'](lang)
          kw["indexes_pages_main"] = self.config['INDEXES_PAGES_MAIN']
          kw["indexes_static"] = self.config['INDEXES_STATIC']
          kw['indexes_pretty_page_url'] = self.config["INDEXES_PRETTY_PAGE_URL"]
          kw['show_index_page_navigation'] = self.config['SHOW_INDEX_PAGE_NAVIGATION']
  
          if additional_dependencies is None:
              additional_dependencies = []
  
          # Split in smaller lists
          lists = []
          if kw["indexes_static"]:
              lists.append(posts[:kw["index_display_post_count"]])
              posts = posts[kw["index_display_post_count"]:]
              while posts:
                  lists.append(posts[-kw["index_display_post_count"]:])
                  posts = posts[:-kw["index_display_post_count"]]
          else:
              while posts:
                  lists.append(posts[:kw["index_display_post_count"]])
                  posts = posts[kw["index_display_post_count"]:]
              if not lists:
                  lists.append([])
          num_pages = len(lists)
          displayed_page_numbers = [utils.get_displayed_page_number(i, num_pages, self) for i in range(num_pages)]
          page_links = [page_link(i, page_number, num_pages, False) for i, page_number in enumerate(displayed_page_numbers)]
          if kw['show_index_page_navigation']:
              # Since the list displayed_page_numbers is not necessarily
              # sorted -- in case INDEXES_STATIC is True, it is of the
              # form [num_pages, 1, 2, ..., num_pages - 1] -- we order it
              # via a map. This allows to not replicate the logic of
              # utils.get_displayed_page_number() here.
              if not kw["indexes_pages_main"] and not kw["indexes_static"]:
                  temp_map = {page_number: link for page_number, link in zip(displayed_page_numbers, page_links)}
              else:
                  temp_map = {page_number - 1: link for page_number, link in zip(displayed_page_numbers, page_links)}
              page_links_context = [temp_map[i] for i in range(num_pages)]
          for i, post_list in enumerate(lists):
              context = context_source.copy()
              if 'pagekind' not in context:
                  context['pagekind'] = ['index']
              if 'has_other_languages' not in context:
                  context['has_other_languages'] = False
              ipages_i = displayed_page_numbers[i]
              if kw["indexes_pages"]:
                  indexes_pages = kw["indexes_pages"] % ipages_i
              else:
                  if kw["indexes_pages_main"]:
                      ipages_msg = "page %d"
                  else:
                      ipages_msg = "old posts, page %d"
                  indexes_pages = " (" + \
                      kw["messages"][lang][ipages_msg] % ipages_i + ")"
              if i > 0 or kw["indexes_pages_main"]:
                  context["title"] = indexes_title + indexes_pages
              else:
                  context["title"] = indexes_title
              context["prevlink"] = None
              context["nextlink"] = None
              context['index_teasers'] = kw['index_teasers']
              prevlink = None
              nextlink = None
              if kw["indexes_static"]:
                  if i > 0:
                      if i < num_pages - 1:
                          prevlink = i + 1
                      elif i == num_pages - 1:
                          prevlink = 0
                  if num_pages > 1:
                      if i > 1:
                          nextlink = i - 1
                      elif i == 0:
                          nextlink = num_pages - 1
              else:
                  if i >= 1:
                      prevlink = i - 1
                  if i < num_pages - 1:
                      nextlink = i + 1
              if prevlink is not None:
                  context["prevlink"] = page_links[prevlink]
                  context["prevfeedlink"] = page_link(prevlink, displayed_page_numbers[prevlink],
                                                      num_pages, False, extension=".atom")
              if nextlink is not None:
                  context["nextlink"] = page_links[nextlink]
                  context["nextfeedlink"] = page_link(nextlink, displayed_page_numbers[nextlink],
                                                      num_pages, False, extension=".atom")
              context['show_index_page_navigation'] = kw['show_index_page_navigation']
              if kw['show_index_page_navigation']:
                  context['page_links'] = page_links_context
                  if not kw["indexes_pages_main"] and not kw["indexes_static"]:
                      context['current_page'] = ipages_i
                  else:
                      context['current_page'] = ipages_i - 1
                  context['prev_next_links_reversed'] = kw['indexes_static']
              context["permalink"] = page_links[i]
              context["is_frontmost_index"] = i == 0
  
              # Add dependencies to featured posts
              if 'featured' in context:
                  for post in context['featured']:
                      additional_dependencies += post.deps_uptodate(lang)
  
              output_name = os.path.join(kw['output_folder'], page_path(i, ipages_i, num_pages, False))
              task = self.generic_post_list_renderer(
                  lang,
                  post_list,
                  output_name,
                  template_name,
                  kw['filters'],
                  context,
              )
              task['uptodate'] = task['uptodate'] + [utils.config_changed(kw, 'nikola.nikola.Nikola.generic_index_renderer')] + additional_dependencies
              task['basename'] = basename
              yield task
  
          if kw["indexes_pages_main"] and kw['indexes_pretty_page_url'](lang):
              # create redirection
              output_name = os.path.join(kw['output_folder'], page_path(0, displayed_page_numbers[0], num_pages, True))
              link = page_links[0]
              yield utils.apply_filters({
                  'basename': basename,
                  'name': output_name,
                  'targets': [output_name],
                  'actions': [(utils.create_redirect, (output_name, link))],
                  'clean': True,
                  'uptodate': [utils.config_changed(kw, 'nikola.nikola.Nikola.generic_index_renderer')],
              }, kw["filters"])
  
      def generic_atom_renderer(self, lang, posts, context_source, kw, basename, classification, kind, additional_dependencies=None):
          """Create an Atom feed.
  
          lang: The language
          posts: A list of posts
          context_source: This will be copied and extended and used as every
                          page's context
          kw: An extended version will be used for uptodate dependencies
          basename: Basename for task
          classification: name of current classification (used to generate links)
          kind: classification kind (used to generate links)
          additional_dependencies: a list of dependencies which will be added
                                   to task['uptodate']
          """
          # Update kw
          kw = kw.copy()
          kw["feed_length"] = self.config['FEED_LENGTH']
          kw['generate_atom'] = self.config["GENERATE_ATOM"]
          kw['feed_links_append_query'] = self.config["FEED_LINKS_APPEND_QUERY"]
          kw['feed_teasers'] = self.config['FEED_TEASERS']
          kw['feed_plain'] = self.config['FEED_PLAIN']
  
          if additional_dependencies is None:
              additional_dependencies = []
  
          post_list = posts[:kw["feed_length"]]
          feedlink = self.link(kind + "_atom", classification, lang)
          feedpath = self.path(kind + "_atom", classification, lang)
  
          context = context_source.copy()
          if 'has_other_languages' not in context:
              context['has_other_languages'] = False
  
          output_name = os.path.join(kw['output_folder'], feedpath)
          context["feedlink"] = feedlink
          task = {
              "basename": basename,
              "name": output_name,
              "file_dep": sorted([_.base_path for _ in post_list]),
              "task_dep": ['render_posts'],
              "targets": [output_name],
              "actions": [(self.atom_feed_renderer,
                           (lang,
                            post_list,
                            output_name,
                            kw['filters'],
                            context,))],
              "clean": True,
              "uptodate": [utils.config_changed(kw, 'nikola.nikola.Nikola.atom_feed_renderer')] + additional_dependencies
          }
          yield utils.apply_filters(task, kw['filters'])
  
      def __repr__(self):
          """Representation of a Nikola site."""
          return '<Nikola Site: {0!r}>'.format(self.config['BLOG_TITLE'](self.config['DEFAULT_LANG']))
  