context: using System;
  using System.Collections.Generic;
  using System.Globalization;
  using System.Linq;
  using System.Threading;
  using System.Data.Entity;
  using System.Web.Mvc;
  using System.Xml.Linq;
  using SmartStore.Core;
  using SmartStore.Core.Data;
  using SmartStore.Core.Domain.Catalog;
  using SmartStore.Core.Domain.Customers;
  using SmartStore.Core.Domain.Security;
  using SmartStore.Core.Domain.Seo;
  using SmartStore.Core.Domain.Topics;
  using SmartStore.Core.Logging;
  using SmartStore.Services.Catalog;
  using SmartStore.Services.Customers;
  using SmartStore.Services.Localization;
  using SmartStore.Services.Search;
  using SmartStore.Services.Topics;
  using SmartStore.Core.IO;
  using SmartStore.Utilities;
  using System.IO;
  using SmartStore.Collections;
  using SmartStore.Core.Domain.Stores;
  using SmartStore.Core.Domain.Localization;
  using System.Threading.Tasks;
  
  namespace SmartStore.Services.Seo
  {
  	public partial class XmlSitemapGenerator : IXmlSitemapGenerator
      {
  		private const string SiteMapsNamespace = "http://www.sitemaps.org/schemas/sitemap/0.9";
  		private const string XhtmlNamespace = "http://www.w3.org/1999/xhtml";
  		private const string SiteMapFileNamePattern = "sitemap-{0}.xml";
  		private const string LockFileNamePattern = "sitemap-{0}-{1}.lock";
  
  		/// <summary>
  		/// The maximum number of sitemaps a sitemap index file can contain.
  		/// </summary>
  		private const int MaximumSiteMapCount = 50000;
  
  		/// <summary>
  		/// The maximum number of sitemap nodes allowed in a sitemap file. The absolute maximum allowed is 50,000 
  		/// according to the specification. See http://www.sitemaps.org/protocol.html but the file size must also be 
  		/// less than 10MB. After some experimentation, a maximum of 2.000 nodes keeps the file size below 10MB.
  		/// </summary>
  		private const int MaximumSiteMapNodeCount = 2000;
  
  		/// <summary>
  		/// The maximum size of a sitemap file in bytes (10MB).
  		/// </summary>
  		private const int MaximumSiteMapSizeInBytes = 10485760;
  
  		private readonly ICategoryService _categoryService;
          private readonly IProductService _productService;
          private readonly IManufacturerService _manufacturerService;
          private readonly ITopicService _topicService;
  		private readonly ILanguageService _languageService;
  		private readonly ICustomerService _customerService;
  		private readonly ICatalogSearchService _catalogSearchService;
  		private readonly IUrlRecordService _urlRecordService;
  		private readonly ICommonServices _services;
  		private readonly ILockFileManager _lockFileManager;
  		private readonly UrlHelper _urlHelper;
  
  		private readonly IVirtualFolder _tenantFolder;
  		private readonly string _baseDir;
  
  		public XmlSitemapGenerator(
  			ICategoryService categoryService,
              IProductService productService, 
  			IManufacturerService manufacturerService,
              ITopicService topicService,
  			ILanguageService languageService,
  			ICustomerService customerService,
  			ICatalogSearchService catalogSearchService,
  			IUrlRecordService urlRecordService,
  			ICommonServices services,
  			ILockFileManager lockFileManager,
  			UrlHelper urlHelper)
          {
              _categoryService = categoryService;
              _productService = productService;
              _manufacturerService = manufacturerService;
              _topicService = topicService;
  			_languageService = languageService;
  			_customerService = customerService;
  			_catalogSearchService = catalogSearchService;
  			_urlRecordService = urlRecordService;
  			_services = services;
  			_lockFileManager = lockFileManager;
  			_urlHelper = urlHelper;
  
  			_tenantFolder = _services.ApplicationEnvironment.TenantFolder;
  			_baseDir = _tenantFolder.Combine("Sitemaps");
  
  			Logger = NullLogger.Instance;
  			
  		}
  
  		public ILogger Logger { get; set; }
  
  		public virtual async Task<XmlSitemapPartition> GetSitemapPartAsync(int index = 0)
  		{
  			return await GetSitemapPartAsync(index, false);
  		}
  
  		private async Task<XmlSitemapPartition> GetSitemapPartAsync(int index, bool isRetry)
  		{
  			Guard.NotNegative(index, nameof(index));
  
  			var store = _services.StoreContext.CurrentStore;
  			var language = _services.WorkContext.WorkingLanguage;
  
  			var exists = SitemapFileExists(store.Id, language.Id, index, out var path, out var name);
  
  			if (exists)
  			{
  				return new XmlSitemapPartition
  				{
  					Index = index,
  					Name = name,
  					LanguageId = language.Id,
  					StoreId = store.Id,
  					ModifiedOnUtc = _tenantFolder.GetFileLastWriteTimeUtc(path),
  					Stream = _tenantFolder.OpenFile(path)
  				};
  			}
  
  			if (isRetry)
  			{
  				var msg = "Could not generate XML sitemap. Index: {0}, Date: {1}".FormatInvariant(index, DateTime.UtcNow);
  				Logger.Error(msg);
  				throw new SmartException(msg);
  			}
  
  			if (index > 0)
  			{
  				// File with index greater 0 has been requested, but it does not exist.
  				// Now we have to determine whether just the passed index is out of range
  				// or the files has never been created before.
  				// If the main file (index 0) exists, the action should return NotFoundResult,
  				// otherwise the rebuild process should be started or waited for.
  
  				if (SitemapFileExists(store.Id, language.Id, 0, out path, out name))
  				{
  					throw new IndexOutOfRangeException("The sitemap file '{0}' does not exist.".FormatInvariant(name));
  				}
  			}
  
  			// The main sitemap document with index 0 does not exist, meaning: the whole sitemap
  			// needs to be created and cached by partitions.
  
  			var wasRebuilding = false;
  			var lockFilePath = GetLockFilePath(store.Id, language.Id);
  
  			while (IsRebuilding(lockFilePath))
  			{
  				// The rebuild process is already running, either started
  				// by the task scheduler or another HTTP request.
  				// We should wait for completion.
  
  				wasRebuilding = true;
  				Thread.Sleep(1000);
  			}
  
  			if (!wasRebuilding)
  			{
  				// No lock. Rebuild now.
  				var buildContext = new XmlSitemapBuildContext(store, new[] { language })
  				{
  					CancellationToken = CancellationToken.None
  				};
  
  				await RebuildAsync(buildContext);
  			}
  
  			// DRY: call self to get sitemap partition object
  			return await GetSitemapPartAsync(index, true);
  		}
  
  		private bool SitemapFileExists(int storeId, int languageId, int index, out string path, out string name)
  		{
  			path = BuildSitemapFilePath(storeId, languageId, index, out name);
  
  			// Does not work reliably with symlinks due to framework caching
  			//var exists = _tenantFolder.FileExists(path);
  
  			var exists = File.Exists(_tenantFolder.MapPath(path));
  
  			if (!exists)
  			{
  				path = null;
  				name = null;
  			}
  
  			return exists;
  		}
  
  		private string BuildSitemapFilePath(int storeId, int languageId, int index, out string fileName)
  		{
  			fileName = SiteMapFileNamePattern.FormatInvariant(index);
  			return _tenantFolder.Combine(BuildSitemapDirPath(storeId, languageId), fileName);
  		}
  
  		private string BuildSitemapDirPath(int storeId, int languageId)
  		{
  			return _tenantFolder.Combine(_baseDir, storeId + "/" + languageId);
  		}
  
  		private string GetLockFilePath(int storeId, int languageId)
  		{
  			var fileName = LockFileNamePattern.FormatInvariant(storeId, languageId);
  			return _tenantFolder.Combine(_baseDir, fileName);
  		}
  
  		public virtual async Task RebuildAsync(XmlSitemapBuildContext ctx)
  		{
  			Guard.NotNull(ctx, nameof(ctx));
  
  			var languageData = new Dictionary<int, LanguageData>();
  
  			foreach (var language in ctx.Languages)
  			{
  				var lockFilePath = GetLockFilePath(ctx.Store.Id, language.Id);
  
  				if (_lockFileManager.TryAcquireLock(lockFilePath, out var lockFile))
  				{
  					// Process only languages that are unlocked right now
  					// It is possible that an HTTP request triggered the generation
  					// of a language specific sitemap.
  
  					var sitemapDir = BuildSitemapDirPath(ctx.Store.Id, language.Id);
  					var data = new LanguageData
  					{
  						Store = ctx.Store,
  						Language = language,
  						LockFile = lockFile,
  						LockFilePath = lockFilePath,
  						TempDir = sitemapDir + "~",
  						FinalDir = sitemapDir,
  						BaseUrl = BuildBaseUrl(ctx.Store, language)
  					};
  
  					_tenantFolder.TryDeleteDirectory(data.TempDir);
  					_tenantFolder.CreateDirectory(data.TempDir);
  
  					languageData[language.Id] = data;
  				}
  			}
  
  			if (languageData.Count == 0)
  			{
  				Logger.Warn("XML sitemap rebuild already in process.");
  				return;
  			}
  
  			var languages = languageData.Values.Select(x => x.Language);
  			var languageIds = languages.Select(x => x.Id).Concat(new[] { 0 }).ToArray();
  
  			// All sitemaps grouped by language
  			var sitemaps = new Multimap<int, XmlSitemapNode>();
  
  			var compositeFileLock = new ActionDisposable(() => 
  			{
  				foreach (var data in languageData.Values)
  				{
  					data.LockFile.Release();
  				}
  			});
  
  			using (compositeFileLock)
  			{
  				// Impersonate
  				var prevCustomer = _services.WorkContext.CurrentCustomer;
  				// no need to vary xml sitemap by customer roles: it's relevant to crawlers only.
  				_services.WorkContext.CurrentCustomer = _customerService.GetCustomerBySystemName(SystemCustomerNames.SearchEngine);
  
  				try
  				{
  					var nodes = new List<XmlSitemapNode>();
  
  					var queries = CreateQueries(ctx);
  					var total = await queries.GetTotalRecordCountAsync();
  
  					var totalSegments = (int)Math.Ceiling(total / (double)MaximumSiteMapNodeCount);
  					var hasIndex = totalSegments > 1;
  
gt: 					var indexNodes = new Multimap<int, XmlSitemapNode>();