context: using System;
  using System.Collections;
  using System.Collections.Generic;
  using System.Text;
  using System.IO;
  using System.Linq;
  using System.Xml;
  using System.Text.RegularExpressions;
  using TabulateSmarterTestContentPackage.Extractors;
  using TabulateSmarterTestContentPackage.Mappers;
  using TabulateSmarterTestContentPackage.Models;
  using TabulateSmarterTestContentPackage.Utilities;
  using TabulateSmarterTestContentPackage.Validators;
  
  namespace TabulateSmarterTestContentPackage
  {
      
      // Using partial class so that we can distribute the functions among source code files
      // while retaining the context of the tabulator class.
      partial class Tabulator : IDisposable
      {
          const string cImsManifest = "imsmanifest.xml";
          const string cItemTypeStim = "stim";
          const string cItemTypeWordlist = "wordList";
          const string cItemTypeTutorial = "tut";
          const string cScoreMetaHand = "HandScored";
          const string cScoreMetaMachine = "Automatic with Machine Rubric";
          const string cSelectedResponseAnswerKey = "SR";
  
          static NameTable sXmlNt;
          static XmlNamespaceManager sXmlNs;
  
          const string cStimulusInteractionType = "Stimulus";
  
          static readonly HashSet<string> sValidWritingTypes = new HashSet<string>(
              new[] {
                  "Explanatory",
                  "Opinion",
                  "Informative",
                  "Argumentative",
                  "Narrative"
              });
  
          // Filenames
          const string cSummaryReportFn = "SummaryReport.txt";
          const string cItemReportFn = "ItemReport.csv";
          const string cStimulusReportFn = "StimulusReport.csv";
          const string cWordlistReportFn = "WordlistReport.csv";
          const string cGlossaryReportFn = "GlossaryReport.csv";
          const string cErrorReportFn = "ErrorReport.csv";
          const string cIdReportFn = "IdReport.csv";
          const string cRubricExportFn = "Rubrics";
  
          static ItemIdentifier cBlankItemId = new ItemIdentifier(string.Empty, 0, 0);
  
          // Per Package variables
          TestPackage mPackage;
          Dictionary<string, string> mFilenameToResourceId = new Dictionary<string, string>();
          HashSet<string> mResourceDependencies = new HashSet<string>();
          DistinctList<ItemIdentifier> mItemQueue = new DistinctList<ItemIdentifier>();
          DistinctList<ItemIdentifier> mStimQueue = new DistinctList<ItemIdentifier>();
          DistinctList<ItemIdentifier> mWordlistQueue = new DistinctList<ItemIdentifier>();
          DistinctList<ItemIdentifier> mTutorialQueue = new DistinctList<ItemIdentifier>();
          Dictionary<string, int> mWordlistRefCounts = new Dictionary<string, int>();   // Reference count for wordlist IDs
          int mProgressCount = 0;
          int mTransferCount = 0;
  
          class RubricInfo
          {
              public ItemIdentifier FirstItemId;
              public int Count;
          }
  
          // Per report variables
          bool mInitialized;
          int mStartTicks;
          int mItemCount = 0;
          int mWordlistCount = 0;
          int mStimCount = 0;
          int mGlossaryTermCount = 0;
          Dictionary<string, int> mTypeCounts = new Dictionary<string, int>();
          Dictionary<string, int> mTermCounts = new Dictionary<string, int>();
          Dictionary<string, int> mTranslationCounts = new Dictionary<string, int>();
          Dictionary<string, int> mAnswerKeyCounts = new Dictionary<string, int>();
  
          StatAccumulator mAslStat = new StatAccumulator();
          string mReportPathPrefix;
          TextWriter mItemReport;
          TextWriter mStimulusReport;
          TextWriter mWordlistReport;
          TextWriter mGlossaryReport;
          TextWriter mSummaryReport;
          string mRubricPathPrefix;
  
  
  
          static Tabulator()
          {
              sXmlNt = new NameTable();
              sXmlNs = new XmlNamespaceManager(sXmlNt);
              sXmlNs.AddNamespace("sa", "http://www.smarterapp.org/ns/1/assessment_item_metadata");
              sXmlNs.AddNamespace("xsi", "http://www.w3.org/2001/XMLSchema-instance");
              sXmlNs.AddNamespace("ims", "http://www.imsglobal.org/xsd/apip/apipv1p0/imscp_v1p1");
  
              StaticInitWordlist();
          }
  
          public Tabulator(string reportPathPrefix)
          {
              mReportPathPrefix = string.Concat(reportPathPrefix, "_");
          }
  
          public void Dispose()
          {
              Conclude();
          }
  
          public bool ReportIds { get; set; }
          public bool ExitAfterSelect { get; set; }
          public bool ExportRubrics { get; set; }
          public bool DeDuplicate { get; set; }
  
          public void SelectItems(IEnumerable<ItemIdentifier> itemIds)
          {
              foreach(var ii in itemIds)
              {
                  if (ii.IsStimulus)
                  {
                      mStimQueue.Add(ii);
                  }
                  else
                  {
                      mItemQueue.Add(ii);
                  }
                  int count = mItemQueue.Count + mStimQueue.Count;
                  if (count % 100 == 0)
                  {
                      Console.Error.Write($"   Selected {count}\r");
                  }
              }
          }
  
          void ReportSelectedItems()
          {
              // File gets deleted at init so we can do append for each set of items.
              using (var report = new StreamWriter(string.Concat(mReportPathPrefix, cIdReportFn), true, Encoding.UTF8))
              {
                  if (report.BaseStream.Length == 0)
                  {
                      report.WriteLine("BankKey,ItemId,ItemType");
                  }
                  foreach (ItemIdentifier ii in mItemQueue)
                  {
                      report.WriteLine(string.Join(",", ii.BankKey, ii.ItemId, ii.ItemType));
                  }
  
                  foreach (ItemIdentifier ii in mStimQueue)
                  {
                      report.WriteLine(string.Join(",", ii.BankKey, ii.ItemId, ii.ItemType));
                  }
              }
          }
  
          // Initialize all files and collections for a tabulation run
          private void Initialize()
          {
              mStartTicks = Environment.TickCount;
  
              // Prep the error report
              ReportingUtility.ErrorReportPath = string.Concat(mReportPathPrefix, cErrorReportFn);
              File.Delete(ReportingUtility.ErrorReportPath); // Delete does not throw exception if file does not exist.
              ReportingUtility.DeDuplicate = DeDuplicate;
  
              // Delete any existing reports
              File.Delete(mReportPathPrefix + cIdReportFn);
              File.Delete(mReportPathPrefix + cItemReportFn);
              File.Delete(mReportPathPrefix + cStimulusReportFn);
              File.Delete(mReportPathPrefix + cWordlistReportFn);
              File.Delete(mReportPathPrefix + cGlossaryReportFn);
  
              // Prep the summary report
              mSummaryReport = new StreamWriter(string.Concat(mReportPathPrefix, cSummaryReportFn), false, Encoding.UTF8);
  
              // Write the header
              {
                  var application = System.Reflection.Assembly.GetExecutingAssembly();
                  mSummaryReport.WriteLine($"TabulateSmarterTestContentPackage v{application.GetName().Version}");
                  mSummaryReport.WriteLine();
              }
  
              // Report options
              mSummaryReport.WriteLine($"Validation Options: {Program.Options}");
  
              // If tabulation not being suppressed, open the other reports
              if (!ExitAfterSelect)
              {
                  mItemReport = new StreamWriter(string.Concat(mReportPathPrefix, cItemReportFn), false, Encoding.UTF8);
                  // DOK is "Depth of Knowledge"
                  // In the case of multiple standards/claims/targets, these headers will not be sufficient
                  // TODO: Add CsvHelper library to allow expandable headers
                  mItemReport.WriteLine("Folder,BankKey,ItemId,ItemType,Version,Subject,Grade,Status,AnswerKey,AnswerOptions,AsmtType,WordlistId,StimId,TutorialId,ASL," +
                                        "BrailleType,Translation,Glossary,Media,Size,DOK,AllowCalculator,MathematicalPractice,MaxPoints," +
                                        "Claim,Target,CCSS,ClaimContentTarget,SecondaryCCSS,SecondaryClaimContentTarget,PtWritingType," +
                                        "CAT_MeasurementModel,CAT_ScorePoints,CAT_Dimension,CAT_Weight,CAT_Parameters,PP_MeasurementModel," +
                                        "PP_ScorePoints,PP_Dimension,PP_Weight,PP_Parameters");
  
                  mStimulusReport = new StreamWriter(string.Concat(mReportPathPrefix, cStimulusReportFn), false, Encoding.UTF8);
                  mStimulusReport.WriteLine("Folder,BankKey,StimulusId,Version,Subject,Status,WordlistId,ASL,BrailleType,Translation,Glossary,Media,Size,WordCount,StimulusLiteraryKnowledgeDemands,StimulusLiteraryNonFiction,StimulusReadabilityFK,StimulusReadabilityLexile");
  
                  mWordlistReport = new StreamWriter(string.Concat(mReportPathPrefix, cWordlistReportFn), false, Encoding.UTF8);
                  mWordlistReport.WriteLine("Folder,BankKey,WIT_ID,RefCount,TermCount,MaxGloss,MinGloss,AvgGloss");
  
                  mGlossaryReport = new StreamWriter(string.Concat(mReportPathPrefix, cGlossaryReportFn), false, Encoding.UTF8);
                  mGlossaryReport.WriteLine(Program.gValidationOptions.IsEnabled("gtr")
                      ? "Folder,BankKey,WIT_ID,ItemId,Index,Term,Language,Length,Audio,AudioSize,Image,ImageSize,Text"
                      : "Folder,BankKey,WIT_ID,ItemId,Index,Term,Language,Length,Audio,AudioSize,Image,ImageSize");
  
                  // If rubrics being exported, ensure the directory exists
                  if (ExportRubrics)
                  {
                      mRubricPathPrefix = string.Concat(mReportPathPrefix, cRubricExportFn);
                      Directory.CreateDirectory(mRubricPathPrefix);
                  }
              }
  
              ReportingUtility.ErrorCount = 0;
              mItemCount = 0;
              mWordlistCount = 0;
              mStimCount = 0;
              mGlossaryTermCount = 0;
  
              mTypeCounts.Clear();
              mTermCounts.Clear();
              mTranslationCounts.Clear();
              mAnswerKeyCounts.Clear();
  
              mAslStat.Clear();
  
              mInitialized = true;
          }
  
          private void Conclude()
          {
              try
              {
                  if (mSummaryReport != null)
                  {
                      SummaryReport(mSummaryReport);
  
                      // Report aggregate results to the console
                      Console.WriteLine("{0} Errors reported.", ReportingUtility.ErrorCount);
                      Console.WriteLine();
                  }
              }
              finally
              {
                  if (mSummaryReport != null)
                  {
                      mSummaryReport.Dispose();
                      mSummaryReport = null;
                  }
                  if (mWordlistReport != null)
                  {
                      mWordlistReport.Dispose();
                      mWordlistReport = null;
                  }
                  if (mStimulusReport != null)
                  {
                      mStimulusReport.Dispose();
                      mStimulusReport = null;
                  }
                  if (mItemReport != null)
                  {
                      mItemReport.Dispose();
                      mItemReport = null;
                  }
                  if (mGlossaryReport != null)
                  {
                      mGlossaryReport.Dispose();
                      mGlossaryReport = null;
                  }
                  ReportingUtility.CloseReport();
              }
          }
  
          public void Tabulate(TestPackage package)
          {
              if (!mInitialized)
              {
                  Initialize();
              }
              
              Console.WriteLine("Tabulating " + package.Name);
              try
              {
                  var startTicks = Environment.TickCount;
                  ReportingUtility.CurrentPackageName = package.Name;
  
                  // Initialize package-specific collections
                  mPackage = package;
                  mFilenameToResourceId.Clear();
                  mResourceDependencies.Clear();
                  mWordlistRefCounts.Clear();
  
                  // Validate the manifest (if it exists)
                  {
                      FileFolder rootFolder;
                      if (package.TryGetItem(null, out rootFolder))   // Retrieves the root folder if the package has one
                      {
                          // Validate manifest
                          if (!ValidateManifest(rootFolder))
                          {
                              return; // Not a valid package
                          }
                      }
                  }
  
                  // Select the items to be tablulated (if a specific set was not already given)
                  string label;
                  if (mItemQueue.Count == 0 && mStimQueue.Count == 0)
                  {
                      SelectItems(package.ItemsAndStimuli);
                      label = "Selected";
                  }
                  else
                  {
                      label = "Preselected";
                  }
  
                  // Sort the queues before reporting IDs
                  mItemQueue.Sort();
                  mStimQueue.Sort();
  
                  // Report
                  if (ReportIds)
                  {
                      ReportSelectedItems();
                  }
                  Console.WriteLine($"   {label} {mItemQueue.Count} items and {mStimQueue.Count} stimuli.");
  
                  // Tabulate the selected items
                  if (!ExitAfterSelect)
                  {
                      // Process Items
                      foreach (var ii in mItemQueue)
                      {
                          ++mProgressCount;
                          try
                          {
                              TabulateItem(ii);
                              ReportProgress();
                          }
                          catch (Exception err)
                          {
                              ReportingUtility.ReportError(ii, err);
                          }
                      }
  
                      // Process stimuli
                      mStimQueue.Sort(); // Sort again - stimuli may have been added while processing items
                      foreach (var ii in mStimQueue)
                      {
                          ++mProgressCount;
                          try
                          {
                              TabulateStimulus(ii);
                              ReportProgress();
                          }
                          catch (Exception err)
                          {
                              ReportingUtility.ReportError(ii, err);
                          }
                      }
  
                      // Process WordLists
                      mWordlistQueue.Sort();
                      foreach (var ii in mWordlistQueue)
                      {
                          ++mProgressCount;
                          try
                          {
                              TabulateWordList(ii);
                              ReportProgress();
                          }
                          catch (Exception err)
                          {
                              ReportingUtility.ReportError(ii, err);
                          }
                      }
  
                      // Process Tutorials (we handle these separately because they are dependent on items)
                      mTutorialQueue.Sort();
                      foreach (var ii in mTutorialQueue)
                      {
                          ++mProgressCount;
                          try
                          {
                              TabulateTutorial(ii);
                              ReportProgress();
                          }
                          catch (Exception err)
                          {
                              ReportingUtility.ReportError(ii, err);
                          }
                      }
                  }
  
                  ReportingUtility.CurrentPackageName = null;
  
                  var elapsedTicks = unchecked((uint)Environment.TickCount - (uint)startTicks);
                  Console.WriteLine("   Package time: {0}.{1:d3} seconds", elapsedTicks / 1000, elapsedTicks % 1000);
                  mSummaryReport.WriteLine("{0}: {1}.{2:d3} seconds", package.Name, elapsedTicks / 1000, elapsedTicks % 1000);
  
                  // Clear the queues
                  mItemQueue.Clear();
                  mStimQueue.Clear();
                  mWordlistQueue.Clear();
                  mTutorialQueue.Clear();
                  mProgressCount = 0;
                  mTransferCount = 0;
              }
              catch (Exception err)
              {
                  Console.WriteLine("   Exception: " + err.Message);
                  ReportingUtility.ReportError(null, err);
              }
          }
  
          static readonly char[] c_progressChars = new char[] { '|', '/', '-', '\\' };
          static int s_progressSpin;
          static int s_lastProgressTicks;
          private void ReportProgress()
          {
              uint elapsedTicks = unchecked((uint)Environment.TickCount - (uint)s_lastProgressTicks);
              if (elapsedTicks > 150) // milliseconds
              {
                  ++s_progressSpin;
                  s_lastProgressTicks = Environment.TickCount;
              }
              int total = mItemQueue.Count + mStimQueue.Count + mTutorialQueue.Count + mWordlistQueue.Count - mTransferCount;
              Console.Error.Write($"   {mProgressCount} of {total} {c_progressChars[s_progressSpin & 3]}\r");
          }
  
          private void TabulateItem(ItemIdentifier ii)
          {
              /* Handling wordList items.
               * 
               * To begin with, we don't know if an item is an actual interaction item so wordList
               * items will come into this function. However, wordLists need to be processed after
               * items so they have their own queue. WordLists get into that queue in one two ways.
               * 
               * 1) When an item is processed that references a wordList then the ID of that wordlist
               * is added into the wordList queue.
               * 
               * 2) In this function, when the item is read and it's type is wordList then it will be
               * added to the queue.
               * 
               * We avoid reading and parsing the wordList XML unnecessarily by checkign whether the
               * item ID is in the wordlist queue before ever reading it in. That will happen if the
               * referencing item comes before the wordlist item in the queue.
               */
  
              // Skip the item if it's already in the wordlist queue.
              if (mWordlistQueue.Contains(ii))
              {
                  --mProgressCount;   // Didn't actually process anything
                  return;
              }
  
              // Skip the item it it's already in the tutorial queue.
              if (mTutorialQueue.Contains(ii))
              {
                  --mProgressCount;   // Didn't actually process anything
                  return;
              }
  
              // Get the item folder
              FileFolder ffItem;
              if (!mPackage.TryGetItem(ii, out ffItem))
              {
                  ReportingUtility.ReportError(ii, ErrorId.T0110);
                  return;
              }
  
              // Read the item XML
              var xml = new XmlDocument(sXmlNt);
              if (!TryLoadXml(ffItem, ffItem.Name + ".xml", xml))
              {
                  ReportingUtility.ReportError(ii, ErrorId.T0025, LoadXmlErrorDetail);
                  return;
              }
  
              // Get the actual item type
              string itemType = null;
              var xmlItem = xml.SelectSingleNode("itemrelease/item") as XmlElement;
              if (xmlItem != null)
              {
                  itemType = xmlItem.XpEval("@format") ?? xmlItem.XpEval("@type");
              }
              if (string.IsNullOrEmpty(itemType))
              {
                  ReportingUtility.ReportError(ii, ErrorId.T0044, LoadXmlErrorDetail);
                  return;
              }
              ii.ItemType = itemType;
  
              // If wordlist, transfer to the wordList queue
              if (ii.ItemType.Equals(cItemTypeWordlist, StringComparison.Ordinal))
              {
                  if (mWordlistQueue.Add(ii))
                  {
                      ++mTransferCount;
                  }
                  --mProgressCount; // Have not yet processed this item
                  return;
              }
  
              // If tutorial, transfer to the tutorial queue
              if (ii.ItemType.Equals(cItemTypeTutorial))
              {
                  if (mTutorialQueue.Add(ii))
                  {
                      ++mTransferCount;
                  }
                  --mProgressCount; // Have not yet processed this item
                  return;
              }
  
              // Count the item
              ++mItemCount;
              mTypeCounts.Increment(itemType);
  
              // Build the item context
              var it = new ItemContext(mPackage, ffItem, ii);
  
              // Handle the item according to type
              switch (it.ItemType)
              {
                  case "EBSR":        // Evidence-Based Selected Response
                  case "eq":          // Equation
                  case "er":          // Extended-Response
                  case "gi":          // Grid Item (graphic)
                  case "htq":         // Hot Text (QTI)
                  case "mc":          // Multiple Choice
                  case "mi":          // Match Interaction
                  case "ms":          // Multi-Select
                  case "sa":          // Short Answer
                  case "ti":          // Table Interaction
                  case "wer":         // Writing Extended Response
                      TabulateInteraction(it);
                      break;
  
                  case "nl":          // Natural Language
                  case "SIM":         // Simulation
                      ReportingUtility.ReportError(it, ErrorId.T0111, "itemType='{0}'", it.ItemType);
                      TabulateInteraction(it);
                      break;
  
                  default:
                      ReportingUtility.ReportError(it, ErrorId.T0080, "itemType='{0}'", it.ItemType);
                      break;
              }
          }
  
          private void TabulateInteraction(ItemContext it)
          {
              // Read the item XML
              var xml = new XmlDocument(sXmlNt);
              if (!TryLoadXml(it.FfItem, it.FfItem.Name + ".xml", xml))
              {
                  ReportingUtility.ReportError(it, ErrorId.T0025, LoadXmlErrorDetail);
                  return;
              }
  
              // Get the version
              var itemVersion = xml.XpEvalE("itemrelease/item/@version");
              it.Version = itemVersion; // Make version available for error reporting.
  
              // Load metadata
              var xmlMetadata = new XmlDocument(sXmlNt);
              if (!TryLoadXml(it.FfItem, "metadata.xml", xmlMetadata))
              {
                  ReportingUtility.ReportError(it, ErrorId.T0112, LoadXmlErrorDetail);
              }
  
              // Get metadata version (which includes the minor version number)
              var metadataVersion = xmlMetadata.XpEvalE("metadata/sa:smarterAppMetadata/sa:Version", sXmlNs);
  
              // Check for consistency between the version number in item xml and metadata xml.
              // The metadata XML stores the version number in "major.minor" format, while 
              // the item xml stores the version in "major" format. Only the "major" number
              // is to be compared.
              var metadataVersionValues = metadataVersion.Split('.');
              if (!itemVersion.Equals(metadataVersionValues[0]))
              {
                  ReportingUtility.ReportError(it, ErrorId.T0114, "Item version='{0}' Metadata major version='{1}'", itemVersion, metadataVersionValues[0]);
              }
              else
              {
                  it.Version = metadataVersion; // Update to include minor version number when present
              }
  
              IList< ItemScoring > scoringInformation = new List<ItemScoring>();
              if (xmlMetadata.HasChildNodes)
              {
                  scoringInformation = IrtExtractor.RetrieveIrtInformation(xmlMetadata.MapToXDocument()).ToList();
              }
              if (!scoringInformation.Any())
              {
                  scoringInformation.Add(new ItemScoring());
              }
  
              // Check interaction type
              var metaItemType = xmlMetadata.XpEvalE("metadata/sa:smarterAppMetadata/sa:InteractionType", sXmlNs);
              if (!string.Equals(metaItemType, it.ItemType.ToUpperInvariant(), StringComparison.Ordinal))
                  ReportingUtility.ReportError(it, ErrorId.T0113, "InteractionType='{0}' Expected='{1}'", metaItemType, it.ItemType.ToUpperInvariant());
  
              // Subject
              var subject = xml.XpEvalE("itemrelease/item/attriblist/attrib[@attid='itm_item_subject']/val");
              var metaSubject = xmlMetadata.XpEvalE("metadata/sa:smarterAppMetadata/sa:Subject", sXmlNs);
              if (string.IsNullOrEmpty(subject))
              {
                  ReportingUtility.ReportError(it, ErrorId.T0060);
                  subject = metaSubject;
                  if (string.IsNullOrEmpty(subject))
                      ReportingUtility.ReportError(it, ErrorId.T0061);
              }
              else
              {
                  if (!string.Equals(subject, metaSubject, StringComparison.Ordinal))
                      ReportingUtility.ReportError(it, ErrorId.T0115, "ItemSubject='{0}' MetadataSubject='{1}'", subject, metaSubject);
              }
  
              // AllowCalculator
              var allowCalculator = AllowCalculatorFromMetadata(xmlMetadata, sXmlNs);
              if (string.IsNullOrEmpty(allowCalculator) && 
                  (string.Equals(metaSubject, "MATH", StringComparison.OrdinalIgnoreCase) || 
                  string.Equals(subject, "MATH", StringComparison.OrdinalIgnoreCase)))
              {
                  ReportingUtility.ReportError(it, ErrorId.T0001);
              }
  
              // MaximumNumberOfPoints
              var maximumNumberOfPoints = MaximumNumberOfPointsFromMetadata(xmlMetadata, sXmlNs);
              {
                  int maxPts = 0;
                  if (string.IsNullOrEmpty(maximumNumberOfPoints) || !int.TryParse(maximumNumberOfPoints, out maxPts))
                  {
                      ReportingUtility.ReportError(it, ErrorId.T0049);
                  }
  
                  else if (it.ItemType.Equals("wer", StringComparison.OrdinalIgnoreCase) && maxPts > 6)
                  {
                      ReportingUtility.ReportError(it, ErrorId.T0050, $"maxPoints='{maxPts}' subject='{subject}'");
                  }
              }
  
              // Grade
              var grade = xml.XpEvalE("itemrelease/item/attriblist/attrib[@attid='itm_att_Grade']/val").Trim();
              var metaGrade = xmlMetadata.XpEvalE("metadata/sa:smarterAppMetadata/sa:IntendedGrade", sXmlNs);
              if (string.IsNullOrEmpty(grade))
              {
                  ReportingUtility.ReportError(it, ErrorId.T0059);
                  grade = metaGrade;
                  if (string.IsNullOrEmpty(grade))
                      ReportingUtility.ReportError(it, ErrorId.T0058);
              }
              else
              {
  
gt:                 if (!string.Equals(grade, metaGrade, StringComparison.Ordinal))