text: //	X1EmailScanner.cpp
  //
  //	Scans email to build X1 indices.
  //
  //	Copyright (c) 2005 by QUALCOMM, Incorporated
  /* Copyright (c) 2016, Computer History Museum 
  All rights reserved. 
  Redistribution and use in source and binary forms, with or without modification, are permitted (subject to 
  the limitations in the disclaimer below) provided that the following conditions are met: 
   * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 
   * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following 
     disclaimer in the documentation and/or other materials provided with the distribution. 
   * Neither the name of Computer History Museum nor the names of its contributors may be used to endorse or promote products 
     derived from this software without specific prior written permission. 
  NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE 
  COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 
  HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 
  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 
  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 
  DAMAGE. */
  
  
  
  #include "stdafx.h"
  #include "X1EmailScanner.h"
  #include "X1EmailIndexItem.h"
  #include "SearchManagerInfo.h"
  #include "SearchManagerTaskInfo.h"
  #include "SearchManagerUtils.h"
  #include "debug.h"
  #include "QCMailboxCommand.h"
  #include "QCMailboxDirector.h"
  #include "QCTaskManager.h"
  
  
  extern QCMailboxDirector		g_theMailboxDirector;
  
  
  // ---------------------------------------------------------------------------
  //		* X1EmailScanner											 [Public]
  // ---------------------------------------------------------------------------
  //	X1EmailScanner constructor
  
  X1EmailScanner::X1EmailScanner(
  	X1::IX1Database *			in_pX1DB)
  	:	m_nRefCount(0), m_pEmailDB(in_pX1DB), m_nConnectionHandle(0),
  		m_nScanCount(0), m_eScanState(ss_Stopped),
  		m_eLastScanType(st_None), m_eScanType(st_None), m_eNextScanType(st_None),
  		m_bProcessingIndexUpdateActions(false), m_szMessageBuffer(),
  		m_nMessageIDLastAddAttempt(0), m_pUpdateActionsTaskInfo(NULL),
  		m_nTotalUpdateActions(0), m_nCurrentUpdateAction(0),
  		m_szMailboxName(), m_nTotalMessagesToRemove(0),
  		m_pMainScansTaskInfo(NULL),
  		m_pMailboxList(NULL), m_posCurrentMailbox(NULL), m_pCurrentMailbox(NULL),
  		m_nTotalMailboxes(0), m_nCurrentMailbox(0),
  		m_pX1IndexedItems(), m_nNumX1IndexedItems(0),
  		m_nX1IndexedItem(0)
  {
  	//	Initialize arrays
  	m_pCurrentTocDoc[sd_MainScans] = NULL;
  	m_bMailboxFileExists[sd_MainScans] = TRUE;
  	m_posCurrentSummary[sd_MainScans] = NULL;
  	m_posLastSummaryAddAttempt[sd_MainScans] = NULL;
  	m_nCurrentMessage[sd_MainScans] = 0;
  	m_nSoFarBase[sd_MainScans] = 0;
  	m_bCheckForIndexedMessages[sd_MainScans] = true;
  	m_bCheckingMessages[sd_MainScans] = false;
  	m_bNeedToAddMessages[sd_MainScans] = false;
  
  	m_pCurrentTocDoc[sd_IndexUpdateActions] = NULL;
  	m_bMailboxFileExists[sd_IndexUpdateActions] = TRUE;
  	m_posCurrentSummary[sd_IndexUpdateActions] = NULL;
  	m_posLastSummaryAddAttempt[sd_IndexUpdateActions] = NULL;
  	m_nCurrentMessage[sd_IndexUpdateActions] = 0;
  	m_nSoFarBase[sd_IndexUpdateActions] = 0;
  	m_bCheckForIndexedMessages[sd_IndexUpdateActions] = true;
  	m_bCheckingMessages[sd_IndexUpdateActions] = false;
  	m_bNeedToAddMessages[sd_IndexUpdateActions] = false;
  }
  
  
  // ---------------------------------------------------------------------------
  //		* ~X1EmailScanner											 [Public]
  // ---------------------------------------------------------------------------
  //	X1EmailScanner destructor
  
  X1EmailScanner::~X1EmailScanner() 
  {
  }
  
  
  // ---------------------------------------------------------------------------
  //		* QueryInterface											 [Public]
  // ---------------------------------------------------------------------------
  //	Standard IUnknown::QueryInterface() implementation
  
  HRESULT __stdcall
  X1EmailScanner::QueryInterface(
  	const IID &				in_iid,
  	void **					out_ppvObject)
  {
  	if ( (in_iid == IID_IUnknown) || (in_iid == X1::IID_IX1Scanner) )
  	{
  		*out_ppvObject = static_cast<X1::IX1Scanner*>(this);
  	}
  	else
  	{
  		*out_ppvObject = NULL;
  		return E_NOINTERFACE;
  	}
  
  	AddRef();
  
  	return S_OK;
  }
  
  
  // ---------------------------------------------------------------------------
  //		* AddRef													 [Public]
  // ---------------------------------------------------------------------------
  //	Standard IUnknown::AddRef() implementation
  
  ULONG __stdcall
  X1EmailScanner::AddRef()
  {
  	return InterlockedIncrement(&m_nRefCount);
  }
  
  
  // ---------------------------------------------------------------------------
  //		* Release													 [Public]
  // ---------------------------------------------------------------------------
  //	Standard IUnknown::Release() implementation
  
  ULONG __stdcall
  X1EmailScanner::Release()
  {
  	if (InterlockedDecrement(&m_nRefCount) == 0)
  	{
  		delete this;
  		return 0;
  	}
  
  	return m_nRefCount;
  }
  
  
  // ---------------------------------------------------------------------------
  //		* raw_ScanItem												 [Public]
  // ---------------------------------------------------------------------------
  //	Called by the scanner manager to index the next item. This should
  //	generally  only scan a single item at a time. pbDone should be set
  //	to FALSE if there are more messages to scan, otherwise it should be
  //	left untouched.
  
  HRESULT __stdcall
  X1EmailScanner::raw_ScanItem(
  	unsigned char *				out_pbDone)
  {	
  	//	Don't do anything if the scanner isn't running...
  	if (m_eScanState != ss_Running)
  	{
  		return S_OK;
  	}
  
  	//	Bump scan count so that we can detect when we've been scanning
  	m_nScanCount++;
  
  	//	Assume that there's more to do
  	*out_pbDone = FALSE;
  
  	//	Provided we're not currently doing a culling scan, we want to process
  	//	index update actions whenever we have any. That way recently received
  	//	email gets added immediately even when we're waiting for a slower
  	//	add/update scan. The st_ProcessingIndexUpdateActions scan type is
  	//	only really used as a placeholder when we're not using any other
  	//	scan type.
  	//
  	//	We don't interrupt a culling scan because as it is currently implemented
  	//	it requires that the index is not modified while it is running.
  	//	(In fact in order to avoid modifying the index while it scans, the
  	//	culling scan queues "remove" index update actions when it finds an index
  	//	item to cull).
  	if ( (m_eScanType != st_CullingScan) &&
  		 SearchManager::Instance()->HasIndexUpdateActions() )
  	{
  		//	Do we have progress displaying yet?
  		if ( !m_pUpdateActionsTaskInfo &&
  			 (GetIniShort(IDS_INI_SHOW_INDEX_UPDATE_PROGRESS) != 0) )
  		{
  			long	nNumberUpdateActions = SearchManager::Instance()->GetNumberIndexUpdateActions();
  			
  			if ( (nNumberUpdateActions > GetIniLong(IDS_INI_INDEX_UPDATE_MIN_TO_SHOW_PROGRESS)) ||
  				 SearchManager::Instance()->HasMailboxUpdateActions() )
  			{
  				QCTaskManager *		pTaskManager = QCGetTaskManager();
  
  				if (pTaskManager)
  				{
  					//	Add task to task manager to display progress
  					m_pUpdateActionsTaskInfo = DEBUG_NEW SearchManagerTaskInfo(false);
  					pTaskManager->Register(m_pUpdateActionsTaskInfo);
  				}
  			}
  		}
  
  		if ( m_pUpdateActionsTaskInfo && (m_nCurrentUpdateAction == m_nTotalUpdateActions) )
  		{
  			//	We starting processing the next "batch" of actions. Set the current action
  			//	to 0, and the total number of actions to the current remaining count.
  			m_nCurrentUpdateAction = 0;
  			m_nTotalUpdateActions = SearchManager::Instance()->GetNumberIndexUpdateActions();
  			m_pUpdateActionsTaskInfo->SetTotal(m_nTotalUpdateActions * 100);
  			m_nSoFarBase[sd_IndexUpdateActions] = 0;
  
  			//	Set the title
  			m_pUpdateActionsTaskInfo->SetTitle( CRString(IDS_UPDATING_INDEX_TITLE) );
  
  			//	Set the initial main progress text
  			CString		szMainText;
  			szMainText.Format(IDS_UPDATING_INDEX_REMAINING, m_nTotalUpdateActions - m_nCurrentUpdateAction);
  			m_pUpdateActionsTaskInfo->SetMainText(szMainText);
  		}
  		
  		ProcessNextIndexUpdateItem();
  
  		if (m_pUpdateActionsTaskInfo)
  		{
  			//	Update the progress text with the latest remaining count
  			CString		szMainText;
  			szMainText.Format(IDS_UPDATING_INDEX_REMAINING, m_nTotalUpdateActions - m_nCurrentUpdateAction);
  			m_pUpdateActionsTaskInfo->SetMainText(szMainText);
  		}
  
  		if ( !SearchManager::Instance()->HasIndexUpdateActions() )
  		{
  			//	We're done (for now) so close the last mailbox that we had open
  			CloseCurrentMailbox(sd_IndexUpdateActions);
  
  			//	Tell SearchManager to return FALSE in SearchManager::Idle so that
  			//	extended idle processing stops until the next idle timer.
  			//	This will give our progress bar a chance to display full.
  			SearchManager::Instance()->SetPauseExtendedIdleProcessing(true);
  		}
  	}
  	else
  	{
  		if (m_pUpdateActionsTaskInfo)
  		{
  			//	We have task info up from processing update actions. We're done
  			//	now, so remove it. (We wait until now, i.e. until another idle
  			//	after we finished, so that the task info has a chance to show
  			//	a full progress bar).
  
  			//	Reset action counts
  			m_nCurrentUpdateAction = 0;
  			m_nTotalUpdateActions = 0;
  
  			//	Kill progress display
  			m_pUpdateActionsTaskInfo->Kill();
  
  			//	Kill triggers deletion of object
  			m_pUpdateActionsTaskInfo = NULL;
  		}
  		
  		switch (m_eScanType)
  		{
  			case st_AddOrUpdateItemsScan:
  				//	Do the add/update items scan
  				DoAddOrUpdateItemsScan();
  				break;
  
  			case st_CullingScan:
  				//	Do the culling scan
  				DoCullingScan();
  				break;
  
  			case st_ProcessingIndexUpdateActions:
  				//	We would only get here if we don't have any index update
  				//	actions. Therefore we're done with the current scan.
  				StopCurrentScan();
  				break;
  
  			case st_ReadyForNextScanType:
  				//	Move to the next scan
  				m_eScanType = m_eNextScanType;
  
  				//	Next scan is now nothing
  				m_eNextScanType = st_None;
  
  				switch (m_eScanType)
  				{
  					case st_AddOrUpdateItemsScan:
  						//	Init add/update items scan
  						InitAddOrUpdateItemsScan();
  						break;
  
  					case st_CullingScan:
  						//	Init culling scan
  						InitCullingScan();
  						break;
  
  					case st_None:
  						//	Stop all scanning
  						StopAllScanning();
  
  						//	There's no more to do
  						*out_pbDone = TRUE;
  						break;
  				}
  				break;
  		}
  	}
  
  	return S_OK;
  }
  
  
  // ---------------------------------------------------------------------------
  //		* IsScanning												 [Public]
  // ---------------------------------------------------------------------------
  //	Returns whether or not we're currently scanning.
  
  bool
  X1EmailScanner::IsScanning() const
  {
  	return (m_eScanState != ss_Stopped);
  }
  
  
  // ---------------------------------------------------------------------------
  //		* IsDoingFullScan											 [Public]
  // ---------------------------------------------------------------------------
  //	Returns whether or not we're currently scanning.
  
  bool
  X1EmailScanner::IsDoingFullScan() const
  {
  	bool	bIsDoingFullScan = false;
  
  	//	Check to see if we're scanning at all
  	if ( IsScanning() )
  	{
  		//	Are we currently doing an add/update scan or culling scan?
  		if ( (m_eScanType == st_AddOrUpdateItemsScan) || (m_eScanType == st_CullingScan) )
  			bIsDoingFullScan = true;
  
  		//	Are we about to do an add/update scan or culling scan?
  		if ( (m_eScanType == st_ReadyForNextScanType) && 
  			 ((m_eScanType == st_AddOrUpdateItemsScan) || (m_eScanType == st_CullingScan)) )
  		{
  			bIsDoingFullScan = true;
  		}
  	}
  	
  	return bIsDoingFullScan;
  }
  
  
  // ---------------------------------------------------------------------------
  //		* StartScan													 [Public]
  // ---------------------------------------------------------------------------
  //	Starts scanning
  
  void
  X1EmailScanner::StartScan(
  	ScanTypeEnum				in_eScanType,
  	ScanTypeEnum				in_eNextScanType,
  	const char *				in_szIndexingMailboxRelativePath)
  {
  	//	Start scan if:
  	if ( (m_eScanState != ss_Running) || (m_eScanType != in_eScanType) )
  	{
  		TRACE("X1 scan started\n");
  
  		//	Set scan state to running
  		m_eScanState = ss_Running;
  
  		//	Setup current and next scan type as indicated
  		m_eLastScanType = st_None;
  		m_eScanType = in_eScanType;
  		m_eNextScanType = in_eNextScanType;
  
  		//	Init scan types appropriately
  		switch (m_eScanType)
  		{
  			case st_AddOrUpdateItemsScan:
  				PutDebugLog( DEBUG_MASK_SEARCH, "Starting search indexing (adding and updating items)" );
  
  				InitAddOrUpdateItemsScan(in_szIndexingMailboxRelativePath);
  				break;
  
  			case st_CullingScan:
  				PutDebugLog( DEBUG_MASK_SEARCH, "Starting search indexing (culling items)" );
  				
  				InitCullingScan(in_szIndexingMailboxRelativePath);
  				break;
  
  			case st_ProcessingIndexUpdateActions:
  				PutDebugLog( DEBUG_MASK_SEARCH, "Starting search indexing (processing index update items)" );
  				break;
  		}
  	}
  }
  
  
  // ---------------------------------------------------------------------------
  //		* PauseScan													 [Public]
  // ---------------------------------------------------------------------------
  //	Pauses scanning
  
  void
  X1EmailScanner::PauseScan(
  	bool						in_bWaitingForIdle)
  {
  	if (m_eScanState == ss_Running)
  	{
  		TRACE("X1 scan paused\n");
  		m_eScanState = ss_Paused;
  	}
  
  	if (in_bWaitingForIdle)
  	{
  		if (m_pUpdateActionsTaskInfo)
  			m_pUpdateActionsTaskInfo->Progress( CRString(IDS_INDEXING_PAUSING) );
  
  		if (m_pMainScansTaskInfo)
  			m_pMainScansTaskInfo->Progress( CRString(IDS_INDEXING_PAUSING) );
  	}
  }
  
  
  // ---------------------------------------------------------------------------
  //		* ResumeScan												 [Public]
  // ---------------------------------------------------------------------------
  //	Resumes scanning
  
  void
  X1EmailScanner::ResumeScan()
  {
  	if (m_eScanState == ss_Paused)
  	{
  		TRACE("X1 scan resumed\n");
  		m_eScanState = ss_Running;
  	}
  }
  
  
  // ---------------------------------------------------------------------------
  //		* StopCurrentScan											 [Public]
  // ---------------------------------------------------------------------------
  //	Stops current scan
  
  void
  X1EmailScanner::StopCurrentScan()
  {
  	if (m_eScanType == st_AddOrUpdateItemsScan)
  		PutDebugLog( DEBUG_MASK_SEARCH, "Finished with search indexing (adding and updating items)" );
  	else if (m_eScanType == st_CullingScan)
  		PutDebugLog( DEBUG_MASK_SEARCH, "Finished with search index (culling items)" );
  
  	if (m_pMainScansTaskInfo)
  	{
  		//	We have task info up from doing main scan processing.
  		//	We're done now, so remove it.
  		//	Kill progress display
  		m_pMainScansTaskInfo->Kill();
  
  		//	Kill triggers deletion of object
  		m_pMainScansTaskInfo = NULL;
  	}
  	
  	m_eLastScanType = m_eScanType;
  	m_eScanType = st_ReadyForNextScanType;
  }
  
  
  // ---------------------------------------------------------------------------
  //		* StopAllScanning											 [Public]
  // ---------------------------------------------------------------------------
  //	Stops all scanning
  
  void
  X1EmailScanner::StopAllScanning()
  {
  	if (m_eScanState != ss_Stopped)
  	{
  		if (m_pMainScansTaskInfo)
  		{
  			//	We have task info up from doing main scan processing.
  			//	We're done now, so remove it.
  			//	Kill progress display
  			m_pMainScansTaskInfo->Kill();
  
  			//	Kill triggers deletion of object
  			m_pMainScansTaskInfo = NULL;
  		}
  		
  		//	Mark ourselves stopped
  		m_eScanState = ss_Stopped;
  
  		//	No current or next scan types anymore
  		m_eScanType = st_None;
  		m_eNextScanType = st_None;
  	
  		//	Output information	
  		TRACE("X1 scan stopped\n");
  		PutDebugLog( DEBUG_MASK_SEARCH, "Stopping search indexing" );
  
  		if ( (m_eLastScanType == st_AddOrUpdateItemsScan) ||
  			 (m_eLastScanType == st_CullingScan) )
  		{
  			//	Notify Search Manager that we're done
  			SearchManager::Instance()->NotifyIndexingStatus(m_eScanType, m_eNextScanType, "");
  		}
  	}
  }
  
  
  // ---------------------------------------------------------------------------
  //		* NotifySumRemoval											 [Public]
  // ---------------------------------------------------------------------------
  //	Called by SearchManager when a summary is being removed from a given TOC.
  
  void
  X1EmailScanner::NotifySumRemoval(
  	const CTocDoc *				in_pTocDoc,
  	POSITION					in_posSum)
  {
  	for (ScanDataEnum i = sd_MainScans; i <= sd_IndexUpdateActions; )
  	{
  		if ( m_pCurrentTocDoc[i] && (in_pTocDoc == m_pCurrentTocDoc[i]) &&
  			 m_posCurrentSummary[i] && (in_posSum == m_posCurrentSummary[i]) )
  		{
  			//	User is deleting the summary that we would have looked at next.
  			//	Move on to the next summary now.
  			CSumList &		listSums = m_pCurrentTocDoc[i]->GetSumList();
  
  			listSums.GetNext( m_posCurrentSummary[i] );
  		}
  
  		//	Incrementing this was a pain, but this should do it
  		i = static_cast<ScanDataEnum>( static_cast<int>(i) + 1 );
  	}
  }
  
  
  // ---------------------------------------------------------------------------
  //		* NotifyTemporarilyCloseMailbox								 [Public]
  // ---------------------------------------------------------------------------
  //	Called by SearchManager when we need to close a specified mailbox file
  //	during mailbox compaction to make sure that we don't have the mailbox
  //	open so that we don't prevent renaming the temporary mailbox to the
  //	final name.
  
  void
  X1EmailScanner::NotifyTemporarilyCloseMailbox(
  	const char *				in_szMailboxRelativePath)
  {
  	//	Check to see if the current mailbox is affected by comparing the
  	//	path of the mailbox being used by the main scan (if any).
  	if ( m_pCurrentTocDoc[sd_MainScans] &&
  		 (m_szMailboxRelativePath[sd_MainScans].CompareNoCase(in_szMailboxRelativePath) == 0) )
  	{
  		//	Close only the current mailbox file. We'll reopen it the next
  		//	we idle (if we can), by which time Eudora will be done
  		//	with compaction.
  		m_CurrentMailboxFile[sd_MainScans].Close();
  	}
  
  	//	Check to see if the current mailbox is affected by comparing the
  	//	path of the mailbox being used by the index update scan (if any).
  	if ( m_pCurrentTocDoc[sd_IndexUpdateActions] &&
  		 (m_szMailboxRelativePath[sd_IndexUpdateActions].CompareNoCase(in_szMailboxRelativePath) == 0) )
  	{
  		//	Close only the current mailbox file. We'll reopen it the next
  		//	we idle (if we can), by which time Eudora will be done
  		//	with compaction.
  		m_CurrentMailboxFile[sd_IndexUpdateActions].Close();
  	}	
  }
  
  
  // ---------------------------------------------------------------------------
  //		* NotifyRestartProcessingMailbox							 [Public]
  // ---------------------------------------------------------------------------
  //	Called when we need to restart processing a specific mailbox because it
  //	has been sorted (so the summaries no longer have the same positions).
  
  void
  X1EmailScanner::NotifyRestartProcessingMailbox(
  	const char *				in_szMailboxRelativePath)
  {
  	//	Check to see if the current mailbox for the main scan is affected.
  	if ( m_pCurrentTocDoc[sd_MainScans] &&
  		 (m_szMailboxRelativePath[sd_MainScans].CompareNoCase(in_szMailboxRelativePath) == 0) )
  	{		
  		//	Restart summary iteration
  		CSumList &		listSums = m_pCurrentTocDoc[sd_MainScans]->GetSumList();
  		m_posCurrentSummary[sd_MainScans] = listSums.GetHeadPosition();
  		m_posLastSummaryAddAttempt[sd_MainScans] = NULL;
  		m_nCurrentMessage[sd_MainScans] = 0;
  	}
  }
  
  
  // ---------------------------------------------------------------------------
  //		* NotifyStopProcessingMailbox								 [Public]
  // ---------------------------------------------------------------------------
  //	Called when we need to stop processing a specific mailbox because it
  //	has been deleted, moved, or renamed.
  
  void
  X1EmailScanner::NotifyStopProcessingMailbox(
  	const char *				in_szMailboxRelativePath)
  {
  	//	Check to see if the current mailbox for the main scan is affected.
  	if ( m_pCurrentTocDoc[sd_MainScans] &&
  		 (m_szMailboxRelativePath[sd_MainScans].CompareNoCase(in_szMailboxRelativePath) == 0) )
  	{
  		//	Close the current mailbox
  		CloseCurrentMailbox(sd_MainScans, false);
  
  		if (m_eScanType == st_AddOrUpdateItemsScan)
  		{
  			//	Move to the next mailbox
  			NextMailbox();
  
  			//	Open the mailbox
  			if (m_pCurrentMailbox)
  				OpenCurrentMailbox(sd_MainScans);
  		}
  	}
  
  	//	Check to see if the mailbox for the index update scan is affected.
  	if ( m_pCurrentTocDoc[sd_IndexUpdateActions] &&
  		 (m_szMailboxRelativePath[sd_IndexUpdateActions].CompareNoCase(in_szMailboxRelativePath) == 0) )
  	{
  		//	Close the current mailbox. The addition of the appropriate index update
  		//	action will handle removing the current index update action if appropriate.
  		CloseCurrentMailbox(sd_IndexUpdateActions, false);
  	}	
  }
  
  
  // ---------------------------------------------------------------------------
  //		* InitAddOrUpdateItemsScan									 [Public]
  // ---------------------------------------------------------------------------
  //	Initializes mailbox iteration for add new items scan.
  
  void
  X1EmailScanner::InitAddOrUpdateItemsScan(
  	const char *				in_szContinueIndexingMailbox)
  {
  	//	Init mailbox scan members
  	m_szMailboxRelativePath[sd_MainScans].Empty();
  	m_pMailboxList = g_theMailboxDirector.GetMailboxList();
  	m_posCurrentMailbox = NULL;
  	m_pCurrentMailbox = NULL;
  	m_pCurrentTocDoc[sd_MainScans] = NULL;
  	m_posCurrentSummary[sd_MainScans] = NULL;
  	m_posLastSummaryAddAttempt[sd_MainScans] = NULL;
  	
  	//	Keep track of whether we found a mailbox with which to continue
  	CString					szResumeMailboxRelativePath;
  	POSITION				posResumeMailbox = NULL;
  	CPtrList *				pResumeMailboxList = g_theMailboxDirector.GetMailboxList();
  
  	ASSERT(pResumeMailboxList);
  	if (pResumeMailboxList)
  	{
  		posResumeMailbox = pResumeMailboxList->GetHeadPosition();
  		ASSERT(posResumeMailbox);
  	}
  
  	if (pResumeMailboxList && posResumeMailbox)
  	{
  		POSITION				posResumeMailbox = pResumeMailboxList->GetHeadPosition();
  		QCMailboxCommand *		pResumeMailbox = reinterpret_cast<QCMailboxCommand *>( pResumeMailboxList->GetNext(posResumeMailbox) );
  
  		//	Init mailbox scan to look for mailbox to continue scan
  		//	and count the total number of mailboxes
  		m_posCurrentMailbox = m_pMailboxList->GetHeadPosition();
  		m_pCurrentMailbox = reinterpret_cast<QCMailboxCommand *>( m_pMailboxList->GetNext(m_posCurrentMailbox) );
  		m_nCurrentMailbox = 0;
  		m_nTotalMailboxes = 0;
  
  		//	Get path of first mailbox, so that we can find it below
  		if (m_pCurrentMailbox)
  			m_szMailboxRelativePath[sd_MainScans] = static_cast<const char *>(m_pCurrentMailbox->GetPathname()) + EudoraDirLen;
  		
  		//	Attempt to find the mailbox that we were last scanning so that we can
  		//	resume our scan with it
  		while (m_pCurrentMailbox)
  		{
  			if ( in_szContinueIndexingMailbox && *in_szContinueIndexingMailbox &&
  				 (m_szMailboxRelativePath[sd_MainScans].CompareNoCase(in_szContinueIndexingMailbox) == 0) )
  			{
  				//	Remember all the mailbox information for the mailbox with which
  				//	we'll be resuming
  				szResumeMailboxRelativePath = m_szMailboxRelativePath[sd_MainScans];
  				pResumeMailboxList = m_pMailboxList;
  				posResumeMailbox = m_posCurrentMailbox;
  				pResumeMailbox = m_pCurrentMailbox;
  				m_nCurrentMailbox = m_nTotalMailboxes;
  			}
  			
  			NextMailbox();
  
  			m_nTotalMailboxes++;
  		}
  
  		//	Resume with the mailbox we found or the beginning of the list
  		m_szMailboxRelativePath[sd_MainScans] = szResumeMailboxRelativePath;
  		m_pMailboxList = pResumeMailboxList;
  		m_posCurrentMailbox = posResumeMailbox;
  		m_pCurrentMailbox = pResumeMailbox;
  		m_nCurrentMessage[sd_MainScans] = 0;
  
  		ASSERT(m_pCurrentMailbox);
  
  		//	Open the current mailbox so that we'll scan it instead of moving
  		//	on to the next mailbox.
  		if (m_pCurrentMailbox)
  		{
  			m_szMailboxRelativePath[sd_MainScans] = static_cast<const char *>(m_pCurrentMailbox->GetPathname()) + EudoraDirLen;
  			OpenCurrentMailbox(sd_MainScans);
  		}
  
  		//	Do we have progress displaying yet?
  		if (!m_pMainScansTaskInfo)
  		{
  			QCTaskManager *		pTaskManager = QCGetTaskManager();
  
  			if (pTaskManager)
  			{
  				//	Add task to task manager to display progress
  				m_pMainScansTaskInfo = DEBUG_NEW SearchManagerTaskInfo(true);
  				pTaskManager->Register(m_pMainScansTaskInfo);
  			}
  		}
  
  		if (m_pMainScansTaskInfo)
  		{
  			//	Set the total progress to the number of mailboxes.
  			m_pMainScansTaskInfo->SetTotal(m_nTotalMailboxes * 100);
  
  			//	Set where we're at with the progress
  			m_nSoFarBase[sd_MainScans] = m_nCurrentMailbox * 100;
  			m_pMainScansTaskInfo->SetSoFar( m_nSoFarBase[sd_MainScans] );
  
  			//	Set the title
  			m_pMainScansTaskInfo->SetTitle( CRString(IDS_ADD_UPDATE_SCAN_TITLE) );
  
  			//	Set the initial main progress text
  			CString		szMainText;
  			szMainText.Format(IDS_ADD_UPDATE_SCAN_MAILBOXES_REMAINING, m_nTotalMailboxes - m_nCurrentMailbox);
  			m_pMainScansTaskInfo->SetMainText(szMainText);
  		}
  	}
  }
  
  
  // ---------------------------------------------------------------------------
  //		* InitCullingScan											 [Public]
  // ---------------------------------------------------------------------------
  //	Initializes mailbox iteration for culling scan.
  
  void
  X1EmailScanner::InitCullingScan(
  	const char *				in_szContinueIndexingMailbox)
  {
  	m_pCurrentTocDoc[sd_MainScans] = NULL;
  	m_szMailboxRelativePath[sd_MainScans].Empty();
  
  	//	If we have an old list of indexed items, release it
  	if (m_pX1IndexedItems.GetInterfacePtr() != NULL)
  		m_pX1IndexedItems.Release();
  
  	try
  	{
  		//	Search the X1 email DB for all items
  		X1::IX1ItemResultPtr	pItemResult = NULL;
  		m_pEmailDB->FindItems(X1::ftContainsPrefix, _bstr_t(), SearchManager::efid_MessageID, &pItemResult);
  
  		ASSERT( pItemResult.GetInterfacePtr() );
  
  		if (pItemResult)
  		{
  			//	Set up what we want the search results to be sorted by
  			COleSafeArray	safeArraySortBy;
  			int *			pData;
  
  			V_VT(&safeArraySortBy) = VT_ARRAY | VT_INT;
  			V_ARRAY(&safeArraySortBy) = SafeArrayCreateVector(VT_INT, 0, 2);
  
  			//	Sort first by mailbox path, then by message ID
  			safeArraySortBy.AccessData( reinterpret_cast<void **>(&pData) );
  			pData[0] = SearchManager::efid_MailboxRelativePath;
  			pData[1] = SearchManager::efid_MessageID;
  			safeArraySortBy.UnaccessData();
  
  			//	Set up that we want the search results to be sorted forwards
  			COleSafeArray	safeArraySortOrder;
  
  			V_VT(&safeArraySortOrder) = VT_ARRAY | VT_INT;
  			V_ARRAY(&safeArraySortOrder) = SafeArrayCreateVector(VT_INT, 0, 2);
  
  			safeArraySortOrder.AccessData( reinterpret_cast<void **>(&pData) );
  			pData[0] = 0;
  			pData[1] = 0;
  			safeArraySortOrder.UnaccessData();
  
  			//	Actually get the search results
  			pItemResult->GetItemList(safeArraySortBy, safeArraySortOrder, &m_pX1IndexedItems);
  
  			ASSERT( m_pX1IndexedItems.GetInterfacePtr() );
  
  			if (m_pX1IndexedItems)
  			{
  				//	Get the number of items in the X1 email DB
  				m_pX1IndexedItems->get_count(&m_nNumX1IndexedItems);
  
  				//	By default we'll start the culling scan with the first indexed item
  				m_nX1IndexedItem = 0;
  
  				if (in_szContinueIndexingMailbox && *in_szContinueIndexingMailbox)
  				{
  					//	Attempt to find the mailbox that we were last scanning so that we can
  					//	resume our scan with it
  					_bstr_t		strLastMailbox;
  
  					for (long i = 0; i < m_nNumX1IndexedItems; i++)
  					{
  						long		nX1ItemNum = m_pX1IndexedItems->Getitem(i);
  
  						BOOL		bIsValidIndex = m_pEmailDB->IsValid(nX1ItemNum);
  
  						ASSERT(bIsValidIndex);
  
  						if (bIsValidIndex)
  						{
  							//	Compare native strings first to save time
  							if ( strLastMailbox != m_pEmailDB->GetFieldString(SearchManager::efid_MailboxRelativePath, nX1ItemNum) )
  							{
  								//	Remember the native string for this mailbox
  								strLastMailbox = m_pEmailDB->GetFieldString(SearchManager::efid_MailboxRelativePath, nX1ItemNum);
  								
  								//	Get the CString version of the mailbox path string
  								CString		szMailboxRelativePath = static_cast<char *>(strLastMailbox);
  
  								//	Did we find the correct mailbox yet?
  								if (szMailboxRelativePath.CompareNoCase(in_szContinueIndexingMailbox) == 0)
  								{
  									//	Start with the first item for this mailbox
  									m_nX1IndexedItem = i;
  
  									//	No need to look any further
  									break;
  								}
  							}
  						}
  					}
  				}
  
  				//	Do we have progress displaying yet?
  				if (!m_pMainScansTaskInfo)
  				{
  					QCTaskManager *		pTaskManager = QCGetTaskManager();
  
  					if (pTaskManager)
  					{
  						//	Add task to task manager to display progress
  						m_pMainScansTaskInfo = DEBUG_NEW SearchManagerTaskInfo(true);
  						pTaskManager->Register(m_pMainScansTaskInfo);
  					}
  				}
  
  				if (m_pMainScansTaskInfo)
  				{
  					//	Set the total progress to the number of mailboxes.
  					m_pMainScansTaskInfo->SetTotal(m_nNumX1IndexedItems);
  
  					//	Set where we're at with the progress
  					m_nSoFarBase[sd_MainScans] = m_nX1IndexedItem;
  					m_pMainScansTaskInfo->SetSoFar( m_nSoFarBase[sd_MainScans] );
  
  					//	Set the title
  					m_pMainScansTaskInfo->SetTitle( CRString(IDS_CULLING_SCAN_TITLE) );
  
  					//	Set the initial main progress text
  					CString		szMainText;
  					szMainText.Format(IDS_CULLING_SCAN_ITEMS_REMAINING, m_nNumX1IndexedItems - m_nX1IndexedItem);
  					m_pMainScansTaskInfo->SetMainText(szMainText);
  				}
  			}
  			else
  			{
  				PutDebugLog( DEBUG_MASK_SEARCH, "Stopping search indexing (culling) because there are no items in the index" );
  				
  				//	Stop the culling scan
  				StopCurrentScan();
  			}
  		}
  	}
  	catch (_com_error & e)
  	{
  		//	Get developer's attention
  		ASSERT(!"X1 COM Error");
  
  		SearchManager::Utils::LogError( "COM", "InitCullingScan", e.ErrorMessage() );
  
  		//	Stop the culling scan
  		StopCurrentScan();
  	}
  	catch (CException * pException)
  	{
  		//	Get developer's attention
  		ASSERT(!"X1 CException");
  		
  		//	Log error
  		SearchManager::Utils::LogCException(pException, "InitCullingScan");
  
  		//	Delete exception object
  		pException->Delete();
  
  		//	Stop the culling scan
  		StopCurrentScan();
  	}
  }
  
  
  // ---------------------------------------------------------------------------
  //		* NextMailbox												 [Public]
  // ---------------------------------------------------------------------------
  //	Moves m_pCurrentMailbox to the next mailbox that could contain messages
  //	in a depth first traversal of the mailbox tree.
  
  void
  X1EmailScanner::NextMailbox()
  {	
  	if (!m_pMailboxList || !m_pCurrentMailbox)
  	{
  		//	We've scanned through all mailboxes
  		//	Stop the current scan
  		StopCurrentScan();
  
  		//	Return - we're done
  		return;
  	}
  
  	//	Depth first traversal of mailboxes
  	QCMailboxCommand *		pStartMailbox = m_pCurrentMailbox;
  
  	while (m_pCurrentMailbox)
  	{
  		//	Dive deeper. Stop when:
  		//	* We find a mailbox that can contain messages
  		//	* We find a mailbox that doesn't have any children (leaf)
  		while ( m_pCurrentMailbox && m_pCurrentMailbox->CanHaveChildren() &&
  				!m_pCurrentMailbox->GetChildList().IsEmpty() )
  		{
  			if ( m_pCurrentMailbox && (m_pCurrentMailbox != pStartMailbox) &&
  				 m_pCurrentMailbox->CanContainMessages() )
  			{
  				break;
  			}
  			
  			//	Get children
  			m_pMailboxList = &m_pCurrentMailbox->GetChildList();
  
  			//	Get first child
  			m_posCurrentMailbox = m_pMailboxList->GetHeadPosition();
  			m_pCurrentMailbox = reinterpret_cast<QCMailboxCommand *>( m_pMailboxList->GetNext(m_posCurrentMailbox) );
  		}
  
  		//	Sanity check
  		ASSERT(m_pMailboxList);
  		ASSERT(m_pCurrentMailbox);
  
  		//	Did we find a mailbox that can contain messages by diving deeper?
  		//	If so, then we're done.
  		if ( (m_pCurrentMailbox != pStartMailbox) && m_pCurrentMailbox->CanContainMessages() )
  			break;
  
  		//	Move to next mailbox when we can't dive deeper
  		if (m_posCurrentMailbox)
  		{
  			//	More siblings - get the next sibling
  			m_pCurrentMailbox = reinterpret_cast<QCMailboxCommand *>( m_pMailboxList->GetNext(m_posCurrentMailbox) );
  		}
  		else
  		{
  			//	No more siblings - move up the tree and find the sibling of an ancestor
  
  			//	Move up a level - find parent of current mailbox
  			m_pCurrentMailbox = g_theMailboxDirector.FindParent(m_pCurrentMailbox);
  			m_posCurrentMailbox = NULL;
  
  			while (m_pCurrentMailbox)
  			{
  				//	Get parent of current mailbox so that we can find a sibling
  				QCMailboxCommand *	pParent = g_theMailboxDirector.FindParent(m_pCurrentMailbox);
  
  				if (pParent)
  				{
  					//	Get parent's children (our siblings)
  					m_pMailboxList = &pParent->GetChildList();
  				}
  				else
  				{
  					//	Assume that we're dealing with a root entry, therefore our siblings
  					//	are found in the root mailbox list.
  					m_pMailboxList = g_theMailboxDirector.GetMailboxList();
  				}
  
  				//	Get parent's children (our siblings)
  				QCMailboxCommand *	pMailbox = NULL;
  				
  				m_posCurrentMailbox = m_pMailboxList->GetHeadPosition();
  
  				//	Look for position of current mailbox in siblings
  				while (m_posCurrentMailbox)
  				{
  					pMailbox = reinterpret_cast<QCMailboxCommand *>( m_pMailboxList->GetNext(m_posCurrentMailbox) );
  					
  					//	We found the current mailbox
  					if (pMailbox == m_pCurrentMailbox)
  						break;
  				}
  
  				//	Sanity check that we found the current mailbox
  				ASSERT(pMailbox == m_pCurrentMailbox);
  
  				if ( (pMailbox == m_pCurrentMailbox) && m_posCurrentMailbox )
  				{
  					//	We have another sibling - move on to it
  					m_pCurrentMailbox = reinterpret_cast<QCMailboxCommand *>( m_pMailboxList->GetNext(m_posCurrentMailbox) );
  
  					//	Break so that we can explore the sibling
  					break;
  				}
  				else
  				{
  					//	No siblings - move up another level. If we were already at
  					//	the root level this will terminate our traversal.
  					m_pCurrentMailbox = pParent;
  				}
  			}
  		}
  	}
  
  	//	Set up the new mailbox path
  	if (m_pCurrentMailbox)
  		m_szMailboxRelativePath[sd_MainScans] = static_cast<const char *>(m_pCurrentMailbox->GetPathname()) + EudoraDirLen;
  	else
  		m_szMailboxRelativePath[sd_MainScans].Empty();
  
  	if (m_pMainScansTaskInfo)
  	{
  		//	Update the progress display
  		m_nCurrentMessage[sd_MainScans] = 0;
  		m_nCurrentMailbox++;
  		m_nSoFarBase[sd_MainScans] = m_nCurrentMailbox * 100;
  		m_pMainScansTaskInfo->SetSoFar( m_nSoFarBase[sd_MainScans] );
  
  		//	Update main progress text
  		CString		szMainText;
  		szMainText.Format(IDS_ADD_UPDATE_SCAN_MAILBOXES_REMAINING, m_nTotalMailboxes - m_nCurrentMailbox);
  		m_pMainScansTaskInfo->SetMainText(szMainText);
  	}
  }
  
  
  // ---------------------------------------------------------------------------
  //		* AddOrUpdateMailboxMessages								 [Public]
  // ---------------------------------------------------------------------------
  //	Scan to add the next item to the index.
  
  bool
  X1EmailScanner::AddOrUpdateMailboxMessages(
  	ScanDataEnum				in_eScanData)
  {
  	//	Verify that the mailbox is open. It could be closed if we were told to
  	//	close it for compaction (via NotifyTemporarilyCloseMailbox) or if we
  	//	failed when we tried to open it.
  	if (m_CurrentMailboxFile[in_eScanData].IsOpen() != S_OK)
  	{
  		CTocDoc *	pTocDoc = m_pCurrentTocDoc[in_eScanData];
  		
  		//	Make sure that mailbox still exists
  		if ( FileExistsMT(pTocDoc->GetMBFileName()) )
  		{
  			//	Attempt to open or re-open mailbox - currently we never give up
  			m_CurrentMailboxFile[in_eScanData].Open(pTocDoc->GetMBFileName(), O_RDONLY);
  		}
  		else
  		{
  			//	The mailbox doesn't exist anymore. Close our TocDoc, etc.
  			CloseCurrentMailbox(in_eScanData);
  
  			//	Reset m_posCurrentSummary so that we move on
  			m_posCurrentSummary[in_eScanData] = NULL;
  		}
  
  		//	Return now so that we don't use up too much time
  		return (m_posCurrentSummary[in_eScanData] == NULL);
  	}
  
  	try
  	{
  		CSumList &			listSums = m_pCurrentTocDoc[in_eScanData]->GetSumList();
  		CSummary *			pSum = listSums.GetAt( m_posCurrentSummary[in_eScanData] );
  		X1::IX1OneUsePtr	pOneUse;
  
  		if (!pSum)
  		{
  			//	Unexpected failure. Log failure and move on so that we don't get stuck.
  			ASSERT(!"Getting summary failed");
  			PutDebugLog( DEBUG_MASK_SEARCH, "Failed to get summary while doing search indexing" );
  			listSums.GetNext( m_posCurrentSummary[in_eScanData] );
  		}
  		else if ( m_pEmailDB && m_pEmailDB->CanModifyIndex(&pOneUse) )
  		{
  			long	nX1ItemNum = -1;
  			UINT	nProgressResID;
  
  			if ( m_bCheckingMessages[in_eScanData] )
  			{
  				//	Do we need to check to see if the current summary is in the index, and if so
  				//	is the current summary already in the index?
  				if ( IsIndexed(m_szMailboxRelativePath[in_eScanData], pSum->GetUniqueMessageId(), nX1ItemNum) )
  				{
  					//	Does the current summary match what's in the index?
  					if ( DoesIndexedItemMatchSummary(pSum, nX1ItemNum) )
  					{
  						//	It matches, so we don't need to add it to the index
  						pSum->SetShouldBeAddedToIndex(false);
  					}
  					else
  					{
  						//	It doesn't match - remove it from the index
  						m_pEmailDB->RemoveFromIndex(pOneUse, nX1ItemNum);
  
  						//	Set that we need to re-add it to the index
  						pSum->SetShouldBeAddedToIndex(true);
  
  						//	Remember that we need to add one or more messages
  						m_bNeedToAddMessages[in_eScanData] = true;
  					}
  				}
  				else
  				{
  					//	Not indexed, so remember that we need to add it to the index
  					pSum->SetShouldBeAddedToIndex(true);
  
  					//	Remember that we need to add one or more messages
  					m_bNeedToAddMessages[in_eScanData] = true;
  				}
  
  				//	Move on to the next message
  				m_nCurrentMessage[in_eScanData]++;
  				listSums.GetNext(m_posCurrentSummary[in_eScanData]);
  
  				//	Use the appropriate progress string
  				nProgressResID = IDS_INDEXING_VERIFYING_MESSAGES_REMAINING;
  			}
  			else
  			{
  				//	If there were already some messages indexed for this mailbox,
  				//	then we checked this message to see if it should be added to
  				//	the index. If it shouldn't be added, then we're already done.
  				bool	bDone = m_bCheckForIndexedMessages[in_eScanData] && !pSum->ShouldBeAddedToIndex();
  
  				//	Remember whether or not this is our second add attempt
  				bool	bSecondAddAttemptForSummary = ( m_posCurrentSummary[in_eScanData] == m_posLastSummaryAddAttempt[in_eScanData] );
  
  				//	If it's not in the index, then add it to the index
  				if ( !bDone && m_pEmailDB->CanModifyIndex(&pOneUse) )
  				{
  					bool	bWaitingForX1 = false;
  
  					bDone = AddToIndex(in_eScanData, pOneUse, pSum, bWaitingForX1);
  
  					if (!bDone && !bWaitingForX1)
  					{
  						//	Adding to the index failed and it wasn't because we were
  						//	waiting for X1. We'll try once more and then move on.
  						m_posLastSummaryAddAttempt[in_eScanData] = m_posCurrentSummary[in_eScanData];
  					}
  				}
  
  				//	If we're done or it's our second attempt, then move on to the next summary
  				if (bDone || bSecondAddAttemptForSummary)
  				{
  					m_nCurrentMessage[in_eScanData]++;
  					listSums.GetNext(m_posCurrentSummary[in_eScanData]);
  
  					if (!bDone)
  					{
  						CString		szLogEntry;
  
  						szLogEntry.Format( "Failed to add message to index twice while doing search indexing (Subject: %s)",
  											pSum->m_Subject );
  						PutDebugLog(DEBUG_MASK_SEARCH, szLogEntry);
  					}
  				}
  
  				//	Use the appropriate progress string
  				nProgressResID = IDS_INDEXING_ADDING_MESSAGES_REMAINING;
  			}
  
  			SearchManagerTaskInfo *		pTaskInfo = NULL;
  			CString						szMailboxName;
  
  			if ( (in_eScanData == sd_MainScans) && m_pMainScansTaskInfo )
  			{
  				pTaskInfo = m_pMainScansTaskInfo;
  				szMailboxName = m_pCurrentMailbox->GetName();
  			}
  			else if ( (in_eScanData == sd_IndexUpdateActions) && m_pUpdateActionsTaskInfo )
  			{
  				pTaskInfo = m_pUpdateActionsTaskInfo;
  				szMailboxName = m_szMailboxName;
  			}
  
  			if (pTaskInfo)
  			{
  				//	Update progress display info
  				CString		szProgress;
  				long		nTotalMessages = listSums.GetCount();
  				long		nMessagesRemaining = listSums.GetCount() - m_nCurrentMessage[in_eScanData];
  
  				//	Saw a negative number once - check to see if I see it again and why
  				ASSERT(nMessagesRemaining >= 0);
  
  				//	If it went negative, just display 0
  				if (nMessagesRemaining < 0)
  					nMessagesRemaining = 0;
  
  				szProgress.Format(nProgressResID, szMailboxName, nMessagesRemaining);
  				pTaskInfo->Progress(szProgress);
  				
  				//	Update the progress bar
  				long	nSoFar = m_nSoFarBase[in_eScanData] + m_nCurrentMessage[in_eScanData] * 100 / nTotalMessages;
  				pTaskInfo->SetSoFar(nSoFar);
  
  				//	Update persona
  				pTaskInfo->SetPersona( pSum->GetPersona() );
  			}
  
  			if ( m_bCheckingMessages[in_eScanData] && (m_posCurrentSummary[in_eScanData] == NULL) )
  			{
  				//	We've finished checking the messages
  				m_bCheckingMessages[in_eScanData] = false;
  
  				if ( m_bNeedToAddMessages[in_eScanData] )
  				{
  					//	Scan through mailbox again, this time actually adding the messages
  					m_posCurrentSummary[in_eScanData] = listSums.GetHeadPosition();
  					m_posLastSummaryAddAttempt[in_eScanData] = NULL;
  					m_nCurrentMessage[in_eScanData] = 0;
  				}
  			}
  		}
  	}
  	catch (_com_error & e)
  	{
  		//	Get developer's attention
  		ASSERT(!"X1 COM Error");
  
  		SearchManager::Utils::LogError( "COM", "AddOrUpdateMailboxMessages", e.ErrorMessage() );
  	}
  	catch (CException * pException)
  	{
  		//	Get developer's attention
  		ASSERT(!"X1 CException");
  		
  		//	Log error
  		SearchManager::Utils::LogCException(pException, "AddOrUpdateMailboxMessages");
  
  		//	Delete exception object
  		pException->Delete();
  	}
  
  	//	Return whether or not we're done.
  	return (m_posCurrentSummary[in_eScanData] == NULL);
  }
  
  
  // ---------------------------------------------------------------------------
  //		* DoAddOrUpdateItemsScan									 [Public]
  // ---------------------------------------------------------------------------
  //	Scan to add the next item to the index.
  
  void
  X1EmailScanner::DoAddOrUpdateItemsScan()
  {	
  	if ( !m_posCurrentSummary[sd_MainScans] )
  	{
  		//	Close the current mailbox
  		CloseCurrentMailbox(sd_MainScans);
  		
  		//	Move to the next mailbox
  		NextMailbox();
  
  		//	Open the mailbox
  		if (m_pCurrentMailbox)
  			OpenCurrentMailbox(sd_MainScans);
  
  		//	Return now so that we don't use up too much time
  		return;
  	}
  	
  	AddOrUpdateMailboxMessages(sd_MainScans);
  }
  
  
  // ---------------------------------------------------------------------------
  //		* DoCullingScan												 [Public]
  // ---------------------------------------------------------------------------
  //	Culling scan to check the next item to see if it should be removed
  //	from the index.
  
  void
  X1EmailScanner::DoCullingScan()
  {
  	try
  	{
  		//	Update progress display info
  		if (m_pMainScansTaskInfo)
  			m_pMainScansTaskInfo->Progress( CRString(IDS_CULLING_SCAN_ACTIVE) );
  		
  		if (m_nX1IndexedItem >= m_nNumX1IndexedItems)
  		{
  			//	Hit the end of X1 indexed items - stop the culling scan
  			StopCurrentScan();
  		}
  
  		if (m_nX1IndexedItem < m_nNumX1IndexedItems)
  		{
  			long		nX1ItemNum = m_pX1IndexedItems->Getitem(m_nX1IndexedItem);
  
  			BOOL		bIsValidIndex = m_pEmailDB->IsValid(nX1ItemNum);
  
  			ASSERT(bIsValidIndex);
  
  			if (bIsValidIndex)
  			{
  				CString		szMailboxRelativePath = static_cast<char *>(
  								m_pEmailDB->GetFieldString(SearchManager::efid_MailboxRelativePath, nX1ItemNum) );
  				
  				if (szMailboxRelativePath.CompareNoCase(m_szMailboxRelativePath[sd_MainScans]) != 0)
  				{
  					//	Close the current mailbox
  					CloseCurrentMailbox(sd_MainScans);
  
  					//	Switch to the new mailbox
  					m_szMailboxRelativePath[sd_MainScans] = szMailboxRelativePath;
  
  					//	Open the new mailbox
  					OpenCurrentMailbox(sd_MainScans);
  
  					//	Don't move on to the next X1 indexed item so that we can process
  					//	it the next Idle.
  				}
  				else
  				{
  					CString		szMessageID = static_cast<char *>(
  									m_pEmailDB->GetFieldString(SearchManager::efid_MessageID, nX1ItemNum) );
  					long		nMessageID = atol(szMessageID);
  					
  					//	In case of errors with the mailbox file:
  					//	* If the mailbox file doesn't exist - remove the item
  					//	* If the mailbox file exists, but we couldn't open the TOC doc - leave
  					//	  the item alone and move on to the next item
  					bool		bRemoveItem = false;
  
  					if (!m_bMailboxFileExists)
  					{
  						//	The mailbox file for the current item doesn't even exist - remove the item
  						bRemoveItem = true;
  					}
  					else if (m_pCurrentTocDoc[sd_MainScans])
  					{
  						//	Check to see if we should remove the current item
  						CSumList &	listSums = m_pCurrentTocDoc[sd_MainScans]->GetSumList();
  						CSummary *	pSummary = listSums.GetByMessageId(nMessageID);
  
  						//	If we couldn't find the summary in the specified mailbox,
  						//	then remove the item
  						if (!pSummary)
  							bRemoveItem = true;
  					}
  
  					if (bRemoveItem)
  					{
  						//	Queue action to remove the item
  						SearchManager::Instance()->AddIndexUpdateAction(
  								SearchManager::ua_RemoveMessage,
  								szMailboxRelativePath,
  								nMessageID );
  
  						//	We don't call NotifyIndexingStatus or attempt to trigger saving
  						//	the SearchManager actions here, because they'll be recreated
  						//	if we crash since we pick up at the start of the most recent mailbox
  						//	being scanned. NotifyIndexingStatus is called when we switch to the
  						//	next mailbox, which suits us just fine.
  					}
  					
  					//	Move on to the next X1 indexed item
  					m_nX1IndexedItem++;
  				}
  			}
  			else
  			{
  				//	Move on to the next X1 indexed item
  				m_nX1IndexedItem++;
  			}
  		}
  
  		if (m_pMainScansTaskInfo)
  		{
  			//	Set where we're at with the progress
  			m_nSoFarBase[sd_MainScans] = m_nX1IndexedItem;
  			m_pMainScansTaskInfo->SetSoFar( m_nSoFarBase[sd_MainScans] );
  
  			//	Set the initial main progress text
  			CString		szMainText;
  			szMainText.Format(IDS_CULLING_SCAN_ITEMS_REMAINING, m_nNumX1IndexedItems - m_nX1IndexedItem);
  			m_pMainScansTaskInfo->SetMainText(szMainText);
  		}
  	}
  	catch (_com_error & e)
  	{
  		//	Get developer's attention
  		ASSERT(!"X1 COM Error");
  
  		SearchManager::Utils::LogError( "COM", "DoCullingScan", e.ErrorMessage() );
  
  		//	Stop the culling scan
  		StopCurrentScan();
  	}
  	catch (CException * pException)
  	{
  		//	Get developer's attention
  		ASSERT(!"X1 CException");
  		
  		//	Log error
  		SearchManager::Utils::LogCException(pException, "DoCullingScan");
  
  		//	Delete exception object
  		pException->Delete();
  
  		//	Stop the culling scan
  		StopCurrentScan();
  	}
  }
  
  
  // ---------------------------------------------------------------------------
  //		* ProcessNextIndexUpdateItem								 [Public]
  // ---------------------------------------------------------------------------
  //	Processes next index update item.
  
  void
  X1EmailScanner::ProcessNextIndexUpdateItem()
  {
  	try
  	{
  		X1::IX1OneUsePtr	pOneUse;
  
  		if ( m_pEmailDB && m_pEmailDB->CanModifyIndex(&pOneUse) )
  		{
  			bool					bDoneProcessingCurrentAction = true;
  			
  			//	Get the next index update action
  			IndexUpdateAction *		pUpdateAction = SearchManager::Instance()->GetFirstIndexUpdateAction();
  
  			ASSERT(pUpdateAction);
  
  			if (pUpdateAction)
  			{
  				switch ( pUpdateAction->GetUpdateAction() )
  				{
  					case SearchManager::ua_RemoveMessage:
  					case SearchManager::ua_UpdateMessage:
  						bDoneProcessingCurrentAction = ProcessRemoveOrUpdateMessage(pUpdateAction, pOneUse);
  						break;
  
  					case SearchManager::ua_AddMessage:
  						bDoneProcessingCurrentAction = ProcessAddMessage(pUpdateAction, pOneUse);
  						break;
  
  					case SearchManager::ua_RemoveMailbox:
  					case SearchManager::ua_ReindexMailbox:
  						bDoneProcessingCurrentAction = ProcessRemoveOrReindexMailbox(pUpdateAction, pOneUse);
  						break;
  
  					case SearchManager::ua_AddMailbox:
  						bDoneProcessingCurrentAction = ProcessAddMailbox(pUpdateAction, pOneUse);
  						break;
  				}
  			}
  
  			if (bDoneProcessingCurrentAction)
  			{
  				//	Done processing index update action so pop it
  				SearchManager::Instance()->PopIndexUpdateAction();
  
  				if (m_pUpdateActionsTaskInfo)
  				{
  					m_nCurrentUpdateAction++;
  					m_nSoFarBase[sd_IndexUpdateActions] = m_nCurrentUpdateAction * 100;
  					m_pUpdateActionsTaskInfo->SetSoFar( m_nSoFarBase[sd_IndexUpdateActions] );
  					m_nTotalMessagesToRemove = 0;
  				}
  			}
  		}
  	}
  	catch (_com_error & e)
  	{
  		//	Get developer's attention
  		ASSERT(!"X1 COM Error");
  
  		SearchManager::Utils::LogError( "COM", "ProcessNextIndexUpdateItem", e.ErrorMessage() );
  
  		//	We had an error while processing the current index update item.
  		//	Move on to the next item so that we don't get stuck trying to
  		//	process the current item. I don't like this at all, but I'm not
  		//	sure how else to avoid getting stuck.
  		SearchManager::Instance()->PopIndexUpdateAction();
  	}
  	catch (CException * pException)
  	{
  		//	Get developer's attention
  		ASSERT(!"X1 CException");
  		
  		//	Log error
  		SearchManager::Utils::LogCException(pException, "ProcessNextIndexUpdateItem");
  
  		//	Delete exception object
  		pException->Delete();
  
  		//	We had an error while processing the current index update item.
  		//	Move on to the next item so that we don't get stuck trying to
  		//	process the current item. I don't like this at all, but I'm not
  		//	sure how else to avoid getting stuck.
  		SearchManager::Instance()->PopIndexUpdateAction();
  	}
  }
  
  
  // ---------------------------------------------------------------------------
  //		* ProcessRemoveOrUpdateMessage								 [Public]
  // ---------------------------------------------------------------------------
  //	Processes removal or update of a message.
  
  bool
  X1EmailScanner::ProcessRemoveOrUpdateMessage(
  	X1EmailScanner::IndexUpdateAction *			in_pUpdateAction,
  	X1::IX1OneUsePtr &							in_pOneUse)
  {
  	UpdateActionEnum	eUpdateAction = in_pUpdateAction->GetUpdateAction();
  	bool				bDoneProcessingCurrentAction = true;
  	
  	//	Update starts functioning just like a remove - it removes
  	//	the message from the index. After it's done removing, however,
  	//	it transforms itself into an add (see below).
  	//
  	//	Remove searches for the specified message and removes it from
  	//	the index if found.
  	long	nX1ItemNum = -1;
  
  	//	Is the specified message still in the index?
  	if ( IsIndexed(in_pUpdateAction->GetMailboxRelativePath(), in_pUpdateAction->GetMessageID(), nX1ItemNum) )
  	{
  		//	Remove it from the index
  		m_pEmailDB->RemoveFromIndex(in_pOneUse, nX1ItemNum);
  	}
  
  	if (eUpdateAction == SearchManager::ua_UpdateMessage)
  	{
  		//	We removed the outdated indexed information for the message.
  		//	In the next idle we'll want to add the updated information
  		//	for the message.
  		in_pUpdateAction->SetUpdateAction(SearchManager::ua_AddMessage);
  		bDoneProcessingCurrentAction = false;
  		
  		//	Remember that this was an update so that we can display a
  		//	better progress message.
  		in_pUpdateAction->SetIsUpdate(true);
  	}
  	else if (m_pUpdateActionsTaskInfo)
  	{
  		//	Display progress for removing
  		m_pUpdateActionsTaskInfo->Progress( CRString(IDS_UPDATING_INDEX_PROGRESS_REMOVING_MESSAGE) );
  	}
  
  	return bDoneProcessingCurrentAction;
  }
  
  
  // ---------------------------------------------------------------------------
  //		* ProcessAddMessage											 [Public]
  // ---------------------------------------------------------------------------
  //	Processes message addition.
  
  bool
  X1EmailScanner::ProcessAddMessage(
  	X1EmailScanner::IndexUpdateAction *			in_pUpdateAction,
  	X1::IX1OneUsePtr &							in_pOneUse)
  {
  	const CString &		szMailboxRelativePath = in_pUpdateAction->GetMailboxRelativePath();
  	bool				bDoneProcessingCurrentAction = true;
  
  	if (szMailboxRelativePath.CompareNoCase(m_szMailboxRelativePath[sd_IndexUpdateActions]) != 0)
  	{
  		//	Mailbox for the current action is different than mailbox
  		//	for the previous action. Close the current mailbox,
  		//	move on to the next mailbox and open it.
  		CloseCurrentMailbox(sd_IndexUpdateActions);
  		m_szMailboxRelativePath[sd_IndexUpdateActions] = szMailboxRelativePath;
  		OpenCurrentMailbox(sd_IndexUpdateActions);
  
  		//	We'll actually process this action the next time we're
  		//	idled. For now we're not done.
  		bDoneProcessingCurrentAction = false;
  
  		//	Reset the last message ID that we attempted to add to 0
  		m_nMessageIDLastAddAttempt = 0;
  	}
  	else if (m_CurrentMailboxFile[sd_IndexUpdateActions].IsOpen() != S_OK)
  	{
  		//	Mailbox file isn't open. It could be closed if we were told to close
  		//	it in order for compaction to work (via NotifyTemporarilyCloseMailbox)
  		//	or if opening it failed previously.
  		CTocDoc *	pTocDoc = m_pCurrentTocDoc[sd_IndexUpdateActions];
  		
  		//	Make sure that mailbox file still exists
  		if ( pTocDoc && FileExistsMT(pTocDoc->GetMBFileName()) )
  		{
  			//	Attempt to open or re-open mailbox - currently we never give up
  			m_CurrentMailboxFile[sd_IndexUpdateActions].Open(pTocDoc->GetMBFileName(), O_RDONLY);
  		}
  		else
  		{
  			//	The mailbox doesn't exist anymore. Close our TocDoc, etc.
  			CloseCurrentMailbox(sd_IndexUpdateActions);
  		}
  	}
  	else
  	{
  		//	We already opened (or attempted to open) the mailbox for
  		//	the current action. Now attempt to process the addition.
  		CString			szProgress;
  
  		//	Check to see if we were able to open the current mailbox
  		if ( m_bMailboxFileExists[sd_IndexUpdateActions] &&
  			 m_pCurrentTocDoc[sd_IndexUpdateActions] )
  		{
  			//	OK we have an open mailbox, we'll want to get the summary
  			CSummary *		pSum = NULL;
  
  			//	Check to see that we have a good message ID first
  			long			nCurrentMessageID = in_pUpdateAction->GetMessageID();
  			ASSERT(nCurrentMessageID != 0);
  			if (nCurrentMessageID != 0)
  			{
  				//	Get the summary by message ID
  				CSumList &	listSums = m_pCurrentTocDoc[sd_IndexUpdateActions]->GetSumList();
  				pSum = listSums.GetByMessageId(nCurrentMessageID);
  
  				//	Verify that we found the summary
  				ASSERT(pSum);
  			}
  
  			//	Check to see if we had a good ID and found the summary
  			if (pSum)
  			{
  				if (m_pUpdateActionsTaskInfo)
  				{
  					//	Display add or update progress
  					UINT		nStringFormatID = in_pUpdateAction->IsUpdate() ?
  												  IDS_UPDATING_INDEX_PROGRESS_UPDATING_MESSAGE :
  												  IDS_UPDATING_INDEX_PROGRESS_ADDING_MESSAGE;
  
  					szProgress.Format( nStringFormatID, pSum->MakeTitle() );
  
  					m_pUpdateActionsTaskInfo->Progress(szProgress);
  
  					//	Update persona
  					m_pUpdateActionsTaskInfo->SetPersona( pSum->GetPersona() );
  				}
  
  				bool		bRepeatingMessage = (m_nMessageIDLastAddAttempt == nCurrentMessageID);
  				bool		bWaitingForX1 = false;
  				
  				//	Attempt to add the message to the index
  				bDoneProcessingCurrentAction = AddToIndex(sd_IndexUpdateActions, in_pOneUse, pSum, bWaitingForX1);
  
  				//	If we're not waiting for X1 and we're repeating a message
  				//	something is going wrong when we try to add it. Continue
  				//	on to the next message.
  				if (!bWaitingForX1 && bRepeatingMessage)
  					bDoneProcessingCurrentAction = true;
  
  				//	Remember the last message that we attempted to add
  				m_nMessageIDLastAddAttempt = nCurrentMessageID;
  			}
  			else if (m_pUpdateActionsTaskInfo)
  			{
  				//	Display progress about missing message
  				szProgress.LoadString(IDS_SKIPPING_UPDATE_MISSING_MESSAGE);
  				m_pUpdateActionsTaskInfo->Progress(szProgress);
  			}
  		}
  		else if (m_pUpdateActionsTaskInfo)
  		{
  			//	Display progress about missing mailbox
  			szProgress.Format( IDS_SKIPPING_UPDATE_MISSING_MAILBOX, m_szMailboxRelativePath[sd_IndexUpdateActions] );
  			m_pUpdateActionsTaskInfo->Progress(szProgress);
  		}						
  	}
  
  	return bDoneProcessingCurrentAction;
  }
  
  
  // ---------------------------------------------------------------------------
  //		* ProcessRemoveOrReindexMailbox								 [Public]
  // ---------------------------------------------------------------------------
  //	Processes mailbox removal.
  
  bool
  X1EmailScanner::ProcessRemoveOrReindexMailbox(
  	X1EmailScanner::IndexUpdateAction *			in_pUpdateAction,
  	X1::IX1OneUsePtr &							in_pOneUse)
  {
  	bool				bDoneProcessingCurrentAction = true;
  	
  	//	Close any mailbox left open after processing a previous action
  	if ( m_pCurrentTocDoc[sd_IndexUpdateActions] )
  		CloseCurrentMailbox(sd_IndexUpdateActions);
  
  	//	Are there indexed messages for this mailbox?
  	X1::IX1ItemListPtr		pMailboxItemList;
  	bDoneProcessingCurrentAction = !HasIndexedMessages( in_pUpdateAction->GetMailboxRelativePath(), pMailboxItemList );
  
  	if (!bDoneProcessingCurrentAction)
  	{
  		long	nCount = 0;
  
  		//	Determine how many indexed messages we found for the given mailbox
  		pMailboxItemList->get_count(&nCount);
  
  		//	If we just started removing items from this mailbox, remember the
  		//	total number of messages to remove.
  		if ( (m_nTotalMessagesToRemove == 0) && (nCount != 0) )
  			m_nTotalMessagesToRemove = nCount;
  
  		//	Should have found messages, but HasIndexedMessages assumes true
  		//	on failure, so make sure.
  		ASSERT(nCount);
  		if (nCount > 0)
  		{
  			//	Remove the first message that we found that isn't removed or invalid.
  			//
  			//	WARNING: By looping here it is possible that we could process longer
  			//			 than we should during idle. So far items that are removed
  			//			 or invalid appearing in the search results seem to be a rare
  			//			 aberration that occasionally blocked us from completing
  			//			 correctly (since we would call RemoveFromIndex every time
  			//			 we're called with the same item). If this turns out to be a
  			//			 more common situation, then this loop will need to be factored
  			//			 out to work correctly in a more time sliced manner.
  			long	nItemIndex = 0;
  			BOOL	bIsRemovedOrInvalid = FALSE;
  
  			do
  			{
  				long	nX1ItemNum = pMailboxItemList->Getitem(nItemIndex);
  
  				bIsRemovedOrInvalid = m_pEmailDB->IsRemovedOrInvalid(nX1ItemNum);
  
  				if (!bIsRemovedOrInvalid)
  					m_pEmailDB->RemoveFromIndex(in_pOneUse, nX1ItemNum);
  
  				nItemIndex++;
  			} while( bIsRemovedOrInvalid && (nItemIndex < nCount) );
  
  			//	We're done processing the current action if we hit the end of the
  			//	messages in the mailbox and didn't find any that we're not removed
  			//	or invalid.
  			if (nItemIndex == nCount)
  				bDoneProcessingCurrentAction;
  		}
  		else
  		{
  			//	We're done processing the current action if we didn't find any messages
  			bDoneProcessingCurrentAction = true;
  		}
  
  		if (m_pUpdateActionsTaskInfo)
  		{
  			//	Display progress about number of messages remaining to be removed
  			CString			szProgress;
  			szProgress.Format( IDS_UPDATING_INDEX_PROGRESS_REMOVING_MAILBOX,
  							   nCount, in_pUpdateAction->GetMailboxRelativePath() );
  			m_pUpdateActionsTaskInfo->Progress(szProgress);
  
  			int		nSoFar = m_nSoFarBase[sd_IndexUpdateActions] + (m_nTotalMessagesToRemove - nCount) * 100 / m_nTotalMessagesToRemove;
  			m_pUpdateActionsTaskInfo->SetSoFar(nSoFar);
  		}
  	}
  
  	if ( bDoneProcessingCurrentAction &&
  		 (in_pUpdateAction->GetUpdateAction() == SearchManager::ua_ReindexMailbox) )
  	{
  		//	We're done with the removal phase of a reindex mailbox action.
  		//	Transform ourselves into an add mailbox action.
  		in_pUpdateAction->SetUpdateAction(SearchManager::ua_AddMailbox);
  		bDoneProcessingCurrentAction = false;
  	}
  
  	return bDoneProcessingCurrentAction;
  }
  
  
  // ---------------------------------------------------------------------------
  //		* ProcessAddMailbox											 [Public]
  // ---------------------------------------------------------------------------
  //	Processes mailbox addition.
  
  bool
  X1EmailScanner::ProcessAddMailbox(
  	X1EmailScanner::IndexUpdateAction *			in_pUpdateAction,
  	X1::IX1OneUsePtr &							in_pOneUse)
  {
  	bool				bDoneProcessingCurrentAction = true;
  	const CString &		szMailboxRelativePath = in_pUpdateAction->GetMailboxRelativePath();
  	
  	if ( !m_pCurrentTocDoc[sd_IndexUpdateActions]  ||
  		 (szMailboxRelativePath.CompareNoCase(m_szMailboxRelativePath[sd_IndexUpdateActions]) != 0) )
  	{
  		//	Current mailbox is not open yet (either no open mailbox or previous mailbox
  		//	is still open). Close the previous mailbox (if any), move to the current
  		//	mailbox and open it.
  		CloseCurrentMailbox(sd_IndexUpdateActions);
  		m_szMailboxRelativePath[sd_IndexUpdateActions] = szMailboxRelativePath;
  		OpenCurrentMailbox(sd_IndexUpdateActions);
  
  		CString		szMailboxFullPath = EudoraDir + szMailboxRelativePath;
  
  		//	Get the mailbox name for progress display
  		QCMailboxCommand *	pMailbox = g_theMailboxDirector.FindByPathname(szMailboxFullPath);
  		if (pMailbox)
  			m_szMailboxName = pMailbox->GetName();
  		else
  			m_szMailboxName.Empty();
  
  		//	We're done for good if the mailbox file does not exist or if it's empty.
  		bDoneProcessingCurrentAction = !m_bMailboxFileExists[sd_IndexUpdateActions] ||
  									   (m_posCurrentSummary[sd_IndexUpdateActions] == NULL);
  	}
  	else
  	{
  		//	Process adding mailbox
  		bDoneProcessingCurrentAction = AddOrUpdateMailboxMessages(sd_IndexUpdateActions);
  	}
  
  	return bDoneProcessingCurrentAction;
  }
  
  
  
  // ---------------------------------------------------------------------------
  //		* OpenCurrentMailbox										 [Public]
  // ---------------------------------------------------------------------------
  //	Opens the current mailbox TOC and MBX.
  
  void
  X1EmailScanner::OpenCurrentMailbox(
  	ScanDataEnum				in_eScanData)
  {
  	//	If we're currently doing a main scan (i.e. add/update or culling), then
  	//	notify Search Manager that we're moving on to the next mailbox.
  	if (in_eScanData == sd_MainScans)
  		SearchManager::Instance()->NotifyIndexingStatus(m_eScanType, m_eNextScanType, m_szMailboxRelativePath[in_eScanData]);
  	
  	CString			szMailboxFullPath = EudoraDir + m_szMailboxRelativePath[in_eScanData];
  	
  	//	Check to see if the mailbox exists
  	m_bMailboxFileExists[in_eScanData] = FileExistsMT(szMailboxFullPath);
  	
  	if ( !m_bMailboxFileExists[in_eScanData] )
  	{
  		if (m_eScanType == st_AddOrUpdateItemsScan)
  		{
  			ASSERT(!"Error mailbox file does not exists!");
  
  			CString		szLogEntry;
  			szLogEntry.Format( "Mailbox file \"%s\" was missing during search indexing",
  							   m_pCurrentMailbox->GetPathname() );
  			PutDebugLog(DEBUG_MASK_SEARCH, szLogEntry);
  		}
  
  		return;
  	}
  
  	const char *	szMailboxName = NULL;
  
  	if ( (in_eScanData == sd_MainScans) && (m_eScanType == st_AddOrUpdateItemsScan) )
  		szMailboxName = m_pCurrentMailbox->GetName();
  	
  	//	Open TocDoc
  	m_pCurrentTocDoc[in_eScanData] = GetToc(szMailboxFullPath, szMailboxName);
  
  	ASSERT( m_pCurrentTocDoc[in_eScanData] );
  
  	if ( m_pCurrentTocDoc[in_eScanData] )
  	{
  		CTocDoc *	pTocDoc = m_pCurrentTocDoc[in_eScanData];
  
  		//	Tell TocDoc not to auto-delete
  		pTocDoc->IncrementPreventAutoDelete();
  
  		//	Only need to open the mailbox file if we could call AddToIndex.
  		//	We only call AddToIndex when we're doing an add/update scan or an index
  		//	update actions scan (i.e. specifically not when doing a culling scan).
  		if ( (in_eScanData == sd_IndexUpdateActions) || (m_eScanType == st_AddOrUpdateItemsScan) )
  		{
  			//	Open the mailbox file
  			m_CurrentMailboxFile[in_eScanData].Open(pTocDoc->GetMBFileName(), O_RDONLY);
  		}
  
  		//	Prepare for possible summary iteration
  		CSumList &	listSums = pTocDoc->GetSumList();
  		m_posCurrentSummary[in_eScanData] = listSums.GetHeadPosition();
  		m_posLastSummaryAddAttempt[in_eScanData] = NULL;
  		m_nCurrentMessage[in_eScanData] = 0;
  
  		X1::IX1ItemListPtr		pMailboxItemList;
  		
  		//	Have any messages been indexed for this mailbox yet?
  		//	If yes, we'll need to check each message to see if it's already
  		//	in the index. If not, we can optimize our initial indexing and
  		//	skip checking for each message.
  		m_bCheckForIndexedMessages[in_eScanData] = HasIndexedMessages( m_szMailboxRelativePath[in_eScanData], pMailboxItemList );
  
  		m_bCheckingMessages[in_eScanData] = m_bCheckForIndexedMessages[in_eScanData];
  		m_bNeedToAddMessages[in_eScanData] = false;
  	}
  }
  
  
  // ---------------------------------------------------------------------------
  //		* CloseCurrentMailbox										 [Public]
  // ---------------------------------------------------------------------------
  //	Closes the current mailbox TOC and MBX.
  
  void
  X1EmailScanner::CloseCurrentMailbox(
  	ScanDataEnum				in_eScanData,
  	bool						in_bOkToCloseTOC)
  {	
  	if ( m_pCurrentTocDoc[in_eScanData] )
  	{
  		CTocDoc *	pTocDoc = m_pCurrentTocDoc[in_eScanData];
  
  		//	Give our go ahead for TocDoc to auto-delete again
  		pTocDoc->DecrementPreventAutoDelete();
  
  		//	If caller says it's ok to close the TOC (i.e. caller is not aware
  		//	of any outstanding CTocDoc *'s), then close the TocDoc if it is
  		//	not is use by any views.
  		if (in_bOkToCloseTOC)
  			CloseTOCIfNotInUse(pTocDoc);
  
  		//	Close the mailbox (we only open it when we have a Toc Doc, so
  		//	checking for a Toc Doc before closing is reasonable)
  		m_CurrentMailboxFile[in_eScanData].Close();
  
  		//	Done with this Toc Doc
  		m_pCurrentTocDoc[in_eScanData] = NULL;
  	}
  
  	//	Empty mailbox path
  	m_szMailboxRelativePath[in_eScanData].Empty();
  }
  
  
  // ---------------------------------------------------------------------------
  //		* HasIndexedMessages										 [Public]
  // ---------------------------------------------------------------------------
  //	Determines whether or not the specified mailbox has been indexed at all
  //	yet. Mailbox is specified by path.
  
  bool
  X1EmailScanner::HasIndexedMessages(
  	const char *				in_szMailboxRelativePath,
  	X1::IX1ItemListPtr &		out_pMailboxItemList)
  {
  	//	Assume that mailbox has indexed messages, so that if anything goes wrong
  	//	we check each message we add before adding it (to avoid duplicates).
  	bool				bHasIndexedMessages = true;
  
  	//	Get the X1 email database and search object
  	X1::IX1Search *		pSearch = SearchManager::Instance()->GetX1Search();
  
  	ASSERT(pSearch);
  
  	if (m_pEmailDB && pSearch)
  	{
  		try
  		{
  			//	Find matching mailbox
  			X1::IX1ItemResultPtr		pMailboxResult;
  			CString						szMailboxPathWithRootTag = SearchManager::kMailboxRelativePathRootTag;
  			szMailboxPathWithRootTag += in_szMailboxRelativePath;
  			_bstr_t						bstrSearchText(szMailboxPathWithRootTag);
  	
  			m_pEmailDB->FindItems( X1::ftContainsExact, bstrSearchText,
  								   SearchManager::efid_MailboxRelativePath, &pMailboxResult );
  
  			ASSERT( pMailboxResult.GetInterfacePtr() );
  
  			if (pMailboxResult)
  			{
  				//	Set up what we want the search results to be sorted by
  				COleSafeArray	safeArraySortBy;
  				int *			pData;
  
  				V_VT(&safeArraySortBy) = VT_ARRAY | VT_INT;
  				V_ARRAY(&safeArraySortBy) = SafeArrayCreateVector(VT_INT, 0, 1);
  
  				safeArraySortBy.AccessData( reinterpret_cast<void **>(&pData) );
  				pData[0] = -1;
  				safeArraySortBy.UnaccessData();
  
  				//	Set up that we want the search results to be sorted forwards
  				COleSafeArray	safeArraySortOrder;
  
  				V_VT(&safeArraySortOrder) = VT_ARRAY | VT_INT;
  				V_ARRAY(&safeArraySortOrder) = SafeArrayCreateVector(VT_INT, 0, 1);
  
  				safeArraySortOrder.AccessData( reinterpret_cast<void **>(&pData) );
  				pData[0] = 0;
  				safeArraySortOrder.UnaccessData();
  
  				//	Actually get the search results
  				pMailboxResult->GetItemList(safeArraySortBy, safeArraySortOrder, &out_pMailboxItemList);
  
  				ASSERT( out_pMailboxItemList.GetInterfacePtr() );
  
  				if (out_pMailboxItemList)
  				{
  					//	Determine how many search results we found
  					long	nCount = 0;
  					out_pMailboxItemList->get_count(&nCount);
  
  					bHasIndexedMessages = (nCount > 0);
  				}
  			}
  		}
  		catch (_com_error & e)
  		{
  			//	Get developer's attention
  			ASSERT(!"X1 COM Error");
  
  			SearchManager::Utils::LogError( "COM", "HasIndexedMessages", e.ErrorMessage() );
  		}
  		catch (CException * pException)
  		{
  			//	Get developer's attention
  			ASSERT(!"X1 CException");
  			
  			//	Log error
  			SearchManager::Utils::LogCException(pException, "HasIndexedMessages");
  
  			//	Delete exception object
  			pException->Delete();
  		}
  	}
  	
  	return bHasIndexedMessages;
  }
  
  
  // ---------------------------------------------------------------------------
  //		* IsIndexed													 [Public]
  // ---------------------------------------------------------------------------
  //	Determines whether or not the specified message is already in the X1
  //	index. Message is specified by mailbox path and message ID.
  
  bool
  X1EmailScanner::IsIndexed(
  	const char *				in_szMailboxRelativePath,
  	long						in_nUniqueMessageID,
  	long &						out_nX1ItemNum)
  {
  	//	Assume that we found something, so that if anything goes wrong
  	//	we don't re-add any items.
  	long				nCount = 1;
  
  	//	Get the X1 email database and search object
  	X1::IX1Search *		pSearch = SearchManager::Instance()->GetX1Search();
  
  	ASSERT(pSearch);
  
  	if (m_pEmailDB && pSearch)
  	{
  		try
  		{
  			//	Find matching mailbox
  			X1::IX1ItemResultPtr		pMailboxResult;
  			CString						szMailboxPathWithRootTag = SearchManager::kMailboxRelativePathRootTag;
  			szMailboxPathWithRootTag += in_szMailboxRelativePath;
  			_bstr_t						bstrSearchText(szMailboxPathWithRootTag);
  	
  			m_pEmailDB->FindItems( X1::ftContainsExact, bstrSearchText,
  								   SearchManager::efid_MailboxRelativePath, &pMailboxResult );
  
  			//	Find matching message IDs (in any mailbox)
  			X1::IX1ItemResultPtr		pMessageIDResult;
  			CString						szUniqueMessageID;
  
  			szUniqueMessageID.Format("%d", in_nUniqueMessageID);
  			bstrSearchText = szUniqueMessageID;
  			m_pEmailDB->FindItems( X1::ftContainsExact, bstrSearchText,
  								   SearchManager::efid_MessageID, &pMessageIDResult );
  
  			//	Combine results to find unique result 
  			X1::IX1ItemResultPtr		pItemResult = NULL;
  			pMailboxResult->OpAnd(pMessageIDResult, &pItemResult);
  
  			ASSERT( pItemResult.GetInterfacePtr() );
  
  			if (pItemResult)
  			{
  				//	Set up what we want the search results to be sorted by
  				COleSafeArray	safeArraySortBy;
  				int *			pData;
  
  				V_VT(&safeArraySortBy) = VT_ARRAY | VT_INT;
  				V_ARRAY(&safeArraySortBy) = SafeArrayCreateVector(VT_INT, 0, 1);
  
  				safeArraySortBy.AccessData( reinterpret_cast<void **>(&pData) );
  				pData[0] = -1;
  				safeArraySortBy.UnaccessData();
  
  				//	Set up that we want the search results to be sorted forwards
  				COleSafeArray	safeArraySortOrder;
  
  				V_VT(&safeArraySortOrder) = VT_ARRAY | VT_INT;
  				V_ARRAY(&safeArraySortOrder) = SafeArrayCreateVector(VT_INT, 0, 1);
  
  				safeArraySortOrder.AccessData( reinterpret_cast<void **>(&pData) );
  				pData[0] = 0;
  				safeArraySortOrder.UnaccessData();
  
  				//	Actually get the search results
  				X1::IX1ItemListPtr		pItemList = NULL;
  				pItemResult->GetItemList(safeArraySortBy, safeArraySortOrder, &pItemList);
  
  				ASSERT( pItemList.GetInterfacePtr() );
  
  				if (pItemList)
  				{
  					//	Determine how many search results we found
  					pItemList->get_count(&nCount);
  					
  					//	If we have at least one result return the item num for the first one
  					if (nCount > 0)
  						out_nX1ItemNum = pItemList->Getitem(0);
  				}
  			}
  		}
  		catch (_com_error & e)
  		{
  			//	Get developer's attention
  			ASSERT(!"X1 COM Error");
  
  			SearchManager::Utils::LogError( "COM", "IsIndexed", e.ErrorMessage() );
  		}
  		catch (CException * pException)
  		{
  			//	Get developer's attention
  			ASSERT(!"X1 CException");
  			
  			//	Log error
  			SearchManager::Utils::LogCException(pException, "IsIndexed");
  
  			//	Delete exception object
  			pException->Delete();
  		}
  	}
  
  	//	We intended to uniquely identify an item. We don't expect to find more
  	//	than one result.
  	ASSERT(nCount < 2);
  	
  	return (nCount >= 1);
  }
  
  
  // ---------------------------------------------------------------------------
  //		* DoesIndexedItemMatchSummary								 [Public]
  // ---------------------------------------------------------------------------
  //	Determines whether or not the specified summary is already in the X1
  //	index.
  
  bool
  X1EmailScanner::DoesIndexedItemMatchSummary(
  	CSummary *					in_pSummary,
  	long						in_nX1ItemNum)
  {
  	bool	bMatches = true;
  	
  	try
  	{
  		//	We compare all the information that we can easily retrieve from the
  		//	summary object. To save time, we do not compare everything. In
  		//	particular we do not load the body of the message.
  		//
  		//	We compare those values that are most likely to change in every day
  		//	use (marking item read, recalculating junk score, editing the subject,
  		//	etc.). Hopefully pencil edit changes of the body will be noticed because
  		//	the size will no longer match.
  		CString		szIndexedValue;
  		int			nIndexedValue;
  
  		//	Does subject match? Compare case insensitively.
  		szIndexedValue = static_cast<char *>( m_pEmailDB->GetFieldString(SearchManager::efid_Subject, in_nX1ItemNum) );
  		bMatches = (szIndexedValue.CompareNoCase(in_pSummary->m_Subject) == 0);
  
  		if (bMatches)
  		{
  			//	Does label match?
  			nIndexedValue = atoi( m_pEmailDB->GetFieldString(SearchManager::efid_Label, in_nX1ItemNum) );
  			bMatches = (in_pSummary->m_Label == nIndexedValue);
  		}
  
  		if (bMatches)
  		{
  			//	Does the state match?
  			//	We support indexing every state except spooled
  			short	nState = (in_pSummary->m_State != MS_SPOOLED) ? in_pSummary->m_State : 0;
  			nIndexedValue = atoi( m_pEmailDB->GetFieldString(SearchManager::efid_State, in_nX1ItemNum) );
  			bMatches = (nState == nIndexedValue);
  		}
  
  		if (bMatches)
  		{
  			//	Does the mood score match?
  			nIndexedValue = atoi( m_pEmailDB->GetFieldString(SearchManager::efid_Mood, in_nX1ItemNum) );
  			bMatches = (in_pSummary->m_nMood == nIndexedValue);
  		}
  		
  		if (bMatches)
  		{
  			//	Does the priority number match?
  			nIndexedValue = atoi( m_pEmailDB->GetFieldString(SearchManager::efid_PriorityNumber, in_nX1ItemNum) );
  			bMatches = (in_pSummary->m_Priority == nIndexedValue);
  		}
  
  		if (bMatches)
  		{
  			//	Does the junk score match?
  			nIndexedValue = atoi( m_pEmailDB->GetFieldString(SearchManager::efid_JunkScoreNumber, in_nX1ItemNum) );
  			bMatches = (in_pSummary->m_ucJunkScore == nIndexedValue);
  		}
  
  		if (bMatches)
  		{
  			//	Does the size match?
  			nIndexedValue = atoi( m_pEmailDB->GetFieldString(SearchManager::efid_SizeNumber, in_nX1ItemNum) );
  			bMatches = (in_pSummary->GetSize() == nIndexedValue);
  		}
  
  		if ( bMatches && in_pSummary->m_TheToc && in_pSummary->m_TheToc->IsImapToc() )
  		{
  			//	Do the IMAP flags match?
  			nIndexedValue = atoi( m_pEmailDB->GetFieldString(SearchManager::efid_ImapFlags, in_nX1ItemNum) );
  			bMatches = ( static_cast<int>(in_pSummary->m_Imflags & IMFLAGS_DELETED) == nIndexedValue );
  		}
  	}
  	catch (_com_error & e)
  	{
  		//	Get developer's attention
  		ASSERT(!"X1 COM Error");
  
  		SearchManager::Utils::LogError( "COM", "DoesIndexedItemMatchSummary", e.ErrorMessage() );
  	}
  	catch (CException * pException)
  	{
  		//	Get developer's attention
  		ASSERT(!"X1 CException");
  		
  		//	Log error
  		SearchManager::Utils::LogCException(pException, "DoesIndexedItemMatchSummary");
  
  		//	Delete exception object
  		pException->Delete();
  	}
  
  	return bMatches;
  }
  
  
  // ---------------------------------------------------------------------------
  //		* AddToIndex												 [Public]
  // ---------------------------------------------------------------------------
  //	Attempts to add specified summary to X1 index.
  
  bool
  X1EmailScanner::AddToIndex(
  	ScanDataEnum				in_eScanData,
  	X1::IX1OneUse *				in_pOneUse,
  	CSummary *					in_pSummary,
  	bool &						out_bWaitingForX1)
  {
  	bool	bAddedToIndex = false;
  
  	//	Assume that we're not waiting for X1
  	out_bWaitingForX1 = false;
  
  	//	Enforce pre-conditions - we must have a valid toc doc and summary right now
  	ASSERT( m_pCurrentTocDoc[in_eScanData] && in_pSummary );
  	if ( !m_pCurrentTocDoc[in_eScanData] || !in_pSummary )
  		return false;
  
  	try
  	{
  		char *		pBuffer = NULL;
  		bool		bAddToIndex = true;
  		
  		if ( !m_pCurrentTocDoc[in_eScanData]->IsImapToc() || in_pSummary->IsIMAPMessageBodyDownloaded() )
  		{
  			//	Assume failure
  			bAddToIndex = false;
  			HRESULT		hr = m_CurrentMailboxFile[in_eScanData].Seek(in_pSummary->m_Offset, SEEK_SET);
  
  			if ( SUCCEEDED(hr) )
  			{
  				pBuffer = m_szMessageBuffer.GetBuffer(in_pSummary->m_Length);
  				
  				hr = m_CurrentMailboxFile[in_eScanData].Read(pBuffer, in_pSummary->m_Length);
  
  				m_szMessageBuffer.ReleaseBuffer(in_pSummary->m_Length);
  
  				//	Allow adding to index to continue
  				bAddToIndex = SUCCEEDED(hr);
  			}
  		}
  
  		if (bAddToIndex)
  		{
  			CComPtr<X1EmailIndexItem>	pIndexItem = DEBUG_NEW X1EmailIndexItem(in_pSummary, pBuffer);
  
  			bAddedToIndex = (m_pEmailDB->AddToIndex(in_pOneUse, pIndexItem, FALSE) == X1::iarKeptAll);
  
  			//	Make sure that we remember that this message does not need to be added
  			//	to the index. Particularly important if we interrupted other indexing
  			//	of the mailbox that contains this message.
  			if (bAddedToIndex)
  				in_pSummary->SetShouldBeAddedToIndex(false);
  
  			//	If we didn't add it to the index it was because we're waiting for X1
  			out_bWaitingForX1 = !bAddedToIndex;
  		}
  
  		//	Done with message buffer
  		m_szMessageBuffer.Empty();
  	}
  	catch (_com_error & e)
  	{
  		//	Get developer's attention
  		ASSERT(!"X1 COM Error");
  
  		SearchManager::Utils::LogError( "COM", "AddToIndex", e.ErrorMessage() );
  	}
  	catch (CException * pException)
  	{
  		//	Get developer's attention
  		ASSERT(!"X1 CException");
  		
  		//	Log error
  		SearchManager::Utils::LogCException(pException, "AddToIndex");
  
  		//	Delete exception object
  		pException->Delete();
  	}
  	
  	return bAddedToIndex;	
  }
  