title: Asynchronously Enumerate Folders
body: <p>I am trying to implement a general purpose file system crawler that would - for example - be able to enumerate all sub-folders starting at a given root. I would like to do that using the async/await/Task paradigm.</p>
  
  <p>Below is my code so far. It works, but I suspect it can be improved. In particular, the annotated <code>Task.WaitAll</code> causes unnecessary waits in a deep directory tree, since the loop is suspended waiting at each tree level instead of immediately proceeding to tackle the new folders being added to the <code>folderQueue</code>.</p>
  
  <p>Somehow I would like to include the new folders being added to the <code>folderQueue</code> in the list of tasks being waited on by <code>Task.WaitAll()</code> <em>while</em> the <code>WaitAll</code> is in progress. Is that even possible?</p>
  
  <pre><code>using System;
  using System.Collections.Generic;
  using System.Diagnostics;
  using System.IO;
  using System.Linq;
  using System.Threading.Tasks;
  
  class FileSystemCrawlerSO
  {
      static void Main(string[] args)
      {
          FileSystemCrawlerSO crawler = new FileSystemCrawlerSO();
          Stopwatch watch = new Stopwatch();
          watch.Start();
          crawler.CollectFolders(@"d:\www");
          watch.Stop();
          Console.WriteLine($"Collected {crawler.NumFolders:N0} folders in {watch.ElapsedMilliseconds} milliseconds.");
          if (Debugger.IsAttached)
              Console.ReadKey();
      }
  
  
      public int NumFolders { get; set; }
  
      private readonly Queue&lt;DirectoryInfo&gt; folderQueue;
  
  
      public FileSystemCrawlerSO()
      {
          folderQueue = new Queue&lt;DirectoryInfo&gt;();
      }
  
  
      public void CollectFolders(string path)
      {
          DirectoryInfo directoryInfo = new DirectoryInfo(path);
          lock (folderQueue)
             folderQueue.Enqueue(directoryInfo);
          List&lt;Task&gt; tasks = new List&lt;Task&gt;();
          do
          {
              tasks.Clear();
              lock (folderQueue)
              {
                  while (folderQueue.Any())
                  {
                      var folder = folderQueue.Dequeue();
                      Task task = Task.Run(() =&gt; CrawlFolder(folder));
                      tasks.Add(task);
                  }
              }
              if (tasks.Any())
              {
                  Console.WriteLine($"Waiting for {tasks.Count} tasks...");
                  Task.WaitAll(tasks.ToArray()); //&lt;== NOTE: THIS IS NOT OPTIMAL
              }
          } while (tasks.Any());
      }
  
  
      private void CrawlFolder(DirectoryInfo dir)
      {
          try
          {
              DirectoryInfo[] directoryInfos = dir.GetDirectories();
              lock (folderQueue)
                  foreach (DirectoryInfo childInfo in directoryInfos)
                      folderQueue.Enqueue(childInfo);
              // Do something with the current folder
              // e.g. Console.WriteLine($"{dir.FullName}");
              NumFolders++;
          }
          catch (Exception ex)
          {
              while (ex != null)
              {
                  Console.WriteLine($"{ex.GetType()} {ex.Message}\n{ex.StackTrace}");
                  ex = ex.InnerException;
              }
          }
      }
  }
  </code></pre>