Sort and Delete duplicate image files I'm trying to make a console application to sort and delete duplicate image files. I have a folder with about 20000 images which take up roughly 70GB of space.
I wrote the following script and it seems to work fine, but the issue is that my PC runs out of available RAM memory (it has 16GB) without reaching halfway through.
static void Main(string[] args)
{
    var uniqueData = new HashSet&lt;string&gt;();
    
    int progress = 0;
    int fileCount = Directory.GetFiles(imageFolderPath).Length;

    foreach (var file in Directory.GetFiles(imageFolderPath))
    {
        string data, year, month;

        using (var image = Image.FromFile(file))
        {

            using (var ms = new MemoryStream())
            {
                image.Save(ms, ImageFormat.Jpeg);
                data = Convert.ToBase64String(ms.ToArray());
            }

            try //try to get image date
            {
                var date = Encoding.UTF8.GetString(image.GetPropertyItem(36867).Value).Split(':');
                year = date[0];
                month = date[1];
            }
            catch
            {
                year = &quot;Unknown&quot;;
                month = &quot;Unknown&quot;;
            }
        }

        if (!uniqueData.Contains(data))
        {
            uniqueData.Add(data);

            string yearDirectory = Path.Combine(outputFolderPath, year);
                
            if (!Directory.Exists(yearDirectory))
            {
                Directory.CreateDirectory(yearDirectory);
            }

            string monthDirectory = Path.Combine(yearDirectory, (int.TryParse(month, out int i) &amp;&amp; i &gt; 0 &amp;&amp; i &lt; 14) ? CultureInfo.CurrentCulture.DateTimeFormat.GetMonthName(i) : month);

            if (!Directory.Exists(monthDirectory))
            {
                Directory.CreateDirectory(monthDirectory);
            }

            File.Move(file, Path.Combine(monthDirectory, Path.GetFileName(file)));
        }
        else
        {
            File.Delete(file);
        }

        progress++;
        Console.WriteLine($&quot;{progress} / {fileCount}&quot;);
    }
}
You don't need to store the whole file contents, compute hash instead. Also converting each image to JPEG is not necessary, use image.RawFormat, it would be a bit faster.
using (SHA256 sha = SHA256.Create())
using (var ms = new MemoryStream((int)new FileInfo(file).Length))
{
    image.Save(ms, image.RawFormat);
    ms.Position = 0;
    data = Convert.ToBase64String(sha.ComputeHash(ms));
}

2nd option
if (!uniqueData.Contains(data))
{
    uniqueData.Add(data);
    // ...
}

Can be simplified to
if (uniqueData.Add(data))
{
    // ...
}

Also twice getting the directoriry files can be once
string[] files = Directory.GetFiles(imageFolderPath);
int fileCount = files.Length;

foreach (var file in files)
{
    // ...
}

And finally this one
if (!Directory.Exists(yearDirectory))
{
    Directory.CreateDirectory(yearDirectory);
}

Directory.CreateDirectory creates directory if it not exists, otherwise it does nothing. You can remove Exists check.
Directory.CreateDirectory(yearDirectory);