Performant C# Socket server
You are holding on to some IDisposable resources that you shouldn't be, which will impact scalability, GC pressure and likely performance. Here's the augmented Listener class:

internal sealed class Listener : IDisposable
{
    private readonly int port;

    private readonly TcpListener tcpListener;

    private readonly IList&lt;Client&gt; clients = new List&lt;Client&gt;();

    public Listener(int port)
    {
        this.port = port;
        this.tcpListener = new TcpListener(IPAddress.Any, this.port);
        this.tcpListener.Start();
        Task.Factory.StartNew(this.ListenLoop);
    }

    public int Port
    {
        get
        {
            return this.port;
        }
    }

    public void Dispose()
    {
        foreach (var client in this.clients)
        {
            client.Dispose();
        }
    }

    private async void ListenLoop()
    {
        while (true)
        {
            var socket = await this.tcpListener.AcceptSocketAsync();

            if (socket == null)
            {
                break;
            }

            var client = new Client(socket, this.ClientDisconnected);

            this.clients.Add(client);
            await Task.Factory.StartNew(client.Do);
        }
    }

    private void ClientDisconnected(Client client)
    {
        if (client == null)
        {
            return;
        }

        client.Dispose();
        this.clients.Remove(client);
    }
}


Here's the augmented Client class:

internal delegate void ClientDisconnectedDelegate(Client client);

internal sealed class Client : IDisposable
{
    private readonly Socket socket;

    private readonly ClientDisconnectedDelegate doDisconnect;

    private readonly Stream networkStream;

    private readonly Stream memoryStream = new MemoryStream();

    private readonly TextReader streamReader;

    public Client(Socket socket, ClientDisconnectedDelegate doDisconnect)
    {
        this.socket = socket;
        this.doDisconnect = doDisconnect;
        Task.Factory.StartNew(this.CheckForConnection);
        this.networkStream = new NetworkStream(this.socket, true);
        this.streamReader = new StreamReader(this.memoryStream);
    }

    public void Dispose()
    {
        this.streamReader.Dispose();
        this.networkStream.Dispose();
        this.socket.Dispose();
        this.memoryStream.Dispose();
    }

    public async void Do()
    {
        var buffer = new byte[4096];
        var bytesRead = await this.networkStream.ReadAsync(buffer, 0, buffer.Length);

        Console.WriteLine("Doing some awesome work that takes 5 seconds.");
        Thread.Sleep(5000);
        Console.WriteLine("Finished doing work.");

        await this.networkStream.WriteAsync(buffer, 0, bytesRead);
        await this.networkStream.FlushAsync();
    }

    private void CheckForConnection()
    {
        while (true)
        {
            bool isDisconnected;

            try
            {
                isDisconnected = this.socket.Poll(1, SelectMode.SelectRead) &amp;&amp; (this.socket.Available == 0);
            }
            catch (SocketException)
            {
                isDisconnected = true;
            }

            if (isDisconnected &amp;&amp; (this.doDisconnect != null))
            {
                this.doDisconnect(this);
                return;
            }

            Thread.Sleep(100);
        }
    }
}


And here's the augmented Client program:

internal static class Program
{
    private static void Main()
    {
        Thread.Sleep(5000);

        Console.WriteLine("Connecting...");

        using (var client = new TcpClient("localhost", 6666))
        {
            Console.WriteLine("Connected.");

            using (var stream = client.GetStream())
            {
                var messageToSend = new byte[3];

                messageToSend[0] = 1;
                messageToSend[1] = 3;

                var messageReceived = new byte[3];

                stream.WriteAsync(messageToSend, 0, 3);

                stream.Flush();

                stream.Read(messageReceived, 0, 3);

                // Echo
                Console.WriteLine(messageReceived[0] + messageReceived[1]);
                Console.ReadLine();
            }
        }
    }
}