title: WCF fails every 10 calls in a silent and infuriating way
body: <p>So I have a simple test case WCF service.  The idea is that one process calls another and registers for callbacks, and the other then calls the original caller when events occur.  WCF native callbacks don't work properly, or for this application, so don't bother suggesting that.  I have a very simple test program that exhibits the behavior.  Every 10 tries, it will pause for a while (the timeout period) and the recover after it.  It recovers without firing any channel events - open/close/whatever.  I have to be missing something obviously, but what...</p>
  
  <pre><code>using System;
  using System.Collections.Generic;
  using System.Linq;
  using System.Text;
  using System.ServiceModel;
  using System.Threading;
  
  namespace WCFTest
  {
   [ServiceContract(Namespace = "http://WCF.WTF")]
   public interface IServerEvents
   {
    [OperationContract(IsOneWay = true)]
    void Heartbeat();
   }
  
   [ServiceContract(Namespace = "http://WCF.WTF")]
   public interface ICallbackEvents
   {
    [OperationContract(IsOneWay = true)]
    void HeartbeatAck();
   }
  
   [ServiceBehavior(ConcurrencyMode = ConcurrencyMode.Multiple, InstanceContextMode =     InstanceContextMode.Single)]
   [CallbackBehavior(IncludeExceptionDetailInFaults = true)]
   class Program : IServerEvents, ICallbackEvents
   {
    static AutoResetEvent CalledBack = new AutoResetEvent(false);
    static ChannelFactory&lt;IServerEvents&gt; ServerChannelFactory;
    static ChannelFactory&lt;ICallbackEvents&gt; ClientChannelFactory;
    static ServiceHost ServerHost;
    static ServiceHost ClientHost;
  
    static int Timeout = 5;
  
    private Program()
    {
    }
  
    static void Main(string[] args)
    {
     NetTcpBinding binding = new NetTcpBinding(SecurityMode.None)
      {
       OpenTimeout = new TimeSpan(0, 0, Timeout),
       SendTimeout = new TimeSpan(0, 0, Timeout),
       ReceiveTimeout = new TimeSpan(0, 0, Timeout),
       MaxConnections = 50,
       ListenBacklog = 50     
      };
     Uri serverUri = new Uri("net.tcp://localhost:3123/WTF");
     ServerChannelFactory = new ChannelFactory&lt;IServerEvents&gt;(binding, new     EndpointAddress(serverUri));
     ClientChannelFactory = new ChannelFactory&lt;ICallbackEvents&gt;(binding, new     EndpointAddress("net.tcp://localhost:3123/WTF/Client"));
     ServerChannelFactory.Closing += new EventHandler((s,x) =&gt;     Console.WriteLine("SClosing"));
     ServerChannelFactory.Opening += new EventHandler((s, x) =&gt; Console.WriteLine("SOpening"));
     ServerChannelFactory.Faulted += new EventHandler((s,x)=&gt; Console.WriteLine("SFaulted"));
     ClientChannelFactory.Closing += new EventHandler((s, x) =&gt; Console.WriteLine("CClosing"));
     ClientChannelFactory.Opening += new EventHandler((s, x) =&gt; Console.WriteLine("COpening"));
     ClientChannelFactory.Faulted += new EventHandler((s, x) =&gt; Console.WriteLine("CFaulted"));
  
     ServerHost = StartServer("/Server", new Program(), typeof(IServerEvents));
     ClientHost = StartServer("/Client", new Program(), typeof(ICallbackEvents));
  
     while (true)
     {
      Thread.Sleep(100);
      try
      {
       new Program().Heartbeat();
       if (!CalledBack.WaitOne(2500, true))
       {
        throw new TimeoutException("Epic fail.");
       }
      }
      catch (Exception x)
      {
       Console.WriteLine("Failed heartbeat.\n{0}", x);
      }
     }
    }
  
    public void Heartbeat()
    {
     Console.Write(".");
     Console.Out.Flush();
     try
     {
      ClientChannelFactory.CreateChannel().HeartbeatAck();
     }
     catch (Exception x)
     {
      Console.WriteLine("Couldn't ACK heartbeat.\n{0}", x);
     }
    }
  
    public void HeartbeatAck()
    {
     Console.Write("!");
     Console.Out.Flush();
     CalledBack.Set();
    }
  
    private static ServiceHost StartServer&lt;T&gt;(string fragment, T remoteObject, Type interfaceType)
    {
     ServiceHost retHost = null;
     using (AutoResetEvent revent = new AutoResetEvent(false))
     {
  bool hostOk = false;
  // The service host has to be started on a non-sync-context thread or bad things (tm) will happen.
  ThreadPool.QueueUserWorkItem((oo) =&gt;
  {
   try
   {
    retHost = new ServiceHost(remoteObject, new Uri("net.tcp://localhost:3123/WTF"));
    var binding = new NetTcpBinding(SecurityMode.None)
        {
         OpenTimeout = new TimeSpan(0, 0, Timeout),
         SendTimeout = new TimeSpan(0, 0, Timeout),
         ReceiveTimeout = new TimeSpan(0, 0, Timeout),
         MaxConnections = 50,
         ListenBacklog = 50           
        };
    retHost.AddServiceEndpoint(interfaceType, binding, fragment);
        retHost.Open();
        hostOk = true;
       }
       catch (Exception xc)
       {
        Console.WriteLine("Couldn't start WCF Service Host!\n{0}", xc);
       }
       finally
       {
        try { revent.Set(); }
        catch { }
       }
      });
      revent.WaitOne(5000, true);
      return hostOk ? retHost : null;
     }
    }
   }
  }
  </code></pre>