completion port thread is leaking when client terminates the connection

Question

I have a ASP.NET WebApi self-hosted application. HttpSelfHostConfiguration is configured as below

HttpSelfHostConfiguration config = new HttpSelfHostConfiguration("http://0.0.0.0:54781")
{
    TransferMode = TransferMode.StreamedResponse,
    MaxConcurrentRequests = 1000,
    SendTimeout = TimeSpan.FromMinutes(60),
};

config.Routes.MapHttpRoute(
    name: "DefaultApi",
    routeTemplate: "{controller}",
    defaults: new { @controller = "Default" }
);

HttpSelfHostServer server = new HttpSelfHostServer(config);
server.OpenAsync().Wait();

for (;;)
{
    int workerThreads, completionPortThreads;
    ThreadPool.GetAvailableThreads(out workerThreads, out completionPortThreads);
    Console.WriteLine("Available Completion Port Threads = {0};", completionPortThreads);
    Console.Out.Flush();
    Thread.Sleep(2000);
}

There is an action accepts HTTP GET request as below

public class DefaultController : ApiController
{
    public HttpResponseMessage GET()
    {
        Console.WriteLine("Receive HTTP GET request");
        Func<Stream, HttpContent, TransportContext, Task> func = (stream, httpContent, transportContext) =>
        {
            return Monitor(httpContent, stream);
        };


        HttpResponseMessage response = Request.CreateResponse();

        response.StatusCode = HttpStatusCode.OK;
        response.Content = new PushStreamContent(func, new MediaTypeHeaderValue("text/plain"));
        return response;
    }

    async Task Monitor(HttpContent httpContent, Stream stream)
    {
        try
        {
            using (StreamWriter sw = new StreamWriter(stream, new UTF8Encoding(false)))
            {
                for (;;)
                {
                    sw.WriteLine(Guid.NewGuid().ToString());
                    sw.Flush();

                    await Task.Delay(1000);
                }
            }
        }
        catch (CommunicationException ce)
        {
            HttpListenerException ex = ce.InnerException as HttpListenerException;
            if (ex != null)
            {
                Console.WriteLine("HttpListenerException occurs, error code = {0}", ex.ErrorCode);
            }
            else
            {
                Console.WriteLine("{0} occurs : {1}", ce.GetType().Name, ce.Message);
            }
        }
        catch (Exception ex)
        {
            Console.WriteLine("{0} occurs : {1}", ex.GetType().Name, ex.Message);
        }
        finally
        {
            stream.Close();
            stream.Dispose();
            httpContent.Dispose();
            Console.WriteLine("Dispose");
        }
    }
}

Open the url http://127.0.0.1:54781/ and I see progressive response comming.

But If the client terminates the connection when the server is sending the response, the completion port thread is taken up and never released.

It able to bring down the application by exhausting the completion port threads pool.

Answer 1

After changing to OWIN self-hosting, this problem disappears. It seems a bug in System.Web.Http.SelfHost

Here is updated code:

class Program
{

    static void Main(string[] args)
    {
        var server = WebApp.Start<Startup>(url: "http://*:54781");


        for (;;)
        {
            int workerThreads, completionPortThreads;
            ThreadPool.GetAvailableThreads(out workerThreads, out completionPortThreads);
            Console.WriteLine("Worker Threads = {0}; Available Completion Port Threads = {1};", workerThreads, completionPortThreads);
            Console.Out.Flush();
            Thread.Sleep(2000);
        }
    }
}

public class Startup
{
    // This code configures Web API. The Startup class is specified as a type
    // parameter in the WebApp.Start method.
    public void Configuration(IAppBuilder appBuilder)
    {
        // Configure Web API for self-host. 
        HttpConfiguration config = new HttpConfiguration();
        config.Routes.MapHttpRoute(
            name: "DefaultApi",
            routeTemplate: "{controller}",
            defaults: new { @controller = "Default" }
        );

        appBuilder.UseWebApi(config);
    }
}

Answer 2

For others coming to this post there is a similar issue in System.Web.Http.SelfHost which by default uses CPU Cores * 100 threads which is the same as the default ThreadPool limit ending in a deadlock as Mono pre-creates all the threads.

The easiest way to get around this is to set the MONO_THREADS_PER_CPU environment variable to a value greater than 1 or use the --server flag when running the Mono application.