HttpWebRequest - Asynchronous Programming Model/Task.Factory.FromAsyc

潜水三年，今天终于忍不住开始了博客旅程。本人文笔不咋的。所以先从翻译外文开始吧。

声明：本篇文章为我第一次翻译外文，如有言辞不准确的地方，请大家查看原文，欢迎转载，但请注明出处，谢谢。

本篇文章主要讲解了用HttpWebRequest多线程Post和Get的实现方式，可以说是最全的方式了。可以值得一看。

原文链接：http://www.matlus.com/httpwebrequest-asynchronous-programming

 

     自从.net1.1发布以来，异步编程模型(APM)一直以来都是众多程序员用来解决异步I/O的首选方案。当遇到I/O阻塞的问题时，大家也多通过多线程来提高程序的性能。但是使用多线程去解决I/O阻塞如：硬盘I/O,网络I/O 在大多数情况下并不能给程序带来速度上的提升，反而会影响程序的性能。如果你的程序涉及大量的计算阻塞并且你程序的运行平台的cpu是多核的而且你的程序确实在计算方面要花费的系统开销要远远多于多线程管理的开销，那么用多线程来提升其性能的优势是相当明显的。具体在这方面的讨论，你可以参照这篇博客：Data Parallel – Parallel Programming in C#/.NET

     造成APM不容易使用的主要原因是因为它的回调机制造成的，这种回调机制也意味着你开始一段逻辑在一个方法里面，但是你得到这个逻辑的结果可能在另一个方法里面。所以当你决定来使用APM来构建你的代码时，你必须认真考虑这种回调机制对你代码造成的影响。在这篇文章中，我将介绍几种方法让你可以使用HttpWebRequest多线程来创建多个Http请求来提高你程序的性能和吞吐量。我反对多线程滥用。

     使用Http多连接一般遇到的的几种情况如下：

     1.多个http同时链接多个站点，而且必须每个站点都返回时，我们程序才继续往下执行。

     2.多个http先后依次发起，而且前一返回结果将作为因素影响后一请求。

     3.多个http同时发起链接多个站点，但在结果返回之前程序的后续逻辑依然运行。这种模式一般在异步工作流中常见。

 上述的的http请求可以是Get也可以是Post。但相对于Get方式， Post方式更加复杂。

这篇稳重我将想你展示使用APM或者Task多种方法去实现上述的情况。

HttpWebRequest 使用APM比较复杂，因为当你以异步方式使用POST数据请求的时候，它涉及到两个异步过程，因此就有两次异步回调。首先，让我们来实现HttpWebResponsed的异步吧。在编码方面为了照顾那些不喜欢Lambda表达式的人，我将不会施工用Lambda。

首先我构建一个HttpSocket类来更加方便的使用异步Http.这个类里面主要有四个方法：

1. PostAsync 
2. GetAsync 
3. PostAsyncTask 
4. GetAsyncTask    

   PostAsync 和 GetAsync 方法 使用成对的Begin/End 方法来实现异步，PostAsyncTask和GetAsyncTask使用Task来实现异步。特别强调的是Task.Factory.FromAsync方法。从程序性能上来说，两种方式的性能差不多，但是使用PostAsyncTask和GetAsyncTask方法的实现方式使用的资源相当较少。这种优势在那些大型项目中当你需要一个重量级的Http请求时，尤为明显。

using System;using System.Collections.Generic;using System.Linq;using System.Text;using System.Net;using System.Collections.Specialized;using System.IO;using System.Threading;using System.Threading.Tasks;using System.Runtime.Serialization.Json;namespace ConsoleApplication3{  public static class HttpSocket  {    static HttpWebRequest CreateHttpWebRequest(string url, string httpMethod, string contentType)    {      var httpWebRequest = (HttpWebRequest)WebRequest.Create(url);      httpWebRequest.ContentType = contentType;      httpWebRequest.Method = httpMethod;      return httpWebRequest;    }    static byte[] GetRequestBytes(NameValueCollection postParameters)    {      if (postParameters == null || postParameters.Count == 0)        return new byte[0];      var sb = new StringBuilder();      foreach (var key in postParameters.AllKeys)        sb.Append(key + "=" + postParameters[key] + "&");      sb.Length = sb.Length - 1;      return Encoding.UTF8.GetBytes(sb.ToString());    }    static void BeginGetRequestStreamCallback(IAsyncResult asyncResult)    {      Stream requestStream = null;      HttpWebRequestAsyncState asyncState = null;      try      {        asyncState = (HttpWebRequestAsyncState)asyncResult.AsyncState;        requestStream = asyncState.HttpWebRequest.EndGetRequestStream(asyncResult);        requestStream.Write(asyncState.RequestBytes, 0, asyncState.RequestBytes.Length);        requestStream.Close();        asyncState.HttpWebRequest.BeginGetResponse(BeginGetResponseCallback,          new HttpWebRequestAsyncState          {            HttpWebRequest = asyncState.HttpWebRequest,            ResponseCallback = asyncState.ResponseCallback,            State = asyncState.State          });      }      catch (Exception ex)      {        if (asyncState != null)          asyncState.ResponseCallback(new HttpWebRequestCallbackState(ex));        else          throw;      }      finally      {        if (requestStream != null)          requestStream.Close();      }    }    static void BeginGetResponseCallback(IAsyncResult asyncResult)    {      WebResponse webResponse = null;      Stream responseStream = null;      HttpWebRequestAsyncState asyncState = null;      try      {        asyncState = (HttpWebRequestAsyncState)asyncResult.AsyncState;        webResponse = asyncState.HttpWebRequest.EndGetResponse(asyncResult);        responseStream = webResponse.GetResponseStream();        var webRequestCallbackState = new HttpWebRequestCallbackState(responseStream, asyncState.State);        asyncState.ResponseCallback(webRequestCallbackState);        responseStream.Close();        responseStream = null;        webResponse.Close();        webResponse = null;      }      catch (Exception ex)      {        if (asyncState != null)          asyncState.ResponseCallback(new HttpWebRequestCallbackState(ex));        else          throw;      }      finally      {        if (responseStream != null)          responseStream.Close();        if (webResponse != null)          webResponse.Close();      }    }    /// <summary>    /// If the response from a remote server is in text form    /// you can use this method to get the text from the ResponseStream    /// This method Disposes the stream before it returns    /// </summary>    /// <param name="responseStream">The responseStream that was provided in the callback delegate's HttpWebRequestCallbackState parameter</param>    /// <returns></returns>    public static string GetResponseText(Stream responseStream)    {      using (var reader = new StreamReader(responseStream))      {        return reader.ReadToEnd();      }    }    /// <summary>    /// This method uses the DataContractJsonSerializer to    /// Deserialize the contents of a stream to an instance    /// of an object of type T.    /// This method disposes the stream before returning    /// </summary>    /// <typeparam name="T"></typeparam>    /// <param name="stream">A Stream. Typically the ResponseStream</param>    /// <returns>An instance of an object of type T</returns>    static T DeSerializeToJson<T>(Stream stream)    {      using (stream)      {        var deserializer = new DataContractJsonSerializer(typeof(T));        return (T)deserializer.ReadObject(stream);      }    }    /// <summary>    /// This method does an Http POST sending any post parameters to the url provided    /// </summary>    /// <param name="url">The url to make an Http POST to</param>    /// <param name="postParameters">The form parameters if any that need to be POSTed</param>    /// <param name="responseCallback">The callback delegate that should be called when the response returns from the remote server</param>    /// <param name="state">Any state information you need to pass along to be available in the callback method when it is called</param>    /// <param name="contentType">The Content-Type of the Http request</param>    public static void PostAsync(string url, NameValueCollection postParameters,      Action<HttpWebRequestCallbackState> responseCallback, object state = null,      string contentType = "application/x-www-form-urlencoded")    {      var httpWebRequest = CreateHttpWebRequest(url, "POST", contentType);      var requestBytes = GetRequestBytes(postParameters);      httpWebRequest.ContentLength = requestBytes.Length;      httpWebRequest.BeginGetRequestStream(BeginGetRequestStreamCallback,        new HttpWebRequestAsyncState()        {          RequestBytes = requestBytes,          HttpWebRequest = httpWebRequest,          ResponseCallback = responseCallback,            State = state        });    }    /// <summary>    /// This method does an Http GET to the provided url and calls the responseCallback delegate    /// providing it with the response returned from the remote server.    /// </summary>    /// <param name="url">The url to make an Http GET to</param>    /// <param name="responseCallback">The callback delegate that should be called when the response returns from the remote server</param>    /// <param name="state">Any state information you need to pass along to be available in the callback method when it is called</param>    /// <param name="contentType">The Content-Type of the Http request</param>    public static void GetAsync(string url, Action<HttpWebRequestCallbackState> responseCallback,      object state = null, string contentType = "application/x-www-form-urlencoded")    {      var httpWebRequest = CreateHttpWebRequest(url, "GET", contentType);      httpWebRequest.BeginGetResponse(BeginGetResponseCallback,        new HttpWebRequestAsyncState()        {          HttpWebRequest = httpWebRequest,          ResponseCallback = responseCallback,          State = state        });    }    public static void PostAsyncTask(string url, NameValueCollection postParameters,      Action<HttpWebRequestCallbackState> responseCallback, object state = null,      string contentType = "application/x-www-form-urlencoded")    {      var httpWebRequest = CreateHttpWebRequest(url, "POST", contentType);      var requestBytes = GetRequestBytes(postParameters);      httpWebRequest.ContentLength = requestBytes.Length;      var asyncState = new HttpWebRequestAsyncState()      {        RequestBytes = requestBytes,        HttpWebRequest = httpWebRequest,        ResponseCallback = responseCallback,        State = state      };      Task.Factory.FromAsync<Stream>(httpWebRequest.BeginGetRequestStream,        httpWebRequest.EndGetRequestStream, asyncState, TaskCreationOptions.None)        .ContinueWith<HttpWebRequestAsyncState>(task =>        {          var asyncState2 = (HttpWebRequestAsyncState)task.AsyncState;          using (var requestStream = task.Result)          {            requestStream.Write(asyncState2.RequestBytes, 0, asyncState2.RequestBytes.Length);          }                    return asyncState2;        })        .ContinueWith(task =>        {          var httpWebRequestAsyncState2 = (HttpWebRequestAsyncState)task.Result;          var hwr2 = httpWebRequestAsyncState2.HttpWebRequest;          Task.Factory.FromAsync<WebResponse>(hwr2.BeginGetResponse,            hwr2.EndGetResponse, httpWebRequestAsyncState2, TaskCreationOptions.None)            .ContinueWith(task2 =>            {              WebResponse webResponse = null;              Stream responseStream = null;              try              {                var asyncState3 = (HttpWebRequestAsyncState)task2.AsyncState;                webResponse = task2.Result;                responseStream = webResponse.GetResponseStream();                responseCallback(new HttpWebRequestCallbackState(responseStream, asyncState3));              }              finally              {                if (responseStream != null)                  responseStream.Close();                if (webResponse != null)                  webResponse.Close();              }            });        });    }    public static void GetAsyncTask(string url, Action<HttpWebRequestCallbackState> responseCallback,      object state = null, string contentType = "application/x-www-form-urlencoded")    {      var httpWebRequest = CreateHttpWebRequest(url, "GET", contentType);      Task.Factory.FromAsync<WebResponse>(httpWebRequest.BeginGetResponse,        httpWebRequest.EndGetResponse, null).ContinueWith(task =>          {            var webResponse = task.Result;            var responseStream = webResponse.GetResponseStream();            responseCallback(new HttpWebRequestCallbackState(webResponse.GetResponseStream(), state));            responseStream.Close();            webResponse.Close();          });    }  }  /// <summary>  /// This class is used to pass on "state" between each Begin/End call  /// It also carries the user supplied "state" object all the way till  /// the end where is then hands off the state object to the  /// HttpWebRequestCallbackState object.  /// </summary>  class HttpWebRequestAsyncState  {    public byte[] RequestBytes { get; set; }    public HttpWebRequest HttpWebRequest { get; set; }    public Action<HttpWebRequestCallbackState> ResponseCallback { get; set; }    public Object State { get; set; }  }  /// <summary>  /// This class is passed on to the user supplied callback method  /// as a parameter. If there was an exception during the process  /// then the Exception property will not be null and will hold  /// a reference to the Exception that was raised.  /// The ResponseStream property will be not null in the case of  /// a sucessful request/response cycle. Use this stream to  /// exctract the response.  /// </summary>  public class HttpWebRequestCallbackState  {    public Stream ResponseStream { get; private set; }    public Exception Exception { get; private set; }    public Object State { get; set; }    public HttpWebRequestCallbackState(Stream responseStream, object state)    {      ResponseStream = responseStream;      State = state;    }    public HttpWebRequestCallbackState(Exception exception)    {      Exception = exception;    }  }}

这个大家可以参考放到自己类库中。

接下来几个是HttpScoket使用案例：

1.在异步工作中使用HttpWebRequests 

使用HttpSocket来实现异步工作相当简单。你只需要调用PostXXX或者GetXXX并传递相应参数给服务器端，并最后定义并使用一个回调函数就可以了。具体的实现例子如下：

ServicePointManager.DefaultConnectionLimit = 50;      var url = "http://localhost/HttpTaskServer/default.aspx";      var iterations = 1000;      for (int i = 0; i < iterations; i++)      {        var postParameters = new NameValueCollection();        postParameters.Add("data", i.ToString());        HttpSocket.PostAsync(url, postParameters, callbackState =>          {            if (callbackState.Exception != null)              throw callbackState.Exception;            Console.WriteLine(HttpSocket.GetResponseText(callbackState.ResponseStream));          });      }

上面例子中我们使用了PostAsync这个方面。每个方法我们都通过一个Lamba表达式定义了一个回调方法。另外值得注意的是，在上述代码的想要很快，因此我并没有阻塞主线程，同时也没有创建其他线程。在每个Response后，都会调用callback，而代码中我们的Callback也只是仅仅是将callback写到console中。另外一件值得注意的事是，Response不会按照请求的顺序依次返回。

如果你发起多个请求的时候，你会发现你只能得到两个Request，这是因为，DefaultconnectionLimit这个值是2.所以在上面代码中，使用了ServicePointManager.DefaultConnectionLimit = 50;来放宽它的值。

 

2.多个并发Request并在所有Request都得到Response后程序才继续执行

接下来，将有五种方法来实现上述功能：

1）使用Task实现

static void Main(string[] args)    {      ServicePointManager.DefaultConnectionLimit = 100;      var url = "http://localhost/HttpTaskServer/default.aspx";      var iterations = 10;      var workItems = (from w in Enumerable.Range(0, 100)                       let postParameters = new NameValueCollection { { "data", w.ToString() } }                       select new Work() { Id = w, PostParameters = postParameters }).ToArray();      Thread.Sleep(1000);      Benchmarker.MeasureExecutionTime("ParallelForEach              ", iterations, CallHttpWebRequestASyncParallelForEachAndWaitOnAll, url, workItems);      Benchmarker.MeasureExecutionTime("AsyncAndWaitOnAll            ", iterations, CallHttpWebRequestAsyncAndWaitOnAll, url, workItems);      Benchmarker.MeasureExecutionTime("ASyncDataParallelAndWaitOnAll", iterations, CallHttpWebRequestASyncDataParallelAndWaitOnAll, url, workItems);      Benchmarker.MeasureExecutionTime("TaskAndWaitOnAll             ", iterations, CallHttpWebRequestTaskAndWaitOnAll, url, workItems);      Benchmarker.MeasureExecutionTime("SyncAndWaitOnAll             ", iterations, CallHttpWebRequestSyncAndWaitOnAll, url, workItems);      Console.WriteLine("All work Done.");            Console.ReadLine();    }  class Work  {    public int Id { get; set; }    public NameValueCollection PostParameters { get; set; }    public string ResponseData { get; set; }    public Exception Exception { get; set; }  }

CallHttpWebRequestTaskAndWaitOnAll方法：

 /// <summary>    /// This method makes a bunch (workItems.Count()) of HttpRequests using Tasks    /// The work each task performs is a synchronous Http request. Essentially each    /// Task is performed on a different thread and when all threads have completed    /// this method returns    /// </summary>    /// <param name="url"></param>    /// <param name="workItems"></param>    static void CallHttpWebRequestTaskAndWaitOnAll(string url, IEnumerable<Work> workItems)    {            var tasks = new List<Task>();      foreach (var workItem in workItems)      {        tasks.Add(Task.Factory.StartNew(wk =>        {          var wrkItem = (Work)wk;          wrkItem.ResponseData = GetWebResponse(url, wrkItem.PostParameters);        }, workItem));      }      Task.WaitAll(tasks.ToArray());    }

2）使用构建多个Thread实现

/// <summary>    /// This method makes a bunch (workItems.Count()) of HttpRequests synchronously    /// using a pool of threads with a certain cap on the number of threads.    /// As soon as a thread finishes a now job is started till no more work items are left.    /// After all http requests are complete, this method returns    /// </summary>    /// <param name="url"></param>    /// <param name="workItems"></param>    static void CallHttpWebRequestSyncAndWaitOnAll(string url, IEnumerable<Work> workItems)    {      //Since the threads will be blocked for the most part (not using the CPU)      //We're using 4 times as many threads as there are cores on the machine.      //Play with this number to ensure you get the performance you keeping an      //eye on resource utlization as well.      int maxThreadCount = Environment.ProcessorCount * 4;      int executingThreads = 0;      //This variable is used to throttle the number of threads      //created to the maxThreadCount      object lockObj = new object();      foreach (var workItem in workItems)      {        ThreadPool.QueueUserWorkItem((state) =>          {            var work = (Work)state;            try            {              work.ResponseData = GetWebResponse(url, work.PostParameters);              Interlocked.Decrement(ref executingThreads);            }            catch (Exception ex)            {              work.Exception = ex;              Interlocked.Decrement(ref executingThreads);            }          }, workItem);        //If maxThreadCount threads have been spawned        //then wait for any of them to finish before        //spawning additional threads. Therby limiting        //the number of executing (and spawned)        //threads to maxThreadCount        lock (lockObj)        {          executingThreads++;          while (executingThreads == maxThreadCount)            Thread.Sleep(1);        }      }      //Wait on all executing threads to complete      while (executingThreads != 0)        Thread.Sleep(1);    }

3）多个HttpRequest并在请求的同时阻塞主线程

 /// <summary>    /// This method makes a bunch (workItems.Count()) of HttpRequests asynchronously    /// The main thread is blocked until all asynchronous jobs are complete.    /// After all http requests are complete, this method returns    /// </summary>    /// <param name="url"></param>    /// <param name="workItems"></param>    private static void CallHttpWebRequestAsyncAndWaitOnAll(string url, IEnumerable<Work> workItems)    {      var pending = workItems.Count();      using (var mre = new ManualResetEvent(false))      {        foreach (var workItem in workItems)        {          HttpSocket.PostAsync(url, workItem.PostParameters, callbackState =>          {            var hwrcbs = (HttpWebRequestCallbackState)callbackState;            using (var responseStream = hwrcbs.ResponseStream)            {              var reader = new StreamReader(responseStream);              ((Work)hwrcbs.State).ResponseData = reader.ReadToEnd();              if (Interlocked.Decrement(ref pending) == 0)                mre.Set();            }          }, workItem);        }        mre.WaitOne();      }    }

4）使用Data Parallel

    /// <summary>    /// This method makes a bunch (workItems.Count()) of HttpRequests asynchronously    /// but partitions the workItem into chunks. The number of chunks is determined    /// by the number of cores in the machine. The main thread is blocked    /// until all asynchronous jobs are complete    /// After all http requests are complete, this method returns    /// </summary>    /// <param name="url"></param>    /// <param name="workItems"></param>    private static void CallHttpWebRequestASyncDataParallelAndWaitOnAll(string url, IEnumerable<Work> workItems)    {      var coreCount = Environment.ProcessorCount;      var itemCount = workItems.Count();      var batchSize = itemCount / coreCount;      var pending = itemCount;      using (var mre = new ManualResetEvent(false))      {        for (int batchCount = 0; batchCount < coreCount; batchCount++)        {          var lower = batchCount * batchSize;          var upper = (batchCount == coreCount - 1) ? itemCount : lower + batchSize;          var workItemsChunk = workItems.Skip(lower).Take(upper).ToArray();          foreach (var workItem in workItemsChunk)          {            HttpSocket.PostAsync(url, workItem.PostParameters, callbackState =>              {                var hwrcbs = (HttpWebRequestCallbackState)callbackState;                using (var responseStream = hwrcbs.ResponseStream)                {                  var reader = new StreamReader(responseStream);                  ((Work)hwrcbs.State).ResponseData = reader.ReadToEnd();                  if (Interlocked.Decrement(ref pending) == 0)                    mre.Set();                }              }, workItem);          }        }        mre.WaitOne();      }    }

5）使用Parallel.ForEach

/// <summary>    /// This method makes a bunch (workItems.Count()) of HttpRequests synchronously    /// using Parallel.ForEach. Behind the scenes Parallel.ForEach partitions the    /// workItem into chunks (Data Parallel). The main thread is blocked    /// until all workItems have been processed.    /// After all http requests are complete, this method returns    /// </summary>    /// <param name="url"></param>    /// <param name="workItems"></param>    private static void CallHttpWebRequestSyncParallelForEachAndWaitOnAll(string url, IEnumerable<Work> workItems)    {      Parallel.ForEach(workItems, work =>      {        try        {          work.ResponseData = GetWebResponse(url, work.PostParameters);        }        catch (Exception ex)        {          work.Exception = ex;        }      });    }

下表是各种方式的性能测试：

 

MethodTotalAverageMinMaxAsyncAndWaitOnAll20114.21312011.42132004.70272055.6508ASyncDataParallelAndWaitOnAll21306.04321306.0432005.83132412.8763SyncAndWaitOnAll23136.39492313.63942011.82772603.461ParallelForEach23438.73412343.87342010.13353154.8706TaskAndWaitOnAll27978.0522797.80522211.21314459.7625

 同步请求代码：

 static string GetWebResponse(string url, NameValueCollection parameters)    {      var httpWebRequest = (HttpWebRequest)WebRequest.Create(url);      httpWebRequest.ContentType = "application/x-www-form-urlencoded";      httpWebRequest.Method = "POST";      var sb = new StringBuilder();      foreach (var key in parameters.AllKeys)        sb.Append(key + "=" + parameters[key] + "&");      sb.Length = sb.Length - 1;      byte[] requestBytes = Encoding.UTF8.GetBytes(sb.ToString());      httpWebRequest.ContentLength = requestBytes.Length;      using (var requestStream = httpWebRequest.GetRequestStream())      {        requestStream.Write(requestBytes, 0, requestBytes.Length);      }      Task<WebResponse> responseTask = Task.Factory.FromAsync<WebResponse>(httpWebRequest.BeginGetResponse, httpWebRequest.EndGetResponse, null);      using (var responseStream = responseTask.Result.GetResponseStream())      {        var reader = new StreamReader(responseStream);        return reader.ReadToEnd();      }    }

测试性能代码：

using System;using System.Collections.Generic;using System.Linq;using System.Text;using System.Diagnostics;namespace Diagnostics.Performance{  public static class Benchmarker  {    private static void WarmUp(Action action)    {      action();      GC.Collect();    }    private static void DisplayResults(string benchmarkName, TimeSpan totalTime, TimeSpan averageTime, TimeSpan minTime, TimeSpan maxTime)    {      Console.WriteLine("---------------------------------");      Console.WriteLine(benchmarkName);      Console.WriteLine("\tTotal time  : " + totalTime.TotalMilliseconds + "ms");      Console.WriteLine("\tAverage time: " + averageTime.TotalMilliseconds + "ms");      Console.WriteLine("\tMin time    : " + minTime.TotalMilliseconds + "ms");      Console.WriteLine("\tMax time    : " + maxTime.TotalMilliseconds + "ms");      Console.WriteLine("---------------------------------");      Console.WriteLine();    }    public static void MeasureExecutionTime(string benchmarkName, int noOfIterations, Action action)    {      var totalTime = new TimeSpan(0);      var averageTime = new TimeSpan(0);      var minTime = TimeSpan.MaxValue;      var maxTime = TimeSpan.MinValue;      WarmUp(action);      GC.Collect();      GC.WaitForPendingFinalizers();      GC.Collect();      var total = new TimeSpan(0);      var sw = Stopwatch.StartNew();       for (int i = 0; i < noOfIterations; i++)      {        sw.Restart();        action();        sw.Stop();        GC.Collect();        GC.WaitForPendingFinalizers();        GC.Collect();        var thisIteration = sw.Elapsed;        total += thisIteration;        if (thisIteration > maxTime)          maxTime = thisIteration;        if (thisIteration < minTime)          minTime = thisIteration;      }      totalTime = total;      averageTime = new TimeSpan(total.Ticks / noOfIterations);      DisplayResults(benchmarkName, totalTime, averageTime, minTime, maxTime);    }  }}