async.js源码阅读笔记

异步编程在python中有twisted库来处理，即使是使用tornado这样的web框架，异步实现也是使用twisted，所以对于安装tornado，twisted是一个必不可少的依赖包。c#的异步编程没有做过，但是听说很是强大，很多东西都被其它语言的实现所借鉴。而对于javascript这样的语言，事件回调机制乃是天生具备，但是要优雅地实现异步编程，单纯使用setTimeout的实现过于丑陋且不便于分析逻辑。老赵曾经实现过一个叫Wind(原名Jscex)的库，功能也很强大，但是没有玩过，有时间也会进行分析的。今天我们的主角是来自caolan的async库（https://github.com/caolan/async）。

接口概览

• 集合函数
  • each -对数组中的每个元素执行迭代函数
  • map -通过将迭代函数应用在数组的每个元素上产生新的数组
  • filter -返回测试通过的数组元素组成的新数组
  • reject -和filter相反，返回原数组中删除通过测试的元素的新数组
  • reduce -进行化简运算
  • detect -检测是否有通过测试的元素，回调参数为通过测试的第一个元素（array element）。
  • sortBy -异步排序
  • some -检测是否有通过测试的元素，回调参数为是否有通过测试测试的元素（true or false）。
  • every -检测是否每个元素都通过测试，使用和some一样
  • concat -连接对每个元素执行map结果集为一个数组
• 流程控制
  • series
  • parallel
  • whilst
  • until
  • forever
  • waterfall
  • compose
  • applyEach
  • queue
  • cargo
  • auto
  • iterator
  • apply
  • nextTick
  • times

集合函数源码分析

• each的分析

在探究each的实现前，我们首先来看三个辅助函数_each,only_once和_eachLimit

var _each = function (arr, iterator) {        if (arr.forEach) {            return arr.forEach(iterator);        }        for (var i = 0; i < arr.length; i += 1) {            iterator(arr[i], i, arr);        }    };

_each函数仅仅是作一个Array.prototype.forEach的兼容实现

function only_once(fn) {        var called = false;        return function() {            if (called) throw new Error("Callback was already called.");            called = true;            fn.apply(root, arguments);        }    }

only_once是为了保证传入的参数只执行一次，这个特性会在xxxSeries函数中用到，Series系列函数与原函数的唯一区别是保证对数组的操作是顺序的，在异步情况下也就是只有完成完成前一个数组元素操作的成功回调才能继续遍历下一个元素.

var _eachLimit = function (limit) {        return function (arr, iterator, callback) {            callback = callback || function () {};            if (!arr.length || limit <= 0) {                return callback();            }            var completed = 0;            var started = 0;            var running = 0;            (function replenish () {                if (completed >= arr.length) {                    return callback();                }                while (running < limit && started < arr.length) {                    started += 1;                    running += 1;                    iterator(arr[started - 1], function (err) {                        if (err) {                            callback(err);                            callback = function () {};                        }                        else {                            completed += 1;                            running -= 1;                            if (completed >= arr.length) {                                callback();                            }                            else {                                replenish();                            }                        }                    });                }            })();        };    };

_eachLimit返回一个包装好的遍历函数。在红色代码部分while循环中在limit限制内依次启动遍历器，如果有完成的回调，同时没有遍历完数组，则启动下一次遍历过程，这样就保证每次只有limit个函数处于运行状态。

async.each = function (arr, iterator, callback) {        callback = callback || function () {};        if (!arr.length) {            return callback();        }        var completed = 0;        _each(arr, function (x) {            iterator(x, only_once(function (err) {                if (err) {                    callback(err);                    callback = function () {};                }                else {                    completed += 1;                    if (completed >= arr.length) {                        callback(null);                    }                }            }));        });    };    async.forEach = async.each;

iterator是回调方式执行的函数，比如fs.stat(path, callback)。使用only_once的目的是为了使传入iterator的回调函数只执行一次，这样避免了在一个函数内多次调用callback使的completed的计数次数与实际数组长度不符。因为不知到哪个iterator对数组元素首先执行完成，也就无法保证执行的先后顺序，最后一个执行完成的遍历函数内执行回调函数。如果在某个遍历函数中出现错误，函数立即执行错误回调函数。（）这部分我用红色字体标识出来了。

对于each对应的eachSeries,在遍历函数对当前元素执行成功后才调用下一个遍历函数，遍历完成则回调错误函数。可以从 iterator(arr[completed]， callback）看到数组遍历的连续性。

async.eachSeries = function (arr, iterator, callback) {        callback = callback || function () {};        if (!arr.length) {            return callback();        }        var completed = 0;        var iterate = function () {            iterator(arr[completed], function (err) {                if (err) {                    callback(err);                    callback = function () {};                }                else {                    completed += 1;                    if (completed >= arr.length) {                        callback(null);                    }                    else {                        iterate();                    }                }            });        };        iterate();    };    async.forEachSeries = async.eachSeries;

eachLimit函数仅仅是加入了并行运行函数的个数限制，eachimit实现主要是使用_eachLimit包装好的函数进行并行限制

async.eachLimit = function (arr, limit, iterator, callback) {        var fn = _eachLimit(limit);        fn.apply(null, [arr, iterator, callback]);    };    async.forEachLimit = async.eachLimit;

为了方便后面函数的实现,对each相关函数的包装得到下面三个函数

    var doParallel = function (fn) {        return function () {            var args = Array.prototype.slice.call(arguments);            return fn.apply(null, [async.each].concat(args));        };    };    var doParallelLimit = function(limit, fn) {        return function () {            var args = Array.prototype.slice.call(arguments);            return fn.apply(null, [_eachLimit(limit)].concat(args));        };    };    var doSeries = function (fn) {        return function () {            var args = Array.prototype.slice.call(arguments);            return fn.apply(null, [async.eachSeries].concat(args));        };    };

doParallel是对async.each的包装,实现并行执行的功能
doParallelLimit是对_eachLimit的包装,实现最大并行数的限制
doSeries是对async.eachSeries的包装,实现顺序执行功能

• map的实现

map的实现主要是对each的调用，将调用函数调用的结果进行回调。eachFn在实现中是调用async.each进行元素的遍历,_map函数主要是将数组映射成键值对的形式{index:i,value:x}.

var _asyncMap = function (eachfn, arr, iterator, callback) {
        var results = [];        arr = _map(arr, function (x, i) {            return {index: i, value: x};        });        eachfn(arr, function (x, callback) {            iterator(x.value, function (err, v) {                results[x.index] = v;                callback(err);            });        }, function (err) {            callback(err, results);        });    };

关于mapSeries,mapLimit可以参照源码进行学习

• reduce的实现

// reduce only has a series version, as doing reduce in parallel won't
    // work in many situations.
    async.reduce = function (arr, memo, iterator, callback) {
        async.eachSeries(arr, function (x, callback) {
            iterator(memo, x, function (err, v) {
                memo = v;
                callback(err);
            });
        }, function (err) {
            callback(err, memo);
        });
    };
    // inject alias
    async.inject = async.reduce;
    // foldl alias
    async.foldl = async.reduce;

从注释中看出,reduce只有顺序版本,而没有并行版本,对于化简操作来说,并行实现是没有意义的.reduce与标准的eachSeries只是在于遍历函数上的不同,对于reduce,需要每次保存上一次的运算结果,所以最简单的实现是将结果作为函数参数进行传递,最后在回调中传入最终结果..

• filter的实现

filter的实现与map的实现类似,只是在遍历的过程中进行了元素的过滤,最后将结果按原来的顺序进行进行排序,然后将结果传入回调函数.

var _filter = function (eachfn, arr, iterator, callback) {
        var results = [];
        arr = _map(arr, function (x, i) {
            return {index: i, value: x};
        });
        eachfn(arr, function (x, callback) {
            iterator(x.value, function (v) {
                if (v) {
                    results.push(x);
                }
                callback();
            });
        }, function (err) {
            callback(_map(results.sort(function (a, b) {
                return a.index - b.index;
            }), function (x) {
                return x.value;
            }));
        });
    };

filter也存在并行和顺序执行两个版本,分别通过doParallel和doSeries来实现

• detect的实现

乘上面几种函数的实现方式,detect在遍历的过程中一旦测试的结果为真,立即向回调函数传入当前元素,否则执行回调函数

function (err) {
            main_callback();
        }

也就是说我们也可以在main_callback函数中完成检测测试的记录功能

var _detect = function (eachfn, arr, iterator, main_callback) {
        eachfn(arr, function (x, callback) {
            iterator(x, function (result) {
                if (result) {
                    main_callback(x);
                    main_callback = function () {};
                }
                else {
                    callback();
                }
            });
        }, function (err) {
            main_callback();
        });
    };

detect同样提供并行和顺序两个版本,some,any,every的实现和detect大同小异,这里就不进行解读了.

• concat的实现

concat用于将遍历中的所有的回调结果连接成一个数组,结果合并调用的是Array.prototype.concat方法.实现与上述map,filter,detect类似

var _concat = function (eachfn, arr, fn, callback) {
        var r = [];
        eachfn(arr, function (x, cb) {
            fn(x, function (err, y) {
                r = r.concat(y || []);
                cb(err);
            });
        }, function (err) {
            callback(err, r);
        });
    };

• sortBy的实现

sortBy首先使用map对数组进行索引和排序字段的键值映射,在map回调函数中对结果根据排序字段的值进行排序,然后向回调函数传递排序结果.

async.sortBy = function (arr, iterator, callback) {
        async.map(arr, function (x, callback) {
            iterator(x, function (err, criteria) {
                if (err) {
                    callback(err);
                }
                else {
                    callback(null, {value: x, criteria: criteria});
                }
            });
        }, function (err, results) {
            if (err) {
                return callback(err);
            }
            else {
                var fn = function (left, right) {
                    var a = left.criteria, b = right.criteria;
                    return a < b ? -1 : a > b ? 1 : 0;
                };
                callback(null, _map(results.sort(fn), function (x) {
                    return x.value;
                }));
            }
        });
    };

流程控制函数源码分析

• auto的实现

auto属于流程控制函数,对于有依赖关系的函数调用和使用auto来实现,比如有三个函数read,write,complete,其中complete函数只有在read函数和write函数结束后才能调用,这当然可以使用each和eachSeries函数进行实现,提供auto函数将会使的有依赖关系的函数编程更加简易.auto函数的实现稍微长了一点,我在源码中加入了注释.

async.auto = function (tasks, callback) {
        callback = callback || function () {};
        var keys = _keys(tasks);
        if (!keys.length) {
            return callback(null);
        }
        var results = {};
        var listeners = [];    //存放有依赖需要监听的函数
        //添加监听者辅助函数
        var addListener = function (fn) {
            listeners.unshift(fn);
        };
        //移除监听者辅助函数
        var removeListener = function (fn) {
            for (var i = 0; i < listeners.length; i += 1) {
                if (listeners[i] === fn) {
                    listeners.splice(i, 1);
                    return;
                }
            }
        };
        //任务执行成功回调函数,主要是遍历监听队列
        var taskComplete = function () {
            _each(listeners.slice(0), function (fn) {
                fn();
            });
        };
        //添加所有函数执行完成的事件监听
        addListener(function () {
            if (_keys(results).length === keys.length) {
                callback(null, results);
                callback = function () {};
            }
        });
        //auto实现的关键部分,遍历所有的任务
        _each(keys, function (k) {
            var task = (tasks[k] instanceof Function) ? [tasks[k]]: tasks[k];
            //包装任务执行回调函数,主要是对传入参数的包装
            var taskCallback = function (err) {
                var args = Array.prototype.slice.call(arguments, 1);
                if (args.length <= 1) {
                    args = args[0];
                }
                if (err) {
                    var safeResults = {};
                    _each(_keys(results), function(rkey) {
                        safeResults[rkey] = results[rkey];
                    });
                    safeResults[k] = args;
                    callback(err, safeResults);
                    // stop subsequent errors hitting callback multiple times
                    callback = function () {};
                }
                else {
                    //任务执行成功时,将键值加入hash表
                    results[k] = args;
                    //稍后执行任务完成回调函数
                    async.setImmediate(taskComplete);
                }
            };
            //依赖数组
            var requires = task.slice(0, Math.abs(task.length - 1)) || [];
            //绑定依赖准备函数,检测当前results结果集中是否存在依赖函数键值同时任务自身又没有执行过
            var ready = function () {
                return _reduce(requires, function (a, x) {
                    return (a && results.hasOwnProperty(x));
                }, true) && !results.hasOwnProperty(k);
            };
            if (ready()) {
                //依赖准备完成,则执行当前任务
                task[task.length - 1](taskCallback, results);
            }
            else {
                //否则,绑定监听者函数,添加到监听队列
                var listener = function () {
                    if (ready()) {
                        removeListener(listener);
                        task[task.length - 1](taskCallback, results);
                    }
                };
                addListener(listener);
            }
        });
    };

auto的实现大量运用局部变量的绑定,传入绑定后的函数体.主要思路是,已执行函数保存在results中,当依赖存在与results中并且自身不在结果集中(removeListen已经保证了函数只执行一次),则执行任务,否则加入到监听者队列,每次有任务完成则执行监听者队列里的函数(同样是通过局部绑定的ready函数进行依赖验证),当所有任务执行按成,最先添加到监听者队列的监听者条用callback进行执行结果的传递.

• waterFall的实现

看完waterFall的源码,觉得caolan的实现确实很巧妙,利用已经实现的iterator函数进行简单的包装就实现了相对来说比iterator复杂不少的函数.

async.waterfall = function (tasks, callback) {
        callback = callback || function () {};
        if (tasks.constructor !== Array) {
          var err = new Error('First argument to waterfall must be an array of functions');
          return callback(err);
        }
        if (!tasks.length) {
            return callback();
        }
        var wrapIterator = function (iterator) {
            return function (err) {
                if (err) {
                    callback.apply(null, arguments);
                    callback = function () {};
                }
                else {
                    var args = Array.prototype.slice.call(arguments, 1);
                    var next = iterator.next();
                    if (next) {
                        args.push(wrapIterator(next));
                    }
                    else {
                        args.push(callback);
                    }
                    //利用wrap后的遍历函数作为iterator的回调函数,能将函数执行将iterator的执行结果直接传入下一个遍历器
                    async.setImmediate(function () {
                        iterator.apply(null, args);
                    });
                }
            };
        };
        wrapIterator(async.iterator(tasks))();
    };

这里的trick就是巧妙地处理了回调函数的参数列表,使得上一次的回调结果直接传如了下一个遍历器,但是使用是要注意tasks表中的前一个函数的回调函数传递的变量个数与下个函数传入的变量个数需要一致,否则无法完成完整的waterFall操作.

• parallel的实现

each用于object,而map运用于数组,对于流程控制,既需要处理键值对参数的情况,也需要处理数组参数的情况.但是根据each和map的实现不同,map默认会将处理结果传入回调参数,而对于each需要将结果根据参数的键值对进行重新映射,parallel传入的函数需要具有下面这种形式,才能进行结果的传递

function(callback) {
    //do some stuff
    callback(null[,arg1][,arg2]);
}

var _parallel = function(eachfn, tasks, callback) {
        callback = callback || function () {};
        if (tasks.constructor === Array) {
            eachfn.map(tasks, function (fn, callback) {
                if (fn) {
                    fn(function (err) {
                        var args = Array.prototype.slice.call(arguments, 1);
                        if (args.length <= 1) {
                            args = args[0];
                        }
                        callback.call(null, err, args);
                    });
                }
            }, callback);
        }
        else {
            var results = {};
            eachfn.each(_keys(tasks), function (k, callback) {
                tasks[k](function (err) {
                    var args = Array.prototype.slice.call(arguments, 1);
                    if (args.length <= 1) {
                        args = args[0];
                    }
                    results[k] = args;
                    callback(err);
                });
            }, function (err) {
                callback(err, results);
            });
        }
    };

从实现可以看出map函数是通过调用callback.call(null, err, args)来进行参数传递的,而each是通过局部变量results对象以键值对的方式在函数执行回调函数中对结果进行存储,当所有的并行任务处理完成后执行回调函数,对于数组类型的参数,回调函数传入的变量类型也是数组,对于键值对方式传入的参数,回调函数传入的参数同样是以简直对的方式进行查询.

parallel同样提供普通和parallelLimit两个版本,后者同样是调用_mapLimit和_eachLimit的实现方式.

• series的实现

series的实现与parallel类似,只是把函数实现内部的版本改为mapSeries和eachSeries.

• iterator的实现

iterator函数实现了传统的迭代器功能,返回的迭代器支持next方法访问遍历器中的下一个对象.

async.iterator = function (tasks) {
        var makeCallback = function (index) {
            var fn = function () {
                if (tasks.length) {
                    tasks[index].apply(null, arguments);
                }
                return fn.next();
            };
            fn.next = function () {
                return (index < tasks.length - 1) ? makeCallback(index + 1): null;
            };
            return fn;
        };
        return makeCallback(0);
    };

没执行一次iterator对象便会返回下一个遍历器对象,同时也支持使用next方式访问.

• apply的分析

apply比较简单,主要是对回调类型的函数进行包装,简化代码复杂度.

async.apply = function (fn) {
        var args = Array.prototype.slice.call(arguments, 1);
        return function () {
            return fn.apply(
                null, args.concat(Array.prototype.slice.call(arguments))
            );
        };
    };

从源码可以看出,apply实际上做了一件很简单的事情,返回一个包装好的函数,将回调函数(比如parallel传入的)推到fn的已有参数后面作为该函数的callback,前面我们说过只有

function(callback) {
    //do some stuff
    callback(null[,arg1][,arg2]);
}

形式的参数才能正确传递参数.

• whilst的分析

whilst不断进行test测试,如通过则循环执行iterator函数,知道不能通过测试为止,成功则执行回调函数callback

async.whilst = function (test, iterator, callback) {
        if (test()) {
            iterator(function (err) {
                if (err) {
                    return callback(err);
                }
                async.whilst(test, iterator, callback);
            });
        }
        else {
            callback();
        }
    };

需要注意的是这里需要保存测试过程中的条件变量,以使得iterator的改变会影响test的结果,通常使用与函数同一个scope的变量即可.

doWhilst只是一个变体,这里不作分析.

• until的分析

与whilst类似,只是终止条件是满足测试,仅贴出源码

async.until = function (test, iterator, callback) {
        if (!test()) {
            iterator(function (err) {
                if (err) {
                    return callback(err);
                }
                async.until(test, iterator, callback);
            });
        }
        else {
            callback();
        }
    };

• queue的分析

queue提供了比较丰富的方法,worker函数同时对concurrency个queue中的元素进行操作,每次执行unshift和push操作都会激发_insert函数中的async.setmmediate(qprocess),开启处理进程,开始处理后workers数+1,当有workers数小于concurrency数时可以持续向队列中插入数据,如果并行达到最大数量,那么只能等待前面的进程执行完毕,在进程回调函数中有q.process()即是启动下一次进程.

async.queue = function (worker, concurrency) {
        //默认并行数为1
        if (concurrency === undefined) {
            concurrency = 1;
        }
        //包装unshift和push操作,插入后开启进程
        function _insert(q, data, pos, callback) {
          if(data.constructor !== Array) {
              data = [data];
          }
          _each(data, function(task) {
              var item = {
                  data: task,
                  callback: typeof callback === 'function' ? callback : null
              };
              if (pos) {
                q.tasks.unshift(item);
              } else {
                q.tasks.push(item);
              }
              //最大并行数饱和时执行saturated回调函数
              if (q.saturated && q.tasks.length === concurrency) {
                  q.saturated();
              }
              //启动进程
              async.setImmediate(q.process);
          });
        }
        var workers = 0;
        var q = {
            tasks: [],
            concurrency: concurrency,
            saturated: null,
            empty: null,
            drain: null,
            push: function (data, callback) {
              _insert(q, data, false, callback);
            },
            unshift: function (data, callback) {
              _insert(q, data, true, callback);
            },
            process: function () {
                if (workers < q.concurrency && q.tasks.length) {
                    var task = q.tasks.shift();
                    //队列中的元素为空时回调empty
                    if (q.empty && q.tasks.length === 0) {
                        q.empty();
                    }
                    workers += 1;
                    var next = function () {
                        workers -= 1;
                        if (task.callback) {
                            task.callback.apply(task, arguments);
                        }
                        //队列数为空,且正在执行的进程为空时执行回调函数drain
                        if (q.drain && q.tasks.length + workers === 0) {
                            q.drain();
                        }
                        q.process();
                    };
                    var cb = only_once(next);
                    worker(task.data, cb);
                }
            },
            length: function () {
                return q.tasks.length;
            },
            running: function () {
                return workers;
            }
        };
        return q;
    };

• cargo的分析

cargo与queue的唯一不同是,cargo中的任务是同时执行,如果执行期间有新条目加入,则只能等待.与queue代码相比,最大的不同是在process函数中首先判断是否在执行,如果正在执行则等待,当有函数成功返回时,process重新启动.还有一点是tasks.splice(0,payload),可见cargo是每次从tasks数组中拿出payload个任务在worker中执行,捏可以提供顺序,并行等方式执行这些任务.

async.cargo = function (worker, payload) {
        var working     = false,
            tasks       = [];
        var cargo = {
            tasks: tasks,
            payload: payload,
            saturated: null,
            empty: null,
            drain: null,
            push: function (data, callback) {
                if(data.constructor !== Array) {
                    data = [data];
                }
                _each(data, function(task) {
                    tasks.push({
                        data: task,
                        callback: typeof callback === 'function' ? callback : null
                    });
                    if (cargo.saturated && tasks.length === payload) {
                        cargo.saturated();
                    }
                });
                //在下一个空闲点执行process,通常是创建cargo加入任务之后
                async.setImmediate(cargo.process);
            },
            process: function process() {
                //如果正在执行,立即返回
                if (working) return;
                if (tasks.length === 0) {
                    if(cargo.drain) cargo.drain();
                    return;
                }
                var ts = typeof payload === 'number'
                            ? tasks.splice(0, payload)
                            : tasks.splice(0);
                var ds = _map(ts, function (task) {
                    return task.data;
                });
                if(cargo.empty) cargo.empty();
                working = true;
                worker(ds, function () {
                    working = false;
                    var args = arguments;
                    _each(ts, function (data) {
                        if (data.callback) {
                            data.callback.apply(null, args);
                        }
                    });
                    //回调成功时重新开启进程
                    process();
                });
            },
            length: function () {
                return tasks.length;
            },
            running: function () {
                return working;
            }
        };
        return cargo;
    };

实用函数分析

async库提供一些实用函数比如日志记录,函数结果缓存,消除冲突函数,这里主要解析一下memoize函数.memoize是这样一个处理流程,如果缓存memo存在hash值对应的回调函数结果,则返回立即传递结果到传入的回调函数执行,否则如果键存在与队列中,则添加到键对应的数组,如果队列中也不存在则调用当前函数,在回调函数中,缓存回调函数的结果,同时对原函数执行的结果执行回调操作.

async.memoize = function (fn, hasher) {
        var memo = {};
        var queues = {};
        hasher = hasher || function (x) {
            return x;
        };
        var memoized = function () {
            var args = Array.prototype.slice.call(arguments);
            var callback = args.pop();
            var key = hasher.apply(null, args);
            if (key in memo) {
                callback.apply(null, memo[key]);
            }
            else if (key in queues) {
                //当fn正在执行时,queues的键对应的数组已经建立,可以继续添加回调函数
                queues[key].push(callback);
            }
            else {
                queues[key] = [callback];
                fn.apply(null, args.concat([function () {
                    memo[key] = arguments;
                    var q = queues[key];
                    delete queues[key];
                    //回调队列中所有的回调函数,包括fn执行过程中添加的回调函数
                    for (var i = 0, l = q.length; i < l; i++) {
                      q[i].apply(null, arguments);
                    }
                }]));
            }
        };
        memoized.memo = memo;
        memoized.unmemoized = fn;
        return memoized;
    };

小结

从源码阅读的情况来看caolan的js确实写的很扎实,看源码的过程中学到很多tricks,有些在平常的编程实践中没有想到但很实用的东西.弄清楚了各部分的回调关系,async的实现也没有那么复杂,阅读源码之后,对async库使用的理解也会加强很多,希望以后能挤出时间写出更多的源码阅读笔记.由于编程功力不深厚,难免有不准确,不全面的地方,望指正.