Openstack Nova(十)----Instance 创建（RPC基础 ）

在前一章中， 学习了Computer API及Conductor调用RPC做下一步处理。那么， 这里就有个问题， Openstack中RPC是怎么工作的？

想要知道RPC是怎么工作的， 那么第一个要了解的是什么是RPC？

RPC（Remote Procedure Call Protocol）——远程过程调用协议，它是一种通过网络从远程计算机程序上请求服务，而不需要了解底层网络技术的协议。RPC协议假定某些传输协议的存在，如TCP或UDP，为通信程序之间携带信息数据。在OSI网络通信模型中，RPC跨越了传输层和应用层。RPC使得开发包括网络分布式多程序在内的应用程序更加容易。

从Openstack的文档中可以看到，Openstack的内部消息是基于AMQP协议的。 而且从代码结构上看， 它支持Kombu, Qpid， ZMQ三种类型。在系统中， 现在使用的是Qpid。

发送

使用方法

就以前一章的Conductor为例， 它是一个RPC的使用者。

def build_instances(self, context, instances, image, filter_properties,            admin_password, injected_files, requested_networks,            security_groups, block_device_mapping, legacy_bdm=True):        instances_p = [jsonutils.to_primitive(inst) for inst in instances]        image_p = jsonutils.to_primitive(image)        cctxt = self.client.prepare(version='1.5')        cctxt.cast(context, 'build_instances',                   instances=instances_p, image=image_p,                   filter_properties=filter_properties,                   admin_password=admin_password,                   injected_files=injected_files,                   requested_networks=requested_networks,                   security_groups=security_groups,                   block_device_mapping=block_device_mapping,                   legacy_bdm=legacy_bdm)

这是RPC的基本使用方法，直接使用cctxt.cast来发送消息。而cctxt是从调用client的prepare方法返回的。那么client又是怎么来的呢？

RPCClient

看Conductorr 的\__init__方法。

class ConductorAPI(rpcclient.RpcProxy):    def __init__(self):            version_cap = self.VERSION_ALIASES.get(CONF.upgrade_levels.conductor,                                                   CONF.upgrade_levels.conductor)            super(ConductorAPI, self).__init__(                topic=CONF.conductor.topic,                default_version=self.BASE_RPC_API_VERSION,                serializer=objects_base.NovaObjectSerializer(),                version_cap=version_cap)            self.client = self.get_client()

class RpcProxy(proxy.RpcProxy):    def get_client(self, namespace=None, server_params=None):        return RPCClient(self,                         namespace=namespace,                         server_params=server_params)

从这里可以看出， client是RPCClient的实例。
再看看RPCClient的实现， 这里代码不长，就全放在下面。

from nova.openstack.common.rpc import proxyclass RPCClient(object):    def __init__(self, proxy, namespace=None, server_params=None):        super(RPCClient, self).__init__()        #RPC的proxy, 相当于client的lib        self.proxy = proxy        self.namespace = namespace        self.server_params = server_params        self.kwargs = {}        self.fanout = None    def prepare(self, **kwargs):        # Clone ourselves        ret = self.__class__(self.proxy, self.namespace, self.server_params)        ret.kwargs.update(self.kwargs)        ret.fanout = self.fanout        # Update according to supplied kwargs        ret.kwargs.update(kwargs)        server = ret.kwargs.pop('server', None)        if server:            ret.kwargs['topic'] = '%s.%s' % (self.proxy.topic, server)        fanout = ret.kwargs.pop('fanout', None)        if fanout:            ret.fanout = True        return ret    def _invoke(self, cast_or_call, ctxt, method, **kwargs):        try:            msg = self.proxy.make_namespaced_msg(method,                                                 self.namespace,                                                 **kwargs)            return cast_or_call(ctxt, msg, **self.kwargs)        finally:            self.kwargs = {}            self.fanout = None    def cast(self, ctxt, method, **kwargs):        if self.server_params:            def cast_to_server(ctxt, msg, **kwargs):                if self.fanout:                    return self.proxy.fanout_cast_to_server(                        ctxt, self.server_params, msg, **kwargs)                else:                    return self.proxy.cast_to_server(                        ctxt, self.server_params, msg, **kwargs)            caster = cast_to_server        else:            caster = self.proxy.fanout_cast if self.fanout else self.proxy.cast        self._invoke(caster, ctxt, method, **kwargs)    def call(self, ctxt, method, **kwargs):        return self._invoke(self.proxy.call, ctxt, method, **kwargs)    def can_send_version(self, version):        return self.proxy.can_send_version(version)

代码不复杂， 最终的调用还是proxy, 而ConductorAPI本身就是RpcProxy的子类， 现在看看RpcProxy类的实现。

RpcProxy

from nova.openstack.common import rpcfrom nova.openstack.common.rpc import common as rpc_commonfrom nova.openstack.common.rpc import serializer as rpc_serializerclass RpcProxy(object):    """A helper class for rpc clients.    This class is a wrapper around the RPC client API.  It allows you to    specify the topic and API version in a single place.  This is intended to    be used as a base class for a class that implements the client side of an    rpc API.    """    # The default namespace, which can be overriden in a subclass.    RPC_API_NAMESPACE = None    def __init__(self, topic, default_version, version_cap=None,                 serializer=None):        """Initialize an RpcProxy.        :param topic: The topic to use for all messages.        :param default_version: The default API version to request in all               outgoing messages.  This can be overridden on a per-message               basis.        :param version_cap: Optionally cap the maximum version used for sent               messages.        :param serializer: Optionaly (de-)serialize entities with a               provided helper.        """        self.topic = topic        self.default_version = default_version        self.version_cap = version_cap        if serializer is None:            serializer = rpc_serializer.NoOpSerializer()        self.serializer = serializer        super(RpcProxy, self).__init__()    def _set_version(self, msg, vers):        """Helper method to set the version in a message.        :param msg: The message having a version added to it.        :param vers: The version number to add to the message.        """        v = vers if vers else self.default_version        if (self.version_cap and not                rpc_common.version_is_compatible(self.version_cap, v)):            raise rpc_common.RpcVersionCapError(version_cap=self.version_cap)        msg['version'] = v    def _get_topic(self, topic):        """Return the topic to use for a message."""        return topic if topic else self.topic    def can_send_version(self, version):        """Check to see if a version is compatible with the version cap."""        return (not self.version_cap or                rpc_common.version_is_compatible(self.version_cap, version))    @staticmethod    def make_namespaced_msg(method, namespace, **kwargs):        return {'method': method, 'namespace': namespace, 'args': kwargs}    def make_msg(self, method, **kwargs):        return self.make_namespaced_msg(method, self.RPC_API_NAMESPACE,                                        **kwargs)    def _serialize_msg_args(self, context, kwargs):        """Helper method called to serialize message arguments.        This calls our serializer on each argument, returning a new        set of args that have been serialized.        :param context: The request context        :param kwargs: The arguments to serialize        :returns: A new set of serialized arguments        """        new_kwargs = dict()        for argname, arg in kwargs.iteritems():            new_kwargs[argname] = self.serializer.serialize_entity(context,                                                                   arg)        return new_kwargs    def call(self, context, msg, topic=None, version=None, timeout=None):        """rpc.call() a remote method.        :param context: The request context        :param msg: The message to send, including the method and args.        :param topic: Override the topic for this message.        :param version: (Optional) Override the requested API version in this               message.        :param timeout: (Optional) A timeout to use when waiting for the               response.  If no timeout is specified, a default timeout will be               used that is usually sufficient.        :returns: The return value from the remote method.        """        self._set_version(msg, version)        msg['args'] = self._serialize_msg_args(context, msg['args'])        real_topic = self._get_topic(topic)        try:            result = rpc.call(context, real_topic, msg, timeout)            return self.serializer.deserialize_entity(context, result)        except rpc.common.Timeout as exc:            raise rpc.common.Timeout(                exc.info, real_topic, msg.get('method'))    def multicall(self, context, msg, topic=None, version=None, timeout=None):        """rpc.multicall() a remote method.        :param context: The request context        :param msg: The message to send, including the method and args.        :param topic: Override the topic for this message.        :param version: (Optional) Override the requested API version in this               message.        :param timeout: (Optional) A timeout to use when waiting for the               response.  If no timeout is specified, a default timeout will be               used that is usually sufficient.        :returns: An iterator that lets you process each of the returned values                  from the remote method as they arrive.        """        self._set_version(msg, version)        msg['args'] = self._serialize_msg_args(context, msg['args'])        real_topic = self._get_topic(topic)        try:            result = rpc.multicall(context, real_topic, msg, timeout)            return self.serializer.deserialize_entity(context, result)        except rpc.common.Timeout as exc:            raise rpc.common.Timeout(                exc.info, real_topic, msg.get('method'))    def cast(self, context, msg, topic=None, version=None):        """rpc.cast() a remote method.        :param context: The request context        :param msg: The message to send, including the method and args.        :param topic: Override the topic for this message.        :param version: (Optional) Override the requested API version in this               message.        :returns: None.  rpc.cast() does not wait on any return value from the                  remote method.        """        self._set_version(msg, version)        msg['args'] = self._serialize_msg_args(context, msg['args'])        rpc.cast(context, self._get_topic(topic), msg)    def fanout_cast(self, context, msg, topic=None, version=None):        """rpc.fanout_cast() a remote method.        :param context: The request context        :param msg: The message to send, including the method and args.        :param topic: Override the topic for this message.        :param version: (Optional) Override the requested API version in this               message.        :returns: None.  rpc.fanout_cast() does not wait on any return value                  from the remote method.        """        self._set_version(msg, version)        msg['args'] = self._serialize_msg_args(context, msg['args'])        rpc.fanout_cast(context, self._get_topic(topic), msg)    def cast_to_server(self, context, server_params, msg, topic=None,                       version=None):        """rpc.cast_to_server() a remote method.        :param context: The request context        :param server_params: Server parameters.  See rpc.cast_to_server() for               details.        :param msg: The message to send, including the method and args.        :param topic: Override the topic for this message.        :param version: (Optional) Override the requested API version in this               message.        :returns: None.  rpc.cast_to_server() does not wait on any                  return values.        """        self._set_version(msg, version)        msg['args'] = self._serialize_msg_args(context, msg['args'])        rpc.cast_to_server(context, server_params, self._get_topic(topic), msg)    def fanout_cast_to_server(self, context, server_params, msg, topic=None,                              version=None):        """rpc.fanout_cast_to_server() a remote method.        :param context: The request context        :param server_params: Server parameters.  See rpc.cast_to_server() for               details.        :param msg: The message to send, including the method and args.        :param topic: Override the topic for this message.        :param version: (Optional) Override the requested API version in this               message.        :returns: None.  rpc.fanout_cast_to_server() does not wait on any                  return values.        """        self._set_version(msg, version)        msg['args'] = self._serialize_msg_args(context, msg['args'])        rpc.fanout_cast_to_server(context, server_params,                                  self._get_topic(topic), msg)

代码虽然不少， 但大部分都是结构相似的代码。做的事情如下：
1. 设置消息的版本信息， 以便收到消息时进行版本检查。
2. 消息序列化
3. 找出目标地址
4. 发送消息

发送消息是通过RPC的模块来发送的。

result = rpc.call(context, real_topic, msg, timeout)

openstack\common\rpc

在\nova\openstack\common\rpc__init__.py中， call方法实现如下：

def call(context, topic, msg, timeout=None, check_for_lock=False):    if check_for_lock:        _check_for_lock()    return _get_impl().call(CONF, context, topic, msg, timeout)

这里可以看出， 只是调用_get_impl().call。再看_get_impl方法。

def _get_impl():    """Delay import of rpc_backend until configuration is loaded."""    global _RPCIMPL    if _RPCIMPL is None:        try:            _RPCIMPL = importutils.import_module(CONF.rpc_backend)        except ImportError:            # For backwards compatibility with older oslo.config.            impl = CONF.rpc_backend.replace('nova.rpc',                                            'nova.openstack.common.rpc')            _RPCIMPL = importutils.import_module(impl)    return _RPCIMPL

从上面的方法可以看出， 这里load的是CONF.rpc_backend，也就是系统配置的RPC消息的后端。

\nova\openstack\common\rpc\impl_qpid.py

def call(conf, context, topic, msg, timeout=None):    """Sends a message on a topic and wait for a response."""    return rpc_amqp.call(        conf, context, topic, msg, timeout,        rpc_amqp.get_connection_pool(conf, Connection))

\nova\openstack\common\rpc\amqp.py

def multicall(conf, context, topic, msg, timeout, connection_pool):    """Make a call that returns multiple times."""    LOG.debug(_('Making synchronous call on %s ...'), topic)    msg_id = uuid.uuid4().hex    msg.update({'_msg_id': msg_id})    LOG.debug(_('MSG_ID is %s') % (msg_id))    _add_unique_id(msg)    pack_context(msg, context)    with _reply_proxy_create_sem:        if not connection_pool.reply_proxy:            connection_pool.reply_proxy = ReplyProxy(conf, connection_pool)    msg.update({'_reply_q': connection_pool.reply_proxy.get_reply_q()})    wait_msg = MulticallProxyWaiter(conf, msg_id, timeout, connection_pool)    with ConnectionContext(conf, connection_pool) as conn:        #使用ConnectionContext对象发送数据        conn.topic_send(topic, rpc_common.serialize_msg(msg), timeout)    return wait_msgdef call(conf, context, topic, msg, timeout, connection_pool):    """Sends a message on a topic and wait for a response."""    rv = multicall(conf, context, topic, msg, timeout, connection_pool)    # NOTE(vish): return the last result from the multicall    rv = list(rv)    if not rv:        return    return rv[-1]

ConnectionContext

class ConnectionContext(rpc_common.Connection):    """The class that is actually returned to the create_connection() caller.    This is essentially a wrapper around Connection that supports 'with'.    It can also return a new Connection, or one from a pool.    The function will also catch when an instance of this class is to be    deleted.  With that we can return Connections to the pool on exceptions    and so forth without making the caller be responsible for catching them.    If possible the function makes sure to return a connection to the pool.    """    def __init__(self, conf, connection_pool, pooled=True, server_params=None):        """Create a new connection, or get one from the pool."""        self.connection = None        self.conf = conf        #从pool中获取connection 对象        self.connection_pool = connection_pool        if pooled:            self.connection = connection_pool.get()        else:            self.connection = connection_pool.connection_cls(                conf,                server_params=server_params)        self.pooled = pooled    def __enter__(self):        """When with ConnectionContext() is used, return self."""        return self    def _done(self):        """If the connection came from a pool, clean it up and put it back.        If it did not come from a pool, close it.        """        if self.connection:            if self.pooled:                # Reset the connection so it's ready for the next caller                # to grab from the pool                self.connection.reset()                self.connection_pool.put(self.connection)            else:                try:                    self.connection.close()                except Exception:                    pass            self.connection = None    def __exit__(self, exc_type, exc_value, tb):        """End of 'with' statement.  We're done here."""        self._done()    def __del__(self):        """Caller is done with this connection.  Make sure we cleaned up."""        self._done()    def close(self):        """Caller is done with this connection."""        self._done()    def create_consumer(self, topic, proxy, fanout=False):        self.connection.create_consumer(topic, proxy, fanout)    def create_worker(self, topic, proxy, pool_name):        self.connection.create_worker(topic, proxy, pool_name)    def join_consumer_pool(self, callback, pool_name, topic, exchange_name,                           ack_on_error=True):        self.connection.join_consumer_pool(callback,                                           pool_name,                                           topic,                                           exchange_name,                                           ack_on_error)    def consume_in_thread(self):        self.connection.consume_in_thread()    def __getattr__(self, key):        """Proxy all other calls to the Connection instance."""        if self.connection:            return getattr(self.connection, key)        else:            raise rpc_common.InvalidRPCConnectionReuse()

这里可以看出， ConnectionContext类没有topic_send方法， 那么这里调用的又是那个呢？
在ConnectionContext的__getattr__方法中， 返回了self.connection的成员。

def __getattr__(self, key):        """Proxy all other calls to the Connection instance."""        if self.connection:            return getattr(self.connection, key)        else:            raise rpc_common.InvalidRPCConnectionReuse()

所以这里调用的其实是Connection的方法。

在\nova\openstack\common\rpc\impl_qpid.py中实现了Connection类，但由于代码比较长，下面只列出两段代码， 一段是创建Connection，一段是发送。

def connection_create(self, broker):        # Create the connection - this does not open the connection        self.connection = qpid_messaging.Connection(broker)        # Check if flags are set and if so set them for the connection        # before we call open        self.connection.username = self.username        self.connection.password = self.password        self.connection.sasl_mechanisms = self.conf.qpid_sasl_mechanisms        # Reconnection is done by self.reconnect()        self.connection.reconnect = False        self.connection.heartbeat = self.conf.qpid_heartbeat        self.connection.transport = self.conf.qpid_protocol        self.connection.tcp_nodelay = self.conf.qpid_tcp_nodelaydef topic_send(self, topic, msg, timeout=None):        """Send a 'topic' message."""        #        # We want to create a message with attributes, e.g. a TTL. We        # don't really need to keep 'msg' in its JSON format any longer        # so let's create an actual qpid message here and get some        # value-add on the go.        #        # WARNING: Request timeout happens to be in the same units as        # qpid's TTL (seconds). If this changes in the future, then this        # will need to be altered accordingly.        #生成qpid的消息        qpid_message = qpid_messaging.Message(content=msg, ttl=timeout)        #使用publisher_send发送消息        self.publisher_send(TopicPublisher, topic, qpid_message)

TopicPublisher

最后会使用TopicPublisher的send方法发送消息。

class Publisher(object):    """Base Publisher class."""    def __init__(self, conf, session, node_name, node_opts=None):        """Init the Publisher class with the exchange_name, routing_key,        and other options        """        self.sender = None        self.session = session        if conf.qpid_topology_version == 1:            addr_opts = {                "create": "always",                "node": {                    "type": "topic",                    "x-declare": {                        "durable": False,                        # auto-delete isn't implemented for exchanges in qpid,                        # but put in here anyway                        "auto-delete": True,                    },                },            }            if node_opts:                addr_opts["node"]["x-declare"].update(node_opts)            self.address = "%s ; %s" % (node_name, jsonutils.dumps(addr_opts))        elif conf.qpid_topology_version == 2:            self.address = node_name        else:            raise_invalid_topology_version()        self.reconnect(session)    def reconnect(self, session):        """Re-establish the Sender after a reconnection."""        self.sender = session.sender(self.address)    def _pack_json_msg(self, msg):        """Qpid cannot serialize dicts containing strings longer than 65535           characters.  This function dumps the message content to a JSON           string, which Qpid is able to handle.        :param msg: May be either a Qpid Message object or a bare dict.        :returns: A Qpid Message with its content field JSON encoded.        """        try:            msg.content = jsonutils.dumps(msg.content)        except AttributeError:            # Need to have a Qpid message so we can set the content_type.            msg = qpid_messaging.Message(jsonutils.dumps(msg))        msg.content_type = JSON_CONTENT_TYPE        return msg    def send(self, msg):        """Send a message."""        try:            # Check if Qpid can encode the message            check_msg = msg            if not hasattr(check_msg, 'content_type'):                check_msg = qpid_messaging.Message(msg)            content_type = check_msg.content_type            enc, dec = qpid_messaging.message.get_codec(content_type)            enc(check_msg.content)        except qpid_codec.CodecException:            # This means the message couldn't be serialized as a dict.            msg = self._pack_json_msg(msg)        self.sender.send(msg)class TopicPublisher(Publisher):    """Publisher class for 'topic'."""    def __init__(self, conf, session, topic):        """init a 'topic' publisher.        """        exchange_name = rpc_amqp.get_control_exchange(conf)        if conf.qpid_topology_version == 1:            node_name = "%s/%s" % (exchange_name, topic)        elif conf.qpid_topology_version == 2:            node_name = "amq.topic/topic/%s/%s" % (exchange_name, topic)        else:            raise_invalid_topology_version()        super(TopicPublisher, self).__init__(conf, session, node_name)

接收

从上面的发送可以看到， 发送的内容有需要调用的函数名称及调用函数的参数。由此可见， 在接收端，不但要接收到消息，还要执行相关的函数。那么这些是怎么做到的呢？

Conductor/main

对于Conductor来说，接收消息的是一个RPC服务，这也是大家所公用的。
在nova\cmd目录中， 有所有服务的启动命令。
在nova\cmd\conductor.py中， 代码如下：

from nova import configfrom nova import objectsfrom nova.openstack.common import log as loggingfrom nova import servicefrom nova import utilsCONF = cfg.CONFCONF.import_opt('topic', 'nova.conductor.api', group='conductor')def main():    #注册数据库对象    objects.register_all()    #参数解析    config.parse_args(sys.argv)    logging.setup("nova")    utils.monkey_patch()    #以nova-conductor为应用程序创建服务    server = service.Service.create(binary='nova-conductor',                                    topic=CONF.conductor.topic,                                    manager=CONF.conductor.manager)    #启动服务                                    service.serve(server, workers=CONF.conductor.workers)    service.wait()

创建Service（server） 对象

可以看出， 这里主要的任务就是创建和启动服务。先看看Service类的静态方法create。

@classmethod    def create(cls, host=None, binary=None, topic=None, manager=None,               report_interval=None, periodic_enable=None,               periodic_fuzzy_delay=None, periodic_interval_max=None,               db_allowed=True):        """Instantiates class and passes back application object.        :param host: defaults to CONF.host        :param binary: defaults to basename of executable        :param topic: defaults to bin_name - 'nova-' part        :param manager: defaults to CONF.<topic>_manager        :param report_interval: defaults to CONF.report_interval        :param periodic_enable: defaults to CONF.periodic_enable        :param periodic_fuzzy_delay: defaults to CONF.periodic_fuzzy_delay        :param periodic_interval_max: if set, the max time to wait between runs        """        #得到host, 上面没有传入，从conf文件中获取。        if not host:            host = CONF.host        #binary='nova-conductor'        if not binary:            binary = os.path.basename(sys.argv[0])        #topic=CONF.conductor.topic        if not topic:            topic = binary.rpartition('nova-')[2]        #manager=CONF.conductor.manager        #cfg.StrOpt('manager',        #       default='nova.conductor.manager.ConductorManager',        #       help='full class name for the Manager for conductor'),        if not manager:            manager_cls = ('%s_manager' %                           binary.rpartition('nova-')[2])            manager = CONF.get(manager_cls, None)        if report_interval is None:            report_interval = CONF.report_interval        if periodic_enable is None:            periodic_enable = CONF.periodic_enable        if periodic_fuzzy_delay is None:            periodic_fuzzy_delay = CONF.periodic_fuzzy_delay        #创建Service对象        service_obj = cls(host, binary, topic, manager,                          report_interval=report_interval,                          periodic_enable=periodic_enable,                          periodic_fuzzy_delay=periodic_fuzzy_delay,                          periodic_interval_max=periodic_interval_max,                          db_allowed=db_allowed)        return service_obj

最后返回的是Service的对象。

启动Service对象

启动对象中间过程会比较复杂， 但对RPC没有太多的影响， 这里的中间过程就不看了。
最终会调用Service的start方法。

def start(self):        verstr = version.version_string_with_package()        LOG.audit(_('Starting %(topic)s node (version %(version)s)'),                  {'topic': self.topic, 'version': verstr})        self.basic_config_check()        self.manager.init_host()        self.model_disconnected = False        ctxt = context.get_admin_context()        try:            self.service_ref = self.conductor_api.service_get_by_args(ctxt,                    self.host, self.binary)            self.service_id = self.service_ref['id']        except exception.NotFound:            self.service_ref = self._create_service_ref(ctxt)        self.manager.pre_start_hook()        if self.backdoor_port is not None:            self.manager.backdoor_port = self.backdoor_port        #创建QPID的connection        self.conn = rpc.create_connection(new=True)        LOG.debug(_("Creating Consumer connection for Service %s") %                  self.topic)        #创建dispatcher对象， dispathcher用来分发收到的消息。也就是说把消息转化成函数调用。        rpc_dispatcher = self.manager.create_rpc_dispatcher(self.backdoor_port)        # Share this same connection for these Consumers        #当前topic的consumer        self.conn.create_consumer(self.topic, rpc_dispatcher, fanout=False)        #当前topic及host的consumer        node_topic = '%s.%s' % (self.topic, self.host)        self.conn.create_consumer(node_topic, rpc_dispatcher, fanout=False)        #广播消息        self.conn.create_consumer(self.topic, rpc_dispatcher, fanout=True)        # Consume from all consumers in a thread        #把consumer放到一个特定的协程中去执行        self.conn.consume_in_thread()        self.manager.post_start_hook()        LOG.debug(_("Join ServiceGroup membership for this service %s")                  % self.topic)        # Add service to the ServiceGroup membership group.        self.servicegroup_api.join(self.host, self.topic, self)        if self.periodic_enable:            if self.periodic_fuzzy_delay:                initial_delay = random.randint(0, self.periodic_fuzzy_delay)            else:                initial_delay = None            self.tg.add_dynamic_timer(self.periodic_tasks,                                     initial_delay=initial_delay,                                     periodic_interval_max=                                        self.periodic_interval_max)

从上面可以看出， 它创建了两个不同范围的topic消费者及一个广播消费者。然后每个消费都都放置在自己的协程中进行处理。

Consumer协程

def consume_in_thread(self):        """Consumer from all queues/consumers in a greenthread."""        @excutils.forever_retry_uncaught_exceptions        def _consumer_thread():            try:                self.consume()            except greenlet.GreenletExit:                return        if self.consumer_thread is None:            #启动协程            self.consumer_thread = eventlet.spawn(_consumer_thread)        return self.consumer_threaddef consume(self, limit=None):        """Consume from all queues/consumers."""        it = self.iterconsume(limit=limit)        while True:            try:                it.next()            except StopIteration:                returndef iterconsume(self, limit=None, timeout=None):        """Return an iterator that will consume from all queues/consumers."""        def _error_callback(exc):            if isinstance(exc, qpid_exceptions.Empty):                LOG.debug(_('Timed out waiting for RPC response: %s') %                          str(exc))                raise rpc_common.Timeout()            else:                LOG.exception(_('Failed to consume message from queue: %s') %                              str(exc))        def _consume():            nxt_receiver = self.session.next_receiver(timeout=timeout)            try:                self._lookup_consumer(nxt_receiver).consume()            except Exception:                LOG.exception(_("Error processing message.  Skipping it."))        for iteration in itertools.count(0):            if limit and iteration >= limit:                raise StopIteration            yield self.ensure(_error_callback, _consume)

ConsumerBase

def consume(self):        """Fetch the message and pass it to the callback object."""        #接收消息        message = self.receiver.fetch()        try:            #解析消息            self._unpack_json_msg(message)            msg = rpc_common.deserialize_msg(message.content)            #调用回调函数            self.callback(msg)        except Exception:            LOG.exception(_("Failed to process message... skipping it."))        finally:            # TODO(sandy): Need support for optional ack_on_error.            self.session.acknowledge(message)

在调用回调时， 最终会调用到之前的RpcDispatcher/dispath方法。

def dispatch(self, ctxt, version, method, namespace, **kwargs):        """Dispatch a message based on a requested version.        :param ctxt: The request context        :param version: The requested API version from the incoming message        :param method: The method requested to be called by the incoming                       message.        :param namespace: The namespace for the requested method.  If None,                          the dispatcher will look for a method on a callback                          object with no namespace set.        :param kwargs: A dict of keyword arguments to be passed to the method.        :returns: Whatever is returned by the underlying method that gets                  called.        """        if not version:            version = '1.0'        had_compatible = False        for proxyobj in self.callbacks:            # Check for namespace compatibility            try:                cb_namespace = proxyobj.RPC_API_NAMESPACE            except AttributeError:                cb_namespace = None            if namespace != cb_namespace:                continue            # Check for version compatibility            try:                rpc_api_version = proxyobj.RPC_API_VERSION            except AttributeError:                rpc_api_version = '1.0'            is_compatible = rpc_common.version_is_compatible(rpc_api_version,                                                             version)            had_compatible = had_compatible or is_compatible            if not hasattr(proxyobj, method):                continue            if is_compatible:                kwargs = self._deserialize_args(ctxt, kwargs)                #方法调用                result = getattr(proxyobj, method)(ctxt, **kwargs)                return self.serializer.serialize_entity(ctxt, result)        if had_compatible:            raise AttributeError("No such RPC function '%s'" % method)        else:            raise rpc_common.UnsupportedRpcVersion(version=version)

至此， 在RPC消息的对端，调用了相关的方法。