查看nova服务进程的运行时状态信息

openstack nova服务启动后，系统中会有nova-scheduler/nova-conductor/nova-compute等进程，如何查看这些进程内部的线程，协程等状态信息？有两种方法。

一种是通过Guru Meditation reports，例如

[root@controller ~]# ps -elf | grep nova-scheduler
0 R root      3515 20210  0  80   0 - 28160 pipe_w 17:33 pts/4    00:00:00 grep --color=auto nova-scheduler
4 S nova     18405     1  0  80   0 - 111579 ep_pol 15:32 ?       00:00:06 /usr/bin/python /usr/bin/nova-scheduler

[root@controller ~]# kill -USR1 18405

然后用下面的命令就可以看到输出了：

journalctl --unit=openstack-nova-scheduler

或者

journalctl -u openstack-nova-scheduler

Guru Meditation reports机制是通过在进程中接收并处理SIGUSR1信号，然后将内存中的运行状态信息采集到之后，输出到标准错误fd中去。

如果你事先在启动这些进程的时候已经把标准错误重定向到某个文件，那么就会在这个文件中记录当前的状态信息。

另外一种方法是通过backdoor，在nova.conf中有一个配置选项backdoor_port：

# Enable eventlet backdoor. Acceptable values are 0, <port>

# and <start>:<end>, where 0 results in listening on a random
# tcp port number, <port> results in listening on the
# specified port number and not enabling backdoorif it is in
# use and <start>:<end> results in listening on the smallest
# unused port number within the specified range of port
# numbers. The chosen port is displayed in the service's log
# file. (string value)

#backdoor_port=<None>

可以将它配置为一段空闲可用的端口号范文，例如10000:10050，那么这些端口号就会依次被各个nova进程所用，从而在每个进程中开一个telnet服务，提供调试shell。

用netstat命令查看每个nova进程对应的端口号，然后telnet到这样的端口上，就可以输入调试命令了。

 可用的命令有：
     backdoor_locals = {
        'exit': _dont_use_this,      # So we don't exit the entire process
        'quit': _dont_use_this,      # So we don't exit the entire process
        'fo': _find_objects,
        'pgt': _print_greenthreads,
        'pnt': _print_nativethreads,
    }

[root@controller ~]# telnet 127.0.0.1 10005
Trying 127.0.0.1...
Connected to 127.0.0.1.
Escape character is '^]'.
Python 2.7.5 (default, Feb 11 2014, 07:46:25)
[GCC 4.8.2 20140120 (Red Hat 4.8.2-13)] on linux2
Type "help", "copyright", "credits" or "license" for more information.
(InteractiveConsole)
>>> pnt()
140346826684224
  File "/usr/lib/python2.7/site-packages/eventlet/backdoor.py", line 54, in run
    console.interact()
  File "/usr/lib64/python2.7/code.py", line 243, in interact
    more = self.push(line)
  File "/usr/lib64/python2.7/code.py", line 265, in push
    more = self.runsource(source, self.filename)
  File "/usr/lib64/python2.7/code.py", line 87, in runsource
    self.runcode(code)
  File "/usr/lib64/python2.7/code.py", line 103, in runcode
    exec code in self.locals
  File "<console>", line 1, in <module>
  File "/usr/lib/python2.7/site-packages/nova/openstack/common/eventlet_backdoor.py", line 82, in _print_nativethreads
    traceback.print_stack(stack)

>>> pgt()
0 <eventlet.backdoor.SocketConsole object at 0x46bae10>
  File "/usr/lib/python2.7/site-packages/eventlet/backdoor.py", line 54, in run
    console.interact()
  File "/usr/lib64/python2.7/code.py", line 243, in interact
    more = self.push(line)
  File "/usr/lib64/python2.7/code.py", line 265, in push
    more = self.runsource(source, self.filename)
  File "/usr/lib64/python2.7/code.py", line 87, in runsource
    self.runcode(code)
  File "/usr/lib64/python2.7/code.py", line 103, in runcode
    exec code in self.locals
  File "<console>", line 1, in <module>
  File "/usr/lib/python2.7/site-packages/nova/openstack/common/eventlet_backdoor.py", line 75, in _print_greenthreads
    traceback.print_stack(gt.gr_frame)

1 <greenlet.greenlet object at 0x2d90cd0>
  File "/usr/lib/python2.7/site-packages/eventlet/hubs/hub.py", line 236, in run
    self.wait(sleep_time)
  File "/usr/lib/python2.7/site-packages/eventlet/hubs/poll.py", line 97, in wait
    readers.get(fileno, noop).cb(fileno)
  File "/usr/lib/python2.7/site-packages/eventlet/backdoor.py", line 62, in switch
    greenlets.greenlet.switch(self, *args, **kw)

2 <greenlet.greenlet object at 0x2d90d70>
  File "/usr/lib/python2.7/site-packages/eventlet/backdoor.py", line 86, in backdoor_server
    socketpair = sock.accept()
  File "/usr/lib/python2.7/site-packages/eventlet/greenio.py", line 183, in accept
    timeout_exc=socket.timeout("timed out"))
  File "/usr/lib/python2.7/site-packages/eventlet/hubs/__init__.py", line 155, in trampoline
    return hub.switch()
  File "/usr/lib/python2.7/site-packages/eventlet/hubs/hub.py", line 187, in switch
    return self.greenlet.switch()

3 <eventlet.greenthread.GreenThread object at 0x2d90e10>
  File "/usr/lib/python2.7/site-packages/eventlet/greenthread.py", line 194, in main
    result = function(*args, **kwargs)
  File "/usr/lib/python2.7/site-packages/nova/openstack/common/service.py", line 487, in run_service
    done.wait()
  File "/usr/lib/python2.7/site-packages/eventlet/event.py", line 116, in wait
    return hubs.get_hub().switch()
  File "/usr/lib/python2.7/site-packages/eventlet/hubs/hub.py", line 187, in switch
    return self.greenlet.switch()

4 <greenlet.greenlet object at 0x7fa50aa8c410>
  File "/usr/bin/nova-scheduler", line 10, in <module>
    sys.exit(main())
  File "/usr/lib/python2.7/site-packages/nova/cmd/scheduler.py", line 44, in main
    service.wait()
  File "/usr/lib/python2.7/site-packages/nova/service.py", line 433, in wait
    _launcher.wait()
  File "/usr/lib/python2.7/site-packages/nova/openstack/common/service.py", line 196, in wait
    status, signo = self._wait_for_exit_or_signal(ready_callback)
  File "/usr/lib/python2.7/site-packages/nova/openstack/common/service.py", line 173, in _wait_for_exit_or_signal
    super(ServiceLauncher, self).wait()
  File "/usr/lib/python2.7/site-packages/nova/openstack/common/service.py", line 136, in wait
    self.services.wait()
  File "/usr/lib/python2.7/site-packages/nova/openstack/common/service.py", line 468, in wait
    self.tg.wait()
  File "/usr/lib/python2.7/site-packages/nova/openstack/common/threadgroup.py", line 117, in wait
    x.wait()
  File "/usr/lib/python2.7/site-packages/nova/openstack/common/threadgroup.py", line 49, in wait
    return self.thread.wait()
  File "/usr/lib/python2.7/site-packages/eventlet/greenthread.py", line 168, in wait
    return self._exit_event.wait()
  File "/usr/lib/python2.7/site-packages/eventlet/event.py", line 116, in wait
    return hubs.get_hub().switch()
  File "/usr/lib/python2.7/site-packages/eventlet/hubs/hub.py", line 187, in switch
    return self.greenlet.switch()

5 <greenlet.greenlet object at 0x46ba410>
  File "/usr/lib/python2.7/site-packages/eventlet/green/thread.py", line 27, in __thread_body
    func(*args, **kwargs)
  File "/usr/lib64/python2.7/threading.py", line 784, in __bootstrap
    self.__bootstrap_inner()
  File "/usr/lib64/python2.7/threading.py", line 811, in __bootstrap_inner
    self.run()
  File "/usr/lib64/python2.7/threading.py", line 764, in run
    self.__target(*self.__args, **self.__kwargs)
  File "/usr/lib/python2.7/site-packages/qpid/selector.py", line 119, in run
    rd, wr, ex = select(self.reading, self.writing, (), timeout)
  File "/usr/lib/python2.7/site-packages/eventlet/green/select.py", line 79, in select
    return hub.switch()
  File "/usr/lib/python2.7/site-packages/eventlet/hubs/hub.py", line 187, in switch
    return self.greenlet.switch()

6 <eventlet.greenthread.GreenThread object at 0x3383370>
  File "/usr/lib/python2.7/site-packages/eventlet/greenthread.py", line 194, in main
    result = function(*args, **kwargs)
  File "/usr/lib/python2.7/site-packages/oslo/messaging/openstack/common/excutils.py", line 78, in inner_func
    return infunc(*args, **kwargs)
  File "/usr/lib/python2.7/site-packages/oslo/messaging/_executors/impl_eventlet.py", line 88, in _executor_thread
    incoming = self.listener.poll()
  File "/usr/lib/python2.7/site-packages/oslo/messaging/_drivers/amqpdriver.py", line 110, in poll
    self.conn.consume(limit=1)
  File "/usr/lib/python2.7/site-packages/oslo/messaging/_drivers/impl_qpid.py", line 676, in consume
    six.next(it)
  File "/usr/lib/python2.7/site-packages/oslo/messaging/_drivers/impl_qpid.py", line 608, in iterconsume
    yield self.ensure(_error_callback, _consume)
  File "/usr/lib/python2.7/site-packages/oslo/messaging/_drivers/impl_qpid.py", line 546, in ensure
    return method(*args, **kwargs)
  File "/usr/lib/python2.7/site-packages/oslo/messaging/_drivers/impl_qpid.py", line 599, in _consume
    nxt_receiver = self.session.next_receiver(timeout=timeout)
  File "<string>", line 6, in next_receiver
  File "/usr/lib/python2.7/site-packages/qpid/messaging/endpoints.py", line 660, in next_receiver
    if self._ecwait(lambda: self.incoming, timeout):
  File "/usr/lib/python2.7/site-packages/qpid/messaging/endpoints.py", line 50, in _ecwait
    result = self._ewait(lambda: self.closed or predicate(), timeout)
  File "/usr/lib/python2.7/site-packages/qpid/messaging/endpoints.py", line 566, in _ewait
    result = self.connection._ewait(lambda: self.error or predicate(), timeout)
  File "/usr/lib/python2.7/site-packages/qpid/messaging/endpoints.py", line 208, in _ewait
    result = self._wait(lambda: self.error or predicate(), timeout)
  File "/usr/lib/python2.7/site-packages/qpid/messaging/endpoints.py", line 193, in _wait
    return self._waiter.wait(predicate, timeout=timeout)
  File "/usr/lib/python2.7/site-packages/qpid/concurrency.py", line 57, in wait
    self.condition.wait(3)
  File "/usr/lib/python2.7/site-packages/qpid/concurrency.py", line 96, in wait
    sw.wait(timeout)
  File "/usr/lib/python2.7/site-packages/qpid/compat.py", line 53, in wait
    ready, _, _ = select([self], [], [], timeout)
  File "/usr/lib/python2.7/site-packages/eventlet/green/select.py", line 79, in select
    return hub.switch()
  File "/usr/lib/python2.7/site-packages/eventlet/hubs/hub.py", line 187, in switch
    return self.greenlet.switch()

7 <greenlet.greenlet object at 0x46baeb0>
  File "/usr/lib/python2.7/site-packages/nova/openstack/common/loopingcall.py", line 86, in _inner
    greenthread.sleep(delay if delay > 0 else 0)
  File "/usr/lib/python2.7/site-packages/eventlet/greenthread.py", line 31, in sleep
    hub.switch()
  File "/usr/lib/python2.7/site-packages/eventlet/hubs/hub.py", line 187, in switch
    return self.greenlet.switch()

8 <eventlet.greenthread.GreenThread object at 0x46ba4b0>
  File "/usr/lib/python2.7/site-packages/eventlet/greenthread.py", line 194, in main
    result = function(*args, **kwargs)
  File "/usr/lib/python2.7/site-packages/nova/openstack/common/loopingcall.py", line 133, in _inner
    greenthread.sleep(idle)
  File "/usr/lib/python2.7/site-packages/eventlet/greenthread.py", line 31, in sleep
    hub.switch()
  File "/usr/lib/python2.7/site-packages/eventlet/hubs/hub.py", line 187, in switch
    return self.greenlet.switch()