cloudfoundry之router源码分析

cloud foundry之router源码分析

1      router目录结构

2      router组件的启动

每个组件启动都需要经过下列命令

exec_cmd("#{ruby_binary} #{vcap_launch} #{command} #{vcap_components["components"].join(" ")} -c #{deployment_config_path} -v #{vcap_home} -l #{deployment_info["deployment_log_path"]}")

 

即：

 

/root/cloudfoundry/vcap/router/bin/router -c /root/cloudfoundry/.deployments/devbox/config/router.yml  

 

 

3      Router.rb源码

 

# Copyright (c) 2009-2011 VMware, Inc.

require 'fileutils'

require 'optparse'

require 'socket'

require 'yaml'

require 'openssl'

require 'set'

require 'zlib'

require 'rubygems'

require 'bundler/setup'

require 'nats/client'

require 'http/parser'

require 'vcap/common'

require 'vcap/component'

require 'vcap/logging'

require 'vcap/rolling_metric'

$:.unshift(File.dirname(__FILE__))

require 'router/const'

require 'router/router'

require 'router/router_uls_server'

config_path = ENV["CLOUD_FOUNDRY_CONFIG_PATH"] || File.join(File.dirname(__FILE__), '../config')

config_file = File.join(config_path, 'router.yml')

port, inet = nil, nil

options = OptionParser.new do |opts|

  opts.banner = 'Usage: router [OPTIONS]'

  opts.on("-p", "--port [ARG]", "Network port") do |opt|

    port = opt.to_i

  end

  opts.on("-i", "--interface [ARG]", "Network Interface") do |opt|

    inet = opt

  end

  opts.on("-c", "--config [ARG]", "Configuration File") do |opt|

    config_file = opt

  end

  opts.on("-h", "--help", "Help") do

    puts opts

    exit

  end

end

options.parse!(ARGV.dup)

begin

  config = File.open(config_file) do |f|

    YAML.load(f)

  end

rescue => e

  puts "Could not read configuration file:  #{e}"

  exit

end

# Placeholder for Component reporting

config['config_file'] = File.expand_path(config_file)

port = config['port'] unless port

inet = config['inet'] unless inet

EM.epoll

EM.run do

  trap("TERM") { Router.stop() }

  trap("INT")  { Router.stop() }

  Router.config(config)

  Router.log.info "Starting VCAP Router (#{Router.version})"

  Router.log.info "Listening on: #{inet}:#{port}" if inet && port

  Router.inet = inet || VCAP.local_ip(config['local_route'])

  Router.port = port

  # If the sock paramater is set, this will override the inet/port

  # for unix domain sockets

  if fn = config['sock']

    File.unlink(fn) if File.exists?(fn)

    Router.log.info "Listening on unix domain socket: '#{fn}'"

  end

  # Hack for running on BVTs on Macs which default to 256 FDs per process

  if RUBY_PLATFORM =~ /darwin/

    begin

      Process.setrlimit(Process::RLIMIT_NOFILE, 4096)

    rescue => e

      Router.log.info "Failed to modify the socket limit: #{e}"

    end

  end

  EM.set_descriptor_table_size(32768) # Requires Root privileges

  Router.log.info "Socket Limit:#{EM.set_descriptor_table_size}"

  Router.log.info "Pid file: %s" % config['pid']

  begin

    Router.pid_file = VCAP::PidFile.new(config['pid'])

  rescue => e

    Router.log.fatal "Can't create router pid file: #{e}"

    exit 1

  end

  NATS.on_error do |e|

    if e.kind_of? NATS::ConnectError

      Router.log.error("EXITING! NATS connection failed: #{e}")

      exit!

    else

      Router.log.error("NATS problem, #{e}")

    end

  end

  EM.error_handler do |e|

    Router.log.error "Eventmachine problem, #{e}"

    Router.log.error("#{e.backtrace.join("\n")}")

  end

  begin

    # TCP/IP Socket

    Router.server = Thin::Server.new(inet, port, RouterULSServer, :signals => false) if inet && port

    Router.local_server = Thin::Server.new(fn, RouterULSServer, :signals => false) if fn

    Router.server.start if Router.server

    Router.local_server.start if Router.local_server

  rescue => e

    Router.log.fatal "Problem starting server, #{e}"

    exit

  end

  # Allow nginx to access..

  FileUtils.chmod(0777, fn) if fn

  # Override reconnect attempts in NATS until the proper option

  # is available inside NATS itself.

  begin

    sv, $-v = $-v, nil

    NATS::MAX_RECONNECT_ATTEMPTS = 150 # 5 minutes total

    NATS::RECONNECT_TIME_WAIT    = 2   # 2 secs

    $-v = sv

  end

  NATS.start(:uri => config['mbus'])

  # Create the register/unregister listeners.

  Router.setup_listeners

  # Register ourselves with the system

  status_config = config['status'] || {}

  VCAP::Component.register(:type => 'Router',

                           :host => VCAP.local_ip(config['local_route']),

                           :index => config['index'],

                           :config => config,

                           :port => status_config['port'],

                           :user => status_config['user'],

                           :password => status_config['password'],

                           :logger => Router.log)

  # Setup some of our varzs..

  VCAP::Component.varz[:requests] = 0

  VCAP::Component.varz[:bad_requests] = 0

  VCAP::Component.varz[:latency] = VCAP::RollingMetric.new(60)

  VCAP::Component.varz[:responses_2xx] = 0

  VCAP::Component.varz[:responses_3xx] = 0

  VCAP::Component.varz[:responses_4xx] = 0

  VCAP::Component.varz[:responses_5xx] = 0

  VCAP::Component.varz[:responses_xxx] = 0

  VCAP::Component.varz[:bad_requests] = 0

  VCAP::Component.varz[:urls] = 0

  VCAP::Component.varz[:droplets] = 0

  VCAP::Component.varz[:tags] = {}

  @router_id = VCAP.secure_uuid

  @hello_message = { :id => @router_id, :version => Router::VERSION }.to_json.freeze

  # This will check on the state of the registered urls, do maintenance, etc..

  Router.setup_sweepers

  # Setup a start sweeper to make sure we have a consistent view of the world.

  EM.next_tick do

    # Announce our existence

    NATS.publish('router.start', @hello_message)

    # Don't let the messages pile up if we are in a reconnecting state

    EM.add_periodic_timer(START_SWEEPER) do

      unless NATS.client.reconnecting?

        NATS.publish('router.start', @hello_message)

      end

    end

  end

end

 

 

 

首先是一些依赖

然后是进行一些基本的配置，例如读取配置文件等等

然后会启动EVENTMACHINE，进行真正的启动

然后是router的一些操作。

4      Router的总体设计

 

router中会集成一个简单的服务器，通过Sinatra开发的，会集成一个nginx服务器，外界的访问首先是到达nginx服务器，然后nginx会调用lua脚本，生成http请求发送到router自己的服务器，然后router会通过外界访问的host的值来找到相应的app的ip+port地址（指向对应的dea），然后再返回，然后nginx再代理到返回的地址就好了。如图：

 

5      Router源码分析

4.1    服务器启动源码

Router服务器的启动

  begin

    # TCP/IP Socket

    Router.server = Thin::Server.new(inet, port, RouterULSServer, :signals => false) if inet && port #一般情况不用

    Router.local_server = Thin::Server.new(fn, RouterULSServer, :signals => false) if fn ##创建#router的服务器，用来与nginx进行交互  

    Router.server.start if Router.server #启动服务

    Router.local_server.start if Router.local_server

  rescue => e

    Router.log.fatal "Problem starting server, #{e}"

    exit

  end

 

 

这里，一般情况下是监听一个本地的sock文件，这样就很容易实现nginx与router自己的服务器之间的通信：/tmp/router.sock

4.2    Router通过nats组件的订阅

接下来是订阅一些消息，例如有新的app部署成功后，那么router需要登记它的名字和ip+port地址，

 

NATS.start(:uri => config['mbus'])

  # Create the register/unregister listeners.

  Router.setup_listeners #主要是订阅一些组件通过nats发布的消息

 

具体代码如下：

 

 def setup_listeners

      NATS.subscribe('router.register') { |msg|  #注册应用

        msg_hash = Yajl::Parser.parse(msg, :symbolize_keys => true)

        return unless uris = msg_hash[:uris]

        uris.each { |uri| register_droplet(uri, msg_hash[:host], msg_hash[:port],

                                           msg_hash[:tags], msg_hash[:app]) }

      }

      NATS.subscribe('router.unregister') { |msg|  #解除应用

        msg_hash = Yajl::Parser.parse(msg, :symbolize_keys => true)

        return unless uris = msg_hash[:uris]

        uris.each { |uri| unregister_droplet(uri, msg_hash[:host], msg_hash[:port]) }

      }

end

 

router.register用于登记新的app，unregistered则是用于删除应用时，将应用信息从router中删除。

 

 

4.3    Router的周期函数

 

Router.setup_sweepers

 

具体代码：

 

    def setup_sweepers

      @rps_timestamp = Time.now

      @current_num_requests = 0

      EM.add_periodic_timer(RPS_SWEEPER) { calc_rps }

      EM.add_periodic_timer(CHECK_SWEEPER) {

        check_registered_urls

      }

      if @enable_nonprod_apps

        EM.add_periodic_timer(@flush_apps_interval) do

          flush_active_apps

        end

      end

    end

 

setup_sweepers主要是用于设置周期函数，用于更新一些实时的数据，广播当前router的一些信息和状态

 

 

6      Router自己服务器

 

服务器源码

require "sinatra/base"

class RouterULSServer < Sinatra::Base

  class ParserError < StandardError; end

  disable :show_exceptions, :dump_errors

  get "/" do

    uls_response = {}

    VCAP::Component.varz[:requests] += 1

    # Get request body

    request.body.rewind # in case someone already read the body

    body = request.body.read

    Router.log.debug "Request body: #{body}"

    # Parse request body

    uls_req = JSON.parse(body, :symbolize_keys => true)

    raise ParserError if uls_req.nil? || !uls_req.is_a?(Hash)

    stats, url = uls_req[ULS_STATS_UPDATE], uls_req[ULS_HOST_QUERY]

    sticky = uls_req[ULS_STICKY_SESSION]

    if stats then

      update_uls_stats(stats)

    end

    if url then

      # Lookup a droplet

      unless droplets = Router.lookup_droplet(url)

        Router.log.debug "No droplet registered for #{url}"

        raise Sinatra::NotFound

      end

      # Pick a droplet based on original backend addr or pick a droplet randomly

      if sticky

        _, host, port = Router.decrypt_session_cookie(sticky)

        droplet = check_original_droplet(droplets, host, port)

      end

      droplet ||= droplets[rand*droplets.size]

      Router.log.debug "Routing #{droplet[:url]} to #{droplet[:host]}:#{droplet[:port]}"

      # Update droplet stats

      update_droplet_stats(droplet)

      # Update active apps

      Router.add_active_app(droplet[:app]) if droplet[:app]

      # Get session cookie for droplet

      new_sticky = Router.get_session_cookie(droplet)

      uls_req_tags = Base64.encode64(Marshal.dump(droplet[:tags])).strip

      uls_response = {

        ULS_STICKY_SESSION => new_sticky,

        ULS_BACKEND_ADDR   => "#{droplet[:host]}:#{droplet[:port]}",

        ULS_REQUEST_TAGS   => uls_req_tags,

        ULS_ROUTER_IP      => Router.inet,

        ULS_APP_ID         => droplet[:app] || 0,

      }

    end

    uls_response.to_json

  end

  not_found do

    VCAP::Component.varz[:bad_requests] += 1

    "VCAP ROUTER: 404 - DESTINATION NOT FOUND"

  end

  error [ JSON::ParserError, ParserError ] do

    VCAP::Component.varz[:bad_requests] += 1

    _, body = request.body.rewind, request.body.read

    Router.log.error "Failed to parse request body: '#{body}'"

    status 400

    "VCAP ROUTER: 400 - FAILED TO PARSE PAYLOAD"

  end

  error do

    VCAP::Component.varz[:bad_requests] += 1

    Router.log.error env['sinatra.error']

    "VCAP ROUTER: 500 - UNKNOWN"

  end

  protected

  def check_original_droplet(droplets, host, port)

    droplet = nil

    if host and port

      Router.log.debug "request has __VCAP_ID__ cookie for #{host}:#{port}"

      # Check host?

      droplets.each do |d|

        if(d[:host] == host && d[:port] == port.to_i)

          droplet = d; break

        end

      end

      Router.log.debug "request's __VCAP_ID__ is stale" unless droplet

    end

    droplet

  end

  def update_uls_stats(stats)

    stats.each do |stat|

      if stat[ULS_REQUEST_TAGS].length > 0

        tags = Marshal.load(Base64.decode64(stat[ULS_REQUEST_TAGS]))

      end

      latency = stat[ULS_RESPONSE_LATENCY]

      samples = stat[ULS_RESPONSE_SAMPLES]

      # We may find a better solution for latency

      1.upto samples do

        VCAP::Component.varz[:latency] << latency

      end

      stat[ULS_RESPONSE_STATUS].each_pair do |k, v|

        response_code_metric = k.to_sym

        VCAP::Component.varz[response_code_metric] += v

        if not tags then next end

        tags.each do |key, value|

          # In case some req tags of syncup state may be invalid at this time

          if not VCAP::Component.varz[:tags][key] or

            not VCAP::Component.varz[:tags][key][value]

            next

          end

          tag_metrics = VCAP::Component.varz[:tags][key][value]

          tag_metrics[response_code_metric] += v

          1.upto samples do

            tag_metrics[:latency] << latency

          end

        end # tags

      end # stat[ULS_RESPONSE_STATUS]

    end # stats.each

  end

  def update_droplet_stats(droplet)

    if droplet[:tags]

      droplet[:tags].each do |key, value|

        tag_metrics = VCAP::Component.varz[:tags][key][value]

        tag_metrics[:requests] += 1

      end

    end

    droplet[:requests] += 1

  end

end

 

 

 

主要是接受经过lua脚本处理过然后nginx传过来的数据，然后router根据host的数据查找相应的app的信息，主要是找到访问这个app的ip+port地址，然后将它返回回去，这样nginx就可以直接通过这个地址来直接到dea来访问对应的app了

具体的Nginx+Router流程图：如下