rocksdb线程池设计

特点

1. 可以设置线程池中线程的优先级，然而只有linux系统支持。
2. 可以动态增加或减少线程池中线程数量。

接口

class ThreadPool { public:  virtual ~ThreadPool() {}  // Wait for all threads to finish.  // Discard those threads that did not start  // executing  virtual void JoinAllThreads() = 0;  // Set the number of background threads that will be executing the  // scheduled jobs.  virtual void SetBackgroundThreads(int num) = 0;  virtual int GetBackgroundThreads() = 0;  // Get the number of jobs scheduled in the ThreadPool queue.  virtual unsigned int GetQueueLen() const = 0;  // Waits for all jobs to complete those  // that already started running and those that did not  // start yet. This ensures that everything that was thrown  // on the TP runs even though  // we may not have specified enough threads for the amount  // of jobs  virtual void WaitForJobsAndJoinAllThreads() = 0;  // Submit a fire and forget jobs  // This allows to submit the same job multiple times  virtual void SubmitJob(const std::function<void()>&) = 0;  // This moves the function in for efficiency  virtual void SubmitJob(std::function<void()>&&) = 0;};

class ThreadPoolImpl : public ThreadPool { public:  ThreadPoolImpl();  ~ThreadPoolImpl();  ThreadPoolImpl(ThreadPoolImpl&&) = delete;  ThreadPoolImpl& operator=(ThreadPoolImpl&&) = delete;  // Implement ThreadPool interfaces  // Wait for all threads to finish.  // Discards all the jobs that did not  // start executing and waits for those running  // to complete  void JoinAllThreads() override;  // Set the number of background threads that will be executing the  // scheduled jobs.  void SetBackgroundThreads(int num) override;  int GetBackgroundThreads() override;  // Get the number of jobs scheduled in the ThreadPool queue.  unsigned int GetQueueLen() const override;  // Waits for all jobs to complete those  // that already started running and those that did not  // start yet  void WaitForJobsAndJoinAllThreads() override;  // Make threads to run at a lower kernel priority  // Currently only has effect on Linux  void LowerIOPriority();  // Ensure there is at aleast num threads in the pool  // but do not kill threads if there are more  void IncBackgroundThreadsIfNeeded(int num);  // Submit a fire and forget job  // These jobs can not be unscheduled  // This allows to submit the same job multiple times  void SubmitJob(const std::function<void()>&) override;  // This moves the function in for efficiency  void SubmitJob(std::function<void()>&&) override;  // Schedule a job with an unschedule tag and unschedule function  // Can be used to filter and unschedule jobs by a tag  // that are still in the queue and did not start running  void Schedule(void (*function)(void* arg1), void* arg, void* tag,                void (*unschedFunction)(void* arg));  // Filter jobs that are still in a queue and match  // the given tag. Remove them from a queue if any  // and for each such job execute an unschedule function  // if such was given at scheduling time.  int UnSchedule(void* tag);  void SetHostEnv(Env* env);  Env* GetHostEnv() const;  // Return the thread priority.  // This would allow its member-thread to know its priority.  Env::Priority GetThreadPriority() const;  // Set the thread priority.  void SetThreadPriority(Env::Priority priority);  static void PthreadCall(const char* label, int result);  struct Impl; private:   // Current public virtual interface does not provide usable   // functionality and thus can not be used internally to   // facade different implementations.   //   // We propose a pimpl idiom in order to easily replace the thread pool impl   // w/o touching the header file but providing a different .cc potentially   // CMake option driven.   //   // Another option is to introduce a Env::MakeThreadPool() virtual interface   // and override the environment. This would require refactoring ThreadPool usage.   //   // We can also combine these two approaches   std::unique_ptr<Impl>   impl_;};

struct ThreadPoolImpl::Impl {  Impl();  ~Impl();  void JoinThreads(bool wait_for_jobs_to_complete);  void SetBackgroundThreadsInternal(int num, bool allow_reduce);  int GetBackgroundThreads();  unsigned int GetQueueLen() const {    return queue_len_.load(std::memory_order_relaxed);  }  void LowerIOPriority();  void WakeUpAllThreads() {    bgsignal_.notify_all();  }  void BGThread(size_t thread_id);  void StartBGThreads();  void Submit(std::function<void()>&& schedule,    std::function<void()>&& unschedule, void* tag);  int UnSchedule(void* arg);  void SetHostEnv(Env* env) { env_ = env; }  Env* GetHostEnv() const { return env_; }  bool HasExcessiveThread() const {    return static_cast<int>(bgthreads_.size()) > total_threads_limit_;  }  // Return true iff the current thread is the excessive thread to terminate.  // Always terminate the running thread that is added last, even if there are  // more than one thread to terminate.  bool IsLastExcessiveThread(size_t thread_id) const {    return HasExcessiveThread() && thread_id == bgthreads_.size() - 1;  }  bool IsExcessiveThread(size_t thread_id) const {    return static_cast<int>(thread_id) >= total_threads_limit_;  }  // Return the thread priority.  // This would allow its member-thread to know its priority.  Env::Priority GetThreadPriority() const { return priority_; }  // Set the thread priority.  void SetThreadPriority(Env::Priority priority) { priority_ = priority; }private:  static void* BGThreadWrapper(void* arg);  bool low_io_priority_;  Env::Priority priority_;  Env*         env_;  int total_threads_limit_;  std::atomic_uint queue_len_;  // Queue length. Used for stats reporting  bool exit_all_threads_;  bool wait_for_jobs_to_complete_;  // Entry per Schedule()/Submit() call  struct BGItem {    void* tag = nullptr;    std::function<void()> function;    std::function<void()> unschedFunction;  };  using BGQueue = std::deque<BGItem>;  BGQueue       queue_;  std::mutex               mu_;  std::condition_variable  bgsignal_;  std::vector<port::Thread> bgthreads_;};inlineThreadPoolImpl::Impl::Impl()    :      low_io_priority_(false),      priority_(Env::LOW),      env_(nullptr),      total_threads_limit_(0),      queue_len_(),      exit_all_threads_(false),      wait_for_jobs_to_complete_(false),      queue_(),      mu_(),      bgsignal_(),      bgthreads_() {}inlineThreadPoolImpl::Impl::~Impl() { assert(bgthreads_.size() == 0U); }//等待线程池的所有线程退出，wait_for_jobs_to_complete决定了是否要等待工作队列//中所有任务都执行完才退出void ThreadPoolImpl::Impl::JoinThreads(bool wait_for_jobs_to_complete) {  std::unique_lock<std::mutex> lock(mu_);  assert(!exit_all_threads_);  wait_for_jobs_to_complete_ = wait_for_jobs_to_complete;  exit_all_threads_ = true;  // prevent threads from being recreated right after they're joined, in case  // the user is concurrently submitting jobs.  total_threads_limit_ = 0;  lock.unlock();  bgsignal_.notify_all();//唤醒所有阻塞的线程，等待他们退出  for (auto& th : bgthreads_) {    th.join();  }  bgthreads_.clear();  exit_all_threads_ = false;  wait_for_jobs_to_complete_ = false;}//将线程池设置为低优先级的线程池，默认是normal线程inlinevoid ThreadPoolImpl::Impl::LowerIOPriority() {  std::lock_guard<std::mutex> lock(mu_);  low_io_priority_ = true;}void ThreadPoolImpl::Impl::BGThread(size_t thread_id) {  bool low_io_priority = false;  while (true) {// Wait until there is an item that is ready to run    std::unique_lock<std::mutex> lock(mu_);    // Stop waiting if the thread needs to do work or needs to terminate. //没有调用jointhreads退出线程池并且该线程不是最后加入的线程且超过了上限    while (!exit_all_threads_ && !IsLastExcessiveThread(thread_id) &&            (queue_.empty() || IsExcessiveThread(thread_id))) {//任务队列为空或者任务队列不空但该线程数超上限      bgsignal_.wait(lock);    }    if (exit_all_threads_) {  // mechanism to let BG threads exit safely      if(!wait_for_jobs_to_complete_ ||          queue_.empty()) {//如果不用等待所有任务完成或者任务队列为空则直接退出while，结束线程        break;       }    }    if (IsLastExcessiveThread(thread_id)) {//是最后加入的线程并且超过了线程数上限      // Current thread is the last generated one and is excessive.      // We always terminate excessive thread in the reverse order of      // generation time.      auto& terminating_thread = bgthreads_.back();      terminating_thread.detach();//终止掉该线程，释放资源      bgthreads_.pop_back();      if (HasExcessiveThread()) {        // There is still at least more excessive thread to terminate.        WakeUpAllThreads();//如果释放了最后一个线程还是超过上线数，继续唤醒其他线程继续释放      }      break;    }    auto func = std::move(queue_.front().function);    queue_.pop_front();    queue_len_.store(static_cast<unsigned int>(queue_.size()),                     std::memory_order_relaxed);    bool decrease_io_priority = (low_io_priority != low_io_priority_);//判断是否降低了线程优先级    lock.unlock();#ifdef OS_LINUX    if (decrease_io_priority) {#define IOPRIO_CLASS_SHIFT (13)#define IOPRIO_PRIO_VALUE(class, data) (((class) << IOPRIO_CLASS_SHIFT) | data)      // Put schedule into IOPRIO_CLASS_IDLE class (lowest)      // These system calls only have an effect when used in conjunction      // with an I/O scheduler that supports I/O priorities. As at      // kernel 2.6.17 the only such scheduler is the Completely      // Fair Queuing (CFQ) I/O scheduler.      // To change scheduler:      //  echo cfq > /sys/block/<device_name>/queue/schedule      // Tunables to consider:      //  /sys/block/<device_name>/queue/slice_idle      //  /sys/block/<device_name>/queue/slice_sync      syscall(SYS_ioprio_set, 1,  // IOPRIO_WHO_PROCESS              0,                  // current thread              IOPRIO_PRIO_VALUE(3, 0));      low_io_priority = true;//只有linux系统支持降低优先级    }#else    (void)decrease_io_priority;  // avoid 'unused variable' error#endif    func();  }}// Helper struct for passing arguments when creating threads.struct BGThreadMetadata {//thread类只接受一个参数，做个包装  ThreadPoolImpl::Impl* thread_pool_;  size_t thread_id_;  // Thread count in the thread.属于线程vector中的第几个  BGThreadMetadata(ThreadPoolImpl::Impl* thread_pool, size_t thread_id)      : thread_pool_(thread_pool), thread_id_(thread_id) {}};//用于thread类的函数，参数就是BGThreadMetadatavoid* ThreadPoolImpl::Impl::BGThreadWrapper(void* arg) {  BGThreadMetadata* meta = reinterpret_cast<BGThreadMetadata*>(arg);  size_t thread_id = meta->thread_id_;  ThreadPoolImpl::Impl* tp = meta->thread_pool_;#ifdef ROCKSDB_USING_THREAD_STATUS  // for thread-status  ThreadStatusUtil::RegisterThread(      tp->GetHostEnv(), (tp->GetThreadPriority() == Env::Priority::HIGH                             ? ThreadStatus::HIGH_PRIORITY                             : ThreadStatus::LOW_PRIORITY));#endif  delete meta;  tp->BGThread(thread_id);#ifdef ROCKSDB_USING_THREAD_STATUS  ThreadStatusUtil::UnregisterThread();#endif  return nullptr;}void ThreadPoolImpl::Impl::SetBackgroundThreadsInternal(int num,  bool allow_reduce) {//设置线程池个数  std::unique_lock<std::mutex> lock(mu_);  if (exit_all_threads_) {//如果遇到join线程池直接退出    lock.unlock();    return;  }  if (num > total_threads_limit_ ||      (num < total_threads_limit_ && allow_reduce)) {//减少线程需要allow_reduce为true才行    total_threads_limit_ = std::max(0, num);//更改线程池总数    WakeUpAllThreads();//唤醒已有线程，如果num便低的话就释放超出的线程    StartBGThreads();//如果num增大的话用来新创建多出的线程  }}int ThreadPoolImpl::Impl::GetBackgroundThreads() {  std::unique_lock<std::mutex> lock(mu_);  return total_threads_limit_;}//创建total_threads_limit_个线程void ThreadPoolImpl::Impl::StartBGThreads() {  // Start background thread if necessary  while ((int)bgthreads_.size() < total_threads_limit_) {    port::Thread p_t(&BGThreadWrapper,      new BGThreadMetadata(this, bgthreads_.size()));// Set the thread name to aid debugging#if defined(_GNU_SOURCE) && defined(__GLIBC_PREREQ)#if __GLIBC_PREREQ(2, 12)    auto th_handle = p_t.native_handle();    char name_buf[16];    snprintf(name_buf, sizeof name_buf, "rocksdb:bg%" ROCKSDB_PRIszt,             bgthreads_.size());    name_buf[sizeof name_buf - 1] = '\0';    pthread_setname_np(th_handle, name_buf);#endif#endif    bgthreads_.push_back(std::move(p_t));  }}//添加新任务到线程池void ThreadPoolImpl::Impl::Submit(std::function<void()>&& schedule,  std::function<void()>&& unschedule, void* tag) {  std::lock_guard<std::mutex> lock(mu_);  if (exit_all_threads_) {//如果线程池正在join等待退出，返回    return;  }  StartBGThreads();  // Add to priority queue  queue_.push_back(BGItem());  auto& item = queue_.back();  item.tag = tag;  item.function = std::move(schedule);  item.unschedFunction = std::move(unschedule);//加入到任务队列，并更新任务队列的任务数  queue_len_.store(static_cast<unsigned int>(queue_.size()),    std::memory_order_relaxed);  if (!HasExcessiveThread()) {    // Wake up at least one waiting thread.    bgsignal_.notify_one();//如果没有超过线程数上限，则唤醒一个来执行  } else {    // Need to wake up all threads to make sure the one woken    // up is not the one to terminate.    WakeUpAllThreads();//超过，则唤醒所有，减低了最后一个线程执行的概率，因为最后一个创建的线程将要释放    //同时刚好唤醒最后一个线程释放掉  }}//将还没来得及执行的符合arg的任务从队列中清除掉，并执行这些任务的unschedFunction回调int ThreadPoolImpl::Impl::UnSchedule(void* arg) {  int count = 0;  std::vector<std::function<void()>> candidates;  {    std::lock_guard<std::mutex> lock(mu_);    // Remove from priority queue    BGQueue::iterator it = queue_.begin();    while (it != queue_.end()) {      if (arg == (*it).tag) {        if (it->unschedFunction) {          candidates.push_back(std::move(it->unschedFunction));        }        it = queue_.erase(it);        count++;      } else {        ++it;      }    }    queue_len_.store(static_cast<unsigned int>(queue_.size()),      std::memory_order_relaxed);  } // Run unschedule functions outside the mutex  for (auto& f : candidates) {    f();  }  return count;}ThreadPoolImpl::ThreadPoolImpl() :  impl_(new Impl()) {}ThreadPoolImpl::~ThreadPoolImpl() {}void ThreadPoolImpl::JoinAllThreads() {  impl_->JoinThreads(false);}void ThreadPoolImpl::SetBackgroundThreads(int num) {  impl_->SetBackgroundThreadsInternal(num, true);//设置线程池线程数，它会自动增长或缩减}int ThreadPoolImpl::GetBackgroundThreads() {  return impl_->GetBackgroundThreads();}unsigned int ThreadPoolImpl::GetQueueLen() const {  return impl_->GetQueueLen();//任务队列待执行任务数}void ThreadPoolImpl::WaitForJobsAndJoinAllThreads() {  impl_->JoinThreads(true);//不同JoinAllThreads，该函数会等待所以任务执行完才释放线程池}void ThreadPoolImpl::LowerIOPriority() {  impl_->LowerIOPriority();//线程池优先级变低}void ThreadPoolImpl::IncBackgroundThreadsIfNeeded(int num) {  impl_->SetBackgroundThreadsInternal(num, false);//确保线程数至少num个，如果多了并不释放掉多的线程资源}void ThreadPoolImpl::SubmitJob(const std::function<void()>& job) {  auto copy(job);  impl_->Submit(std::move(copy), std::function<void()>(), nullptr);}void ThreadPoolImpl::SubmitJob(std::function<void()>&& job) {  impl_->Submit(std::move(job), std::function<void()>(), nullptr);}void ThreadPoolImpl::Schedule(void(*function)(void* arg1), void* arg,  void* tag, void(*unschedFunction)(void* arg)) {    //封装成了无参数的回调，其实执行的是有参数的回调  std::function<void()> fn = [arg, function] { function(arg); };  std::function<void()> unfn;  if (unschedFunction != nullptr) {    auto uf = [arg, unschedFunction] { unschedFunction(arg); };    unfn = std::move(uf);  }  impl_->Submit(std::move(fn), std::move(unfn), tag);}int ThreadPoolImpl::UnSchedule(void* arg) {  return impl_->UnSchedule(arg);}void ThreadPoolImpl::SetHostEnv(Env* env) { impl_->SetHostEnv(env); }Env* ThreadPoolImpl::GetHostEnv() const { return impl_->GetHostEnv(); }// Return the thread priority.// This would allow its member-thread to know its priority.Env::Priority ThreadPoolImpl::GetThreadPriority() const {  return impl_->GetThreadPriority();}// Set the thread priority.void ThreadPoolImpl::SetThreadPriority(Env::Priority priority) {  impl_->SetThreadPriority(priority);}ThreadPool* NewThreadPool(int num_threads) {  ThreadPoolImpl* thread_pool = new ThreadPoolImpl();  thread_pool->SetBackgroundThreads(num_threads);  return thread_pool;}