spark core 2.0 ExecutorSource Metrics统计

Spark executor 使用ExecutorSource统计Executor的各性能的使用指标。

private[spark]class ExecutorSource(threadPool: ThreadPoolExecutor, executorId: String) extends Source {  private def fileStats(scheme: String) : Option[FileSystem.Statistics] =    FileSystem.getAllStatistics.asScala.find(s => s.getScheme.equals(scheme))  private def registerFileSystemStat[T](        scheme: String, name: String, f: FileSystem.Statistics => T, defaultValue: T) = {    metricRegistry.register(MetricRegistry.name("filesystem", scheme, name), new Gauge[T] {      override def getValue: T = fileStats(scheme).map(f).getOrElse(defaultValue)    })  }  override val metricRegistry = new MetricRegistry()  override val sourceName = "executor"  // Gauge for executor thread pool's actively executing task counts  metricRegistry.register(MetricRegistry.name("threadpool", "activeTasks"), new Gauge[Int] {    override def getValue: Int = threadPool.getActiveCount()  })  // Gauge for executor thread pool's approximate total number of tasks that have been completed  metricRegistry.register(MetricRegistry.name("threadpool", "completeTasks"), new Gauge[Long] {    override def getValue: Long = threadPool.getCompletedTaskCount()  })  // Gauge for executor thread pool's current number of threads  metricRegistry.register(MetricRegistry.name("threadpool", "currentPool_size"), new Gauge[Int] {    override def getValue: Int = threadPool.getPoolSize()  })  // Gauge got executor thread pool's largest number of threads that have ever simultaneously  // been in th pool  metricRegistry.register(MetricRegistry.name("threadpool", "maxPool_size"), new Gauge[Int] {    override def getValue: Int = threadPool.getMaximumPoolSize()  })  // Gauge for file system stats of this executor  for (scheme <- Array("hdfs", "file")) {    registerFileSystemStat(scheme, "read_bytes", _.getBytesRead(), 0L)    registerFileSystemStat(scheme, "write_bytes", _.getBytesWritten(), 0L)    registerFileSystemStat(scheme, "read_ops", _.getReadOps(), 0)    registerFileSystemStat(scheme, "largeRead_ops", _.getLargeReadOps(), 0)    registerFileSystemStat(scheme, "write_ops", _.getWriteOps(), 0)  }}

最后的registerFileSystemStat使用的是FileSystem.Statistics统计。如下所示，以getByteRead()为例。

    /**     * Get the total number of bytes read     * @return the number of bytes     */    public long getBytesRead() {      return visitAll(new StatisticsAggregator<Long>() {        private long bytesRead = 0;        @Override        public void accept(StatisticsData data) {          bytesRead += data.bytesRead;        }        public Long aggregate() {          return bytesRead;        }      });    }

StatisticsAggregator是一个接口，如下所示。

 private interface StatisticsAggregator<T> {      void accept(StatisticsData data);      T aggregate();    }

visitAll方法把这个统计对象相关的所有StatisticsData进行聚合，调用accept方法，如上面的getBytesRead，只对object的bytesRead属性进行聚合。

    /**     * Apply the given aggregator to all StatisticsData objects associated with     * this Statistics object.     *     * For each StatisticsData object, we will call accept on the visitor.     * Finally, at the end, we will call aggregate to get the final total.      *     * @param         The visitor to use.     * @return        The total.     */    private synchronized <T> T visitAll(StatisticsAggregator<T> visitor) {      visitor.accept(rootData);      if (allData != null) {        for (Iterator<StatisticsData> iter = allData.iterator();            iter.hasNext(); ) {          StatisticsData data = iter.next();          visitor.accept(data);          if (data.owner.get() == null) {            /*             * If the thread that created this thread-local data no             * longer exists, remove the StatisticsData from our list             * and fold the values into rootData.             */            rootData.add(data);            iter.remove();          }        }      }      return visitor.aggregate();    }

/**     * Statistics data.     *      * There is only a single writer to thread-local StatisticsData objects.     * Hence, volatile is adequate here-- we do not need AtomicLong or similar     * to prevent lost updates.     * The Java specification guarantees that updates to volatile longs will     * be perceived as atomic with respect to other threads, which is all we     * need.     */    public static class StatisticsData {      volatile long bytesRead;      volatile long bytesWritten;      volatile int readOps;      volatile int largeReadOps;      volatile int writeOps;      /**       * Stores a weak reference to the thread owning this StatisticsData.       * This allows us to remove StatisticsData objects that pertain to       * threads that no longer exist.       */      final WeakReference<Thread> owner;      StatisticsData(WeakReference<Thread> owner) {        this.owner = owner;      }      /**       * Add another StatisticsData object to this one.       */      void add(StatisticsData other) {        this.bytesRead += other.bytesRead;        this.bytesWritten += other.bytesWritten;        this.readOps += other.readOps;        this.largeReadOps += other.largeReadOps;        this.writeOps += other.writeOps;      }      /**       * Negate the values of all statistics.       */      void negate() {        this.bytesRead = -this.bytesRead;        this.bytesWritten = -this.bytesWritten;        this.readOps = -this.readOps;        this.largeReadOps = -this.largeReadOps;        this.writeOps = -this.writeOps;      }      @Override      public String toString() {        return bytesRead + " bytes read, " + bytesWritten + " bytes written, "            + readOps + " read ops, " + largeReadOps + " large read ops, "            + writeOps + " write ops";      }            public long getBytesRead() {        return bytesRead;      }            public long getBytesWritten() {        return bytesWritten;      }            public int getReadOps() {        return readOps;      }            public int getLargeReadOps() {        return largeReadOps;      }            public int getWriteOps() {        return writeOps;      }    }