OpenTSDB源码分析之TSDB表操作(新增)

OpenTSDB数据存储架构图如下，它分四个步骤：

1.     主机报告检测结果给本地的tcollection。

2.     tcollection发送检测结果给远程的tsd。

3.     tsd构造记录，并把数据写入HBase。

4.     HBase存储数据，并确认写入请求。

tcollector使用类Telnet的协议的发送监控信息的示例代码如下：

package test.telnet;import java.io.PrintStream;import org.apache.commons.net.telnet.TelnetClient; public class TelnetUtil {    private TelnetClient telnetClient = new TelnetClient();    private PrintStream out;     public TelnetUtil(String ip, Integer port) {       try {           telnetClient.connect(ip, port);           out = new PrintStream(telnetClient.getOutputStream());       } catch (Exception e) {          e.printStackTrace();       }    }     public static void main(String[] args) {    testGet("localhost", 4242);    }        public static void testGet(String url, Integer port) {    TelnetUtil telnetTest = new TelnetUtil(url, port);    String strTime = "" + System.currentTimeMillis();    System.out.println(strTime);    telnetTest.write("put t22 " + strTime + " 11 host=foo");    telnetTest.disconnect();    }    public void write(String command) {       try {           out.println(command);           out.flush();           System.out.println("本次执行的telnet命令:" + command);       } catch (Exception e) {           e.printStackTrace();       }    }     public void disconnect() {       try {           Thread.sleep(10);           telnetClient.disconnect();       } catch (Exception e) {           e.printStackTrace();       }    } }

tsd服务主要用来接收监控信息、构造记录并写入HBase。OpenTSDB在TSDBMain.main()方法中启动netty服务器用来接收来自远程的Http或RPC请求，针对不同的请求封装成HttpRpc 或TelnetRpc来进行处理。

TSDBMain类方法：public static void main(String[] args) throws IOException {// ./build/tsdb tsd --port=4242 --staticroot=build/staticroot// --cachedir=/tmp/tsdtmp –zkquorum=localhostargs = new String[] { "--port=4242", "--staticroot=F:/workspace/OpenTsdb/staticroot","--cachedir=D:/doc_hadoop/_00_hbase/opentsdb/opentsdb2_linux/build/tmp" };Logger log = LoggerFactory.getLogger(TSDMain.class);log.info("Starting.");log.info(BuildData.revisionString());log.info(BuildData.buildString());try {System.in.close(); // Release a FD we don't need.} catch (Exception e) {log.warn("Failed to close stdin", e);}final ArgP argp = new ArgP();CliOptions.addCommon(argp);argp.addOption("--port", "NUM", "TCP port to listen on.");argp.addOption("--bind", "ADDR", "Address to bind to (default: 0.0.0.0).");argp.addOption("--staticroot", "PATH", "Web root from which to serve static files (/s URLs).");argp.addOption("--cachedir", "PATH", "Directory under which to cache result of requests.");argp.addOption("--worker-threads", "NUM", "Number for async io workers (default: cpu * 2).");argp.addOption("--async-io", "true|false", "Use async NIO (default true) or traditional blocking io");argp.addOption("--backlog", "NUM", "Size of connection attempt queue (default: 3072 or kernel" + " somaxconn.");argp.addOption("--flush-interval", "MSEC", "Maximum time for which a new data point can be buffered" + " (default: " + DEFAULT_FLUSH_INTERVAL+ ").");CliOptions.addAutoMetricFlag(argp);args = CliOptions.parse(argp, args);args = null; // free().// get a config objectConfig config = CliOptions.getConfig(argp);// check for the required parameterstry {if (config.getString("tsd.http.staticroot").isEmpty())usage(argp, "Missing static root directory", 1);} catch (NullPointerException npe) {usage(argp, "Missing static root directory", 1);}try {if (config.getString("tsd.http.cachedir").isEmpty())usage(argp, "Missing cache directory", 1);} catch (NullPointerException npe) {usage(argp, "Missing cache directory", 1);}try {if (!config.hasProperty("tsd.network.port"))usage(argp, "Missing network port", 1);config.getInt("tsd.network.port");} catch (NumberFormatException nfe) {usage(argp, "Invalid network port setting", 1);}// validate the cache and staticroot directoriestry {checkDirectory(config.getString("tsd.http.staticroot"), DONT_CREATE, !MUST_BE_WRITEABLE);checkDirectory(config.getString("tsd.http.cachedir"), CREATE_IF_NEEDED, MUST_BE_WRITEABLE);} catch (IllegalArgumentException e) {usage(argp, e.getMessage(), 3);}final ServerSocketChannelFactory factory;if (config.getBoolean("tsd.network.async_io")) {// 默认为true(在Config中设置)/** * 默认为处理器的两倍 */int workers = Runtime.getRuntime().availableProcessors() * 2;if (config.hasProperty("tsd.network.worker_threads")) {try {workers = config.getInt("tsd.network.worker_threads");} catch (NumberFormatException nfe) {usage(argp, "Invalid worker thread count", 1);}}// System.out.println("workers=" + workers);factory =                    new NioServerSocketChannelFactory(Executors.newCachedThreadPool(),                    Executors.newCachedThreadPool(), workers);} else {factory =                     new OioServerSocketChannelFactory(Executors.newCachedThreadPool(),                     Executors.newCachedThreadPool());}TSDB tsdb = null;try {tsdb = new TSDB(config);tsdb.initializePlugins(true);// Make sure we don't even start if we can't find our tables.tsdb.checkNecessaryTablesExist().joinUninterruptibly();// 注册关闭程序registerShutdownHook(tsdb);final ServerBootstrap server = new ServerBootstrap(factory);/** * 此方法是本类的重点 * PipelineFactory内部负责netty服务器的具体实现 */server.setPipelineFactory(new PipelineFactory(tsdb));if (config.hasProperty("tsd.network.backlog")) {server.setOption("backlog", config.getInt("tsd.network.backlog"));}server.setOption("child.tcpNoDelay",                       config.getBoolean("tsd.network.tcp_no_delay"));server.setOption("child.keepAlive",                        config.getBoolean("tsd.network.keep_alive"));server.setOption("reuseAddress",                        config.getBoolean("tsd.network.reuse_address"));// null is interpreted as the wildcard address.InetAddress bindAddress = null;if (config.hasProperty("tsd.network.bind")) {bindAddress =                          InetAddress.getByName(config.getString("tsd.network.bind"));}// we validated the network port config earlierfinal InetSocketAddress addr = new InetSocketAddress(bindAddress,                        config.getInt("tsd.network.port"));server.bind(addr);log.info("Ready to serve on " + addr);} catch (Throwable e) {factory.releaseExternalResources();try {if (tsdb != null)tsdb.shutdown().joinUninterruptibly();} catch (Exception e2) {log.error("Failed to shutdown HBase client", e2);}throw new RuntimeException("Initialization failed", e);}// The server is now running in separate threads, we can exit main.}RpcHandler类方法     /** * TSDB服务可以监听telnet和http两种信号 */@Overridepublic void messageReceived(final ChannelHandlerContext ctx,                  final MessageEvent msgevent) {try {final Object message = msgevent.getMessage();/** * telnet信号  */if (message instanceof String[]) {handleTelnetRpc(msgevent.getChannel(), (String[]) message);/** * http信号 */} else if (message instanceof HttpRequest) {handleHttpQuery(tsdb, msgevent.getChannel(), (HttpRequest) message);} else {logError(msgevent.getChannel(), "Unexpected message type " +                                message.getClass() + ": " + message);exceptions_caught.incrementAndGet();}} catch (Exception e) {Object pretty_message = msgevent.getMessage();if (pretty_message instanceof String[]) {pretty_message = Arrays.toString((String[]) pretty_message);}logError(msgevent.getChannel(), "Unexpected exception caught" +                      " while serving " + pretty_message, e);exceptions_caught.incrementAndGet();}}

在telnet示例代码向tsd服务发送监控信息之后，tsd将接受的信息封装成PutDataPointRpc(implements TelnetRpc, HttpRpc)并实现其excute()方法，并调用TSDB类的addPointInternal()方法向HBase中添加数据。

public void execute(final TSDB tsdb, final HttpQuery query) throws IOException {requests.incrementAndGet();// only accept POSTif (query.method() != HttpMethod.POST) {throw new BadRequestException(HttpResponseStatus.METHOD_NOT_ALLOWED,                        "Method not allowed", "The HTTP method [" + query.method().getName()+ "] is not permitted for this endpoint");}final List<IncomingDataPoint> dps = query.serializer().parsePutV1();if (dps.size() < 1) {throw new BadRequestException("No datapoints found in content");}final boolean show_details = query.hasQueryStringParam("details");final boolean show_summary = query.hasQueryStringParam("summary");final ArrayList<HashMap<String, Object>> details = show_details ? new ArrayList<HashMap<String, Object>>() : null;long success = 0;long total = 0;for (IncomingDataPoint dp : dps) {total++;try {if (dp.getMetric() == null || dp.getMetric().isEmpty()) {if (show_details) {details.add(this.getHttpDetails("Metric name was empty", dp));}LOG.warn("Metric name was empty: " + dp);continue;}if (dp.getTimestamp() <= 0) {if (show_details) {details.add(this.getHttpDetails("Invalid timestamp", dp));}LOG.warn("Invalid timestamp: " + dp);continue;}if (dp.getValue() == null || dp.getValue().isEmpty()) {if (show_details) {details.add(this.getHttpDetails("Empty value", dp));}LOG.warn("Empty value: " + dp);continue;}if (dp.getTags() == null || dp.getTags().size() < 1) {if (show_details) {details.add(this.getHttpDetails("Missing tags", dp));}LOG.warn("Missing tags: " + dp);continue;}if (Tags.looksLikeInteger(dp.getValue())) {tsdb.addPoint(dp.getMetric(), dp.getTimestamp(), Tags.parseLong(dp.getValue()), dp.getTags());} else {tsdb.addPoint(dp.getMetric(), dp.getTimestamp(), Float.parseFloat(dp.getValue()), dp.getTags());}success++;} catch (NumberFormatException x) {if (show_details) {details.add(this.getHttpDetails("Unable to parse value to a number", dp));}LOG.warn("Unable to parse value to a number: " + dp);invalid_values.incrementAndGet();} catch (IllegalArgumentException iae) {if (show_details) {details.add(this.getHttpDetails(iae.getMessage(), dp));}LOG.warn(iae.getMessage() + ": " + dp);illegal_arguments.incrementAndGet();} catch (NoSuchUniqueName nsu) {if (show_details) {details.add(this.getHttpDetails("Unknown metric", dp));}LOG.warn("Unknown metric: " + dp);unknown_metrics.incrementAndGet();}}final long failures = total - success;if (!show_summary && !show_details) {if (failures > 0) {throw new BadRequestException(HttpResponseStatus.BAD_REQUEST, "One or more data points had errors","Please see the TSD logs or append \"details\" to the put request");} else {query.sendReply(HttpResponseStatus.NO_CONTENT, "".getBytes());}} else {final HashMap<String, Object> summary = new HashMap<String, Object>();summary.put("success", success);summary.put("failed", failures);if (show_details) {summary.put("errors", details);}if (failures > 0) {query.sendReply(HttpResponseStatus.BAD_REQUEST, query.serializer().formatPutV1(summary));} else {query.sendReply(query.serializer().formatPutV1(summary));}}}private Deferred<Object> addPointInternal(final String metric, final long timestamp, final byte[] value, final Map<String, String> tags,final short flags) {// we only accept unix epoch timestamps in seconds or millisecondsif ((timestamp & Const.SECOND_MASK) != 0 && (timestamp < 1000000000000L || timestamp > 9999999999999L)) {throw new IllegalArgumentException((timestamp < 0 ? "negative " : "bad") + " timestamp=" + timestamp + " when trying to add value="+ Arrays.toString(value) + '/' + flags + " to metric=" + metric + ", tags=" + tags);}/** * 有效字符验证 */IncomingDataPoints.checkMetricAndTags(metric, tags);class AddPointCB implements Callback<Deferred<Object>, byte[]> {/** * 此方法调用时会传入行键rowkey */public Deferred<Object> call(final byte[] row) {final long base_time;/** * 构造列修饰符 */final byte[] qualifier = Internal.buildQualifier(timestamp, flags);/** * 完善rowKey 将部分时间戳放入rowKey */if ((timestamp & Const.SECOND_MASK) != 0) {// drop the ms timestamp to seconds to calculate the base// timestampbase_time = ((timestamp / 1000) - ((timestamp / 1000) % Const.MAX_TIMESPAN));} else {base_time = (timestamp - (timestamp % Const.MAX_TIMESPAN));}Bytes.setInt(row, (int) base_time, metrics.width());scheduleForCompaction(row, (int) base_time);/** * 此处执行插入操作 */final PutRequest point = new PutRequest(table, row, FAMILY, qualifier, value);// TODO(tsuna): Add a callback to time the latency of HBase and// store the// timing in a moving Histogram (once we have a class for this).Deferred<Object> result = client.put(point);if (!config.enable_realtime_ts() && !config.enable_tsuid_incrementing() && rt_publisher == null) {return result;}final byte[] tsuid = UniqueId.getTSUIDFromKey(row, METRICS_WIDTH, Const.TIMESTAMP_BYTES);if (config.enable_tsuid_incrementing() || config.enable_realtime_ts()) {TSMeta.incrementAndGetCounter(TSDB.this, tsuid);}if (rt_publisher != null) {/** * Simply logs real time publisher errors when they're * thrown. Without this, exceptions will just disappear * (unless logged by the plugin) since we don't wait for a * result. */final class RTError implements Callback<Object, Exception> {@Overridepublic Object call(final Exception e) throws Exception {LOG.error("Exception from Real Time Publisher", e);return null;}}rt_publisher.sinkDataPoint(metric, timestamp, value, tags, tsuid, flags).addErrback(new RTError());}return result;}}/** * 先通过metric、tags生成rowKey的结构（byte[3+4+3+3]） * 再通过回调函数完善rowKey，并生成PutRequest将数据保存到tsdb表中 */return IncomingDataPoints.rowKeyTemplate(this, metric, tags).addCallbackDeferring(new AddPointCB());}