hbase 12.1 regionserver 宕机报错记录——1

    "Append_min" : 0,
    "Append_max" : 0,
    "Append_mean" : 0.0,
    "Append_median" : 0.0,
    "Append_75th_percentile" : 0.0,
    "Append_90th_percentile" : 0.0,
    "Append_95th_percentile" : 0.0,
    "Append_99th_percentile" : 0.0,
    "SplitTime_num_ops" : 14,
    "SplitTime_min" : 1198,
    "SplitTime_max" : 4528,
    "SplitTime_mean" : 2753.4285714285716,
    "SplitTime_median" : 2668.0,
    "SplitTime_75th_percentile" : 3402.0,
    "SplitTime_90th_percentile" : 3531.0,
    "SplitTime_95th_percentile" : 3531.0,
    "SplitTime_99th_percentile" : 3531.0,
    "SplitTime_TimeRangeCount_1000-3000" : 9,
    "SplitTime_TimeRangeCount_3000-10000" : 5
  } ]
}
2016-07-09 16:11:18,157 INFO  [main-EventThread] regionserver.HRegionServer: STOPPED: regionserver:16020-0x154fb34c4132eb5, quorum=10.80.248.12:2181,ut01:2181,ut03:2181, baseZNode=/hbase regionserver:16020-0x154fb34c4132eb5 received expired from ZooKeeper, aborting
2016-07-09 16:11:18,158 INFO  [regionserver/ut09/10.80.248.19:16020.logRoller] regionserver.LogRoller: LogRoller exiting.
2016-07-09 16:11:18,158 INFO  [main-EventThread] zookeeper.ClientCnxn: EventThread shut down
2016-07-09 16:11:18,162 INFO  [regionserver/ut09/10.80.248.19:16020] regionserver.SplitLogWorker: Sending interrupt to stop the worker thread
2016-07-09 16:11:18,163 INFO  [regionserver/ut09/10.80.248.19:16020] regionserver.HRegionServer: Stopping infoServer
2016-07-09 16:11:18,413 INFO  [SplitLogWorker-ut09:16020] regionserver.SplitLogWorker: SplitLogWorker interrupted. Exiting. 
2016-07-09 16:11:18,419 INFO  [SplitLogWorker-ut09:16020] regionserver.SplitLogWorker: SplitLogWorker ut09,16020,1467006160072 exiting
2016-07-09 16:11:18,570 INFO  [regionserver/ut09/10.80.248.19:16020] mortbay.log: Stopped SelectChannelConnector@0.0.0.0:16030
2016-07-09 16:11:18,676 INFO  [regionserver/ut09/10.80.248.19:16020] regionserver.HeapMemoryManager: Stoping HeapMemoryTuner chore.
2016-07-09 16:11:18,676 INFO  [MemStoreFlusher.0] regionserver.MemStoreFlusher: MemStoreFlusher.0 exiting
2016-07-09 16:11:18,676 INFO  [MemStoreFlusher.1] regionserver.MemStoreFlusher: MemStoreFlusher.1 exiting
2016-07-09 16:11:18,722 INFO  [regionserver/ut09/10.80.248.19:16020] flush.RegionServerFlushTableProcedureManager: Stopping region server flush procedure manager abruptly.
2016-07-09 16:11:18,772 INFO  [regionserver/ut09/10.80.248.19:16020] snapshot.RegionServerSnapshotManager: Stopping RegionServerSnapshotManager abruptly.
2016-07-09 16:11:18,795 INFO  [regionserver/ut09/10.80.248.19:16020] regionserver.HRegionServer: aborting server ut09,16020,1467006160072
2016-07-09 16:11:19,381 INFO  [regionserver/ut09/10.80.248.19:16020] regionserver.HRegionServer: Waiting on 15 regions to close
2016-07-09 16:11:19,406 INFO  [StoreCloserThread-wlan_log,51:93:32 40-70-6102|HWES242999910179800,1467911977102.9a12d3f1055f50a02f54bf6a89edbd52.-1] regionserver.HStore: Closed cf
2016-07-09 16:11:19,408 INFO  [StoreCloserThread-wlan_log,62:55:21 60-70-6102|HWSZ2329999107291552,1468031436966.519c62fce7dad85485309852aeb8278e.-1] regionserver.HStore: Closed cf
2016-07-09 16:11:19,408 ERROR [RS_CLOSE_REGION-ut09:16020-2] regionserver.HRegion: Memstore size is 127460728
2016-07-09 16:11:19,408 ERROR [RS_CLOSE_REGION-ut09:16020-0] regionserver.HRegion: Memstore size is 103098896
2016-07-09 16:11:19,408 INFO  [StoreCloserThread-wlan_log,63:50:32 40-70-6102|HWSY2279999104891720,1468035975548.ca2997727d29175e748e4681c4f4d4a7.-1] regionserver.HStore: Closed cf
2016-07-09 16:11:19,409 INFO  [RS_CLOSE_REGION-ut09:16020-0] regionserver.HRegion: Closed wlan_log,62:55:21 60-70-6102|HWSZ2329999107291552,1468031436966.519c62fce7dad85485309852aeb8278e.
2016-07-09 16:11:19,409 INFO  [RS_CLOSE_REGION-ut09:16020-2] regionserver.HRegion: Closed wlan_log,51:93:32 40-70-6102|HWES242999910179800,1467911977102.9a12d3f1055f50a02f54bf6a89edbd52.
2016-07-09 16:11:19,409 ERROR [RS_CLOSE_REGION-ut09:16020-1] regionserver.HRegion: Memstore size is 52396816
2016-07-09 16:11:19,409 INFO  [RS_CLOSE_REGION-ut09:16020-1] regionserver.HRegion: Closed wlan_log,63:50:32 40-70-6102|HWSY2279999104891720,1468035975548.ca2997727d29175e748e4681c4f4d4a7.
2016-07-09 16:11:19,410 INFO  [StoreCloserThread-viewlog2,25434148660|2016-01-06 14:16:44,1466758620104.3c6031bb034f773ec97762af5c83fb26.-1] regionserver.HStore: Closed cf1
2016-07-09 16:11:19,411 INFO  [StoreCloserThread-wlan_log,65:33:81 40-70-6102|HWHS24799991043992,1468038959115.3c3e34a9b09d4f37dd592a7210c9ac0f.-1] regionserver.HStore: Closed cf
2016-07-09 16:11:19,411 INFO  [StoreCloserThread-viewlog2,25434148660|2016-01-06 14:16:44,1466758620104.3c6031bb034f773ec97762af5c83fb26.-1] regionserver.HStore: Closed cf2
2016-07-09 16:11:19,424 ERROR [RS_CLOSE_REGION-ut09:16020-0] regionserver.HRegion: Memstore size is 21415000
2016-07-09 16:11:19,424 INFO  [StoreCloserThread-viewlog2,25434148660|2016-01-06 14:16:44,1466758620104.3c6031bb034f773ec97762af5c83fb26.-1] regionserver.HStore: Closed cf3
2016-07-09 16:11:19,425 INFO  [RS_CLOSE_REGION-ut09:16020-0] regionserver.HRegion: Closed wlan_log,65:33:81 40-70-6102|HWHS24799991043992,1468038959115.3c3e34a9b09d4f37dd592a7210c9ac0f.
2016-07-09 16:11:19,477 INFO  [RS_CLOSE_REGION-ut09:16020-1] regionserver.HRegion: Closed viewlog2,25434148660|2016-01-06 14:16:44,1466758620104.3c6031bb034f773ec97762af5c83fb26.
2016-07-09 16:11:19,479 INFO  [StoreCloserThread-wlan_log,61:63:10 60-70-6102|HWJZ2339999104592144,1468031436966.6e445f6fa6fd26b709d7c8861bedd84b.-1] regionserver.HStore: Closed cf
2016-07-09 16:11:19,481 ERROR [RS_CLOSE_REGION-ut09:16020-2] regionserver.HRegion: Memstore size is 41586064
2016-07-09 16:11:19,482 INFO  [RS_CLOSE_REGION-ut09:16020-2] regionserver.HRegion: Closed wlan_log,61:63:10 60-70-6102|HWJZ2339999104592144,1468031436966.6e445f6fa6fd26b709d7c8861bedd84b.
2016-07-09 16:11:19,483 INFO  [StoreCloserThread-wlan_log,64:41:70 50-70-6102|HWXN2239999109892408,1468035975548.c8fff7795e3d5f9d2fe56c25f8d4a2ca.-1] regionserver.HStore: Closed cf
2016-07-09 16:11:19,485 ERROR [RS_CLOSE_REGION-ut09:16020-1] regionserver.HRegion: Memstore size is 37542896
2016-07-09 16:11:19,485 INFO  [StoreCloserThread-wlan_log,55:73:91 40-70-6102|HWXY241999911229416,1467908535398.ccdd6f99f4850977e462e0549ebd3411.-1] regionserver.HStore: Closed cf
2016-07-09 16:11:19,486 INFO  [RS_CLOSE_REGION-ut09:16020-1] regionserver.HRegion: Closed wlan_log,64:41:70 50-70-6102|HWXN2239999109892408,1468035975548.c8fff7795e3d5f9d2fe56c25f8d4a2ca.
2016-07-09 16:11:19,486 ERROR [RS_CLOSE_REGION-ut09:16020-0] regionserver.HRegion: Memstore size is 50564544
2016-07-09 16:11:19,487 INFO  [RS_CLOSE_REGION-ut09:16020-0] regionserver.HRegion: Closed wlan_log,55:73:91 40-70-6102|HWXY241999911229416,1467908535398.ccdd6f99f4850977e462e0549ebd3411.
2016-07-09 16:11:19,492 INFO  [StoreCloserThread-wlan_log,53:82:41 40-70-6102|HWXG2619999116991232,1467908535398.5a3fa0f519ee2edc09f1f716453dd022.-1] regionserver.HStore: Closed cf
2016-07-09 16:11:19,493 ERROR [RS_CLOSE_REGION-ut09:16020-2] regionserver.HRegion: Memstore size is 42498488
2016-07-09 16:11:19,494 INFO  [RS_CLOSE_REGION-ut09:16020-2] regionserver.HRegion: Closed wlan_log,53:82:41 40-70-6102|HWXG2619999116991232,1467908535398.5a3fa0f519ee2edc09f1f716453dd022.
2016-07-09 16:11:19,494 INFO  [StoreCloserThread-wlan_log,70:25:91 60-70-6102|HWXN2329999105792054,1468038959115.74cddc0039296df89941f83157147e53.-1] regionserver.HStore: Closed cf
2016-07-09 16:11:19,495 INFO  [StoreCloserThread-wlan_log,50:10:10 60-70-6102|HWQJ223999910239177,1467911977102.00ee2a8c9da4c3e4fcb8c3e18c0b6ca5.-1] regionserver.HStore: Closed cf
2016-07-09 16:11:19,495 ERROR [RS_CLOSE_REGION-ut09:16020-0] regionserver.HRegion: Memstore size is 1459816
2016-07-09 16:11:19,496 ERROR [RS_CLOSE_REGION-ut09:16020-1] regionserver.HRegion: Memstore size is 131093056
2016-07-09 16:11:19,496 INFO  [RS_CLOSE_REGION-ut09:16020-0] regionserver.HRegion: Closed wlan_log,70:25:91 60-70-6102|HWXN2329999105792054,1468038959115.74cddc0039296df89941f83157147e53.
2016-07-09 16:11:19,497 INFO  [RS_CLOSE_REGION-ut09:16020-1] regionserver.HRegion: Closed wlan_log,50:10:10 60-70-6102|HWQJ223999910239177,1467911977102.00ee2a8c9da4c3e4fcb8c3e18c0b6ca5.
2016-07-09 16:11:19,497 INFO  [StoreCloserThread-viewlog2,22831642660|2016-01-01 00:22:12,1466758620104.d20897c8307b64208cfb8cf43c68d445.-1] regionserver.HStore: Closed cf1
2016-07-09 16:11:19,497 INFO  [StoreCloserThread-viewlog2,22831642660|2016-01-01 00:22:12,1466758620104.d20897c8307b64208cfb8cf43c68d445.-1] regionserver.HStore: Closed cf2
2016-07-09 16:11:19,497 INFO  [StoreCloserThread-viewlog2,22831642660|2016-01-01 00:22:12,1466758620104.d20897c8307b64208cfb8cf43c68d445.-1] regionserver.HStore: Closed cf3
2016-07-09 16:11:19,498 INFO  [RS_CLOSE_REGION-ut09:16020-2] regionserver.HRegion: Closed viewlog2,22831642660|2016-01-01 00:22:12,1466758620104.d20897c8307b64208cfb8cf43c68d445.
2016-07-09 16:11:19,500 INFO  [StoreCloserThread-viewlog2,715665240670|2016-01-05 22:50:22,1466758078110.e965df4398ced3b2cdd5c7d9bc23875e.-1] regionserver.HStore: Closed cf1
2016-07-09 16:11:19,500 INFO  [StoreCloserThread-viewlog2,31459301570|2016-01-02 21:49:47,1466758360450.00764b3a75f2c4b29b997d803f473e2e.-1] regionserver.HStore: Closed cf1
2016-07-09 16:11:19,500 INFO  [StoreCloserThread-viewlog2,715665240670|2016-01-05 22:50:22,1466758078110.e965df4398ced3b2cdd5c7d9bc23875e.-1] regionserver.HStore: Closed cf2
2016-07-09 16:11:19,503 INFO  [StoreCloserThread-viewlog2,715665240670|2016-01-05 22:50:22,1466758078110.e965df4398ced3b2cdd5c7d9bc23875e.-1] regionserver.HStore: Closed cf3
2016-07-09 16:11:19,501 INFO  [StoreCloserThread-viewlog2,31459301570|2016-01-02 21:49:47,1466758360450.00764b3a75f2c4b29b997d803f473e2e.-1] regionserver.HStore: Closed cf2
2016-07-09 16:11:19,503 INFO  [StoreCloserThread-viewlog2,31459301570|2016-01-02 21:49:47,1466758360450.00764b3a75f2c4b29b997d803f473e2e.-1] regionserver.HStore: Closed cf3
2016-07-09 16:11:19,503 INFO  [RS_CLOSE_REGION-ut09:16020-1] regionserver.HRegion: Closed viewlog2,715665240670|2016-01-05 22:50:22,1466758078110.e965df4398ced3b2cdd5c7d9bc23875e.
2016-07-09 16:11:19,503 INFO  [RS_CLOSE_REGION-ut09:16020-2] regionserver.HRegion: Closed viewlog2,31459301570|2016-01-02 21:49:47,1466758360450.00764b3a75f2c4b29b997d803f473e2e.
2016-07-09 16:11:19,503 INFO  [StoreCloserThread-wlan_log,71:21:02 40-70-6102|HWES24399991017984,1467678944057.c321c0d12cb9817c6676b92c8954ba29.-1] regionserver.HStore: Closed cf
2016-07-09 16:11:19,503 ERROR [RS_CLOSE_REGION-ut09:16020-0] regionserver.HRegion: Memstore size is 71103616
2016-07-09 16:11:19,503 INFO  [RS_CLOSE_REGION-ut09:16020-0] regionserver.HRegion: Closed wlan_log,71:21:02 40-70-6102|HWES24399991017984,1467678944057.c321c0d12cb9817c6676b92c8954ba29.
2016-07-09 16:11:19,581 INFO  [regionserver/ut09/10.80.248.19:16020] regionserver.HRegionServer: stopping server ut09,16020,1467006160072; all regions closed.
2016-07-09 16:11:19,637 WARN  [regionserver/ut09/10.80.248.19:16020] wal.ProtobufLogWriter: Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException): No lease on /hbase/WALs/ut09,16020,1467006160072-splitting/ut09%2C16020%2C1467006160072.default.1468051308829 (inode 849427): File is not open for writing. Holder DFSClient_NONMAPREDUCE_-1100162849_1 does not have any open files.
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3443)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalDatanode(FSNamesystem.java:3345)
at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAdditionalDatanode(NameNodeRpcServer.java:758)
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAdditionalDatanode(ClientNamenodeProtocolServerSideTranslatorPB.java:515)
at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:616)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:969)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2049)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2045)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2043)


at org.apache.hadoop.ipc.Client.call(Client.java:1475)
at org.apache.hadoop.ipc.Client.call(Client.java:1412)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:229)
at com.sun.proxy.$Proxy16.getAdditionalDatanode(Unknown Source)
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAdditionalDatanode(ClientNamenodeProtocolTranslatorPB.java:443)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:191)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102)
at com.sun.proxy.$Proxy17.getAdditionalDatanode(Unknown Source)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:279)
at com.sun.proxy.$Proxy18.getAdditionalDatanode(Unknown Source)
at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:984)
at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1160)
at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:876)
at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:402)
2016-07-09 16:11:19,699 ERROR [regionserver/ut09/10.80.248.19:16020] regionserver.HRegionServer: Shutdown / close of WAL failed: org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException: No lease on /hbase/WALs/ut09,16020,1467006160072-splitting/ut09%2C16020%2C1467006160072.default.1468051308829 (inode 849427): File is not open for writing. Holder DFSClient_NONMAPREDUCE_-1100162849_1 does not have any open files.
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3443)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalDatanode(FSNamesystem.java:3345)
at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAdditionalDatanode(NameNodeRpcServer.java:758)
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAdditionalDatanode(ClientNamenodeProtocolServerSideTranslatorPB.java:515)
at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:616)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:969)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2049)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2045)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2043)


2016-07-09 16:11:19,709 INFO  [regionserver/ut09/10.80.248.19:16020] regionserver.Leases: regionserver/ut09/10.80.248.19:16020 closing leases
2016-07-09 16:11:19,710 INFO  [regionserver/ut09/10.80.248.19:16020] regionserver.Leases: regionserver/ut09/10.80.248.19:16020 closed leases
2016-07-09 16:11:20,044 INFO  [regionserver/ut09/10.80.248.19:16020] hbase.ChoreService: Chore service for: ut09,16020,1467006160072 had [[ScheduledChore: Name: MovedRegionsCleaner for region ut09,16020,1467006160072 Period: 120000 Unit: MILLISECONDS], [ScheduledChore: Name: ut09,16020,1467006160072-MemstoreFlusherChore Period: 10000 Unit: MILLISECONDS]] on shutdown
2016-07-09 16:11:20,045 INFO  [regionserver/ut09/10.80.248.19:16020] regionserver.CompactSplitThread: Waiting for Split Thread to finish...
2016-07-09 16:11:20,045 INFO  [regionserver/ut09/10.80.248.19:16020] regionserver.CompactSplitThread: Waiting for Merge Thread to finish...
2016-07-09 16:11:20,045 INFO  [regionserver/ut09/10.80.248.19:16020] regionserver.CompactSplitThread: Waiting for Large Compaction Thread to finish...
2016-07-09 16:11:20,045 INFO  [regionserver/ut09/10.80.248.19:16020] regionserver.CompactSplitThread: Waiting for Small Compaction Thread to finish...
2016-07-09 16:11:21,387 INFO  [regionserver/ut09/10.80.248.19:16020.leaseChecker] regionserver.Leases: regionserver/ut09/10.80.248.19:16020.leaseChecker closing leases
2016-07-09 16:11:21,389 INFO  [regionserver/ut09/10.80.248.19:16020.leaseChecker] regionserver.Leases: regionserver/ut09/10.80.248.19:16020.leaseChecker closed leases
2016-07-09 16:11:24,409 INFO  [HBase-Metrics2-1] impl.MetricsSystemImpl: Stopping HBase metrics system...
2016-07-09 16:11:24,409 INFO  [HBase-Metrics2-1] impl.MetricsSystemImpl: HBase metrics system stopped.
2016-07-09 16:11:24,909 INFO  [HBase-Metrics2-1] impl.MetricsConfig: loaded properties from hadoop-metrics2-hbase.properties
2016-07-09 16:11:24,910 INFO  [HBase-Metrics2-1] impl.MetricsSystemImpl: Scheduled snapshot period at 10 second(s).
2016-07-09 16:11:24,910 INFO  [HBase-Metrics2-1] impl.MetricsSystemImpl: HBase metrics system started
2016-07-09 16:11:35,085 ERROR [regionserver/ut09/10.80.248.19:16020] zookeeper.RecoverableZooKeeper: ZooKeeper getChildren failed after 4 attempts
2016-07-09 16:11:35,228 WARN  [regionserver/ut09/10.80.248.19:16020] zookeeper.ZKUtil: regionserver:16020-0x154fb34c4132eb5, quorum=10.80.248.12:2181,ut01:2181,ut03:2181, baseZNode=/hbase Unable to list children of znode /hbase/replication/rs/ut09,16020,1467006160072 
org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /hbase/replication/rs/ut09,16020,1467006160072
at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getChildren(RecoverableZooKeeper.java:295)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.listChildrenAndWatchForNewChildren(ZKUtil.java:456)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.listChildrenAndWatchThem(ZKUtil.java:484)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.listChildrenBFSAndWatchThem(ZKUtil.java:1476)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.deleteNodeRecursivelyMultiOrSequential(ZKUtil.java:1398)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.deleteNodeRecursively(ZKUtil.java:1280)
at org.apache.hadoop.hbase.replication.ReplicationQueuesZKImpl.removeAllQueues(ReplicationQueuesZKImpl.java:187)
at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager.join(ReplicationSourceManager.java:309)
at org.apache.hadoop.hbase.replication.regionserver.Replication.join(Replication.java:180)
at org.apache.hadoop.hbase.replication.regionserver.Replication.stopReplicationService(Replication.java:172)
at org.apache.hadoop.hbase.regionserver.HRegionServer.stopServiceThreads(HRegionServer.java:2153)
at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:1084)
at java.lang.Thread.run(Thread.java:745)
2016-07-09 16:11:35,228 ERROR [regionserver/ut09/10.80.248.19:16020] zookeeper.ZooKeeperWatcher: regionserver:16020-0x154fb34c4132eb5, quorum=10.80.248.12:2181,ut01:2181,ut03:2181, baseZNode=/hbase Received unexpected KeeperException, re-throwing exception
org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /hbase/replication/rs/ut09,16020,1467006160072
at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.getChildren(RecoverableZooKeeper.java:295)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.listChildrenAndWatchForNewChildren(ZKUtil.java:456)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.listChildrenAndWatchThem(ZKUtil.java:484)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.listChildrenBFSAndWatchThem(ZKUtil.java:1476)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.deleteNodeRecursivelyMultiOrSequential(ZKUtil.java:1398)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.deleteNodeRecursively(ZKUtil.java:1280)
at org.apache.hadoop.hbase.replication.ReplicationQueuesZKImpl.removeAllQueues(ReplicationQueuesZKImpl.java:187)
at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceManager.join(ReplicationSourceManager.java:309)
at org.apache.hadoop.hbase.replication.regionserver.Replication.join(Replication.java:180)
at org.apache.hadoop.hbase.replication.regionserver.Replication.stopReplicationService(Replication.java:172)
at org.apache.hadoop.hbase.regionserver.HRegionServer.stopServiceThreads(HRegionServer.java:2153)
at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:1084)
at java.lang.Thread.run(Thread.java:745)
2016-07-09 16:11:35,234 INFO  [regionserver/ut09/10.80.248.19:16020] ipc.RpcServer: Stopping server on 16020
2016-07-09 16:11:35,236 INFO  [RpcServer.listener,port=16020] ipc.RpcServer: RpcServer.listener,port=16020: stopping
2016-07-09 16:11:35,277 INFO  [RpcServer.responder] ipc.RpcServer: RpcServer.responder: stopped
2016-07-09 16:11:35,277 INFO  [RpcServer.responder] ipc.RpcServer: RpcServer.responder: stopping
2016-07-09 16:11:51,590 ERROR [regionserver/ut09/10.80.248.19:16020] zookeeper.RecoverableZooKeeper: ZooKeeper delete failed after 4 attempts
2016-07-09 16:11:51,590 WARN  [regionserver/ut09/10.80.248.19:16020] regionserver.HRegionServer: Failed deleting my ephemeral node
org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /hbase/rs/ut09,16020,1467006160072
at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:873)
at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.delete(RecoverableZooKeeper.java:178)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.deleteNode(ZKUtil.java:1236)
at org.apache.hadoop.hbase.zookeeper.ZKUtil.deleteNode(ZKUtil.java:1225)
at org.apache.hadoop.hbase.regionserver.HRegionServer.deleteMyEphemeralNode(HRegionServer.java:1418)
at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:1092)
at java.lang.Thread.run(Thread.java:745)
2016-07-09 16:11:51,590 INFO  [regionserver/ut09/10.80.248.19:16020] regionserver.HRegionServer: stopping server ut09,16020,1467006160072; zookeeper connection closed.
2016-07-09 16:11:51,591 INFO  [regionserver/ut09/10.80.248.19:16020] regionserver.HRegionServer: regionserver/ut09/10.80.248.19:16020 exiting
2016-07-09 16:11:53,809 ERROR [main] regionserver.HRegionServerCommandLine: Region server exiting
java.lang.RuntimeException: HRegionServer Aborted
at org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.start(HRegionServerCommandLine.java:68)
at org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.run(HRegionServerCommandLine.java:87)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70)
at org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:126)
at org.apache.hadoop.hbase.regionserver.HRegionServer.main(HRegionServer.java:2667)
2016-07-09 16:11:54,380 INFO  [Thread-5] regionserver.ShutdownHook: Shutdown hook starting; hbase.shutdown.hook=true; fsShutdownHook=org.apache.hadoop.fs.FileSystem$Cache$ClientFinalizer@1e044120
2016-07-09 16:11:54,380 INFO  [Thread-5] regionserver.ShutdownHook: Starting fs shutdown hook thread.
2016-07-09 16:11:54,408 INFO  [Thread-5] regionserver.ShutdownHook: Shutdown hook finished.