at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:815) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:803) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:625) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:147) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:5615) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:5766) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:5553) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2413) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32205) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2127) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108) at java.lang.Thread.run(Thread.java:745) 2017-08-05 00:01:54,042 INFO [B.defaultRpcServer.handler=10,queue=0,port=16020] hdfs.DFSClient: Access token was invalid when connecting to /hadoop4:50010 : org.apache.hadoop.security.token.SecretManager$InvalidToken: access control error while attempting to set up short-circuit access to /apps/hbase/data/data/default/CampaignMailBounce/0ad76bdd20489f07f496a4d53b2211ab/info/466200153cd04f18a703dcfeff5c3ce6 2017-08-05 00:02:11,670 INFO [B.defaultRpcServer.handler=25,queue=0,port=16020] hdfs.DFSClient: Access token was invalid when connecting to /hadoop5:50010 : org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException: Got access token error, status message , for OP_READ_BLOCK, self=/hadoop4:46602, remote=/hadoop5:50010, for file /apps/hbase/data/data/default/CampaignMailBounce/858650f09dfc5a81c6656c24a426b526/info/47eb254b75584c0ba43423dbc8757fc7, for pool BP-1810172115-hadoop2-1478343078462 block 1086907493_13222380 2017-08-05 00:02:15,642 INFO [regionserver/aps-hadoop4/hadoop4:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501891019129 with entries=301, filesize=121.97 MB; new WAL /apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501891335600 2017-08-05 00:02:25,133 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=6.13 GB, freeSize=254.96 MB, max=6.38 GB, blockCount=98574, accesses=436289279, hits=225844844, hitRatio=51.76%, , cachingAccesses=432526000, cachingHits=225843334, cachingHitsRatio=52.21%, evictions=55430, evicted=204894128, evictedPerRun=3696.4482421875 2017-08-05 00:02:29,481 INFO [sync.1] wal.FSHLog: Slow sync cost: 1783 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-9ba4f08a-d996-4490-b27d-6c8ca9a67152,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK]] 2017-08-05 00:02:29,484 INFO [sync.2] wal.FSHLog: Slow sync cost: 1755 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-9ba4f08a-d996-4490-b27d-6c8ca9a67152,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK]] 2017-08-05 00:02:29,493 INFO [sync.3] wal.FSHLog: Slow sync cost: 1706 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-9ba4f08a-d996-4490-b27d-6c8ca9a67152,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK]] 2017-08-05 00:03:44,956 INFO [B.defaultRpcServer.handler=2,queue=2,port=16020] shortcircuit.ShortCircuitCache: ShortCircuitCache(0x11b44177): could not load 1086651581_BP-1810172115-hadoop2-1478343078462 due to InvalidToken exception. org.apache.hadoop.security.token.SecretManager$InvalidToken: access control error while attempting to set up short-circuit access to /apps/hbase/data/data/default/CampaignBlastScheduleTransaction/04b08c4b94d6579e5bff09bb9adbf643/info/7b443b01196b4a2d9e34104b65c270f9 at org.apache.hadoop.hdfs.BlockReaderFactory.requestFileDescriptors(BlockReaderFactory.java:589) at org.apache.hadoop.hdfs.BlockReaderFactory.createShortCircuitReplicaInfo(BlockReaderFactory.java:488) at org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.create(ShortCircuitCache.java:784) at org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.fetchOrCreate(ShortCircuitCache.java:718) at org.apache.hadoop.hdfs.BlockReaderFactory.getBlockReaderLocal(BlockReaderFactory.java:422) at org.apache.hadoop.hdfs.BlockReaderFactory.build(BlockReaderFactory.java:333) at org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:662) at org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:898) at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:955) at java.io.DataInputStream.read(DataInputStream.java:149) at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:199) at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1407) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1677) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1504) at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:441) at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$ScannerV2.seekTo(HFileReaderV2.java:910) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:290) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:194) at org.apache.hadoop.hbase.regionserver.NonLazyKeyValueScanner.doRealSeek(NonLazyKeyValueScanner.java:55) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:312) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:268) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:815) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekAsDirection(StoreScanner.java:803) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:625) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:147) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:5615) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:5766) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:5553) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2413) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32205) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2127) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108) at java.lang.Thread.run(Thread.java:745) 2017-08-05 00:03:44,956 INFO [B.defaultRpcServer.handler=2,queue=2,port=16020] hdfs.DFSClient: Access token was invalid when connecting to /hadoop4:50010 : org.apache.hadoop.security.token.SecretManager$InvalidToken: access control error while attempting to set up short-circuit access to /apps/hbase/data/data/default/CampaignBlastScheduleTransaction/04b08c4b94d6579e5bff09bb9adbf643/info/7b443b01196b4a2d9e34104b65c270f9 2017-08-05 00:04:55,004 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183., current region memstore size 128.28 MB, and 1/1 column families' memstores are being flushed. 2017-08-05 00:04:55,803 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=65586385, memsize=128.3 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/8083c44b3c7bce24d150b22ed70ea183/.tmp/098ea6167350478ea2331edf860b0266 2017-08-05 00:04:55,817 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/8083c44b3c7bce24d150b22ed70ea183/info/098ea6167350478ea2331edf860b0266, entries=49677, sequenceid=65586385, filesize=29.1 M 2017-08-05 00:04:55,819 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.28 MB/134510480, currentsize=2.48 MB/2605256 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183. in 815ms, sequenceid=65586385, compaction requested=true 2017-08-05 00:04:55,820 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183. 2017-08-05 00:04:55,820 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/8083c44b3c7bce24d150b22ed70ea183/.tmp, totalSize=87.4 M 2017-08-05 00:04:55,824 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=100657, currentSize=6718117488, freeSize=132360592, maxSize=6850478080, heapSize=6718117488, minSize=6507954176, minFactor=0.95, multiSize=3253977088, multiFactor=0.5, singleSize=1626988544, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-08-05 00:04:57,176 INFO [sync.2] wal.FSHLog: Slow sync cost: 118 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-9ba4f08a-d996-4490-b27d-6c8ca9a67152,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK]] 2017-08-05 00:04:57,695 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183. into b39b14c77a6a4168b8e675b296af8854(size=29.2 M), total size for store is 152.4 M. This selection was in queue for 0sec, and took 1sec to execute. 2017-08-05 00:04:57,696 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183., storeName=info, fileCount=3, fileSize=87.4 M, priority=6, time=25976258138203125; duration=1sec 2017-08-05 00:07:25,133 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=6.06 GB, freeSize=326.69 MB, max=6.38 GB, blockCount=97479, accesses=436555163, hits=226006073, hitRatio=51.77%, , cachingAccesses=432790121, cachingHits=226004563, cachingHitsRatio=52.22%, evictions=55467, evicted=204998118, evictedPerRun=3695.857177734375 2017-08-05 00:07:29,400 INFO [sync.0] wal.FSHLog: Slow sync cost: 1306 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-9ba4f08a-d996-4490-b27d-6c8ca9a67152,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK]] 2017-08-05 00:07:32,316 INFO [sync.2] wal.FSHLog: Slow sync cost: 1021 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-9ba4f08a-d996-4490-b27d-6c8ca9a67152,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK]] 2017-08-05 00:07:34,310 INFO [regionserver/aps-hadoop4/hadoop4:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501891335600 with entries=305, filesize=121.94 MB; new WAL /apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501891654254 2017-08-05 00:10:01,370 WARN [B.defaultRpcServer.handler=27,queue=2,port=16020] ipc.RpcServer: (responseTooSlow): {"call":"Scan(org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ScanRequest)","starttimems":1501891741298,"responsesize":416,"method":"Scan","processingtimems":60072,"client":"hadoop4:50258","queuetimems":0,"class":"HRegionServer"} 2017-08-05 00:11:04,245 WARN [B.defaultRpcServer.handler=30,queue=0,port=16020] ipc.RpcServer: (responseTooSlow): {"call":"Scan(org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ScanRequest)","starttimems":1501891852441,"responsesize":416,"method":"Scan","processingtimems":11804,"client":"hadoop4:58186","queuetimems":0,"class":"HRegionServer"} 2017-08-05 00:12:25,133 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=6.24 GB, freeSize=147.10 MB, max=6.38 GB, blockCount=100332, accesses=437645040, hits=226550989, hitRatio=51.77%, , cachingAccesses=433879998, cachingHits=226549479, cachingHitsRatio=52.21%, evictions=55603, evicted=205540223, evictedPerRun=3696.567138671875 2017-08-05 00:13:46,576 INFO [sync.3] wal.FSHLog: Slow sync cost: 1366 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK], DatanodeInfoWithStorage[hadoop7:50010,DS-9a10c707-ebd5-4fae-8f82-b381f706fa57,DISK]] 2017-08-05 00:13:46,757 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183., current region memstore size 128.81 MB, and 1/1 column families' memstores are being flushed. 2017-08-05 00:13:47,591 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=65586863, memsize=128.8 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/8083c44b3c7bce24d150b22ed70ea183/.tmp/400e6c2f8ccb43db82a6bde669df918f 2017-08-05 00:13:47,609 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/8083c44b3c7bce24d150b22ed70ea183/info/400e6c2f8ccb43db82a6bde669df918f, entries=49677, sequenceid=65586863, filesize=29.1 M 2017-08-05 00:13:47,611 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~128.81 MB/135063640, currentsize=0 B/0 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183. in 854ms, sequenceid=65586863, compaction requested=true 2017-08-05 00:13:52,091 INFO [sync.0] wal.FSHLog: Slow sync cost: 1953 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK], DatanodeInfoWithStorage[hadoop7:50010,DS-9a10c707-ebd5-4fae-8f82-b381f706fa57,DISK]] 2017-08-05 00:13:57,355 INFO [sync.2] wal.FSHLog: Slow sync cost: 2063 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK], DatanodeInfoWithStorage[hadoop7:50010,DS-9a10c707-ebd5-4fae-8f82-b381f706fa57,DISK]] 2017-08-05 00:13:57,356 INFO [sync.3] wal.FSHLog: Slow sync cost: 2038 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK], DatanodeInfoWithStorage[hadoop7:50010,DS-9a10c707-ebd5-4fae-8f82-b381f706fa57,DISK]] 2017-08-05 00:14:41,796 WARN [ResponseProcessor for block BP-1810172115-hadoop2-1478343078462:blk_1087207171_13526135] hdfs.DFSClient: Slow ReadProcessor read fields took 44115ms (threshold=30000ms); ack: seqno: 1006 reply: SUCCESS reply: SUCCESS reply: SUCCESS downstreamAckTimeNanos: 41417948215 flag: 0 flag: 0 flag: 0, targets: [DatanodeInfoWithStorage[hadoop4:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK], DatanodeInfoWithStorage[hadoop7:50010,DS-9a10c707-ebd5-4fae-8f82-b381f706fa57,DISK]] 2017-08-05 00:14:41,812 WARN [sync.1] hdfs.DFSClient: Slow waitForAckedSeqno took 41434ms (threshold=30000ms) 2017-08-05 00:14:41,812 INFO [sync.1] wal.FSHLog: Slow sync cost: 41433 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK], DatanodeInfoWithStorage[hadoop7:50010,DS-9a10c707-ebd5-4fae-8f82-b381f706fa57,DISK]] 2017-08-05 00:14:41,813 WARN [B.defaultRpcServer.handler=4,queue=4,port=16020] ipc.RpcServer: (responseTooSlow): {"call":"Multi(org.apache.hadoop.hbase.protobuf.generated.ClientProtos$MultiRequest)","starttimems":1501892040374,"responsesize":285,"method":"Multi","processingtimems":41438,"client":"hadoop3:41014","queuetimems":0,"class":"HRegionServer"} 2017-08-05 00:15:21,736 WARN [B.defaultRpcServer.handler=38,queue=3,port=16020] ipc.RpcServer: (responseTooSlow): {"call":"Scan(org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ScanRequest)","starttimems":1501892104541,"responsesize":416,"method":"Scan","processingtimems":17194,"client":"hadoop4:45388","queuetimems":0,"class":"HRegionServer"} 2017-08-05 00:17:25,133 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=6.18 GB, freeSize=207.39 MB, max=6.38 GB, blockCount=99380, accesses=438801947, hits=227129497, hitRatio=51.76%, , cachingAccesses=435036905, cachingHits=227127987, cachingHitsRatio=52.21%, evictions=55743, evicted=206119572, evictedPerRun=3697.67626953125 2017-08-05 00:17:46,692 INFO [aps-hadoop4,16020,1501679842018_ChoreService_1] regionserver.HRegionServer: aps-hadoop4,16020,1501679842018-MemstoreFlusherChore requesting flush for region CampaignFactsByLinkClicksMDC,cccccccc,1478428072397.e2fdff075d0d93921fbe26ed912db4cf. after a delay of 7114 2017-08-05 00:17:53,806 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for CampaignFactsByLinkClicksMDC,cccccccc,1478428072397.e2fdff075d0d93921fbe26ed912db4cf., current region memstore size 33.23 KB, and 1/1 column families' memstores are being flushed. 2017-08-05 00:17:53,972 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=614658, memsize=33.2 K, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/CampaignFactsByLinkClicksMDC/e2fdff075d0d93921fbe26ed912db4cf/.tmp/43a30530ad954a91a501fb356af7692a 2017-08-05 00:17:53,994 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/CampaignFactsByLinkClicksMDC/e2fdff075d0d93921fbe26ed912db4cf/info/43a30530ad954a91a501fb356af7692a, entries=132, sequenceid=614658, filesize=7.6 K 2017-08-05 00:17:53,998 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~33.23 KB/34024, currentsize=0 B/0 for region CampaignFactsByLinkClicksMDC,cccccccc,1478428072397.e2fdff075d0d93921fbe26ed912db4cf. in 192ms, sequenceid=614658, compaction requested=false 2017-08-05 00:18:38,635 WARN [B.defaultRpcServer.handler=10,queue=0,port=16020] ipc.RpcServer: (responseTooSlow): {"call":"Scan(org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ScanRequest)","starttimems":1501892303284,"responsesize":416,"method":"Scan","processingtimems":15351,"client":"hadoop4:39984","queuetimems":0,"class":"HRegionServer"} 2017-08-05 00:19:55,879 INFO [regionserver/aps-hadoop4/hadoop4:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501891654254 with entries=257, filesize=122.08 MB; new WAL /apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501892395738 2017-08-05 00:19:55,880 INFO [regionserver/aps-hadoop4/hadoop4:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501888649854 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop4%2C16020%2C1501679842018.default.1501888649854 2017-08-05 00:19:55,896 INFO [regionserver/aps-hadoop4/hadoop4:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501889077580 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop4%2C16020%2C1501679842018.default.1501889077580 2017-08-05 00:19:55,913 INFO [regionserver/aps-hadoop4/hadoop4:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501889381424 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop4%2C16020%2C1501679842018.default.1501889381424 2017-08-05 00:19:55,927 INFO [regionserver/aps-hadoop4/hadoop4:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501889682292 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop4%2C16020%2C1501679842018.default.1501889682292 2017-08-05 00:22:25,133 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=6.17 GB, freeSize=220.09 MB, max=6.38 GB, blockCount=99181, accesses=439786480, hits=227621778, hitRatio=51.76%, , cachingAccesses=436021438, cachingHits=227620269, cachingHitsRatio=52.20%, evictions=55862, evicted=206612025, evictedPerRun=3698.614990234375 2017-08-05 00:23:21,598 INFO [sync.3] wal.FSHLog: Slow sync cost: 4211 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK]] 2017-08-05 00:23:24,505 INFO [sync.4] wal.FSHLog: Slow sync cost: 1561 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK]] 2017-08-05 00:23:27,042 INFO [sync.3] wal.FSHLog: Slow sync cost: 214 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK]] 2017-08-05 00:23:30,020 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183., current region memstore size 128.05 MB, and 1/1 column families' memstores are being flushed. 2017-08-05 00:23:30,887 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=65587340, memsize=128.1 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/8083c44b3c7bce24d150b22ed70ea183/.tmp/aa944c79c84f49a19286a02e22b270b5 2017-08-05 00:23:30,903 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/8083c44b3c7bce24d150b22ed70ea183/info/aa944c79c84f49a19286a02e22b270b5, entries=49677, sequenceid=65587340, filesize=29.1 M 2017-08-05 00:23:30,906 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.05 MB/134274352, currentsize=2.90 MB/3039424 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183. in 886ms, sequenceid=65587340, compaction requested=true 2017-08-05 00:23:30,909 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183. 2017-08-05 00:23:30,909 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/8083c44b3c7bce24d150b22ed70ea183/.tmp, totalSize=87.4 M 2017-08-05 00:23:30,925 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=100441, currentSize=6703245264, freeSize=147232816, maxSize=6850478080, heapSize=6703245264, minSize=6507954176, minFactor=0.95, multiSize=3253977088, multiFactor=0.5, singleSize=1626988544, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-08-05 00:23:32,016 INFO [sync.4] wal.FSHLog: Slow sync cost: 355 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK]] 2017-08-05 00:23:33,668 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183. into 702b039b6d0947beaac24ab80668f8b3(size=29.2 M), total size for store is 152.4 M. This selection was in queue for 0sec, and took 2sec to execute. 2017-08-05 00:23:33,668 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183., storeName=info, fileCount=3, fileSize=87.4 M, priority=6, time=25977373226641331; duration=2sec 2017-08-05 00:23:37,052 INFO [sync.4] wal.FSHLog: Slow sync cost: 335 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK]] 2017-08-05 00:24:26,527 INFO [sync.2] wal.FSHLog: Slow sync cost: 8661 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK]] 2017-08-05 00:27:25,133 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=6.28 GB, freeSize=103.61 MB, max=6.38 GB, blockCount=101018, accesses=441116098, hits=228285681, hitRatio=51.75%, , cachingAccesses=437349293, cachingHits=228284171, cachingHitsRatio=52.20%, evictions=56022, evicted=207274139, evictedPerRun=3699.870361328125 2017-08-05 00:27:25,972 WARN [ResponseProcessor for block BP-1810172115-hadoop2-1478343078462:blk_1087207309_13526280] hdfs.DFSClient: Slow ReadProcessor read fields took 61325ms (threshold=30000ms); ack: seqno: 1330 reply: SUCCESS reply: SUCCESS reply: SUCCESS downstreamAckTimeNanos: 36473869454 flag: 0 flag: 0 flag: 0, targets: [DatanodeInfoWithStorage[hadoop4:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK]] 2017-08-05 00:27:25,972 WARN [sync.2] hdfs.DFSClient: Slow waitForAckedSeqno took 36474ms (threshold=30000ms) 2017-08-05 00:27:25,972 INFO [sync.2] wal.FSHLog: Slow sync cost: 36474 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK]] 2017-08-05 00:27:25,972 WARN [B.defaultRpcServer.handler=46,queue=1,port=16020] ipc.RpcServer: (responseTooSlow): {"call":"Multi(org.apache.hadoop.hbase.protobuf.generated.ClientProtos$MultiRequest)","starttimems":1501892809497,"responsesize":74,"method":"Multi","processingtimems":36475,"client":"hadoop3:41542","queuetimems":0,"class":"HRegionServer"} 2017-08-05 00:28:03,481 INFO [sync.3] wal.FSHLog: Slow sync cost: 354 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK]] 2017-08-05 00:28:06,394 INFO [regionserver/aps-hadoop4/hadoop4:16020.logRoller] wal.FSHLog: Slow sync cost: 177 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK]] 2017-08-05 00:28:06,411 INFO [regionserver/aps-hadoop4/hadoop4:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501892395738 with entries=350, filesize=121.61 MB; new WAL /apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501892886067 2017-08-05 00:28:06,413 INFO [regionserver/aps-hadoop4/hadoop4:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501891654254 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop4%2C16020%2C1501679842018.default.1501891654254 2017-08-05 00:31:40,051 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183., current region memstore size 128.24 MB, and 1/1 column families' memstores are being flushed. 2017-08-05 00:31:40,953 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=65587820, memsize=128.2 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/8083c44b3c7bce24d150b22ed70ea183/.tmp/6fd42698391b4ebab30b2493d562792a 2017-08-05 00:31:41,006 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/8083c44b3c7bce24d150b22ed70ea183/info/6fd42698391b4ebab30b2493d562792a, entries=49677, sequenceid=65587820, filesize=29.1 M 2017-08-05 00:31:41,008 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.24 MB/134472264, currentsize=1.16 MB/1219808 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183. in 957ms, sequenceid=65587820, compaction requested=true 2017-08-05 00:32:25,133 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=6.12 GB, freeSize=264.59 MB, max=6.38 GB, blockCount=98471, accesses=442038164, hits=228746783, hitRatio=51.75%, , cachingAccesses=438271359, cachingHits=228745273, cachingHitsRatio=52.19%, evictions=56134, evicted=207737649, evictedPerRun=3700.74560546875 2017-08-05 00:35:37,166 INFO [sync.2] wal.FSHLog: Slow sync cost: 1568 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-1940f918-25a9-4136-8034-e6fd0972e5a2,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-9ba4f08a-d996-4490-b27d-6c8ca9a67152,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK]] 2017-08-05 00:35:42,096 INFO [sync.1] wal.FSHLog: Slow sync cost: 1670 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-1940f918-25a9-4136-8034-e6fd0972e5a2,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-9ba4f08a-d996-4490-b27d-6c8ca9a67152,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK]] 2017-08-05 00:35:47,704 INFO [sync.2] wal.FSHLog: Slow sync cost: 1871 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-1940f918-25a9-4136-8034-e6fd0972e5a2,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-9ba4f08a-d996-4490-b27d-6c8ca9a67152,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK]] 2017-08-05 00:35:50,856 INFO [sync.1] wal.FSHLog: Slow sync cost: 463 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-1940f918-25a9-4136-8034-e6fd0972e5a2,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-9ba4f08a-d996-4490-b27d-6c8ca9a67152,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK]] 2017-08-05 00:37:25,133 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=6.24 GB, freeSize=141.86 MB, max=6.38 GB, blockCount=100410, accesses=443209035, hits=229332220, hitRatio=51.74%, , cachingAccesses=439442230, cachingHits=229330710, cachingHitsRatio=52.19%, evictions=56275, evicted=208321147, evictedPerRun=3701.841796875 2017-08-05 00:37:39,238 INFO [sync.2] wal.FSHLog: Slow sync cost: 5090 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-1940f918-25a9-4136-8034-e6fd0972e5a2,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-9ba4f08a-d996-4490-b27d-6c8ca9a67152,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK]] 2017-08-05 00:40:05,533 WARN [ResponseProcessor for block BP-1810172115-hadoop2-1478343078462:blk_1087207358_13526329] hdfs.DFSClient: Slow ReadProcessor read fields took 42998ms (threshold=30000ms); ack: seqno: 2101 reply: SUCCESS reply: SUCCESS reply: SUCCESS downstreamAckTimeNanos: 42954450107 flag: 0 flag: 0 flag: 0, targets: [DatanodeInfoWithStorage[hadoop4:50010,DS-1940f918-25a9-4136-8034-e6fd0972e5a2,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-9ba4f08a-d996-4490-b27d-6c8ca9a67152,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK]] 2017-08-05 00:40:05,534 WARN [sync.0] hdfs.DFSClient: Slow waitForAckedSeqno took 42955ms (threshold=30000ms) 2017-08-05 00:40:05,534 INFO [sync.0] wal.FSHLog: Slow sync cost: 42954 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-1940f918-25a9-4136-8034-e6fd0972e5a2,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-9ba4f08a-d996-4490-b27d-6c8ca9a67152,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK]] 2017-08-05 00:40:05,534 WARN [B.defaultRpcServer.handler=39,queue=4,port=16020] ipc.RpcServer: (responseTooSlow): {"call":"Multi(org.apache.hadoop.hbase.protobuf.generated.ClientProtos$MultiRequest)","starttimems":1501893562578,"responsesize":8,"method":"Multi","processingtimems":42956,"client":"hadoop3:42026","queuetimems":0,"class":"HRegionServer"} 2017-08-05 00:40:05,554 INFO [sync.1] wal.FSHLog: Slow sync cost: 23675 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-1940f918-25a9-4136-8034-e6fd0972e5a2,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-9ba4f08a-d996-4490-b27d-6c8ca9a67152,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK]] 2017-08-05 00:40:05,554 WARN [B.defaultRpcServer.handler=17,queue=2,port=16020] ipc.RpcServer: (responseTooSlow): {"call":"Multi(org.apache.hadoop.hbase.protobuf.generated.ClientProtos$MultiRequest)","starttimems":1501893581876,"responsesize":74,"method":"Multi","processingtimems":23678,"client":"hadoop3:42026","queuetimems":0,"class":"HRegionServer"} 2017-08-05 00:40:10,952 INFO [sync.3] wal.FSHLog: Slow sync cost: 212 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-1940f918-25a9-4136-8034-e6fd0972e5a2,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-9ba4f08a-d996-4490-b27d-6c8ca9a67152,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK]] 2017-08-05 00:40:11,594 INFO [regionserver/aps-hadoop4/hadoop4:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501892886067 with entries=338, filesize=121.83 MB; new WAL /apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501893611545 2017-08-05 00:40:16,917 INFO [sync.0] wal.FSHLog: Slow sync cost: 265 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-1940f918-25a9-4136-8034-e6fd0972e5a2,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK]] 2017-08-05 00:40:19,591 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183., current region memstore size 128.39 MB, and 1/1 column families' memstores are being flushed. 2017-08-05 00:40:20,384 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=65588299, memsize=128.4 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/8083c44b3c7bce24d150b22ed70ea183/.tmp/d2dd39bb6a45468b9da4d181afffdfc0 2017-08-05 00:40:20,408 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/8083c44b3c7bce24d150b22ed70ea183/info/d2dd39bb6a45468b9da4d181afffdfc0, entries=49677, sequenceid=65588299, filesize=29.1 M 2017-08-05 00:40:20,456 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.39 MB/134628016, currentsize=2.39 MB/2509200 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183. in 865ms, sequenceid=65588299, compaction requested=true 2017-08-05 00:40:20,461 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183. 2017-08-05 00:40:20,461 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/8083c44b3c7bce24d150b22ed70ea183/.tmp, totalSize=87.4 M 2017-08-05 00:40:20,463 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=99853, currentSize=6664480880, freeSize=185997200, maxSize=6850478080, heapSize=6664480880, minSize=6507954176, minFactor=0.95, multiSize=3253977088, multiFactor=0.5, singleSize=1626988544, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-08-05 00:40:22,481 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183. into fde79a0054584c0b859c20b00a7ba8a9(size=29.2 M), total size for store is 152.4 M. This selection was in queue for 0sec, and took 2sec to execute. 2017-08-05 00:40:22,481 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183., storeName=info, fileCount=3, fileSize=87.4 M, priority=6, time=25978382778361144; duration=2sec 2017-08-05 00:40:54,390 INFO [sync.1] wal.FSHLog: Slow sync cost: 8384 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-1940f918-25a9-4136-8034-e6fd0972e5a2,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK]] 2017-08-05 00:42:25,133 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=6.30 GB, freeSize=78.22 MB, max=6.38 GB, blockCount=101420, accesses=444313589, hits=229883599, hitRatio=51.74%, , cachingAccesses=440545021, cachingHits=229882089, cachingHitsRatio=52.18%, evictions=56408, evicted=208871547, evictedPerRun=3702.87109375 2017-08-05 00:44:48,940 INFO [sync.3] wal.FSHLog: Slow sync cost: 204 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-1940f918-25a9-4136-8034-e6fd0972e5a2,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK]] 2017-08-05 00:44:54,783 INFO [sync.0] wal.FSHLog: Slow sync cost: 128 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-1940f918-25a9-4136-8034-e6fd0972e5a2,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK]] 2017-08-05 00:44:54,798 INFO [sync.1] wal.FSHLog: Slow sync cost: 124 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-1940f918-25a9-4136-8034-e6fd0972e5a2,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK]] 2017-08-05 00:47:25,133 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=6.13 GB, freeSize=255.59 MB, max=6.38 GB, blockCount=98614, accesses=445640484, hits=230547035, hitRatio=51.73%, , cachingAccesses=441871916, cachingHits=230545525, cachingHitsRatio=52.17%, evictions=56569, evicted=209537812, evictedPerRun=3704.110107421875 2017-08-05 00:47:42,989 INFO [sync.2] wal.FSHLog: Slow sync cost: 283 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-1940f918-25a9-4136-8034-e6fd0972e5a2,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK]] 2017-08-05 00:47:46,692 INFO [aps-hadoop4,16020,1501679842018_ChoreService_1] regionserver.HRegionServer: aps-hadoop4,16020,1501679842018-MemstoreFlusherChore requesting flush for region CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. after a delay of 13922 2017-08-05 00:47:47,110 INFO [regionserver/aps-hadoop4/hadoop4:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501893611545 with entries=312, filesize=122.40 MB; new WAL /apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501894067065 2017-08-05 00:47:53,564 INFO [sync.1] wal.FSHLog: Slow sync cost: 1494 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK]] 2017-08-05 00:47:53,566 INFO [sync.2] wal.FSHLog: Slow sync cost: 1464 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK]] 2017-08-05 00:47:54,773 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183., current region memstore size 128.80 MB, and 1/1 column families' memstores are being flushed. 2017-08-05 00:47:55,789 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=65588781, memsize=129.0 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/8083c44b3c7bce24d150b22ed70ea183/.tmp/489473b97fb6492e949d475c01b9ae53 2017-08-05 00:47:55,809 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/8083c44b3c7bce24d150b22ed70ea183/info/489473b97fb6492e949d475c01b9ae53, entries=49677, sequenceid=65588781, filesize=29.1 M 2017-08-05 00:47:55,813 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~129.04 MB/135307024, currentsize=3.19 MB/3349696 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183. in 1040ms, sequenceid=65588781, compaction requested=true 2017-08-05 00:47:56,692 INFO [aps-hadoop4,16020,1501679842018_ChoreService_1] regionserver.HRegionServer: aps-hadoop4,16020,1501679842018-MemstoreFlusherChore requesting flush for region CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. after a delay of 5726 2017-08-05 00:47:58,833 INFO [sync.2] wal.FSHLog: Slow sync cost: 1365 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK]] 2017-08-05 00:47:58,834 INFO [sync.3] wal.FSHLog: Slow sync cost: 1328 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK]] 2017-08-05 00:48:00,615 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494., current region memstore size 2.69 MB, and 1/12 column families' memstores are being flushed. 2017-08-05 00:48:00,615 INFO [MemStoreFlusher.1] regionserver.HRegion: Flushing Column Family: info which was occupying 736.34 KB of memstore. 2017-08-05 00:48:00,670 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=158236863, memsize=735.9 K, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/CampaignGoalMDC/6cf8953db275a43b98ab944d2d614494/.tmp/c5a50817a71d4b80a7363843543702fb 2017-08-05 00:48:00,688 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/CampaignGoalMDC/6cf8953db275a43b98ab944d2d614494/info/c5a50817a71d4b80a7363843543702fb, entries=2538, sequenceid=158236863, filesize=39.8 K 2017-08-05 00:48:00,690 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~735.95 KB/753608, currentsize=1.97 MB/2066688 for region CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. in 75ms, sequenceid=158236863, compaction requested=false 2017-08-05 00:48:03,862 INFO [sync.0] wal.FSHLog: Slow sync cost: 1317 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK]] 2017-08-05 00:50:16,692 INFO [aps-hadoop4,16020,1501679842018_ChoreService_1] regionserver.HRegionServer: aps-hadoop4,16020,1501679842018-MemstoreFlusherChore requesting flush for region CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. after a delay of 3338 2017-08-05 00:50:20,030 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494., current region memstore size 1.97 MB, and 1/12 column families' memstores are being flushed. 2017-08-05 00:50:20,030 INFO [MemStoreFlusher.0] regionserver.HRegion: Flushing Column Family: Em which was occupying 1.29 MB of memstore. 2017-08-05 00:50:20,100 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=158236866, memsize=1.3 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/CampaignGoalMDC/6cf8953db275a43b98ab944d2d614494/.tmp/88d6185b0256457a893c21d54d34295a 2017-08-05 00:50:20,124 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/CampaignGoalMDC/6cf8953db275a43b98ab944d2d614494/Em/88d6185b0256457a893c21d54d34295a, entries=4526, sequenceid=158236866, filesize=59.6 K 2017-08-05 00:50:20,128 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~1.29 MB/1348920, currentsize=700.95 KB/717768 for region CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. in 98ms, sequenceid=158236866, compaction requested=true 2017-08-05 00:50:20,134 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] regionserver.HRegion: Starting compaction on Em in region CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. 2017-08-05 00:50:20,134 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] regionserver.HStore: Starting compaction of 3 file(s) in Em of CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/CampaignGoalMDC/6cf8953db275a43b98ab944d2d614494/.tmp, totalSize=3.1 M 2017-08-05 00:50:20,164 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=97762, currentSize=6525873200, freeSize=324604880, maxSize=6850478080, heapSize=6525873200, minSize=6507954176, minFactor=0.95, multiSize=3253977088, multiFactor=0.5, singleSize=1626988544, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-08-05 00:50:20,609 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] regionserver.HStore: Completed compaction of 3 (all) file(s) in Em of CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. into 9a810a091fd842308fdea113987646db(size=3.0 M), total size for store is 3.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-08-05 00:50:20,609 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] regionserver.CompactSplitThread: Completed compaction: Request = regionName=CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494., storeName=Em, fileCount=3, fileSize=3.1 M, priority=7, time=25978982451803395; duration=0sec 2017-08-05 00:52:25,151 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=6.08 GB, freeSize=308.33 MB, max=6.38 GB, blockCount=97781, accesses=446687960, hits=231123382, hitRatio=51.74%, , cachingAccesses=442919200, cachingHits=231121872, cachingHitsRatio=52.18%, evictions=56683, evicted=210009582, evictedPerRun=3704.983642578125 2017-08-05 00:54:27,793 INFO [sync.3] wal.FSHLog: Slow sync cost: 157 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK]] 2017-08-05 00:57:25,133 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=6.24 GB, freeSize=145.62 MB, max=6.38 GB, blockCount=100354, accesses=448182829, hits=231870790, hitRatio=51.74%, , cachingAccesses=444414069, cachingHits=231869280, cachingHitsRatio=52.17%, evictions=56863, evicted=210754471, evictedPerRun=3706.35498046875 2017-08-05 00:58:16,144 WARN [ResponseProcessor for block BP-1810172115-hadoop2-1478343078462:blk_1087207533_13526504] hdfs.DFSClient: Slow ReadProcessor read fields took 65062ms (threshold=30000ms); ack: seqno: -2 reply: SUCCESS reply: SUCCESS reply: ERROR downstreamAckTimeNanos: 0 flag: 0 flag: 0 flag: 1, targets: [DatanodeInfoWithStorage[hadoop4:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK]] 2017-08-05 00:58:16,144 WARN [ResponseProcessor for block BP-1810172115-hadoop2-1478343078462:blk_1087207533_13526504] hdfs.DFSClient: DFSOutputStream ResponseProcessor exception for block BP-1810172115-hadoop2-1478343078462:blk_1087207533_13526504 java.io.IOException: Bad response ERROR for block BP-1810172115-hadoop2-1478343078462:blk_1087207533_13526504 from datanode DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK] at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer$ResponseProcessor.run(DFSOutputStream.java:785) 2017-08-05 00:58:16,144 WARN [DataStreamer for file /apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501894067065 block BP-1810172115-hadoop2-1478343078462:blk_1087207533_13526504] hdfs.DFSClient: Error Recovery for block BP-1810172115-hadoop2-1478343078462:blk_1087207533_13526504 in pipeline DatanodeInfoWithStorage[hadoop4:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK]: bad datanode DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK] 2017-08-05 00:58:21,371 WARN [sync.3] hdfs.DFSClient: Slow waitForAckedSeqno took 70027ms (threshold=30000ms) 2017-08-05 00:58:21,401 INFO [sync.3] wal.FSHLog: Slow sync cost: 70055 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-d7a093e4-6437-4935-b4a2-0decf44fabea,DISK]] 2017-08-05 00:58:21,401 WARN [B.defaultRpcServer.handler=0,queue=0,port=16020] ipc.RpcServer: (responseTooSlow): {"call":"Multi(org.apache.hadoop.hbase.protobuf.generated.ClientProtos$MultiRequest)","starttimems":1501894631338,"responsesize":321,"method":"Multi","processingtimems":70063,"client":"hadoop3:43198","queuetimems":0,"class":"HRegionServer"} 2017-08-05 00:58:24,823 INFO [regionserver/aps-hadoop4/hadoop4:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501894067065 with entries=307, filesize=121.67 MB; new WAL /apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501894704773 2017-08-05 00:58:24,824 INFO [regionserver/aps-hadoop4/hadoop4:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501890289721 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop4%2C16020%2C1501679842018.default.1501890289721 2017-08-05 00:58:24,837 INFO [regionserver/aps-hadoop4/hadoop4:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501890719070 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop4%2C16020%2C1501679842018.default.1501890719070 2017-08-05 00:58:26,692 INFO [aps-hadoop4,16020,1501679842018_ChoreService_1] regionserver.HRegionServer: aps-hadoop4,16020,1501679842018-MemstoreFlusherChore requesting flush for region CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. after a delay of 10566 2017-08-05 00:58:28,955 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183., current region memstore size 128.84 MB, and 1/1 column families' memstores are being flushed. 2017-08-05 00:58:29,748 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=65589264, memsize=128.8 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/8083c44b3c7bce24d150b22ed70ea183/.tmp/c1a7972864b74c7ea69f814674015eff 2017-08-05 00:58:29,762 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/8083c44b3c7bce24d150b22ed70ea183/info/c1a7972864b74c7ea69f814674015eff, entries=49677, sequenceid=65589264, filesize=29.1 M 2017-08-05 00:58:29,763 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.84 MB/135095704, currentsize=2.31 MB/2420640 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183. in 808ms, sequenceid=65589264, compaction requested=true 2017-08-05 00:58:29,766 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183. 2017-08-05 00:58:29,766 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/8083c44b3c7bce24d150b22ed70ea183/.tmp, totalSize=87.4 M 2017-08-05 00:58:29,768 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=101500, currentSize=6773201984, freeSize=77276096, maxSize=6850478080, heapSize=6773201984, minSize=6507954176, minFactor=0.95, multiSize=3253977088, multiFactor=0.5, singleSize=1626988544, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-08-05 00:58:30,170 INFO [sync.0] wal.FSHLog: Slow sync cost: 353 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK]] 2017-08-05 00:58:32,154 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183. into eedcfb8ca5e444ba9d312a92e8fde241(size=29.2 M), total size for store is 152.4 M. This selection was in queue for 0sec, and took 2sec to execute. 2017-08-05 00:58:32,155 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183., storeName=info, fileCount=3, fileSize=87.4 M, priority=6, time=25979472083520861; duration=2sec 2017-08-05 00:58:35,557 INFO [sync.4] wal.FSHLog: Slow sync cost: 358 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK]] 2017-08-05 00:58:35,574 INFO [sync.0] wal.FSHLog: Slow sync cost: 336 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK]] 2017-08-05 00:58:36,692 INFO [aps-hadoop4,16020,1501679842018_ChoreService_1] regionserver.HRegionServer: aps-hadoop4,16020,1501679842018-MemstoreFlusherChore requesting flush for region CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. after a delay of 22033 2017-08-05 00:58:37,258 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494., current region memstore size 1.01 MB, and 1/12 column families' memstores are being flushed. 2017-08-05 00:58:37,259 INFO [MemStoreFlusher.1] regionserver.HRegion: Flushing Column Family: Mb which was occupying 701.34 KB of memstore. 2017-08-05 00:58:37,303 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=158236970, memsize=700.9 K, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/CampaignGoalMDC/6cf8953db275a43b98ab944d2d614494/.tmp/9b65012d980e43e1a9e085b7490f701b 2017-08-05 00:58:37,322 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/CampaignGoalMDC/6cf8953db275a43b98ab944d2d614494/Mb/9b65012d980e43e1a9e085b7490f701b, entries=2266, sequenceid=158236970, filesize=29.7 K 2017-08-05 00:58:37,324 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~700.95 KB/717768, currentsize=334.37 KB/342392 for region CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. in 66ms, sequenceid=158236970, compaction requested=false 2017-08-05 01:02:25,133 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=6.15 GB, freeSize=239.84 MB, max=6.38 GB, blockCount=98867, accesses=449174782, hits=232365870, hitRatio=51.73%, , cachingAccesses=445404259, cachingHits=232364360, cachingHitsRatio=52.17%, evictions=56983, evicted=211251066, evictedPerRun=3707.264892578125 2017-08-05 01:02:39,552 WARN [ResponseProcessor for block BP-1810172115-hadoop2-1478343078462:blk_1087207625_13526600] hdfs.DFSClient: Slow ReadProcessor read fields took 49196ms (threshold=30000ms); ack: seqno: 598 reply: SUCCESS reply: SUCCESS reply: SUCCESS downstreamAckTimeNanos: 34795262259 flag: 0 flag: 0 flag: 0, targets: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK]] 2017-08-05 01:02:39,552 WARN [sync.2] hdfs.DFSClient: Slow waitForAckedSeqno took 34795ms (threshold=30000ms) 2017-08-05 01:02:39,552 INFO [sync.2] wal.FSHLog: Slow sync cost: 34795 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK]] 2017-08-05 01:02:39,552 WARN [B.defaultRpcServer.handler=18,queue=3,port=16020] ipc.RpcServer: (responseTooSlow): {"call":"Multi(org.apache.hadoop.hbase.protobuf.generated.ClientProtos$MultiRequest)","starttimems":1501894924750,"responsesize":74,"method":"Multi","processingtimems":34802,"client":"hadoop3:43198","queuetimems":0,"class":"HRegionServer"} 2017-08-05 01:03:14,222 WARN [ResponseProcessor for block BP-1810172115-hadoop2-1478343078462:blk_1087207625_13526600] hdfs.DFSClient: Slow ReadProcessor read fields took 34670ms (threshold=30000ms); ack: seqno: 599 reply: SUCCESS reply: SUCCESS reply: SUCCESS downstreamAckTimeNanos: 30995703267 flag: 0 flag: 0 flag: 0, targets: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK]] 2017-08-05 01:03:14,229 WARN [sync.3] hdfs.DFSClient: Slow waitForAckedSeqno took 31002ms (threshold=30000ms) 2017-08-05 01:03:14,229 INFO [sync.3] wal.FSHLog: Slow sync cost: 31002 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK]] 2017-08-05 01:03:14,229 WARN [B.defaultRpcServer.handler=16,queue=1,port=16020] ipc.RpcServer: (responseTooSlow): {"call":"Multi(org.apache.hadoop.hbase.protobuf.generated.ClientProtos$MultiRequest)","starttimems":1501894963222,"responsesize":183,"method":"Multi","processingtimems":31007,"client":"hadoop3:43198","queuetimems":0,"class":"HRegionServer"} 2017-08-05 01:04:09,285 INFO [sync.4] wal.FSHLog: Slow sync cost: 428 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK]] 2017-08-05 01:04:23,779 INFO [sync.0] wal.FSHLog: Slow sync cost: 1433 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK]] 2017-08-05 01:04:32,159 INFO [sync.1] wal.FSHLog: Slow sync cost: 8375 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK]] 2017-08-05 01:07:25,133 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=6.11 GB, freeSize=279.64 MB, max=6.38 GB, blockCount=98235, accesses=450439946, hits=232998532, hitRatio=51.73%, , cachingAccesses=446669423, cachingHits=232997022, cachingHitsRatio=52.16%, evictions=57136, evicted=211884202, evictedPerRun=3708.418701171875 2017-08-05 01:07:36,512 INFO [sync.1] wal.FSHLog: Slow sync cost: 18771 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK]] 2017-08-05 01:07:36,512 WARN [B.defaultRpcServer.handler=31,queue=1,port=16020] ipc.RpcServer: (responseTooSlow): {"call":"Multi(org.apache.hadoop.hbase.protobuf.generated.ClientProtos$MultiRequest)","starttimems":1501895237735,"responsesize":225,"method":"Multi","processingtimems":18777,"client":"hadoop3:43198","queuetimems":1,"class":"HRegionServer"} 2017-08-05 01:07:36,515 INFO [sync.2] wal.FSHLog: Slow sync cost: 18736 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK]] 2017-08-05 01:07:36,515 WARN [B.defaultRpcServer.handler=7,queue=2,port=16020] ipc.RpcServer: (responseTooSlow): {"call":"Multi(org.apache.hadoop.hbase.protobuf.generated.ClientProtos$MultiRequest)","starttimems":1501895237777,"responsesize":62,"method":"Multi","processingtimems":18738,"client":"hadoop3:43198","queuetimems":0,"class":"HRegionServer"} 2017-08-05 01:07:42,545 INFO [sync.0] wal.FSHLog: Slow sync cost: 908 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK]] 2017-08-05 01:07:42,547 INFO [sync.1] wal.FSHLog: Slow sync cost: 852 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK]] 2017-08-05 01:07:45,155 INFO [regionserver/aps-hadoop4/hadoop4:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501894704773 with entries=456, filesize=121.61 MB; new WAL /apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501895265068 2017-08-05 01:07:45,156 INFO [regionserver/aps-hadoop4/hadoop4:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501891019129 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop4%2C16020%2C1501679842018.default.1501891019129 2017-08-05 01:07:45,173 INFO [regionserver/aps-hadoop4/hadoop4:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501891335600 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop4%2C16020%2C1501679842018.default.1501891335600 2017-08-05 01:07:47,683 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183., current region memstore size 128.12 MB, and 1/1 column families' memstores are being flushed. 2017-08-05 01:07:48,576 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=65589746, memsize=128.1 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/8083c44b3c7bce24d150b22ed70ea183/.tmp/89b1e47dfac34297a7c72b7662b79665 2017-08-05 01:07:48,616 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/8083c44b3c7bce24d150b22ed70ea183/info/89b1e47dfac34297a7c72b7662b79665, entries=49677, sequenceid=65589746, filesize=29.1 M 2017-08-05 01:07:48,618 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~128.12 MB/134339824, currentsize=2.57 MB/2696336 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183. in 935ms, sequenceid=65589746, compaction requested=true 2017-08-05 01:12:25,133 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=6.09 GB, freeSize=299.02 MB, max=6.38 GB, blockCount=97928, accesses=451515287, hits=233536192, hitRatio=51.72%, , cachingAccesses=447744764, cachingHits=233534682, cachingHitsRatio=52.16%, evictions=57266, evicted=212422189, evictedPerRun=3709.39453125 2017-08-05 01:12:47,011 INFO [sync.1] wal.FSHLog: Slow sync cost: 534 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[hadoop7:50010,DS-485b95df-5888-4104-8455-5448e0f7846b,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK]] 2017-08-05 01:15:47,659 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183., current region memstore size 128.17 MB, and 1/1 column families' memstores are being flushed. 2017-08-05 01:15:47,734 INFO [regionserver/aps-hadoop4/hadoop4:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501895265068 with entries=305, filesize=122.01 MB; new WAL /apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501895747659 2017-08-05 01:15:48,489 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=65590223, memsize=128.2 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/8083c44b3c7bce24d150b22ed70ea183/.tmp/fb266630a06d4d6baf2e2cec31c5ea19 2017-08-05 01:15:48,519 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/8083c44b3c7bce24d150b22ed70ea183/info/fb266630a06d4d6baf2e2cec31c5ea19, entries=49677, sequenceid=65590223, filesize=29.1 M 2017-08-05 01:15:48,552 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~128.17 MB/134394664, currentsize=100.79 KB/103208 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183. in 893ms, sequenceid=65590223, compaction requested=true 2017-08-05 01:15:48,559 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183. 2017-08-05 01:15:48,559 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/8083c44b3c7bce24d150b22ed70ea183/.tmp, totalSize=87.4 M 2017-08-05 01:15:48,562 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=98715, currentSize=6589043000, freeSize=261435080, maxSize=6850478080, heapSize=6589043000, minSize=6507954176, minFactor=0.95, multiSize=3253977088, multiFactor=0.5, singleSize=1626988544, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-08-05 01:15:48,665 INFO [sync.4] wal.FSHLog: Slow sync cost: 107 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-08-05 01:15:48,687 INFO [sync.0] wal.FSHLog: Slow sync cost: 115 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-08-05 01:15:50,428 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183. into e59e270fb0964529a95dee1baa250f7b(size=29.2 M), total size for store is 152.4 M. This selection was in queue for 0sec, and took 1sec to execute. 2017-08-05 01:15:50,428 INFO [regionserver/aps-hadoop4/hadoop4:16020-shortCompactions-1501679883189] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183., storeName=info, fileCount=3, fileSize=87.4 M, priority=6, time=25980510876876032; duration=1sec 2017-08-05 01:17:25,133 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=6.10 GB, freeSize=285.46 MB, max=6.38 GB, blockCount=98139, accesses=452069952, hits=233812652, hitRatio=51.72%, , cachingAccesses=448297666, cachingHits=233811142, cachingHitsRatio=52.16%, evictions=57333, evicted=212698421, evictedPerRun=3709.877685546875 2017-08-05 01:18:56,456 INFO [sync.3] wal.FSHLog: Slow sync cost: 111 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-08-05 01:18:56,921 INFO [sync.0] wal.FSHLog: Slow sync cost: 112 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-08-05 01:18:57,413 INFO [sync.2] wal.FSHLog: Slow sync cost: 110 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-08-05 01:18:58,254 INFO [sync.1] wal.FSHLog: Slow sync cost: 127 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-08-05 01:19:00,596 INFO [sync.0] wal.FSHLog: Slow sync cost: 102 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-08-05 01:19:04,819 INFO [sync.1] wal.FSHLog: Slow sync cost: 100 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-08-05 01:19:05,375 INFO [sync.3] wal.FSHLog: Slow sync cost: 114 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-08-05 01:19:10,220 INFO [sync.2] wal.FSHLog: Slow sync cost: 1521 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-08-05 01:19:27,630 INFO [sync.4] wal.FSHLog: Slow sync cost: 2417 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-08-05 01:19:28,304 INFO [sync.0] wal.FSHLog: Slow sync cost: 669 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-08-05 01:21:50,988 INFO [sync.4] wal.FSHLog: Slow sync cost: 15237 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-08-05 01:21:50,988 WARN [B.defaultRpcServer.handler=19,queue=4,port=16020] ipc.RpcServer: (responseTooSlow): {"call":"Multi(org.apache.hadoop.hbase.protobuf.generated.ClientProtos$MultiRequest)","starttimems":1501896095749,"responsesize":8,"method":"Multi","processingtimems":15239,"client":"hadoop3:44374","queuetimems":0,"class":"HRegionServer"} 2017-08-05 01:21:51,047 INFO [sync.0] wal.FSHLog: Slow sync cost: 14653 ms, current pipeline: [DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-08-05 01:21:51,047 WARN [B.defaultRpcServer.handler=29,queue=4,port=16020] ipc.RpcServer: (responseTooSlow): {"call":"Multi(org.apache.hadoop.hbase.protobuf.generated.ClientProtos$MultiRequest)","starttimems":1501896096390,"responsesize":183,"method":"Multi","processingtimems":14657,"client":"hadoop3:44374","queuetimems":0,"class":"HRegionServer"} 2017-08-05 01:22:25,133 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=6.27 GB, freeSize=108.27 MB, max=6.38 GB, blockCount=100943, accesses=452539338, hits=234047494, hitRatio=51.72%, , cachingAccesses=448767052, cachingHits=234045984, cachingHitsRatio=52.15%, evictions=57389, evicted=212930163, evictedPerRun=3710.295654296875 2017-08-05 01:22:46,692 INFO [aps-hadoop4,16020,1501679842018_ChoreService_1] regionserver.HRegionServer: aps-hadoop4,16020,1501679842018-MemstoreFlusherChore requesting flush for region CampaignFactsByLinkClicksMDC,cccccccc,1478428072397.e2fdff075d0d93921fbe26ed912db4cf. after a delay of 20988 2017-08-05 01:22:56,692 INFO [aps-hadoop4,16020,1501679842018_ChoreService_1] regionserver.HRegionServer: aps-hadoop4,16020,1501679842018-MemstoreFlusherChore requesting flush for region CampaignFactsByLinkClicksMDC,cccccccc,1478428072397.e2fdff075d0d93921fbe26ed912db4cf. after a delay of 12938 2017-08-05 01:23:03,709 WARN [ResponseProcessor for block BP-1810172115-hadoop2-1478343078462:blk_1087207729_13526709] hdfs.DFSClient: DFSOutputStream ResponseProcessor exception for block BP-1810172115-hadoop2-1478343078462:blk_1087207729_13526709 java.io.EOFException: Premature EOF: no length prefix available at org.apache.hadoop.hdfs.protocolPB.PBHelper.vintPrefixed(PBHelper.java:2293) at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer$ResponseProcessor.run(DFSOutputStream.java:748) 2017-08-05 01:23:03,709 WARN [DataStreamer for file /apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501895747659 block BP-1810172115-hadoop2-1478343078462:blk_1087207729_13526709] hdfs.DFSClient: Error Recovery for block BP-1810172115-hadoop2-1478343078462:blk_1087207729_13526709 in pipeline DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]: bad datanode DatanodeInfoWithStorage[hadoop4:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK] 2017-08-05 01:23:06,692 INFO [aps-hadoop4,16020,1501679842018_ChoreService_1] regionserver.HRegionServer: aps-hadoop4,16020,1501679842018-MemstoreFlusherChore requesting flush for region CampaignFactsByLinkClicksMDC,cccccccc,1478428072397.e2fdff075d0d93921fbe26ed912db4cf. after a delay of 11635 2017-08-05 01:23:07,335 INFO [DataStreamer for file /apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501895747659 block BP-1810172115-hadoop2-1478343078462:blk_1087207729_13526709] hdfs.DFSClient: Exception in createBlockOutputStream java.io.IOException: Got error, status message , ack with firstBadLink as hadoop5:50010 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:140) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.createBlockOutputStream(DFSOutputStream.java:1393) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1217) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-08-05 01:23:07,335 WARN [DataStreamer for file /apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501895747659 block BP-1810172115-hadoop2-1478343078462:blk_1087207729_13526709] hdfs.DFSClient: Error Recovery for block BP-1810172115-hadoop2-1478343078462:blk_1087207729_13526709 in pipeline DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK], DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK]: bad datanode DatanodeInfoWithStorage[hadoop5:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK] 2017-08-05 01:23:07,680 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for CampaignFactsByLinkClicksMDC,cccccccc,1478428072397.e2fdff075d0d93921fbe26ed912db4cf., current region memstore size 22.61 KB, and 1/1 column families' memstores are being flushed. 2017-08-05 01:23:10,434 INFO [DataStreamer for file /apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501895747659 block BP-1810172115-hadoop2-1478343078462:blk_1087207729_13526709] hdfs.DFSClient: Exception in createBlockOutputStream java.io.IOException: Got error, status message , ack with firstBadLink as hadoop7:50010 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:140) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.createBlockOutputStream(DFSOutputStream.java:1393) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1217) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-08-05 01:23:10,435 WARN [DataStreamer for file /apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501895747659 block BP-1810172115-hadoop2-1478343078462:blk_1087207729_13526709] hdfs.DFSClient: Error Recovery for block BP-1810172115-hadoop2-1478343078462:blk_1087207729_13526709 in pipeline DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK], DatanodeInfoWithStorage[hadoop7:50010,DS-9a10c707-ebd5-4fae-8f82-b381f706fa57,DISK]: bad datanode DatanodeInfoWithStorage[hadoop7:50010,DS-9a10c707-ebd5-4fae-8f82-b381f706fa57,DISK] 2017-08-05 01:23:10,436 WARN [DataStreamer for file /apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501895747659 block BP-1810172115-hadoop2-1478343078462:blk_1087207729_13526709] hdfs.DFSClient: DataStreamer Exception java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]], original=[DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:947) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1021) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1189) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-08-05 01:23:10,437 WARN [sync.4] hdfs.DFSClient: Error while syncing java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]], original=[DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:947) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1021) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1189) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-08-05 01:23:10,437 WARN [sync.0] hdfs.DFSClient: Error while syncing java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]], original=[DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:947) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1021) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1189) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-08-05 01:23:10,437 ERROR [sync.4] wal.FSHLog: Error syncing, request close of WAL java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]], original=[DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:947) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1021) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1189) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-08-05 01:23:10,437 ERROR [sync.0] wal.FSHLog: Error syncing, request close of WAL java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]], original=[DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:947) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1021) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1189) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-08-05 01:23:10,437 INFO [sync.4] wal.FSHLog: Slow sync cost: 68063 ms, current pipeline: [DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-08-05 01:23:10,437 INFO [sync.0] wal.FSHLog: Slow sync cost: 2756 ms, current pipeline: [DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-08-05 01:23:10,446 WARN [B.defaultRpcServer.handler=45,queue=0,port=16020] ipc.RpcServer: (responseTooSlow): {"call":"Multi(org.apache.hadoop.hbase.protobuf.generated.ClientProtos$MultiRequest)","starttimems":1501896122367,"responsesize":55048,"method":"Multi","processingtimems":68078,"client":"hadoop3:44374","queuetimems":0,"class":"HRegionServer"} 2017-08-05 01:23:10,465 ERROR [sync.1] wal.FSHLog: Error syncing, request close of WAL java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]], original=[DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:947) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1021) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1189) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-08-05 01:23:10,466 ERROR [regionserver/aps-hadoop4/hadoop4:16020.logRoller] wal.FSHLog: Failed close of WAL writer hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501895747659, unflushedEntries=4 org.apache.hadoop.hbase.regionserver.wal.FailedSyncBeforeLogCloseException: java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]], original=[DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SafePointZigZagLatch.waitSafePoint(FSHLog.java:1893) at org.apache.hadoop.hbase.regionserver.wal.FSHLog.replaceWriter(FSHLog.java:964) at org.apache.hadoop.hbase.regionserver.wal.FSHLog.rollWriter(FSHLog.java:733) at org.apache.hadoop.hbase.regionserver.LogRoller.run(LogRoller.java:148) at java.lang.Thread.run(Thread.java:745) Caused by: java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]], original=[DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:947) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1021) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1189) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-08-05 01:23:10,468 FATAL [regionserver/aps-hadoop4/hadoop4:16020.logRoller] regionserver.HRegionServer: ABORTING region server aps-hadoop4,16020,1501679842018: Failed log close in log roller org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException: hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop4,16020,1501679842018/aps-hadoop4%2C16020%2C1501679842018.default.1501895747659, unflushedEntries=4 at org.apache.hadoop.hbase.regionserver.wal.FSHLog.replaceWriter(FSHLog.java:1014) at org.apache.hadoop.hbase.regionserver.wal.FSHLog.rollWriter(FSHLog.java:733) at org.apache.hadoop.hbase.regionserver.LogRoller.run(LogRoller.java:148) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.hadoop.hbase.regionserver.wal.FailedSyncBeforeLogCloseException: java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]], original=[DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SafePointZigZagLatch.waitSafePoint(FSHLog.java:1893) at org.apache.hadoop.hbase.regionserver.wal.FSHLog.replaceWriter(FSHLog.java:964) ... 3 more Caused by: java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]], original=[DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:947) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1021) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1189) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-08-05 01:23:10,468 FATAL [regionserver/aps-hadoop4/hadoop4:16020.logRoller] regionserver.HRegionServer: RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint] 2017-08-05 01:23:10,500 ERROR [MemStoreFlusher.0] regionserver.MemStoreFlusher: Cache flush failed for region CampaignFactsByLinkClicksMDC,cccccccc,1478428072397.e2fdff075d0d93921fbe26ed912db4cf. java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]], original=[DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:947) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1021) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1189) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-08-05 01:23:10,548 INFO [regionserver/aps-hadoop4/hadoop4:16020.logRoller] regionserver.HRegionServer: Dump of metrics as JSON on abort: { "beans" : [ { "name" : "java.lang:type=Memory", "modelerType" : "sun.management.MemoryImpl", "ObjectPendingFinalizationCount" : 0, "NonHeapMemoryUsage" : { "committed" : 102744064, "init" : 2555904, "max" : -1, "used" : 101159496 }, "Verbose" : true, "HeapMemoryUsage" : { "committed" : 15557640192, "init" : 10737418240, "max" : 17126195200, "used" : 10408852320 }, "ObjectName" : "java.lang:type=Memory" } ], "beans" : [ { "name" : "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType" : "RegionServer,sub=IPC", "tag.Context" : "regionserver", "tag.Hostname" : "aps-hadoop4", "queueSize" : 0, "numCallsInGeneralQueue" : 0, "numCallsInReplicationQueue" : 0, "numCallsInPriorityQueue" : 0, "numOpenConnections" : 3, "numActiveHandler" : 0, "receivedBytes" : 75922324980, "exceptions.RegionMovedException" : 39, "authenticationSuccesses" : 0, "authorizationFailures" : 0, "TotalCallTime_num_ops" : 188199552, "TotalCallTime_min" : 0, "TotalCallTime_max" : 429843, "TotalCallTime_mean" : 3.6425165613571706, "TotalCallTime_median" : 1.0, "TotalCallTime_75th_percentile" : 1.0, "TotalCallTime_95th_percentile" : 12.0, "TotalCallTime_99th_percentile" : 15.529999999999973, "exceptions.RegionTooBusyException" : 0, "exceptions.FailedSanityCheckException" : 0, "exceptions.UnknownScannerException" : 0, "exceptions.OutOfOrderScannerNextException" : 0, "exceptions" : 76, "ProcessCallTime_num_ops" : 188199552, "ProcessCallTime_min" : 0, "ProcessCallTime_max" : 429843, "ProcessCallTime_mean" : 3.620914979648836, "ProcessCallTime_median" : 1.0, "ProcessCallTime_75th_percentile" : 1.0, "ProcessCallTime_95th_percentile" : 11.0, "ProcessCallTime_99th_percentile" : 16.0, "exceptions.NotServingRegionException" : 31, "authorizationSuccesses" : 8432, "sentBytes" : 5525810121868, "QueueCallTime_num_ops" : 188199552, "QueueCallTime_min" : 0, "QueueCallTime_max" : 781, "QueueCallTime_mean" : 0.021601581708334778, "QueueCallTime_median" : 0.0, "QueueCallTime_75th_percentile" : 0.0, "QueueCallTime_95th_percentile" : 0.0, "QueueCallTime_99th_percentile" : 1.0, "authenticationFailures" : 0 } ], "beans" : [ { "name" : "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType" : "RegionServer,sub=Replication", "tag.Context" : "regionserver", "tag.Hostname" : "aps-hadoop4", "sink.appliedHFiles" : 0, "sink.appliedOps" : 0, "sink.ageOfLastAppliedOp" : 0, "sink.appliedBatches" : 0 } ], "beans" : [ { "name" : "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType" : "RegionServer,sub=Server", "tag.zookeeperQuorum" : "aps-hadoop6:2181,aps-hadoop7:2181,aps-hadoop4:2181", "tag.serverName" : "aps-hadoop4,16020,1501679842018", "tag.clusterId" : "fea6e6f3-b462-4e4c-946c-f35c02a0d2f4", "tag.Context" : "regionserver", "tag.Hostname" : "aps-hadoop4", "regionCount" : 124, "storeCount" : 225, "hlogFileCount" : 7, "hlogFileSize" : 766639184, "storeFileCount" : 114, "memStoreSize" : 117345320, "storeFileSize" : 44841302046, "regionServerStartTime" : 1501679842018, "totalRequestCount" : 220526071, "readRequestCount" : 14124656780, "writeRequestCount" : 27924777, "checkMutateFailedCount" : 0, "checkMutatePassedCount" : 0, "storeFileIndexSize" : 2104096, "staticIndexSize" : 205423715, "staticBloomSize" : 325870990, "mutationsWithoutWALCount" : 0, "mutationsWithoutWALSize" : 0, "percentFilesLocal" : 78, "percentFilesLocalSecondaryRegions" : 0, "splitQueueLength" : 0, "compactionQueueLength" : 0, "flushQueueLength" : 0, "blockCacheFreeSize" : 71883936, "blockCacheCount" : 101571, "blockCacheSize" : 6778594144, "blockCacheHitCount" : 234077079, "blockCacheHitCountPrimary" : 234077079, "blockCacheMissCount" : 218521432, "blockCacheMissCountPrimary" : 218521432, "blockCacheEvictionCount" : 212959124, "blockCacheEvictionCountPrimary" : 212959124, "blockCacheCountHitPercent" : 51.0, "blockCacheExpressHitPercent" : 52, "updatesBlockedTime" : 0, "flushedCellsCount" : 291743446, "compactedCellsCount" : 910340286, "majorCompactedCellsCount" : 418344158, "flushedCellsSize" : 104296880088, "compactedCellsSize" : 56467199971, "majorCompactedCellsSize" : 19809860251, "blockedRequestCount" : 284, "Mutate_num_ops" : 407755, "Mutate_min" : 0, "Mutate_max" : 429843, "Mutate_mean" : 60.544635872031, "Mutate_median" : 14.0, "Mutate_75th_percentile" : 25.0, "Mutate_95th_percentile" : 54.0, "Mutate_99th_percentile" : 1484.669999999999, "slowAppendCount" : 0, "slowDeleteCount" : 0, "Increment_num_ops" : 0, "Increment_min" : 0, "Increment_max" : 0, "Increment_mean" : 0.0, "Increment_median" : 0.0, "Increment_75th_percentile" : 0.0, "Increment_95th_percentile" : 0.0, "Increment_99th_percentile" : 0.0, "Replay_num_ops" : 0, "Replay_min" : 0, "Replay_max" : 0, "Replay_mean" : 0.0, "Replay_median" : 0.0, "Replay_75th_percentile" : 0.0, "Replay_95th_percentile" : 0.0, "Replay_99th_percentile" : 0.0, "FlushTime_num_ops" : 987, "FlushTime_min" : 65, "FlushTime_max" : 114715, "FlushTime_mean" : 5377.096251266464, "FlushTime_median" : 1328.0, "FlushTime_75th_percentile" : 3346.0, "FlushTime_95th_percentile" : 19566.79999999997, "FlushTime_99th_percentile" : 58383.279999999875, "Delete_num_ops" : 0, "Delete_min" : 0, "Delete_max" : 0, "Delete_mean" : 0.0, "Delete_median" : 0.0, "Delete_75th_percentile" : 0.0, "Delete_95th_percentile" : 0.0, "Delete_99th_percentile" : 0.0, "splitRequestCount" : 0, "splitSuccessCount" : 0, "slowGetCount" : 0, "Get_num_ops" : 0, "Get_min" : 0, "Get_max" : 0, "Get_mean" : 0.0, "Get_median" : 0.0, "Get_75th_percentile" : 0.0, "Get_95th_percentile" : 0.0, "Get_99th_percentile" : 0.0, "ScanNext_num_ops" : 187739562, "ScanNext_min" : 0, "ScanNext_max" : 2653573, "ScanNext_mean" : 28998.587885961937, "ScanNext_median" : 18688.0, "ScanNext_75th_percentile" : 18744.0, "ScanNext_95th_percentile" : 18780.0, "ScanNext_99th_percentile" : 25400.0, "slowPutCount" : 1499, "slowIncrementCount" : 0, "Append_num_ops" : 0, "Append_min" : 0, "Append_max" : 0, "Append_mean" : 0.0, "Append_median" : 0.0, "Append_75th_percentile" : 0.0, "Append_95th_percentile" : 0.0, "Append_99th_percentile" : 0.0, "SplitTime_num_ops" : 0, "SplitTime_min" : 0, "SplitTime_max" : 0, "SplitTime_mean" : 0.0, "SplitTime_median" : 0.0, "SplitTime_75th_percentile" : 0.0, "SplitTime_95th_percentile" : 0.0, "SplitTime_99th_percentile" : 0.0 } ] } 2017-08-05 01:23:10,609 INFO [regionserver/aps-hadoop4/hadoop4:16020.logRoller] regionserver.HRegionServer: STOPPED: Failed log close in log roller 2017-08-05 01:23:10,609 INFO [regionserver/aps-hadoop4/hadoop4:16020.logRoller] regionserver.LogRoller: LogRoller exiting. 2017-08-05 01:23:10,609 INFO [regionserver/aps-hadoop4/hadoop4:16020] regionserver.SplitLogWorker: Sending interrupt to stop the worker thread 2017-08-05 01:23:10,611 INFO [regionserver/aps-hadoop4/hadoop4:16020] regionserver.HRegionServer: Stopping infoServer 2017-08-05 01:23:10,611 INFO [SplitLogWorker-aps-hadoop4:16020] regionserver.SplitLogWorker: SplitLogWorker interrupted. Exiting. 2017-08-05 01:23:10,611 INFO [SplitLogWorker-aps-hadoop4:16020] regionserver.SplitLogWorker: SplitLogWorker aps-hadoop4,16020,1501679842018 exiting 2017-08-05 01:23:10,639 INFO [regionserver/aps-hadoop4/hadoop4:16020] mortbay.log: Stopped SelectChannelConnector@0.0.0.0:16030 2017-08-05 01:23:10,742 INFO [regionserver/aps-hadoop4/hadoop4:16020] regionserver.HeapMemoryManager: Stoping HeapMemoryTuner chore. 2017-08-05 01:23:10,742 INFO [MemStoreFlusher.0] regionserver.MemStoreFlusher: MemStoreFlusher.0 exiting 2017-08-05 01:23:10,742 INFO [MemStoreFlusher.1] regionserver.MemStoreFlusher: MemStoreFlusher.1 exiting 2017-08-05 01:23:10,742 INFO [regionserver/aps-hadoop4/hadoop4:16020] flush.RegionServerFlushTableProcedureManager: Stopping region server flush procedure manager abruptly. 2017-08-05 01:23:10,742 INFO [regionserver/aps-hadoop4/hadoop4:16020] snapshot.RegionServerSnapshotManager: Stopping RegionServerSnapshotManager abruptly. 2017-08-05 01:23:10,743 INFO [regionserver/aps-hadoop4/hadoop4:16020] regionserver.HRegionServer: aborting server aps-hadoop4,16020,1501679842018 2017-08-05 01:23:10,743 INFO [regionserver/aps-hadoop4/hadoop4:16020] client.ConnectionManager$HConnectionImplementation: Closing zookeeper sessionid=0x15c3e07cc79c4b5 2017-08-05 01:23:10,745 INFO [regionserver/aps-hadoop4/hadoop4:16020] zookeeper.ZooKeeper: Session: 0x15c3e07cc79c4b5 closed 2017-08-05 01:23:10,745 INFO [regionserver/aps-hadoop4/hadoop4:16020-EventThread] zookeeper.ClientCnxn: EventThread shut down 2017-08-05 01:23:10,759 INFO [regionserver/aps-hadoop4/hadoop4:16020] regionserver.HRegionServer: Waiting on 124 regions to close 2017-08-05 01:23:10,765 INFO [StoreCloserThread-CampaignSummaryTrends_New,33333333,1493328805561.74243e9568c593b1103f47afc976a6bf.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,765 INFO [StoreCloserThread-CampaignMailBounce,b8955bcf_a1ea_4434_8919_c75472d06b44_000_9dc5f876-1564-402e-9207-9e61645d6268_f5db3d47-7def-11e4-924f-06ab9bb94413_45316bb3-e071-4ac5-b5eb-f031396d6b0f_20150611063757,1500534659793.556b6ce940af63522ea1250af1b8c9c1.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,765 INFO [StoreCloserThread-CampaignMailBounce,zAcwjKf5X4BPKf20150825071313,1486099305438.9e75db0e3d8cfb75c50977dee786db56.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,765 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed CampaignSummaryTrends_New,33333333,1493328805561.74243e9568c593b1103f47afc976a6bf. 2017-08-05 01:23:10,765 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed CampaignMailBounce,b8955bcf_a1ea_4434_8919_c75472d06b44_000_9dc5f876-1564-402e-9207-9e61645d6268_f5db3d47-7def-11e4-924f-06ab9bb94413_45316bb3-e071-4ac5-b5eb-f031396d6b0f_20150611063757,1500534659793.556b6ce940af63522ea1250af1b8c9c1. 2017-08-05 01:23:10,766 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed CampaignMailBounce,zAcwjKf5X4BPKf20150825071313,1486099305438.9e75db0e3d8cfb75c50977dee786db56. 2017-08-05 01:23:10,766 INFO [StoreCloserThread-CampaignSummaryBusinessSegment,,1478428086929.aff40cab1335fb43c1a30c9d5dcddcd5.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,766 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed CampaignSummaryBusinessSegment,,1478428086929.aff40cab1335fb43c1a30c9d5dcddcd5. 2017-08-05 01:23:10,766 INFO [StoreCloserThread-Recipients,zAcwX6DQ63,1480293735365.f3de9991728ea4e3d3d2df811016233d.-1] regionserver.HStore: Closed Prop 2017-08-05 01:23:10,766 INFO [StoreCloserThread-WorkflowJobStatus,33333333,1478428303673.f59ea6027eed4692224ee7fd19bd0721.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,766 INFO [StoreCloserThread-Recipients,zAcwX6DQ63,1480293735365.f3de9991728ea4e3d3d2df811016233d.-1] regionserver.HStore: Closed Seg 2017-08-05 01:23:10,766 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed WorkflowJobStatus,33333333,1478428303673.f59ea6027eed4692224ee7fd19bd0721. 2017-08-05 01:23:10,766 INFO [StoreCloserThread-ModelsConfigSync,66666666,1478428291186.9d6f30ba4337c5538495e2b26f78e17c.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,766 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed ModelsConfigSync,66666666,1478428291186.9d6f30ba4337c5538495e2b26f78e17c. 2017-08-05 01:23:10,766 INFO [StoreCloserThread-CampaignBenchMark,,1478428131332.38b6ed0dc73ee50bad729bc5430ad766.-1] regionserver.HStore: Closed Em 2017-08-05 01:23:10,766 INFO [StoreCloserThread-CampaignBenchMark,,1478428131332.38b6ed0dc73ee50bad729bc5430ad766.-1] regionserver.HStore: Closed Fb 2017-08-05 01:23:10,766 INFO [StoreCloserThread-CampaignBenchMark,,1478428131332.38b6ed0dc73ee50bad729bc5430ad766.-1] regionserver.HStore: Closed Fp 2017-08-05 01:23:10,766 INFO [StoreCloserThread-CampaignBenchMark,,1478428131332.38b6ed0dc73ee50bad729bc5430ad766.-1] regionserver.HStore: Closed Lnk 2017-08-05 01:23:10,766 INFO [StoreCloserThread-CampaignBenchMark,,1478428131332.38b6ed0dc73ee50bad729bc5430ad766.-1] regionserver.HStore: Closed Mob 2017-08-05 01:23:10,767 INFO [StoreCloserThread-CampaignBenchMark,,1478428131332.38b6ed0dc73ee50bad729bc5430ad766.-1] regionserver.HStore: Closed QR 2017-08-05 01:23:10,767 INFO [StoreCloserThread-CampaignBenchMark,,1478428131332.38b6ed0dc73ee50bad729bc5430ad766.-1] regionserver.HStore: Closed Tw 2017-08-05 01:23:10,767 INFO [StoreCloserThread-CampaignBenchMark,,1478428131332.38b6ed0dc73ee50bad729bc5430ad766.-1] regionserver.HStore: Closed Yt 2017-08-05 01:23:10,767 INFO [StoreCloserThread-CampaignBenchMark,,1478428131332.38b6ed0dc73ee50bad729bc5430ad766.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,767 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed CampaignBenchMark,,1478428131332.38b6ed0dc73ee50bad729bc5430ad766. 2017-08-05 01:23:10,767 INFO [StoreCloserThread-CampaignSummaryDetailGeographyByCity,,1478428044795.0d7dc768580bf53b4e5310a049090e50.-1] regionserver.HStore: Closed Geo 2017-08-05 01:23:10,767 INFO [StoreCloserThread-CampaignSummaryDetailGeographyByCity,,1478428044795.0d7dc768580bf53b4e5310a049090e50.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,767 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed CampaignSummaryDetailGeographyByCity,,1478428044795.0d7dc768580bf53b4e5310a049090e50. 2017-08-05 01:23:10,774 INFO [StoreCloserThread-Campaigns,,1478354871976.b549cc0d22314c1b898911f0c44c8b2d.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,774 INFO [StoreCloserThread-Recipients,zAcwX6DQ63,1480293735365.f3de9991728ea4e3d3d2df811016233d.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,774 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed Campaigns,,1478354871976.b549cc0d22314c1b898911f0c44c8b2d. 2017-08-05 01:23:10,774 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed Recipients,zAcwX6DQ63,1480293735365.f3de9991728ea4e3d3d2df811016233d. 2017-08-05 01:23:10,775 INFO [StoreCloserThread-CampaignSummaryOverview,,1478427980057.7acd0b6f16fd8a07a004ae4cb6d01e79.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,775 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed CampaignSummaryOverview,,1478427980057.7acd0b6f16fd8a07a004ae4cb6d01e79. 2017-08-05 01:23:10,775 INFO [StoreCloserThread-HsearchIndexConfig,66666666,1478428288064.29477673de1c98436387fb8f6a391978.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,775 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed HsearchIndexConfig,66666666,1478428288064.29477673de1c98436387fb8f6a391978. 2017-08-05 01:23:10,775 INFO [StoreCloserThread-CampaignSummaryFacts,,1478427939763.febbdbcc1080e2a413acab24c7e795e1.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,775 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed CampaignSummaryFacts,,1478427939763.febbdbcc1080e2a413acab24c7e795e1. 2017-08-05 01:23:10,779 INFO [StoreCloserThread-EmailCampaignStatus,33333333,1490570366016.d8dd073e236ff3c6cbc5ad0d7782f05b.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,779 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed EmailCampaignStatus,33333333,1490570366016.d8dd073e236ff3c6cbc5ad0d7782f05b. 2017-08-05 01:23:10,781 INFO [StoreCloserThread-CampaignSMSLinkClicksTransaction,33333333,1478427776736.f4dd8f0bb635f0766ae1707a25d84579.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,781 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed CampaignSMSLinkClicksTransaction,33333333,1478427776736.f4dd8f0bb635f0766ae1707a25d84579. 2017-08-05 01:23:10,783 INFO [StoreCloserThread-CampaignSMSTransaction,66666666,1479218455453.26d1ebb5d03132c78e8d8b351fe2027f.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,784 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed CampaignSMSTransaction,66666666,1479218455453.26d1ebb5d03132c78e8d8b351fe2027f. 2017-08-05 01:23:10,786 INFO [StoreCloserThread-CampaignSummaryGeographyByCityMDC,66666666,1478428033615.b0b7ac203e84762d96860e6316094577.-1] regionserver.HStore: Closed Geo 2017-08-05 01:23:10,786 INFO [StoreCloserThread-CampaignMailBounce,zAcwbcLb0ZZ7ZA,1500889161970.858650f09dfc5a81c6656c24a426b526.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,786 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed CampaignMailBounce,zAcwbcLb0ZZ7ZA,1500889161970.858650f09dfc5a81c6656c24a426b526. 2017-08-05 01:23:10,786 INFO [StoreCloserThread-CampaignSMSTransaction_testing,cccccccc,1493727222020.9728676a0cd501e6353330ba4cfd3c1d.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,786 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed CampaignSMSTransaction_testing,cccccccc,1493727222020.9728676a0cd501e6353330ba4cfd3c1d. 2017-08-05 01:23:10,788 INFO [StoreCloserThread-sms_linkclick_test,,1500374505196.ae03af459c21d5f1cbeae898b0db9cab.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,788 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed sms_linkclick_test,,1500374505196.ae03af459c21d5f1cbeae898b0db9cab. 2017-08-05 01:23:10,788 INFO [StoreCloserThread-UserAnalytics360OverviewChannels,cccccccc,1478428300551.a8bf49000e24c906699c97b198920afb.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,788 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed UserAnalytics360OverviewChannels,cccccccc,1478428300551.a8bf49000e24c906699c97b198920afb. 2017-08-05 01:23:10,789 INFO [StoreCloserThread-DashboardCampaignLeadsChannelWise,99999999,1478428306794.f6bff0ce86072056c7e9c0b81a7bd66c.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,789 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed DashboardCampaignLeadsChannelWise,99999999,1478428306794.f6bff0ce86072056c7e9c0b81a7bd66c. 2017-08-05 01:23:10,789 INFO [StoreCloserThread-UserAnalytics360RecipientCampaign,,1478428494703.7b28a0e3c49138df622f03dca3e9ee04.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,789 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed UserAnalytics360RecipientCampaign,,1478428494703.7b28a0e3c49138df622f03dca3e9ee04. 2017-08-05 01:23:10,790 INFO [StoreCloserThread-CampaignSummaryGeographyByCityMDC,66666666,1478428033615.b0b7ac203e84762d96860e6316094577.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,790 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed CampaignSummaryGeographyByCityMDC,66666666,1478428033615.b0b7ac203e84762d96860e6316094577. 2017-08-05 01:23:10,795 INFO [StoreCloserThread-BenchMark,INANA9609123Male56500,1479004862262.cc9074ccd72196eb6d94e5e8747c5184.-1] regionserver.HStore: Closed Em 2017-08-05 01:23:10,796 INFO [StoreCloserThread-CampaignOpenTransaction,,1478427715271.f8eff8c1add9ccea16fb01e0ea703113.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,796 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed CampaignOpenTransaction,,1478427715271.f8eff8c1add9ccea16fb01e0ea703113. 2017-08-05 01:23:10,798 INFO [StoreCloserThread-CampaignMailBounce,cccccccc,1501133697023.f0ee398edbab02f878fcff94f82ad05f.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,798 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed CampaignMailBounce,cccccccc,1501133697023.f0ee398edbab02f878fcff94f82ad05f. 2017-08-05 01:23:10,802 INFO [StoreCloserThread-BenchMark,INANA9609123Male56500,1479004862262.cc9074ccd72196eb6d94e5e8747c5184.-1] regionserver.HStore: Closed Fb 2017-08-05 01:23:10,803 INFO [StoreCloserThread-BenchMark,INANA9609123Male56500,1479004862262.cc9074ccd72196eb6d94e5e8747c5184.-1] regionserver.HStore: Closed Fp 2017-08-05 01:23:10,803 INFO [StoreCloserThread-BenchMark,INANA9609123Male56500,1479004862262.cc9074ccd72196eb6d94e5e8747c5184.-1] regionserver.HStore: Closed Lnk 2017-08-05 01:23:10,803 INFO [StoreCloserThread-CampaignSMSTransaction,99999999,1493279834786.d5adacd3b69c6ace6e8bf702b8ce47c9.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,803 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed CampaignSMSTransaction,99999999,1493279834786.d5adacd3b69c6ace6e8bf702b8ce47c9. 2017-08-05 01:23:10,807 INFO [StoreCloserThread-BenchMark,INANA9609123Male56500,1479004862262.cc9074ccd72196eb6d94e5e8747c5184.-1] regionserver.HStore: Closed Mob 2017-08-05 01:23:10,810 INFO [StoreCloserThread-CampaignMailBounce,zAcwCABNXD5N2,1489993661185.0ad76bdd20489f07f496a4d53b2211ab.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,810 INFO [StoreCloserThread-WebAnalyticsUserFlow,cDGwefkRACityGunnaur, India01496026037,1497007044065.0d4d310e9aa1c1d104d7980f4dcdc748.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,810 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed WebAnalyticsUserFlow,cDGwefkRACityGunnaur, India01496026037,1497007044065.0d4d310e9aa1c1d104d7980f4dcdc748. 2017-08-05 01:23:10,810 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed CampaignMailBounce,zAcwCABNXD5N2,1489993661185.0ad76bdd20489f07f496a4d53b2211ab. 2017-08-05 01:23:10,812 INFO [StoreCloserThread-BenchMark,INANA9609123Male56500,1479004862262.cc9074ccd72196eb6d94e5e8747c5184.-1] regionserver.HStore: Closed QR 2017-08-05 01:23:10,812 INFO [StoreCloserThread-BenchMark,INANA9609123Male56500,1479004862262.cc9074ccd72196eb6d94e5e8747c5184.-1] regionserver.HStore: Closed Tw 2017-08-05 01:23:10,812 INFO [StoreCloserThread-BenchMark,INANA9609123Male56500,1479004862262.cc9074ccd72196eb6d94e5e8747c5184.-1] regionserver.HStore: Closed Yt 2017-08-05 01:23:10,816 INFO [StoreCloserThread-CampaignConversion,33333333,1478427841199.3aa8adbaae87d15525503abad12955b0.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,816 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed CampaignConversion,33333333,1478427841199.3aa8adbaae87d15525503abad12955b0. 2017-08-05 01:23:10,820 INFO [StoreCloserThread-CampaignFactsByLinkClicksMDC,cccccccc,1478428072397.e2fdff075d0d93921fbe26ed912db4cf.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,820 INFO [StoreCloserThread-WebAnalyticsUserFlow,cDGwpFDRACityBengaluru, India11496038161,1496990869191.c83c1576e5197eecc1f71832be6be65e.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,820 ERROR [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Memstore size is 23152 2017-08-05 01:23:10,820 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed WebAnalyticsUserFlow,cDGwpFDRACityBengaluru, India11496038161,1496990869191.c83c1576e5197eecc1f71832be6be65e. 2017-08-05 01:23:10,820 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed CampaignFactsByLinkClicksMDC,cccccccc,1478428072397.e2fdff075d0d93921fbe26ed912db4cf. 2017-08-05 01:23:10,820 INFO [StoreCloserThread-UserAnalytics360UserCampaignReports,99999999,1478428512513.115e7d68d5e912285ca55740eb0af289.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,820 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed UserAnalytics360UserCampaignReports,99999999,1478428512513.115e7d68d5e912285ca55740eb0af289. 2017-08-05 01:23:10,822 INFO [StoreCloserThread-DashboardCampaignPerformance,cccccccc,1478428360041.3df79740665e77a4a3ea47fe6fbc0718.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,822 INFO [StoreCloserThread-BenchMark,INANA9609123Male56500,1479004862262.cc9074ccd72196eb6d94e5e8747c5184.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,822 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed DashboardCampaignPerformance,cccccccc,1478428360041.3df79740665e77a4a3ea47fe6fbc0718. 2017-08-05 01:23:10,822 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed BenchMark,INANA9609123Male56500,1479004862262.cc9074ccd72196eb6d94e5e8747c5184. 2017-08-05 01:23:10,822 INFO [StoreCloserThread-SocialMediaPages,33333333,1478427791732.aec6031e51fae21dc99a4286014066f6.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,822 INFO [StoreCloserThread-SocialMediaPages,66666666,1478427791732.c96590e78a22c42f6f6d2662407c9c9a.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,822 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed SocialMediaPages,33333333,1478427791732.aec6031e51fae21dc99a4286014066f6. 2017-08-05 01:23:10,822 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed SocialMediaPages,66666666,1478427791732.c96590e78a22c42f6f6d2662407c9c9a. 2017-08-05 01:23:10,822 INFO [StoreCloserThread-CampaignSummaryFactsByEMailClient,cccccccc,1478428266204.66b046e29ccf739427e1d6751d426918.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,822 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed CampaignSummaryFactsByEMailClient,cccccccc,1478428266204.66b046e29ccf739427e1d6751d426918. 2017-08-05 01:23:10,822 INFO [StoreCloserThread-ModelsConfigSync,33333333,1478428291186.a3031397845bbbf714f6c4f2309c3f4e.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,822 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed ModelsConfigSync,33333333,1478428291186.a3031397845bbbf714f6c4f2309c3f4e. 2017-08-05 01:23:10,829 INFO [StoreCloserThread-CampaignBlastScheduleTransaction,zAcwqk81jS9EA,1489287275951.393939ae482caf94d881c21043b9e8e5.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,829 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed CampaignBlastScheduleTransaction,zAcwqk81jS9EA,1489287275951.393939ae482caf94d881c21043b9e8e5. 2017-08-05 01:23:10,830 INFO [StoreCloserThread-CampaignBlastScheduleTransaction,33333333,1478427693033.b584a415a18d1c254074303145d2e4e5.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,830 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed CampaignBlastScheduleTransaction,33333333,1478427693033.b584a415a18d1c254074303145d2e4e5. 2017-08-05 01:23:10,830 INFO [StoreCloserThread-CampaignSummaryDetailGeographyByCityMDC,99999999,1478428063068.cb335bf9c8a5cb12a20102311253c227.-1] regionserver.HStore: Closed Geo 2017-08-05 01:23:10,835 INFO [StoreCloserThread-WebAnalyticsUserFlow,cDGw45zRACountryin11494415496,1495590996235.55667ebba2d8e8355da0c7b6087c89dd.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,835 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed WebAnalyticsUserFlow,cDGw45zRACountryin11494415496,1495590996235.55667ebba2d8e8355da0c7b6087c89dd. 2017-08-05 01:23:10,835 INFO [StoreCloserThread-WebAnalyticsUserFlow_Old,33333333,1478513262242.4a3642f2de64cfaf1879da87dd5e826f.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,835 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed WebAnalyticsUserFlow_Old,33333333,1478513262242.4a3642f2de64cfaf1879da87dd5e826f. 2017-08-05 01:23:10,835 INFO [StoreCloserThread-CampaignSummaryInsight,99999999,1478428013094.b5db23e67f4323a2e1947f5ef8c24a49.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,835 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed CampaignSummaryInsight,99999999,1478428013094.b5db23e67f4323a2e1947f5ef8c24a49. 2017-08-05 01:23:10,835 INFO [StoreCloserThread-CampaignSummaryIndustrySegment,,1478428096368.bb225673e1cdda4e705d3f144b490d34.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,835 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed CampaignSummaryIndustrySegment,,1478428096368.bb225673e1cdda4e705d3f144b490d34. 2017-08-05 01:23:10,835 INFO [StoreCloserThread-DashboardAverageTimeToConversion,99999999,1478428382583.d7dc9782440ebd333405e3432f28fe08.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,835 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed DashboardAverageTimeToConversion,99999999,1478428382583.d7dc9782440ebd333405e3432f28fe08. 2017-08-05 01:23:10,839 INFO [StoreCloserThread-CampaignSummaryTrends_New,66666666,1478427970677.ae8b0e9341b11f06b65dd20f4ef829b3.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,839 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed CampaignSummaryTrends_New,66666666,1478427970677.ae8b0e9341b11f06b65dd20f4ef829b3. 2017-08-05 01:23:10,839 INFO [StoreCloserThread-CampaignSummaryIndustrySegment,99999999,1478428096368.b2b638af07f53576a15b94bdb83697c0.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,839 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed CampaignSummaryIndustrySegment,99999999,1478428096368.b2b638af07f53576a15b94bdb83697c0. 2017-08-05 01:23:10,843 INFO [StoreCloserThread-CampaignSummaryDetailGeographyByCityMDC,99999999,1478428063068.cb335bf9c8a5cb12a20102311253c227.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,843 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed CampaignSummaryDetailGeographyByCityMDC,99999999,1478428063068.cb335bf9c8a5cb12a20102311253c227. 2017-08-05 01:23:10,844 INFO [StoreCloserThread-CampaignROIForecasting,99999999,1478428263066.3ba3f10e830a2685637f3911c2ea9991.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,844 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed CampaignROIForecasting,99999999,1478428263066.3ba3f10e830a2685637f3911c2ea9991. 2017-08-05 01:23:10,845 INFO [StoreCloserThread-CampaignSMSLinkClicksTransaction,cccccccc,1488471469014.97e16a2e0b57885a84182c756426bf32.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,845 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed CampaignSMSLinkClicksTransaction,cccccccc,1488471469014.97e16a2e0b57885a84182c756426bf32. 2017-08-05 01:23:10,845 INFO [StoreCloserThread-CampaignQRCodeTransaction,33333333,1478427783205.755ba704de43ccb18877cacff82487ed.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,845 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed CampaignQRCodeTransaction,33333333,1478427783205.755ba704de43ccb18877cacff82487ed. 2017-08-05 01:23:10,846 INFO [StoreCloserThread-RecipientFieldCount,99999999,1478427677155.5272a7c2689aa7706eaec531cf5d95cb.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,846 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed RecipientFieldCount,99999999,1478427677155.5272a7c2689aa7706eaec531cf5d95cb. 2017-08-05 01:23:10,846 INFO [StoreCloserThread-RecipientFieldCount,cccccccc,1478427677155.7caf6dd7de06844ac48b95551dde6cb4.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,846 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed RecipientFieldCount,cccccccc,1478427677155.7caf6dd7de06844ac48b95551dde6cb4. 2017-08-05 01:23:10,848 INFO [StoreCloserThread-SMSCampaignStatus,zAcwYZCnKBWN7,1492102956993.2cdd9ba7e4ed3e8e1c02b403f28ed11d.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,848 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed SMSCampaignStatus,zAcwYZCnKBWN7,1492102956993.2cdd9ba7e4ed3e8e1c02b403f28ed11d. 2017-08-05 01:23:10,849 INFO [StoreCloserThread-EmailCampaignStatus,zAcw897RbOPEN6G3HS,1495184280967.af26ae169a05719cfcaba1b70a57db8b.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,849 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed EmailCampaignStatus,zAcw897RbOPEN6G3HS,1495184280967.af26ae169a05719cfcaba1b70a57db8b. 2017-08-05 01:23:10,849 INFO [StoreCloserThread-CampaignBenchMark,33333333,1478428131332.1588e3d44f7a3b31d2f745f80196dd8f.-1] regionserver.HStore: Closed Em 2017-08-05 01:23:10,849 INFO [StoreCloserThread-CampaignBenchMark,33333333,1478428131332.1588e3d44f7a3b31d2f745f80196dd8f.-1] regionserver.HStore: Closed Fb 2017-08-05 01:23:10,849 INFO [StoreCloserThread-CampaignBenchMark,33333333,1478428131332.1588e3d44f7a3b31d2f745f80196dd8f.-1] regionserver.HStore: Closed Fp 2017-08-05 01:23:10,849 INFO [StoreCloserThread-CampaignBenchMark,33333333,1478428131332.1588e3d44f7a3b31d2f745f80196dd8f.-1] regionserver.HStore: Closed Lnk 2017-08-05 01:23:10,849 INFO [StoreCloserThread-CampaignBenchMark,33333333,1478428131332.1588e3d44f7a3b31d2f745f80196dd8f.-1] regionserver.HStore: Closed Mob 2017-08-05 01:23:10,849 INFO [StoreCloserThread-CampaignBenchMark,33333333,1478428131332.1588e3d44f7a3b31d2f745f80196dd8f.-1] regionserver.HStore: Closed QR 2017-08-05 01:23:10,849 INFO [StoreCloserThread-CampaignBenchMark,33333333,1478428131332.1588e3d44f7a3b31d2f745f80196dd8f.-1] regionserver.HStore: Closed Tw 2017-08-05 01:23:10,849 INFO [StoreCloserThread-CampaignBenchMark,33333333,1478428131332.1588e3d44f7a3b31d2f745f80196dd8f.-1] regionserver.HStore: Closed Yt 2017-08-05 01:23:10,849 INFO [StoreCloserThread-CampaignBenchMark,33333333,1478428131332.1588e3d44f7a3b31d2f745f80196dd8f.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,849 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed CampaignBenchMark,33333333,1478428131332.1588e3d44f7a3b31d2f745f80196dd8f. 2017-08-05 01:23:10,849 INFO [StoreCloserThread-test1,cccccccc,1480081852792.de85cbdc154401a72f9fed6614ba81c8.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,849 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed test1,cccccccc,1480081852792.de85cbdc154401a72f9fed6614ba81c8. 2017-08-05 01:23:10,851 INFO [StoreCloserThread-CampaignSMSTransaction,33333333,1478427770516.0dd8460d94f481e4f8d3dd59a32834a6.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,851 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed CampaignSMSTransaction,33333333,1478427770516.0dd8460d94f481e4f8d3dd59a32834a6. 2017-08-05 01:23:10,851 INFO [StoreCloserThread-SocialMediaUserBehaviour,,1478427797767.513b67d6217f4a93319844b549a12e7c.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,851 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed SocialMediaUserBehaviour,,1478427797767.513b67d6217f4a93319844b549a12e7c. 2017-08-05 01:23:10,851 INFO [StoreCloserThread-CampaignMailUnsubscribe,,1478427744280.f16ed6a015054b0dc1f169466565b26a.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,851 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed CampaignMailUnsubscribe,,1478427744280.f16ed6a015054b0dc1f169466565b26a. 2017-08-05 01:23:10,851 INFO [StoreCloserThread-CampaignSummaryImpact,99999999,1478427995538.4b6593435f97c30880b807a906b9e526.-1] regionserver.HStore: Closed Dg 2017-08-05 01:23:10,851 INFO [StoreCloserThread-CampaignSummaryImpact,99999999,1478427995538.4b6593435f97c30880b807a906b9e526.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,851 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed CampaignSummaryImpact,99999999,1478427995538.4b6593435f97c30880b807a906b9e526. 2017-08-05 01:23:10,851 INFO [StoreCloserThread-WeekDayList,66666666,1478427657453.d4f6571bcecbd8df235fcb45c4e908a2.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,852 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed WeekDayList,66666666,1478427657453.d4f6571bcecbd8df235fcb45c4e908a2. 2017-08-05 01:23:10,852 INFO [StoreCloserThread-ClientsBenchMark,33333333,1478428269332.2afebba1708e881b6efeb79437249bb1.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,852 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed ClientsBenchMark,33333333,1478428269332.2afebba1708e881b6efeb79437249bb1. 2017-08-05 01:23:10,852 INFO [StoreCloserThread-UserAnalytics360RecipientCampaign,cccccccc,1478428494703.1967dfcfef3faeebc6d9fd7d05747f5f.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,852 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed UserAnalytics360RecipientCampaign,cccccccc,1478428494703.1967dfcfef3faeebc6d9fd7d05747f5f. 2017-08-05 01:23:10,852 INFO [StoreCloserThread-CampaignSummaryInsight,66666666,1478428013094.886e6214704531b66c716f3daf08515e.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,852 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed CampaignSummaryInsight,66666666,1478428013094.886e6214704531b66c716f3daf08515e. 2017-08-05 01:23:10,852 INFO [StoreCloserThread-Recipients,,1478427584994.8c28d2a564d72b729ef4fdb3d3a39424.-1] regionserver.HStore: Closed Prop 2017-08-05 01:23:10,852 INFO [StoreCloserThread-Recipients,,1478427584994.8c28d2a564d72b729ef4fdb3d3a39424.-1] regionserver.HStore: Closed Seg 2017-08-05 01:23:10,852 INFO [StoreCloserThread-CampaignSMSTransaction,egzAcwHIcHv6BNA53201603171209,1479277468523.9babb6ae92d23edc32fc127985a42808.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,852 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed CampaignSMSTransaction,egzAcwHIcHv6BNA53201603171209,1479277468523.9babb6ae92d23edc32fc127985a42808. 2017-08-05 01:23:10,853 INFO [StoreCloserThread-SocialMediaInsightsDetail_New_Bkp160517,99999999,1478427825373.b5e3d38e88f21b6a0c82c7ab60afe2ec.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,853 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed SocialMediaInsightsDetail_New_Bkp160517,99999999,1478427825373.b5e3d38e88f21b6a0c82c7ab60afe2ec. 2017-08-05 01:23:10,853 INFO [StoreCloserThread-CampaignSummaryFacts,33333333,1478427939763.3dac1719eae9e393c639be04fed72004.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,853 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed CampaignSummaryFacts,33333333,1478427939763.3dac1719eae9e393c639be04fed72004. 2017-08-05 01:23:10,853 INFO [StoreCloserThread-CampaignBlastScheduleTransactionByRecipientGUID,66666666,1478428471741.5340b4d689225f12467d95994effb65c.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,853 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed CampaignBlastScheduleTransactionByRecipientGUID,66666666,1478428471741.5340b4d689225f12467d95994effb65c. 2017-08-05 01:23:10,856 INFO [StoreCloserThread-Recipients,,1478427584994.8c28d2a564d72b729ef4fdb3d3a39424.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,856 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed Recipients,,1478427584994.8c28d2a564d72b729ef4fdb3d3a39424. 2017-08-05 01:23:10,861 INFO [StoreCloserThread-CampaignConversion,66666666,1478427841199.8eacb0eb4299655cb87525a22bfdf909.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,861 INFO [StoreCloserThread-SMSCampaignStatus,zAcwGHtXrJ6RYT,1489986614646.29caec44986c84fa6dc78d79b2ea6dd6.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,861 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed SMSCampaignStatus,zAcwGHtXrJ6RYT,1489986614646.29caec44986c84fa6dc78d79b2ea6dd6. 2017-08-05 01:23:10,861 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed CampaignConversion,66666666,1478427841199.8eacb0eb4299655cb87525a22bfdf909. 2017-08-05 01:23:10,862 INFO [StoreCloserThread-EmailCampaignStatus,zAcwcJMcABLASTSK5HK,1501479807352.173df1b90a8ded42566c33e0db403394.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,862 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed EmailCampaignStatus,zAcwcJMcABLASTSK5HK,1501479807352.173df1b90a8ded42566c33e0db403394. 2017-08-05 01:23:10,865 INFO [StoreCloserThread-WebAnalyticsUserFlow,cDGwKcHRACityBengaluru, India41495432574,1496113619076.ad8f1ba78a6c00c7af406989a575e012.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,865 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed WebAnalyticsUserFlow,cDGwKcHRACityBengaluru, India41495432574,1496113619076.ad8f1ba78a6c00c7af406989a575e012. 2017-08-05 01:23:10,866 INFO [StoreCloserThread-UserAnalytics360UserReports,cccccccc,1478428457233.1db91d804fc1473958b13271513ecf5d.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,866 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed UserAnalytics360UserReports,cccccccc,1478428457233.1db91d804fc1473958b13271513ecf5d. 2017-08-05 01:23:10,866 INFO [StoreCloserThread-UserAnalytics360RecipientCampaign,99999999,1478428494703.d667cac8a5d66d3f1f37472da62349c9.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,866 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed UserAnalytics360RecipientCampaign,99999999,1478428494703.d667cac8a5d66d3f1f37472da62349c9. 2017-08-05 01:23:10,867 INFO [StoreCloserThread-SocialMediaAnalyticsRecipients,66666666,1478427863544.e67c1301dd29e91602c633432b1414f0.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,867 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed SocialMediaAnalyticsRecipients,66666666,1478427863544.e67c1301dd29e91602c633432b1414f0. 2017-08-05 01:23:10,868 INFO [StoreCloserThread-SocialMediaRecipients,cccccccc,1478427832245.14f6bcf5177513e11f72031c6d8dc990.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,868 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed SocialMediaRecipients,cccccccc,1478427832245.14f6bcf5177513e11f72031c6d8dc990. 2017-08-05 01:23:10,869 INFO [StoreCloserThread-EmailCampaignStatus,NOBMs49bTOPENeLirb,1494686125741.64064a93c70a053f86656a5b14b588ec.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,870 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed EmailCampaignStatus,NOBMs49bTOPENeLirb,1494686125741.64064a93c70a053f86656a5b14b588ec. 2017-08-05 01:23:10,871 INFO [StoreCloserThread-CampaignMailForwardTransaction,66666666,1478427734777.f6eab7cdb3ef9914fc3de2854d096856.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,871 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed CampaignMailForwardTransaction,66666666,1478427734777.f6eab7cdb3ef9914fc3de2854d096856. 2017-08-05 01:23:10,871 INFO [StoreCloserThread-CampaignSummaryFactsMDC,,1478427948111.6ac61aca525b2e3f533242e47051518a.-1] regionserver.HStore: Closed Em 2017-08-05 01:23:10,871 INFO [StoreCloserThread-CampaignSummaryFactsMDC,,1478427948111.6ac61aca525b2e3f533242e47051518a.-1] regionserver.HStore: Closed Fb 2017-08-05 01:23:10,871 INFO [StoreCloserThread-CampaignSummaryFactsMDC,,1478427948111.6ac61aca525b2e3f533242e47051518a.-1] regionserver.HStore: Closed Fp 2017-08-05 01:23:10,871 INFO [StoreCloserThread-CampaignSummaryFactsMDC,,1478427948111.6ac61aca525b2e3f533242e47051518a.-1] regionserver.HStore: Closed GP 2017-08-05 01:23:10,871 INFO [StoreCloserThread-CampaignBlastScheduleTransactionByRecipientGUID,99999999,1478428471741.1636391c920ab3a7c1b13276f671c34c.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,871 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed CampaignBlastScheduleTransactionByRecipientGUID,99999999,1478428471741.1636391c920ab3a7c1b13276f671c34c. 2017-08-05 01:23:10,875 INFO [StoreCloserThread-CampaignSummaryFactsMDC,,1478427948111.6ac61aca525b2e3f533242e47051518a.-1] regionserver.HStore: Closed Mob 2017-08-05 01:23:10,875 INFO [StoreCloserThread-CampaignSummaryFactsMDC,,1478427948111.6ac61aca525b2e3f533242e47051518a.-1] regionserver.HStore: Closed Pi 2017-08-05 01:23:10,877 INFO [StoreCloserThread-CampaignGoalMDC,66666666,1478427913612.d203ee0385f99a4648972742c3debddb.-1] regionserver.HStore: Closed Em 2017-08-05 01:23:10,877 INFO [StoreCloserThread-CampaignGoalMDC,66666666,1478427913612.d203ee0385f99a4648972742c3debddb.-1] regionserver.HStore: Closed Fb 2017-08-05 01:23:10,877 INFO [StoreCloserThread-CampaignGoalMDC,66666666,1478427913612.d203ee0385f99a4648972742c3debddb.-1] regionserver.HStore: Closed Fp 2017-08-05 01:23:10,877 INFO [StoreCloserThread-CampaignGoalMDC,66666666,1478427913612.d203ee0385f99a4648972742c3debddb.-1] regionserver.HStore: Closed Ga 2017-08-05 01:23:10,877 INFO [StoreCloserThread-CampaignGoalMDC,66666666,1478427913612.d203ee0385f99a4648972742c3debddb.-1] regionserver.HStore: Closed Lnk 2017-08-05 01:23:10,877 INFO [StoreCloserThread-CampaignSummaryFactsMDC,,1478427948111.6ac61aca525b2e3f533242e47051518a.-1] regionserver.HStore: Closed Qr 2017-08-05 01:23:10,877 INFO [StoreCloserThread-CampaignSummaryFactsMDC,,1478427948111.6ac61aca525b2e3f533242e47051518a.-1] regionserver.HStore: Closed Tw 2017-08-05 01:23:10,877 INFO [StoreCloserThread-CampaignSummaryFactsMDC,,1478427948111.6ac61aca525b2e3f533242e47051518a.-1] regionserver.HStore: Closed Yt 2017-08-05 01:23:10,880 INFO [StoreCloserThread-CampaignSummaryFactsMDC,,1478427948111.6ac61aca525b2e3f533242e47051518a.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,880 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed CampaignSummaryFactsMDC,,1478427948111.6ac61aca525b2e3f533242e47051518a. 2017-08-05 01:23:10,880 INFO [StoreCloserThread-RecipientsSocialMediaInfo,99999999,1478427803572.a834062fab648fe7251a8b3f671d2ed1.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,880 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed RecipientsSocialMediaInfo,99999999,1478427803572.a834062fab648fe7251a8b3f671d2ed1. 2017-08-05 01:23:10,881 INFO [StoreCloserThread-CampaignSummaryFacts,66666666,1478427939763.c81813e696112c8a36b46b01c928bbdc.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,881 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed CampaignSummaryFacts,66666666,1478427939763.c81813e696112c8a36b46b01c928bbdc. 2017-08-05 01:23:10,882 INFO [StoreCloserThread-CampaignGoalMDC,66666666,1478427913612.d203ee0385f99a4648972742c3debddb.-1] regionserver.HStore: Closed Mb 2017-08-05 01:23:10,884 INFO [StoreCloserThread-SMSCampaignStatus,zAcw88WY7LB3N,1489878761901.38703cccab25f015971b58d86cccce9e.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,884 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed SMSCampaignStatus,zAcw88WY7LB3N,1489878761901.38703cccab25f015971b58d86cccce9e. 2017-08-05 01:23:10,885 INFO [StoreCloserThread-CampaignGoalMDC,66666666,1478427913612.d203ee0385f99a4648972742c3debddb.-1] regionserver.HStore: Closed Qr 2017-08-05 01:23:10,885 INFO [StoreCloserThread-CampaignGoalMDC,66666666,1478427913612.d203ee0385f99a4648972742c3debddb.-1] regionserver.HStore: Closed Ra 2017-08-05 01:23:10,885 INFO [StoreCloserThread-CampaignGoalMDC,66666666,1478427913612.d203ee0385f99a4648972742c3debddb.-1] regionserver.HStore: Closed Tw 2017-08-05 01:23:10,885 INFO [StoreCloserThread-CampaignGoalMDC,66666666,1478427913612.d203ee0385f99a4648972742c3debddb.-1] regionserver.HStore: Closed We 2017-08-05 01:23:10,885 INFO [StoreCloserThread-CampaignGoalMDC,66666666,1478427913612.d203ee0385f99a4648972742c3debddb.-1] regionserver.HStore: Closed Yt 2017-08-05 01:23:10,887 INFO [StoreCloserThread-SMSCampaignStatus,zAcwXPK7tBUTV2,1494704681403.13f0709e1757c082c1d370b61d4c2264.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,887 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed SMSCampaignStatus,zAcwXPK7tBUTV2,1494704681403.13f0709e1757c082c1d370b61d4c2264. 2017-08-05 01:23:10,887 INFO [StoreCloserThread-ClientsBenchMark,66666666,1478428269332.a9e1ef22f445d57b64e82378703e0fd3.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,887 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed ClientsBenchMark,66666666,1478428269332.a9e1ef22f445d57b64e82378703e0fd3. 2017-08-05 01:23:10,888 INFO [StoreCloserThread-CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494.-1] regionserver.HStore: Closed Em 2017-08-05 01:23:10,888 INFO [StoreCloserThread-CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494.-1] regionserver.HStore: Closed Fb 2017-08-05 01:23:10,888 INFO [StoreCloserThread-CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494.-1] regionserver.HStore: Closed Fp 2017-08-05 01:23:10,888 INFO [StoreCloserThread-CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494.-1] regionserver.HStore: Closed Ga 2017-08-05 01:23:10,888 INFO [StoreCloserThread-CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494.-1] regionserver.HStore: Closed Lnk 2017-08-05 01:23:10,889 INFO [StoreCloserThread-CampaignGoalMDC,66666666,1478427913612.d203ee0385f99a4648972742c3debddb.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,889 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed CampaignGoalMDC,66666666,1478427913612.d203ee0385f99a4648972742c3debddb. 2017-08-05 01:23:10,889 INFO [StoreCloserThread-CampaignSummaryGeographyByCity,66666666,1478428019768.a182c014ade9845809df38586b1a291d.-1] regionserver.HStore: Closed Geo 2017-08-05 01:23:10,889 INFO [StoreCloserThread-CampaignSummaryGeographyByCity,66666666,1478428019768.a182c014ade9845809df38586b1a291d.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,889 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed CampaignSummaryGeographyByCity,66666666,1478428019768.a182c014ade9845809df38586b1a291d. 2017-08-05 01:23:10,890 INFO [StoreCloserThread-IndustryBenchMark,cccccccc,1478428278697.110cbdd3996f4937bfaa7e442abefc66.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,890 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed IndustryBenchMark,cccccccc,1478428278697.110cbdd3996f4937bfaa7e442abefc66. 2017-08-05 01:23:10,890 INFO [StoreCloserThread-UserAnalytics360ViralityMap,,1478428479264.b979f32d86456c3f54e225892c9108d0.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,890 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed UserAnalytics360ViralityMap,,1478428479264.b979f32d86456c3f54e225892c9108d0. 2017-08-05 01:23:10,891 INFO [StoreCloserThread-CampaignSummaryFactsMDC,33333333,1478427948111.97c67df4b304db2ee9591a9347fd0273.-1] regionserver.HStore: Closed Em 2017-08-05 01:23:10,891 INFO [StoreCloserThread-CampaignSummaryFactsMDC,33333333,1478427948111.97c67df4b304db2ee9591a9347fd0273.-1] regionserver.HStore: Closed Fb 2017-08-05 01:23:10,891 INFO [StoreCloserThread-CampaignSummaryFactsMDC,33333333,1478427948111.97c67df4b304db2ee9591a9347fd0273.-1] regionserver.HStore: Closed Fp 2017-08-05 01:23:10,891 INFO [StoreCloserThread-CampaignSummaryFactsMDC,33333333,1478427948111.97c67df4b304db2ee9591a9347fd0273.-1] regionserver.HStore: Closed GP 2017-08-05 01:23:10,893 INFO [StoreCloserThread-CampaignSummaryTrends_New,34nwnoeMobileH1577516,1501638938779.f8b4de0216e731b7847af7ee4e3d87eb.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,893 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed CampaignSummaryTrends_New,34nwnoeMobileH1577516,1501638938779.f8b4de0216e731b7847af7ee4e3d87eb. 2017-08-05 01:23:10,893 INFO [StoreCloserThread-DashboardCampaignLeadsChannelWise,cccccccc,1478428306794.17da2404716950425deff2bfa3e13daf.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,893 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed DashboardCampaignLeadsChannelWise,cccccccc,1478428306794.17da2404716950425deff2bfa3e13daf. 2017-08-05 01:23:10,894 INFO [StoreCloserThread-CampaignSummaryFactsMDC,33333333,1478427948111.97c67df4b304db2ee9591a9347fd0273.-1] regionserver.HStore: Closed Mob 2017-08-05 01:23:10,894 INFO [StoreCloserThread-CampaignSummaryFactsMDC,33333333,1478427948111.97c67df4b304db2ee9591a9347fd0273.-1] regionserver.HStore: Closed Pi 2017-08-05 01:23:10,896 INFO [StoreCloserThread-SocialMediaInsights_New,cccccccc,1494941509384.4b7cb5bf990b818746d9a5fbe1626deb.-1] regionserver.HStore: Closed ch 2017-08-05 01:23:10,897 INFO [StoreCloserThread-CampaignSummaryFactsMDC,33333333,1478427948111.97c67df4b304db2ee9591a9347fd0273.-1] regionserver.HStore: Closed Qr 2017-08-05 01:23:10,897 INFO [StoreCloserThread-CampaignSummaryFactsMDC,33333333,1478427948111.97c67df4b304db2ee9591a9347fd0273.-1] regionserver.HStore: Closed Tw 2017-08-05 01:23:10,897 INFO [StoreCloserThread-CampaignSummaryFactsMDC,33333333,1478427948111.97c67df4b304db2ee9591a9347fd0273.-1] regionserver.HStore: Closed Yt 2017-08-05 01:23:10,899 INFO [StoreCloserThread-CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494.-1] regionserver.HStore: Closed Mb 2017-08-05 01:23:10,900 INFO [StoreCloserThread-SocialMediaInsights_New,cccccccc,1494941509384.4b7cb5bf990b818746d9a5fbe1626deb.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,900 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed SocialMediaInsights_New,cccccccc,1494941509384.4b7cb5bf990b818746d9a5fbe1626deb. 2017-08-05 01:23:10,900 INFO [StoreCloserThread-RecipientFieldCount,33333333,1478427677155.5898989a93d2ddd54eb187511b069a37.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,900 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed RecipientFieldCount,33333333,1478427677155.5898989a93d2ddd54eb187511b069a37. 2017-08-05 01:23:10,901 INFO [StoreCloserThread-CampaignSummaryFactsMDC,33333333,1478427948111.97c67df4b304db2ee9591a9347fd0273.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,901 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed CampaignSummaryFactsMDC,33333333,1478427948111.97c67df4b304db2ee9591a9347fd0273. 2017-08-05 01:23:10,902 INFO [StoreCloserThread-CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494.-1] regionserver.HStore: Closed Qr 2017-08-05 01:23:10,902 INFO [StoreCloserThread-CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494.-1] regionserver.HStore: Closed Ra 2017-08-05 01:23:10,902 INFO [StoreCloserThread-CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494.-1] regionserver.HStore: Closed Tw 2017-08-05 01:23:10,902 INFO [StoreCloserThread-CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494.-1] regionserver.HStore: Closed We 2017-08-05 01:23:10,902 INFO [StoreCloserThread-CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494.-1] regionserver.HStore: Closed Yt 2017-08-05 01:23:10,904 INFO [StoreCloserThread-CampaignSMSTransaction,FEzAcw4dATiS95HA201702251800,1490095737415.8f0a2d3fafd021b853a5f90c08b4c41a.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,904 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed CampaignSMSTransaction,FEzAcw4dATiS95HA201702251800,1490095737415.8f0a2d3fafd021b853a5f90c08b4c41a. 2017-08-05 01:23:10,907 INFO [StoreCloserThread-CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,907 ERROR [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Memstore size is 2666120 2017-08-05 01:23:10,907 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. 2017-08-05 01:23:10,909 INFO [StoreCloserThread-CampaignSMSTransaction,FEzAcwT86zINRZB201610101205,1495726131026.fa3fb69ce9e136c4f4aa82200d5d2436.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,909 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed CampaignSMSTransaction,FEzAcwT86zINRZB201610101205,1495726131026.fa3fb69ce9e136c4f4aa82200d5d2436. 2017-08-05 01:23:10,909 INFO [StoreCloserThread-UserAnalytics360ViralityMap,33333333,1478428479264.6aabeae029e218679b6e01f3a1941d5b.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,909 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed UserAnalytics360ViralityMap,33333333,1478428479264.6aabeae029e218679b6e01f3a1941d5b. 2017-08-05 01:23:10,910 INFO [StoreCloserThread-IndustryBenchMark,99999999,1478428278697.0f1960ffcf9ff8e7f2fb1a0f6d9daada.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,910 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed IndustryBenchMark,99999999,1478428278697.0f1960ffcf9ff8e7f2fb1a0f6d9daada. 2017-08-05 01:23:10,910 INFO [StoreCloserThread-UserAnalytics360RecipientCampaign,66666666,1478428494703.3471642746ae08a3d2e0eea4ed66b3ee.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,910 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed UserAnalytics360RecipientCampaign,66666666,1478428494703.3471642746ae08a3d2e0eea4ed66b3ee. 2017-08-05 01:23:10,910 INFO [StoreCloserThread-CampaignConversionTracking,cccccccc,1478427850748.65f062da0f0374e8996f4301dc654230.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,910 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed CampaignConversionTracking,cccccccc,1478427850748.65f062da0f0374e8996f4301dc654230. 2017-08-05 01:23:10,910 INFO [StoreCloserThread-DashboardAverageTimeToConversion,,1478428382583.55621d078219838922fe86e92d434da5.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,910 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed DashboardAverageTimeToConversion,,1478428382583.55621d078219838922fe86e92d434da5. 2017-08-05 01:23:10,910 INFO [StoreCloserThread-CampaignBenchMark,cccccccc,1478428131332.4dd348bfccd5d2f1d7a29838780cc80d.-1] regionserver.HStore: Closed Em 2017-08-05 01:23:10,910 INFO [StoreCloserThread-CampaignBenchMark,cccccccc,1478428131332.4dd348bfccd5d2f1d7a29838780cc80d.-1] regionserver.HStore: Closed Fb 2017-08-05 01:23:10,910 INFO [StoreCloserThread-CampaignBenchMark,cccccccc,1478428131332.4dd348bfccd5d2f1d7a29838780cc80d.-1] regionserver.HStore: Closed Fp 2017-08-05 01:23:10,910 INFO [StoreCloserThread-CampaignBenchMark,cccccccc,1478428131332.4dd348bfccd5d2f1d7a29838780cc80d.-1] regionserver.HStore: Closed Lnk 2017-08-05 01:23:10,910 INFO [StoreCloserThread-CampaignBenchMark,cccccccc,1478428131332.4dd348bfccd5d2f1d7a29838780cc80d.-1] regionserver.HStore: Closed Mob 2017-08-05 01:23:10,910 INFO [StoreCloserThread-CampaignBenchMark,cccccccc,1478428131332.4dd348bfccd5d2f1d7a29838780cc80d.-1] regionserver.HStore: Closed QR 2017-08-05 01:23:10,910 INFO [StoreCloserThread-CampaignBenchMark,cccccccc,1478428131332.4dd348bfccd5d2f1d7a29838780cc80d.-1] regionserver.HStore: Closed Tw 2017-08-05 01:23:10,910 INFO [StoreCloserThread-CampaignBenchMark,cccccccc,1478428131332.4dd348bfccd5d2f1d7a29838780cc80d.-1] regionserver.HStore: Closed Yt 2017-08-05 01:23:10,910 INFO [StoreCloserThread-CampaignBenchMark,cccccccc,1478428131332.4dd348bfccd5d2f1d7a29838780cc80d.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,911 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed CampaignBenchMark,cccccccc,1478428131332.4dd348bfccd5d2f1d7a29838780cc80d. 2017-08-05 01:23:10,912 INFO [StoreCloserThread-CampaignBlastScheduleTransaction,srzAcwp3D5XFFASRV,1492484474153.18190d106607299d031ceadd8cf40034.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,912 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed CampaignBlastScheduleTransaction,srzAcwp3D5XFFASRV,1492484474153.18190d106607299d031ceadd8cf40034. 2017-08-05 01:23:10,912 INFO [StoreCloserThread-DashboardChannelWisePerformance,,1478428370167.ea51863f0d80b5b9a4f09d039b95eeb6.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,912 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed DashboardChannelWisePerformance,,1478428370167.ea51863f0d80b5b9a4f09d039b95eeb6. 2017-08-05 01:23:10,913 INFO [StoreCloserThread-Recipients,33333333,1478427584994.f3317a31f7b3ff0e70b00c74f1f6e10f.-1] regionserver.HStore: Closed Prop 2017-08-05 01:23:10,913 INFO [StoreCloserThread-Recipients,33333333,1478427584994.f3317a31f7b3ff0e70b00c74f1f6e10f.-1] regionserver.HStore: Closed Seg 2017-08-05 01:23:10,913 INFO [StoreCloserThread-CampaignSMSTransaction,,1479213298205.cd4ae2f706fce231946c72f8b09784be.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,913 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed CampaignSMSTransaction,,1479213298205.cd4ae2f706fce231946c72f8b09784be. 2017-08-05 01:23:10,917 INFO [StoreCloserThread-EmailCampaignStatus,zAcwefNntBLASTEL3E,1490799888294.f4598840a75fc365232aabdc2524701d.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,917 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed EmailCampaignStatus,zAcwefNntBLASTEL3E,1490799888294.f4598840a75fc365232aabdc2524701d. 2017-08-05 01:23:10,917 INFO [StoreCloserThread-UserAnalytics360Details,99999999,1478428500855.0ca3c303e54bc6abb52eed0ef8f5c4d5.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,917 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed UserAnalytics360Details,99999999,1478428500855.0ca3c303e54bc6abb52eed0ef8f5c4d5. 2017-08-05 01:23:10,917 INFO [StoreCloserThread-UserAnalytics360OverviewBehaviour,99999999,1478428297431.6227051f8a3f958079601ad7a4b288cc.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,917 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed UserAnalytics360OverviewBehaviour,99999999,1478428297431.6227051f8a3f958079601ad7a4b288cc. 2017-08-05 01:23:10,917 INFO [StoreCloserThread-CampaignSummaryBusinessSegment,99999999,1478428086929.841583408059121ced9bb587f1d8c2df.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,917 INFO [StoreCloserThread-Recipients,33333333,1478427584994.f3317a31f7b3ff0e70b00c74f1f6e10f.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,917 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed CampaignSummaryBusinessSegment,99999999,1478428086929.841583408059121ced9bb587f1d8c2df. 2017-08-05 01:23:10,917 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed Recipients,33333333,1478427584994.f3317a31f7b3ff0e70b00c74f1f6e10f. 2017-08-05 01:23:10,917 INFO [StoreCloserThread-DashboardCampaignDetails,99999999,1478428398909.3fdab817d35622b6fcc732fe199fae39.-1] regionserver.HStore: Closed DCD 2017-08-05 01:23:10,917 INFO [StoreCloserThread-DashboardCampaignDetails,99999999,1478428398909.3fdab817d35622b6fcc732fe199fae39.-1] regionserver.HStore: Closed Lds 2017-08-05 01:23:10,917 INFO [StoreCloserThread-DashboardCampaignDetails,99999999,1478428398909.3fdab817d35622b6fcc732fe199fae39.-1] regionserver.HStore: Closed Per 2017-08-05 01:23:10,917 INFO [StoreCloserThread-DashboardCampaignDetails,99999999,1478428398909.3fdab817d35622b6fcc732fe199fae39.-1] regionserver.HStore: Closed Prop 2017-08-05 01:23:10,917 INFO [StoreCloserThread-DashboardCampaignDetails,99999999,1478428398909.3fdab817d35622b6fcc732fe199fae39.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,917 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed DashboardCampaignDetails,99999999,1478428398909.3fdab817d35622b6fcc732fe199fae39. 2017-08-05 01:23:10,918 INFO [StoreCloserThread-CampaignSummaryDemographics,,1478427988763.bef9ebf6ba4beace1b24ce296606defb.-1] regionserver.HStore: Closed Dg 2017-08-05 01:23:10,919 INFO [StoreCloserThread-ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,919 ERROR [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Memstore size is 113673248 2017-08-05 01:23:10,919 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1498380336261.8083c44b3c7bce24d150b22ed70ea183. 2017-08-05 01:23:10,920 INFO [StoreCloserThread-CustomerPropensitySurrogate,66666666,1478427602931.91ef04b976a7156ff344312d16946c8f.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,920 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed CustomerPropensitySurrogate,66666666,1478427602931.91ef04b976a7156ff344312d16946c8f. 2017-08-05 01:23:10,922 INFO [StoreCloserThread-BenchMark,,1479074875620.ca3b04f87378984f6ae728ce2c612634.-1] regionserver.HStore: Closed Em 2017-08-05 01:23:10,924 INFO [StoreCloserThread-CampaignSummaryDemographics,,1478427988763.bef9ebf6ba4beace1b24ce296606defb.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,924 INFO [StoreCloserThread-WebAnalyticsUserFlow,cDGwOP2RACountryin01494766555,1496795479470.74cfcf80e29ba2438655444e2464aa2b.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,924 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed CampaignSummaryDemographics,,1478427988763.bef9ebf6ba4beace1b24ce296606defb. 2017-08-05 01:23:10,924 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed WebAnalyticsUserFlow,cDGwOP2RACountryin01494766555,1496795479470.74cfcf80e29ba2438655444e2464aa2b. 2017-08-05 01:23:10,924 INFO [StoreCloserThread-BenchMark,,1479074875620.ca3b04f87378984f6ae728ce2c612634.-1] regionserver.HStore: Closed Fb 2017-08-05 01:23:10,924 INFO [StoreCloserThread-Recipients,zAcwDSTPY,1482409859871.6f3f95e37bdc7406746f4e2a233a10fd.-1] regionserver.HStore: Closed Prop 2017-08-05 01:23:10,924 INFO [StoreCloserThread-BenchMark,,1479074875620.ca3b04f87378984f6ae728ce2c612634.-1] regionserver.HStore: Closed Fp 2017-08-05 01:23:10,924 INFO [StoreCloserThread-Recipients,zAcwDSTPY,1482409859871.6f3f95e37bdc7406746f4e2a233a10fd.-1] regionserver.HStore: Closed Seg 2017-08-05 01:23:10,924 INFO [StoreCloserThread-BenchMark,,1479074875620.ca3b04f87378984f6ae728ce2c612634.-1] regionserver.HStore: Closed Lnk 2017-08-05 01:23:10,924 INFO [StoreCloserThread-CampaignRecommendation,,1478428313095.fe0da2708b463ff78625ca61aa72bd25.-1] regionserver.HStore: Closed Em 2017-08-05 01:23:10,924 INFO [StoreCloserThread-CampaignRecommendation,,1478428313095.fe0da2708b463ff78625ca61aa72bd25.-1] regionserver.HStore: Closed Mb 2017-08-05 01:23:10,924 INFO [StoreCloserThread-CampaignRecommendation,,1478428313095.fe0da2708b463ff78625ca61aa72bd25.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,924 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed CampaignRecommendation,,1478428313095.fe0da2708b463ff78625ca61aa72bd25. 2017-08-05 01:23:10,927 INFO [StoreCloserThread-BenchMark,,1479074875620.ca3b04f87378984f6ae728ce2c612634.-1] regionserver.HStore: Closed Mob 2017-08-05 01:23:10,928 INFO [StoreCloserThread-Recipients,zAcwDSTPY,1482409859871.6f3f95e37bdc7406746f4e2a233a10fd.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,928 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed Recipients,zAcwDSTPY,1482409859871.6f3f95e37bdc7406746f4e2a233a10fd. 2017-08-05 01:23:10,929 INFO [StoreCloserThread-BenchMark,,1479074875620.ca3b04f87378984f6ae728ce2c612634.-1] regionserver.HStore: Closed QR 2017-08-05 01:23:10,929 INFO [StoreCloserThread-BenchMark,,1479074875620.ca3b04f87378984f6ae728ce2c612634.-1] regionserver.HStore: Closed Tw 2017-08-05 01:23:10,929 INFO [StoreCloserThread-BenchMark,,1479074875620.ca3b04f87378984f6ae728ce2c612634.-1] regionserver.HStore: Closed Yt 2017-08-05 01:23:10,931 INFO [StoreCloserThread-BenchMark,,1479074875620.ca3b04f87378984f6ae728ce2c612634.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,931 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed BenchMark,,1479074875620.ca3b04f87378984f6ae728ce2c612634. 2017-08-05 01:23:10,934 INFO [StoreCloserThread-CampaignSMSTransaction,00zAcwklUHaVSFYC6201508270717,1479280046343.061c7096dc96a669709456d9e1496c5e.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,934 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-2] regionserver.HRegion: Closed CampaignSMSTransaction,00zAcwklUHaVSFYC6201508270717,1479280046343.061c7096dc96a669709456d9e1496c5e. 2017-08-05 01:23:10,941 INFO [StoreCloserThread-CampaignBlastScheduleTransaction,cccccccc,1497164540277.04b08c4b94d6579e5bff09bb9adbf643.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,941 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-0] regionserver.HRegion: Closed CampaignBlastScheduleTransaction,cccccccc,1497164540277.04b08c4b94d6579e5bff09bb9adbf643. 2017-08-05 01:23:10,951 INFO [StoreCloserThread-CampaignSMSTransaction,00zAcwCbQsF2D7FQ201512120743,1479290202747.a065e67f22dbf3f3ddd6beb4cabbf222.-1] regionserver.HStore: Closed info 2017-08-05 01:23:10,951 INFO [RS_CLOSE_REGION-aps-hadoop4:16020-1] regionserver.HRegion: Closed CampaignSMSTransaction,00zAcwCbQsF2D7FQ201512120743,1479290202747.a065e67f22dbf3f3ddd6beb4cabbf222. 2017-08-05 01:23:10,959 INFO [regionserver/aps-hadoop4/hadoop4:16020] regionserver.HRegionServer: stopping server aps-hadoop4,16020,1501679842018; all regions closed. 2017-08-05 01:23:10,959 WARN [regionserver/aps-hadoop4/hadoop4:16020] wal.ProtobufLogWriter: Failed to write trailer, non-fatal, continuing... java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]], original=[DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:947) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1021) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1189) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-08-05 01:23:10,960 ERROR [regionserver/aps-hadoop4/hadoop4:16020] regionserver.HRegionServer: Shutdown / close of WAL failed: java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]], original=[DatanodeInfoWithStorage[hadoop1:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[hadoop6:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. 2017-08-05 01:23:10,970 INFO [regionserver/aps-hadoop4/hadoop4:16020] regionserver.Leases: regionserver/aps-hadoop4/hadoop4:16020 closing leases 2017-08-05 01:23:10,970 INFO [regionserver/aps-hadoop4/hadoop4:16020] regionserver.Leases: regionserver/aps-hadoop4/hadoop4:16020 closed leases 2017-08-05 01:23:10,970 INFO [regionserver/aps-hadoop4/hadoop4:16020] hbase.ChoreService: Chore service for: aps-hadoop4,16020,1501679842018 had [[ScheduledChore: Name: aps-hadoop4,16020,1501679842018-MemstoreFlusherChore Period: 10000 Unit: MILLISECONDS], [ScheduledChore: Name: MovedRegionsCleaner for region aps-hadoop4,16020,1501679842018 Period: 120000 Unit: MILLISECONDS]] on shutdown 2017-08-05 01:23:10,971 INFO [regionserver/aps-hadoop4/hadoop4:16020] regionserver.CompactSplitThread: Waiting for Split Thread to finish... 2017-08-05 01:23:10,971 INFO [regionserver/aps-hadoop4/hadoop4:16020] regionserver.CompactSplitThread: Waiting for Merge Thread to finish... 2017-08-05 01:23:10,971 INFO [regionserver/aps-hadoop4/hadoop4:16020] regionserver.CompactSplitThread: Waiting for Large Compaction Thread to finish... 2017-08-05 01:23:10,971 INFO [regionserver/aps-hadoop4/hadoop4:16020] regionserver.CompactSplitThread: Waiting for Small Compaction Thread to finish... 2017-08-05 01:23:10,981 INFO [regionserver/aps-hadoop4/hadoop4:16020] ipc.RpcServer: Stopping server on 16020 2017-08-05 01:23:10,981 INFO [RpcServer.listener,port=16020] ipc.RpcServer: RpcServer.listener,port=16020: stopping 2017-08-05 01:23:10,982 INFO [RpcServer.responder] ipc.RpcServer: RpcServer.responder: stopped 2017-08-05 01:23:10,982 INFO [RpcServer.responder] ipc.RpcServer: RpcServer.responder: stopping 2017-08-05 01:23:10,997 INFO [regionserver/aps-hadoop4/hadoop4:16020] zookeeper.ZooKeeper: Session: 0x35c3e07cb26cc2f closed 2017-08-05 01:23:10,997 INFO [main-EventThread] zookeeper.ClientCnxn: EventThread shut down 2017-08-05 01:23:10,997 INFO [regionserver/aps-hadoop4/hadoop4:16020] regionserver.HRegionServer: stopping server aps-hadoop4,16020,1501679842018; zookeeper connection closed. 2017-08-05 01:23:10,997 INFO [regionserver/aps-hadoop4/hadoop4:16020] regionserver.HRegionServer: regionserver/aps-hadoop4/hadoop4:16020 exiting 2017-08-05 01:23:10,997 ERROR [main] regionserver.HRegionServerCommandLine: Region server exiting java.lang.RuntimeException: HRegionServer Aborted at org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.start(HRegionServerCommandLine.java:68) at org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.run(HRegionServerCommandLine.java:87) at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) at org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:126) at org.apache.hadoop.hbase.regionserver.HRegionServer.main(HRegionServer.java:2655) 2017-08-05 01:23:11,025 INFO [pool-2-thread-1] regionserver.ShutdownHook: Shutdown hook starting; hbase.shutdown.hook=true; fsShutdownHook=org.apache.hadoop.fs.FileSystem$Cache$ClientFinalizer@7a11c4c7 2017-08-05 01:23:11,026 INFO [pool-2-thread-1] regionserver.ShutdownHook: Starting fs shutdown hook thread. 2017-08-05 01:23:11,038 INFO [pool-2-thread-1] regionserver.ShutdownHook: Shutdown hook finished.