org.apache.hadoop.security.token.SecretManager$InvalidToken: access control error while attempting to set up short-circuit access to /apps/hbase/data/data/default/CampaignSMSTransaction/d5adacd3b69c6ace6e8bf702b8ce47c9/info/484e5ed8d6434da5a91f22b48624336b at org.apache.hadoop.hdfs.BlockReaderFactory.requestFileDescriptors(BlockReaderFactory.java:589) at org.apache.hadoop.hdfs.BlockReaderFactory.createShortCircuitReplicaInfo(BlockReaderFactory.java:488) at org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.create(ShortCircuitCache.java:784) at org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache.fetchOrCreate(ShortCircuitCache.java:718) at org.apache.hadoop.hdfs.BlockReaderFactory.getBlockReaderLocal(BlockReaderFactory.java:422) at org.apache.hadoop.hdfs.BlockReaderFactory.build(BlockReaderFactory.java:333) at org.apache.hadoop.hdfs.DFSInputStream.blockSeekTo(DFSInputStream.java:662) at org.apache.hadoop.hdfs.DFSInputStream.readWithStrategy(DFSInputStream.java:898) at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:955) at java.io.DataInputStream.read(DataInputStream.java:149) at org.apache.hadoop.hbase.io.hfile.HFileBlock.readWithExtra(HFileBlock.java:679) at org.apache.hadoop.hbase.io.hfile.HFileBlock$AbstractFSReader.readAtOffset(HFileBlock.java:1412) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockDataInternal(HFileBlock.java:1625) at org.apache.hadoop.hbase.io.hfile.HFileBlock$FSReaderImpl.readBlockData(HFileBlock.java:1504) at org.apache.hadoop.hbase.io.hfile.HFileReaderV2.readBlock(HFileReaderV2.java:441) at org.apache.hadoop.hbase.io.hfile.HFileBlockIndex$BlockIndexReader.loadDataBlockWithScanInfo(HFileBlockIndex.java:269) at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.seekTo(HFileReaderV2.java:642) at org.apache.hadoop.hbase.io.hfile.HFileReaderV2$AbstractScannerV2.reseekTo(HFileReaderV2.java:622) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseekAtOrAfter(StoreFileScanner.java:280) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.reseek(StoreFileScanner.java:194) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.enforceSeek(StoreFileScanner.java:400) at org.apache.hadoop.hbase.regionserver.StoreFileScanner.requestSeek(StoreFileScanner.java:363) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.generalizedSeek(KeyValueHeap.java:310) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.requestSeek(KeyValueHeap.java:268) at org.apache.hadoop.hbase.regionserver.StoreScanner.reseek(StoreScanner.java:815) at org.apache.hadoop.hbase.regionserver.StoreScanner.seekToNextRow(StoreScanner.java:792) at org.apache.hadoop.hbase.regionserver.StoreScanner.next(StoreScanner.java:592) at org.apache.hadoop.hbase.regionserver.KeyValueHeap.next(KeyValueHeap.java:147) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.populateResult(HRegion.java:5615) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextInternal(HRegion.java:5766) at org.apache.hadoop.hbase.regionserver.HRegion$RegionScannerImpl.nextRaw(HRegion.java:5553) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2413) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:32205) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2127) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:133) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:108) at java.lang.Thread.run(Thread.java:745) 2017-05-03 19:44:29,159 INFO [B.defaultRpcServer.handler=13,queue=1,port=16020] hdfs.DFSClient: Access token was invalid when connecting to /1..1..1..:50010 : org.apache.hadoop.security.token.SecretManager$InvalidToken: access control error while attempting to set up short-circuit access to /apps/hbase/data/data/default/CampaignSMSTransaction/d5adacd3b69c6ace6e8bf702b8ce47c9/info/484e5ed8d6434da5a91f22b48624336b 2017-05-03 19:45:14,404 INFO [sync.1] wal.FSHLog: Slow sync cost: 284 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-9a10c707-ebd5-4fae-8f82-b381f706fa57,DISK]] 2017-05-03 19:45:14,527 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=3.08 GB, freeSize=101.47 MB, max=3.18 GB, blockCount=50867, accesses=359999915, hits=181661956, hitRatio=50.46%, , cachingAccesses=354219752, cachingHits=181617818, cachingHitsRatio=51.27%, evictions=90467, evicted=172225732, evictedPerRun=1903.7408447265625 2017-05-03 19:45:16,717 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.33 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 19:45:17,277 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50148202, memsize=128.3 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/0dc0e7d2fe694ffabc27c46a803be773 2017-05-03 19:45:17,289 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/0dc0e7d2fe694ffabc27c46a803be773, entries=96668, sequenceid=50148202, filesize=8.2 M 2017-05-03 19:45:17,291 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~128.33 MB/134561384, currentsize=120 KB/122880 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 573ms, sequenceid=50148202, compaction requested=true 2017-05-03 19:45:34,862 INFO [sync.4] wal.FSHLog: Slow sync cost: 271 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-9a10c707-ebd5-4fae-8f82-b381f706fa57,DISK]] 2017-05-03 19:45:39,911 INFO [sync.3] wal.FSHLog: Slow sync cost: 316 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-9a10c707-ebd5-4fae-8f82-b381f706fa57,DISK]] 2017-05-03 19:45:42,370 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493840601082 with entries=925, filesize=121.69 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493840742329 2017-05-03 19:47:04,450 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.41 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 19:47:05,020 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50148611, memsize=128.4 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/4d95b4e7e8cb4d1d91722f45b02dfeeb 2017-05-03 19:47:05,037 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/4d95b4e7e8cb4d1d91722f45b02dfeeb, entries=96668, sequenceid=50148611, filesize=8.2 M 2017-05-03 19:47:05,040 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~128.41 MB/134650992, currentsize=6.81 MB/7142320 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 590ms, sequenceid=50148611, compaction requested=true 2017-05-03 19:47:05,043 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 19:47:05,043 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 19:47:05,045 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=51827, currentSize=3371722360, freeSize=42781832, maxSize=3414504192, heapSize=3371722360, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 19:47:05,963 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into 1cc515d404f046ff838ff600d2f9f490(size=8.1 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 19:47:05,963 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17676018120057370; duration=0sec 2017-05-03 19:48:00,013 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493840742329 with entries=2207, filesize=122.02 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493840879923 2017-05-03 19:48:52,826 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.50 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 19:48:53,533 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50149043, memsize=128.5 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/1989e46caa59475abc586498efe57706 2017-05-03 19:48:53,547 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/1989e46caa59475abc586498efe57706, entries=96668, sequenceid=50149043, filesize=8.2 M 2017-05-03 19:48:53,548 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~128.50 MB/134737880, currentsize=4.87 MB/5103928 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 722ms, sequenceid=50149043, compaction requested=true 2017-05-03 19:50:08,568 INFO [sync.2] wal.FSHLog: Slow sync cost: 320 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-9a10c707-ebd5-4fae-8f82-b381f706fa57,DISK]] 2017-05-03 19:50:08,570 INFO [sync.3] wal.FSHLog: Slow sync cost: 321 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-9a10c707-ebd5-4fae-8f82-b381f706fa57,DISK]] 2017-05-03 19:50:14,527 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=3.10 GB, freeSize=85.46 MB, max=3.18 GB, blockCount=51120, accesses=360816229, hits=182070601, hitRatio=50.46%, , cachingAccesses=355035104, cachingHits=182026463, cachingHitsRatio=51.27%, evictions=90664, evicted=172632185, evictedPerRun=1904.0875244140625 2017-05-03 19:50:17,903 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493840879923 with entries=898, filesize=121.97 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493841017842 2017-05-03 19:50:18,409 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.46 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 19:50:18,970 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50149468, memsize=128.5 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/b344b5fe557b4087b7f97e671f3eb137 2017-05-03 19:50:18,984 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/b344b5fe557b4087b7f97e671f3eb137, entries=96668, sequenceid=50149468, filesize=8.2 M 2017-05-03 19:50:18,986 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.46 MB/134701632, currentsize=4.85 MB/5086560 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 577ms, sequenceid=50149468, compaction requested=true 2017-05-03 19:50:18,988 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 19:50:18,988 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 19:50:18,991 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=50352, currentSize=3274093720, freeSize=140410472, maxSize=3414504192, heapSize=3274093720, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 19:50:19,888 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into 0fd877efc23a4fcc8f78484a3b6d45d9(size=8.1 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 19:50:19,888 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17676212065370278; duration=0sec 2017-05-03 19:51:27,702 INFO [sync.2] wal.FSHLog: Slow sync cost: 1211 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-9a10c707-ebd5-4fae-8f82-b381f706fa57,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-5c91e4fa-90df-4f20-8217-049e5d671c16,DISK]] 2017-05-03 19:52:05,289 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.44 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 19:52:05,928 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50149876, memsize=128.4 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/96e6ef101cf14a398360c39278788211 2017-05-03 19:52:05,939 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/96e6ef101cf14a398360c39278788211, entries=96668, sequenceid=50149876, filesize=8.2 M 2017-05-03 19:52:05,940 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.44 MB/134676400, currentsize=1.95 MB/2042328 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 651ms, sequenceid=50149876, compaction requested=true 2017-05-03 19:52:36,770 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493841017842 with entries=1587, filesize=121.69 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493841156733 2017-05-03 19:52:41,126 INFO [sync.4] wal.FSHLog: Slow sync cost: 261 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK]] 2017-05-03 19:52:46,136 INFO [sync.1] wal.FSHLog: Slow sync cost: 301 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK]] 2017-05-03 19:52:47,116 INFO [sync.2] wal.FSHLog: Slow sync cost: 302 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK]] 2017-05-03 19:53:55,754 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.55 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 19:53:56,309 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50150319, memsize=128.6 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/72597db14baf43b8a8050b0734ce43ac 2017-05-03 19:53:56,334 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/72597db14baf43b8a8050b0734ce43ac, entries=96668, sequenceid=50150319, filesize=8.2 M 2017-05-03 19:53:56,335 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.55 MB/134794680, currentsize=3.09 MB/3242896 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 581ms, sequenceid=50150319, compaction requested=true 2017-05-03 19:53:56,337 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 19:53:56,337 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 19:53:56,340 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=51207, currentSize=3330674360, freeSize=83829832, maxSize=3414504192, heapSize=3330674360, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 19:53:57,255 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into f7d41ac30166453881195cd325db1efe(size=8.1 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 19:53:57,255 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17676429414662618; duration=0sec 2017-05-03 19:54:55,025 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493841156733 with entries=2316, filesize=121.71 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493841294995 2017-05-03 19:55:10,239 INFO [sync.2] wal.FSHLog: Slow sync cost: 119 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-120b90e1-5d80-447d-b8db-b8ea53661e88,DISK]] 2017-05-03 19:55:10,239 INFO [sync.1] wal.FSHLog: Slow sync cost: 159 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-120b90e1-5d80-447d-b8db-b8ea53661e88,DISK]] 2017-05-03 19:55:10,404 INFO [sync.4] wal.FSHLog: Slow sync cost: 117 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-120b90e1-5d80-447d-b8db-b8ea53661e88,DISK]] 2017-05-03 19:55:14,527 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=3.06 GB, freeSize=127.79 MB, max=3.18 GB, blockCount=50451, accesses=361078175, hits=182201313, hitRatio=50.46%, , cachingAccesses=355295126, cachingHits=182157175, cachingHitsRatio=51.27%, evictions=90727, evicted=172762165, evictedPerRun=1904.1978759765625 2017-05-03 19:55:15,244 INFO [sync.1] wal.FSHLog: Slow sync cost: 284 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-120b90e1-5d80-447d-b8db-b8ea53661e88,DISK]] 2017-05-03 19:55:15,244 INFO [sync.2] wal.FSHLog: Slow sync cost: 244 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-120b90e1-5d80-447d-b8db-b8ea53661e88,DISK]] 2017-05-03 19:55:16,984 INFO [sync.0] wal.FSHLog: Slow sync cost: 1644 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-120b90e1-5d80-447d-b8db-b8ea53661e88,DISK]] 2017-05-03 19:55:16,985 INFO [sync.1] wal.FSHLog: Slow sync cost: 1610 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-120b90e1-5d80-447d-b8db-b8ea53661e88,DISK]] 2017-05-03 19:55:20,334 INFO [sync.1] wal.FSHLog: Slow sync cost: 312 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-120b90e1-5d80-447d-b8db-b8ea53661e88,DISK]] 2017-05-03 19:55:21,906 INFO [sync.2] wal.FSHLog: Slow sync cost: 1830 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-120b90e1-5d80-447d-b8db-b8ea53661e88,DISK]] 2017-05-03 19:55:21,906 INFO [sync.3] wal.FSHLog: Slow sync cost: 1789 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-120b90e1-5d80-447d-b8db-b8ea53661e88,DISK]] 2017-05-03 19:55:21,906 INFO [sync.4] wal.FSHLog: Slow sync cost: 1568 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-120b90e1-5d80-447d-b8db-b8ea53661e88,DISK]] 2017-05-03 19:55:24,020 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.06 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 19:55:24,597 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50150733, memsize=128.1 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/f2bd9930bf1e4da98d41b8477bfe32e8 2017-05-03 19:55:24,613 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/f2bd9930bf1e4da98d41b8477bfe32e8, entries=96668, sequenceid=50150733, filesize=8.2 M 2017-05-03 19:55:24,618 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.06 MB/134280824, currentsize=7.43 MB/7790208 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 598ms, sequenceid=50150733, compaction requested=true 2017-05-03 19:55:27,107 INFO [sync.2] wal.FSHLog: Slow sync cost: 1660 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-120b90e1-5d80-447d-b8db-b8ea53661e88,DISK]] 2017-05-03 19:55:27,108 INFO [sync.3] wal.FSHLog: Slow sync cost: 1642 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-120b90e1-5d80-447d-b8db-b8ea53661e88,DISK]] 2017-05-03 19:57:52,461 WARN [ResponseProcessor for block BP-1810172115-10.64.228.157-1478343078462:blk_1079559557_5836266] hdfs.DFSClient: Slow ReadProcessor read fields took 44774ms (threshold=30000ms); ack: seqno: 3150 reply: SUCCESS reply: SUCCESS reply: SUCCESS downstreamAckTimeNanos: 5109180 flag: 0 flag: 0 flag: 0, targets: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-120b90e1-5d80-447d-b8db-b8ea53661e88,DISK]] 2017-05-03 19:57:52,475 WARN [sync.0] hdfs.DFSClient: Slow waitForAckedSeqno took 30455ms (threshold=30000ms) 2017-05-03 19:57:52,475 WARN [sync.4] hdfs.DFSClient: Slow waitForAckedSeqno took 44796ms (threshold=30000ms) 2017-05-03 19:57:52,475 INFO [sync.1] wal.FSHLog: Slow sync cost: 16766 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-120b90e1-5d80-447d-b8db-b8ea53661e88,DISK]] 2017-05-03 19:57:52,476 WARN [B.defaultRpcServer.handler=6,queue=0,port=16020] ipc.RpcServer: (responseTooSlow): {"call":"Multi(org.apache.hadoop.hbase.protobuf.generated.ClientProtos$MultiRequest)","starttimems":1493841442020,"responsesize":8,"method":"Multi","processingtimems":30455,"client":"10.64.228.171:51374","queuetimems":0,"class":"HRegionServer"} 2017-05-03 19:57:52,476 INFO [sync.4] wal.FSHLog: Slow sync cost: 44796 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-120b90e1-5d80-447d-b8db-b8ea53661e88,DISK]] 2017-05-03 19:57:52,475 INFO [sync.0] wal.FSHLog: Slow sync cost: 30455 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-120b90e1-5d80-447d-b8db-b8ea53661e88,DISK]] 2017-05-03 19:57:52,476 WARN [B.defaultRpcServer.handler=22,queue=1,port=16020] ipc.RpcServer: (responseTooSlow): {"call":"Multi(org.apache.hadoop.hbase.protobuf.generated.ClientProtos$MultiRequest)","starttimems":1493841427662,"responsesize":603,"method":"Multi","processingtimems":44814,"client":"10.64.228.171:51374","queuetimems":0,"class":"HRegionServer"} 2017-05-03 19:57:52,476 WARN [B.defaultRpcServer.handler=0,queue=0,port=16020] ipc.RpcServer: (responseTooSlow): {"call":"Multi(org.apache.hadoop.hbase.protobuf.generated.ClientProtos$MultiRequest)","starttimems":1493841455708,"responsesize":8,"method":"Multi","processingtimems":16767,"client":"10.64.228.171:51374","queuetimems":0,"class":"HRegionServer"} 2017-05-03 19:57:52,995 INFO [sync.3] wal.FSHLog: Slow sync cost: 471 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-120b90e1-5d80-447d-b8db-b8ea53661e88,DISK]] 2017-05-03 19:57:53,003 INFO [sync.4] wal.FSHLog: Slow sync cost: 473 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-120b90e1-5d80-447d-b8db-b8ea53661e88,DISK]] 2017-05-03 19:57:54,238 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 129.71 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 19:57:54,786 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50151156, memsize=129.7 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/59b4b39503f44385a6b1ead86232b3a8 2017-05-03 19:57:54,820 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/59b4b39503f44385a6b1ead86232b3a8, entries=96668, sequenceid=50151156, filesize=8.2 M 2017-05-03 19:57:54,821 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~129.71 MB/136011784, currentsize=3.90 MB/4090376 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 583ms, sequenceid=50151156, compaction requested=true 2017-05-03 19:57:54,823 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 19:57:54,823 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 19:57:54,825 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=51259, currentSize=3334118328, freeSize=80385864, maxSize=3414504192, heapSize=3334118328, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 19:57:55,682 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into f0fef390c6694abab2dcb04145a952ab(size=8.1 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 19:57:55,682 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17676667900632607; duration=0sec 2017-05-03 19:57:56,388 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493841294995 with entries=1776, filesize=121.62 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493841476351 2017-05-03 19:58:27,307 INFO [sync.4] wal.FSHLog: Slow sync cost: 409 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-6e0a3279-95d2-4bff-ac72-ed8e01ad47ab,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK]] 2017-05-03 19:59:16,105 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.33 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 19:59:16,670 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50151594, memsize=128.3 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/6c145eb525b04ca48a7a3aad0063d92a 2017-05-03 19:59:16,684 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/6c145eb525b04ca48a7a3aad0063d92a, entries=96668, sequenceid=50151594, filesize=8.2 M 2017-05-03 19:59:16,685 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~128.33 MB/134561384, currentsize=120 KB/122880 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 580ms, sequenceid=50151594, compaction requested=true 2017-05-03 20:00:12,528 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493841476351 with entries=1485, filesize=121.85 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493841612491 2017-05-03 20:00:14,527 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=3.10 GB, freeSize=77.87 MB, max=3.18 GB, blockCount=51240, accesses=361298987, hits=182311599, hitRatio=50.46%, , cachingAccesses=355514976, cachingHits=182267461, cachingHitsRatio=51.27%, evictions=90780, evicted=172870939, evictedPerRun=1904.284423828125 2017-05-03 20:00:35,998 INFO [sync.2] wal.FSHLog: Slow sync cost: 172 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-0ad77247-babc-4c36-9cd0-c04ad47e0894,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-9a10c707-ebd5-4fae-8f82-b381f706fa57,DISK]] 2017-05-03 20:00:36,704 INFO [sync.1] wal.FSHLog: Slow sync cost: 246 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-0ad77247-babc-4c36-9cd0-c04ad47e0894,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-9a10c707-ebd5-4fae-8f82-b381f706fa57,DISK]] 2017-05-03 20:00:40,779 INFO [sync.0] wal.FSHLog: Slow sync cost: 353 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-0ad77247-babc-4c36-9cd0-c04ad47e0894,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-9a10c707-ebd5-4fae-8f82-b381f706fa57,DISK]] 2017-05-03 20:01:05,482 INFO [sync.2] wal.FSHLog: Slow sync cost: 352 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-0ad77247-babc-4c36-9cd0-c04ad47e0894,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-9a10c707-ebd5-4fae-8f82-b381f706fa57,DISK]] 2017-05-03 20:01:05,790 INFO [sync.3] wal.FSHLog: Slow sync cost: 152 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-0ad77247-babc-4c36-9cd0-c04ad47e0894,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-9a10c707-ebd5-4fae-8f82-b381f706fa57,DISK]] 2017-05-03 20:01:08,524 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.41 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:01:09,156 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50152003, memsize=128.4 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/c63376f189d244248b15481ac6eb8878 2017-05-03 20:01:09,169 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/c63376f189d244248b15481ac6eb8878, entries=96668, sequenceid=50152003, filesize=8.2 M 2017-05-03 20:01:09,171 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.41 MB/134650992, currentsize=7.28 MB/7637272 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 647ms, sequenceid=50152003, compaction requested=true 2017-05-03 20:01:09,174 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 20:01:09,174 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 20:01:09,177 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=50745, currentSize=3299989704, freeSize=114514488, maxSize=3414504192, heapSize=3299989704, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 20:01:10,057 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into cd8a73d72e724954bf1321e9b89e6f9e(size=8.1 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 20:01:10,057 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17676862251323411; duration=0sec 2017-05-03 20:02:34,901 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493841612491 with entries=1087, filesize=121.60 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493841754862 2017-05-03 20:02:56,630 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.50 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:02:57,208 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50152435, memsize=128.5 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/4c10a9037ecb4790bd699e392f08863d 2017-05-03 20:02:57,220 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/4c10a9037ecb4790bd699e392f08863d, entries=96668, sequenceid=50152435, filesize=8.2 M 2017-05-03 20:02:57,221 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~128.50 MB/134737880, currentsize=3.01 MB/3157400 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 591ms, sequenceid=50152435, compaction requested=true 2017-05-03 20:03:06,056 INFO [sync.1] wal.FSHLog: Slow sync cost: 221 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-5c91e4fa-90df-4f20-8217-049e5d671c16,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK]] 2017-05-03 20:03:11,226 INFO [sync.0] wal.FSHLog: Slow sync cost: 311 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-5c91e4fa-90df-4f20-8217-049e5d671c16,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK]] 2017-05-03 20:03:11,226 INFO [sync.1] wal.FSHLog: Slow sync cost: 233 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-5c91e4fa-90df-4f20-8217-049e5d671c16,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK]] 2017-05-03 20:03:26,307 INFO [sync.1] wal.FSHLog: Slow sync cost: 335 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-5c91e4fa-90df-4f20-8217-049e5d671c16,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK]] 2017-05-03 20:03:26,307 INFO [sync.2] wal.FSHLog: Slow sync cost: 306 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-5c91e4fa-90df-4f20-8217-049e5d671c16,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK]] 2017-05-03 20:04:20,219 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.46 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:04:20,793 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50152860, memsize=128.5 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/1cbe85a41aea488b9557201265b5a8f5 2017-05-03 20:04:20,805 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/1cbe85a41aea488b9557201265b5a8f5, entries=96668, sequenceid=50152860, filesize=8.2 M 2017-05-03 20:04:20,807 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~128.46 MB/134701632, currentsize=4.85 MB/5086560 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 588ms, sequenceid=50152860, compaction requested=true 2017-05-03 20:04:20,808 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 20:04:20,808 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 20:04:20,811 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49895, currentSize=3243751024, freeSize=170753168, maxSize=3414504192, heapSize=3243751024, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 20:04:21,671 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into 54191ab391ce4b91bc13a6e8f385d987(size=8.1 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 20:04:21,671 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17677053885611322; duration=0sec 2017-05-03 20:04:35,628 INFO [sync.1] wal.FSHLog: Slow sync cost: 229 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-5c91e4fa-90df-4f20-8217-049e5d671c16,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK]] 2017-05-03 20:04:46,992 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493841754862 with entries=2416, filesize=121.74 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493841886961 2017-05-03 20:04:54,932 INFO [aps-hadoop5,16020,1493618413009_ChoreService_1] regionserver.HRegionServer: aps-hadoop5,16020,1493618413009-MemstoreFlusherChore requesting flush for region CampaignPerformance,66666666,1478428211809.82260485c06f9076b59f25064eeb47cc. after a delay of 5176 2017-05-03 20:05:00,109 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for CampaignPerformance,66666666,1478428211809.82260485c06f9076b59f25064eeb47cc., current region memstore size 36.26 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:05:00,292 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=23835716, memsize=36.3 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/CampaignPerformance/82260485c06f9076b59f25064eeb47cc/.tmp/53aa736b71404b8095f7d4ef4c23f0fa 2017-05-03 20:05:00,303 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/CampaignPerformance/82260485c06f9076b59f25064eeb47cc/info/53aa736b71404b8095f7d4ef4c23f0fa, entries=28155, sequenceid=23835716, filesize=319.5 K 2017-05-03 20:05:00,305 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~36.26 MB/38023944, currentsize=0 B/0 for region CampaignPerformance,66666666,1478428211809.82260485c06f9076b59f25064eeb47cc. in 196ms, sequenceid=23835716, compaction requested=true 2017-05-03 20:05:00,332 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region CampaignPerformance,66666666,1478428211809.82260485c06f9076b59f25064eeb47cc. 2017-05-03 20:05:00,332 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of CampaignPerformance,66666666,1478428211809.82260485c06f9076b59f25064eeb47cc. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/CampaignPerformance/82260485c06f9076b59f25064eeb47cc/.tmp, totalSize=1.4 M 2017-05-03 20:05:00,334 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49897, currentSize=3243834792, freeSize=170669400, maxSize=3414504192, heapSize=3243834792, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 20:05:00,567 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 (all) file(s) in info of CampaignPerformance,66666666,1478428211809.82260485c06f9076b59f25064eeb47cc. into 3588a3c4db1c4750a37c6999d5cc16ae(size=764.4 K), total size for store is 764.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 20:05:00,567 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=CampaignPerformance,66666666,1478428211809.82260485c06f9076b59f25064eeb47cc., storeName=info, fileCount=3, fileSize=1.4 M, priority=7, time=17677093409500470; duration=0sec 2017-05-03 20:05:14,527 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=3.02 GB, freeSize=162.82 MB, max=3.18 GB, blockCount=49895, accesses=361356160, hits=182341827, hitRatio=50.46%, , cachingAccesses=355570124, cachingHits=182297600, cachingHitsRatio=51.27%, evictions=90814, evicted=172897294, evictedPerRun=1903.8616943359375 2017-05-03 20:05:30,655 INFO [RS_OPEN_META-aps-hadoop5:16020-0-MetaLogRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009..meta.1493838330469.meta with entries=0, filesize=91 B; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009..meta.1493841930617.meta 2017-05-03 20:05:30,656 INFO [RS_OPEN_META-aps-hadoop5:16020-0-MetaLogRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009..meta.1493838330469.meta to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009..meta.1493838330469.meta 2017-05-03 20:06:08,810 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.44 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:06:09,310 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50153268, memsize=128.4 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/d8f4fe332bcb4f11878533c2cb5e5325 2017-05-03 20:06:09,322 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/d8f4fe332bcb4f11878533c2cb5e5325, entries=96668, sequenceid=50153268, filesize=8.2 M 2017-05-03 20:06:09,324 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.44 MB/134676400, currentsize=120.59 KB/123480 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 514ms, sequenceid=50153268, compaction requested=true 2017-05-03 20:07:27,594 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493841886961 with entries=539, filesize=123.16 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493842047442 2017-05-03 20:07:27,594 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493838158507 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493838158507 2017-05-03 20:07:27,604 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493838327110 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493838327110 2017-05-03 20:07:54,229 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.55 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:07:54,795 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50153711, memsize=128.6 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/705e2fef87cd4fbe8dcde24d700a6be4 2017-05-03 20:07:54,806 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/705e2fef87cd4fbe8dcde24d700a6be4, entries=96668, sequenceid=50153711, filesize=8.2 M 2017-05-03 20:07:54,807 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.55 MB/134794680, currentsize=3.09 MB/3242896 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 578ms, sequenceid=50153711, compaction requested=true 2017-05-03 20:07:54,808 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 20:07:54,808 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 20:07:54,810 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49895, currentSize=3243777096, freeSize=170727096, maxSize=3414504192, heapSize=3243777096, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 20:07:55,646 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into ba21a74786824dee8583c7cb1d95e981(size=8.1 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 20:07:55,646 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17677267885565529; duration=0sec 2017-05-03 20:09:17,402 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.06 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:09:18,004 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50154125, memsize=128.1 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/24e71f24522947229d754b97b0a8ef39 2017-05-03 20:09:18,015 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/24e71f24522947229d754b97b0a8ef39, entries=96668, sequenceid=50154125, filesize=8.2 M 2017-05-03 20:09:18,017 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~128.06 MB/134280824, currentsize=7.90 MB/8285016 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 615ms, sequenceid=50154125, compaction requested=true 2017-05-03 20:09:42,004 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493842047442 with entries=537, filesize=121.76 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493842181974 2017-05-03 20:10:14,527 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=3.02 GB, freeSize=162.82 MB, max=3.18 GB, blockCount=49895, accesses=361364683, hits=182349388, hitRatio=50.46%, , cachingAccesses=355577685, cachingHits=182305161, cachingHitsRatio=51.27%, evictions=90844, evicted=172897294, evictedPerRun=1903.23291015625 2017-05-03 20:11:00,441 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 129.71 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:11:00,992 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50154548, memsize=129.7 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/a3005d1fb8274919b6e2645a79816cbf 2017-05-03 20:11:01,003 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/a3005d1fb8274919b6e2645a79816cbf, entries=96668, sequenceid=50154548, filesize=8.2 M 2017-05-03 20:11:01,004 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~129.71 MB/136011784, currentsize=3.90 MB/4090376 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 564ms, sequenceid=50154548, compaction requested=true 2017-05-03 20:11:01,005 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 20:11:01,005 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 20:11:01,007 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49895, currentSize=3243777096, freeSize=170727096, maxSize=3414504192, heapSize=3243777096, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 20:11:01,914 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into c7893693186843d58cb29b09d018c5d9(size=8.2 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 20:11:01,914 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17677454082211944; duration=0sec 2017-05-03 20:11:50,754 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493842181974 with entries=1781, filesize=122.62 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493842310717 2017-05-03 20:11:56,423 INFO [sync.0] wal.FSHLog: Slow sync cost: 196 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-d7a093e4-6437-4935-b4a2-0decf44fabea,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-9ba4f08a-d996-4490-b27d-6c8ca9a67152,DISK]] 2017-05-03 20:12:19,427 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.33 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:12:19,946 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50154986, memsize=128.3 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/7359f7f4731647b782953825c414125c 2017-05-03 20:12:19,960 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/7359f7f4731647b782953825c414125c, entries=96668, sequenceid=50154986, filesize=8.2 M 2017-05-03 20:12:19,961 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.33 MB/134561384, currentsize=120 KB/122880 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 534ms, sequenceid=50154986, compaction requested=true 2017-05-03 20:12:34,932 INFO [aps-hadoop5,16020,1493618413009_ChoreService_1] regionserver.HRegionServer: aps-hadoop5,16020,1493618413009-MemstoreFlusherChore requesting flush for region CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. after a delay of 10671 2017-05-03 20:12:44,932 INFO [aps-hadoop5,16020,1493618413009_ChoreService_1] regionserver.HRegionServer: aps-hadoop5,16020,1493618413009-MemstoreFlusherChore requesting flush for region CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. after a delay of 16854 2017-05-03 20:12:45,603 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494., current region memstore size 104.37 MB, and 3/12 column families' memstores are being flushed. 2017-05-03 20:12:45,603 INFO [MemStoreFlusher.0] regionserver.HRegion: Flushing Column Family: info which was occupying 25.41 MB of memstore. 2017-05-03 20:12:45,603 INFO [MemStoreFlusher.0] regionserver.HRegion: Flushing Column Family: Em which was occupying 50.14 MB of memstore. 2017-05-03 20:12:45,603 INFO [MemStoreFlusher.0] regionserver.HRegion: Flushing Column Family: Mb which was occupying 28.82 MB of memstore. 2017-05-03 20:12:45,769 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=113673934, memsize=50.1 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/CampaignGoalMDC/6cf8953db275a43b98ab944d2d614494/.tmp/27f15c49053e4b0590d304c999a0ebb6 2017-05-03 20:12:45,870 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=113673934, memsize=28.8 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/CampaignGoalMDC/6cf8953db275a43b98ab944d2d614494/.tmp/e85568721df24ca19b9bdfc19209f058 2017-05-03 20:12:45,964 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=113673934, memsize=25.4 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/CampaignGoalMDC/6cf8953db275a43b98ab944d2d614494/.tmp/311bb39f20d141bf8d4e7e1fadb78d0d 2017-05-03 20:12:45,975 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/CampaignGoalMDC/6cf8953db275a43b98ab944d2d614494/Em/27f15c49053e4b0590d304c999a0ebb6, entries=1023, sequenceid=113673934, filesize=20.9 K 2017-05-03 20:12:45,983 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/CampaignGoalMDC/6cf8953db275a43b98ab944d2d614494/Mb/e85568721df24ca19b9bdfc19209f058, entries=1100, sequenceid=113673934, filesize=20.4 K 2017-05-03 20:12:45,991 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/CampaignGoalMDC/6cf8953db275a43b98ab944d2d614494/info/311bb39f20d141bf8d4e7e1fadb78d0d, entries=747, sequenceid=113673934, filesize=15.1 K 2017-05-03 20:12:45,992 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~104.37 MB/109434680, currentsize=0 B/0 for region CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. in 389ms, sequenceid=113673934, compaction requested=true 2017-05-03 20:12:45,994 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on Em in region CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. 2017-05-03 20:12:45,994 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in Em of CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/CampaignGoalMDC/6cf8953db275a43b98ab944d2d614494/.tmp, totalSize=2.6 M 2017-05-03 20:12:45,996 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49895, currentSize=3243777096, freeSize=170727096, maxSize=3414504192, heapSize=3243777096, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 20:12:46,328 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 (all) file(s) in Em of CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. into dccfac420fc24678a8b09ed0cd83c5f9(size=2.6 M), total size for store is 2.6 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 20:12:46,328 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494., storeName=Em, fileCount=3, fileSize=2.6 M, priority=7, time=17677559070930496; duration=0sec 2017-05-03 20:13:39,166 INFO [sync.3] wal.FSHLog: Slow sync cost: 868 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-d7a093e4-6437-4935-b4a2-0decf44fabea,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-9ba4f08a-d996-4490-b27d-6c8ca9a67152,DISK]] 2017-05-03 20:13:59,166 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493842310717 with entries=1055, filesize=122.05 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493842439135 2017-05-03 20:13:59,167 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493838729067 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493838729067 2017-05-03 20:13:59,178 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493838895074 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493838895074 2017-05-03 20:13:59,187 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493839033370 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493839033370 2017-05-03 20:14:00,557 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.41 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:14:01,129 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50155395, memsize=128.4 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/a7b389e392e74327ad2da48e9d64371f 2017-05-03 20:14:01,142 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/a7b389e392e74327ad2da48e9d64371f, entries=96668, sequenceid=50155395, filesize=8.2 M 2017-05-03 20:14:01,143 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~128.41 MB/134650992, currentsize=6.81 MB/7142320 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 586ms, sequenceid=50155395, compaction requested=true 2017-05-03 20:14:01,145 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 20:14:01,145 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 20:14:01,147 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49895, currentSize=3243777096, freeSize=170727096, maxSize=3414504192, heapSize=3243777096, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 20:14:02,045 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into a1721827693a42c2bdd69cb1e91bb32c(size=8.1 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 20:14:02,045 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17677634222045015; duration=0sec 2017-05-03 20:15:14,527 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=3.02 GB, freeSize=162.82 MB, max=3.18 GB, blockCount=49895, accesses=361369388, hits=182352005, hitRatio=50.46%, , cachingAccesses=355580302, cachingHits=182307778, cachingHitsRatio=51.27%, evictions=90874, evicted=172897294, evictedPerRun=1902.6046142578125 2017-05-03 20:15:40,965 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.50 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:15:41,478 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50155827, memsize=128.5 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/0fb25b73792743ae8b0976c510bbac44 2017-05-03 20:15:41,498 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/0fb25b73792743ae8b0976c510bbac44, entries=96668, sequenceid=50155827, filesize=8.2 M 2017-05-03 20:15:41,512 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~128.50 MB/134737880, currentsize=4.87 MB/5103928 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 547ms, sequenceid=50155827, compaction requested=true 2017-05-03 20:16:07,914 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493842439135 with entries=917, filesize=121.65 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493842567860 2017-05-03 20:16:38,667 INFO [sync.2] wal.FSHLog: Slow sync cost: 169 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-05-03 20:17:02,232 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.46 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:17:02,890 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50156252, memsize=128.5 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/6a555598038748068b0205c0eef2bdf1 2017-05-03 20:17:02,902 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/6a555598038748068b0205c0eef2bdf1, entries=96668, sequenceid=50156252, filesize=8.2 M 2017-05-03 20:17:02,903 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.46 MB/134701632, currentsize=4.85 MB/5086560 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 671ms, sequenceid=50156252, compaction requested=true 2017-05-03 20:17:02,904 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 20:17:02,904 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 20:17:02,908 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49895, currentSize=3243777096, freeSize=170727096, maxSize=3414504192, heapSize=3243777096, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 20:17:03,800 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into 3289f21e3b424407a9a0881d7aa0b189(size=8.1 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 20:17:03,800 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17677815981692854; duration=0sec 2017-05-03 20:17:24,741 INFO [sync.4] wal.FSHLog: Slow sync cost: 219 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-05-03 20:18:16,280 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493842567860 with entries=1657, filesize=122.05 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493842696249 2017-05-03 20:18:41,861 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.44 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:18:42,366 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50156660, memsize=128.4 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/6dacfa4a0bec49acbfb769415a9d6db7 2017-05-03 20:18:42,379 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/6dacfa4a0bec49acbfb769415a9d6db7, entries=96668, sequenceid=50156660, filesize=8.2 M 2017-05-03 20:18:42,380 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.44 MB/134676400, currentsize=120.59 KB/123480 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 519ms, sequenceid=50156660, compaction requested=true 2017-05-03 20:18:50,632 INFO [sync.1] wal.FSHLog: Slow sync cost: 317 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-1940f918-25a9-4136-8034-e6fd0972e5a2,DISK]] 2017-05-03 20:20:14,527 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=3.02 GB, freeSize=162.82 MB, max=3.18 GB, blockCount=49895, accesses=361375231, hits=182356886, hitRatio=50.46%, , cachingAccesses=355585183, cachingHits=182312659, cachingHitsRatio=51.27%, evictions=90904, evicted=172897294, evictedPerRun=1901.976806640625 2017-05-03 20:20:21,543 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.55 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:20:22,079 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50157103, memsize=128.6 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/3f503003c14f437e8a253e7e5c6b039e 2017-05-03 20:20:22,091 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/3f503003c14f437e8a253e7e5c6b039e, entries=96668, sequenceid=50157103, filesize=8.2 M 2017-05-03 20:20:22,092 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.55 MB/134794680, currentsize=4.66 MB/4881472 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 549ms, sequenceid=50157103, compaction requested=true 2017-05-03 20:20:22,093 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 20:20:22,094 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 20:20:22,096 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49895, currentSize=3243777096, freeSize=170727096, maxSize=3414504192, heapSize=3243777096, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 20:20:22,964 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into fa34f933f7c44de9830289512790ded9(size=8.2 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 20:20:22,964 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17678015170832633; duration=0sec 2017-05-03 20:20:24,040 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493842696249 with entries=1193, filesize=121.98 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493842824002 2017-05-03 20:21:14,932 INFO [aps-hadoop5,16020,1493618413009_ChoreService_1] regionserver.HRegionServer: aps-hadoop5,16020,1493618413009-MemstoreFlusherChore requesting flush for region CampaignSummaryDetailGeographyByCityMDC,cccccccc,1478428063068.2860d686a4589ad57488c2b034722a16. after a delay of 13497 2017-05-03 20:21:24,932 INFO [aps-hadoop5,16020,1493618413009_ChoreService_1] regionserver.HRegionServer: aps-hadoop5,16020,1493618413009-MemstoreFlusherChore requesting flush for region CampaignSummaryDetailGeographyByCityMDC,cccccccc,1478428063068.2860d686a4589ad57488c2b034722a16. after a delay of 10954 2017-05-03 20:21:28,429 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for CampaignSummaryDetailGeographyByCityMDC,cccccccc,1478428063068.2860d686a4589ad57488c2b034722a16., current region memstore size 19.14 MB, and 1/2 column families' memstores are being flushed. 2017-05-03 20:21:28,429 INFO [MemStoreFlusher.0] regionserver.HRegion: Flushing Column Family: info which was occupying 19.14 MB of memstore. 2017-05-03 20:21:28,558 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=35230919, memsize=19.1 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/CampaignSummaryDetailGeographyByCityMDC/2860d686a4589ad57488c2b034722a16/.tmp/d1dd656276e54552b7e0ed2a010c012e 2017-05-03 20:21:28,572 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/CampaignSummaryDetailGeographyByCityMDC/2860d686a4589ad57488c2b034722a16/info/d1dd656276e54552b7e0ed2a010c012e, entries=47264, sequenceid=35230919, filesize=754.0 K 2017-05-03 20:21:28,573 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~19.14 MB/20070152, currentsize=0 B/0 for region CampaignSummaryDetailGeographyByCityMDC,cccccccc,1478428063068.2860d686a4589ad57488c2b034722a16. in 144ms, sequenceid=35230919, compaction requested=true 2017-05-03 20:21:28,578 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region CampaignSummaryDetailGeographyByCityMDC,cccccccc,1478428063068.2860d686a4589ad57488c2b034722a16. 2017-05-03 20:21:28,578 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of CampaignSummaryDetailGeographyByCityMDC,cccccccc,1478428063068.2860d686a4589ad57488c2b034722a16. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/CampaignSummaryDetailGeographyByCityMDC/2860d686a4589ad57488c2b034722a16/.tmp, totalSize=114.5 M 2017-05-03 20:21:28,582 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49895, currentSize=3243777096, freeSize=170727096, maxSize=3414504192, heapSize=3243777096, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 20:21:39,175 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.06 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:21:39,700 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50157517, memsize=128.1 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/2b7ec319c5ad40f0ab40af4eb6d689f7 2017-05-03 20:21:39,715 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/2b7ec319c5ad40f0ab40af4eb6d689f7, entries=96668, sequenceid=50157517, filesize=8.2 M 2017-05-03 20:21:39,716 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.06 MB/134280824, currentsize=7.43 MB/7790208 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 541ms, sequenceid=50157517, compaction requested=true 2017-05-03 20:21:40,284 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 (all) file(s) in info of CampaignSummaryDetailGeographyByCityMDC,cccccccc,1478428063068.2860d686a4589ad57488c2b034722a16. into 7819c4c0ca8d4c7f83861fb1bff0b0c8(size=113.0 M), total size for store is 113.0 M. This selection was in queue for 0sec, and took 11sec to execute. 2017-05-03 20:21:40,286 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=CampaignSummaryDetailGeographyByCityMDC,cccccccc,1478428063068.2860d686a4589ad57488c2b034722a16., storeName=info, fileCount=3, fileSize=114.5 M, priority=7, time=17678081655425053; duration=11sec 2017-05-03 20:22:02,844 INFO [sync.2] wal.FSHLog: Slow sync cost: 138 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-9ba4f08a-d996-4490-b27d-6c8ca9a67152,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-6e0a3279-95d2-4bff-ac72-ed8e01ad47ab,DISK]] 2017-05-03 20:22:30,262 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493842824002 with entries=1021, filesize=122.02 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493842950190 2017-05-03 20:22:30,263 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493839179034 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493839179034 2017-05-03 20:22:30,284 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493839339615 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493839339615 2017-05-03 20:22:30,306 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493839526858 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493839526858 2017-05-03 20:22:30,331 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493839667391 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493839667391 2017-05-03 20:22:30,352 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493839808943 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493839808943 2017-05-03 20:22:30,374 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493839954228 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493839954228 2017-05-03 20:23:19,323 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 129.71 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:23:19,834 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50157940, memsize=129.7 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/d28e2c8d59bb4a7690056fc26fe0f88b 2017-05-03 20:23:19,848 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/d28e2c8d59bb4a7690056fc26fe0f88b, entries=96668, sequenceid=50157940, filesize=8.2 M 2017-05-03 20:23:19,849 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~129.71 MB/136011784, currentsize=3.90 MB/4090376 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 527ms, sequenceid=50157940, compaction requested=true 2017-05-03 20:23:19,850 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 20:23:19,850 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 20:23:19,853 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49890, currentSize=3243728576, freeSize=170775616, maxSize=3414504192, heapSize=3243728576, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 20:23:20,702 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into 75890bf074144ab58ae2bffc7c0eccf9(size=8.1 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 20:23:20,702 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17678192927633928; duration=0sec 2017-05-03 20:24:37,459 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493842950190 with entries=811, filesize=121.83 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493843077419 2017-05-03 20:24:37,841 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.33 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:24:38,354 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50158378, memsize=128.3 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/13010d69f2bf4a4da0d071d83c24ca69 2017-05-03 20:24:38,391 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/13010d69f2bf4a4da0d071d83c24ca69, entries=96668, sequenceid=50158378, filesize=8.2 M 2017-05-03 20:24:38,392 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~128.33 MB/134561736, currentsize=120 KB/122880 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 551ms, sequenceid=50158378, compaction requested=true 2017-05-03 20:25:14,527 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=3.02 GB, freeSize=162.86 MB, max=3.18 GB, blockCount=49890, accesses=361387289, hits=182357908, hitRatio=50.46%, , cachingAccesses=355586206, cachingHits=182313677, cachingHitsRatio=51.27%, evictions=90934, evicted=172897304, evictedPerRun=1901.3494873046875 2017-05-03 20:26:19,692 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.41 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:26:20,248 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50158787, memsize=128.4 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/a472f3ddaf274cf1ac50efc7ba8f5ac1 2017-05-03 20:26:20,260 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/a472f3ddaf274cf1ac50efc7ba8f5ac1, entries=96668, sequenceid=50158787, filesize=8.2 M 2017-05-03 20:26:20,261 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.41 MB/134651696, currentsize=7.28 MB/7637272 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 569ms, sequenceid=50158787, compaction requested=true 2017-05-03 20:26:20,263 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 20:26:20,263 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 20:26:20,266 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49890, currentSize=3243728576, freeSize=170775616, maxSize=3414504192, heapSize=3243728576, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 20:26:21,193 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into c5fcc5556630464eaefaa74807a69648(size=8.1 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 20:26:21,193 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17678373340240962; duration=0sec 2017-05-03 20:26:42,962 INFO [sync.2] wal.FSHLog: Slow sync cost: 161 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-01d24dbb-ac60-4a67-8c97-11733f4d3f3b,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-1940f918-25a9-4136-8034-e6fd0972e5a2,DISK]] 2017-05-03 20:26:46,723 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493843077419 with entries=845, filesize=121.67 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493843206691 2017-05-03 20:27:37,713 INFO [sync.1] wal.FSHLog: Slow sync cost: 138 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-5c91e4fa-90df-4f20-8217-049e5d671c16,DISK]] 2017-05-03 20:28:00,692 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.50 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:28:01,359 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50159219, memsize=128.5 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/546ebdab50994c64aa36f056b8318f31 2017-05-03 20:28:01,371 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/546ebdab50994c64aa36f056b8318f31, entries=96668, sequenceid=50159219, filesize=8.2 M 2017-05-03 20:28:01,372 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~128.50 MB/134738408, currentsize=4.87 MB/5103928 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 680ms, sequenceid=50159219, compaction requested=true 2017-05-03 20:28:56,221 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493843206691 with entries=2673, filesize=121.72 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493843336186 2017-05-03 20:29:20,925 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.46 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:29:21,456 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50159644, memsize=128.5 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/19683265fbfe4aa6b06ba8da9a82ec04 2017-05-03 20:29:21,472 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/19683265fbfe4aa6b06ba8da9a82ec04, entries=96668, sequenceid=50159644, filesize=8.2 M 2017-05-03 20:29:21,473 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~128.46 MB/134702336, currentsize=4.85 MB/5086560 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 548ms, sequenceid=50159644, compaction requested=true 2017-05-03 20:29:21,475 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 20:29:21,475 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 20:29:21,478 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49890, currentSize=3243728576, freeSize=170775616, maxSize=3414504192, heapSize=3243728576, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 20:29:22,396 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into 222d4fd258a54f84bcaa512058e3d259(size=8.1 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 20:29:22,396 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17678554552232165; duration=0sec 2017-05-03 20:30:14,527 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=3.02 GB, freeSize=162.86 MB, max=3.18 GB, blockCount=49890, accesses=361395511, hits=182364206, hitRatio=50.46%, , cachingAccesses=355592504, cachingHits=182319975, cachingHitsRatio=51.27%, evictions=90964, evicted=172897304, evictedPerRun=1900.722412109375 2017-05-03 20:31:01,072 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.44 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:31:01,712 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50160052, memsize=128.4 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/27bbe692daa147c9a43c1e74cdbba16f 2017-05-03 20:31:01,726 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/27bbe692daa147c9a43c1e74cdbba16f, entries=96668, sequenceid=50160052, filesize=8.2 M 2017-05-03 20:31:01,727 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~128.44 MB/134676928, currentsize=120.76 KB/123656 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 655ms, sequenceid=50160052, compaction requested=true 2017-05-03 20:31:24,768 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493843336186 with entries=586, filesize=122.41 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493843484732 2017-05-03 20:31:52,641 INFO [sync.2] wal.FSHLog: Slow sync cost: 115 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-6e0a3279-95d2-4bff-ac72-ed8e01ad47ab,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]] 2017-05-03 20:32:41,315 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.55 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:32:41,843 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50160495, memsize=128.6 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/50db6774d84f414b952fc5cd1f361fc9 2017-05-03 20:32:41,855 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/50db6774d84f414b952fc5cd1f361fc9, entries=96668, sequenceid=50160495, filesize=8.2 M 2017-05-03 20:32:41,856 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~128.55 MB/134795384, currentsize=5.52 MB/5791320 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 541ms, sequenceid=50160495, compaction requested=true 2017-05-03 20:32:41,858 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 20:32:41,858 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 20:32:41,861 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49890, currentSize=3243728576, freeSize=170775616, maxSize=3414504192, heapSize=3243728576, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 20:32:42,729 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into dcc5686e9da5491cb51edc5e8d6c11f6(size=8.1 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 20:32:42,729 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17678754934927475; duration=0sec 2017-05-03 20:33:32,094 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493843484732 with entries=543, filesize=121.82 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493843612046 2017-05-03 20:33:59,871 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.06 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:34:01,688 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50160909, memsize=128.1 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/38c7e5325b5c46d8b232080ed28ec290 2017-05-03 20:34:01,700 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/38c7e5325b5c46d8b232080ed28ec290, entries=96668, sequenceid=50160909, filesize=8.2 M 2017-05-03 20:34:01,702 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.06 MB/134281528, currentsize=9.91 MB/10388840 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 1831ms, sequenceid=50160909, compaction requested=true 2017-05-03 20:34:04,531 INFO [sync.1] wal.FSHLog: Slow sync cost: 252 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-d7a093e4-6437-4935-b4a2-0decf44fabea,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-9ba4f08a-d996-4490-b27d-6c8ca9a67152,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-5f5eea04-33fc-449d-8bb6-d372303a99c7,DISK]] 2017-05-03 20:35:14,527 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=3.02 GB, freeSize=162.86 MB, max=3.18 GB, blockCount=49890, accesses=361398706, hits=182366439, hitRatio=50.46%, , cachingAccesses=355594737, cachingHits=182322208, cachingHitsRatio=51.27%, evictions=90994, evicted=172897304, evictedPerRun=1900.095703125 2017-05-03 20:35:39,547 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493843612046 with entries=1128, filesize=122.27 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493843739506 2017-05-03 20:35:40,073 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 129.71 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:35:40,617 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50161332, memsize=129.7 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/5af7dbaf71d9423abe6d69ad808f1b7e 2017-05-03 20:35:40,630 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/5af7dbaf71d9423abe6d69ad808f1b7e, entries=96668, sequenceid=50161332, filesize=8.2 M 2017-05-03 20:35:40,631 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~129.71 MB/136012312, currentsize=3.90 MB/4090552 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 558ms, sequenceid=50161332, compaction requested=true 2017-05-03 20:35:40,633 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 20:35:40,633 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 20:35:40,636 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49890, currentSize=3243728576, freeSize=170775616, maxSize=3414504192, heapSize=3243728576, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 20:35:41,490 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into 5d2fef8868384960943e42d64584a9c9(size=8.1 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 20:35:41,490 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17678933710595727; duration=0sec 2017-05-03 20:36:13,532 INFO [sync.3] wal.FSHLog: Slow sync cost: 112 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-0ad77247-babc-4c36-9cd0-c04ad47e0894,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK]] 2017-05-03 20:36:36,325 INFO [sync.1] wal.FSHLog: Slow sync cost: 164 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-0ad77247-babc-4c36-9cd0-c04ad47e0894,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK]] 2017-05-03 20:36:57,452 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.33 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:36:58,048 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50161770, memsize=128.3 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/c6278de13dac430084f93759410aa8c5 2017-05-03 20:36:58,062 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/c6278de13dac430084f93759410aa8c5, entries=96668, sequenceid=50161770, filesize=8.2 M 2017-05-03 20:36:58,063 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.33 MB/134562088, currentsize=120 KB/122880 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 611ms, sequenceid=50161770, compaction requested=true 2017-05-03 20:37:24,932 INFO [aps-hadoop5,16020,1493618413009_ChoreService_1] regionserver.HRegionServer: aps-hadoop5,16020,1493618413009-MemstoreFlusherChore requesting flush for region CampaignSummaryFactsMDC,66666666,1478427948111.ad61a965e252fb9f6854dea225e4e2e3. after a delay of 15002 2017-05-03 20:37:34,932 INFO [aps-hadoop5,16020,1493618413009_ChoreService_1] regionserver.HRegionServer: aps-hadoop5,16020,1493618413009-MemstoreFlusherChore requesting flush for region CampaignSummaryFactsMDC,66666666,1478427948111.ad61a965e252fb9f6854dea225e4e2e3. after a delay of 20954 2017-05-03 20:37:39,934 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for CampaignSummaryFactsMDC,66666666,1478427948111.ad61a965e252fb9f6854dea225e4e2e3., current region memstore size 215.53 KB, and 2/10 column families' memstores are being flushed. 2017-05-03 20:37:39,934 INFO [MemStoreFlusher.0] regionserver.HRegion: Flushing Column Family: info which was occupying 47.60 KB of memstore. 2017-05-03 20:37:39,934 INFO [MemStoreFlusher.0] regionserver.HRegion: Flushing Column Family: Em which was occupying 168.73 KB of memstore. 2017-05-03 20:37:40,005 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=19136, memsize=168.3 K, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/CampaignSummaryFactsMDC/ad61a965e252fb9f6854dea225e4e2e3/.tmp/08739de73b7a4c368ab3372e50a7d527 2017-05-03 20:37:40,048 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=19136, memsize=47.2 K, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/CampaignSummaryFactsMDC/ad61a965e252fb9f6854dea225e4e2e3/.tmp/02b58fd1037d41879e65e4730f3e93a9 2017-05-03 20:37:40,063 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/CampaignSummaryFactsMDC/ad61a965e252fb9f6854dea225e4e2e3/Em/08739de73b7a4c368ab3372e50a7d527, entries=54, sequenceid=19136, filesize=5.3 K 2017-05-03 20:37:40,075 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/CampaignSummaryFactsMDC/ad61a965e252fb9f6854dea225e4e2e3/info/02b58fd1037d41879e65e4730f3e93a9, entries=15, sequenceid=19136, filesize=5.0 K 2017-05-03 20:37:40,077 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~215.53 KB/220704, currentsize=0 B/0 for region CampaignSummaryFactsMDC,66666666,1478427948111.ad61a965e252fb9f6854dea225e4e2e3. in 142ms, sequenceid=19136, compaction requested=true 2017-05-03 20:37:40,078 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on Em in region CampaignSummaryFactsMDC,66666666,1478427948111.ad61a965e252fb9f6854dea225e4e2e3. 2017-05-03 20:37:40,078 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in Em of CampaignSummaryFactsMDC,66666666,1478427948111.ad61a965e252fb9f6854dea225e4e2e3. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/CampaignSummaryFactsMDC/ad61a965e252fb9f6854dea225e4e2e3/.tmp, totalSize=32.0 K 2017-05-03 20:37:40,081 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49890, currentSize=3243728576, freeSize=170775616, maxSize=3414504192, heapSize=3243728576, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 20:37:40,191 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 (all) file(s) in Em of CampaignSummaryFactsMDC,66666666,1478427948111.ad61a965e252fb9f6854dea225e4e2e3. into 8814e4c06b614a099aa983ecefea8b05(size=21.3 K), total size for store is 21.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 20:37:40,191 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=CampaignSummaryFactsMDC,66666666,1478427948111.ad61a965e252fb9f6854dea225e4e2e3., storeName=Em, fileCount=3, fileSize=32.0 K, priority=7, time=17679053155562591; duration=0sec 2017-05-03 20:37:44,491 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493843739506 with entries=2130, filesize=121.97 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493843864458 2017-05-03 20:37:44,492 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493840168512 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493840168512 2017-05-03 20:37:44,502 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493840314059 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493840314059 2017-05-03 20:37:44,511 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493840460739 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493840460739 2017-05-03 20:37:44,521 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493840601082 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493840601082 2017-05-03 20:37:44,532 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493840742329 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493840742329 2017-05-03 20:37:44,542 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493840879923 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493840879923 2017-05-03 20:37:44,551 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493841017842 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493841017842 2017-05-03 20:37:44,560 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493841156733 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493841156733 2017-05-03 20:37:44,568 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493841294995 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493841294995 2017-05-03 20:37:44,578 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493841476351 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493841476351 2017-05-03 20:37:44,595 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493841612491 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493841612491 2017-05-03 20:37:44,606 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493841754862 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493841754862 2017-05-03 20:37:44,616 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493841886961 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493841886961 2017-05-03 20:37:44,624 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493842047442 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493842047442 2017-05-03 20:38:36,934 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.41 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:38:37,471 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50162179, memsize=128.4 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/bd6bb6b4f2be48c4b205ea21bc95035f 2017-05-03 20:38:37,486 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/bd6bb6b4f2be48c4b205ea21bc95035f, entries=96668, sequenceid=50162179, filesize=8.2 M 2017-05-03 20:38:37,490 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.41 MB/134651696, currentsize=6.81 MB/7142320 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 556ms, sequenceid=50162179, compaction requested=true 2017-05-03 20:38:37,492 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 20:38:37,492 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 20:38:37,495 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49890, currentSize=3243658544, freeSize=170845648, maxSize=3414504192, heapSize=3243658544, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 20:38:38,352 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into 9cd20979fae148099399bc1d2c328868(size=8.1 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 20:38:38,352 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17679110569434827; duration=0sec 2017-05-03 20:39:09,483 INFO [sync.2] wal.FSHLog: Slow sync cost: 1371 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-0ad77247-babc-4c36-9cd0-c04ad47e0894,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-5c91e4fa-90df-4f20-8217-049e5d671c16,DISK]] 2017-05-03 20:39:52,132 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493843864458 with entries=965, filesize=122.89 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493843992100 2017-05-03 20:40:14,527 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=3.02 GB, freeSize=162.93 MB, max=3.18 GB, blockCount=49890, accesses=361402554, hits=182368360, hitRatio=50.46%, , cachingAccesses=355596658, cachingHits=182324127, cachingHitsRatio=51.27%, evictions=91024, evicted=172897306, evictedPerRun=1899.469482421875 2017-05-03 20:40:16,628 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.50 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:40:17,162 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50162611, memsize=128.5 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/fb9c04895f7d4c3091bbfdf941ae5f87 2017-05-03 20:40:17,188 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/fb9c04895f7d4c3091bbfdf941ae5f87, entries=96668, sequenceid=50162611, filesize=8.2 M 2017-05-03 20:40:17,189 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.50 MB/134738408, currentsize=3.01 MB/3157400 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 562ms, sequenceid=50162611, compaction requested=true 2017-05-03 20:40:25,458 INFO [sync.4] wal.FSHLog: Slow sync cost: 137 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-d7a093e4-6437-4935-b4a2-0decf44fabea,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK]] 2017-05-03 20:41:35,674 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.46 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:41:36,181 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50163036, memsize=128.5 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/c17a1fe9a70145a38869cb2d0a179eb5 2017-05-03 20:41:36,194 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/c17a1fe9a70145a38869cb2d0a179eb5, entries=96668, sequenceid=50163036, filesize=8.2 M 2017-05-03 20:41:36,195 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.46 MB/134702336, currentsize=4.85 MB/5086560 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 521ms, sequenceid=50163036, compaction requested=true 2017-05-03 20:41:36,196 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 20:41:36,196 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 20:41:36,200 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49890, currentSize=3243658544, freeSize=170845648, maxSize=3414504192, heapSize=3243658544, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 20:41:37,058 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into c643808078784d06b33c9b0da0d113a5(size=8.1 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 20:41:37,058 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17679289273727704; duration=0sec 2017-05-03 20:41:58,864 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493843992100 with entries=1686, filesize=122.24 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493844118833 2017-05-03 20:43:13,351 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.44 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:43:13,919 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50163444, memsize=128.4 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/8faa726fd3d34a659d31a3032e0c00fe 2017-05-03 20:43:13,931 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/8faa726fd3d34a659d31a3032e0c00fe, entries=96668, sequenceid=50163444, filesize=8.2 M 2017-05-03 20:43:13,934 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.44 MB/134676928, currentsize=1.49 MB/1563776 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 583ms, sequenceid=50163444, compaction requested=true 2017-05-03 20:43:42,048 INFO [sync.3] wal.FSHLog: Slow sync cost: 419 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-9a10c707-ebd5-4fae-8f82-b381f706fa57,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]] 2017-05-03 20:44:03,051 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493844118833 with entries=1741, filesize=121.71 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493844243015 2017-05-03 20:44:52,173 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.55 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:44:52,772 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50163887, memsize=128.6 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/1b7c5df257ff49a5b68c11407210cede 2017-05-03 20:44:52,791 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/1b7c5df257ff49a5b68c11407210cede, entries=96668, sequenceid=50163887, filesize=8.2 M 2017-05-03 20:44:52,792 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.55 MB/134795384, currentsize=5.52 MB/5791320 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 619ms, sequenceid=50163887, compaction requested=true 2017-05-03 20:44:52,794 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 20:44:52,794 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 20:44:52,796 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49890, currentSize=3243658544, freeSize=170845648, maxSize=3414504192, heapSize=3243658544, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 20:44:53,688 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into bd3b58f2837b447caef201ffc1d9b5ef(size=8.1 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 20:44:53,688 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17679485871039658; duration=0sec 2017-05-03 20:45:14,527 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=3.02 GB, freeSize=162.93 MB, max=3.18 GB, blockCount=49890, accesses=361405873, hits=182369755, hitRatio=50.46%, , cachingAccesses=355598053, cachingHits=182325522, cachingHitsRatio=51.27%, evictions=91054, evicted=172897306, evictedPerRun=1898.8436279296875 2017-05-03 20:46:09,158 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493844243015 with entries=1385, filesize=121.98 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493844369103 2017-05-03 20:46:09,858 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.06 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:46:10,398 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50164301, memsize=128.1 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/0f458a58c102483cb9f9f61fa4c56326 2017-05-03 20:46:10,419 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/0f458a58c102483cb9f9f61fa4c56326, entries=96668, sequenceid=50164301, filesize=8.2 M 2017-05-03 20:46:10,420 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~128.06 MB/134281528, currentsize=6.96 MB/7293112 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 562ms, sequenceid=50164301, compaction requested=true 2017-05-03 20:46:11,033 INFO [sync.1] wal.FSHLog: Slow sync cost: 162 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-6e0a3279-95d2-4bff-ac72-ed8e01ad47ab,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-5c91e4fa-90df-4f20-8217-049e5d671c16,DISK]] 2017-05-03 20:46:41,980 INFO [sync.2] wal.FSHLog: Slow sync cost: 107 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-50fbd188-eca8-404b-9a7a-a85a07a1a66b,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-6e0a3279-95d2-4bff-ac72-ed8e01ad47ab,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-5c91e4fa-90df-4f20-8217-049e5d671c16,DISK]] 2017-05-03 20:47:48,831 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 129.71 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:47:49,397 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50164724, memsize=129.7 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/ac9184222c9e4e1882d698f0e498a80e 2017-05-03 20:47:49,412 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/ac9184222c9e4e1882d698f0e498a80e, entries=96668, sequenceid=50164724, filesize=8.2 M 2017-05-03 20:47:49,413 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~129.71 MB/136012312, currentsize=3.90 MB/4090552 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 582ms, sequenceid=50164724, compaction requested=true 2017-05-03 20:47:49,415 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 20:47:49,415 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 20:47:49,418 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49890, currentSize=3243658544, freeSize=170845648, maxSize=3414504192, heapSize=3243658544, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 20:47:50,276 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into f732553fbb1d4212891734a3e95723f1(size=8.1 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 20:47:50,276 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17679662491985730; duration=0sec 2017-05-03 20:48:14,900 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493844369103 with entries=1096, filesize=121.72 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493844494860 2017-05-03 20:49:07,386 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.33 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:49:07,987 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50165162, memsize=128.3 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/cc0238dc91e14daa8aa9fbe8fe4148c7 2017-05-03 20:49:07,999 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/cc0238dc91e14daa8aa9fbe8fe4148c7, entries=96668, sequenceid=50165162, filesize=8.2 M 2017-05-03 20:49:08,001 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~128.33 MB/134562088, currentsize=120 KB/122880 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 615ms, sequenceid=50165162, compaction requested=true 2017-05-03 20:50:14,527 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=3.02 GB, freeSize=162.93 MB, max=3.18 GB, blockCount=49890, accesses=361407628, hits=182370548, hitRatio=50.46%, , cachingAccesses=355598846, cachingHits=182326315, cachingHitsRatio=51.27%, evictions=91084, evicted=172897306, evictedPerRun=1898.21826171875 2017-05-03 20:50:21,732 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493844494860 with entries=871, filesize=122.03 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493844621671 2017-05-03 20:50:46,724 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.41 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:50:47,234 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50165571, memsize=128.4 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/d37145a112f046b2bbac0b0cdd21c3c7 2017-05-03 20:50:47,249 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/d37145a112f046b2bbac0b0cdd21c3c7, entries=96668, sequenceid=50165571, filesize=8.2 M 2017-05-03 20:50:47,249 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~128.41 MB/134651696, currentsize=7.28 MB/7637272 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 525ms, sequenceid=50165571, compaction requested=true 2017-05-03 20:50:47,251 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 20:50:47,251 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 20:50:47,253 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49890, currentSize=3243658544, freeSize=170845648, maxSize=3414504192, heapSize=3243658544, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 20:50:48,121 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into ef9dd39eb85a4fcd8c01589cc29e07c7(size=8.2 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 20:50:48,121 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17679840328348201; duration=0sec 2017-05-03 20:52:25,963 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.50 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:52:26,556 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50166003, memsize=128.5 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/b18dcead39ae4af8beb487e47f879e27 2017-05-03 20:52:26,570 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/b18dcead39ae4af8beb487e47f879e27, entries=96668, sequenceid=50166003, filesize=8.2 M 2017-05-03 20:52:26,571 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~128.50 MB/134738408, currentsize=4.87 MB/5103928 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 608ms, sequenceid=50166003, compaction requested=true 2017-05-03 20:52:27,837 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493844621671 with entries=3777, filesize=121.63 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493844747801 2017-05-03 20:53:43,393 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.46 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:53:43,926 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50166428, memsize=128.5 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/0e8610745be744caaa9be26240c84045 2017-05-03 20:53:43,940 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/0e8610745be744caaa9be26240c84045, entries=96668, sequenceid=50166428, filesize=8.2 M 2017-05-03 20:53:43,940 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~128.46 MB/134702336, currentsize=4.85 MB/5086560 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 547ms, sequenceid=50166428, compaction requested=true 2017-05-03 20:53:43,942 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 20:53:43,942 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 20:53:43,945 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49890, currentSize=3243658544, freeSize=170845648, maxSize=3414504192, heapSize=3243658544, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 20:53:44,794 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into 105da53b70074c5f8e945e1fa44c7dd2(size=8.1 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 20:53:44,794 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17680017019565116; duration=0sec 2017-05-03 20:54:30,214 INFO [sync.1] wal.FSHLog: Slow sync cost: 186 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-9ba4f08a-d996-4490-b27d-6c8ca9a67152,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-9a10c707-ebd5-4fae-8f82-b381f706fa57,DISK]] 2017-05-03 20:54:33,436 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493844747801 with entries=1087, filesize=121.79 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493844873397 2017-05-03 20:55:14,527 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=3.02 GB, freeSize=162.93 MB, max=3.18 GB, blockCount=49890, accesses=361411473, hits=182372469, hitRatio=50.46%, , cachingAccesses=355600767, cachingHits=182328236, cachingHitsRatio=51.27%, evictions=91114, evicted=172897306, evictedPerRun=1897.59326171875 2017-05-03 20:55:22,790 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.44 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:55:23,326 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50166836, memsize=128.4 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/361b2a45ad7740849a247293d50fd452 2017-05-03 20:55:23,341 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/361b2a45ad7740849a247293d50fd452, entries=96668, sequenceid=50166836, filesize=8.2 M 2017-05-03 20:55:23,342 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.44 MB/134676928, currentsize=120.76 KB/123656 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 552ms, sequenceid=50166836, compaction requested=true 2017-05-03 20:56:42,463 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493844873397 with entries=563, filesize=121.69 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493845002429 2017-05-03 20:57:04,803 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.55 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:57:05,378 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50167279, memsize=128.6 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/22c896aef6a544cd9e477c8bf3ee8d93 2017-05-03 20:57:05,393 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/22c896aef6a544cd9e477c8bf3ee8d93, entries=96668, sequenceid=50167279, filesize=8.2 M 2017-05-03 20:57:05,394 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.55 MB/134795384, currentsize=5.52 MB/5791320 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 591ms, sequenceid=50167279, compaction requested=true 2017-05-03 20:57:05,396 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 20:57:05,396 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 20:57:05,399 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49890, currentSize=3243658544, freeSize=170845648, maxSize=3414504192, heapSize=3243658544, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 20:57:06,289 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into 83461285e9534760aba99bb5348742fb(size=8.1 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 20:57:06,289 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17680218473431410; duration=0sec 2017-05-03 20:58:24,544 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.06 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 20:58:25,066 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50167693, memsize=128.1 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/596973018b4e4cdb9ab4663539408e69 2017-05-03 20:58:25,081 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/596973018b4e4cdb9ab4663539408e69, entries=96668, sequenceid=50167693, filesize=8.2 M 2017-05-03 20:58:25,082 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.06 MB/134281528, currentsize=6.96 MB/7293112 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 538ms, sequenceid=50167693, compaction requested=true 2017-05-03 20:58:51,828 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493845002429 with entries=2368, filesize=121.84 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493845131787 2017-05-03 20:59:14,293 INFO [sync.4] wal.FSHLog: Slow sync cost: 282 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK]] 2017-05-03 21:00:06,761 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 129.71 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 21:00:07,357 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50168116, memsize=129.7 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/e66f0ab3630a4fbc879ceaef3741c179 2017-05-03 21:00:07,369 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/e66f0ab3630a4fbc879ceaef3741c179, entries=96668, sequenceid=50168116, filesize=8.2 M 2017-05-03 21:00:07,370 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~129.71 MB/136012312, currentsize=3.90 MB/4090552 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 609ms, sequenceid=50168116, compaction requested=true 2017-05-03 21:00:07,372 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 21:00:07,372 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 21:00:07,374 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49890, currentSize=3243658544, freeSize=170845648, maxSize=3414504192, heapSize=3243658544, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 21:00:08,260 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into 48e0040dc41f43d6ac44f902a50d728c(size=8.2 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 21:00:08,260 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17680400448897520; duration=0sec 2017-05-03 21:00:14,527 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=3.02 GB, freeSize=162.93 MB, max=3.18 GB, blockCount=49890, accesses=361414031, hits=182373103, hitRatio=50.46%, , cachingAccesses=355601401, cachingHits=182328870, cachingHitsRatio=51.27%, evictions=91144, evicted=172897306, evictedPerRun=1896.9686279296875 2017-05-03 21:00:56,532 INFO [sync.1] wal.FSHLog: Slow sync cost: 1367 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK]] 2017-05-03 21:00:56,532 INFO [sync.2] wal.FSHLog: Slow sync cost: 1336 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK]] 2017-05-03 21:01:00,960 INFO [sync.0] wal.FSHLog: Slow sync cost: 768 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK]] 2017-05-03 21:01:00,983 INFO [sync.1] wal.FSHLog: Slow sync cost: 757 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK]] 2017-05-03 21:01:00,983 INFO [sync.2] wal.FSHLog: Slow sync cost: 643 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK]] 2017-05-03 21:01:03,694 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493845131787 with entries=1494, filesize=121.95 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493845263616 2017-05-03 21:01:29,040 INFO [sync.3] wal.FSHLog: Slow sync cost: 1117 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-05-03 21:01:29,061 INFO [sync.4] wal.FSHLog: Slow sync cost: 1126 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-05-03 21:01:29,061 INFO [sync.1] wal.FSHLog: Slow sync cost: 1004 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-05-03 21:01:29,061 INFO [sync.0] wal.FSHLog: Slow sync cost: 1089 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-05-03 21:01:33,058 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.33 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 21:01:33,607 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50168554, memsize=128.3 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/cdcce2dbdb3f40de88d7da1c0f33d3cb 2017-05-03 21:01:33,651 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/cdcce2dbdb3f40de88d7da1c0f33d3cb, entries=96668, sequenceid=50168554, filesize=8.2 M 2017-05-03 21:01:33,653 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.33 MB/134562088, currentsize=120 KB/122880 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 595ms, sequenceid=50168554, compaction requested=true 2017-05-03 21:01:37,599 INFO [sync.1] wal.FSHLog: Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-05-03 21:01:37,599 INFO [sync.2] wal.FSHLog: Slow sync cost: 198 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-05-03 21:01:37,599 INFO [sync.3] wal.FSHLog: Slow sync cost: 134 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-05-03 21:01:38,242 INFO [sync.3] wal.FSHLog: Slow sync cost: 199 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-05-03 21:01:38,242 INFO [sync.2] wal.FSHLog: Slow sync cost: 264 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-9f6829c5-3834-436e-8ec2-39df06418ca4,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK]] 2017-05-03 21:03:17,085 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.41 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 21:03:17,652 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50168963, memsize=128.4 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/9450e930bb5647708b31be5b2881e4f5 2017-05-03 21:03:17,670 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/9450e930bb5647708b31be5b2881e4f5, entries=96668, sequenceid=50168963, filesize=8.2 M 2017-05-03 21:03:17,672 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~128.41 MB/134651696, currentsize=7.28 MB/7637272 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 586ms, sequenceid=50168963, compaction requested=true 2017-05-03 21:03:17,674 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 21:03:17,674 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 21:03:17,676 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=51759, currentSize=3368330504, freeSize=46173688, maxSize=3414504192, heapSize=3368330504, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 21:03:18,080 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493845263616 with entries=2134, filesize=122.01 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493845398023 2017-05-03 21:03:18,625 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into 6d08ebd46a93452f9627b8cc563b0c03(size=8.1 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 21:03:18,626 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17680590750967105; duration=0sec 2017-05-03 21:04:07,419 INFO [sync.4] wal.FSHLog: Slow sync cost: 132 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-6e0a3279-95d2-4bff-ac72-ed8e01ad47ab,DISK]] 2017-05-03 21:04:07,431 INFO [sync.0] wal.FSHLog: Slow sync cost: 100 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-6e0a3279-95d2-4bff-ac72-ed8e01ad47ab,DISK]] 2017-05-03 21:04:09,375 INFO [sync.0] wal.FSHLog: Slow sync cost: 971 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-6e0a3279-95d2-4bff-ac72-ed8e01ad47ab,DISK]] 2017-05-03 21:04:09,375 INFO [sync.1] wal.FSHLog: Slow sync cost: 944 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-6e0a3279-95d2-4bff-ac72-ed8e01ad47ab,DISK]] 2017-05-03 21:04:58,665 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.50 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 21:04:59,221 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50169395, memsize=128.5 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/25be83ac6ba14a0684f3d4e79758ee84 2017-05-03 21:04:59,238 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/25be83ac6ba14a0684f3d4e79758ee84, entries=96668, sequenceid=50169395, filesize=8.2 M 2017-05-03 21:04:59,239 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.50 MB/134738408, currentsize=4.87 MB/5103928 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 574ms, sequenceid=50169395, compaction requested=true 2017-05-03 21:05:14,527 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=3.10 GB, freeSize=82.04 MB, max=3.18 GB, blockCount=51172, accesses=362113060, hits=182726650, hitRatio=50.46%, , cachingAccesses=356299468, cachingHits=182682417, cachingHitsRatio=51.27%, evictions=91313, evicted=173240544, evictedPerRun=1897.2166748046875 2017-05-03 21:05:28,057 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493845398023 with entries=1883, filesize=121.63 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493845528027 2017-05-03 21:05:30,756 INFO [RS_OPEN_META-aps-hadoop5:16020-0-MetaLogRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009..meta.1493841930617.meta with entries=0, filesize=91 B; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009..meta.1493845530723.meta 2017-05-03 21:05:30,757 INFO [RS_OPEN_META-aps-hadoop5:16020-0-MetaLogRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009..meta.1493841930617.meta to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009..meta.1493841930617.meta 2017-05-03 21:06:18,685 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.46 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 21:06:19,267 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50169820, memsize=128.5 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/ff57287683854e3f8bb1187e05066368 2017-05-03 21:06:19,280 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/ff57287683854e3f8bb1187e05066368, entries=96668, sequenceid=50169820, filesize=8.2 M 2017-05-03 21:06:19,281 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.46 MB/134702336, currentsize=4.85 MB/5086560 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 596ms, sequenceid=50169820, compaction requested=true 2017-05-03 21:06:19,283 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 21:06:19,283 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 21:06:19,286 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=51741, currentSize=3366274192, freeSize=48230000, maxSize=3414504192, heapSize=3366274192, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 21:06:20,152 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into 58c6620cbc9d471cb91790aa8370b317(size=8.1 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 21:06:20,152 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17680772360512652; duration=0sec 2017-05-03 21:07:42,369 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493845528027 with entries=1508, filesize=121.61 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493845662215 2017-05-03 21:08:03,700 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.44 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 21:08:04,217 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50170228, memsize=128.4 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/e8f29cccccd245f8a4bcb4b550069b38 2017-05-03 21:08:04,230 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/e8f29cccccd245f8a4bcb4b550069b38, entries=96668, sequenceid=50170228, filesize=8.2 M 2017-05-03 21:08:04,232 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~128.44 MB/134676928, currentsize=120.76 KB/123656 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 532ms, sequenceid=50170228, compaction requested=true 2017-05-03 21:09:08,927 INFO [sync.2] wal.FSHLog: Slow sync cost: 104 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-9a10c707-ebd5-4fae-8f82-b381f706fa57,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-5c91e4fa-90df-4f20-8217-049e5d671c16,DISK]] 2017-05-03 21:09:18,899 INFO [sync.0] wal.FSHLog: Slow sync cost: 148 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-9a10c707-ebd5-4fae-8f82-b381f706fa57,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-5c91e4fa-90df-4f20-8217-049e5d671c16,DISK]] 2017-05-03 21:10:14,527 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=3.02 GB, freeSize=162.85 MB, max=3.18 GB, blockCount=49890, accesses=362167906, hits=182755926, hitRatio=50.46%, , cachingAccesses=356353352, cachingHits=182711693, cachingHitsRatio=51.27%, evictions=91347, evicted=173266434, evictedPerRun=1896.7939453125 2017-05-03 21:10:18,565 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.55 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 21:10:19,162 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50170671, memsize=128.6 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/e69e8da12a9c4dfd9261bb79cbd0a822 2017-05-03 21:10:19,178 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/e69e8da12a9c4dfd9261bb79cbd0a822, entries=96668, sequenceid=50170671, filesize=8.2 M 2017-05-03 21:10:19,180 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.55 MB/134795384, currentsize=4.04 MB/4232096 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 615ms, sequenceid=50170671, compaction requested=true 2017-05-03 21:10:19,182 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 21:10:19,182 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 21:10:19,184 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49890, currentSize=3243744160, freeSize=170760032, maxSize=3414504192, heapSize=3243744160, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 21:10:20,063 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into f1918eebc5944e5b905472d55e951cb6(size=8.2 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 21:10:20,063 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17681012259356140; duration=0sec 2017-05-03 21:10:34,932 INFO [aps-hadoop5,16020,1493618413009_ChoreService_1] regionserver.HRegionServer: aps-hadoop5,16020,1493618413009-MemstoreFlusherChore requesting flush for region AverageCampaignPerformance,33333333,1478428256763.573482bf2396e92f7ea555322e355f0c. after a delay of 15326 2017-05-03 21:10:44,932 INFO [aps-hadoop5,16020,1493618413009_ChoreService_1] regionserver.HRegionServer: aps-hadoop5,16020,1493618413009-MemstoreFlusherChore requesting flush for region CampaignPerformance,66666666,1478428211809.82260485c06f9076b59f25064eeb47cc. after a delay of 7686 2017-05-03 21:10:44,932 INFO [aps-hadoop5,16020,1493618413009_ChoreService_1] regionserver.HRegionServer: aps-hadoop5,16020,1493618413009-MemstoreFlusherChore requesting flush for region AverageCampaignPerformance,33333333,1478428256763.573482bf2396e92f7ea555322e355f0c. after a delay of 8466 2017-05-03 21:10:50,260 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for AverageCampaignPerformance,33333333,1478428256763.573482bf2396e92f7ea555322e355f0c., current region memstore size 5.89 KB, and 1/1 column families' memstores are being flushed. 2017-05-03 21:10:50,321 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=4277, memsize=5.9 K, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/AverageCampaignPerformance/573482bf2396e92f7ea555322e355f0c/.tmp/9cee54ba771544b4a5a3fe6075dddb5d 2017-05-03 21:10:51,687 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Slow sync cost: 1377 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-9a10c707-ebd5-4fae-8f82-b381f706fa57,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-5c91e4fa-90df-4f20-8217-049e5d671c16,DISK]] 2017-05-03 21:10:51,689 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/AverageCampaignPerformance/573482bf2396e92f7ea555322e355f0c/info/9cee54ba771544b4a5a3fe6075dddb5d, entries=36, sequenceid=4277, filesize=5.2 K 2017-05-03 21:10:51,705 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493845662215 with entries=2684, filesize=122.32 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493845850295 2017-05-03 21:10:51,708 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~5.89 KB/6032, currentsize=0 B/0 for region AverageCampaignPerformance,33333333,1478428256763.573482bf2396e92f7ea555322e355f0c. in 1448ms, sequenceid=4277, compaction requested=false 2017-05-03 21:10:52,618 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for CampaignPerformance,66666666,1478428211809.82260485c06f9076b59f25064eeb47cc., current region memstore size 37.34 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 21:10:52,799 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=23848520, memsize=37.3 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/CampaignPerformance/82260485c06f9076b59f25064eeb47cc/.tmp/bc91a4a3f0a34732a5ad768e318827d6 2017-05-03 21:10:52,813 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/CampaignPerformance/82260485c06f9076b59f25064eeb47cc/info/bc91a4a3f0a34732a5ad768e318827d6, entries=28155, sequenceid=23848520, filesize=319.6 K 2017-05-03 21:10:52,813 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~37.34 MB/39153184, currentsize=5.99 KB/6136 for region CampaignPerformance,66666666,1478428211809.82260485c06f9076b59f25064eeb47cc. in 195ms, sequenceid=23848520, compaction requested=false 2017-05-03 21:12:18,005 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.06 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 21:12:18,538 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50171085, memsize=128.1 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/4eae1aa06d1647fa8d8095e030764015 2017-05-03 21:12:18,551 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/4eae1aa06d1647fa8d8095e030764015, entries=96668, sequenceid=50171085, filesize=8.2 M 2017-05-03 21:12:18,552 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~128.06 MB/134281528, currentsize=8.37 MB/8778800 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 548ms, sequenceid=50171085, compaction requested=true 2017-05-03 21:13:06,883 INFO [sync.2] wal.FSHLog: Slow sync cost: 405 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-6a874575-5e2a-4b1d-8914-c70e465dba0e,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-9ba4f08a-d996-4490-b27d-6c8ca9a67152,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK]] 2017-05-03 21:13:20,153 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493845850295 with entries=2510, filesize=121.61 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493846000121 2017-05-03 21:13:20,154 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493842181974 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493842181974 2017-05-03 21:13:54,932 INFO [aps-hadoop5,16020,1493618413009_ChoreService_1] regionserver.HRegionServer: aps-hadoop5,16020,1493618413009-MemstoreFlusherChore requesting flush for region CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. after a delay of 11198 2017-05-03 21:14:04,932 INFO [aps-hadoop5,16020,1493618413009_ChoreService_1] regionserver.HRegionServer: aps-hadoop5,16020,1493618413009-MemstoreFlusherChore requesting flush for region CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. after a delay of 12637 2017-05-03 21:14:06,130 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494., current region memstore size 124.64 MB, and 3/12 column families' memstores are being flushed. 2017-05-03 21:14:06,130 INFO [MemStoreFlusher.0] regionserver.HRegion: Flushing Column Family: info which was occupying 29.87 MB of memstore. 2017-05-03 21:14:06,130 INFO [MemStoreFlusher.0] regionserver.HRegion: Flushing Column Family: Em which was occupying 64.28 MB of memstore. 2017-05-03 21:14:06,130 INFO [MemStoreFlusher.0] regionserver.HRegion: Flushing Column Family: Mb which was occupying 30.49 MB of memstore. 2017-05-03 21:14:06,336 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=113714234, memsize=64.3 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/CampaignGoalMDC/6cf8953db275a43b98ab944d2d614494/.tmp/49b4966e1c444cf69d0710b2a5c2cf27 2017-05-03 21:14:06,443 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=113714234, memsize=30.5 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/CampaignGoalMDC/6cf8953db275a43b98ab944d2d614494/.tmp/3d7d2bcb6ffa4ec594d7aa830ffb5270 2017-05-03 21:14:09,616 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 129.71 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 21:14:10,327 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50171508, memsize=129.7 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/e7b27ef74d62402591397990aa8dff68 2017-05-03 21:14:10,341 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/e7b27ef74d62402591397990aa8dff68, entries=96668, sequenceid=50171508, filesize=8.2 M 2017-05-03 21:14:10,342 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~129.71 MB/136012312, currentsize=3.90 MB/4090552 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 726ms, sequenceid=50171508, compaction requested=true 2017-05-03 21:14:10,344 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 21:14:10,344 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 21:14:10,348 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49890, currentSize=3243744160, freeSize=170760032, maxSize=3414504192, heapSize=3243744160, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 21:14:11,299 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into 2694e7feac764b8098c3b69680b459ff(size=8.2 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 21:14:11,299 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17681243421449364; duration=0sec 2017-05-03 21:14:14,245 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=113714234, memsize=29.9 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/CampaignGoalMDC/6cf8953db275a43b98ab944d2d614494/.tmp/ae5d8af271474ccbbb967a08e37f9016 2017-05-03 21:14:14,259 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/CampaignGoalMDC/6cf8953db275a43b98ab944d2d614494/Em/49b4966e1c444cf69d0710b2a5c2cf27, entries=1426, sequenceid=113714234, filesize=28.8 K 2017-05-03 21:14:14,268 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/CampaignGoalMDC/6cf8953db275a43b98ab944d2d614494/Mb/3d7d2bcb6ffa4ec594d7aa830ffb5270, entries=1034, sequenceid=113714234, filesize=19.2 K 2017-05-03 21:14:14,278 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/CampaignGoalMDC/6cf8953db275a43b98ab944d2d614494/info/ae5d8af271474ccbbb967a08e37f9016, entries=837, sequenceid=113714234, filesize=16.3 K 2017-05-03 21:14:14,279 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~124.64 MB/130694872, currentsize=0 B/0 for region CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. in 8149ms, sequenceid=113714234, compaction requested=true 2017-05-03 21:14:14,280 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on Mb in region CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. 2017-05-03 21:14:14,280 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in Mb of CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/CampaignGoalMDC/6cf8953db275a43b98ab944d2d614494/.tmp, totalSize=7.2 M 2017-05-03 21:14:14,283 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49890, currentSize=3243744160, freeSize=170760032, maxSize=3414504192, heapSize=3243744160, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 21:14:15,145 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 (all) file(s) in Mb of CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. into b6a3ff802b044fe38be5ec380fcd92b5(size=7.1 M), total size for store is 7.1 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 21:14:15,145 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494., storeName=Mb, fileCount=3, fileSize=7.2 M, priority=7, time=17681247357610784; duration=0sec 2017-05-03 21:14:15,147 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. 2017-05-03 21:14:15,147 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/CampaignGoalMDC/6cf8953db275a43b98ab944d2d614494/.tmp, totalSize=3.6 M 2017-05-03 21:14:15,150 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49890, currentSize=3243744160, freeSize=170760032, maxSize=3414504192, heapSize=3243744160, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 21:14:15,614 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 (all) file(s) in info of CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. into 4c99b9fcaf494998962db5208f17a2e8(size=3.6 M), total size for store is 3.6 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 21:14:15,614 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494., storeName=info, fileCount=3, fileSize=3.6 M, priority=7, time=17681248224700655; duration=0sec 2017-05-03 21:15:14,527 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=3.02 GB, freeSize=162.85 MB, max=3.18 GB, blockCount=49890, accesses=362171527, hits=182756951, hitRatio=50.46%, , cachingAccesses=356354377, cachingHits=182712718, cachingHitsRatio=51.27%, evictions=91377, evicted=173266434, evictedPerRun=1896.171142578125 2017-05-03 21:15:30,019 INFO [sync.4] wal.FSHLog: Slow sync cost: 1093 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-0ad77247-babc-4c36-9cd0-c04ad47e0894,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-9ba4f08a-d996-4490-b27d-6c8ca9a67152,DISK]] 2017-05-03 21:15:33,744 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.33 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 21:15:34,272 INFO [MemStoreFlusher.0] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50171946, memsize=128.3 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/7b07a25f13744a56946e96de098804af 2017-05-03 21:15:34,286 INFO [MemStoreFlusher.0] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/7b07a25f13744a56946e96de098804af, entries=96668, sequenceid=50171946, filesize=8.2 M 2017-05-03 21:15:34,287 INFO [MemStoreFlusher.0] regionserver.HRegion: Finished memstore flush of ~128.33 MB/134562088, currentsize=120 KB/122880 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 543ms, sequenceid=50171946, compaction requested=true 2017-05-03 21:15:56,339 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493846000121 with entries=1383, filesize=121.83 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493846156304 2017-05-03 21:15:56,339 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493842310717 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493842310717 2017-05-03 21:15:56,352 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493842439135 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493842439135 2017-05-03 21:15:56,367 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493842567860 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493842567860 2017-05-03 21:15:56,376 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493842696249 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493842696249 2017-05-03 21:15:56,386 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Archiving hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493843484732 to hdfs://mycluster/apps/hbase/data/oldWALs/aps-hadoop5%2C16020%2C1493618413009.default.1493843484732 2017-05-03 21:16:29,054 INFO [sync.1] wal.FSHLog: Slow sync cost: 158 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK]] 2017-05-03 21:16:31,083 INFO [sync.3] wal.FSHLog: Slow sync cost: 291 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK]] 2017-05-03 21:17:18,080 INFO [sync.1] wal.FSHLog: Slow sync cost: 1232 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK]] 2017-05-03 21:17:18,080 INFO [sync.2] wal.FSHLog: Slow sync cost: 1135 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK]] 2017-05-03 21:17:19,018 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.41 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 21:17:19,533 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50172355, memsize=128.4 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/1f0f320e342f4e558d725cf6bfb100bc 2017-05-03 21:17:19,548 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/1f0f320e342f4e558d725cf6bfb100bc, entries=96668, sequenceid=50172355, filesize=8.2 M 2017-05-03 21:17:19,549 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.41 MB/134651696, currentsize=6.81 MB/7142320 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 531ms, sequenceid=50172355, compaction requested=true 2017-05-03 21:17:19,551 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 21:17:19,551 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 21:17:19,553 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49890, currentSize=3243744160, freeSize=170760032, maxSize=3414504192, heapSize=3243744160, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 21:17:20,397 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into a849acec2d1540b29d7b337c9f88d46c(size=8.1 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 21:17:20,397 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17681432628550621; duration=0sec 2017-05-03 21:17:27,209 INFO [sync.4] wal.FSHLog: Slow sync cost: 148 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK]] 2017-05-03 21:17:31,472 INFO [sync.4] wal.FSHLog: Slow sync cost: 132 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK]] 2017-05-03 21:17:47,321 INFO [sync.1] wal.FSHLog: Slow sync cost: 321 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK]] 2017-05-03 21:17:47,322 INFO [sync.2] wal.FSHLog: Slow sync cost: 179 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK]] 2017-05-03 21:18:01,298 INFO [sync.0] wal.FSHLog: Slow sync cost: 139 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK]] 2017-05-03 21:18:07,369 INFO [sync.1] wal.FSHLog: Slow sync cost: 162 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-50b6de57-9040-41d4-80e9-bc62db9d1c5c,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-3109a88b-277e-4d45-8cfb-8a5f3bf57adb,DISK]] 2017-05-03 21:18:10,060 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493846156304 with entries=2559, filesize=122.04 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493846290022 2017-05-03 21:18:36,048 INFO [sync.4] wal.FSHLog: Slow sync cost: 214 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-d7a093e4-6437-4935-b4a2-0decf44fabea,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-1940f918-25a9-4136-8034-e6fd0972e5a2,DISK]] 2017-05-03 21:19:06,230 INFO [sync.3] wal.FSHLog: Slow sync cost: 312 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-d7a093e4-6437-4935-b4a2-0decf44fabea,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-1940f918-25a9-4136-8034-e6fd0972e5a2,DISK]] 2017-05-03 21:19:06,230 INFO [sync.2] wal.FSHLog: Slow sync cost: 381 ms, current pipeline: [DatanodeInfoWithStorage[1..1..1..:50010,DS-d7a093e4-6437-4935-b4a2-0decf44fabea,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-08f50d3d-2a14-4033-b117-b162b0cae2ce,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-1940f918-25a9-4136-8034-e6fd0972e5a2,DISK]] 2017-05-03 21:19:06,488 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.50 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 21:19:07,014 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50172787, memsize=128.5 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/dc28612996be4b5fb170836d36d7b21a 2017-05-03 21:19:07,031 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/dc28612996be4b5fb170836d36d7b21a, entries=96668, sequenceid=50172787, filesize=8.2 M 2017-05-03 21:19:07,032 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.50 MB/134738408, currentsize=3.01 MB/3157400 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 544ms, sequenceid=50172787, compaction requested=true 2017-05-03 21:20:14,527 INFO [LruBlockCacheStatsExecutor] hfile.LruBlockCache: totalSize=3.02 GB, freeSize=162.85 MB, max=3.18 GB, blockCount=49890, accesses=362174464, hits=182758926, hitRatio=50.46%, , cachingAccesses=356356352, cachingHits=182714693, cachingHitsRatio=51.27%, evictions=91407, evicted=173266434, evictedPerRun=1895.548828125 2017-05-03 21:20:32,917 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Rolled WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493846290022 with entries=614, filesize=121.89 MB; new WAL /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493846432867 2017-05-03 21:20:36,067 INFO [MemStoreFlusher.1] regionserver.HRegion: Started memstore flush for ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., current region memstore size 128.46 MB, and 1/1 column families' memstores are being flushed. 2017-05-03 21:20:36,672 INFO [MemStoreFlusher.1] regionserver.DefaultStoreFlusher: Flushed, sequenceid=50173212, memsize=128.5 M, hasBloomFilter=true, into tmp file hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp/7bcf5ac6ee4a4ecba07cd55c18a2cd28 2017-05-03 21:20:36,688 INFO [MemStoreFlusher.1] regionserver.HStore: Added hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/info/7bcf5ac6ee4a4ecba07cd55c18a2cd28, entries=96668, sequenceid=50173212, filesize=8.2 M 2017-05-03 21:20:36,689 INFO [MemStoreFlusher.1] regionserver.HRegion: Finished memstore flush of ~128.46 MB/134702336, currentsize=4.85 MB/5086560 for region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. in 622ms, sequenceid=50173212, compaction requested=true 2017-05-03 21:20:36,692 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HRegion: Starting compaction on info in region ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 21:20:36,692 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Starting compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into tmpdir=hdfs://mycluster/apps/hbase/data/data/default/ORMDetails/96ab0ca6ba008280cd0cc841abeeb413/.tmp, totalSize=24.5 M 2017-05-03 21:20:36,694 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] hfile.CacheConfig: blockCache=LruBlockCache{blockCount=49890, currentSize=3243744160, freeSize=170760032, maxSize=3414504192, heapSize=3243744160, minSize=3243778816, minFactor=0.95, multiSize=1621889408, multiFactor=0.5, singleSize=810944704, singleFactor=0.25}, cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false 2017-05-03 21:20:37,557 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.HStore: Completed compaction of 3 file(s) in info of ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. into c3029966607643a58be74e504542401d(size=8.1 M), total size for store is 123.0 M. This selection was in queue for 0sec, and took 0sec to execute. 2017-05-03 21:20:37,557 INFO [regionserver/aps-hadoop5/1..1..1..:16020-shortCompactions-1493618706540] regionserver.CompactSplitThread: Completed compaction: Request = regionName=ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413., storeName=info, fileCount=3, fileSize=24.5 M, priority=6, time=17681629769137293; duration=0sec 2017-05-03 21:21:44,932 INFO [aps-hadoop5,16020,1493618413009_ChoreService_1] regionserver.HRegionServer: aps-hadoop5,16020,1493618413009-MemstoreFlusherChore requesting flush for region CampaignSummaryDetailGeographyByCityMDC,cccccccc,1478428063068.2860d686a4589ad57488c2b034722a16. after a delay of 22598 2017-05-03 21:21:54,932 INFO [aps-hadoop5,16020,1493618413009_ChoreService_1] regionserver.HRegionServer: aps-hadoop5,16020,1493618413009-MemstoreFlusherChore requesting flush for region CampaignSummaryDetailGeographyByCityMDC,cccccccc,1478428063068.2860d686a4589ad57488c2b034722a16. after a delay of 6328 2017-05-03 21:22:04,932 INFO [aps-hadoop5,16020,1493618413009_ChoreService_1] regionserver.HRegionServer: aps-hadoop5,16020,1493618413009-MemstoreFlusherChore requesting flush for region CampaignSummaryDetailGeographyByCityMDC,cccccccc,1478428063068.2860d686a4589ad57488c2b034722a16. after a delay of 5246 2017-05-03 21:22:07,531 INFO [MemStoreFlusher.0] regionserver.HRegion: Started memstore flush for CampaignSummaryDetailGeographyByCityMDC,cccccccc,1478428063068.2860d686a4589ad57488c2b034722a16., current region memstore size 21.15 MB, and 1/2 column families' memstores are being flushed. 2017-05-03 21:22:07,531 INFO [MemStoreFlusher.0] regionserver.HRegion: Flushing Column Family: info which was occupying 21.15 MB of memstore. 2017-05-03 21:22:38,729 WARN [ResponseProcessor for block BP-1810172115-10.64.228.157-1478343078462:blk_1079562185_5838908] hdfs.DFSClient: DFSOutputStream ResponseProcessor exception for block BP-1810172115-10.64.228.157-1478343078462:blk_1079562185_5838908 java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:142) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) at java.io.FilterInputStream.read(FilterInputStream.java:83) at java.io.FilterInputStream.read(FilterInputStream.java:83) at org.apache.hadoop.hdfs.protocolPB.PBHelper.vintPrefixed(PBHelper.java:2291) at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer$ResponseProcessor.run(DFSOutputStream.java:748) 2017-05-03 21:22:38,729 WARN [DataStreamer for file /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493846432867 block BP-1810172115-10.64.228.157-1478343078462:blk_1079562185_5838908] hdfs.DFSClient: Error Recovery for block BP-1810172115-10.64.228.157-1478343078462:blk_1079562185_5838908 in pipeline DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK], DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]: bad datanode DatanodeInfoWithStorage[1..1..1..:50010,DS-751946a0-5a6f-4485-ad27-61f061359410,DISK] 2017-05-03 21:22:41,744 INFO [DataStreamer for file /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493846432867 block BP-1810172115-10.64.228.157-1478343078462:blk_1079562185_5838908] hdfs.DFSClient: Exception in createBlockOutputStream java.io.IOException: Got error, status message , ack with firstBadLink as 10.64.228.164:50010 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:140) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.createBlockOutputStream(DFSOutputStream.java:1393) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1217) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-05-03 21:22:41,745 WARN [DataStreamer for file /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493846432867 block BP-1810172115-10.64.228.157-1478343078462:blk_1079562185_5838908] hdfs.DFSClient: Error Recovery for block BP-1810172115-10.64.228.157-1478343078462:blk_1079562185_5838908 in pipeline DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[10.64.228.164:50010,DS-9ba4f08a-d996-4490-b27d-6c8ca9a67152,DISK]: bad datanode DatanodeInfoWithStorage[10.64.228.164:50010,DS-9ba4f08a-d996-4490-b27d-6c8ca9a67152,DISK] 2017-05-03 21:22:44,779 INFO [DataStreamer for file /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493846432867 block BP-1810172115-10.64.228.157-1478343078462:blk_1079562185_5838908] hdfs.DFSClient: Exception in createBlockOutputStream java.io.IOException: Got error, status message , ack with firstBadLink as 10.64.228.141:50010 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:140) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.createBlockOutputStream(DFSOutputStream.java:1393) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1217) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-05-03 21:22:44,779 WARN [DataStreamer for file /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493846432867 block BP-1810172115-10.64.228.157-1478343078462:blk_1079562185_5838908] hdfs.DFSClient: Error Recovery for block BP-1810172115-10.64.228.157-1478343078462:blk_1079562185_5838908 in pipeline DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK], DatanodeInfoWithStorage[10.64.228.141:50010,DS-120b90e1-5d80-447d-b8db-b8ea53661e88,DISK]: bad datanode DatanodeInfoWithStorage[10.64.228.141:50010,DS-120b90e1-5d80-447d-b8db-b8ea53661e88,DISK] 2017-05-03 21:22:44,781 WARN [DataStreamer for file /apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493846432867 block BP-1810172115-10.64.228.157-1478343078462:blk_1079562185_5838908] hdfs.DFSClient: DataStreamer Exception java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]], original=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:947) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1021) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1189) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-05-03 21:22:44,781 WARN [sync.0] hdfs.DFSClient: Error while syncing java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]], original=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:947) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1021) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1189) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-05-03 21:22:44,782 WARN [sync.4] hdfs.DFSClient: Error while syncing java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]], original=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:947) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1021) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1189) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-05-03 21:22:44,782 ERROR [sync.4] wal.FSHLog: Error syncing, request close of WAL java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]], original=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:947) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1021) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1189) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-05-03 21:22:44,782 WARN [sync.1] hdfs.DFSClient: Error while syncing java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]], original=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:947) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1021) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1189) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-05-03 21:22:44,782 ERROR [sync.0] wal.FSHLog: Error syncing, request close of WAL java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]], original=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:947) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1021) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1189) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-05-03 21:22:44,781 WARN [sync.2] hdfs.DFSClient: Error while syncing java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]], original=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:947) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1021) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1189) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-05-03 21:22:44,782 INFO [sync.0] wal.FSHLog: Slow sync cost: 72065 ms, current pipeline: [DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]] 2017-05-03 21:22:44,782 ERROR [sync.1] wal.FSHLog: Error syncing, request close of WAL java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]], original=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:947) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1021) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1189) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-05-03 21:22:44,782 INFO [sync.4] wal.FSHLog: Slow sync cost: 74396 ms, current pipeline: [DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]] 2017-05-03 21:22:44,782 WARN [sync.3] hdfs.DFSClient: Error while syncing java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]], original=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:947) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1021) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1189) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-05-03 21:22:44,782 ERROR [sync.3] wal.FSHLog: Error syncing, request close of WAL java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]], original=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:947) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1021) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1189) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-05-03 21:22:44,782 INFO [sync.1] wal.FSHLog: Slow sync cost: 37245 ms, current pipeline: [DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]] 2017-05-03 21:22:44,782 ERROR [sync.2] wal.FSHLog: Error syncing, request close of WAL java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]], original=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:947) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1021) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1189) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-05-03 21:22:44,783 INFO [sync.3] wal.FSHLog: Slow sync cost: 20395 ms, current pipeline: [DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]] 2017-05-03 21:22:44,783 INFO [sync.2] wal.FSHLog: Slow sync cost: 21182 ms, current pipeline: [DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]] 2017-05-03 21:22:44,783 WARN [B.defaultRpcServer.handler=13,queue=1,port=16020] ipc.RpcServer: (responseTooSlow): {"call":"Multi(org.apache.hadoop.hbase.protobuf.generated.ClientProtos$MultiRequest)","starttimems":1493846543599,"responsesize":1254,"method":"Multi","processingtimems":21184,"client":"10.64.228.171:51374","queuetimems":0,"class":"HRegionServer"} 2017-05-03 21:22:44,783 WARN [B.defaultRpcServer.handler=9,queue=0,port=16020] ipc.RpcServer: (responseTooSlow): {"call":"Multi(org.apache.hadoop.hbase.protobuf.generated.ClientProtos$MultiRequest)","starttimems":1493846544387,"responsesize":1254,"method":"Multi","processingtimems":20396,"client":"10.64.228.171:51374","queuetimems":0,"class":"HRegionServer"} 2017-05-03 21:22:44,784 WARN [B.defaultRpcServer.handler=21,queue=0,port=16020] ipc.RpcServer: (responseTooSlow): {"call":"Multi(org.apache.hadoop.hbase.protobuf.generated.ClientProtos$MultiRequest)","starttimems":1493846490385,"responsesize":1254,"method":"Multi","processingtimems":74399,"client":"10.64.228.171:51374","queuetimems":0,"class":"HRegionServer"} 2017-05-03 21:22:44,803 WARN [B.defaultRpcServer.handler=16,queue=1,port=16020] ipc.RpcServer: (responseTooSlow): {"call":"Multi(org.apache.hadoop.hbase.protobuf.generated.ClientProtos$MultiRequest)","starttimems":1493846492705,"responsesize":125104,"method":"Multi","processingtimems":72098,"client":"10.64.228.171:51374","queuetimems":0,"class":"HRegionServer"} 2017-05-03 21:22:44,819 ERROR [MemStoreFlusher.0] regionserver.MemStoreFlusher: Cache flush failed for region CampaignSummaryDetailGeographyByCityMDC,cccccccc,1478428063068.2860d686a4589ad57488c2b034722a16. java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]], original=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:947) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1021) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1189) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-05-03 21:22:44,829 ERROR [sync.4] wal.FSHLog: Error syncing, request close of WAL java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]], original=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:947) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1021) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1189) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-05-03 21:22:44,829 ERROR [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] wal.FSHLog: Failed close of WAL writer hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493846432867, unflushedEntries=10 org.apache.hadoop.hbase.regionserver.wal.FailedSyncBeforeLogCloseException: java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]], original=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SafePointZigZagLatch.waitSafePoint(FSHLog.java:1893) at org.apache.hadoop.hbase.regionserver.wal.FSHLog.replaceWriter(FSHLog.java:964) at org.apache.hadoop.hbase.regionserver.wal.FSHLog.rollWriter(FSHLog.java:733) at org.apache.hadoop.hbase.regionserver.LogRoller.run(LogRoller.java:148) at java.lang.Thread.run(Thread.java:745) Caused by: java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]], original=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:947) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1021) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1189) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-05-03 21:22:44,830 FATAL [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] regionserver.HRegionServer: ABORTING region server aps-hadoop5,16020,1493618413009: Failed log close in log roller org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException: hdfs://mycluster/apps/hbase/data/WALs/aps-hadoop5,16020,1493618413009/aps-hadoop5%2C16020%2C1493618413009.default.1493846432867, unflushedEntries=10 at org.apache.hadoop.hbase.regionserver.wal.FSHLog.replaceWriter(FSHLog.java:1014) at org.apache.hadoop.hbase.regionserver.wal.FSHLog.rollWriter(FSHLog.java:733) at org.apache.hadoop.hbase.regionserver.LogRoller.run(LogRoller.java:148) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.hadoop.hbase.regionserver.wal.FailedSyncBeforeLogCloseException: java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]], original=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SafePointZigZagLatch.waitSafePoint(FSHLog.java:1893) at org.apache.hadoop.hbase.regionserver.wal.FSHLog.replaceWriter(FSHLog.java:964) ... 3 more Caused by: java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]], original=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:947) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1021) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1189) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-05-03 21:22:44,830 FATAL [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] regionserver.HRegionServer: RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint, org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2017-05-03 21:22:44,916 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] regionserver.HRegionServer: Dump of metrics as JSON on abort: { "beans" : [ { "name" : "java.lang:type=Memory", "modelerType" : "sun.management.MemoryImpl", "ObjectPendingFinalizationCount" : 0, "NonHeapMemoryUsage" : { "committed" : 104951808, "init" : 2555904, "max" : -1, "used" : 101281112 }, "Verbose" : true, "HeapMemoryUsage" : { "committed" : 8536260608, "init" : 8589934592, "max" : 8536260608, "used" : 4785267896 }, "ObjectName" : "java.lang:type=Memory" } ], "beans" : [ { "name" : "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType" : "RegionServer,sub=IPC", "tag.Context" : "regionserver", "tag.Hostname" : "aps-hadoop5", "queueSize" : 0, "numCallsInGeneralQueue" : 0, "numCallsInReplicationQueue" : 0, "numCallsInPriorityQueue" : 0, "numOpenConnections" : 8, "numActiveHandler" : 0, "receivedBytes" : 200056552928, "exceptions.RegionMovedException" : 18, "authenticationSuccesses" : 0, "authorizationFailures" : 0, "TotalCallTime_num_ops" : 136533295, "TotalCallTime_min" : 0, "TotalCallTime_max" : 105372, "TotalCallTime_mean" : 2.2611075049496168, "TotalCallTime_median" : 2.0, "TotalCallTime_75th_percentile" : 4.0, "TotalCallTime_95th_percentile" : 20.0, "TotalCallTime_99th_percentile" : 33.059999999999945, "exceptions.RegionTooBusyException" : 0, "exceptions.FailedSanityCheckException" : 0, "exceptions.UnknownScannerException" : 24, "exceptions.OutOfOrderScannerNextException" : 0, "exceptions" : 63, "ProcessCallTime_num_ops" : 136533295, "ProcessCallTime_min" : 0, "ProcessCallTime_max" : 105372, "ProcessCallTime_mean" : 2.237145386405565, "ProcessCallTime_median" : 2.0, "ProcessCallTime_75th_percentile" : 4.0, "ProcessCallTime_95th_percentile" : 21.0, "ProcessCallTime_99th_percentile" : 41.889999999999645, "exceptions.NotServingRegionException" : 17, "authorizationSuccesses" : 29124, "sentBytes" : 3960132399278, "QueueCallTime_num_ops" : 136533295, "QueueCallTime_min" : 0, "QueueCallTime_max" : 1089, "QueueCallTime_mean" : 0.023962118544051836, "QueueCallTime_median" : 0.0, "QueueCallTime_75th_percentile" : 0.0, "QueueCallTime_95th_percentile" : 0.0, "QueueCallTime_99th_percentile" : 1.0, "authenticationFailures" : 0 } ], "beans" : [ { "name" : "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType" : "RegionServer,sub=Replication", "tag.Context" : "regionserver", "tag.Hostname" : "aps-hadoop5", "sink.appliedHFiles" : 0, "sink.appliedOps" : 0, "sink.ageOfLastAppliedOp" : 0, "sink.appliedBatches" : 0 } ], "beans" : [ { "name" : "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType" : "RegionServer,sub=Server", "tag.zookeeperQuorum" : "aps-hadoop6:2181,aps-hadoop7:2181,aps-hadoop4:2181", "tag.serverName" : "aps-hadoop5,16020,1493618413009", "tag.clusterId" : "fea6e6f3-b462-4e4c-946c-f35c02a0d2f4", "tag.Context" : "regionserver", "tag.Hostname" : "aps-hadoop5", "regionCount" : 108, "storeCount" : 187, "hlogFileCount" : 28, "hlogFileSize" : 3324263412, "storeFileCount" : 97, "memStoreSize" : 62518680, "storeFileSize" : 35807045717, "regionServerStartTime" : 1493618413009, "totalRequestCount" : 195629005, "readRequestCount" : 11927033647, "writeRequestCount" : 39695918, "checkMutateFailedCount" : 0, "checkMutatePassedCount" : 0, "storeFileIndexSize" : 738768, "staticIndexSize" : 147725776, "staticBloomSize" : 202830282, "mutationsWithoutWALCount" : 0, "mutationsWithoutWALSize" : 0, "percentFilesLocal" : 76, "percentFilesLocalSecondaryRegions" : 0, "splitQueueLength" : 0, "compactionQueueLength" : 0, "flushQueueLength" : 0, "blockCacheFreeSize" : 170760032, "blockCacheCount" : 49890, "blockCacheSize" : 3243744160, "blockCacheHitCount" : 182758961, "blockCacheHitCountPrimary" : 182758961, "blockCacheMissCount" : 179416500, "blockCacheMissCountPrimary" : 179416500, "blockCacheEvictionCount" : 173266434, "blockCacheEvictionCountPrimary" : 173266434, "blockCacheCountHitPercent" : 50.0, "blockCacheExpressHitPercent" : 51, "updatesBlockedTime" : 0, "flushedCellsCount" : 628337353, "compactedCellsCount" : 933193415, "majorCompactedCellsCount" : 368056660, "flushedCellsSize" : 209351367904, "compactedCellsSize" : 83719177083, "majorCompactedCellsSize" : 21049412113, "blockedRequestCount" : 204, "Mutate_num_ops" : 2106747, "Mutate_min" : 0, "Mutate_max" : 105372, "Mutate_mean" : 16.259541843420212, "Mutate_median" : 2.0, "Mutate_75th_percentile" : 12.0, "Mutate_95th_percentile" : 22.0, "Mutate_99th_percentile" : 37.58999999999992, "slowAppendCount" : 0, "slowDeleteCount" : 0, "Increment_num_ops" : 0, "Increment_min" : 0, "Increment_max" : 0, "Increment_mean" : 0.0, "Increment_median" : 0.0, "Increment_75th_percentile" : 0.0, "Increment_95th_percentile" : 0.0, "Increment_99th_percentile" : 0.0, "Replay_num_ops" : 0, "Replay_min" : 0, "Replay_max" : 0, "Replay_mean" : 0.0, "Replay_median" : 0.0, "Replay_75th_percentile" : 0.0, "Replay_95th_percentile" : 0.0, "Replay_99th_percentile" : 0.0, "FlushTime_num_ops" : 3119, "FlushTime_min" : 40, "FlushTime_max" : 94978, "FlushTime_mean" : 2237.1910868868226, "FlushTime_median" : 604.0, "FlushTime_75th_percentile" : 685.0, "FlushTime_95th_percentile" : 16036.099999999999, "FlushTime_99th_percentile" : 21930.009999999995, "Delete_num_ops" : 2, "Delete_min" : 1, "Delete_max" : 3, "Delete_mean" : 2.0, "Delete_median" : 2.0, "Delete_75th_percentile" : 3.0, "Delete_95th_percentile" : 3.0, "Delete_99th_percentile" : 3.0, "splitRequestCount" : 0, "splitSuccessCount" : 0, "slowGetCount" : 0, "Get_num_ops" : 0, "Get_min" : 0, "Get_max" : 0, "Get_mean" : 0.0, "Get_median" : 0.0, "Get_75th_percentile" : 0.0, "Get_95th_percentile" : 0.0, "Get_99th_percentile" : 0.0, "ScanNext_num_ops" : 134189861, "ScanNext_min" : 0, "ScanNext_max" : 2132622, "ScanNext_mean" : 29075.240877967673, "ScanNext_median" : 14800.0, "ScanNext_75th_percentile" : 15068.0, "ScanNext_95th_percentile" : 15200.0, "ScanNext_99th_percentile" : 69600.0, "slowPutCount" : 1938, "slowIncrementCount" : 0, "Append_num_ops" : 0, "Append_min" : 0, "Append_max" : 0, "Append_mean" : 0.0, "Append_median" : 0.0, "Append_75th_percentile" : 0.0, "Append_95th_percentile" : 0.0, "Append_99th_percentile" : 0.0, "SplitTime_num_ops" : 0, "SplitTime_min" : 0, "SplitTime_max" : 0, "SplitTime_mean" : 0.0, "SplitTime_median" : 0.0, "SplitTime_75th_percentile" : 0.0, "SplitTime_95th_percentile" : 0.0, "SplitTime_99th_percentile" : 0.0 } ] } 2017-05-03 21:22:44,970 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] regionserver.HRegionServer: STOPPED: Failed log close in log roller 2017-05-03 21:22:44,970 INFO [regionserver/aps-hadoop5/1..1..1..:16020.logRoller] regionserver.LogRoller: LogRoller exiting. 2017-05-03 21:22:44,970 INFO [regionserver/aps-hadoop5/1..1..1..:16020] regionserver.SplitLogWorker: Sending interrupt to stop the worker thread 2017-05-03 21:22:44,971 INFO [regionserver/aps-hadoop5/1..1..1..:16020] regionserver.HRegionServer: Stopping infoServer 2017-05-03 21:22:44,978 INFO [regionserver/aps-hadoop5/1..1..1..:16020] mortbay.log: Stopped SelectChannelConnector@0.0.0.0:16030 2017-05-03 21:22:44,980 INFO [SplitLogWorker-aps-hadoop5:16020] regionserver.SplitLogWorker: SplitLogWorker interrupted. Exiting. 2017-05-03 21:22:44,980 INFO [SplitLogWorker-aps-hadoop5:16020] regionserver.SplitLogWorker: SplitLogWorker aps-hadoop5,16020,1493618413009 exiting 2017-05-03 21:22:45,079 INFO [regionserver/aps-hadoop5/1..1..1..:16020] regionserver.HeapMemoryManager: Stoping HeapMemoryTuner chore. 2017-05-03 21:22:45,079 INFO [regionserver/aps-hadoop5/1..1..1..:16020] flush.RegionServerFlushTableProcedureManager: Stopping region server flush procedure manager abruptly. 2017-05-03 21:22:45,079 INFO [MemStoreFlusher.0] regionserver.MemStoreFlusher: MemStoreFlusher.0 exiting 2017-05-03 21:22:45,079 INFO [MemStoreFlusher.1] regionserver.MemStoreFlusher: MemStoreFlusher.1 exiting 2017-05-03 21:22:45,079 INFO [regionserver/aps-hadoop5/1..1..1..:16020] snapshot.RegionServerSnapshotManager: Stopping RegionServerSnapshotManager abruptly. 2017-05-03 21:22:45,080 INFO [StoreCloserThread-SocialMediaUserBehaviour,33333333,1478427797767.8fbbc679af9b7fb9dabc13a3ff3bc564.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,080 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed SocialMediaUserBehaviour,33333333,1478427797767.8fbbc679af9b7fb9dabc13a3ff3bc564. 2017-05-03 21:22:45,080 INFO [regionserver/aps-hadoop5/1..1..1..:16020] regionserver.HRegionServer: aborting server aps-hadoop5,16020,1493618413009 2017-05-03 21:22:45,080 INFO [regionserver/aps-hadoop5/1..1..1..:16020] client.ConnectionManager$HConnectionImplementation: Closing zookeeper sessionid=0x25887d92d0a8efe 2017-05-03 21:22:45,085 INFO [StoreCloserThread-HsearchIndexConfig,,1478428288064.c202098b27beefc3fb20a3d8fbbd4130.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,085 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed HsearchIndexConfig,,1478428288064.c202098b27beefc3fb20a3d8fbbd4130. 2017-05-03 21:22:45,085 INFO [StoreCloserThread-SocialMediaAnalyticsRecipients,,1478427863544.18fe1b7b6515c19926516abaffd3609d.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,085 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed SocialMediaAnalyticsRecipients,,1478427863544.18fe1b7b6515c19926516abaffd3609d. 2017-05-03 21:22:45,085 INFO [StoreCloserThread-CampaignSummaryOverview,99999999,1478427980057.219411b1cedb2fdc62988c9cf3c5130f.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,085 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed CampaignSummaryOverview,99999999,1478427980057.219411b1cedb2fdc62988c9cf3c5130f. 2017-05-03 21:22:45,085 INFO [StoreCloserThread-ModelsConfigSync,66666666,1478428291186.9d6f30ba4337c5538495e2b26f78e17c.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,085 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed ModelsConfigSync,66666666,1478428291186.9d6f30ba4337c5538495e2b26f78e17c. 2017-05-03 21:22:45,086 INFO [StoreCloserThread-CampaignSummaryDetailGeographyByCity,,1478428044795.0d7dc768580bf53b4e5310a049090e50.-1] regionserver.HStore: Closed Geo 2017-05-03 21:22:45,086 INFO [StoreCloserThread-CampaignSummaryDetailGeographyByCity,,1478428044795.0d7dc768580bf53b4e5310a049090e50.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,086 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed CampaignSummaryDetailGeographyByCity,,1478428044795.0d7dc768580bf53b4e5310a049090e50. 2017-05-03 21:22:45,088 INFO [StoreCloserThread-SMSCampaignStatus,zAcw12XmlZVPVA,1490858687987.84f8816bf598ded193753bc31fcc31b7.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,088 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed SMSCampaignStatus,zAcw12XmlZVPVA,1490858687987.84f8816bf598ded193753bc31fcc31b7. 2017-05-03 21:22:45,091 INFO [StoreCloserThread-CampaignSMSLinkClicksTransaction,33333333,1478427776736.f4dd8f0bb635f0766ae1707a25d84579.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,091 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed CampaignSMSLinkClicksTransaction,33333333,1478427776736.f4dd8f0bb635f0766ae1707a25d84579. 2017-05-03 21:22:45,092 INFO [StoreCloserThread-Campaigns,,1478354871976.b549cc0d22314c1b898911f0c44c8b2d.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,092 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed Campaigns,,1478354871976.b549cc0d22314c1b898911f0c44c8b2d. 2017-05-03 21:22:45,092 INFO [StoreCloserThread-CampaignSummaryDetailGeographyByCity,33333333,1478428044795.24132903fbe24036935e7929a777e894.-1] regionserver.HStore: Closed Geo 2017-05-03 21:22:45,092 INFO [StoreCloserThread-CampaignSummaryDetailGeographyByCity,33333333,1478428044795.24132903fbe24036935e7929a777e894.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,092 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed CampaignSummaryDetailGeographyByCity,33333333,1478428044795.24132903fbe24036935e7929a777e894. 2017-05-03 21:22:45,092 INFO [StoreCloserThread-CampaignSummaryOverview,,1478427980057.7acd0b6f16fd8a07a004ae4cb6d01e79.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,092 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed CampaignSummaryOverview,,1478427980057.7acd0b6f16fd8a07a004ae4cb6d01e79. 2017-05-03 21:22:45,094 INFO [StoreCloserThread-Campaigns,66666666,1478354871976.be5ece4d1769cbe600cef2510c6e7c8d.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,094 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed Campaigns,66666666,1478354871976.be5ece4d1769cbe600cef2510c6e7c8d. 2017-05-03 21:22:45,096 INFO [StoreCloserThread-CampaignBlastScheduleTransaction,lozAcwCDDipDD4JPK,1492484474153.745603a73cef6c58667e71a8cc985705.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,096 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed CampaignBlastScheduleTransaction,lozAcwCDDipDD4JPK,1492484474153.745603a73cef6c58667e71a8cc985705. 2017-05-03 21:22:45,099 INFO [StoreCloserThread-SMSCampaignStatus,zAcweT3AfSTSKL,1489129022632.68378a5fea855a228a5514a2cc696500.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,099 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed SMSCampaignStatus,zAcweT3AfSTSKL,1489129022632.68378a5fea855a228a5514a2cc696500. 2017-05-03 21:22:45,101 INFO [StoreCloserThread-CampaignPerformance,66666666,1478428211809.82260485c06f9076b59f25064eeb47cc.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,101 ERROR [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Memstore size is 5332736 2017-05-03 21:22:45,101 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed CampaignPerformance,66666666,1478428211809.82260485c06f9076b59f25064eeb47cc. 2017-05-03 21:22:45,101 INFO [StoreCloserThread-RecommendationToleranceValues,cccccccc,1478428275574.144f7d716b98a217ac6f969887dbc5e0.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,101 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed RecommendationToleranceValues,cccccccc,1478428275574.144f7d716b98a217ac6f969887dbc5e0. 2017-05-03 21:22:45,101 INFO [StoreCloserThread-UserAnalytics360OverviewBehaviour,cccccccc,1478428297431.5461123b9dc0917d7996e3e6cac530ba.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,101 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed UserAnalytics360OverviewBehaviour,cccccccc,1478428297431.5461123b9dc0917d7996e3e6cac530ba. 2017-05-03 21:22:45,101 INFO [StoreCloserThread-CampaignSMSTransaction,66666666,1479218455453.26d1ebb5d03132c78e8d8b351fe2027f.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,102 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed CampaignSMSTransaction,66666666,1479218455453.26d1ebb5d03132c78e8d8b351fe2027f. 2017-05-03 21:22:45,104 INFO [StoreCloserThread-CampaignSummaryFactsMDC,66666666,1478427948111.ad61a965e252fb9f6854dea225e4e2e3.-1] regionserver.HStore: Closed Em 2017-05-03 21:22:45,104 INFO [StoreCloserThread-CampaignSummaryFactsMDC,66666666,1478427948111.ad61a965e252fb9f6854dea225e4e2e3.-1] regionserver.HStore: Closed Fb 2017-05-03 21:22:45,104 INFO [StoreCloserThread-CampaignSummaryFactsMDC,66666666,1478427948111.ad61a965e252fb9f6854dea225e4e2e3.-1] regionserver.HStore: Closed Fp 2017-05-03 21:22:45,104 INFO [StoreCloserThread-CampaignSummaryFactsMDC,66666666,1478427948111.ad61a965e252fb9f6854dea225e4e2e3.-1] regionserver.HStore: Closed GP 2017-05-03 21:22:45,104 INFO [StoreCloserThread-CampaignSummaryGeographyByCityMDC,,1478428033615.b96cf41f23ada02429c7e5aefa3568dd.-1] regionserver.HStore: Closed Geo 2017-05-03 21:22:45,106 INFO [StoreCloserThread-CampaignSummaryFactsMDC,66666666,1478427948111.ad61a965e252fb9f6854dea225e4e2e3.-1] regionserver.HStore: Closed Mob 2017-05-03 21:22:45,106 INFO [StoreCloserThread-CampaignSummaryFactsMDC,66666666,1478427948111.ad61a965e252fb9f6854dea225e4e2e3.-1] regionserver.HStore: Closed Pi 2017-05-03 21:22:45,106 INFO [StoreCloserThread-CampaignSummaryGeographyByCityMDC,,1478428033615.b96cf41f23ada02429c7e5aefa3568dd.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,106 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed CampaignSummaryGeographyByCityMDC,,1478428033615.b96cf41f23ada02429c7e5aefa3568dd. 2017-05-03 21:22:45,108 INFO [StoreCloserThread-CampaignSummaryFactsMDC,66666666,1478427948111.ad61a965e252fb9f6854dea225e4e2e3.-1] regionserver.HStore: Closed Qr 2017-05-03 21:22:45,108 INFO [StoreCloserThread-CampaignSummaryFactsMDC,66666666,1478427948111.ad61a965e252fb9f6854dea225e4e2e3.-1] regionserver.HStore: Closed Tw 2017-05-03 21:22:45,108 INFO [StoreCloserThread-CampaignSummaryFactsMDC,66666666,1478427948111.ad61a965e252fb9f6854dea225e4e2e3.-1] regionserver.HStore: Closed Yt 2017-05-03 21:22:45,109 INFO [StoreCloserThread-CampaignBlastScheduleTransaction,00zAcwyzKG7HNFKPM,1493319850990.186f5de6827df65054da9e1780efac9e.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,109 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed CampaignBlastScheduleTransaction,00zAcwyzKG7HNFKPM,1493319850990.186f5de6827df65054da9e1780efac9e. 2017-05-03 21:22:45,110 INFO [StoreCloserThread-UserAnalytics360OverviewChannels,33333333,1478428300551.370362b5f1102029ab61e1c12cb551df.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,110 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed UserAnalytics360OverviewChannels,33333333,1478428300551.370362b5f1102029ab61e1c12cb551df. 2017-05-03 21:22:45,112 INFO [StoreCloserThread-EmailCampaignStatus,99999999,1490799576103.d6116cba04f2f7fb7afe52d1422e0834.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,112 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed EmailCampaignStatus,99999999,1490799576103.d6116cba04f2f7fb7afe52d1422e0834. 2017-05-03 21:22:45,112 INFO [StoreCloserThread-CampaignSummaryFactsMDC,66666666,1478427948111.ad61a965e252fb9f6854dea225e4e2e3.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,113 ERROR [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Memstore size is 147136 2017-05-03 21:22:45,113 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed CampaignSummaryFactsMDC,66666666,1478427948111.ad61a965e252fb9f6854dea225e4e2e3. 2017-05-03 21:22:45,116 INFO [regionserver/aps-hadoop5/1..1..1..:16020] zookeeper.ZooKeeper: Session: 0x25887d92d0a8efe closed 2017-05-03 21:22:45,117 INFO [regionserver/aps-hadoop5/1..1..1..:16020] regionserver.CompactSplitThread: Waiting for Split Thread to finish... 2017-05-03 21:22:45,117 INFO [regionserver/aps-hadoop5/1..1..1..:16020] regionserver.CompactSplitThread: Waiting for Merge Thread to finish... 2017-05-03 21:22:45,117 INFO [regionserver/aps-hadoop5/1..1..1..:16020] regionserver.CompactSplitThread: Waiting for Large Compaction Thread to finish... 2017-05-03 21:22:45,117 INFO [regionserver/aps-hadoop5/1..1..1..:16020] regionserver.CompactSplitThread: Waiting for Small Compaction Thread to finish... 2017-05-03 21:22:45,117 INFO [regionserver/aps-hadoop5/1..1..1..:16020] regionserver.HRegionServer: Waiting on 85 regions to close 2017-05-03 21:22:45,124 INFO [StoreCloserThread-DashboardROITrend,66666666,1478428376068.01b554541831d66c9ed0933ca2952256.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,124 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed DashboardROITrend,66666666,1478428376068.01b554541831d66c9ed0933ca2952256. 2017-05-03 21:22:45,124 INFO [StoreCloserThread-UserAnalytics360Overview,99999999,1478428294311.4767e8dbd53f27ca5fcf406a34a23803.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,125 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed UserAnalytics360Overview,99999999,1478428294311.4767e8dbd53f27ca5fcf406a34a23803. 2017-05-03 21:22:45,126 INFO [StoreCloserThread-CampaignSMSTransaction,FEzAcwHIIE5YL38,1488452469954.315844264f598f452e11c8480364f385.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,126 INFO [StoreCloserThread-CampaignMailBounce,zAcwjKf5X4BPKf20150825071313,1486099305438.9e75db0e3d8cfb75c50977dee786db56.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,127 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed CampaignSMSTransaction,FEzAcwHIIE5YL38,1488452469954.315844264f598f452e11c8480364f385. 2017-05-03 21:22:45,127 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-2] regionserver.HRegion: Closed CampaignMailBounce,zAcwjKf5X4BPKf20150825071313,1486099305438.9e75db0e3d8cfb75c50977dee786db56. 2017-05-03 21:22:45,127 INFO [StoreCloserThread-hbase:meta,,1.1588230740-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,127 INFO [StoreCloserThread-CampaignSummaryGeographyByCity,cccccccc,1478428019768.c8bd20a43a3df953cbb47ce31f18cb8f.-1] regionserver.HStore: Closed Geo 2017-05-03 21:22:45,127 INFO [StoreCloserThread-CampaignSummaryGeographyByCity,cccccccc,1478428019768.c8bd20a43a3df953cbb47ce31f18cb8f.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,127 INFO [RS_CLOSE_META-aps-hadoop5:16020-0] regionserver.HRegion: Closed hbase:meta,,1.1588230740 2017-05-03 21:22:45,127 INFO [StoreCloserThread-DashboardAverageTimeToConversion,33333333,1478428382583.bb76f8efe3c00d0e0c72e1e7388805a7.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,127 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-2] regionserver.HRegion: Closed CampaignSummaryGeographyByCity,cccccccc,1478428019768.c8bd20a43a3df953cbb47ce31f18cb8f. 2017-05-03 21:22:45,127 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed DashboardAverageTimeToConversion,33333333,1478428382583.bb76f8efe3c00d0e0c72e1e7388805a7. 2017-05-03 21:22:45,127 INFO [StoreCloserThread-SocialMediaPages,66666666,1478427791732.c96590e78a22c42f6f6d2662407c9c9a.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,127 INFO [StoreCloserThread-DashboardSummary_testing,66666666,1492429917459.0f97e8df31c00555642131ca759e8450.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,127 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-2] regionserver.HRegion: Closed SocialMediaPages,66666666,1478427791732.c96590e78a22c42f6f6d2662407c9c9a. 2017-05-03 21:22:45,127 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed DashboardSummary_testing,66666666,1492429917459.0f97e8df31c00555642131ca759e8450. 2017-05-03 21:22:45,127 INFO [StoreCloserThread-Recipients,zAcwGNW6Q,1482409867665.0eeb9b55ab67fb9da32253cc26ec21bf.-1] regionserver.HStore: Closed Prop 2017-05-03 21:22:45,127 INFO [StoreCloserThread-Recipients,zAcwGNW6Q,1482409867665.0eeb9b55ab67fb9da32253cc26ec21bf.-1] regionserver.HStore: Closed Seg 2017-05-03 21:22:45,127 INFO [StoreCloserThread-CampaignSummaryFactsByEMailClient,cccccccc,1478428266204.66b046e29ccf739427e1d6751d426918.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,128 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed CampaignSummaryFactsByEMailClient,cccccccc,1478428266204.66b046e29ccf739427e1d6751d426918. 2017-05-03 21:22:45,128 INFO [StoreCloserThread-SocialMediaUserBehaviour,99999999,1478427797767.556592ee81df734945850724feb2f931.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,128 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed SocialMediaUserBehaviour,99999999,1478427797767.556592ee81df734945850724feb2f931. 2017-05-03 21:22:45,128 INFO [StoreCloserThread-DashboardChannelWisePerformance,33333333,1478428370167.4d873483c34240b7afdfd1e39e3c3d87.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,128 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed DashboardChannelWisePerformance,33333333,1478428370167.4d873483c34240b7afdfd1e39e3c3d87. 2017-05-03 21:22:45,128 INFO [StoreCloserThread-CampaignSummaryIndustrySegment,,1478428096368.bb225673e1cdda4e705d3f144b490d34.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,128 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed CampaignSummaryIndustrySegment,,1478428096368.bb225673e1cdda4e705d3f144b490d34. 2017-05-03 21:22:45,130 INFO [StoreCloserThread-Recipients,zAcwGNW6Q,1482409867665.0eeb9b55ab67fb9da32253cc26ec21bf.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,130 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-2] regionserver.HRegion: Closed Recipients,zAcwGNW6Q,1482409867665.0eeb9b55ab67fb9da32253cc26ec21bf. 2017-05-03 21:22:45,131 INFO [regionserver/aps-hadoop5/1..1..1..:16020-EventThread] zookeeper.ClientCnxn: EventThread shut down 2017-05-03 21:22:45,131 INFO [StoreCloserThread-CampaignMailBounce,b8955bcf_a1ea_4434_8919_c75472d06b44_000_5e4a129b-77cb-493f-abf6-5f06c32ccc12_c5d62550-58e9-11e4-89b2-06ab9bb94413_3a23d9ab-c334-4fc4-a91d-1ac9ee39ab78_20150610102126,1489993247729.17564b5b0f3f5c903c620950814a3a9c.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,131 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed CampaignMailBounce,b8955bcf_a1ea_4434_8919_c75472d06b44_000_5e4a129b-77cb-493f-abf6-5f06c32ccc12_c5d62550-58e9-11e4-89b2-06ab9bb94413_3a23d9ab-c334-4fc4-a91d-1ac9ee39ab78_20150610102126,1489993247729.17564b5b0f3f5c903c620950814a3a9c. 2017-05-03 21:22:45,131 INFO [StoreCloserThread-CampaignSMSTransaction_testing,66666666,1493727222020.3c215ba34acfbfe9966c8192c4e7a320.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,131 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed CampaignSMSTransaction_testing,66666666,1493727222020.3c215ba34acfbfe9966c8192c4e7a320. 2017-05-03 21:22:45,132 INFO [StoreCloserThread-CampaignMailBounce,99999999,1489993247729.3984b76da98de8d90f416d59782461bd.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,132 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-2] regionserver.HRegion: Closed CampaignMailBounce,99999999,1489993247729.3984b76da98de8d90f416d59782461bd. 2017-05-03 21:22:45,132 INFO [StoreCloserThread-DashboardCampaignPerformance,33333333,1478428360041.56871c93d1283b987c9e587fb7d23888.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,132 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-2] regionserver.HRegion: Closed DashboardCampaignPerformance,33333333,1478428360041.56871c93d1283b987c9e587fb7d23888. 2017-05-03 21:22:45,133 INFO [StoreCloserThread-CampaignSummaryIndustrySegment,99999999,1478428096368.b2b638af07f53576a15b94bdb83697c0.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,133 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-2] regionserver.HRegion: Closed CampaignSummaryIndustrySegment,99999999,1478428096368.b2b638af07f53576a15b94bdb83697c0. 2017-05-03 21:22:45,133 INFO [StoreCloserThread-CampaignSMSTransaction,99999999,1493279834786.d5adacd3b69c6ace6e8bf702b8ce47c9.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,133 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed CampaignSMSTransaction,99999999,1493279834786.d5adacd3b69c6ace6e8bf702b8ce47c9. 2017-05-03 21:22:45,133 INFO [StoreCloserThread-UserAnalytics360ViralityMap,cccccccc,1478428479264.1e30bf052601f8a0d20809cb289bf5d3.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,134 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed UserAnalytics360ViralityMap,cccccccc,1478428479264.1e30bf052601f8a0d20809cb289bf5d3. 2017-05-03 21:22:45,141 INFO [StoreCloserThread-SMSCampaignStatus,zAcwYZCnKBWN7,1492102956993.2cdd9ba7e4ed3e8e1c02b403f28ed11d.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,141 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed SMSCampaignStatus,zAcwYZCnKBWN7,1492102956993.2cdd9ba7e4ed3e8e1c02b403f28ed11d. 2017-05-03 21:22:45,145 INFO [StoreCloserThread-AverageCampaignPerformance,33333333,1478428256763.573482bf2396e92f7ea555322e355f0c.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,145 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed AverageCampaignPerformance,33333333,1478428256763.573482bf2396e92f7ea555322e355f0c. 2017-05-03 21:22:45,147 INFO [StoreCloserThread-test1,cccccccc,1480081852792.de85cbdc154401a72f9fed6614ba81c8.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,147 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed test1,cccccccc,1480081852792.de85cbdc154401a72f9fed6614ba81c8. 2017-05-03 21:22:45,152 INFO [StoreCloserThread-CampaignFacts,,1478427883316.0bd5902b2373e4730587ec7d7dfccaef.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,152 ERROR [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Memstore size is 22400 2017-05-03 21:22:45,152 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed CampaignFacts,,1478427883316.0bd5902b2373e4730587ec7d7dfccaef. 2017-05-03 21:22:45,155 INFO [StoreCloserThread-ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,156 ERROR [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Memstore size is 43616904 2017-05-03 21:22:45,156 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed ORMDetails,GkMwabF20152rs_c150f3f5d4a239a5ee3593654e1a6c16D13newsNegativeen20170122224825,1493286144460.96ab0ca6ba008280cd0cc841abeeb413. 2017-05-03 21:22:45,156 INFO [StoreCloserThread-CampaignPerformance,99999999,1478428211809.93242887b60d2de4e548bf4d53e76d37.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,156 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed CampaignPerformance,99999999,1478428211809.93242887b60d2de4e548bf4d53e76d37. 2017-05-03 21:22:45,159 INFO [StoreCloserThread-CampaignSMSLinkClicksTransaction,cccccccc,1488471469014.97e16a2e0b57885a84182c756426bf32.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,159 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed CampaignSMSLinkClicksTransaction,cccccccc,1488471469014.97e16a2e0b57885a84182c756426bf32. 2017-05-03 21:22:45,159 INFO [StoreCloserThread-WeekDayList,66666666,1478427657453.d4f6571bcecbd8df235fcb45c4e908a2.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,160 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed WeekDayList,66666666,1478427657453.d4f6571bcecbd8df235fcb45c4e908a2. 2017-05-03 21:22:45,161 INFO [StoreCloserThread-CampaignSMSTransaction,FEzAcw2qdJ1C87CZ201703101800,1493279834786.a275c53c602ba187af3a6a0a57bf8758.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,161 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed CampaignSMSTransaction,FEzAcw2qdJ1C87CZ201703101800,1493279834786.a275c53c602ba187af3a6a0a57bf8758. 2017-05-03 21:22:45,161 INFO [StoreCloserThread-CampaignSummaryTrends_New,34nwldhMobileH2153208,1486102387783.a0652debed1258c1fd31f2b27b561b44.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,161 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed CampaignSummaryTrends_New,34nwldhMobileH2153208,1486102387783.a0652debed1258c1fd31f2b27b561b44. 2017-05-03 21:22:45,162 INFO [StoreCloserThread-CampaignSMSLinkClicksTransaction,fizAcwfg3Z8NX5ba20160610151547,1488471469014.a015ecaddcf767a1bdd464e7a3615e6a.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,162 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-2] regionserver.HRegion: Closed CampaignSMSLinkClicksTransaction,fizAcwfg3Z8NX5ba20160610151547,1488471469014.a015ecaddcf767a1bdd464e7a3615e6a. 2017-05-03 21:22:45,163 INFO [StoreCloserThread-CampaignConversion,66666666,1478427841199.8eacb0eb4299655cb87525a22bfdf909.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,163 INFO [StoreCloserThread-CampaignConversionTracking,66666666,1478427850748.fd9b0a1c14e5d8e49156ab69e1ac470d.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,163 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed CampaignConversion,66666666,1478427841199.8eacb0eb4299655cb87525a22bfdf909. 2017-05-03 21:22:45,163 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed CampaignConversionTracking,66666666,1478427850748.fd9b0a1c14e5d8e49156ab69e1ac470d. 2017-05-03 21:22:45,163 INFO [StoreCloserThread-Recipients,zAcw44PJD,1490855470855.f134dc7afaceeb7305bec7119b799530.-1] regionserver.HStore: Closed Prop 2017-05-03 21:22:45,163 INFO [StoreCloserThread-Recipients,zAcw44PJD,1490855470855.f134dc7afaceeb7305bec7119b799530.-1] regionserver.HStore: Closed Seg 2017-05-03 21:22:45,163 INFO [StoreCloserThread-CampaignSummaryGeographyByCity,,1478428019768.02dfc8309104bc69b562b955e44f21e0.-1] regionserver.HStore: Closed Geo 2017-05-03 21:22:45,163 INFO [StoreCloserThread-CampaignSummaryGeographyByCity,,1478428019768.02dfc8309104bc69b562b955e44f21e0.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,163 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed CampaignSummaryGeographyByCity,,1478428019768.02dfc8309104bc69b562b955e44f21e0. 2017-05-03 21:22:45,164 INFO [StoreCloserThread-Recipients,zAcw44PJD,1490855470855.f134dc7afaceeb7305bec7119b799530.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,165 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed Recipients,zAcw44PJD,1490855470855.f134dc7afaceeb7305bec7119b799530. 2017-05-03 21:22:45,165 INFO [StoreCloserThread-SocialMediaAnalyticsRecipients,66666666,1478427863544.e67c1301dd29e91602c633432b1414f0.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,165 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed SocialMediaAnalyticsRecipients,66666666,1478427863544.e67c1301dd29e91602c633432b1414f0. 2017-05-03 21:22:45,165 INFO [StoreCloserThread-CampaignSummaryOverview,cccccccc,1478427980057.509d7a61c7d6510b1c693f1cdf777692.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,165 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed CampaignSummaryOverview,cccccccc,1478427980057.509d7a61c7d6510b1c693f1cdf777692. 2017-05-03 21:22:45,166 INFO [StoreCloserThread-CampaignSummaryFactsMDC,99999999,1478427948111.fcf0380fd986f7c3d3f3bf298e91e504.-1] regionserver.HStore: Closed Em 2017-05-03 21:22:45,166 INFO [StoreCloserThread-CampaignSummaryFactsMDC,99999999,1478427948111.fcf0380fd986f7c3d3f3bf298e91e504.-1] regionserver.HStore: Closed Fb 2017-05-03 21:22:45,166 INFO [StoreCloserThread-CampaignSummaryFactsMDC,99999999,1478427948111.fcf0380fd986f7c3d3f3bf298e91e504.-1] regionserver.HStore: Closed Fp 2017-05-03 21:22:45,166 INFO [StoreCloserThread-CampaignSummaryFactsMDC,99999999,1478427948111.fcf0380fd986f7c3d3f3bf298e91e504.-1] regionserver.HStore: Closed GP 2017-05-03 21:22:45,166 INFO [StoreCloserThread-CampaignSummaryFactsMDC,,1478427948111.6ac61aca525b2e3f533242e47051518a.-1] regionserver.HStore: Closed Em 2017-05-03 21:22:45,166 INFO [StoreCloserThread-CampaignSummaryFactsMDC,,1478427948111.6ac61aca525b2e3f533242e47051518a.-1] regionserver.HStore: Closed Fb 2017-05-03 21:22:45,166 INFO [StoreCloserThread-CampaignSummaryFactsMDC,,1478427948111.6ac61aca525b2e3f533242e47051518a.-1] regionserver.HStore: Closed Fp 2017-05-03 21:22:45,166 INFO [StoreCloserThread-CampaignSummaryFactsMDC,,1478427948111.6ac61aca525b2e3f533242e47051518a.-1] regionserver.HStore: Closed GP 2017-05-03 21:22:45,167 INFO [StoreCloserThread-CampaignMailForwardTransaction,66666666,1478427734777.f6eab7cdb3ef9914fc3de2854d096856.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,167 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed CampaignMailForwardTransaction,66666666,1478427734777.f6eab7cdb3ef9914fc3de2854d096856. 2017-05-03 21:22:45,168 INFO [StoreCloserThread-CampaignSummaryFactsMDC,,1478427948111.6ac61aca525b2e3f533242e47051518a.-1] regionserver.HStore: Closed Mob 2017-05-03 21:22:45,168 INFO [StoreCloserThread-CampaignSummaryFactsMDC,,1478427948111.6ac61aca525b2e3f533242e47051518a.-1] regionserver.HStore: Closed Pi 2017-05-03 21:22:45,168 INFO [StoreCloserThread-CampaignSummaryFactsMDC,99999999,1478427948111.fcf0380fd986f7c3d3f3bf298e91e504.-1] regionserver.HStore: Closed Mob 2017-05-03 21:22:45,168 INFO [StoreCloserThread-CampaignSummaryFactsMDC,99999999,1478427948111.fcf0380fd986f7c3d3f3bf298e91e504.-1] regionserver.HStore: Closed Pi 2017-05-03 21:22:45,169 INFO [StoreCloserThread-CampaignSummaryFactsMDC,,1478427948111.6ac61aca525b2e3f533242e47051518a.-1] regionserver.HStore: Closed Qr 2017-05-03 21:22:45,169 INFO [StoreCloserThread-CampaignBlastScheduleTransaction,,1493319850990.7fdf9a1d2001f9d037a0679a91c09ed7.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,169 INFO [StoreCloserThread-CampaignSummaryFactsMDC,,1478427948111.6ac61aca525b2e3f533242e47051518a.-1] regionserver.HStore: Closed Tw 2017-05-03 21:22:45,169 INFO [StoreCloserThread-CampaignSummaryFactsMDC,,1478427948111.6ac61aca525b2e3f533242e47051518a.-1] regionserver.HStore: Closed Yt 2017-05-03 21:22:45,169 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed CampaignBlastScheduleTransaction,,1493319850990.7fdf9a1d2001f9d037a0679a91c09ed7. 2017-05-03 21:22:45,169 INFO [StoreCloserThread-CampaignBlastScheduleTransactionByRecipientGUID,99999999,1478428471741.1636391c920ab3a7c1b13276f671c34c.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,169 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed CampaignBlastScheduleTransactionByRecipientGUID,99999999,1478428471741.1636391c920ab3a7c1b13276f671c34c. 2017-05-03 21:22:45,169 INFO [StoreCloserThread-CampaignSummaryDetailGeographyByCityMDC,cccccccc,1478428063068.2860d686a4589ad57488c2b034722a16.-1] regionserver.HStore: Closed Geo 2017-05-03 21:22:45,170 INFO [StoreCloserThread-CampaignSummaryFactsMDC,99999999,1478427948111.fcf0380fd986f7c3d3f3bf298e91e504.-1] regionserver.HStore: Closed Qr 2017-05-03 21:22:45,170 INFO [StoreCloserThread-CampaignSummaryFactsMDC,99999999,1478427948111.fcf0380fd986f7c3d3f3bf298e91e504.-1] regionserver.HStore: Closed Tw 2017-05-03 21:22:45,170 INFO [StoreCloserThread-CampaignSummaryFactsMDC,99999999,1478427948111.fcf0380fd986f7c3d3f3bf298e91e504.-1] regionserver.HStore: Closed Yt 2017-05-03 21:22:45,171 INFO [StoreCloserThread-CampaignSummaryFactsMDC,,1478427948111.6ac61aca525b2e3f533242e47051518a.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,171 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed CampaignSummaryFactsMDC,,1478427948111.6ac61aca525b2e3f533242e47051518a. 2017-05-03 21:22:45,171 INFO [StoreCloserThread-Recipients,zAcw6BVHS2,1482409860416.dcdb893c4076eef7214e49c63160e022.-1] regionserver.HStore: Closed Prop 2017-05-03 21:22:45,171 INFO [StoreCloserThread-Recipients,zAcw6BVHS2,1482409860416.dcdb893c4076eef7214e49c63160e022.-1] regionserver.HStore: Closed Seg 2017-05-03 21:22:45,171 INFO [StoreCloserThread-CampaignSummaryDetailGeographyByCityMDC,cccccccc,1478428063068.2860d686a4589ad57488c2b034722a16.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,171 ERROR [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Memstore size is 22177488 2017-05-03 21:22:45,171 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed CampaignSummaryDetailGeographyByCityMDC,cccccccc,1478428063068.2860d686a4589ad57488c2b034722a16. 2017-05-03 21:22:45,171 INFO [StoreCloserThread-CampaignRecommendation,66666666,1478428313095.776626c69dcf7552c9c169862d0acea1.-1] regionserver.HStore: Closed Em 2017-05-03 21:22:45,171 INFO [StoreCloserThread-CampaignRecommendation,66666666,1478428313095.776626c69dcf7552c9c169862d0acea1.-1] regionserver.HStore: Closed Mb 2017-05-03 21:22:45,171 INFO [StoreCloserThread-CampaignRecommendation,66666666,1478428313095.776626c69dcf7552c9c169862d0acea1.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,171 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed CampaignRecommendation,66666666,1478428313095.776626c69dcf7552c9c169862d0acea1. 2017-05-03 21:22:45,171 INFO [StoreCloserThread-UserAnalytics360RecipientCampaign,33333333,1478428494703.1671d7ea8b2ee29c65e1198b4570543e.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,171 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed UserAnalytics360RecipientCampaign,33333333,1478428494703.1671d7ea8b2ee29c65e1198b4570543e. 2017-05-03 21:22:45,172 INFO [StoreCloserThread-HsearchIndexConfig,33333333,1478428288064.78ea07d7ee21ad26fb1da5658851d311.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,172 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed HsearchIndexConfig,33333333,1478428288064.78ea07d7ee21ad26fb1da5658851d311. 2017-05-03 21:22:45,172 INFO [StoreCloserThread-Recipients,zAcw6BVHS2,1482409860416.dcdb893c4076eef7214e49c63160e022.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,172 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed Recipients,zAcw6BVHS2,1482409860416.dcdb893c4076eef7214e49c63160e022. 2017-05-03 21:22:45,173 INFO [StoreCloserThread-CampaignSummaryFactsMDC,99999999,1478427948111.fcf0380fd986f7c3d3f3bf298e91e504.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,173 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-2] regionserver.HRegion: Closed CampaignSummaryFactsMDC,99999999,1478427948111.fcf0380fd986f7c3d3f3bf298e91e504. 2017-05-03 21:22:45,176 INFO [StoreCloserThread-CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494.-1] regionserver.HStore: Closed Em 2017-05-03 21:22:45,176 INFO [StoreCloserThread-CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494.-1] regionserver.HStore: Closed Fb 2017-05-03 21:22:45,176 INFO [StoreCloserThread-CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494.-1] regionserver.HStore: Closed Fp 2017-05-03 21:22:45,176 INFO [StoreCloserThread-CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494.-1] regionserver.HStore: Closed Ga 2017-05-03 21:22:45,176 INFO [StoreCloserThread-CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494.-1] regionserver.HStore: Closed Lnk 2017-05-03 21:22:45,176 INFO [StoreCloserThread-CampaignSMSLinkClicksTransaction,,1481276951756.90aac44a072294a0f0c1083dc7aee2ca.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,176 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed CampaignSMSLinkClicksTransaction,,1481276951756.90aac44a072294a0f0c1083dc7aee2ca. 2017-05-03 21:22:45,177 INFO [StoreCloserThread-IndustryBenchMark,cccccccc,1478428278697.110cbdd3996f4937bfaa7e442abefc66.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,177 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed IndustryBenchMark,cccccccc,1478428278697.110cbdd3996f4937bfaa7e442abefc66. 2017-05-03 21:22:45,177 INFO [StoreCloserThread-UserAnalytics360UserReports,33333333,1478428457233.cdb2abc48692634b7bd8f08011c0b7bb.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,177 INFO [StoreCloserThread-CampaignSummaryFactsMDC,33333333,1478427948111.97c67df4b304db2ee9591a9347fd0273.-1] regionserver.HStore: Closed Em 2017-05-03 21:22:45,177 INFO [StoreCloserThread-CampaignSummaryFactsMDC,33333333,1478427948111.97c67df4b304db2ee9591a9347fd0273.-1] regionserver.HStore: Closed Fb 2017-05-03 21:22:45,177 INFO [StoreCloserThread-CampaignSummaryFactsMDC,33333333,1478427948111.97c67df4b304db2ee9591a9347fd0273.-1] regionserver.HStore: Closed Fp 2017-05-03 21:22:45,177 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed UserAnalytics360UserReports,33333333,1478428457233.cdb2abc48692634b7bd8f08011c0b7bb. 2017-05-03 21:22:45,177 INFO [StoreCloserThread-CampaignSummaryFactsMDC,33333333,1478427948111.97c67df4b304db2ee9591a9347fd0273.-1] regionserver.HStore: Closed GP 2017-05-03 21:22:45,177 INFO [StoreCloserThread-UserAnalytics360ViralityMap,,1478428479264.b979f32d86456c3f54e225892c9108d0.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,177 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed UserAnalytics360ViralityMap,,1478428479264.b979f32d86456c3f54e225892c9108d0. 2017-05-03 21:22:45,177 INFO [StoreCloserThread-WebAnalyticsUserFlow,33333333,1478513262242.97bc62eb21ddbf3a5656ceea704c42d1.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,177 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed WebAnalyticsUserFlow,33333333,1478513262242.97bc62eb21ddbf3a5656ceea704c42d1. 2017-05-03 21:22:45,177 INFO [StoreCloserThread-CampaignBlastScheduleTransactionCount,99999999,1478428104476.58d98ef45cf3925b2339b65d15e35166.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,178 INFO [StoreCloserThread-CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494.-1] regionserver.HStore: Closed Mb 2017-05-03 21:22:45,178 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed CampaignBlastScheduleTransactionCount,99999999,1478428104476.58d98ef45cf3925b2339b65d15e35166. 2017-05-03 21:22:45,178 INFO [StoreCloserThread-SocialMediaInsightsDetail_New,33333333,1478427825373.135df1b4962d7859c8a42671dc536a82.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,178 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed SocialMediaInsightsDetail_New,33333333,1478427825373.135df1b4962d7859c8a42671dc536a82. 2017-05-03 21:22:45,179 INFO [StoreCloserThread-CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494.-1] regionserver.HStore: Closed Qr 2017-05-03 21:22:45,179 INFO [StoreCloserThread-CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494.-1] regionserver.HStore: Closed Ra 2017-05-03 21:22:45,179 INFO [StoreCloserThread-CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494.-1] regionserver.HStore: Closed Tw 2017-05-03 21:22:45,179 INFO [StoreCloserThread-CampaignSummaryFactsMDC,33333333,1478427948111.97c67df4b304db2ee9591a9347fd0273.-1] regionserver.HStore: Closed Mob 2017-05-03 21:22:45,179 INFO [StoreCloserThread-CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494.-1] regionserver.HStore: Closed We 2017-05-03 21:22:45,179 INFO [StoreCloserThread-CampaignSummaryFactsMDC,33333333,1478427948111.97c67df4b304db2ee9591a9347fd0273.-1] regionserver.HStore: Closed Pi 2017-05-03 21:22:45,179 INFO [StoreCloserThread-CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494.-1] regionserver.HStore: Closed Yt 2017-05-03 21:22:45,182 INFO [StoreCloserThread-CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,182 ERROR [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Memstore size is 12046056 2017-05-03 21:22:45,182 INFO [StoreCloserThread-CampaignGoalMDC,33333333,1478427913612.29d8ed7f7a9c5e6cd3537015ad67fd01.-1] regionserver.HStore: Closed Em 2017-05-03 21:22:45,182 INFO [StoreCloserThread-CampaignGoalMDC,33333333,1478427913612.29d8ed7f7a9c5e6cd3537015ad67fd01.-1] regionserver.HStore: Closed Fb 2017-05-03 21:22:45,182 INFO [StoreCloserThread-CampaignGoalMDC,33333333,1478427913612.29d8ed7f7a9c5e6cd3537015ad67fd01.-1] regionserver.HStore: Closed Fp 2017-05-03 21:22:45,182 INFO [StoreCloserThread-CampaignGoalMDC,33333333,1478427913612.29d8ed7f7a9c5e6cd3537015ad67fd01.-1] regionserver.HStore: Closed Ga 2017-05-03 21:22:45,182 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed CampaignGoalMDC,cccccccc,1478427913612.6cf8953db275a43b98ab944d2d614494. 2017-05-03 21:22:45,182 INFO [StoreCloserThread-CampaignGoalMDC,33333333,1478427913612.29d8ed7f7a9c5e6cd3537015ad67fd01.-1] regionserver.HStore: Closed Lnk 2017-05-03 21:22:45,182 INFO [StoreCloserThread-CampaignSummaryFactsMDC,33333333,1478427948111.97c67df4b304db2ee9591a9347fd0273.-1] regionserver.HStore: Closed Qr 2017-05-03 21:22:45,183 INFO [StoreCloserThread-DashboardCampaignLeadsChannelWise,cccccccc,1478428306794.17da2404716950425deff2bfa3e13daf.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,183 INFO [StoreCloserThread-CampaignSummaryFactsMDC,33333333,1478427948111.97c67df4b304db2ee9591a9347fd0273.-1] regionserver.HStore: Closed Tw 2017-05-03 21:22:45,183 INFO [StoreCloserThread-CampaignSummaryFactsMDC,33333333,1478427948111.97c67df4b304db2ee9591a9347fd0273.-1] regionserver.HStore: Closed Yt 2017-05-03 21:22:45,183 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed DashboardCampaignLeadsChannelWise,cccccccc,1478428306794.17da2404716950425deff2bfa3e13daf. 2017-05-03 21:22:45,183 INFO [StoreCloserThread-UserAnalytics360Overview,,1478428294311.18534adff7177f17b9f91ff08226c6ba.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,183 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed UserAnalytics360Overview,,1478428294311.18534adff7177f17b9f91ff08226c6ba. 2017-05-03 21:22:45,184 INFO [StoreCloserThread-CampaignGoalMDC,33333333,1478427913612.29d8ed7f7a9c5e6cd3537015ad67fd01.-1] regionserver.HStore: Closed Mb 2017-05-03 21:22:45,184 INFO [StoreCloserThread-CampaignSummaryFactsMDC,33333333,1478427948111.97c67df4b304db2ee9591a9347fd0273.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,184 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-2] regionserver.HRegion: Closed CampaignSummaryFactsMDC,33333333,1478427948111.97c67df4b304db2ee9591a9347fd0273. 2017-05-03 21:22:45,184 INFO [StoreCloserThread-SocialMediaRecipients,,1478427832245.cd7def8ccf96199154c2b6808a4f3c16.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,185 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-2] regionserver.HRegion: Closed SocialMediaRecipients,,1478427832245.cd7def8ccf96199154c2b6808a4f3c16. 2017-05-03 21:22:45,185 INFO [StoreCloserThread-CampaignGoalMDC,33333333,1478427913612.29d8ed7f7a9c5e6cd3537015ad67fd01.-1] regionserver.HStore: Closed Qr 2017-05-03 21:22:45,185 INFO [StoreCloserThread-CampaignGoalMDC,33333333,1478427913612.29d8ed7f7a9c5e6cd3537015ad67fd01.-1] regionserver.HStore: Closed Ra 2017-05-03 21:22:45,185 INFO [StoreCloserThread-CampaignGoalMDC,33333333,1478427913612.29d8ed7f7a9c5e6cd3537015ad67fd01.-1] regionserver.HStore: Closed Tw 2017-05-03 21:22:45,185 INFO [StoreCloserThread-CampaignGoalMDC,33333333,1478427913612.29d8ed7f7a9c5e6cd3537015ad67fd01.-1] regionserver.HStore: Closed We 2017-05-03 21:22:45,185 INFO [StoreCloserThread-CampaignGoalMDC,33333333,1478427913612.29d8ed7f7a9c5e6cd3537015ad67fd01.-1] regionserver.HStore: Closed Yt 2017-05-03 21:22:45,223 INFO [StoreCloserThread-CampaignConversionTracking,cccccccc,1478427850748.65f062da0f0374e8996f4301dc654230.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,223 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-2] regionserver.HRegion: Closed CampaignConversionTracking,cccccccc,1478427850748.65f062da0f0374e8996f4301dc654230. 2017-05-03 21:22:45,224 INFO [StoreCloserThread-CampaignGoalMDC,33333333,1478427913612.29d8ed7f7a9c5e6cd3537015ad67fd01.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,224 INFO [StoreCloserThread-DashboardAverageTimeToConversion,,1478428382583.55621d078219838922fe86e92d434da5.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,224 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed CampaignGoalMDC,33333333,1478427913612.29d8ed7f7a9c5e6cd3537015ad67fd01. 2017-05-03 21:22:45,224 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-2] regionserver.HRegion: Closed DashboardAverageTimeToConversion,,1478428382583.55621d078219838922fe86e92d434da5. 2017-05-03 21:22:45,224 INFO [StoreCloserThread-UserAnalytics360ViralityMap,99999999,1478428479264.9fabc6c22db13e2a21b2e5974bb2688f.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,224 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed UserAnalytics360ViralityMap,99999999,1478428479264.9fabc6c22db13e2a21b2e5974bb2688f. 2017-05-03 21:22:45,224 INFO [StoreCloserThread-RecipientsSocialMediaInfo,33333333,1478427803572.8517234053d212aa7d55d117a613186f.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,224 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-2] regionserver.HRegion: Closed RecipientsSocialMediaInfo,33333333,1478427803572.8517234053d212aa7d55d117a613186f. 2017-05-03 21:22:45,225 INFO [StoreCloserThread-Recipients,NOBMUmb,1478766090155.6b464780dd9ff29d769a5c87d007e6de.-1] regionserver.HStore: Closed Prop 2017-05-03 21:22:45,225 INFO [StoreCloserThread-Recipients,NOBMUmb,1478766090155.6b464780dd9ff29d769a5c87d007e6de.-1] regionserver.HStore: Closed Seg 2017-05-03 21:22:45,226 INFO [StoreCloserThread-CampaignMailBounce,zAcwSTE3LAD6420150825071313,1489815449837.07f8e516bcef00b0068fd4d53b4d0919.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,226 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-2] regionserver.HRegion: Closed CampaignMailBounce,zAcwSTE3LAD6420150825071313,1489815449837.07f8e516bcef00b0068fd4d53b4d0919. 2017-05-03 21:22:45,226 INFO [StoreCloserThread-DashboardChannelWisePerformance,,1478428370167.ea51863f0d80b5b9a4f09d039b95eeb6.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,226 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-2] regionserver.HRegion: Closed DashboardChannelWisePerformance,,1478428370167.ea51863f0d80b5b9a4f09d039b95eeb6. 2017-05-03 21:22:45,226 INFO [StoreCloserThread-CampaignBlastScheduleTransactionByMessageID,cccccccc,1478428005110.e34707f008f4b390b4553cecab8fb804.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,226 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-2] regionserver.HRegion: Closed CampaignBlastScheduleTransactionByMessageID,cccccccc,1478428005110.e34707f008f4b390b4553cecab8fb804. 2017-05-03 21:22:45,226 INFO [StoreCloserThread-CampaignSummaryImpact,cccccccc,1478427995538.cbe227556c858a5507939b264173c203.-1] regionserver.HStore: Closed Dg 2017-05-03 21:22:45,226 INFO [StoreCloserThread-CampaignSummaryImpact,cccccccc,1478427995538.cbe227556c858a5507939b264173c203.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,226 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-2] regionserver.HRegion: Closed CampaignSummaryImpact,cccccccc,1478427995538.cbe227556c858a5507939b264173c203. 2017-05-03 21:22:45,226 INFO [StoreCloserThread-CampaignSummaryDetailGeographyByCity,cccccccc,1478428044795.7cbe4ad8f9dd7f3fb0b8a44b27b3bb03.-1] regionserver.HStore: Closed Geo 2017-05-03 21:22:45,226 INFO [StoreCloserThread-CampaignSummaryDetailGeographyByCity,cccccccc,1478428044795.7cbe4ad8f9dd7f3fb0b8a44b27b3bb03.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,226 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-2] regionserver.HRegion: Closed CampaignSummaryDetailGeographyByCity,cccccccc,1478428044795.7cbe4ad8f9dd7f3fb0b8a44b27b3bb03. 2017-05-03 21:22:45,227 INFO [StoreCloserThread-Recipients,NOBMUmb,1478766090155.6b464780dd9ff29d769a5c87d007e6de.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,227 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed Recipients,NOBMUmb,1478766090155.6b464780dd9ff29d769a5c87d007e6de. 2017-05-03 21:22:45,227 INFO [StoreCloserThread-CampaignSummaryGeographyByCity,99999999,1478428019768.c2b1c9c11742bc744f07e0ff91cca65e.-1] regionserver.HStore: Closed Geo 2017-05-03 21:22:45,227 INFO [StoreCloserThread-CampaignSummaryGeographyByCity,99999999,1478428019768.c2b1c9c11742bc744f07e0ff91cca65e.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,227 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed CampaignSummaryGeographyByCity,99999999,1478428019768.c2b1c9c11742bc744f07e0ff91cca65e. 2017-05-03 21:22:45,228 INFO [StoreCloserThread-CampaignSMSTransaction,FEzAcwOdp8zYTZ3201701171543,1490792676052.877c4cfaaf901966b360ba4d938d1d0b.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,228 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed CampaignSMSTransaction,FEzAcwOdp8zYTZ3201701171543,1490792676052.877c4cfaaf901966b360ba4d938d1d0b. 2017-05-03 21:22:45,228 INFO [StoreCloserThread-WebAnalyticsUserFlow,,1478513262242.7495f855e20b40c172a1f19e64cb6956.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,229 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed WebAnalyticsUserFlow,,1478513262242.7495f855e20b40c172a1f19e64cb6956. 2017-05-03 21:22:45,229 INFO [StoreCloserThread-RecipientsSocialMediaInfo,,1478427803572.2fc10b953daabf4e549875f640a15c08.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,229 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed RecipientsSocialMediaInfo,,1478427803572.2fc10b953daabf4e549875f640a15c08. 2017-05-03 21:22:45,229 INFO [StoreCloserThread-CampaignSummaryInsight,cccccccc,1478428013094.8744d406c6480e4f7c272f7243aae65a.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,229 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed CampaignSummaryInsight,cccccccc,1478428013094.8744d406c6480e4f7c272f7243aae65a. 2017-05-03 21:22:45,229 INFO [StoreCloserThread-CampaignSummaryDemographics,,1478427988763.bef9ebf6ba4beace1b24ce296606defb.-1] regionserver.HStore: Closed Dg 2017-05-03 21:22:45,229 INFO [StoreCloserThread-CampaignSMSTransaction,FEzAcwgmpczXS99R201703231805,1493295423261.f6b6419ab6a9a7e5ef1637d7ae685413.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,229 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-2] regionserver.HRegion: Closed CampaignSMSTransaction,FEzAcwgmpczXS99R201703231805,1493295423261.f6b6419ab6a9a7e5ef1637d7ae685413. 2017-05-03 21:22:45,230 INFO [StoreCloserThread-CampaignSummaryIndustrySegment,33333333,1478428096368.01e656740f344d98a12c0de7d2af2e2f.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,230 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-2] regionserver.HRegion: Closed CampaignSummaryIndustrySegment,33333333,1478428096368.01e656740f344d98a12c0de7d2af2e2f. 2017-05-03 21:22:45,230 INFO [StoreCloserThread-SocialMediaInsightsDetail_New_bkp,66666666,1493112706395.d0424c7dacd60a1e6b14e07d0c577988.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,230 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-2] regionserver.HRegion: Closed SocialMediaInsightsDetail_New_bkp,66666666,1493112706395.d0424c7dacd60a1e6b14e07d0c577988. 2017-05-03 21:22:45,230 INFO [StoreCloserThread-CampaignSummaryDemographics,,1478427988763.bef9ebf6ba4beace1b24ce296606defb.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,230 INFO [StoreCloserThread-CampaignSummaryBusinessSegment,cccccccc,1478428086929.6698745deedc2e9c642114013f4b9f9a.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,230 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed CampaignSummaryDemographics,,1478427988763.bef9ebf6ba4beace1b24ce296606defb. 2017-05-03 21:22:45,230 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-2] regionserver.HRegion: Closed CampaignSummaryBusinessSegment,cccccccc,1478428086929.6698745deedc2e9c642114013f4b9f9a. 2017-05-03 21:22:45,230 INFO [StoreCloserThread-DashboardCustomerPropensity,33333333,1478428350617.f38153002996c58ea5de3814a4a2d797.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,230 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-2] regionserver.HRegion: Closed DashboardCustomerPropensity,33333333,1478428350617.f38153002996c58ea5de3814a4a2d797. 2017-05-03 21:22:45,230 INFO [StoreCloserThread-SocialMediaRecipients,66666666,1478427832245.25a506bfd69d6ac5210599ade237ac1d.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,230 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-2] regionserver.HRegion: Closed SocialMediaRecipients,66666666,1478427832245.25a506bfd69d6ac5210599ade237ac1d. 2017-05-03 21:22:45,231 INFO [StoreCloserThread-CampaignSummaryGeographyByCityMDC,33333333,1478428033615.8fb9ba7f974a45757b875c50407e9fad.-1] regionserver.HStore: Closed Geo 2017-05-03 21:22:45,231 INFO [StoreCloserThread-CampaignBlastScheduleTransaction,cccccccc,1491991266331.6b988b9cd6ddd30c15aa254b3831809a.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,231 INFO [StoreCloserThread-CampaignSummaryGeographyByCityMDC,33333333,1478428033615.8fb9ba7f974a45757b875c50407e9fad.-1] regionserver.HStore: Closed info 2017-05-03 21:22:45,231 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-1] regionserver.HRegion: Closed CampaignBlastScheduleTransaction,cccccccc,1491991266331.6b988b9cd6ddd30c15aa254b3831809a. 2017-05-03 21:22:45,231 INFO [RS_CLOSE_REGION-aps-hadoop5:16020-0] regionserver.HRegion: Closed CampaignSummaryGeographyByCityMDC,33333333,1478428033615.8fb9ba7f974a45757b875c50407e9fad. 2017-05-03 21:22:45,317 INFO [regionserver/aps-hadoop5/1..1..1..:16020] regionserver.HRegionServer: stopping server aps-hadoop5,16020,1493618413009; all regions closed. 2017-05-03 21:22:45,333 WARN [regionserver/aps-hadoop5/1..1..1..:16020] wal.ProtobufLogWriter: Failed to write trailer, non-fatal, continuing... java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]], original=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:947) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1021) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1189) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.processDatanodeError(DFSOutputStream.java:904) at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:411) 2017-05-03 21:22:45,333 ERROR [regionserver/aps-hadoop5/1..1..1..:16020] regionserver.HRegionServer: Shutdown / close of WAL failed: java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]], original=[DatanodeInfoWithStorage[10.64.228.140:50010,DS-8ab76f9c-ee05-4ec0-897a-8718ab89635f,DISK], DatanodeInfoWithStorage[10.64.228.150:50010,DS-57010fb6-92c0-4c3e-8b9e-11233ceb7bfa,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration. 2017-05-03 21:22:45,343 INFO [regionserver/aps-hadoop5/1..1..1..:16020] regionserver.Leases: regionserver/aps-hadoop5/1..1..1..:16020 closing leases 2017-05-03 21:22:45,343 INFO [regionserver/aps-hadoop5/1..1..1..:16020] regionserver.Leases: regionserver/aps-hadoop5/1..1..1..:16020 closed leases 2017-05-03 21:22:45,347 INFO [regionserver/aps-hadoop5/1..1..1..:16020] hbase.ChoreService: Chore service for: aps-hadoop5,16020,1493618413009 had [[ScheduledChore: Name: aps-hadoop5,16020,1493618413009-MemstoreFlusherChore Period: 10000 Unit: MILLISECONDS], [ScheduledChore: Name: MovedRegionsCleaner for region aps-hadoop5,16020,1493618413009 Period: 120000 Unit: MILLISECONDS]] on shutdown 2017-05-03 21:22:50,172 INFO [regionserver/aps-hadoop5/1..1..1..:16020.leaseChecker] regionserver.Leases: regionserver/aps-hadoop5/1..1..1..:16020.leaseChecker closing leases 2017-05-03 21:22:50,173 INFO [regionserver/aps-hadoop5/1..1..1..:16020.leaseChecker] regionserver.Leases: regionserver/aps-hadoop5/1..1..1..:16020.leaseChecker closed leases 2017-05-03 21:22:50,778 INFO [RS_OPEN_META-aps-hadoop5:16020-0-MetaLogRoller] regionserver.LogRoller: LogRoller exiting. 2017-05-03 21:22:50,847 INFO [regionserver/aps-hadoop5/1..1..1..:16020] ipc.RpcServer: Stopping server on 16020 2017-05-03 21:22:50,847 INFO [RpcServer.listener,port=16020] ipc.RpcServer: RpcServer.listener,port=16020: stopping 2017-05-03 21:22:50,847 INFO [RpcServer.responder] ipc.RpcServer: RpcServer.responder: stopped 2017-05-03 21:22:50,847 INFO [RpcServer.responder] ipc.RpcServer: RpcServer.responder: stopping 2017-05-03 21:22:50,851 INFO [regionserver/aps-hadoop5/1..1..1..:16020] zookeeper.ZooKeeper: Session: 0x25887d92d0a8efa closed 2017-05-03 21:22:50,851 INFO [main-EventThread] zookeeper.ClientCnxn: EventThread shut down 2017-05-03 21:22:50,851 INFO [regionserver/aps-hadoop5/1..1..1..:16020] regionserver.HRegionServer: stopping server aps-hadoop5,16020,1493618413009; zookeeper connection closed. 2017-05-03 21:22:50,851 INFO [regionserver/aps-hadoop5/1..1..1..:16020] regionserver.HRegionServer: regionserver/aps-hadoop5/1..1..1..:16020 exiting 2017-05-03 21:22:50,888 ERROR [main] regionserver.HRegionServerCommandLine: Region server exiting java.lang.RuntimeException: HRegionServer Aborted at org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.start(HRegionServerCommandLine.java:68) at org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.run(HRegionServerCommandLine.java:87) at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) at org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:126) at org.apache.hadoop.hbase.regionserver.HRegionServer.main(HRegionServer.java:2655) 2017-05-03 21:22:50,905 INFO [pool-2-thread-1] regionserver.ShutdownHook: Shutdown hook starting; hbase.shutdown.hook=true; fsShutdownHook=org.apache.hadoop.fs.FileSystem$Cache$ClientFinalizer@7a11c4c7 2017-05-03 21:22:50,905 INFO [pool-2-thread-1] regionserver.ShutdownHook: Starting fs shutdown hook thread. 2017-05-03 21:22:50,931 INFO [pool-2-thread-1] regionserver.ShutdownHook: Shutdown hook finished.