-------------------------------------------------------------------------------- VERTICES STATUS TOTAL COMPLETED RUNNING PENDING FAILED KILLED -------------------------------------------------------------------------------- Map 1 RUNNING 1 0 0 1 4 0 -------------------------------------------------------------------------------- VERTICES: 00/01 [>>--------------------------] 0% ELAPSED TIME: 710.40 s -------------------------------------------------------------------------------- ERROR : Status: Failed ERROR : Vertex failed, vertexName=Map 1, vertexId=vertex_1508230545807_0009_2_00, diagnostics=[Task failed, taskId=task_1508230545807_0009_2_00_000000, diagnostics=[TaskAttempt 0 failed, info=[Error: Failure while running task:java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: java.io.IOException: java.io.IOException: Couldn't get next key/value from mongodb: at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:173) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:139) at org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:347) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable$1.run(TezTaskRunner.java:194) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable$1.run(TezTaskRunner.java:185) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1866) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable.callInternal(TezTaskRunner.java:185) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable.callInternal(TezTaskRunner.java:181) at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: java.io.IOException: java.io.IOException: Couldn't get next key/value from mongodb: at org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.pushRecord(MapRecordSource.java:76) at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.run(MapRecordProcessor.java:325) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:150) ... 14 more Caused by: java.io.IOException: java.io.IOException: Couldn't get next key/value from mongodb: at org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain.handleRecordReaderNextException(HiveIOExceptionHandlerChain.java:121) at org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil.handleRecordReaderNextException(HiveIOExceptionHandlerUtil.java:77) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.doNext(HiveContextAwareRecordReader.java:355) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:79) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:33) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.next(HiveContextAwareRecordReader.java:116) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.next(TezGroupedSplitsInputFormat.java:141) at org.apache.tez.mapreduce.lib.MRReaderMapred.next(MRReaderMapred.java:113) at org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.pushRecord(MapRecordSource.java:66) ... 16 more Caused by: java.io.IOException: Couldn't get next key/value from mongodb: at com.mongodb.hadoop.mapred.input.MongoRecordReader.nextKeyValue(MongoRecordReader.java:113) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:118) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:34) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.doNext(HiveContextAwareRecordReader.java:350) ... 22 more Caused by: com.mongodb.MongoInternalException: Unexpected exception at com.mongodb.connection.InternalStreamConnection.translateReadException(InternalStreamConnection.java:469) at com.mongodb.connection.InternalStreamConnection.receiveMessage(InternalStreamConnection.java:214) at com.mongodb.connection.UsageTrackingInternalConnection.receiveMessage(UsageTrackingInternalConnection.java:96) at com.mongodb.connection.DefaultConnectionPool$PooledConnection.receiveMessage(DefaultConnectionPool.java:438) at com.mongodb.connection.CommandProtocol.execute(CommandProtocol.java:105) at com.mongodb.connection.DefaultServer$DefaultServerProtocolExecutor.execute(DefaultServer.java:159) at com.mongodb.connection.DefaultServerConnection.executeProtocol(DefaultServerConnection.java:289) at com.mongodb.connection.DefaultServerConnection.command(DefaultServerConnection.java:176) at com.mongodb.operation.QueryBatchCursor.getMore(QueryBatchCursor.java:209) at com.mongodb.operation.QueryBatchCursor.hasNext(QueryBatchCursor.java:105) at com.mongodb.MongoBatchCursorAdapter.hasNext(MongoBatchCursorAdapter.java:46) at com.mongodb.DBCursor.hasNext(DBCursor.java:145) at com.mongodb.hadoop.mapred.input.MongoRecordReader.nextKeyValue(MongoRecordReader.java:100) ... 25 more Caused by: java.lang.OutOfMemoryError: Java heap space at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at com.mongodb.internal.connection.PowerOfTwoBufferPool.createNew(PowerOfTwoBufferPool.java:84) at com.mongodb.internal.connection.PowerOfTwoBufferPool.access$000(PowerOfTwoBufferPool.java:33) at com.mongodb.internal.connection.PowerOfTwoBufferPool$1.create(PowerOfTwoBufferPool.java:57) at com.mongodb.internal.connection.PowerOfTwoBufferPool$1.create(PowerOfTwoBufferPool.java:54) at com.mongodb.internal.connection.ConcurrentPool.createNewAndReleasePermitIfFailure(ConcurrentPool.java:164) at com.mongodb.internal.connection.ConcurrentPool.get(ConcurrentPool.java:132) at com.mongodb.internal.connection.ConcurrentPool.get(ConcurrentPool.java:109) at com.mongodb.internal.connection.PowerOfTwoBufferPool.getBuffer(PowerOfTwoBufferPool.java:76) at com.mongodb.connection.SocketStream.read(SocketStream.java:80) at com.mongodb.connection.InternalStreamConnection.receiveResponseBuffers(InternalStreamConnection.java:486) at com.mongodb.connection.InternalStreamConnection.receiveMessage(InternalStreamConnection.java:210) at com.mongodb.connection.UsageTrackingInternalConnection.receiveMessage(UsageTrackingInternalConnection.java:96) at com.mongodb.connection.DefaultConnectionPool$PooledConnection.receiveMessage(DefaultConnectionPool.java:438) at com.mongodb.connection.CommandProtocol.execute(CommandProtocol.java:105) at com.mongodb.connection.DefaultServer$DefaultServerProtocolExecutor.execute(DefaultServer.java:159) at com.mongodb.connection.DefaultServerConnection.executeProtocol(DefaultServerConnection.java:289) at com.mongodb.connection.DefaultServerConnection.command(DefaultServerConnection.java:176) at com.mongodb.operation.QueryBatchCursor.getMore(QueryBatchCursor.java:209) at com.mongodb.operation.QueryBatchCursor.hasNext(QueryBatchCursor.java:105) at com.mongodb.MongoBatchCursorAdapter.hasNext(MongoBatchCursorAdapter.java:46) at com.mongodb.DBCursor.hasNext(DBCursor.java:145) at com.mongodb.hadoop.mapred.input.MongoRecordReader.nextKeyValue(MongoRecordReader.java:100) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:118) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:34) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.doNext(HiveContextAwareRecordReader.java:350) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:79) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:33) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.next(HiveContextAwareRecordReader.java:116) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.next(TezGroupedSplitsInputFormat.java:141) at org.apache.tez.mapreduce.lib.MRReaderMapred.next(MRReaderMapred.java:113) ], TaskAttempt 1 failed, info=[Error: Failure while running task:java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: java.io.IOException: java.io.IOException: Couldn't get next key/value from mongodb: at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:173) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:139) at org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:347) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable$1.run(TezTaskRunner.java:194) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable$1.run(TezTaskRunner.java:185) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1866) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable.callInternal(TezTaskRunner.java:185) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable.callInternal(TezTaskRunner.java:181) at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: java.io.IOException: java.io.IOException: Couldn't get next key/value from mongodb: at org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.pushRecord(MapRecordSource.java:76) at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.run(MapRecordProcessor.java:325) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:150) ... 14 more Caused by: java.io.IOException: java.io.IOException: Couldn't get next key/value from mongodb: at org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain.handleRecordReaderNextException(HiveIOExceptionHandlerChain.java:121) at org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil.handleRecordReaderNextException(HiveIOExceptionHandlerUtil.java:77) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.doNext(HiveContextAwareRecordReader.java:355) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:79) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:33) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.next(HiveContextAwareRecordReader.java:116) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.next(TezGroupedSplitsInputFormat.java:141) at org.apache.tez.mapreduce.lib.MRReaderMapred.next(MRReaderMapred.java:113) at org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.pushRecord(MapRecordSource.java:66) ... 16 more Caused by: java.io.IOException: Couldn't get next key/value from mongodb: at com.mongodb.hadoop.mapred.input.MongoRecordReader.nextKeyValue(MongoRecordReader.java:113) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:118) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:34) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.doNext(HiveContextAwareRecordReader.java:350) ... 22 more Caused by: com.mongodb.MongoInternalException: Unexpected exception at com.mongodb.connection.InternalStreamConnection.translateReadException(InternalStreamConnection.java:469) at com.mongodb.connection.InternalStreamConnection.receiveMessage(InternalStreamConnection.java:214) at com.mongodb.connection.UsageTrackingInternalConnection.receiveMessage(UsageTrackingInternalConnection.java:96) at com.mongodb.connection.DefaultConnectionPool$PooledConnection.receiveMessage(DefaultConnectionPool.java:438) at com.mongodb.connection.CommandProtocol.execute(CommandProtocol.java:105) at com.mongodb.connection.DefaultServer$DefaultServerProtocolExecutor.execute(DefaultServer.java:159) at com.mongodb.connection.DefaultServerConnection.executeProtocol(DefaultServerConnection.java:289) at com.mongodb.connection.DefaultServerConnection.command(DefaultServerConnection.java:176) at com.mongodb.operation.QueryBatchCursor.getMore(QueryBatchCursor.java:209) at com.mongodb.operation.QueryBatchCursor.hasNext(QueryBatchCursor.java:105) at com.mongodb.MongoBatchCursorAdapter.hasNext(MongoBatchCursorAdapter.java:46) at com.mongodb.DBCursor.hasNext(DBCursor.java:145) at com.mongodb.hadoop.mapred.input.MongoRecordReader.nextKeyValue(MongoRecordReader.java:100) ... 25 more Caused by: java.lang.OutOfMemoryError: Java heap space at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at com.mongodb.internal.connection.PowerOfTwoBufferPool.createNew(PowerOfTwoBufferPool.java:84) at com.mongodb.internal.connection.PowerOfTwoBufferPool.access$000(PowerOfTwoBufferPool.java:33) at com.mongodb.internal.connection.PowerOfTwoBufferPool$1.create(PowerOfTwoBufferPool.java:57) at com.mongodb.internal.connection.PowerOfTwoBufferPool$1.create(PowerOfTwoBufferPool.java:54) at com.mongodb.internal.connection.ConcurrentPool.createNewAndReleasePermitIfFailure(ConcurrentPool.java:164) at com.mongodb.internal.connection.ConcurrentPool.get(ConcurrentPool.java:132) at com.mongodb.internal.connection.ConcurrentPool.get(ConcurrentPool.java:109) at com.mongodb.internal.connection.PowerOfTwoBufferPool.getBuffer(PowerOfTwoBufferPool.java:76) at com.mongodb.connection.SocketStream.read(SocketStream.java:80) at com.mongodb.connection.InternalStreamConnection.receiveResponseBuffers(InternalStreamConnection.java:486) at com.mongodb.connection.InternalStreamConnection.receiveMessage(InternalStreamConnection.java:210) at com.mongodb.connection.UsageTrackingInternalConnection.receiveMessage(UsageTrackingInternalConnection.java:96) at com.mongodb.connection.DefaultConnectionPool$PooledConnection.receiveMessage(DefaultConnectionPool.java:438) at com.mongodb.connection.CommandProtocol.execute(CommandProtocol.java:105) at com.mongodb.connection.DefaultServer$DefaultServerProtocolExecutor.execute(DefaultServer.java:159) at com.mongodb.connection.DefaultServerConnection.executeProtocol(DefaultServerConnection.java:289) at com.mongodb.connection.DefaultServerConnection.command(DefaultServerConnection.java:176) at com.mongodb.operation.QueryBatchCursor.getMore(QueryBatchCursor.java:209) at com.mongodb.operation.QueryBatchCursor.hasNext(QueryBatchCursor.java:105) at com.mongodb.MongoBatchCursorAdapter.hasNext(MongoBatchCursorAdapter.java:46) at com.mongodb.DBCursor.hasNext(DBCursor.java:145) at com.mongodb.hadoop.mapred.input.MongoRecordReader.nextKeyValue(MongoRecordReader.java:100) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:118) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:34) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.doNext(HiveContextAwareRecordReader.java:350) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:79) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:33) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.next(HiveContextAwareRecordReader.java:116) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.next(TezGroupedSplitsInputFormat.java:141) at org.apache.tez.mapreduce.lib.MRReaderMapred.next(MRReaderMapred.java:113) ], TaskAttempt 2 failed, info=[Error: Failure while running task:java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: java.io.IOException: java.io.IOException: Couldn't get next key/value from mongodb: at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:173) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:139) at org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:347) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable$1.run(TezTaskRunner.java:194) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable$1.run(TezTaskRunner.java:185) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1866) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable.callInternal(TezTaskRunner.java:185) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable.callInternal(TezTaskRunner.java:181) at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: java.io.IOException: java.io.IOException: Couldn't get next key/value from mongodb: at org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.pushRecord(MapRecordSource.java:76) at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.run(MapRecordProcessor.java:325) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:150) ... 14 more Caused by: java.io.IOException: java.io.IOException: Couldn't get next key/value from mongodb: at org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain.handleRecordReaderNextException(HiveIOExceptionHandlerChain.java:121) at org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil.handleRecordReaderNextException(HiveIOExceptionHandlerUtil.java:77) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.doNext(HiveContextAwareRecordReader.java:355) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:79) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:33) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.next(HiveContextAwareRecordReader.java:116) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.next(TezGroupedSplitsInputFormat.java:141) at org.apache.tez.mapreduce.lib.MRReaderMapred.next(MRReaderMapred.java:113) at org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.pushRecord(MapRecordSource.java:66) ... 16 more Caused by: java.io.IOException: Couldn't get next key/value from mongodb: at com.mongodb.hadoop.mapred.input.MongoRecordReader.nextKeyValue(MongoRecordReader.java:113) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:118) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:34) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.doNext(HiveContextAwareRecordReader.java:350) ... 22 more Caused by: com.mongodb.MongoInternalException: Unexpected exception at com.mongodb.connection.InternalStreamConnection.translateReadException(InternalStreamConnection.java:469) at com.mongodb.connection.InternalStreamConnection.receiveMessage(InternalStreamConnection.java:214) at com.mongodb.connection.UsageTrackingInternalConnection.receiveMessage(UsageTrackingInternalConnection.java:96) at com.mongodb.connection.DefaultConnectionPool$PooledConnection.receiveMessage(DefaultConnectionPool.java:438) at com.mongodb.connection.CommandProtocol.execute(CommandProtocol.java:105) at com.mongodb.connection.DefaultServer$DefaultServerProtocolExecutor.execute(DefaultServer.java:159) at com.mongodb.connection.DefaultServerConnection.executeProtocol(DefaultServerConnection.java:289) at com.mongodb.connection.DefaultServerConnection.command(DefaultServerConnection.java:176) at com.mongodb.operation.QueryBatchCursor.getMore(QueryBatchCursor.java:209) at com.mongodb.operation.QueryBatchCursor.hasNext(QueryBatchCursor.java:105) at com.mongodb.MongoBatchCursorAdapter.hasNext(MongoBatchCursorAdapter.java:46) at com.mongodb.DBCursor.hasNext(DBCursor.java:145) at com.mongodb.hadoop.mapred.input.MongoRecordReader.nextKeyValue(MongoRecordReader.java:100) ... 25 more Caused by: java.lang.OutOfMemoryError: Java heap space at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at com.mongodb.internal.connection.PowerOfTwoBufferPool.createNew(PowerOfTwoBufferPool.java:84) at com.mongodb.internal.connection.PowerOfTwoBufferPool.access$000(PowerOfTwoBufferPool.java:33) at com.mongodb.internal.connection.PowerOfTwoBufferPool$1.create(PowerOfTwoBufferPool.java:57) at com.mongodb.internal.connection.PowerOfTwoBufferPool$1.create(PowerOfTwoBufferPool.java:54) at com.mongodb.internal.connection.ConcurrentPool.createNewAndReleasePermitIfFailure(ConcurrentPool.java:164) at com.mongodb.internal.connection.ConcurrentPool.get(ConcurrentPool.java:132) at com.mongodb.internal.connection.ConcurrentPool.get(ConcurrentPool.java:109) at com.mongodb.internal.connection.PowerOfTwoBufferPool.getBuffer(PowerOfTwoBufferPool.java:76) at com.mongodb.connection.SocketStream.read(SocketStream.java:80) at com.mongodb.connection.InternalStreamConnection.receiveResponseBuffers(InternalStreamConnection.java:486) at com.mongodb.connection.InternalStreamConnection.receiveMessage(InternalStreamConnection.java:210) at com.mongodb.connection.UsageTrackingInternalConnection.receiveMessage(UsageTrackingInternalConnection.java:96) at com.mongodb.connection.DefaultConnectionPool$PooledConnection.receiveMessage(DefaultConnectionPool.java:438) at com.mongodb.connection.CommandProtocol.execute(CommandProtocol.java:105) at com.mongodb.connection.DefaultServer$DefaultServerProtocolExecutor.execute(DefaultServer.java:159) at com.mongodb.connection.DefaultServerConnection.executeProtocol(DefaultServerConnection.java:289) at com.mongodb.connection.DefaultServerConnection.command(DefaultServerConnection.java:176) at com.mongodb.operation.QueryBatchCursor.getMore(QueryBatchCursor.java:209) at com.mongodb.operation.QueryBatchCursor.hasNext(QueryBatchCursor.java:105) at com.mongodb.MongoBatchCursorAdapter.hasNext(MongoBatchCursorAdapter.java:46) at com.mongodb.DBCursor.hasNext(DBCursor.java:145) at com.mongodb.hadoop.mapred.input.MongoRecordReader.nextKeyValue(MongoRecordReader.java:100) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:118) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:34) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.doNext(HiveContextAwareRecordReader.java:350) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:79) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:33) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.next(HiveContextAwareRecordReader.java:116) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.next(TezGroupedSplitsInputFormat.java:141) at org.apache.tez.mapreduce.lib.MRReaderMapred.next(MRReaderMapred.java:113) ], TaskAttempt 3 failed, info=[Error: Failure while running task:java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: java.io.IOException: java.io.IOException: Couldn't get next key/value from mongodb: at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:173) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:139) at org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:347) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable$1.run(TezTaskRunner.java:194) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable$1.run(TezTaskRunner.java:185) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1866) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable.callInternal(TezTaskRunner.java:185) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable.callInternal(TezTaskRunner.java:181) at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: java.io.IOException: java.io.IOException: Couldn't get next key/value from mongodb: at org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.pushRecord(MapRecordSource.java:76) at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.run(MapRecordProcessor.java:325) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:150) ... 14 more Caused by: java.io.IOException: java.io.IOException: Couldn't get next key/value from mongodb: at org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain.handleRecordReaderNextException(HiveIOExceptionHandlerChain.java:121) at org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil.handleRecordReaderNextException(HiveIOExceptionHandlerUtil.java:77) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.doNext(HiveContextAwareRecordReader.java:355) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:79) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:33) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.next(HiveContextAwareRecordReader.java:116) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.next(TezGroupedSplitsInputFormat.java:141) at org.apache.tez.mapreduce.lib.MRReaderMapred.next(MRReaderMapred.java:113) at org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.pushRecord(MapRecordSource.java:66) ... 16 more Caused by: java.io.IOException: Couldn't get next key/value from mongodb: at com.mongodb.hadoop.mapred.input.MongoRecordReader.nextKeyValue(MongoRecordReader.java:113) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:118) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:34) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.doNext(HiveContextAwareRecordReader.java:350) ... 22 more Caused by: com.mongodb.MongoInternalException: Unexpected exception at com.mongodb.connection.InternalStreamConnection.translateReadException(InternalStreamConnection.java:469) at com.mongodb.connection.InternalStreamConnection.receiveMessage(InternalStreamConnection.java:214) at com.mongodb.connection.UsageTrackingInternalConnection.receiveMessage(UsageTrackingInternalConnection.java:96) at com.mongodb.connection.DefaultConnectionPool$PooledConnection.receiveMessage(DefaultConnectionPool.java:438) at com.mongodb.connection.CommandProtocol.execute(CommandProtocol.java:105) at com.mongodb.connection.DefaultServer$DefaultServerProtocolExecutor.execute(DefaultServer.java:159) at com.mongodb.connection.DefaultServerConnection.executeProtocol(DefaultServerConnection.java:289) at com.mongodb.connection.DefaultServerConnection.command(DefaultServerConnection.java:176) at com.mongodb.operation.QueryBatchCursor.getMore(QueryBatchCursor.java:209) at com.mongodb.operation.QueryBatchCursor.hasNext(QueryBatchCursor.java:105) at com.mongodb.MongoBatchCursorAdapter.hasNext(MongoBatchCursorAdapter.java:46) at com.mongodb.DBCursor.hasNext(DBCursor.java:145) at com.mongodb.hadoop.mapred.input.MongoRecordReader.nextKeyValue(MongoRecordReader.java:100) ... 25 more Caused by: java.lang.OutOfMemoryError: Java heap space at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at com.mongodb.internal.connection.PowerOfTwoBufferPool.createNew(PowerOfTwoBufferPool.java:84) at com.mongodb.internal.connection.PowerOfTwoBufferPool.access$000(PowerOfTwoBufferPool.java:33) at com.mongodb.internal.connection.PowerOfTwoBufferPool$1.create(PowerOfTwoBufferPool.java:57) at com.mongodb.internal.connection.PowerOfTwoBufferPool$1.create(PowerOfTwoBufferPool.java:54) at com.mongodb.internal.connection.ConcurrentPool.createNewAndReleasePermitIfFailure(ConcurrentPool.java:164) at com.mongodb.internal.connection.ConcurrentPool.get(ConcurrentPool.java:132) at com.mongodb.internal.connection.ConcurrentPool.get(ConcurrentPool.java:109) at com.mongodb.internal.connection.PowerOfTwoBufferPool.getBuffer(PowerOfTwoBufferPool.java:76) at com.mongodb.connection.SocketStream.read(SocketStream.java:80) at com.mongodb.connection.InternalStreamConnection.receiveResponseBuffers(InternalStreamConnection.java:486) at com.mongodb.connection.InternalStreamConnection.receiveMessage(InternalStreamConnection.java:210) at com.mongodb.connection.UsageTrackingInternalConnection.receiveMessage(UsageTrackingInternalConnection.java:96) at com.mongodb.connection.DefaultConnectionPool$PooledConnection.receiveMessage(DefaultConnectionPool.java:438) at com.mongodb.connection.CommandProtocol.execute(CommandProtocol.java:105) at com.mongodb.connection.DefaultServer$DefaultServerProtocolExecutor.execute(DefaultServer.java:159) at com.mongodb.connection.DefaultServerConnection.executeProtocol(DefaultServerConnection.java:289) at com.mongodb.connection.DefaultServerConnection.command(DefaultServerConnection.java:176) at com.mongodb.operation.QueryBatchCursor.getMore(QueryBatchCursor.java:209) at com.mongodb.operation.QueryBatchCursor.hasNext(QueryBatchCursor.java:105) at com.mongodb.MongoBatchCursorAdapter.hasNext(MongoBatchCursorAdapter.java:46) at com.mongodb.DBCursor.hasNext(DBCursor.java:145) at com.mongodb.hadoop.mapred.input.MongoRecordReader.nextKeyValue(MongoRecordReader.java:100) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:118) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:34) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.doNext(HiveContextAwareRecordReader.java:350) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:79) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:33) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.next(HiveContextAwareRecordReader.java:116) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.next(TezGroupedSplitsInputFormat.java:141) at org.apache.tez.mapreduce.lib.MRReaderMapred.next(MRReaderMapred.java:113) ]], Vertex did not succeed due to OWN_TASK_FAILURE, failedTasks:1 killedTasks:0, Vertex vertex_1508230545807_0009_2_00 [Map 1] killed/failed due to:OWN_TASK_FAILURE] ERROR : DAG did not succeed due to VERTEX_FAILURE. failedVertices:1 killedVertices:0 Error: Error while processing statement: FAILED: Execution Error, return code 2 from org.apache.hadoop.hive.ql.exec.tez.TezTask. Vertex failed, vertexName=Map 1, vertexId=vertex_1508230545807_0009_2_00, diagnostics=[Task failed, taskId=task_1508230545807_0009_2_00_000000, diagnostics=[TaskAttempt 0 failed, info=[Error: Failure while running task:java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: java.io.IOException: java.io.IOException: Couldn't get next key/value from mongodb: at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:173) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:139) at org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:347) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable$1.run(TezTaskRunner.java:194) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable$1.run(TezTaskRunner.java:185) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1866) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable.callInternal(TezTaskRunner.java:185) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable.callInternal(TezTaskRunner.java:181) at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: java.io.IOException: java.io.IOException: Couldn't get next key/value from mongodb: at org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.pushRecord(MapRecordSource.java:76) at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.run(MapRecordProcessor.java:325) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:150) ... 14 more Caused by: java.io.IOException: java.io.IOException: Couldn't get next key/value from mongodb: at org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain.handleRecordReaderNextException(HiveIOExceptionHandlerChain.java:121) at org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil.handleRecordReaderNextException(HiveIOExceptionHandlerUtil.java:77) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.doNext(HiveContextAwareRecordReader.java:355) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:79) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:33) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.next(HiveContextAwareRecordReader.java:116) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.next(TezGroupedSplitsInputFormat.java:141) at org.apache.tez.mapreduce.lib.MRReaderMapred.next(MRReaderMapred.java:113) at org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.pushRecord(MapRecordSource.java:66) ... 16 more Caused by: java.io.IOException: Couldn't get next key/value from mongodb: at com.mongodb.hadoop.mapred.input.MongoRecordReader.nextKeyValue(MongoRecordReader.java:113) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:118) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:34) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.doNext(HiveContextAwareRecordReader.java:350) ... 22 more Caused by: com.mongodb.MongoInternalException: Unexpected exception at com.mongodb.connection.InternalStreamConnection.translateReadException(InternalStreamConnection.java:469) at com.mongodb.connection.InternalStreamConnection.receiveMessage(InternalStreamConnection.java:214) at com.mongodb.connection.UsageTrackingInternalConnection.receiveMessage(UsageTrackingInternalConnection.java:96) at com.mongodb.connection.DefaultConnectionPool$PooledConnection.receiveMessage(DefaultConnectionPool.java:438) at com.mongodb.connection.CommandProtocol.execute(CommandProtocol.java:105) at com.mongodb.connection.DefaultServer$DefaultServerProtocolExecutor.execute(DefaultServer.java:159) at com.mongodb.connection.DefaultServerConnection.executeProtocol(DefaultServerConnection.java:289) at com.mongodb.connection.DefaultServerConnection.command(DefaultServerConnection.java:176) at com.mongodb.operation.QueryBatchCursor.getMore(QueryBatchCursor.java:209) at com.mongodb.operation.QueryBatchCursor.hasNext(QueryBatchCursor.java:105) at com.mongodb.MongoBatchCursorAdapter.hasNext(MongoBatchCursorAdapter.java:46) at com.mongodb.DBCursor.hasNext(DBCursor.java:145) at com.mongodb.hadoop.mapred.input.MongoRecordReader.nextKeyValue(MongoRecordReader.java:100) ... 25 more Caused by: java.lang.OutOfMemoryError: Java heap space at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at com.mongodb.internal.connection.PowerOfTwoBufferPool.createNew(PowerOfTwoBufferPool.java:84) at com.mongodb.internal.connection.PowerOfTwoBufferPool.access$000(PowerOfTwoBufferPool.java:33) at com.mongodb.internal.connection.PowerOfTwoBufferPool$1.create(PowerOfTwoBufferPool.java:57) at com.mongodb.internal.connection.PowerOfTwoBufferPool$1.create(PowerOfTwoBufferPool.java:54) at com.mongodb.internal.connection.ConcurrentPool.createNewAndReleasePermitIfFailure(ConcurrentPool.java:164) at com.mongodb.internal.connection.ConcurrentPool.get(ConcurrentPool.java:132) at com.mongodb.internal.connection.ConcurrentPool.get(ConcurrentPool.java:109) at com.mongodb.internal.connection.PowerOfTwoBufferPool.getBuffer(PowerOfTwoBufferPool.java:76) at com.mongodb.connection.SocketStream.read(SocketStream.java:80) at com.mongodb.connection.InternalStreamConnection.receiveResponseBuffers(InternalStreamConnection.java:486) at com.mongodb.connection.InternalStreamConnection.receiveMessage(InternalStreamConnection.java:210) at com.mongodb.connection.UsageTrackingInternalConnection.receiveMessage(UsageTrackingInternalConnection.java:96) at com.mongodb.connection.DefaultConnectionPool$PooledConnection.receiveMessage(DefaultConnectionPool.java:438) at com.mongodb.connection.CommandProtocol.execute(CommandProtocol.java:105) at com.mongodb.connection.DefaultServer$DefaultServerProtocolExecutor.execute(DefaultServer.java:159) at com.mongodb.connection.DefaultServerConnection.executeProtocol(DefaultServerConnection.java:289) at com.mongodb.connection.DefaultServerConnection.command(DefaultServerConnection.java:176) at com.mongodb.operation.QueryBatchCursor.getMore(QueryBatchCursor.java:209) at com.mongodb.operation.QueryBatchCursor.hasNext(QueryBatchCursor.java:105) at com.mongodb.MongoBatchCursorAdapter.hasNext(MongoBatchCursorAdapter.java:46) at com.mongodb.DBCursor.hasNext(DBCursor.java:145) at com.mongodb.hadoop.mapred.input.MongoRecordReader.nextKeyValue(MongoRecordReader.java:100) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:118) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:34) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.doNext(HiveContextAwareRecordReader.java:350) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:79) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:33) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.next(HiveContextAwareRecordReader.java:116) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.next(TezGroupedSplitsInputFormat.java:141) at org.apache.tez.mapreduce.lib.MRReaderMapred.next(MRReaderMapred.java:113) ], TaskAttempt 1 failed, info=[Error: Failure while running task:java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: java.io.IOException: java.io.IOException: Couldn't get next key/value from mongodb: at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:173) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:139) at org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:347) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable$1.run(TezTaskRunner.java:194) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable$1.run(TezTaskRunner.java:185) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1866) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable.callInternal(TezTaskRunner.java:185) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable.callInternal(TezTaskRunner.java:181) at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: java.io.IOException: java.io.IOException: Couldn't get next key/value from mongodb: at org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.pushRecord(MapRecordSource.java:76) at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.run(MapRecordProcessor.java:325) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:150) ... 14 more Caused by: java.io.IOException: java.io.IOException: Couldn't get next key/value from mongodb: at org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain.handleRecordReaderNextException(HiveIOExceptionHandlerChain.java:121) at org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil.handleRecordReaderNextException(HiveIOExceptionHandlerUtil.java:77) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.doNext(HiveContextAwareRecordReader.java:355) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:79) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:33) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.next(HiveContextAwareRecordReader.java:116) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.next(TezGroupedSplitsInputFormat.java:141) at org.apache.tez.mapreduce.lib.MRReaderMapred.next(MRReaderMapred.java:113) at org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.pushRecord(MapRecordSource.java:66) ... 16 more Caused by: java.io.IOException: Couldn't get next key/value from mongodb: at com.mongodb.hadoop.mapred.input.MongoRecordReader.nextKeyValue(MongoRecordReader.java:113) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:118) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:34) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.doNext(HiveContextAwareRecordReader.java:350) ... 22 more Caused by: com.mongodb.MongoInternalException: Unexpected exception at com.mongodb.connection.InternalStreamConnection.translateReadException(InternalStreamConnection.java:469) at com.mongodb.connection.InternalStreamConnection.receiveMessage(InternalStreamConnection.java:214) at com.mongodb.connection.UsageTrackingInternalConnection.receiveMessage(UsageTrackingInternalConnection.java:96) at com.mongodb.connection.DefaultConnectionPool$PooledConnection.receiveMessage(DefaultConnectionPool.java:438) at com.mongodb.connection.CommandProtocol.execute(CommandProtocol.java:105) at com.mongodb.connection.DefaultServer$DefaultServerProtocolExecutor.execute(DefaultServer.java:159) at com.mongodb.connection.DefaultServerConnection.executeProtocol(DefaultServerConnection.java:289) at com.mongodb.connection.DefaultServerConnection.command(DefaultServerConnection.java:176) at com.mongodb.operation.QueryBatchCursor.getMore(QueryBatchCursor.java:209) at com.mongodb.operation.QueryBatchCursor.hasNext(QueryBatchCursor.java:105) at com.mongodb.MongoBatchCursorAdapter.hasNext(MongoBatchCursorAdapter.java:46) at com.mongodb.DBCursor.hasNext(DBCursor.java:145) at com.mongodb.hadoop.mapred.input.MongoRecordReader.nextKeyValue(MongoRecordReader.java:100) ... 25 more Caused by: java.lang.OutOfMemoryError: Java heap space at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at com.mongodb.internal.connection.PowerOfTwoBufferPool.createNew(PowerOfTwoBufferPool.java:84) at com.mongodb.internal.connection.PowerOfTwoBufferPool.access$000(PowerOfTwoBufferPool.java:33) at com.mongodb.internal.connection.PowerOfTwoBufferPool$1.create(PowerOfTwoBufferPool.java:57) at com.mongodb.internal.connection.PowerOfTwoBufferPool$1.create(PowerOfTwoBufferPool.java:54) at com.mongodb.internal.connection.ConcurrentPool.createNewAndReleasePermitIfFailure(ConcurrentPool.java:164) at com.mongodb.internal.connection.ConcurrentPool.get(ConcurrentPool.java:132) at com.mongodb.internal.connection.ConcurrentPool.get(ConcurrentPool.java:109) at com.mongodb.internal.connection.PowerOfTwoBufferPool.getBuffer(PowerOfTwoBufferPool.java:76) at com.mongodb.connection.SocketStream.read(SocketStream.java:80) at com.mongodb.connection.InternalStreamConnection.receiveResponseBuffers(InternalStreamConnection.java:486) at com.mongodb.connection.InternalStreamConnection.receiveMessage(InternalStreamConnection.java:210) at com.mongodb.connection.UsageTrackingInternalConnection.receiveMessage(UsageTrackingInternalConnection.java:96) at com.mongodb.connection.DefaultConnectionPool$PooledConnection.receiveMessage(DefaultConnectionPool.java:438) at com.mongodb.connection.CommandProtocol.execute(CommandProtocol.java:105) at com.mongodb.connection.DefaultServer$DefaultServerProtocolExecutor.execute(DefaultServer.java:159) at com.mongodb.connection.DefaultServerConnection.executeProtocol(DefaultServerConnection.java:289) at com.mongodb.connection.DefaultServerConnection.command(DefaultServerConnection.java:176) at com.mongodb.operation.QueryBatchCursor.getMore(QueryBatchCursor.java:209) at com.mongodb.operation.QueryBatchCursor.hasNext(QueryBatchCursor.java:105) at com.mongodb.MongoBatchCursorAdapter.hasNext(MongoBatchCursorAdapter.java:46) at com.mongodb.DBCursor.hasNext(DBCursor.java:145) at com.mongodb.hadoop.mapred.input.MongoRecordReader.nextKeyValue(MongoRecordReader.java:100) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:118) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:34) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.doNext(HiveContextAwareRecordReader.java:350) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:79) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:33) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.next(HiveContextAwareRecordReader.java:116) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.next(TezGroupedSplitsInputFormat.java:141) at org.apache.tez.mapreduce.lib.MRReaderMapred.next(MRReaderMapred.java:113) ], TaskAttempt 2 failed, info=[Error: Failure while running task:java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: java.io.IOException: java.io.IOException: Couldn't get next key/value from mongodb: at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:173) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:139) at org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:347) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable$1.run(TezTaskRunner.java:194) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable$1.run(TezTaskRunner.java:185) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1866) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable.callInternal(TezTaskRunner.java:185) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable.callInternal(TezTaskRunner.java:181) at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: java.io.IOException: java.io.IOException: Couldn't get next key/value from mongodb: at org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.pushRecord(MapRecordSource.java:76) at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.run(MapRecordProcessor.java:325) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:150) ... 14 more Caused by: java.io.IOException: java.io.IOException: Couldn't get next key/value from mongodb: at org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain.handleRecordReaderNextException(HiveIOExceptionHandlerChain.java:121) at org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil.handleRecordReaderNextException(HiveIOExceptionHandlerUtil.java:77) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.doNext(HiveContextAwareRecordReader.java:355) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:79) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:33) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.next(HiveContextAwareRecordReader.java:116) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.next(TezGroupedSplitsInputFormat.java:141) at org.apache.tez.mapreduce.lib.MRReaderMapred.next(MRReaderMapred.java:113) at org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.pushRecord(MapRecordSource.java:66) ... 16 more Caused by: java.io.IOException: Couldn't get next key/value from mongodb: at com.mongodb.hadoop.mapred.input.MongoRecordReader.nextKeyValue(MongoRecordReader.java:113) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:118) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:34) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.doNext(HiveContextAwareRecordReader.java:350) ... 22 more Caused by: com.mongodb.MongoInternalException: Unexpected exception at com.mongodb.connection.InternalStreamConnection.translateReadException(InternalStreamConnection.java:469) at com.mongodb.connection.InternalStreamConnection.receiveMessage(InternalStreamConnection.java:214) at com.mongodb.connection.UsageTrackingInternalConnection.receiveMessage(UsageTrackingInternalConnection.java:96) at com.mongodb.connection.DefaultConnectionPool$PooledConnection.receiveMessage(DefaultConnectionPool.java:438) at com.mongodb.connection.CommandProtocol.execute(CommandProtocol.java:105) at com.mongodb.connection.DefaultServer$DefaultServerProtocolExecutor.execute(DefaultServer.java:159) at com.mongodb.connection.DefaultServerConnection.executeProtocol(DefaultServerConnection.java:289) at com.mongodb.connection.DefaultServerConnection.command(DefaultServerConnection.java:176) at com.mongodb.operation.QueryBatchCursor.getMore(QueryBatchCursor.java:209) at com.mongodb.operation.QueryBatchCursor.hasNext(QueryBatchCursor.java:105) at com.mongodb.MongoBatchCursorAdapter.hasNext(MongoBatchCursorAdapter.java:46) at com.mongodb.DBCursor.hasNext(DBCursor.java:145) at com.mongodb.hadoop.mapred.input.MongoRecordReader.nextKeyValue(MongoRecordReader.java:100) ... 25 more Caused by: java.lang.OutOfMemoryError: Java heap space at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at com.mongodb.internal.connection.PowerOfTwoBufferPool.createNew(PowerOfTwoBufferPool.java:84) at com.mongodb.internal.connection.PowerOfTwoBufferPool.access$000(PowerOfTwoBufferPool.java:33) at com.mongodb.internal.connection.PowerOfTwoBufferPool$1.create(PowerOfTwoBufferPool.java:57) at com.mongodb.internal.connection.PowerOfTwoBufferPool$1.create(PowerOfTwoBufferPool.java:54) at com.mongodb.internal.connection.ConcurrentPool.createNewAndReleasePermitIfFailure(ConcurrentPool.java:164) at com.mongodb.internal.connection.ConcurrentPool.get(ConcurrentPool.java:132) at com.mongodb.internal.connection.ConcurrentPool.get(ConcurrentPool.java:109) at com.mongodb.internal.connection.PowerOfTwoBufferPool.getBuffer(PowerOfTwoBufferPool.java:76) at com.mongodb.connection.SocketStream.read(SocketStream.java:80) at com.mongodb.connection.InternalStreamConnection.receiveResponseBuffers(InternalStreamConnection.java:486) at com.mongodb.connection.InternalStreamConnection.receiveMessage(InternalStreamConnection.java:210) at com.mongodb.connection.UsageTrackingInternalConnection.receiveMessage(UsageTrackingInternalConnection.java:96) at com.mongodb.connection.DefaultConnectionPool$PooledConnection.receiveMessage(DefaultConnectionPool.java:438) at com.mongodb.connection.CommandProtocol.execute(CommandProtocol.java:105) at com.mongodb.connection.DefaultServer$DefaultServerProtocolExecutor.execute(DefaultServer.java:159) at com.mongodb.connection.DefaultServerConnection.executeProtocol(DefaultServerConnection.java:289) at com.mongodb.connection.DefaultServerConnection.command(DefaultServerConnection.java:176) at com.mongodb.operation.QueryBatchCursor.getMore(QueryBatchCursor.java:209) at com.mongodb.operation.QueryBatchCursor.hasNext(QueryBatchCursor.java:105) at com.mongodb.MongoBatchCursorAdapter.hasNext(MongoBatchCursorAdapter.java:46) at com.mongodb.DBCursor.hasNext(DBCursor.java:145) at com.mongodb.hadoop.mapred.input.MongoRecordReader.nextKeyValue(MongoRecordReader.java:100) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:118) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:34) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.doNext(HiveContextAwareRecordReader.java:350) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:79) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:33) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.next(HiveContextAwareRecordReader.java:116) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.next(TezGroupedSplitsInputFormat.java:141) at org.apache.tez.mapreduce.lib.MRReaderMapred.next(MRReaderMapred.java:113) ], TaskAttempt 3 failed, info=[Error: Failure while running task:java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: java.io.IOException: java.io.IOException: Couldn't get next key/value from mongodb: at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:173) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:139) at org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:347) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable$1.run(TezTaskRunner.java:194) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable$1.run(TezTaskRunner.java:185) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1866) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable.callInternal(TezTaskRunner.java:185) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable.callInternal(TezTaskRunner.java:181) at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: java.io.IOException: java.io.IOException: Couldn't get next key/value from mongodb: at org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.pushRecord(MapRecordSource.java:76) at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.run(MapRecordProcessor.java:325) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:150) ... 14 more Caused by: java.io.IOException: java.io.IOException: Couldn't get next key/value from mongodb: at org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain.handleRecordReaderNextException(HiveIOExceptionHandlerChain.java:121) at org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil.handleRecordReaderNextException(HiveIOExceptionHandlerUtil.java:77) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.doNext(HiveContextAwareRecordReader.java:355) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:79) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:33) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.next(HiveContextAwareRecordReader.java:116) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.next(TezGroupedSplitsInputFormat.java:141) at org.apache.tez.mapreduce.lib.MRReaderMapred.next(MRReaderMapred.java:113) at org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.pushRecord(MapRecordSource.java:66) ... 16 more Caused by: java.io.IOException: Couldn't get next key/value from mongodb: at com.mongodb.hadoop.mapred.input.MongoRecordReader.nextKeyValue(MongoRecordReader.java:113) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:118) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:34) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.doNext(HiveContextAwareRecordReader.java:350) ... 22 more Caused by: com.mongodb.MongoInternalException: Unexpected exception at com.mongodb.connection.InternalStreamConnection.translateReadException(InternalStreamConnection.java:469) at com.mongodb.connection.InternalStreamConnection.receiveMessage(InternalStreamConnection.java:214) at com.mongodb.connection.UsageTrackingInternalConnection.receiveMessage(UsageTrackingInternalConnection.java:96) at com.mongodb.connection.DefaultConnectionPool$PooledConnection.receiveMessage(DefaultConnectionPool.java:438) at com.mongodb.connection.CommandProtocol.execute(CommandProtocol.java:105) at com.mongodb.connection.DefaultServer$DefaultServerProtocolExecutor.execute(DefaultServer.java:159) at com.mongodb.connection.DefaultServerConnection.executeProtocol(DefaultServerConnection.java:289) at com.mongodb.connection.DefaultServerConnection.command(DefaultServerConnection.java:176) at com.mongodb.operation.QueryBatchCursor.getMore(QueryBatchCursor.java:209) at com.mongodb.operation.QueryBatchCursor.hasNext(QueryBatchCursor.java:105) at com.mongodb.MongoBatchCursorAdapter.hasNext(MongoBatchCursorAdapter.java:46) at com.mongodb.DBCursor.hasNext(DBCursor.java:145) at com.mongodb.hadoop.mapred.input.MongoRecordReader.nextKeyValue(MongoRecordReader.java:100) ... 25 more Caused by: java.lang.OutOfMemoryError: Java heap space at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at com.mongodb.internal.connection.PowerOfTwoBufferPool.createNew(PowerOfTwoBufferPool.java:84) at com.mongodb.internal.connection.PowerOfTwoBufferPool.access$000(PowerOfTwoBufferPool.java:33) at com.mongodb.internal.connection.PowerOfTwoBufferPool$1.create(PowerOfTwoBufferPool.java:57) at com.mongodb.internal.connection.PowerOfTwoBufferPool$1.create(PowerOfTwoBufferPool.java:54) at com.mongodb.internal.connection.ConcurrentPool.createNewAndReleasePermitIfFailure(ConcurrentPool.java:164) at com.mongodb.internal.connection.ConcurrentPool.get(ConcurrentPool.java:132) at com.mongodb.internal.connection.ConcurrentPool.get(ConcurrentPool.java:109) at com.mongodb.internal.connection.PowerOfTwoBufferPool.getBuffer(PowerOfTwoBufferPool.java:76) at com.mongodb.connection.SocketStream.read(SocketStream.java:80) at com.mongodb.connection.InternalStreamConnection.receiveResponseBuffers(InternalStreamConnection.java:486) at com.mongodb.connection.InternalStreamConnection.receiveMessage(InternalStreamConnection.java:210) at com.mongodb.connection.UsageTrackingInternalConnection.receiveMessage(UsageTrackingInternalConnection.java:96) at com.mongodb.connection.DefaultConnectionPool$PooledConnection.receiveMessage(DefaultConnectionPool.java:438) at com.mongodb.connection.CommandProtocol.execute(CommandProtocol.java:105) at com.mongodb.connection.DefaultServer$DefaultServerProtocolExecutor.execute(DefaultServer.java:159) at com.mongodb.connection.DefaultServerConnection.executeProtocol(DefaultServerConnection.java:289) at com.mongodb.connection.DefaultServerConnection.command(DefaultServerConnection.java:176) at com.mongodb.operation.QueryBatchCursor.getMore(QueryBatchCursor.java:209) at com.mongodb.operation.QueryBatchCursor.hasNext(QueryBatchCursor.java:105) at com.mongodb.MongoBatchCursorAdapter.hasNext(MongoBatchCursorAdapter.java:46) at com.mongodb.DBCursor.hasNext(DBCursor.java:145) at com.mongodb.hadoop.mapred.input.MongoRecordReader.nextKeyValue(MongoRecordReader.java:100) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:118) at com.mongodb.hadoop.mapred.input.MongoRecordReader.next(MongoRecordReader.java:34) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.doNext(HiveContextAwareRecordReader.java:350) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:79) at org.apache.hadoop.hive.ql.io.HiveRecordReader.doNext(HiveRecordReader.java:33) at org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader.next(HiveContextAwareRecordReader.java:116) at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.next(TezGroupedSplitsInputFormat.java:141) at org.apache.tez.mapreduce.lib.MRReaderMapred.next(MRReaderMapred.java:113) ]], Vertex did not succeed due to OWN_TASK_FAILURE, failedTasks:1 killedTasks:0, Vertex vertex_1508230545807_0009_2_00 [Map 1] killed/failed due to:OWN_TASK_FAILURE]DAG did not succeed due to VERTEX_FAILURE. failedVertices:1 killedVertices:0 (state=08S01,code=2)