Hi, I am trying to take our daily HDFS logs from Ranger and convert those into AVRO and creating a HIVE table on top of it for reporting. I used InferAvroSchema to produce AVRO schema and was able to convert all most all of the old logs except a few which are failing with below messages. sometimes with this error 2017-08-08 10:25:27,983 WARN [Timer-Driven Process Thread-7] o.a.n.c.t.ContinuallyRunProcessorTask java.lang.RuntimeException: Unexpected end-of-input in VALUE_STRING at [Source: org.apache.nifi.controller.repository.io.FlowFileAccessInputStream@2c191ce8; line: 2871, column: 2864] at com.fasterxml.jackson.databind.MappingIterator.next(MappingIterator.java:196) ~[jackson-databind-2.6.1.jar:2.6.1] at org.kitesdk.shaded.com.google.common.collect.Iterators$8.next(Iterators.java:811) ~[na:na] at org.kitesdk.data.spi.filesystem.JSONFileReader.next(JSONFileReader.java:121) ~[na:na] at org.apache.nifi.processors.kite.ConvertJSONToAvro$1.process(ConvertJSONToAvro.java:148) ~[na:na] at org.apache.nifi.controller.repository.StandardProcessSession.write(StandardProcessSession.java:2578) ~[nifi-framework-core-1.1.0.2.1.0.0-165.jar:1.1.0.2.1.0.0-165] at org.apache.nifi.processors.kite.ConvertJSONToAvro.onTrigger(ConvertJSONToAvro.java:139) ~[na:na] at org.apache.nifi.processor.AbstractProcessor.onTrigger(AbstractProcessor.java:27) ~[nifi-api-1.1.0.2.1.0.0-165.jar:1.1.0.2.1.0.0-165] at org.apache.nifi.controller.StandardProcessorNode.onTrigger(StandardProcessorNode.java:1099) ~[nifi-framework-core-1.1.0.2.1.0.0-165.jar:1.1.0.2.1.0.0-165] at org.apache.nifi.controller.tasks.ContinuallyRunProcessorTask.call(ContinuallyRunProcessorTask.java:136) [nifi-framework-core-1.1.0.2.1.0.0-165.jar:1.1.0.2.1.0.0-165] at org.apache.nifi.controller.tasks.ContinuallyRunProcessorTask.call(ContinuallyRunProcessorTask.java:47) [nifi-framework-core-1.1.0.2.1.0.0-165.jar:1.1.0.2.1.0.0-165] at org.apache.nifi.controller.scheduling.TimerDrivenSchedulingAgent$1.run(TimerDrivenSchedulingAgent.java:132) [nifi-framework-core-1.1.0.2.1.0.0-165.jar:1.1.0.2.1.0.0-165] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_112] at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) [na:1.8.0_112] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) [na:1.8.0_112] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) [na:1.8.0_112] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [na:1.8.0_112] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [na:1.8.0_112] at java.lang.Thread.run(Thread.java:745) [na:1.8.0_112] Caused by: com.fasterxml.jackson.core.JsonParseException: Unexpected end-of-input in VALUE_STRING at [Source: org.apache.nifi.controller.repository.io.FlowFileAccessInputStream@2c191ce8; line: 2871, column: 2864] at com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:1581) ~[jackson-core-2.6.1.jar:2.6.1] at com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:533) ~[jackson-core-2.6.1.jar:2.6.1] at com.fasterxml.jackson.core.base.ParserMinimalBase._reportInvalidEOF(ParserMinimalBase.java:470) ~[jackson-core-2.6.1.jar:2.6.1] at com.fasterxml.jackson.core.base.ParserMinimalBase._reportInvalidEOF(ParserMinimalBase.java:466) ~[jackson-core-2.6.1.jar:2.6.1] at com.fasterxml.jackson.core.base.ParserBase.loadMoreGuaranteed(ParserBase.java:459) ~[jackson-core-2.6.1.jar:2.6.1] at com.fasterxml.jackson.core.json.UTF8StreamJsonParser._finishAndReturnString(UTF8StreamJsonParser.java:2389) ~[jackson-core-2.6.1.jar:2.6.1] at com.fasterxml.jackson.core.json.UTF8StreamJsonParser.getText(UTF8StreamJsonParser.java:285) ~[jackson-core-2.6.1.jar:2.6.1] at com.fasterxml.jackson.databind.deser.std.BaseNodeDeserializer.deserializeObject(JsonNodeDeserializer.java:233) ~[jackson-databind-2.6.1.jar:2.6.1] at com.fasterxml.jackson.databind.deser.std.JsonNodeDeserializer.deserialize(JsonNodeDeserializer.java:69) ~[jackson-databind-2.6.1.jar:2.6.1] at com.fasterxml.jackson.databind.deser.std.JsonNodeDeserializer.deserialize(JsonNodeDeserializer.java:15) ~[jackson-databind-2.6.1.jar:2.6.1] at com.fasterxml.jackson.databind.MappingIterator.nextValue(MappingIterator.java:277) ~[jackson-databind-2.6.1.jar:2.6.1] at com.fasterxml.jackson.databind.MappingIterator.next(MappingIterator.java:192) ~[jackson-databind-2.6.1.jar:2.6.1] ... 17 common frames omitted and sometimes with this ERROR 2017-08-08 10:01:19,391 ERROR [Timer-Driven Process Thread-2] o.a.n.processors.kite.ConvertJSONToAvro ConvertJSONToAvro[id=0995e03c-40f5-4156-a065-4cda05b4efa1] ConvertJSONToAvro[id=0995e03c-40f5-4156-a065-4cda05b4efa1] failed to process due to java.lang.RuntimeException: Unexpected end-of-input in field name at [Source: org.apache.nifi.controller.repository.io.FlowFileAccessInputStream@ccb3a65; line: 440072, column: 2258]; rolling back session: java.lang.RuntimeException: Unexpected end-of-input in field name at [Source: org.apache.nifi.controller.repository.io.FlowFileAccessInputStream@ccb3a65; line: 440072, column: 2258] 2017-08-08 10:01:19,392 ERROR [Timer-Driven Process Thread-2] o.a.n.processors.kite.ConvertJSONToAvro java.lang.RuntimeException: Unexpected end-of-input in field name at [Source: org.apache.nifi.controller.repository.io.FlowFileAccessInputStream@ccb3a65; line: 440072, column: 2258] at com.fasterxml.jackson.databind.MappingIterator.next(MappingIterator.java:196) ~[jackson-databind-2.6.1.jar:2.6.1] at org.kitesdk.shaded.com.google.common.collect.Iterators$8.next(Iterators.java:811) ~[kite-data-core-1.0.0.jar:na] at org.kitesdk.data.spi.filesystem.JSONFileReader.next(JSONFileReader.java:121) ~[kite-data-core-1.0.0.jar:na] at org.apache.nifi.processors.kite.ConvertJSONToAvro$1.process(ConvertJSONToAvro.java:148) ~[nifi-kite-processors-1.1.0.2.1.0.0-165.jar:1.1.0.2.1.0.0-165] at org.apache.nifi.controller.repository.StandardProcessSession.write(StandardProcessSession.java:2578) ~[nifi-framework-core-1.1.0.2.1.0.0-165.jar:1.1.0.2.1.0.0-165] at org.apache.nifi.processors.kite.ConvertJSONToAvro.onTrigger(ConvertJSONToAvro.java:139) ~[nifi-kite-processors-1.1.0.2.1.0.0-165.jar:1.1.0.2.1.0.0-165] at org.apache.nifi.processor.AbstractProcessor.onTrigger(AbstractProcessor.java:27) ~[nifi-api-1.1.0.2.1.0.0-165.jar:1.1.0.2.1.0.0-165] at org.apache.nifi.controller.StandardProcessorNode.onTrigger(StandardProcessorNode.java:1099) [nifi-framework-core-1.1.0.2.1.0.0-165.jar:1.1.0.2.1.0.0-165] at org.apache.nifi.controller.tasks.ContinuallyRunProcessorTask.call(ContinuallyRunProcessorTask.java:136) [nifi-framework-core-1.1.0.2.1.0.0-165.jar:1.1.0.2.1.0.0-165] at org.apache.nifi.controller.tasks.ContinuallyRunProcessorTask.call(ContinuallyRunProcessorTask.java:47) [nifi-framework-core-1.1.0.2.1.0.0-165.jar:1.1.0.2.1.0.0-165] at org.apache.nifi.controller.scheduling.TimerDrivenSchedulingAgent$1.run(TimerDrivenSchedulingAgent.java:132) [nifi-framework-core-1.1.0.2.1.0.0-165.jar:1.1.0.2.1.0.0-165] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_112] at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) [na:1.8.0_112] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) [na:1.8.0_112] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) [na:1.8.0_112] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [na:1.8.0_112] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [na:1.8.0_112] at java.lang.Thread.run(Thread.java:745) [na:1.8.0_112] Caused by: com.fasterxml.jackson.core.JsonParseException: Unexpected end-of-input in field name at [Source: org.apache.nifi.controller.repository.io.FlowFileAccessInputStream@ccb3a65; line: 440072, column: 2258] at com.fasterxml.jackson.core.JsonParser._constructError(JsonParser.java:1581) ~[jackson-core-2.6.1.jar:2.6.1] at com.fasterxml.jackson.core.base.ParserMinimalBase._reportError(ParserMinimalBase.java:533) ~[jackson-core-2.6.1.jar:2.6.1] at com.fasterxml.jackson.core.base.ParserMinimalBase._reportInvalidEOF(ParserMinimalBase.java:470) ~[jackson-core-2.6.1.jar:2.6.1] at com.fasterxml.jackson.core.json.UTF8StreamJsonParser.parseEscapedName(UTF8StreamJsonParser.java:1966) ~[jackson-core-2.6.1.jar:2.6.1] at com.fasterxml.jackson.core.json.UTF8StreamJsonParser.slowParseName(UTF8StreamJsonParser.java:1867) ~[jackson-core-2.6.1.jar:2.6.1] at com.fasterxml.jackson.core.json.UTF8StreamJsonParser._parseName(UTF8StreamJsonParser.java:1651) ~[jackson-core-2.6.1.jar:2.6.1] at com.fasterxml.jackson.core.json.UTF8StreamJsonParser.nextFieldName(UTF8StreamJsonParser.java:1007) ~[jackson-core-2.6.1.jar:2.6.1] at com.fasterxml.jackson.databind.deser.std.BaseNodeDeserializer.deserializeObject(JsonNodeDeserializer.java:219) ~[jackson-databind-2.6.1.jar:2.6.1] at com.fasterxml.jackson.databind.deser.std.JsonNodeDeserializer.deserialize(JsonNodeDeserializer.java:69) ~[jackson-databind-2.6.1.jar:2.6.1] at com.fasterxml.jackson.databind.deser.std.JsonNodeDeserializer.deserialize(JsonNodeDeserializer.java:15) ~[jackson-databind-2.6.1.jar:2.6.1] at com.fasterxml.jackson.databind.MappingIterator.nextValue(MappingIterator.java:277) ~[jackson-databind-2.6.1.jar:2.6.1] at com.fasterxml.jackson.databind.MappingIterator.next(MappingIterator.java:192) ~[jackson-databind-2.6.1.jar:2.6.1] ... 17 common frames omitted So it looks like those files are not in correct format , i was expecting NiFi to move those files on to Failure relationship so that we can examine those or do something else. But it still keep them in the queue. Which make it to try and process these files again and again..How to solve this.? ... View more

Hi, I have my JSON file with each record as below {"repoType":1,"repo":"Dev_hadoop","reqUser":"tarapasa","evtTime":"2017-01-04 23:59:54.606","access":"READ_EXECUTE","resource":"/user/putarapasa","resType":"path","action":"read","result":1,"policy":-1,"reason":"/user/putarapasa","enforcer":"hadoop-acl","cliIP":"172.16.2.4","agentHost":"HDPDEV.COM","logType":"RangerAudit","id":"d3285a85-caf9-40bc-bb5d-c4208d0c4b8f-2314470","seq_num":3457350,"event_count":5,"event_dur_ms":4008,"tags":[]} I created a HIVE table with below sql. CREATE TABLE HDFS_log( repoType string, repo string, reqUser string, evtTime string, access string, resource string, resType string, action string, result string, policy string, reason string, enforcer string, cliIP string, agentHost string, logType string, id string, seq_num string, event_count string, event_dur_ms string, tags array<string> ) ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe' STORED AS TEXTFILE LOCATION '/user/putarapasa/HDFS Logs' i copied some files in to the above folder and i was able to query the records without any issue. when i copied many more files , i am getting an error like below. it looks like one of the files contained a record where its missing one tags. what do you do in these scenarios.? ERROR : Vertex failed, vertexName=Map 1, vertexId=vertex_1493386875011_0001_2_00, diagnostics=[Task failed, taskId=task_1493386875011_0001_2_00_000000, diagnostics=[TaskAttempt 0 failed, info=[Error: Failure while running task:java.lang.RuntimeException: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable {"repoType":1,"repo":"NestlePurinaDev_hadoop","reqUser":"hbase","evtTime":"2016-12-27 09:49:00.951","access":"WRITE","resource":"/apps/hbase/data/data/hbase/namespace/2fdbb2aa9731bb723a48bfd157b60af2/recovered.edits/67.seqid","resType":"path","result":1,"po at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:173) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.run(TezProcessor.java:139) at org.apache.tez.runtime.LogicalIOProcessorRuntimeTask.run(LogicalIOProcessorRuntimeTask.java:347) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable$1.run(TezTaskRunner.java:194) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable$1.run(TezTaskRunner.java:185) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1724) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable.callInternal(TezTaskRunner.java:185) at org.apache.tez.runtime.task.TezTaskRunner$TaskRunnerCallable.callInternal(TezTaskRunner.java:181) at org.apache.tez.common.CallableWithNdc.call(CallableWithNdc.java:36) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable {"repoType":1,"repo":"NestlePurinaDev_hadoop","reqUser":"hbase","evtTime":"2016-12-27 09:49:00.951","access":"WRITE","resource":"/apps/hbase/data/data/hbase/namespace/2fdbb2aa9731bb723a48bfd157b60af2/recovered.edits/67.seqid","resType":"path","result":1,"po at org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.processRow(MapRecordSource.java:91) at org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.pushRecord(MapRecordSource.java:68) at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.run(MapRecordProcessor.java:325) at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:150) ... 14 more Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable {"repoType":1,"repo":"NestlePurinaDev_hadoop","reqUser":"hbase","evtTime":"2016-12-27 09:49:00.951","access":"WRITE","resource":"/apps/hbase/data/data/hbase/namespace/2fdbb2aa9731bb723a48bfd157b60af2/recovered.edits/67.seqid","resType":"path","result":1,"po at org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:563) at org.apache.hadoop.hive.ql.exec.tez.MapRecordSource.processRow(MapRecordSource.java:83) ... 17 more Caused by: org.apache.hadoop.hive.serde2.SerDeException: org.codehaus.jackson.JsonParseException: Unexpected character ('r' (code 114)): was expecting a colon to separate field name and value at [Source: java.io.ByteArrayInputStream@40a1ead; line: 1, column: 263] at org.apache.hive.hcatalog.data.JsonSerDe.deserialize(JsonSerDe.java:179) at org.apache.hadoop.hive.ql.exec.MapOperator$MapOpCtx.readRow(MapOperator.java:149) at org.apache.hadoop.hive.ql.exec.MapOperator$MapOpCtx.access$200(MapOperator.java:113) at org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:554) ... 18 more Caused by: org.codehaus.jackson.JsonParseException: Unexpected character ('r' (code 114)): was expecting a colon to separate field name and value at [Source: java.io.ByteArrayInputStream@40a1ead; line: 1, column: 263] at org.codehaus.jackson.JsonParser._constructError(JsonParser.java:1432) at org.codehaus.jackson.impl.JsonParserMinimalBase._reportError(JsonParserMinimalBase.java:385) at org.codehaus.jackson.impl.JsonParserMinimalBase._reportUnexpectedChar(JsonParserMinimalBase.java:306) at org.codehaus.jackson.impl.Utf8StreamParser.nextToken(Utf8StreamParser.java:494) at org.apache.hive.hcatalog.data.JsonSerDe.deserialize(JsonSerDe.java:172) ... 21 more ... View more