[root@node2 ~]# pig -useHCatalog -x tez
WARNING: Use "yarn jar" to launch YARN applications.
16/05/04 15:21:15 INFO pig.ExecTypeProvider: Trying ExecType : LOCAL
16/05/04 15:21:15 INFO pig.ExecTypeProvider: Trying ExecType : MAPREDUCE
16/05/04 15:21:15 INFO pig.ExecTypeProvider: Trying ExecType : TEZ_LOCAL
16/05/04 15:21:15 INFO pig.ExecTypeProvider: Trying ExecType : TEZ
16/05/04 15:21:15 INFO pig.ExecTypeProvider: Picked TEZ as the ExecType
2016-05-04 15:21:15,800 [main] INFO org.apache.pig.Main - Apache Pig version 0.15.0.2.4.0.0-169 (rexported) compiled Feb 10 2016, 07:50:04
2016-05-04 15:21:15,800 [main] INFO org.apache.pig.Main - Logging error messages to: /root/pig_1462364475798.log
2016-05-04 15:21:15,843 [main] INFO org.apache.pig.impl.util.Utils - Default bootup file /root/.pigbootup not found
2016-05-04 15:21:16,441 [main] INFO org.apache.pig.backend.hadoop.executionengine.HExecutionEngine - Connecting to hadoop file system at: hdfs://node1.hdp.host:8020
2016-05-04 15:21:17,844 [main] INFO org.apache.pig.PigServer - Pig Script ID for the session: PIG-default-bc994238-ce7e-4cdb-9a72-096b55822402
2016-05-04 15:21:18,335 [main] INFO org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl - Timeline service address: http://node2.hdp.host:8188/ws/v1/timeline/
2016-05-04 15:21:18,599 [main] INFO org.apache.pig.backend.hadoop.ATSService - Created ATS Hook
grunt> a = LOAD 'geolocation' USING org.apache.hive.hcatalog.pig.HCatLoader();
2016-05-04 15:21:29,850 [main] INFO hive.metastore - Trying to connect to metastore with URI thrift://node2.hdp.host:9083
2016-05-04 15:21:29,907 [main] INFO hive.metastore - Connected to metastore.
grunt> b = FILTER a BY event != 'normal';
grunt> c = FOREACH b generate driverid, event, (int) '1' as occurance;
grunt> d = GROUP c BY driverid;
grunt> e = foreach d generate group as driverid, SUM(c.occurance) as t_occ;
grunt> g = LOAD 'drivermileage' using org.apache.hive.hcatalog.pig.HCatLoader();
grunt> h = join e by driverid, g by driverid;
grunt> final_data = foreach h generate $0 as driverid, $1 as events, $3 as totmiles, (float) $3/$1 as riskfactor;
2016-05-04 15:22:19,173 [main] WARN org.apache.pig.newplan.BaseOperatorPlan - Encountered Warning IMPLICIT_CAST_TO_FLOAT 1 time(s).
grunt> store final_data into 'riskfactor' using org.apache.hive.hcatalog.pig.HCatStorer();
2016-05-04 15:22:28,034 [main] WARN org.apache.pig.newplan.BaseOperatorPlan - Encountered Warning IMPLICIT_CAST_TO_FLOAT 1 time(s).
2016-05-04 15:22:28,315 [main] WARN org.apache.pig.newplan.BaseOperatorPlan - Encountered Warning IMPLICIT_CAST_TO_FLOAT 1 time(s).
2016-05-04 15:22:28,881 [main] INFO org.apache.pig.tools.pigstats.ScriptState - Pig features used in the script: HASH_JOIN,GROUP_BY,FILTER
2016-05-04 15:22:28,922 [main] INFO org.apache.pig.data.SchemaTupleBackend - Key [pig.schematuple] was not set... will not generate code.
2016-05-04 15:22:28,976 [main] INFO org.apache.pig.newplan.logical.optimizer.LogicalPlanOptimizer - {RULES_ENABLED=[AddForEach, ColumnMapKeyPrune, ConstantCalculator, GroupByConstParallelSetter, LimitOptimizer, LoadTypeCastInserter, MergeFilter, MergeForEach, PartitionFilterOptimizer, PredicatePushdownOptimizer, PushDownForEachFlatten, PushUpFilter, SplitFilter, StreamTypeCastInserter]}
2016-05-04 15:22:29,225 [main] INFO org.apache.pig.newplan.logical.rules.ColumnPruneVisitor - Columns pruned for a: $0, $3, $4, $5, $6, $7, $8, $9
2016-05-04 15:22:29,392 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.TezLauncher - Tez staging directory is /tmp/root/staging and resources directory is /tmp/temp18217824
2016-05-04 15:22:29,473 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.plan.TezCompiler - File concatenation threshold: 100 optimistic? false
2016-05-04 15:22:29,518 [main] INFO org.apache.pig.backend.hadoop.executionengine.util.CombinerOptimizerUtil - Choosing to move algebraic foreach to combiner
2016-05-04 15:22:29,876 [main] INFO org.apache.hadoop.hive.ql.log.PerfLogger -
2016-05-04 15:22:29,885 [main] INFO org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - ORC pushdown predicate: null
2016-05-04 15:22:29,970 [main] INFO org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - FooterCacheHitRatio: 0/0
2016-05-04 15:22:29,971 [main] INFO org.apache.hadoop.hive.ql.log.PerfLogger -
2016-05-04 15:22:29,975 [main] INFO org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil - Total input paths (combined) to process : 1
2016-05-04 15:22:30,219 [main] INFO org.apache.hadoop.hive.ql.log.PerfLogger -
2016-05-04 15:22:30,219 [main] INFO org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - ORC pushdown predicate: null
2016-05-04 15:22:30,240 [main] INFO org.apache.hadoop.hive.ql.io.orc.OrcInputFormat - FooterCacheHitRatio: 0/0
2016-05-04 15:22:30,240 [main] INFO org.apache.hadoop.hive.ql.log.PerfLogger -
2016-05-04 15:22:30,240 [main] INFO org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil - Total input paths (combined) to process : 1
2016-05-04 15:22:37,694 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.TezJobCompiler - Local resource: libfb303-0.9.2.jar
2016-05-04 15:22:37,695 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.TezJobCompiler - Local resource: hive-exec-1.2.1000.2.4.0.0-169.jar
2016-05-04 15:22:37,695 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.TezJobCompiler - Local resource: pig-0.15.0.2.4.0.0-169-core-h2.jar
2016-05-04 15:22:37,695 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.TezJobCompiler - Local resource: jdo-api-3.0.1.jar
2016-05-04 15:22:37,695 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.TezJobCompiler - Local resource: automaton-1.11-8.jar
2016-05-04 15:22:37,695 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.TezJobCompiler - Local resource: hive-metastore-1.2.1000.2.4.0.0-169.jar
2016-05-04 15:22:37,695 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.TezJobCompiler - Local resource: hive-hcatalog-core-1.2.1000.2.4.0.0-169.jar
2016-05-04 15:22:37,695 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.TezJobCompiler - Local resource: antlr-runtime-3.4.jar
2016-05-04 15:22:37,695 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.TezJobCompiler - Local resource: hive-hbase-handler-1.2.1000.2.4.0.0-169.jar
2016-05-04 15:22:37,695 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.TezJobCompiler - Local resource: libthrift-0.9.2.jar
2016-05-04 15:22:37,695 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.TezJobCompiler - Local resource: hive-hcatalog-pig-adapter-1.2.1000.2.4.0.0-169.jar
2016-05-04 15:22:37,889 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.io.sort.mb to 859 from MR setting mapreduce.task.io.sort.mb
2016-05-04 15:22:37,889 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.read.timeout to 180000 from MR setting mapreduce.reduce.shuffle.read.timeout
2016-05-04 15:22:37,889 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.ifile.readahead.bytes to 4194304 from MR setting mapreduce.ifile.readahead.bytes
2016-05-04 15:22:37,889 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.ssl.enable to false from MR setting mapreduce.shuffle.ssl.enabled
2016-05-04 15:22:37,889 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.sort.spill.percent to 0.7 from MR setting mapreduce.map.sort.spill.percent
2016-05-04 15:22:37,889 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.ifile.readahead to true from MR setting mapreduce.ifile.readahead
2016-05-04 15:22:37,889 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.merge.percent to 0.66 from MR setting mapreduce.reduce.shuffle.merge.percent
2016-05-04 15:22:37,889 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.parallel.copies to 30 from MR setting mapreduce.reduce.shuffle.parallelcopies
2016-05-04 15:22:37,889 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.memory.limit.percent to 0.25 from MR setting mapreduce.reduce.shuffle.memory.limit.percent
2016-05-04 15:22:37,889 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.io.sort.factor to 100 from MR setting mapreduce.task.io.sort.factor
2016-05-04 15:22:37,890 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.compress to false from MR setting mapreduce.map.output.compress
2016-05-04 15:22:37,890 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.connect.timeout to 180000 from MR setting mapreduce.reduce.shuffle.connect.timeout
2016-05-04 15:22:37,890 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.task.input.post-merge.buffer.percent to 0.0 from MR setting mapreduce.reduce.input.buffer.percent
2016-05-04 15:22:37,890 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.compress.codec to org.apache.hadoop.io.compress.DefaultCodec from MR setting mapreduce.map.output.compress.codec
2016-05-04 15:22:37,890 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.merge.progress.records to 10000 from MR setting mapreduce.task.merge.progress.records
2016-05-04 15:22:37,890 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.internal.sorter.class to org.apache.hadoop.util.QuickSort from MR setting map.sort.class
2016-05-04 15:22:37,890 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.fetch.buffer.percent to 0.7 from MR setting mapreduce.reduce.shuffle.input.buffer.percent
2016-05-04 15:22:38,045 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.TezJobCompiler - For vertex - scope-51: parallelism=1, memory=1536, java opts=-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB
2016-05-04 15:22:38,207 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.io.sort.mb to 859 from MR setting mapreduce.task.io.sort.mb
2016-05-04 15:22:38,207 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.read.timeout to 180000 from MR setting mapreduce.reduce.shuffle.read.timeout
2016-05-04 15:22:38,207 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.ifile.readahead.bytes to 4194304 from MR setting mapreduce.ifile.readahead.bytes
2016-05-04 15:22:38,207 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.ssl.enable to false from MR setting mapreduce.shuffle.ssl.enabled
2016-05-04 15:22:38,207 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.sort.spill.percent to 0.7 from MR setting mapreduce.map.sort.spill.percent
2016-05-04 15:22:38,207 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.ifile.readahead to true from MR setting mapreduce.ifile.readahead
2016-05-04 15:22:38,207 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.merge.percent to 0.66 from MR setting mapreduce.reduce.shuffle.merge.percent
2016-05-04 15:22:38,207 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.parallel.copies to 30 from MR setting mapreduce.reduce.shuffle.parallelcopies
2016-05-04 15:22:38,207 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.memory.limit.percent to 0.25 from MR setting mapreduce.reduce.shuffle.memory.limit.percent
2016-05-04 15:22:38,207 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.io.sort.factor to 100 from MR setting mapreduce.task.io.sort.factor
2016-05-04 15:22:38,207 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.compress to false from MR setting mapreduce.map.output.compress
2016-05-04 15:22:38,207 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.connect.timeout to 180000 from MR setting mapreduce.reduce.shuffle.connect.timeout
2016-05-04 15:22:38,207 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.task.input.post-merge.buffer.percent to 0.0 from MR setting mapreduce.reduce.input.buffer.percent
2016-05-04 15:22:38,207 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.compress.codec to org.apache.hadoop.io.compress.DefaultCodec from MR setting mapreduce.map.output.compress.codec
2016-05-04 15:22:38,207 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.merge.progress.records to 10000 from MR setting mapreduce.task.merge.progress.records
2016-05-04 15:22:38,207 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.internal.sorter.class to org.apache.hadoop.util.QuickSort from MR setting map.sort.class
2016-05-04 15:22:38,207 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.fetch.buffer.percent to 0.7 from MR setting mapreduce.reduce.shuffle.input.buffer.percent
2016-05-04 15:22:38,319 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.TezJobCompiler - Set auto parallelism for vertex scope-52
2016-05-04 15:22:38,319 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.TezJobCompiler - For vertex - scope-52: parallelism=1, memory=2048, java opts=-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB
2016-05-04 15:22:38,414 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.io.sort.mb to 859 from MR setting mapreduce.task.io.sort.mb
2016-05-04 15:22:38,414 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.read.timeout to 180000 from MR setting mapreduce.reduce.shuffle.read.timeout
2016-05-04 15:22:38,414 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.ifile.readahead.bytes to 4194304 from MR setting mapreduce.ifile.readahead.bytes
2016-05-04 15:22:38,414 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.ssl.enable to false from MR setting mapreduce.shuffle.ssl.enabled
2016-05-04 15:22:38,414 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.sort.spill.percent to 0.7 from MR setting mapreduce.map.sort.spill.percent
2016-05-04 15:22:38,414 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.ifile.readahead to true from MR setting mapreduce.ifile.readahead
2016-05-04 15:22:38,414 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.merge.percent to 0.66 from MR setting mapreduce.reduce.shuffle.merge.percent
2016-05-04 15:22:38,414 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.parallel.copies to 30 from MR setting mapreduce.reduce.shuffle.parallelcopies
2016-05-04 15:22:38,414 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.memory.limit.percent to 0.25 from MR setting mapreduce.reduce.shuffle.memory.limit.percent
2016-05-04 15:22:38,414 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.io.sort.factor to 100 from MR setting mapreduce.task.io.sort.factor
2016-05-04 15:22:38,414 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.compress to false from MR setting mapreduce.map.output.compress
2016-05-04 15:22:38,414 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.connect.timeout to 180000 from MR setting mapreduce.reduce.shuffle.connect.timeout
2016-05-04 15:22:38,414 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.task.input.post-merge.buffer.percent to 0.0 from MR setting mapreduce.reduce.input.buffer.percent
2016-05-04 15:22:38,414 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.compress.codec to org.apache.hadoop.io.compress.DefaultCodec from MR setting mapreduce.map.output.compress.codec
2016-05-04 15:22:38,414 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.merge.progress.records to 10000 from MR setting mapreduce.task.merge.progress.records
2016-05-04 15:22:38,415 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.internal.sorter.class to org.apache.hadoop.util.QuickSort from MR setting map.sort.class
2016-05-04 15:22:38,415 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.fetch.buffer.percent to 0.7 from MR setting mapreduce.reduce.shuffle.input.buffer.percent
2016-05-04 15:22:38,520 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.io.sort.mb to 859 from MR setting mapreduce.task.io.sort.mb
2016-05-04 15:22:38,520 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.read.timeout to 180000 from MR setting mapreduce.reduce.shuffle.read.timeout
2016-05-04 15:22:38,520 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.ifile.readahead.bytes to 4194304 from MR setting mapreduce.ifile.readahead.bytes
2016-05-04 15:22:38,520 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.ssl.enable to false from MR setting mapreduce.shuffle.ssl.enabled
2016-05-04 15:22:38,520 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.sort.spill.percent to 0.7 from MR setting mapreduce.map.sort.spill.percent
2016-05-04 15:22:38,520 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.ifile.readahead to true from MR setting mapreduce.ifile.readahead
2016-05-04 15:22:38,520 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.merge.percent to 0.66 from MR setting mapreduce.reduce.shuffle.merge.percent
2016-05-04 15:22:38,520 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.parallel.copies to 30 from MR setting mapreduce.reduce.shuffle.parallelcopies
2016-05-04 15:22:38,520 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.memory.limit.percent to 0.25 from MR setting mapreduce.reduce.shuffle.memory.limit.percent
2016-05-04 15:22:38,520 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.io.sort.factor to 100 from MR setting mapreduce.task.io.sort.factor
2016-05-04 15:22:38,520 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.compress to false from MR setting mapreduce.map.output.compress
2016-05-04 15:22:38,520 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.connect.timeout to 180000 from MR setting mapreduce.reduce.shuffle.connect.timeout
2016-05-04 15:22:38,520 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.task.input.post-merge.buffer.percent to 0.0 from MR setting mapreduce.reduce.input.buffer.percent
2016-05-04 15:22:38,520 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.compress.codec to org.apache.hadoop.io.compress.DefaultCodec from MR setting mapreduce.map.output.compress.codec
2016-05-04 15:22:38,520 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.merge.progress.records to 10000 from MR setting mapreduce.task.merge.progress.records
2016-05-04 15:22:38,520 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.internal.sorter.class to org.apache.hadoop.util.QuickSort from MR setting map.sort.class
2016-05-04 15:22:38,520 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.fetch.buffer.percent to 0.7 from MR setting mapreduce.reduce.shuffle.input.buffer.percent
2016-05-04 15:22:38,550 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.TezJobCompiler - For vertex - scope-53: parallelism=1, memory=1536, java opts=-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB
2016-05-04 15:22:38,632 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.io.sort.mb to 859 from MR setting mapreduce.task.io.sort.mb
2016-05-04 15:22:38,632 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.read.timeout to 180000 from MR setting mapreduce.reduce.shuffle.read.timeout
2016-05-04 15:22:38,632 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.ifile.readahead.bytes to 4194304 from MR setting mapreduce.ifile.readahead.bytes
2016-05-04 15:22:38,632 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.ssl.enable to false from MR setting mapreduce.shuffle.ssl.enabled
2016-05-04 15:22:38,632 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.sort.spill.percent to 0.7 from MR setting mapreduce.map.sort.spill.percent
2016-05-04 15:22:38,633 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.ifile.readahead to true from MR setting mapreduce.ifile.readahead
2016-05-04 15:22:38,633 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.merge.percent to 0.66 from MR setting mapreduce.reduce.shuffle.merge.percent
2016-05-04 15:22:38,633 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.parallel.copies to 30 from MR setting mapreduce.reduce.shuffle.parallelcopies
2016-05-04 15:22:38,633 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.memory.limit.percent to 0.25 from MR setting mapreduce.reduce.shuffle.memory.limit.percent
2016-05-04 15:22:38,633 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.io.sort.factor to 100 from MR setting mapreduce.task.io.sort.factor
2016-05-04 15:22:38,633 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.compress to false from MR setting mapreduce.map.output.compress
2016-05-04 15:22:38,633 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.connect.timeout to 180000 from MR setting mapreduce.reduce.shuffle.connect.timeout
2016-05-04 15:22:38,633 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.task.input.post-merge.buffer.percent to 0.0 from MR setting mapreduce.reduce.input.buffer.percent
2016-05-04 15:22:38,633 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.compress.codec to org.apache.hadoop.io.compress.DefaultCodec from MR setting mapreduce.map.output.compress.codec
2016-05-04 15:22:38,633 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.merge.progress.records to 10000 from MR setting mapreduce.task.merge.progress.records
2016-05-04 15:22:38,633 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.internal.sorter.class to org.apache.hadoop.util.QuickSort from MR setting map.sort.class
2016-05-04 15:22:38,633 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.fetch.buffer.percent to 0.7 from MR setting mapreduce.reduce.shuffle.input.buffer.percent
2016-05-04 15:22:38,703 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.TezJobCompiler - Set auto parallelism for vertex scope-54
2016-05-04 15:22:38,703 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.TezJobCompiler - For vertex - scope-54: parallelism=2, memory=2048, java opts=-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB
2016-05-04 15:22:38,880 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.io.sort.mb to 859 from MR setting mapreduce.task.io.sort.mb
2016-05-04 15:22:38,880 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.read.timeout to 180000 from MR setting mapreduce.reduce.shuffle.read.timeout
2016-05-04 15:22:38,880 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.ifile.readahead.bytes to 4194304 from MR setting mapreduce.ifile.readahead.bytes
2016-05-04 15:22:38,880 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.ssl.enable to false from MR setting mapreduce.shuffle.ssl.enabled
2016-05-04 15:22:38,880 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.sort.spill.percent to 0.7 from MR setting mapreduce.map.sort.spill.percent
2016-05-04 15:22:38,880 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.ifile.readahead to true from MR setting mapreduce.ifile.readahead
2016-05-04 15:22:38,880 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.merge.percent to 0.66 from MR setting mapreduce.reduce.shuffle.merge.percent
2016-05-04 15:22:38,880 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.parallel.copies to 30 from MR setting mapreduce.reduce.shuffle.parallelcopies
2016-05-04 15:22:38,880 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.memory.limit.percent to 0.25 from MR setting mapreduce.reduce.shuffle.memory.limit.percent
2016-05-04 15:22:38,880 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.io.sort.factor to 100 from MR setting mapreduce.task.io.sort.factor
2016-05-04 15:22:38,880 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.compress to false from MR setting mapreduce.map.output.compress
2016-05-04 15:22:38,880 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.connect.timeout to 180000 from MR setting mapreduce.reduce.shuffle.connect.timeout
2016-05-04 15:22:38,880 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.task.input.post-merge.buffer.percent to 0.0 from MR setting mapreduce.reduce.input.buffer.percent
2016-05-04 15:22:38,880 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.compress.codec to org.apache.hadoop.io.compress.DefaultCodec from MR setting mapreduce.map.output.compress.codec
2016-05-04 15:22:38,880 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.merge.progress.records to 10000 from MR setting mapreduce.task.merge.progress.records
2016-05-04 15:22:38,880 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.internal.sorter.class to org.apache.hadoop.util.QuickSort from MR setting map.sort.class
2016-05-04 15:22:38,881 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.fetch.buffer.percent to 0.7 from MR setting mapreduce.reduce.shuffle.input.buffer.percent
2016-05-04 15:22:38,990 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.io.sort.mb to 859 from MR setting mapreduce.task.io.sort.mb
2016-05-04 15:22:38,990 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.read.timeout to 180000 from MR setting mapreduce.reduce.shuffle.read.timeout
2016-05-04 15:22:38,990 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.ifile.readahead.bytes to 4194304 from MR setting mapreduce.ifile.readahead.bytes
2016-05-04 15:22:38,990 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.ssl.enable to false from MR setting mapreduce.shuffle.ssl.enabled
2016-05-04 15:22:38,990 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.sort.spill.percent to 0.7 from MR setting mapreduce.map.sort.spill.percent
2016-05-04 15:22:38,990 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.ifile.readahead to true from MR setting mapreduce.ifile.readahead
2016-05-04 15:22:38,991 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.merge.percent to 0.66 from MR setting mapreduce.reduce.shuffle.merge.percent
2016-05-04 15:22:38,991 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.parallel.copies to 30 from MR setting mapreduce.reduce.shuffle.parallelcopies
2016-05-04 15:22:38,991 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.memory.limit.percent to 0.25 from MR setting mapreduce.reduce.shuffle.memory.limit.percent
2016-05-04 15:22:38,991 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.io.sort.factor to 100 from MR setting mapreduce.task.io.sort.factor
2016-05-04 15:22:38,991 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.compress to false from MR setting mapreduce.map.output.compress
2016-05-04 15:22:38,991 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.connect.timeout to 180000 from MR setting mapreduce.reduce.shuffle.connect.timeout
2016-05-04 15:22:38,991 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.task.input.post-merge.buffer.percent to 0.0 from MR setting mapreduce.reduce.input.buffer.percent
2016-05-04 15:22:38,991 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.compress.codec to org.apache.hadoop.io.compress.DefaultCodec from MR setting mapreduce.map.output.compress.codec
2016-05-04 15:22:38,991 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.merge.progress.records to 10000 from MR setting mapreduce.task.merge.progress.records
2016-05-04 15:22:38,991 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.internal.sorter.class to org.apache.hadoop.util.QuickSort from MR setting map.sort.class
2016-05-04 15:22:38,991 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.util.MRToTezHelper - Setting tez.runtime.shuffle.fetch.buffer.percent to 0.7 from MR setting mapreduce.reduce.shuffle.input.buffer.percent
2016-05-04 15:22:39,145 [PigTezLauncher-0] INFO org.apache.pig.tools.pigstats.tez.TezScriptState - Pig script settings are added to the job
2016-05-04 15:22:39,279 [PigTezLauncher-0] INFO org.apache.tez.client.TezClient - Tez Client Version: [ component=tez-api, version=0.7.0.2.4.0.0-169, revision=3c1431f45faaca982ecc8dad13a107787b834696, SCM-URL=scm:git:https://git-wip-us.apache.org/repos/asf/tez.git, buildTime=20160210-0711 ]
2016-05-04 15:22:39,429 [PigTezLauncher-0] INFO org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl - Timeline service address: http://node2.hdp.host:8188/ws/v1/timeline/
2016-05-04 15:22:39,436 [PigTezLauncher-0] INFO org.apache.hadoop.yarn.client.RMProxy - Connecting to ResourceManager at node2.hdp.host/79.143.180.21:8050
2016-05-04 15:22:39,629 [PigTezLauncher-0] INFO org.apache.tez.client.TezClient - Using org.apache.tez.dag.history.ats.acls.ATSV15HistoryACLPolicyManager to manage Timeline ACLs
2016-05-04 15:22:39,788 [PigTezLauncher-0] INFO org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl - Timeline service address: http://node2.hdp.host:8188/ws/v1/timeline/
2016-05-04 15:22:39,810 [PigTezLauncher-0] INFO org.apache.tez.client.TezClient - Session mode. Starting session.
2016-05-04 15:22:39,818 [PigTezLauncher-0] INFO org.apache.tez.client.TezClientUtils - Using tez.lib.uris value from configuration: /hdp/apps/2.4.0.0-169/tez/tez.tar.gz
2016-05-04 15:22:40,105 [PigTezLauncher-0] INFO org.apache.tez.client.TezClient - Tez system stage directory hdfs://node1.hdp.host:8020/tmp/root/staging/.tez/application_1462345329592_0014 doesn't exist and is created
2016-05-04 15:22:40,207 [PigTezLauncher-0] INFO org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl - Writing domains for appattempt_1462345329592_0014_000001 to /ats/active/application_1462345329592_0014/appattempt_1462345329592_0014_000001/domainlog-appattempt_1462345329592_0014_000001
2016-05-04 15:22:40,269 [PigTezLauncher-0] INFO org.apache.tez.dag.history.ats.acls.ATSV15HistoryACLPolicyManager - Created Timeline Domain for History ACLs, domainId=Tez_ATS_application_1462345329592_0014
2016-05-04 15:22:40,963 [PigTezLauncher-0] INFO org.apache.hadoop.yarn.client.api.impl.YarnClientImpl - Submitted application application_1462345329592_0014
2016-05-04 15:22:40,965 [PigTezLauncher-0] INFO org.apache.tez.client.TezClient - The url to track the Tez Session: http://node2.hdp.host:8088/proxy/application_1462345329592_0014/
2016-05-04 15:22:47,780 [PigTezLauncher-0] INFO org.apache.pig.backend.hadoop.executionengine.tez.TezJob - Submitting DAG PigLatin:DefaultJobName-0_scope-0
2016-05-04 15:22:47,780 [PigTezLauncher-0] INFO org.apache.tez.client.TezClient - Submitting dag to TezSession, sessionName=PigLatin:DefaultJobName, applicationId=application_1462345329592_0014, dagName=PigLatin:DefaultJobName-0_scope-0, callerContext={ context=PIG, callerType=PIG_SCRIPT_ID, callerId=PIG-default-bc994238-ce7e-4cdb-9a72-096b55822402 }
2016-05-04 15:22:49,449 [PigTezLauncher-0] INFO org.apache.tez.client.TezClient - Submitted dag to TezSession, sessionName=PigLatin:DefaultJobName, applicationId=application_1462345329592_0014, dagName=PigLatin:DefaultJobName-0_scope-0
2016-05-04 15:22:49,820 [PigTezLauncher-0] INFO org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl - Timeline service address: http://node2.hdp.host:8188/ws/v1/timeline/
2016-05-04 15:22:49,821 [PigTezLauncher-0] INFO org.apache.hadoop.yarn.client.RMProxy - Connecting to ResourceManager at node2.hdp.host/79.143.180.21:8050
2016-05-04 15:22:49,860 [PigTezLauncher-0] INFO org.apache.pig.backend.hadoop.executionengine.tez.TezJob - Submitted DAG PigLatin:DefaultJobName-0_scope-0. Application id: application_1462345329592_0014
2016-05-04 15:22:50,124 [main] INFO org.apache.pig.backend.hadoop.executionengine.tez.TezLauncher - HadoopJobId: job_1462345329592_0014
2016-05-04 15:23:07,101 [PigTezLauncher-0] INFO org.apache.tez.common.counters.Limits - Counter limits initialized with parameters: GROUP_NAME_MAX=256, MAX_GROUPS=3000, COUNTER_NAME_MAX=64, MAX_COUNTERS=10000
2016-05-04 15:23:07,117 [PigTezLauncher-0] INFO org.apache.pig.backend.hadoop.executionengine.tez.TezJob - DAG Status: status=SUCCEEDED, progress=TotalTasks: 4 Succeeded: 4 Running: 0 Failed: 0 Killed: 0, diagnostics=, counters=Counters: 171
org.apache.tez.common.counters.DAGCounter
NUM_SUCCEEDED_TASKS=4
TOTAL_LAUNCHED_TASKS=4
DATA_LOCAL_TASKS=2
AM_CPU_MILLISECONDS=5220
AM_GC_TIME_MILLIS=147
File System Counters
FILE_BYTES_READ=3397
FILE_BYTES_WRITTEN=2335
FILE_READ_OPS=0
FILE_LARGE_READ_OPS=0
FILE_WRITE_OPS=0
HDFS_BYTES_READ=25953
HDFS_BYTES_WRITTEN=1467
HDFS_READ_OPS=9
HDFS_LARGE_READ_OPS=0
HDFS_WRITE_OPS=2
org.apache.tez.common.counters.TaskCounter
REDUCE_INPUT_GROUPS=298
REDUCE_INPUT_RECORDS=298
COMBINE_INPUT_RECORDS=0
SPILLED_RECORDS=596
NUM_SHUFFLED_INPUTS=5
NUM_SKIPPED_INPUTS=0
NUM_FAILED_SHUFFLE_INPUTS=0
MERGED_MAP_OUTPUTS=5
GC_TIME_MILLIS=676
CPU_MILLISECONDS=13440
PHYSICAL_MEMORY_BYTES=2239758336
VIRTUAL_MEMORY_BYTES=14433169408
COMMITTED_HEAP_BYTES=2239758336
INPUT_RECORDS_PROCESSED=8100
OUTPUT_RECORDS=755
OUTPUT_BYTES=7458
OUTPUT_BYTES_WITH_OVERHEAD=4265
OUTPUT_BYTES_PHYSICAL=2191
ADDITIONAL_SPILLS_BYTES_WRITTEN=0
ADDITIONAL_SPILLS_BYTES_READ=2191
ADDITIONAL_SPILL_COUNT=0
SHUFFLE_CHUNK_COUNT=3
SHUFFLE_BYTES=2191
SHUFFLE_BYTES_DECOMPRESSED=4265
SHUFFLE_BYTES_TO_MEM=0
SHUFFLE_BYTES_TO_DISK=0
SHUFFLE_BYTES_DISK_DIRECT=2191
NUM_MEM_TO_DISK_MERGES=0
NUM_DISK_TO_DISK_MERGES=0
SHUFFLE_PHASE_TIME=210
MERGE_PHASE_TIME=341
FIRST_EVENT_RECEIVED=174
LAST_EVENT_RECEIVED=181
Shuffle Errors
BAD_ID=0
CONNECTION=0
IO_ERROR=0
WRONG_LENGTH=0
WRONG_MAP=0
WRONG_REDUCE=0
Shuffle Errors_scope_52_INPUT_scope_51
BAD_ID=0
CONNECTION=0
IO_ERROR=0
WRONG_LENGTH=0
WRONG_MAP=0
WRONG_REDUCE=0
Shuffle Errors_scope_54_INPUT_scope_52
BAD_ID=0
CONNECTION=0
IO_ERROR=0
WRONG_LENGTH=0
WRONG_MAP=0
WRONG_REDUCE=0
Shuffle Errors_scope_54_INPUT_scope_53
BAD_ID=0
CONNECTION=0
IO_ERROR=0
WRONG_LENGTH=0
WRONG_MAP=0
WRONG_REDUCE=0
TaskCounter_scope_51_INPUT_scope_2
INPUT_RECORDS_PROCESSED=8000
TaskCounter_scope_51_OUTPUT_scope_52
ADDITIONAL_SPILLS_BYTES_READ=0
ADDITIONAL_SPILLS_BYTES_WRITTEN=0
ADDITIONAL_SPILL_COUNT=0
OUTPUT_BYTES=4992
OUTPUT_BYTES_PHYSICAL=605
OUTPUT_BYTES_WITH_OVERHEAD=1377
OUTPUT_RECORDS=457
SHUFFLE_CHUNK_COUNT=1
SPILLED_RECORDS=99
TaskCounter_scope_52_INPUT_scope_51
ADDITIONAL_SPILLS_BYTES_READ=605
ADDITIONAL_SPILLS_BYTES_WRITTEN=0
COMBINE_INPUT_RECORDS=0
FIRST_EVENT_RECEIVED=82
LAST_EVENT_RECEIVED=82
MERGED_MAP_OUTPUTS=1
MERGE_PHASE_TIME=172
NUM_DISK_TO_DISK_MERGES=0
NUM_FAILED_SHUFFLE_INPUTS=0
NUM_MEM_TO_DISK_MERGES=0
NUM_SHUFFLED_INPUTS=1
NUM_SKIPPED_INPUTS=0
REDUCE_INPUT_GROUPS=99
REDUCE_INPUT_RECORDS=99
SHUFFLE_BYTES=605
SHUFFLE_BYTES_DECOMPRESSED=1377
SHUFFLE_BYTES_DISK_DIRECT=605
SHUFFLE_BYTES_TO_DISK=0
SHUFFLE_BYTES_TO_MEM=0
SHUFFLE_PHASE_TIME=94
SPILLED_RECORDS=99
TaskCounter_scope_52_OUTPUT_scope_54
ADDITIONAL_SPILLS_BYTES_READ=0
ADDITIONAL_SPILLS_BYTES_WRITTEN=0
ADDITIONAL_SPILL_COUNT=0
OUTPUT_BYTES=1074
OUTPUT_BYTES_PHYSICAL=683
OUTPUT_BYTES_WITH_OVERHEAD=1284
OUTPUT_RECORDS=99
SHUFFLE_CHUNK_COUNT=1
SPILLED_RECORDS=99
TaskCounter_scope_53_INPUT_scope_24
INPUT_RECORDS_PROCESSED=100
TaskCounter_scope_53_OUTPUT_scope_54
ADDITIONAL_SPILLS_BYTES_READ=0
ADDITIONAL_SPILLS_BYTES_WRITTEN=0
ADDITIONAL_SPILL_COUNT=0
OUTPUT_BYTES=1392
OUTPUT_BYTES_PHYSICAL=903
OUTPUT_BYTES_WITH_OVERHEAD=1604
OUTPUT_RECORDS=100
SHUFFLE_CHUNK_COUNT=1
SPILLED_RECORDS=100
TaskCounter_scope_54_INPUT_scope_52
ADDITIONAL_SPILLS_BYTES_READ=683
ADDITIONAL_SPILLS_BYTES_WRITTEN=0
COMBINE_INPUT_RECORDS=0
FIRST_EVENT_RECEIVED=62
LAST_EVENT_RECEIVED=69
MERGED_MAP_OUTPUTS=2
MERGE_PHASE_TIME=89
NUM_DISK_TO_DISK_MERGES=0
NUM_FAILED_SHUFFLE_INPUTS=0
NUM_MEM_TO_DISK_MERGES=0
NUM_SHUFFLED_INPUTS=2
NUM_SKIPPED_INPUTS=0
REDUCE_INPUT_GROUPS=99
REDUCE_INPUT_RECORDS=99
SHUFFLE_BYTES=683
SHUFFLE_BYTES_DECOMPRESSED=1284
SHUFFLE_BYTES_DISK_DIRECT=683
SHUFFLE_BYTES_TO_DISK=0
SHUFFLE_BYTES_TO_MEM=0
SHUFFLE_PHASE_TIME=72
SPILLED_RECORDS=99
TaskCounter_scope_54_INPUT_scope_53
ADDITIONAL_SPILLS_BYTES_READ=903
ADDITIONAL_SPILLS_BYTES_WRITTEN=0
COMBINE_INPUT_RECORDS=0
FIRST_EVENT_RECEIVED=30
LAST_EVENT_RECEIVED=30
MERGED_MAP_OUTPUTS=2
MERGE_PHASE_TIME=80
NUM_DISK_TO_DISK_MERGES=0
NUM_FAILED_SHUFFLE_INPUTS=0
NUM_MEM_TO_DISK_MERGES=0
NUM_SHUFFLED_INPUTS=2
NUM_SKIPPED_INPUTS=0
REDUCE_INPUT_GROUPS=100
REDUCE_INPUT_RECORDS=100
SHUFFLE_BYTES=903
SHUFFLE_BYTES_DECOMPRESSED=1604
SHUFFLE_BYTES_DISK_DIRECT=903
SHUFFLE_BYTES_TO_DISK=0
SHUFFLE_BYTES_TO_MEM=0
SHUFFLE_PHASE_TIME=44
SPILLED_RECORDS=100
TaskCounter_scope_54_OUTPUT_scope_50
OUTPUT_RECORDS=99
org.apache.hadoop.mapreduce.TaskCounter
COMBINE_INPUT_RECORDS=99
COMBINE_OUTPUT_RECORDS=457
org.apache.hadoop.mapreduce.TaskCounter_scope_51_OUTPUT_scope_52
COMBINE_INPUT_RECORDS=99
COMBINE_OUTPUT_RECORDS=457
org.apache.hadoop.mapreduce.TaskCounter_scope_52_INPUT_scope_51
COMBINE_INPUT_RECORDS=0
COMBINE_OUTPUT_RECORDS=0
2016-05-04 15:23:08,154 [main] INFO org.apache.pig.tools.pigstats.tez.TezPigScriptStats - Script Statistics:
HadoopVersion: 2.7.1.2.4.0.0-169
PigVersion: 0.15.0.2.4.0.0-169
TezVersion: 0.7.0.2.4.0.0-169
UserId: root
FileName:
StartedAt: 2016-05-04 15:22:29
FinishedAt: 2016-05-04 15:23:08
Features: HASH_JOIN,GROUP_BY,FILTER
Success!
DAG PigLatin:DefaultJobName-0_scope-0:
ApplicationId: job_1462345329592_0014
TotalLaunchedTasks: 4
FileBytesRead: 3397
FileBytesWritten: 2335
HdfsBytesRead: 25953
HdfsBytesWritten: 1467
Input(s):
Successfully read 100 records (1301 bytes) from: "drivermileage"
Successfully read 8000 records (24652 bytes) from: "geolocation"
Output(s):
Successfully stored 99 records (1467 bytes) in: "riskfactor"