Support Questions
Find answers, ask questions, and share your expertise

flume run is ok, But why i cann't any Metric Detais about flume ?, how to config it ? I use CDH5.3.0


flume run is ok, But why i cann't any Metric Detais about flume ?, how to config it ? I use CDH5.3.0

New Contributor
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# The configuration file needs to define the sources, 
# the channels and the sinks.
# Sources, channels and sinks are defined per collector1, 
# in this case called 'collector1'

collector1.sources = src_avro_c1 
collector1.channels = ch_file ch_mem 
collector1.sinks = sink_hdfs sink_spark 

# For each one of the sources, the type is defined
collector1.sources.src_avro_c1.type = avro 
collector1.sources.src_avro_c1.port= 41414

# The channel can be defined as follows.
collector1.sources.src_avro_c1.channels = ch_file ch_mem 
collector1.sources.src_avro_c1.selector.type = replicating 

# Each sink's type must be defined
collector1.sinks.sink_hdfs.type = hdfs
#collector1.sinks.sink_hdfs.type = logger
collector1.sinks.sink_hdfs.hdfs.path = hdfs://XXXXXXXXX:8020/staging/%{}/%Y-%m-%d
collector1.sinks.sink_hdfs.hdfs.inUsePrefix = .  
collector1.sinks.sink_hdfs.hdfs.maxOpenFiles = 5000   
collector1.sinks.sink_hdfs.hdfs.batchSize= 1000  
collector1.sinks.sink_hdfs.hdfs.fileType = DataStream  
collector1.sinks.sink_hdfs.hdfs.writeFormat =Text  
collector1.sinks.sink_hdfs.hdfs.rollSize = 2048000000  
collector1.sinks.sink_hdfs.hdfs.rollCount = 0  
collector1.sinks.sink_hdfs.hdfs.rollInterval = 3600  
collector1.sinks.sink_hdfs.hdfs.round = true  
collector1.sinks.sink_hdfs.hdfs.roundValue = 1 
collector1.sinks.sink_hdfs.hdfs.roundUnit = hour  
collector1.sinks.sink_hdfs.hdfs.useLocalTimeStamp = true 
#Specify the channel the sink should use = ch_file 

# sink_spark
collector1.sinks.sink_spark.type = org.apache.spark.streaming.flume.sink.SparkSink
collector1.sinks.sink_spark.hostname = hadoop-2
collector1.sinks.sink_spark.port = 50001 ch_mem 

# Each channel's type is defined.
collector1.channels.ch_file.type = file 
collector1.channels.ch_file.checkpointDIR = /data/01flume/checkpoint 
collector1.channels.ch_file.dataDirs = /data01/flume/data

# Other config values specific to each type of channel(sink or source)
# can be defined as well
# In this case, it specifies the capacity of the memory channel
collector1.channels.ch_mem.type= memory
collector1.channels.ch_mem.capacity = 1000000
collector1.channels.ch_mem.transactionCapacity  = 1000000