flume的几个经典配置案例

一、source是spooldir、channel是memory，sink是hbase

a1.sources = r1

a1.sinks = hbaseSink

a1.channels = memChannel

##########the type of channel is
memory#########

a1.channels.memChannel.type = memory

a1.channels.memChannel.capacity = 10000

a1.channels.memChannel.transactionCapacity = 1000

a1.channels.memChannel.byteCapacityBufferPercentage = 20

a1.channles.memChannel.byteCapacity = 80000

##########the type of
channel is memory#########

##########the type of sink is hbase#########

a1.sinks.hbaseSink.type = hbase

a1.sinks.hbaseSink.table=mytable

a1.sinks.hbaseSink.columnFamily=cf

a1.sinks.hbaseSink.serializer=org.apache.flume.sink.hbase.RegexHbaseEventSerializer

a1.sinks.hbaseSink.zookeeperQuorum=flumekafka1，flumekafka2，flumekafka3

##########the type of sink is hbase#########

a1.sources.r1.channels =
memChannel

a1.sinks.k1.channel = memChannel

二、source是spooldir、channel是memory，sink是hdfs

a1.sources = r1

a1.sinks = hdfsSink

a1.channels = memChannel

##########the type of channel is memory#########

a1.channels.memChannel.type = memory

a1.channels.memChannel.capacity = 10000

a1.channels.memChannel.transactionCapacity = 1000

a1.channels.memChannel.byteCapacityBufferPercentage = 20

a1.channles.memChannel.byteCapacity = 80000

##########the type of channel is memory#########

##########the type of sink is hdfs#########

a1.sinks.hdfsSink.type=hdfs

a1.sinks.hdfsSink.hdfs.path=hdfs://master:9000/flume1

a1.sinks.hdfsSink.hdfs.filePrefix=lwq

a1.sinks.sink1.hdfs.round = true

a1.sinks.hdfsSink.hdfs.rollInterval = 36

a1.sinks.hdfsSink.hdfs.rollSize = 128000000

a1.sinks.hdfsSink.hdfs.rollCount = 0

a1.sinks.hdfsSink.hdfs.batchSize = 1000

a1.sinks.hdfsSink.hdfs.roundValue = 1

a1.sinks.hdfsSink.hdfs.roundUnit = minute

a1.sinks.hdfsSink.hdfs.useLocalTimeStamp = true

a1.sinks.hdfsSink.hdfs.fileType = DataStream

a1.sinks.hdfsSink.hdfs.writeFormat = Text

##########the type of sink is hdfs#########

a1.sources.r1.channels =
memChannel

a1.sinks.k1.channel = memChannel

三、source是spooldir、channel是memory，sink是kafka

a1.sources = r1
a1.sinks = kafkaSink

a1.channels =
memChannel

##########the type of channel is memory#########

a1.channels.memChannel.type = memory

a1.channels.memChannel.capacity = 10000

a1.channels.memChannel.transactionCapacity = 1000

a1.channels.memChannel.byteCapacityBufferPercentage = 20

a1.channles.memChannel.byteCapacity = 80000

##########the type of channel is memory#########

##########the type of sink is kafka#########

a1.sinks.kafkaSink.type = org.apache.flume.sink.kafka.KafkaSink

a1.sinks.kafkaSink.topic=flume1

a1.sinks.kafkaSink.brokerList=flumekafka1:9092,flumekafka2:9092,flumekafka3:9092

a1.sinks.kafkaSink.requiredAcks=1

a1.sinks.kafkaSink.batchSize = 20

##########the type of sink is kafka#########

a1.sources.r1.channels =
memChannel

a1.sinks.kafkaSink.channel =
memChannel

四、source是spooldir、channel是memory，sink是kafka和hdfs

a1.sources = r1

a1.sinks = kafkaSink hdfsSink

a1.channels = kafkaChannel hdfsChannel

a1.sources.r1.type = spooldir

a1.sources.r1.spoolDir = /home/bms/flumeSpool

a1.sources.r1.fileHeader = true

a1.sources.r1.deletePolicy = immediate

##########the type of channel is kafka#########

a1.channels.kafkaChannel .type = memory

a1.channels.kafkaChannel .capacity = 10000

a1.channels.kafkaChannel .transactionCapacity = 1000

a1.channels.kafkaChannel .byteCapacityBufferPercentage = 20

a1.channles.kafkaChannel .byteCapacity = 80000

##########the type of channel is kafka#########

##########the type of channel is hdfs#########

a1.channels.hdfsChannel.type = memory

a1.channels.hdfsChannel.capacity = 10000

a1.channels.hdfsChannel.transactionCapacity = 1000

a1.channels.hdfsChannel.byteCapacityBufferPercentage = 20

a1.channles.hdfsChannel.byteCapacity = 80000

##########the type of channel is
hdfs#########

##########the type of sink is kafka#########

a1.sinks.kafkaSink .type = org.apache.flume.sink.kafka.KafkaSink

a1.sinks.kafkaSink .topic=flume1

a1.sinks.kafkaSink .brokerList=flumekafka1:9092,flumekafka2:9092,flumekafka3:9092

a1.sinks.kafkaSink .requiredAcks=1

a1.sinks.kafkaSink .batchSize = 20

##########the type of sink is kafka#########

##########the type of sink is kafka#########

a1.sinks.hdfsSink.type=hdfs

a1.sinks.hdfsSink.hdfs.path=hdfs://master:9000/flume1

a1.sinks.hdfsSink.hdfs.filePrefix=lwq

a1.sinks.sink1.hdfs.round = true

a1.sinks.hdfsSink.hdfs.rollInterval = 36

a1.sinks.hdfsSink.hdfs.rollSize = 128000000

a1.sinks.hdfsSink.hdfs.rollCount = 0

a1.sinks.hdfsSink.hdfs.batchSize = 1000

a1.sinks.hdfsSink.hdfs.roundValue = 1

a1.sinks.hdfsSink.hdfs.roundUnit = minute

a1.sinks.hdfsSink.hdfs.useLocalTimeStamp = true

a1.sinks.hdfsSink.hdfs.fileType = DataStream

a1.sinks.hdfsSink.hdfs.writeFormat = Text

##########the type of sink is hdfs#########

a1.sources.r1.channels =
kafkaChannel hdfsChannel

a1.sinks.kafkaSink .channel =
kafkaChannel

a1.sinks.hdfsSink.channel = hdfsChannel

a1.sources.r1.selector.type = replicating