Flume实现日志文件夹数据加载到HDFS

Flume是一种分布式，可靠和可用的服务，用于高效收集，聚合和移动大量日志数据。 它具有基于数据流的简单和可伸缩的架构。 它具有可靠性机制和故障切换和恢复机制的鲁棒性和容错能力。

vi corp_base_info.conf

a1.sources = r1

a1.sinks = k1

a1.channels = c1

# Describe/configure the source

a1.sources.r1.type = spooldir

a1.sources.r1.spoolDir=/home/flume/testdata/test

a1.sources.r1.includePattern=^AUEIC.C_CONS([0-9a-zA-Z]|[._-])*$

a1.sources.r1.ignorePattern=^.*COMPLETED$

a1.sources.r1.inputCharset=UTF-8

a1.sources.r1.pollDelay=300000 #5分针采集一次

加粗的属性1.7以上才有

#Use a channel which buffers events in memory

a1.channels=c1

a1.channels.c1.capacity=1000000

a1.channels.c1.transactionCapacity=1000000

a1.channels.c1.type=memory

#Describe the sink

a1.sinks=k1

a1.sinks.k1.channel=c1

a1.sinks.k1.hdfs.fileType=DataStream

a1.sinks.k1.hdfs.path=hdfs://mynameservice/apps/hive/warehouse/flume.db/corp_base_info/ymd=%Y%m%d

a1.sinks.k1.hdfs.rollCount=0

a1.sinks.k1.hdfs.rollInterval=0

a1.sinks.k1.hdfs.rollSize=10240000

a1.sinks.k1.hdfs.idleTimeout=60

a1.sinks.k1.hdfs.writeFormat=Text

a1.sinks.k1.type=hdfs

# Bind the source and sink to the channel

a1.sources.r1.channels = c1

a1.sinks.k1.channel = c1