#计划

##收集hive的日志，存到hdfs

/opt/cdh-5.3.6/hive-0.13.1-cdh5.3.6/logs/hive.log

命令：tail -f    

source：Exec source 在启动时运行一个给定的Unix命令，并期望该进程在标准输出上不断地生成数据

channel：memory

sink：hdfs    #/user/root/flume/hive-logs/

##准备agent配置文件：flume-tail.conf 如下：

# The configuration file needs to define the sources,

# the channels and the sinks.

####define agent

a2.sources = r2

a2.channels = c2

a2.sinks = k2

###define sources

a2.sources.r2.type = exec

a2.sources.r2.command = tail -f /opt/cdh-5.3.6/hive-0.13.1-cdh5.3.6/logs/hive.log

a2.sources.r2.shell = /bin/bash -c

###define channel

a2.channels.c2.type = memory

a2.channels.c2.capacity = 1000

a2.channels.c2.transactionCapacity = 100

###define sink

a2.sinks.k2.type = hdfs

a2.sinks.k2.hdfs.path = hdfs://hadoop-senior.ibeifeng.com:8020/user/root/flume/hive-logs/

a2.sinks.k2.hdfs.fileType = DataStream

a2.sinks.k2.hdfs.writeFormat = Text

a2.sinks.k2.hdfs.batchSize = 10

###bind the soures and sink to the channel

a2.sources.r2.channels = c2

a2.sinks.k2.channel = c2

##flume开始实时监听抽取

[root@hadoop-senior flume-1.5.0-cdh5.3.6]# bin/flume-ng agent -c conf -n a2 -f conf/flume-tail.conf -Dflume.root.logger=DEBUG,console

##此时可以去hive中执行一些命令，产生日志

##hdfs查看，已经抽取到很多文件了

[root@hadoop-senior hadoop-2.5.0-cdh5.3.6]# bin/hdfs dfs -ls -R /user/root/flume/hive-logs/

-rw-r--r--   3 root supergroup       1133 2019-05-08 13:43 /user/root/flume/hive-logs/FlumeData.1557294191838

-rw-r--r--   3 root supergroup        534 2019-05-08 13:43 /user/root/flume/hive-logs/FlumeData.1557294191839

-rw-r--r--   3 root supergroup       1056 2019-05-08 13:59 /user/root/flume/hive-logs/FlumeData.1557295160087

-rw-r--r--   3 root supergroup        408 2019-05-08 13:59 /user/root/flume/hive-logs/FlumeData.1557295160088

-rw-r--r--   3 root supergroup       1319 2019-05-08 13:59 /user/root/flume/hive-logs/FlumeData.1557295160089

-rw-r--r--   3 root supergroup        240 2019-05-08 13:59 /user/root/flume/hive-logs/FlumeData.1557295160090

-rw-r--r--   3 root supergroup       1083 2019-05-08 13:59 /user/root/flume/hive-logs/FlumeData.1557295160091

-rw-r--r--   3 root supergroup        255 2019-05-08 13:59 /user/root/flume/hive-logs/FlumeData.1557295160092

-rw-r--r--   3 root supergroup        122 2019-05-08 13:59 /user/root/flume/hive-logs/FlumeData.1557295160093

-rw-r--r--   3 root supergroup        956 2019-05-08 13:59 /user/root/flume/hive-logs/FlumeData.1557295160094

-rw-r--r--   3 root supergroup        515 2019-05-08 13:59 /user/root/flume/hive-logs/FlumeData.1557295160095.tmp

##当hdfs的架构为HA时，define sink字段定义：

a2.sinks.k2.type = hdfs

a2.sinks.k2.hdfs.path = hdfs://代理名:8020/user/root/flume/hive-logs/

把主机名改为HA的代理名称，然后可以直接把core-site.xml   hdfs-site.xml放进flume的conf目录中，让flume读取；