nginx+flume+sqoop

nginx跟踪日志，flume将日志采集到hdfs，通过sqoop将数据放到mysql中
vi /usr/local/nginx/conf/nginx.conf

#user  nobody;
worker_processes  1;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;

events {
worker_connections  1024;
}

http {
include       mime.types;
default_type  application/octet-stream;

log_format  main  '$remote_addr' '$remote_user' '$time_local' '$request' '$status';

#access_log  logs/access.log  main;

sendfile        on;
#tcp_nopush     on;

#keepalive_timeout  0;
keepalive_timeout  65;

#gzip  on;
upstream frame-tomcat {
server hdp-1:8989 ;
}
server {
listen       80;
server_name  hdp-0;

#charset koi8-r;
access_log  logs/log.BiSheThree.access.log  main;
#access_log  logs/log.frame.access.log  main;

location / {
# root   html;
# index  index.html index.htm;
proxy_pass http://frame-tomcat;
}

error_page   500 502 503 504  /50x.html;
location = /50x.html {
root   html;
}
}
server {
listen       80;
server_name  localhost;

#charset koi8-r;

#access_log  logs/host.access.log  main;

location / {
root   html;
index  index.html index.htm;
}

#error_page  404              /404.html;

# redirect server error pages to the static page /50x.html
#
error_page   500 502 503 504  /50x.html;
location = /50x.html {
root   html;
}

# proxy the PHP scripts to Apache listening on 127.0.0.1:80
#
#location ~ \.php$ {
#    proxy_pass   http://127.0.0.1;
#}

# pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
#
#location ~ \.php$ {
#    root           html;
#    fastcgi_pass   127.0.0.1:9000;
#    fastcgi_index  index.php;
#    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
#    include        fastcgi_params;
#}

# deny access to .htaccess files, if Apache's document root
# concurs with nginx's one
#
#location ~ /\.ht {
#    deny  all;
#}
}

# another virtual host using mix of IP-, name-, and port-based configuration
#
#server {
#    listen       8000;
#    listen       somename:8080;
#    server_name  somename  alias  another.alias;

#    location / {
#        root   html;
#        index  index.html index.htm;
#    }
#}

# HTTPS server
#
#server {
#    listen       443;
#    server_name  localhost;

#    ssl                  on;
#    ssl_certificate      cert.pem;
#    ssl_certificate_key  cert.key;

#    ssl_session_timeout  5m;

#    ssl_protocols  SSLv2 SSLv3 TLSv1;
#    ssl_ciphers  HIGH:!aNULL:!MD5;
#    ssl_prefer_server_ciphers   on;

#    location / {
#        root   html;
#        index  index.html index.htm;
#    }
#}

}

$remote_addr （ip地址）
$remote_user （用户名称）
$time_local （访问时间）
$request （请求的url与http协议）
$status （请求状态）
$body_bytes_sent （客户端文件主体内容大小）
$http_referer （页面链接）
$http_user_agent （客户端浏览器的相关信息）

./nginx -s reload
flume:
vi bishe.conf

ag1.sources = source1
ag1.sinks = sink1
ag1.channels = channel1

ag1.sources.source1.type = exec
ag1.sources.source1.command = tail -F /usr/local/nginx/logs/log.BiSheThree.access.log

ag1.sinks.sink1.type = hdfs
ag1.sinks.sink1.hdfs.path =hdfs://hdp-0:9000/bishe/%y-%m-%d/%H-%M
ag1.sinks.sink1.hdfs.filePrefix = app_log
ag1.sinks.sink1.hdfs.fileSuffix = .log
ag1.sinks.sink1.hdfs.batchSize= 100
ag1.sinks.sink1.hdfs.fileType = DataStream
ag1.sinks.sink1.hdfs.writeFormat = Text

ag1.sinks.sink1.hdfs.rollSize = 512000
ag1.sinks.sink1.hdfs.rollCount = 1000000
ag1.sinks.sink1.hdfs.rollInterval = 60
ag1.sinks.sink1.hdfs.round = true
ag1.sinks.sink1.hdfs.roundValue = 10
ag1.sinks.sink1.hdfs.roundUnit = minute
ag1.sinks.sink1.hdfs.useLocalTimeStamp = true

ag1.channels.channel1.type = memory
ag1.channels.channel1.capacity = 500000
ag1.channels.channel1.transactionCapacity = 600

ag1.sources.source1.channels = channel1
ag1.sinks.sink1.channel = channel1

bin目录下：
./flume-ng agent -C …/conf/ -f …/dir-hdfs.conf -n ag1 -Dflume.root.logger=INFO.console

sqoop：
bin目录下：

sqoop export  \
--connect jdbc:mysql://hdp-0:3306/bishe \
--username root \
--password 1234 \
--table pv \
--export-dir /bishe/19-12-03/16-30/app_log.1575362336260.log \
--input-fields-terminated-by ' '

• 点赞
• 收藏
• 分享
• 20000 文章举报