sqoop+hive+shell+oozie 示例
2015-12-11 15:55
645 查看
示例
create.q
job.properties
procedure.q
script.sh
workflow.xml
自己写的
workflowhue.xml
hue配置的
create.q
create external table IF NOT EXISTS default.SYS_ACCOUNT ( account_id double, account_name string, account_password string, status int, last_login_device_id string, band_device_id string, key string, last_login_ip string, account_type int, lock_time DATE, account_password_ver string ) partitioned by (date_time string) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' location '/data/hive/sys_account';
job.properties
nameNode=hdfs://fuze250:8020 jobTracker=fuze250:8032 #nameNode=hdfs://nameservice #jobTracker=yarnRM queueName=default oozieRoot=oozie hiveRoot=hive oozie.use.system.libpath=true oozie.libpath=/user/oozie/share/lib oozie.wf.application.path=${nameNode}/user/${user.name}/${oozieRoot}/sqoop/sys_account_user_vip
procedure.q
ALTER TABLE default.SYS_ACCOUNT ADD PARTITION (date_time=${date_time}) location '/data/hive/sys_account/date_time=${date_time}';
script.sh
#!/bin/bash DATE=$(date +%Y%m%d); echo date_time=$DATE
workflow.xml
自己写的
<?xml version="1.0" encoding="UTF-8"?> <!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. --> <workflow-app name="sys_account" xmlns="uri:oozie:workflow:0.5"> <start to="sys_account_create" /> <action name="sys_account_create" cred="hive2"> <hive2 xmlns="uri:oozie:hive2-action:0.1"> <job-tracker>${jobTracker}</job-tracker> <name-node>${nameNode}</name-node> <jdbc-url>jdbc:hive2://fuze248:10000/default</jdbc-url> <configuration> <property> <name>mapred.job.queue.name</name> <value>${queueName}</value> </property> </configuration> <script>create.q</script> </hive2> <ok to="shell-date" /> <error to="fail" /> </action> <action name="shell-date"> <shell xmlns="uri:oozie:shell-action:0.2"> <job-tracker>${jobTracker}</job-tracker> <name-node>${nameNode}</name-node> <configuration> <property> <name>mapred.job.queue.name</name> <value>${queueName}</value> </property> </configuration> <exec>script.sh</exec> <file>script.sh</file> <capture-output /> </shell> <ok to="sys_account_import" /> <error to="fail" /> </action> <action name="sys_account_import"> <sqoop xmlns="uri:oozie:sqoop-action:0.2"> <job-tracker>${jobTracker}</job-tracker> <name-node>${nameNode}</name-node> <configuration> <property> <name>mapred.job.queue.name</name> <value>${queueName}</value> </property> </configuration> <arg>import</arg> <arg>--connect</arg> <arg>jdbc:oracle:thin:@xxx:1521:xxx</arg> <arg>--username</arg> <arg>FUZEPASSPORT</arg> <arg>--SYS_ACCOUNT</arg> <arg>FUZEPASSPORT_PWD</arg> <arg>--query</arg> <arg>select SA.* from SYS_ACCOUNT SA where $CONDITIONS</arg> <arg>--delete-target-dir</arg> <arg>--target-dir</arg> <arg>/data/hive/sys_account/date_time=${(wf:actionData('shell-date')['date_time'])} </arg> <arg>--fields-terminated-by</arg> <arg>\001</arg> <arg>-m</arg> <arg>1</arg> </sqoop> <ok to="sys_account_procedure" /> <error to="fail" /> </action> <action name="sys_account_procedure" cred="hive2"> <hive2 xmlns="uri:oozie:hive2-action:0.1"> <job-tracker>${jobTracker}</job-tracker> <name-node>${nameNode}</name-node> <jdbc-url>jdbc:hive2://fuze248:10000/default</jdbc-url> <configuration> <property> <name>mapred.job.queue.name</name> <value>${queueName}</value> </property> </configuration> <script>procedure.q</script> <param>date_time=${(wf:actionData('shell-date')['date_time'])} </param> </hive2> <ok to="end" /> <error to="fail" /> </action> <kill name="fail"> <message>Sqoop failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message> </kill> <end name="end" /> </workflow-app>
workflowhue.xml
hue配置的
<workflow-app name="sys_account" xmlns="uri:oozie:workflow:0.5"> <start to="hive2-13ef"/> <kill name="Kill"> <message>操作失败,错误消息[${wf:errorMessage(wf:lastErrorNode())}]</message> </kill> <action name="hive2-13ef" cred="hive2"> <hive2 xmlns="uri:oozie:hive2-action:0.1"> <job-tracker>${jobTracker}</job-tracker> <name-node>${nameNode}</name-node> <jdbc-url>jdbc:hive2://fuze248:10000/default</jdbc-url> <script>/user/root/oozie/sqoop/sys_account_user_vip/create.q</script> </hive2> <ok to="shell-a94d"/> <error to="Kill"/> </action> <action name="shell-a94d"> <shell xmlns="uri:oozie:shell-action:0.1"> <job-tracker>${jobTracker}</job-tracker> <name-node>${nameNode}</name-node> <exec>/user/root/oozie/sqoop/sys_account_user_vip/script.sh</exec> <file>/user/root/oozie/sqoop/sys_account_user_vip/script.sh#script.sh</file> <capture-output/> </shell> <ok to="sqoop-a084"/> <error to="Kill"/> </action> <action name="sqoop-a084"> <sqoop xmlns="uri:oozie:sqoop-action:0.2"> <job-tracker>${jobTracker}</job-tracker> <name-node>${nameNode}</name-node> <arg>import</arg> <arg>--connect</arg> <arg>jdbc:oracle:thin:xxx:1521:xxx</arg> <arg>--username</arg> <arg>FUZEPASSPORT</arg> <arg>--password</arg> <arg>FUZEPASSPORT_PWD</arg> <arg>--query</arg> <arg>select SA.* from SYS_ACCOUNT SA where $CONDITIONS</arg> <arg>--delete-target-dir</arg> <arg>--target-dir</arg> <arg>/data/hive/sys_account/date_time=${(wf:actionData('shell-a94d')['date_time'])} </arg> <arg>--fields-terminated-by</arg> <arg>\001</arg> <arg>-m</arg> <arg>1</arg> </sqoop> <ok to="hive2-46d1"/> <error to="Kill"/> </action> <action name="hive2-46d1" cred="hive2"> <hive2 xmlns="uri:oozie:hive2-action:0.1"> <job-tracker>${jobTracker}</job-tracker> <name-node>${nameNode}</name-node> <jdbc-url>jdbc:hive2://fuze248:10000/default</jdbc-url> <script>/user/root/oozie/sqoop/sys_account_user_vip/procedure.q</script> <param>date_time=${(wf:actionData('shell-a94d')['date_time'])}</param> </hive2> <ok to="End"/> <error to="Kill"/> </action> <end name="End"/> </workflow-app>
相关文章推荐
- 如何在HP-UX中安装bash?
- 再谈CSHELL对C程序员的价值
- powershell 汉洛塔
- shell根据系统硬盘大小删除文件
- shell if 多条件判断 时间大小比较
- shell之监控tomcat(一个tomcat)端口是否正常
- shell常用脚本
- shell脚本无法使用source的原因及解决方法
- python调用shell脚本
- shell调用脚本--直接退出不执行后续命令
- 5. storm常用命令(客户端storm shell)
- hive shell执行命令报错:Specified key was too long; max key length is 767 bytes
- 14.bash中 2>&1 & 的解释
- 13.工作命令汇集
- shell之逆向解析以生日生成的MD5
- 9.一些shell编程中命令的累积
- 8.shell中文本处理三剑客
- 7.shell分支与循环结构
- 4.shell变量替换技术
- 3.shell变量的子串应用