python脚本 用sqoop把mysql数据导入hive
2016-11-07 10:20
531 查看
主要功能：用python把mysql数据库的数据导入到hive中，该过程主要是通过python语言操作sqoop.
not in run_hive_query("show tables"):
os.system("sqoop import --connect jdbc:mysql://%s:%s/%s --username %s --password %s --table %s --hive-database %s -m 10 --create-hive-table --hive-import --hive-overwrite "%(
host,port,database,user,passwd,table,database))
else:
os.system("sqoop import --connect jdbc:mysql://%s:%s/%s --username %s --password %s --table %s --hive-database %s -m 10 --hive-import --hive-overwrite "%(
host,port,database,user,passwd,table,database))
mysql_to_hive(mysql_info["host"],mysql_info["port"],mysql_info["user"],mysql_info["passwd"].replace("(","\("),"wwn","cm_vip")
#! /usr/bin/env python
# coding:utf-8
# --------------------------------
# Created by coco on 16/2/23
# ---------------------------------
# Comment: 主要功能说明 :初始化业务数据库
import os
import pyhs2
conn=pyhs2.connect(host="192.168.8.94",port=10000,authMechanism="PLAIN",user="hdfs")
mysql_info={"host":"192.168.8.94","port":3306,"user":"root","passwd":"gc895316"}
print mysql_info
def run_hive_query(sql):
with conn.cursor() as cursor:
cursor.execute(sql)
return cursor.fetchall()
def mysql_to_hive(host,port,user,passwd,database,table):
#os.system("hadoop fs -rm -r /user/task/%s"%table)
if [database] not in run_hive_query("show databases"):
with conn.cursor() as cursor:
cursor.execute("create database " +database)
with conn.cursor() as cursor:
cursor.execute("use "+database)
if 