上亿级数据量-JDBC插入数据库

package com.database;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
/**
* 将亿级数据量,通过JDBC方式,插入到数据库中.关键技术说明
*   一、为了避免内存溢出问题,每100万条重新建立一次jdbc连接.
*   二、将连接属性设置为不自动提交,每5万条批量执行一次提交.
* @author Administrator
*/
public class Import {
private static final String JDBC_URL = "jdbc:oracle:thin:@localhost:1521:orcl";
private static final String JDBC_USER = "ttt";
private static final String JDBC_PASSWORD = "pw123456";
private static final String JDBC_SQL = "insert into tb_test (col1,col2) values (?,?)";

public static void main(String[] args) throws Exception {
Class.forName("oracle.jdbc.driver.OracleDriver");
exe(100000005);//1亿数据量测试通过
}

public static void exe(int sl) throws SQLException{
int SIZE = 1000000;//每批的记录数(建议值100万)
int page = sl/SIZE;//计算批数

for (int i = 0; i < page; i++) {//批量执行
execute(i*SIZE,(i+1)*SIZE);
}
if(sl%SIZE != 0){
execute(page*SIZE,sl);
}
}

private static void execute(int begin, int end) throws SQLException {
Connection conn = null;
PreparedStatement pst= null;
try {
conn = DriverManager.getConnection(JDBC_URL, JDBC_USER,JDBC_PASSWORD);
System.out.println("重新打开连接 :" + begin + " - " + end);

conn.setAutoCommit(false);// 设置不自动提交
pst = conn.prepareStatement(JDBC_SQL);

int recordNum = 0; // 计数器
int commit_size = 50000;// 每次提交记录数5万
for (int i = begin; i < end; i++) {
recordNum++; // 计数
pst.setString(1, "n" + i);
pst.setString(2, "v" + i);
pst.addBatch();
if (recordNum % commit_size == 0) {
pst.executeBatch();
conn.commit();
System.out.println("提交:" + i);
conn.setAutoCommit(false);
pst = conn.prepareStatement(JDBC_SQL);
}
}
if (recordNum % commit_size != 0) {
pst.executeBatch();
conn.commit();
System.out.println("提交:" + recordNum);
}
} catch (SQLException e) {
e.printStackTrace();
} finally {
try {
if (pst != null) pst.close();
} catch (SQLException e) {
e.printStackTrace();
}finally{
try{
if (conn != null) conn.close();
System.out.println("成功关闭连接！");
} catch (SQLException e) {
e.printStackTrace();
}
}
}
}

}