java通过ftp方式读取文件,并解析入库
2015-01-07 20:59
465 查看
对于自己是个总结,下次使用的时候直接copy,要注意的地方是ftp访问文件的方式,访问的是ftp用户的home目录,而不是从绝对路径访问。 private final static String DELETEUSER = "delete from customermanager_user where cmanagerno in "; private final static String QUERYUSER = "select cmanagerno from customermanager_user where cmanagerno in "; private final static String INSERTUSER = "insert into customermanager_user(cmanagerno, cmanagercity, cmanagercontry, cmanagername, cmanagercontact, cmanagerstatus) values(?,?,?,?,?,?)"; private CustomManagerJobDAO managerJobDAO; public CustomManagerJobDAO getManagerJobDAO() { return managerJobDAO; } public void setManagerJobDAO(CustomManagerJobDAO managerJobDAO) { this.managerJobDAO = managerJobDAO; } /** * customermanager_group 表增删查 */ private final static String DELETEGROUP = "delete from customermanager_group where id in "; private final static String QUERYGROUP = "select id from customermanager_group where id in "; private final static String INSERTGROUP = "insert into customermanager_group(id, cmgroupno, cmgroupcity, cmgroupcontry, cmgroupcustomername, cmgroupcorno, cmgroupcoradd, cmgroupcustomertype, cmgroupiffirst, cmgroupcustomerlevel, cmgroupcontact, cmgroupslalevel, cmgroupaddress, cmgroupsatus) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; private static Integer file1length = Integer.valueOf(6); private static Integer file2length = Integer.valueOf(14); private final static String REMOVEFLAG = "1"; private final static String UPDATEFLAG = "0"; private List removekeylist = new LinkedList(); private List insertkeylist = new LinkedList(); private List insertlist = new LinkedList(); static { String file1 = StaticMethod.nullObject2String(XmlManage.getFile( "/config/customanager-ftp-boss.xml").getProperty("file1length")); if (file1 != null && file1.length() >= 0) { file1length = Integer.valueOf(file1); } String file2 = StaticMethod.nullObject2String(XmlManage.getFile( "/config/customanager-ftp-boss.xml").getProperty("file2length")); if (file1 != null && file2.length() >= 0) { file2length = Integer.valueOf(file2); } } public void parseData(String filepath) { int table1size = file1length.intValue(); int table2size = file2length.intValue(); int count = 1; try { long begintime = System.currentTimeMillis(); URL url = new URL(filepath); URLConnection urlconn = url.openConnection(); BufferedReader br = new BufferedReader(new InputStreamReader(urlconn.getInputStream())); String line = null; String keyflag = null; // 移除和删除标记 while ((line = br.readLine()) != null) { count++; String[] fileds = line.split("\\|"); if (table1size == fileds.length) { keyflag = fileds[table1size - 1]; if (REMOVEFLAG.equals(keyflag)) { removekeylist.add(fileds[0]); } else if (UPDATEFLAG.equals(keyflag)) { insertkeylist.add(fileds[0]); fileds = changeAreaNametoId(fileds); insertlist.add(fileds);// 再把所有数组给存起来 } } else if (table2size == fileds.length) { keyflag = fileds[table2size - 1]; if (REMOVEFLAG.equals(keyflag)) { removekeylist.add(fileds[0]); } else if (UPDATEFLAG.equals(keyflag)) { insertkeylist.add(fileds[0]); fileds = changeAreaNametoId(fileds); insertlist.add(fileds);// 再把所有数组给存起来 } } if (count % 5000 == 0) { queryAndDelete(removekeylist, insertlist); // 先查一把!然后再删 queryAndInsert(insertkeylist, insertlist);// 先查一把!然后再新增 removekeylist.clear(); insertkeylist.clear(); insertlist.clear(); } queryAndDelete(removekeylist, insertlist); // 先查一把!然后再删 queryAndInsert(insertkeylist, insertlist);// 先查一把!然后再新增 removekeylist.clear(); insertkeylist.clear(); insertlist.clear(); } br.close(); long endtime = System.currentTimeMillis(); System.out.println((endtime - begintime) / 3600); } catch (FileNotFoundException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } } public void getFileByFtp() { String ftpserver = StaticMethod.nullObject2String(XmlManage.getFile("/config/customanager-ftp-boss.xml").getProperty("ftpserver")); String userLogin = StaticMethod.nullObject2String(XmlManage.getFile("/config/customanager-ftp-boss.xml").getProperty("ftpuserLogin")); String pwdLogin = StaticMethod.nullObject2String(XmlManage.getFile("/config/customanager-ftp-boss.xml").getProperty("ftppwdLogin")); String serverPath = StaticMethod.nullObject2String(XmlManage.getFile("/config/customanager-ftp-boss.xml").getProperty("ftpserverPath")); try { // ftp服务器上下载每天的工单流水文件 String retMessage = connectToFtpServer(ftpserver, userLogin, pwdLogin); if (!retMessage.equals("connect success")) { System.out.println("下载时文件时无法和FTP服务器连接"); } String filename = StaticMethod.getYYYYMMDD(StaticMethod .getLocalString(-1)); System.out.println("filename===" + filename); String filepath1 = "ftp://" + userLogin + ":" + pwdLogin + "@" + ftpserver + serverPath + filename + "_001.txt"; String filepath2 = "ftp://" + userLogin + ":" + pwdLogin + "@" + ftpserver + serverPath + filename + "_002.txt"; System.out.println("path===" + filepath1); URL url = new URL(filepath1); URLConnection urlconn = url.openConnection(); BufferedReader br = new BufferedReader(new InputStreamReader( urlconn.getInputStream())); String line = null; while ((line = br.readLine()) != null) { System.out.println("line======" + line); } parseData(filepath1); parseData(filepath2); br.close(); closeFtpConnect(); } catch (MalformedURLException e1) { e1.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } } /** * 把地市名称换成id * * @param keylist */ public String[] changeAreaNametoId(String[] records) { if (file1length.intValue() == records.length) { // 数组长度为6,为第一张表 records[1] = getAreaid(records[1]); } else if (file2length.intValue() == records.length) { records[2] = getAreaid(records[2]); } return records; } /** * 对数据进行查询,如果存在则删除 * * @param keylist */ public void queryAndDelete(List keylist, List datalist) { String sql = null; String[] records = null; if (datalist != null && datalist.size() > 0) { records = (String[]) datalist.get(0); } List retlist = null; try { if (file1length.intValue() == records.length) { // 数组长度为6,为第一张表 sql = getExecuteSql(QUERYUSER, keylist); retlist = this.getManagerJobDAO().getKeyList(sql); // 根据key把库中已经存在的记录查出来。 if (retlist != null && retlist.size() > 0) { sql = getExecuteSql(DELETEUSER, retlist); // 组装成需要的sql System.out.println("delete id from customermanager_user where flag = 1:===" + retlist); this.getManagerJobDAO().delteRecord(sql);// 根据key把库中已经存在的删除 } } else if (file2length.intValue() == records.length) { sql = getExecuteSql(QUERYGROUP, keylist); retlist = this.getManagerJobDAO().getKeyList(sql); // 根据key把库中已经存在的记录查出来。 if (retlist != null && retlist.size() > 0) { sql = getExecuteSql(DELETEGROUP, retlist); System.out.println("delete id from customermanager_group where flag = 1 ===="+ sql); this.getManagerJobDAO().delteRecord(sql); } } } catch (SQLException e) { e.printStackTrace(); } } public void queryAndInsert(List keylist, List datalist) { String sql = null; String[] records = null; if (datalist != null && datalist.size() > 0) { records = (String[]) datalist.get(0); } List retlist = new ArrayList(); try { if (file1length.intValue() == records.length && retlist != null) { // 数组长度为6,为第一张表 sql = getExecuteSql(QUERYUSER, keylist); retlist = this.getManagerJobDAO().getKeyList(sql); // 根据key把库中已经存在的记录查出来。 if (retlist.size() == 0) { // 如果没有查到一个,则全部入库 this.getManagerJobDAO().insertTable1(INSERTUSER, datalist); // 把所有字段全部入库 } else { sql = getExecuteSql(DELETEUSER, retlist); // 如果存在重复,则更新和新增分开操作 System.out.println("delete id from customermanager_user where flag = 1 ===="+ retlist); this.getManagerJobDAO().delteRecord(sql);// 根据key把库中已经存在的删除 System.out.println("insert * into customermanager_user where flag = 0 ===="+ datalist); this.getManagerJobDAO().insertTable1(INSERTUSER, datalist); // 把所有字段全部入库 } } else if (file2length.intValue() == records.length && retlist != null) { sql = getExecuteSql(QUERYGROUP, keylist); retlist = this.getManagerJobDAO().getKeyList(sql); if (retlist.size() == 0) { this.getManagerJobDAO().insertTable2(INSERTGROUP, datalist); } else { sql = getExecuteSql(DELETEGROUP, retlist); System.out.println("delete id from customermanager_group where flag = 1 ===="+ retlist); this.getManagerJobDAO().delteRecord(sql); System.out.println("insert * into customermanager_group where flag = 0 ===="+ datalist); this.getManagerJobDAO().insertTable2(INSERTGROUP, datalist); } } } catch (SQLException e) { e.printStackTrace(); } } public static String getExecuteSql(final String querySql, List retList) { String record = null; String keys = null; StringBuffer sqlsb = new StringBuffer(); for (int i = 0; i < retList.size(); i++) { keys = StaticMethod.nullObject2String(retList.get(i)); sqlsb.append("'").append(keys).append("',"); } record = sqlsb.toString(); String sqlwhere = record.substring(0, record.lastIndexOf(",")); return querySql + "(" + sqlwhere + ")"; } public Logger logger = Logger.getLogger(this.getClass()); FtpClient ftpClient=null; /* * 获取FTP服务 */ public String connectServer( String ip, int port, String user, String password, String path) { String returnResult = "true"; try { ftpClient = new FtpClient(); ftpClient.openServer(ip, port); ftpClient.login(user, password); if (path.length() != 0) ftpClient.cd(path); ftpClient.binary(); } catch (IOException ex) { returnResult = "false"; } return returnResult; } private String connectToFtpServer(String ftpserver, String userLogin, String pwdLogin) { FtpClient fc = new FtpClient(); if ((ftpserver == null) || (ftpserver.equals(""))) return "FTP服务器名设置不正确!"; try { fc.openServer(ftpserver); fc.login(userLogin, pwdLogin); fc.binary(); } catch (FtpLoginException e) { return "没有与FTP服务器连接的权限,或用户名密码设置不正确!"; } catch (IOException e) { return "与FTP服务器连接失败!"; } catch (SecurityException e) { return "没有权限与FTP服务器连接"; } return "connect success"; } /* * 关闭FTP */ public void closeFtpConnect() { try { ftpClient.closeServer(); } catch (IOException ex) { System.out.println("关闭时出错======:"+ex.getMessage()); } } }
相关文章推荐
- java通过ftp方式读取文件,并解析入库
- java通过ftp方式读取文件,并解析入库
- Java通过jxl解析Excel文件入库,及日期格式处理方式 (附源代码)
- Java眼中的XML--文件读取--2 应用SAX方式解析XML
- java解析dbf之通过javadbf包生成和读取dbf文件
- Java眼中的XML--文件读取--2 应用SAX方式解析XML
- Java眼中的XML--文件读取--1 应用DOM方式解析XML
- java中,通过FastJson对海量数据的Json文件,边读取边解析
- java开发:读取ftp服务器文件通过浏览器下载
- Java眼中的XML--文件读取--2 应用SAX方式解析XML
- java通过判断文本的编码方式再读取文件
- java通过poi解析Excel文件,直接读取客户端文件
- 通过ftp读取文件入库
- java读取文件入库
- JAVA解析XML配置文件的读取操作
- 通过java提供的URL类包读取网上的文件
- Java读取远程文件[Http,ftp],并保存
- 【Java】运用多种方式读取文件内容,包括按字节、字符和按行为单位读取文件内容
- java中读取文件的方式
- JAVA读取文件方式