您的位置:首页 > 数据库

功能齐全、效率一流的免费开源数据库导入导出工具(c#开发,支持SQL server、SQLite、ACCESS三种数据库),每月借此处理数据5G以上

2014-06-28 21:50 1406 查看
软件名:DataPie

功能:支持SQL server、SQLite、ACCESS数据库的导入、导出、存储过程调用,支持EXCEL2007、EXCEL2003、ACCESS2007、 CSV文件导入数据库,支持EXCEL、CSV、ZIP、ACCESS文件方式导出,支持数据拆分导出及自定义SQL查询与导出。

开发背景:作者从事财务管理工作,主要是出具集团的内部财务报表,随着公司精细化管理的需求,管理报表的数据量急速增长, 依赖EXCEL加工处理数据已经变得极为困难,因此团队全面转向关系数据库进行数据处理,为减少财务人员使用数据库的难度,因此专门针对财务报表核算需要,开发了该工具。目前,我月度报表处理的数据量超过5G,最大的单次运算量记录接近千万,该工具主要发挥的作用就是将收集到的数据, 导入SQL SERVER数据库,进行报表运算,并且输出各类财务报表,对于几十万级的数据输入、输出基本上能够轻松应付。

源码下载地址:https://github.com/yfl8910/DataPie

软件界面及主要代码:

1.导入界面,支持EXCEL2007、EXCEL2003、ACCESS2007、 CSV文件导入数据库,支持CSV文件夹整体导入相应表。财务工作的数据源较多,有业务提供的excel数据,也有从系统中取出的csv、excel或者text文档,为满足多种数据源的输入,所以多种数据源导入数据库。目前SQL SERVER导入整体效率较高,ACCESS下,excel文件导入效率最高(几十万的数据量可以很快导入),其他方式效率还有待提升。



涉及核心代码如下

public static DataTable GetDataTableFromFile(string path, string tname)
{
string ace = "Microsoft.ACE.OLEDB.12.0";
string jet = "Microsoft.Jet.OLEDB.4.0";
string xl2007 = "Excel 12.0 Xml";
string xl2003 = "Excel 8.0";
string imex = "IMEX=1";
string hdr = "Yes";
string conn = "Provider={0};Data Source={1};Extended Properties=\"{2};HDR={3};{4}\";";
string select = "";
string ext = Path.GetExtension(path);
OleDbDataAdapter oda;
DataTable dt = new DataTable(tname);
switch (ext.ToLower())
{
case ".xlsx":
conn = String.Format(conn, ace, Path.GetFullPath(path), xl2007, hdr, imex);
select = string.Format("SELECT * FROM [{0}$]", tname);
break;
case ".xls":
conn = String.Format(conn, jet, Path.GetFullPath(path), xl2003, hdr, imex);
select = string.Format("SELECT * FROM [{0}$]", tname);
break;
case ".accdb":
conn = String.Format("Provider=Microsoft.ACE.OLEDB.12.0;Data Source= {0};Persist Security Info=False;", path);
select = string.Format("SELECT * FROM [{0}]", tname);
break;
case ".csv":
conn = String.Format(conn, ace, path.Substring(0, path.LastIndexOf('\\')), "text;Excel 12.0", hdr, imex);
select = string.Format("SELECT * FROM [{0}]", Path.GetFileName(path));
break;
default:
throw new Exception("File Not Supported!");
}
OleDbConnection con = new OleDbConnection(conn);
con.Open();
oda = new OleDbDataAdapter(select, con);
oda.Fill(dt);
con.Close();
return dt;
}


2.数据导出界面,支持EXCEL2007、CSV、ACCESS、zip压缩文件、分拆导出几个功能。我目前从事的岗位,报表整体数据量接近千万级,其中单表需要导出分发到业务手中的数据量最大也超过了百万级,目前达到百万级数据量的表导出,一般采用CSV文件或者压缩包的形式,或者分多个表导出。



涉及核心代码。excel导出主要源码 :

public static void SaveExcel(string FileName, string sql, string SheetName)
{

FileInfo newFile = new FileInfo(FileName);
if (newFile.Exists)
{
newFile.Delete();
newFile = new FileInfo(FileName);
}
using (ExcelPackage package = new ExcelPackage(newFile))
{
try
{
ExcelWorksheet ws = package.Workbook.Worksheets.Add(SheetName);

IDataReader reader = DBConfig.db.DBProvider.ExecuteReader(sql);
ws.Cells["A1"].LoadFromDataReader(reader, true);
}
catch (Exception ex)
{
throw ex;
}
package.Save();
}

}

/// <summary>
/// 工作簿中添加新的SHEET
/// </summary>
public static bool SaveExcel(ExcelPackage package, string sql, string SheetName)
{

try
{
ExcelWorksheet ws = package.Workbook.Worksheets.Add(SheetName);
IDataReader reader = DBConfig.db.DBProvider.ExecuteReader(sql);
ws.Cells["A1"].LoadFromDataReader(reader, true);

return true;
}
catch (Exception ex)
{
throw ex;
}
}
/// <summary>
/// 单表格导出到一个EXCEL工作簿
/// </summary>
public static int ExportExcel(string FileName, string sql, string SheetName)
{
if (FileName != null)
{
Stopwatch watch = Stopwatch.StartNew();
watch.Start();

SaveExcel(FileName, sql, SheetName);
watch.Stop();
return Convert.ToInt32(watch.ElapsedMilliseconds / 1000);
}

return -1;
}

public static async Task<int> ExportExcelAsync(string FileName, string sql, string SheetName)
{

return await Task.Run(
() => { return ExportExcel(FileName,sql,SheetName); }
);

}

/// <summary>
/// 多表格导出到一个EXCEL工作簿
/// </summary>
public static int ExportExcel(string[] TabelNameArray, string filename)
{
if (filename != null)
{
Stopwatch watch = Stopwatch.StartNew();
watch.Start();

FileInfo newFile = new FileInfo(filename);
if (newFile.Exists)
{
newFile.Delete();
newFile = new FileInfo(filename);
}

using (ExcelPackage package = new ExcelPackage(newFile))
{
for (int i = 0; i < TabelNameArray.Length; i++)
{
string sql = "select * from  [" + TabelNameArray[i] + "]";
IDataReader reader = DBConfig.db.DBProvider.ExecuteReader(sql);
SaveExcel(package, sql, TabelNameArray[i]);

}
package.Save();
}
watch.Stop();
return Convert.ToInt32(watch.ElapsedMilliseconds / 1000);
}

return -1;
}

public static  async Task<int> ExportExcelAsync(string[] TabelNameArray, string filename) {

return await Task.Run(
() => { return ExportExcel( TabelNameArray,filename); }
);

}

/// <summary>
/// 分拆导出
/// </summary>
public static int ExportExcel(string[] TabelNameArray, string filename, string[] whereSQLArr)
{
if (filename != null)
{
Stopwatch watch = Stopwatch.StartNew();
watch.Start();
FileInfo file = new FileInfo(filename);
int wherecount = whereSQLArr.Length;
int count = TabelNameArray.Length;
for (int i = 0; i < wherecount; i++)
{
string s = filename.Substring(0, filename.LastIndexOf("\\"));
StringBuilder newfileName = new StringBuilder(s);
int index = whereSQLArr[i].LastIndexOf("=");
string sp = whereSQLArr[i].Substring(index + 2, whereSQLArr[i].Length - index - 3);
newfileName.Append("\\" + file.Name.Substring(0, file.Name.LastIndexOf(".")) + "_" + sp + ".xlsx");
FileInfo newFile = new FileInfo(newfileName.ToString());
if (newFile.Exists)
{
newFile.Delete();
newFile = new FileInfo(newfileName.ToString());
}

for (int j = 0; j < count; j++)
{
string sql = "select * from [" + TabelNameArray[j] + "]" + whereSQLArr[i];
IDataReader reader = DBConfig.db.DBProvider.ExecuteReader(sql);
SaveExcel(newfileName.ToString(), sql, TabelNameArray[j]);

}

}

watch.Stop();
return Convert.ToInt32(watch.ElapsedMilliseconds / 1000);
}

return -1;
}

public static async Task<int> ExportExcelAsync(string[] TabelNameArray, string filename, string[] whereSQLArr)
{

return await Task.Run(
() => { return ExportExcel(TabelNameArray,filename,whereSQLArr); }
);

}


csv导出主要源码 (其他代码请参github上的源码):

private static void WriteHeader(IDataReader reader, StreamWriter sw)
{
for (int i = 0; i < reader.FieldCount; i++)
{
if (i > 0)
sw.Write(',');
sw.Write(reader.GetName(i));
}
sw.Write(Environment.NewLine);
}
private static void WriteContent(IDataReader reader, StreamWriter sw)
{
for (int i = 0; i < reader.FieldCount; i++)
{
if (i > 0)
sw.Write(',');
String v = reader[i].ToString();
if (v.Contains(',') || v.Contains('\n') || v.Contains('\r') || v.Contains('"'))
{
sw.Write('"');
sw.Write(v.Replace("\"", "\"\""));
sw.Write('"');
}
else
{
sw.Write(v);
}
}
sw.Write(Environment.NewLine);
}

public static int SaveCsv(IDataReader reader, string filename)
{
Stopwatch watch = Stopwatch.StartNew();
watch.Start();

StreamWriter sw = new StreamWriter(filename, false, Encoding.GetEncoding("gb2312"));
WriteHeader(reader, sw);

while (reader.Read())
{
WriteContent(reader, sw);
}

sw.Flush();
sw.Close();//释放资源

watch.Stop();
return Convert.ToInt32(watch.ElapsedMilliseconds / 1000);
}
public static async Task<int> ExportCsvAsync(IDataReader reader, string filename)
{
return await Task.Run( () => { return SaveCsv( reader,  filename);} );
}

public static StreamWriter GetStreamWriter(string filename, int outCount)
{
string s = filename.Substring(0, filename.LastIndexOf("."));
StringBuilder newfileName = new StringBuilder(s);
newfileName.Append(outCount + 1 + ".csv");
FileInfo newFile = new FileInfo(newfileName.ToString());
if (newFile.Exists)
{
newFile.Delete();
newFile = new FileInfo(newfileName.ToString());
}
StreamWriter sw = new StreamWriter(newfileName.ToString(), false, Encoding.GetEncoding("gb2312"));
return sw;
}

public static int SaveCsv(IDataReader reader, string filename, int pagesize)
{
int innerCount = 0, outCount = 0;
Stopwatch watch = Stopwatch.StartNew();
watch.Start();
StreamWriter sw = GetStreamWriter(filename, outCount);
WriteHeader(reader, sw);
while (reader.Read())
{
if (innerCount < pagesize)
{
WriteContent(reader, sw);
innerCount++;
}
else
{
innerCount = 0;
outCount++;
sw.Flush();
sw.Close();
sw = GetStreamWriter(filename, outCount);
WriteHeader(reader, sw);
WriteContent(reader, sw);
innerCount++;
}
}
sw.Flush();
sw.Close();
watch.Stop();
return Convert.ToInt32(watch.ElapsedMilliseconds / 1000);

}
public static async Task<int> ExportCsvAsync(IDataReader reader, string filename, int pagesize)
{
return await Task.Run(() => { return SaveCsv(reader, filename,pagesize); });
}


3.设置拆分导出条件。财务工作中,涉及向业务单位分发各自的财务数据,如果采用手工筛选,然后分发效率较低,所以开发此功能,快速实现数据的拆分发送。



4.自定义SQL代码导出。通过双击左边的表名即可自动生成导出SQL脚本,点击添加条件,还可以增加导出的条件。该工具在数据核查时,使用比较方便,虽然SQL server数据库也自带相应的功能,但是导出功能缺无法满足我的需求,所以开发一个方便自己使用的界面还是很有必要的。



其他各项功能,及代码请参考github上的源码。
内容来自用户分享和网络整理,不保证内容的准确性,如有侵权内容,可联系管理员处理 点击这里给我发消息
标签: 
相关文章推荐