检测重复文件功能实现(改进版本1)

针对大文件md5计算时间偏长问题，觉得不需要完全计算整个文件的md5也可以初步确定文件是否相同。换作计算一些属性的md5值，极大提高速度，也可以满足需求：

1. 文件名

2. 文件修改时间——根据实验，两个重复文件如果仅仅是复制，创建时间和最后访问时间都会改变，但最后修改时间在内容没有修改时不会是变的。这是根据文件系统特征决定的。

3. 文件大小

以上三点使用以下结构来表示。typedef struct FileInfoForCompare
{
DWORD dwFileAttributes;
FILETIME ftLastWriteTime;
DWORD nFileSizeHigh;
DWORD nFileSizeLow;
WCHAR cFileName[ 260 ];
}FileInfoForCompare_;
以下版本更改计算md5的数据，更正了GetMultimapForDirList函数的while循环没有把迭代器加1的bug。改进之后的代码如下：
#include <windows.h>
#include "md5.h"

typedef struct MIX_WIN32_FIND_DATA
{
std::wstring fullpath;
WIN32_FIND_DATA wfd;
}MIX_WIN32_FIND_DATA_;

typedef std::multimap<INT64,MIX_WIN32_FIND_DATA> map_type;
typedef std::list<std::wstring> list_type;
union union_MD5
{
INT64 i64_md5;
unsigned char c_md5[16];
};

typedef struct FileInfoForCompare
{
DWORD dwFileAttributes;
FILETIME ftLastWriteTime;
DWORD nFileSizeHigh;
DWORD nFileSizeLow;
WCHAR cFileName[ 260 ];
}FileInfoForCompare_;

void GetMd5(MIX_WIN32_FIND_DATA &mwfd,union_MD5 &uMd5)
{
// use file data md5.
//MD5File(mwfd.fullpath.c_str(), uMd5.c_md5);

// use file attribute md5
FileInfoForCompare fic;
fic.dwFileAttributes = mwfd.wfd.dwFileAttributes;
fic.ftLastWriteTime = mwfd.wfd.ftLastWriteTime;
fic.nFileSizeHigh = mwfd.wfd.nFileSizeHigh;
fic.nFileSizeLow = mwfd.wfd.nFileSizeLow;
_tcscpy_s(fic.cFileName,MAX_PATH,mwfd.wfd.cFileName);
MD5Data((unsigned char*)(&fic),sizeof(fic),uMd5.c_md5);
}

DWORD RecurseGetFileList(std::wstring &fullpath,map_type &map_data)
{
DWORD count = 0;
MIX_WIN32_FIND_DATA mwfd;
WIN32_FIND_DATA &fd = mwfd.wfd;
HANDLE hFile = ::FindFirstFile(fullpath.c_str(),&fd);
if ( INVALID_HANDLE_VALUE == hFile )
{
return 0;
}
do
{
if ( ( 0 != _tcscmp(fd.cFileName,_T(".")))
&& ( 0 != _tcscmp(fd.cFileName,_T("..")) ) )
{
if ( fd.dwFileAttributes == INVALID_FILE_ATTRIBUTES )
{
return 0;
}
if ( FILE_ATTRIBUTE_DIRECTORY & fd.dwFileAttributes )
{
std::wstring newdir = fullpath.substr(0,fullpath.length()-2);
newdir.append(_T("\\")).append(fd.cFileName).append(_T("\\*"));
count += RecurseGetFileList(newdir,map_data);
}
else
{
union_MD5 uMd5;
mwfd.fullpath = fullpath.substr(0,fullpath.length()-2);
mwfd.fullpath.append(_T("\\")).append(fd.cFileName);

GetMd5(mwfd,uMd5);

map_data.insert(std::make_pair(uMd5.i64_md5,mwfd));
int lowpart = uMd5.i64_md5>>32;
int highpart = uMd5.i64_md5&0xFFFFFFFF;
_tprintf(_T("MD5:%04x %04x filepath: %s\n"),highpart,lowpart,mwfd.fullpath.c_str());
count++;
}
}

if (::FindNextFile(hFile,&fd) == FALSE)
{
break;
}

}while(1);

return count;
}

DWORD GetMultimapForDirList(list_type dirlist,map_type &out_map)
{
DWORD count = 0;
WIN32_FILE_ATTRIBUTE_DATA fad;
list_type::iterator it = dirlist.begin();
while (it!=dirlist.end())
{
count += RecurseGetFileList(*it,out_map);
it++;
}
return count;
}

int main(int argc,char ** argv)
{
list_type find_list;
map_type find_map;
find_list.push_front(std::wstring(_T("D:\\Setup Files\\*")));
GetMultimapForDirList(find_list,find_map);
return 0;
}