myvtm/Module/mod_upload/upload.cpp

#include "stdafx.h"
#include "SpBase.h"
#include "SpIni.h"
#include "upload.h"

#include <memutil.h>
#include <fileutil.h>
#include <array.h>
#include "XZip.h"
#include <tchar.h>
#include <string>
#include "sstream"

#include "ZipZilb.h"
#include "stdio.h"

using namespace std;
//获取文件夹实际路径
static ErrorCodeEnum expand_dir(IEntityFunction *pEntityFunc, const char *str, CSimpleStringA &out)
{
	ErrorCodeEnum Error = Error_Unexpect;
	if (str) {
		const char *p = str;
		const char *sfirst;
		int first = 0;
		char tmp[1024];
		int k = 0;
		while (*p) {
			if (first) {
				if (*p == ')') {
					char key[MAX_PATH];
					key[0] = 0;
					CSimpleStringA strKeyPath;;
					memcpy(key, sfirst, p - sfirst);
					key[p-sfirst] = 0;
					Error = pEntityFunc->GetPath(key, strKeyPath);
					if (Error != Error_Succeed) {
						Dbg("sys path $(%s)$ cannot evaluate!", key);
						return Error;
					}
					strcpy(&tmp[k], (LPCSTR)strKeyPath);\
					k += strKeyPath.GetLength();
					first = 0;
				}
			} else {
				if (*p == '$' && *(p+1) == '(') {
					p++;
					first = 1;
					sfirst = p+1;
				} else {
					tmp[k++] = *p;
				}
			}
			p++;
		}
		if (k != 0) {
			tmp[k] = 0;
			out = tmp;
			return Error_Succeed;
		}
	} else {
		return Error_Param;
	}
	return Error;
}

static upload_dir_t *upload_dir_load(IEntityFunction *pEntityFunc, IConfigInfo *pConfig, const char *dir, int default_silent_time, int default_limitation)
{
	upload_dir_t *updir = ZALLOC_T(upload_dir_t);
	if (updir) 
	{
		char tmp[MAX_PATH];
		CSimpleStringA str;
		ErrorCodeEnum Error = pConfig->ReadConfigValue(dir, "Path", str);
		if (Error == Error_Succeed) 
		{
			INIT_LIST_HEAD(&updir->candidate_list);
			CSimpleStringA strRealPath;
			Error = expand_dir(pEntityFunc, (LPCSTR)str, strRealPath);
			if (Error == Error_Succeed) 
			{
				INIT_LIST_HEAD(&updir->candidate_list);
				updir->name = _strdup(dir);
				updir->path = _strdup(strRealPath);
				pConfig->ReadConfigValue(dir, "MovePath", str);
				if (str.GetLength() > 0) 
				{
					updir->flags |= UPLOAD_FLAG_MOVEPATH;
					Error = expand_dir(pEntityFunc, str, strRealPath);
					if (Error != Error_Succeed) 
					{
						return NULL;
					}
					updir->movepath = _strdup(strRealPath);
				} else {
					pConfig->ReadConfigValue(dir, "AutoDelete", str);
					if (_stricmp(str, "true") == 0) 
					{
						updir->flags |= UPLOAD_FLAG_AUTODELETE;
					}
				}
				pConfig->ReadConfigValue(dir, "Zip", str);
				if (_stricmp(str, "true") == 0) 
				{
					updir->flags |= UPLOAD_FLAG_ZIP;
				}
				pConfig->ReadConfigValueInt(dir, "SilentTime", updir->silent_time);
				if (updir->silent_time == 0) 
				{
					updir->silent_time = default_silent_time;
				}
				pConfig->ReadConfigValueInt(dir, "Limitation", updir->child_count_limitation);
				if (updir->child_count_limitation == 0) 
				{
					updir->child_count_limitation = default_limitation;
				}
				updir->fileCount=0;//初始化设置为0
				updir->fileLenSum=0;//初始化设置为0
			}
		}
	}
	return updir;
}

static void upload_dir_clear_candidate_list(upload_dir_t *updir);


static void file_destroy(file_t *file)
{
	free(file->name);
	free(file->path);
	free(file);
}

void upload_file_destroy(file_t *file)
{
	file_destroy(file);
}

void updir_del_file(file_t *file)
{
	(file->owner->fileCount)--;//文件夹文件个数减1
	file->owner->fileLenSum=file->owner->fileLenSum-file->length/1024;//减去文件长度
	list_del(&file->entry);//从链表删除文件
	upload_file_destroy(file);//删除文件内存

}


//初始化要上传的文件夹配置
int upload_create(struct list_head *list, IEntityFunction *pEntityFunc, IConfigInfo *pConfig, CSimpleStringA &checkDir)
{
	assert(list_empty(list));
	assert(pConfig);
	ErrorCodeEnum Error;
	char type_str[1024];
	char *p, *c;
	int default_silent_time = 0;
	int default_limitation = 0;

	{
		CSimpleStringA str;
		Error = pConfig->ReadConfigValue("Main", "Type", str);
		if (Error == Error_Succeed) 
		{
			strcpy(type_str, (LPCSTR)str);
		} 
		else 
		{
			return Error;
		}
		//添加lwt，读入日结时需要检查的文件类型参数
		Error = pConfig->ReadConfigValue("Main", "CheckType", str);
		if (Error == Error_Succeed) 
		{
			checkDir = str;
		} 
		else 
		{
			return Error;
		}
		pConfig->ReadConfigValueInt("Main", "SilentTime", default_silent_time);//间隔时间
		pConfig->ReadConfigValueInt("Main", "Limitation", default_limitation);//最大文件数
		if (default_limitation == 0) 
		{
			default_limitation = INT_MAX;
		}
	}

	p = strtok_s(type_str, ", ", &c);
	while (p) {
		upload_dir_t *dir = upload_dir_load(pEntityFunc, pConfig, p, default_silent_time, default_limitation);
		if (!dir) 
		{
			return Error_Unexpect;
		}
		Dbg("load %s ok", p);
		list_add_tail(&dir->entry, list);
		p = strtok_s(NULL, ", ", &c);
	}

	return 0;
}

static int updir_exist_file(upload_dir_t *dir, const char *path)
{
	file_t *pos;
	list_for_each_entry(pos, &dir->candidate_list, file_t, entry) {
		if (_stricmp(pos->path, path) == 0)
			return TRUE;
	}
	return FALSE;
}

#define FT_2000_1_1_0_0_0	125911584000000000UL

static int check_zero_ref(const char *path)
{
	HANDLE hFile = CreateFileA(path, 
		GENERIC_READ, 0, NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); // try open
	if (hFile != INVALID_HANDLE_VALUE) {
		CloseHandle(hFile);
		return TRUE;
	} else {
		DWORD dwRet = GetLastError();
		return FALSE;
	}
}

static void updir_add_file(upload_dir_t *dir, const char *path)
{
	WIN32_FILE_ATTRIBUTE_DATA attr;
	if (GetFileAttributesExA(path, GetFileExInfoStandard, &attr)) 
	{
		file_t *f = ZALLOC_T(file_t);
		if (f) 
		{
			int offset = strlen(dir->path);
			if (dir->path[offset-1] != '\\')
				offset++;
			f->path = _strdup(path);
			f->name = _strdup(path + offset);
			f->owner = dir;
			char *p = f->name;
			while (p = strchr(p, '\\')) 
			{
				*p = '$';
			}
			union 
			{
				FILETIME ft;
				unsigned __int64 v;
			}u;
			u.ft.dwHighDateTime = attr.ftCreationTime.dwHighDateTime;
			u.ft.dwLowDateTime = attr.ftCreationTime.dwLowDateTime;
			f->create_time = (unsigned int)((u.v - FT_2000_1_1_0_0_0) / 10000000UL);
			f->length = attr.nFileSizeLow;

			if (strcmp(dir->name,"Debug") == 0) 
			{
				string str(f->path);
				if ( str.substr(str.find_last_of('.')+1,str.length())!="zip")
				{
					if(attr.nFileSizeHigh>0||attr.nFileSizeLow>200*1024*1024)
					{
						//删文件
						Dbg("file %s name(%s) lenth (nFileSizeHigh=%d,nFileSizeLow=%d) is over 200m，prepare delete!", f->path, f->name,attr.nFileSizeHigh,attr.nFileSizeLow);
						WORD dwAttr = GetFileAttributesA(f->path);
						dwAttr &= ~FILE_ATTRIBUTE_READONLY;
						SetFileAttributesA(f->path, dwAttr);
						DeleteFileA(f->path);
						Dbg("file %s name(%s) lenth is over 200m，delete !", f->path, f->name);
						return;
					}
					string createtime;
					ostringstream oss;
					oss<<f->create_time;
					createtime = oss.str();
					str = str.substr(0,str.find_last_of('.'))+"&"+createtime+"."+"zip";
					if (ZipData((LPCTSTR)str.c_str(),(LPCTSTR)f->path))
					{
						//delete source file
						WORD dwAttr = GetFileAttributesA(f->path);
						dwAttr &= ~FILE_ATTRIBUTE_READONLY;
						SetFileAttributesA(f->path, dwAttr);
						DeleteFileA(f->path);
						//insert zip file info
						string name(f->name);
						name = name.substr(0,name.find_last_of('.')+1)+"zip";
						f->name = _strdup(name.c_str());
						f->path = _strdup(str.c_str());
						WIN32_FILE_ATTRIBUTE_DATA attr;
						if (GetFileAttributesExA(f->path, GetFileExInfoStandard, &attr)) 
						{
							f->length = attr.nFileSizeLow;
						}
					}	
				}
				else
				{
					string createdata(f->path);		
					std::size_t found = createdata.find('&');  
					if (found!=std::string::npos)  
					{
						int index0 = createdata.find_last_of('\\')+1;
						int index = createdata.find_last_of('&');
						int index2 = createdata.find_last_of('.');
						string filetime = createdata.substr(index0,index-index0);
						createdata = createdata.substr(index+1,index2-index);
						string fullname(f->name);
						//Dbg("get full name = %s",fullname.c_str());
						string name = fullname.substr(0,fullname.find_last_of('$')+1)+filetime+"."+"zip";
						//Dbg("upload name = %s",name.c_str());
						f->name = _strdup(name.c_str());
						f->create_time = atoi(createdata.c_str());
					}

				}
			}
			//把文件按照时间从小到大排序放置
			file_t *pos;
			list_for_each_entry(pos, &dir->candidate_list, file_t, entry) 
			{
				if (f->create_time < pos->create_time) 
				{
					__list_add(&f->entry, pos->entry.prev, &pos->entry);
					(dir->fileCount)++;//文件夹文件个数加1
					dir->fileLenSum=dir->fileLenSum+f->length/1024; //统计传送文件长度
					Dbg("::file %s name(%s) added!", f->path, f->name);
					return;
				}
			}

			Dbg("file %s name(%s) added!", f->path, f->name);
			list_add_tail(&f->entry, &dir->candidate_list);//把文件加入要上传的列表
			(dir->fileCount)++;//文件夹文件个数加1
			dir->fileLenSum=dir->fileLenSum+f->length/1024; //统计传送文件长度
		}
	}
}

static void dir_fresh(upload_dir_t *dir, const char *path)
{
	array_header_t *arr_files = fileutil_get_sub_files2_a(path, dir->child_count_limitation);
	if (arr_files) {
		int i;
		for (i = 0; i < arr_files->nelts; ++i) 
		{
			char *file_path = ARRAY_IDX(arr_files, i, char*);
			WIN32_FILE_ATTRIBUTE_DATA attr;
			if (GetFileAttributesExA(file_path, GetFileExInfoStandard, &attr)) 
			{
				SYSTEMTIME st;
				SYSTEMTIME filest;
				FILETIME ft;
				FILETIME fttmp;
				GetLocalTime(&st);
				SystemTimeToFileTime(&st, &ft);
				if (CompareFileTime(&ft, &attr.ftLastWriteTime) > 0) 
				{
					//LARGE_INTEGER *p1 = (LARGE_INTEGER *)&attr.ftLastWriteTime;
					FileTimeToLocalFileTime(&attr.ftLastWriteTime,&fttmp);
					FileTimeToSystemTime(&fttmp,&filest);
					LARGE_INTEGER *p2 = (LARGE_INTEGER *)&ft;
					LARGE_INTEGER *p1 = (LARGE_INTEGER *)&fttmp;
					LONGLONG diff = p2->QuadPart - p1->QuadPart;
					diff = diff / 10000000L; // convert to seconds FILETIME是以100纳秒(ns)为单位	
					//if silent_time == 1 days
					if (dir->silent_time >= 86400)
					{
						int nsecondsInterval = (st.wYear-filest.wYear)*32140800+(st.wMonth-filest.wMonth)*2678400+(st.wDay-filest.wDay)*86400;
						if((nsecondsInterval>=dir->silent_time))
						{
							if (check_zero_ref(file_path)) 
							{
								if (!updir_exist_file(dir, file_path)) 
								{
									updir_add_file(dir, file_path);
									
								}
							}
							//Dbg("addfile %s added! diff:%d", file_path,nsecondsInterval);
						}
						
					}
					else
					{
						if (diff >= dir->silent_time) 
						{
							if (check_zero_ref(file_path)) 
							{
								if (!updir_exist_file(dir, file_path)) 
								{
									updir_add_file(dir, file_path);
								}
							}
							//Dbg("addfile %s added! diff:%d", file_path,diff);
						}
						
					}
				}
				else
				{
					Dbg("the time error!");
				}
			}
		}
		toolkit_array_free2(arr_files);
	}

	array_header_t *arr_dir = fileutil_get_sub_dirs_a(path);
	if (arr_dir) {
		int i;
		for (i = 0; i < arr_dir->nelts; ++i) {
			char *dir_path = ARRAY_IDX(arr_dir, i, char*);
			dir_fresh(dir, dir_path);
		}
	}
}

int upload_fresh(struct list_head *list)
{
	upload_dir_t *pos;

	list_for_each_entry(pos, list, upload_dir_t, entry) 
	{
		dir_fresh(pos, pos->path);
	}

	return 0;
}

int check_dir_fresh(const char *path,int limitation,int silentTime,int &fileSumlen){
	int count=0;//当前文件夹下面的文件个数
	int sum = 0;//当前文件夹下面的子文件夹下面的文件个数
	array_header_t *arr_files = fileutil_get_sub_files2_a(path, limitation);
	if (arr_files) {
		int i;
		for (i = 0; i < arr_files->nelts; ++i) 
		{
			char *file_path = ARRAY_IDX(arr_files, i, char*);
			WIN32_FILE_ATTRIBUTE_DATA attr;
			if (GetFileAttributesExA(file_path, GetFileExInfoStandard, &attr)) 
			{
				SYSTEMTIME st;
				SYSTEMTIME filest;
				FILETIME ft;
				FILETIME fttmp;
				GetLocalTime(&st);
				SystemTimeToFileTime(&st, &ft);
				if (CompareFileTime(&ft, &attr.ftLastWriteTime) > 0) 
				{
					//LARGE_INTEGER *p1 = (LARGE_INTEGER *)&attr.ftLastWriteTime;
					FileTimeToLocalFileTime(&attr.ftLastWriteTime,&fttmp);
					FileTimeToSystemTime(&fttmp,&filest);
					LARGE_INTEGER *p2 = (LARGE_INTEGER *)&ft;
					LARGE_INTEGER *p1 = (LARGE_INTEGER *)&fttmp;
					LONGLONG diff = p2->QuadPart - p1->QuadPart;
					diff = diff / 10000000L; // convert to seconds FILETIME是以100纳秒(ns)为单位	
					
					//增加特殊文件不计入上传处理
					char* p=strstr(file_path,"SystemInitial.log");
					if(p!=NULL){
						continue;
					}
					 p=strstr(file_path,"G_");
					if(p!=NULL){
						continue;
					}

					if (silentTime >= 86400)
					{
						int nsecondsInterval = (st.wYear-filest.wYear)*32140800+(st.wMonth-filest.wMonth)*2678400+(st.wDay-filest.wDay)*86400;
						if((nsecondsInterval>=silentTime))
						{
							//个数加1
							count++;
							//获取文件长度
							WIN32_FILE_ATTRIBUTE_DATA attr;
							if (GetFileAttributesExA(file_path, GetFileExInfoStandard, &attr)) 
							{
								int fileSize = attr.nFileSizeLow/1024;//单位为k
								fileSumlen+=fileSize;
								
							}else{
								
							}
							Dbg("checkfile %s ! diff:%d", file_path,nsecondsInterval);
						}
					}
					else
					{
						if (diff >= silentTime) 
						{
							
							//个数加1
							count++;
							//获取文件长度
							WIN32_FILE_ATTRIBUTE_DATA attr;
							if (GetFileAttributesExA(file_path, GetFileExInfoStandard, &attr)) 
							{
								int fileSize = attr.nFileSizeLow/1024;//单位为k
								fileSumlen+=fileSize;
								
							}else{

							}
							Dbg("checkfile %s ! diff:%d", file_path,diff);
						}
					}
				}
				else
				{
					Dbg("the time error!");
				}
			}
		}
		toolkit_array_free2(arr_files);
	}

	array_header_t *arr_dir = fileutil_get_sub_dirs_a(path);
	if (arr_dir) {
		int i;
		for (i = 0; i < arr_dir->nelts; ++i) {
			char *dir_path = ARRAY_IDX(arr_dir, i, char*);
			int fileLen=0;
			sum += check_dir_fresh(dir_path,limitation,silentTime,fileLen);
			fileSumlen=fileSumlen+fileLen;
		}
	}
	count = sum+count;
	return count;
}

bool ZipData(LPCTSTR lpszZipArchive, LPCTSTR lpszSrcFile)
{
	BOOL bResult = TRUE;
	if (!lpszZipArchive)
	{
		Dbg("lpszZipArchive is NULL");
		return false;
	}

	if (!lpszSrcFile )
	{
		Dbg("lpszSrcFile is NULL");
		return false;
	}

	// does zip source file exist?
	//if (_waccess((wchar_t *)lpszSrcFile, 0) == -1)
	//{
	//	Dbg("WARNING: zip source file '%s' cannot be found,operation aborted",lpszSrcFile);
	//	return false;
	//}

	// use only the file name for zip file entry
	TCHAR * cp = (TCHAR *)_tcsrchr(lpszSrcFile, _T('\\'));
	if (cp == NULL)
		cp = (TCHAR *) lpszSrcFile;
	else
		cp++;

	HZIP hz = CreateZip((void *)lpszZipArchive, 0, ZIP_FILENAME);

	if (hz)
	{
		ZRESULT zr = ZipAdd(hz, cp, (void *)lpszSrcFile, 0, ZIP_FILENAME);

		CloseZip(hz);

		// did add work?
		if (zr == ZR_OK)
		{
			//Dbg("added '%s' to zip file '%s'",lpszSrcFile, lpszZipArchive);
			bResult = true;
		}
		else
		{
			Dbg("WARNING: failed to add zip source file '%s'",lpszSrcFile);
			bResult = false;
		}
	}
	else
	{
		Dbg("ERROR: failed to create zip file '%s'",lpszZipArchive);
		bResult = false;
	}

	return bResult;
}