diff --git a/wbackup/Program.cs b/wbackup/Program.cs index c8da4de..858ba60 100644 --- a/wbackup/Program.cs +++ b/wbackup/Program.cs @@ -5,6 +5,7 @@ using System.IO.Enumeration; using System.Linq; using System.Threading; +using System.Security.Cryptography; using System.Security.Cryptography.X509Certificates; using ICSharpCode.SharpZipLib.GZip; using ICSharpCode.SharpZipLib.Tar; @@ -15,31 +16,67 @@ class Program { public static CommandParser param; + public static int maxThreadLimit = 4; + + class FileInformation + { + public string fileName; + public uint hash; + }; static void Main(string[] args) { var sw = new System.Diagnostics.Stopwatch(); - string[] storedTarget = { ".mov", ".mp4", ".mpg", ".mpeg", ".jpg", ".jpeg", ".png", ".gif", ".zip" }; - param = new CommandParser(args); - ConcurrentQueue srceFilesQueue = new ConcurrentQueue(Directory.GetFiles(param.sourceDir, "*.*", SearchOption.AllDirectories)); - string postFixString = DateTime.Now.ToString("yyyyMMddHHmmss"); + string[] storedTarget = { ".mov", ".mp4", ".mpg", ".mpeg", ".jpg", ".jpeg", ".png", ".gif", ".zip" }; StreamWriter logws = null; if (param.logFileName != "") { logws = File.CreateText(param.logFileName + "-" + postFixString + ".log"); } - int totalFileCount = srceFilesQueue.Count; + + List fileNameList = new List(Directory.GetFiles(param.sourceDir, "*.*", SearchOption.AllDirectories)); + + // 圧縮率を高めるためにファイルをソートする + var hashAlgorism = SHA256.Create(); + List sortedTemp = new List(); + foreach (var fileName in fileNameList) + { + byte []hashData = hashAlgorism.ComputeHash(File.OpenRead(fileName)); + uint hashTemp = (uint)hashData[0] ^ (((uint)hashData[1]) << 8) ^ (((uint)hashData[2]) << 16) ^ (((uint)hashData[3]) << 24); + sortedTemp.Add(new FileInformation() { fileName = fileName, hash = hashTemp }); + } + List sortedList = new List(sortedTemp.OrderBy((x) => x.hash)); + + // ハッシュ化したデータを各Threadに振り分ける + ConcurrentQueue[] srceFilesQueue = new ConcurrentQueue[maxThreadLimit]; + for (int i = 0;i < maxThreadLimit;i++) + { + srceFilesQueue[i] = new ConcurrentQueue(); + int beginIndex = (i * fileNameList.Count / maxThreadLimit); + int lastIndex = ((i + 1) * fileNameList.Count / maxThreadLimit); + if (maxThreadLimit == (i + 1)) + { + lastIndex = fileNameList.Count; + } + + for (int j = beginIndex; j < lastIndex; j++) + { + srceFilesQueue[i].Enqueue(sortedList[j].fileName); + } + } + + int totalFileCount = fileNameList.Count; int archiveFileCount = 0; int exceptionCount = 0; Mutex consoleLock = new Mutex(); sw.Start(); - Parallel.For(0, 1, i => + Parallel.For(0, maxThreadLimit, i => { // アーカイブ先ファイルのインスタンスを作成 string zipFileName = param.destinationDir + "-" + postFixString + "-" + i.ToString("00") + ".tar.gz"; @@ -56,7 +93,7 @@ readBuffer[0] = new byte[buffSize]; readBuffer[1] = new byte[buffSize]; - while (srceFilesQueue.TryDequeue(out string srceFileName)) + while (srceFilesQueue[i].TryDequeue(out string srceFileName)) { try { @@ -122,7 +159,7 @@ } catch (Exception exp) { - srceFilesQueue.Enqueue(srceFileName); + srceFilesQueue[i].Enqueue(srceFileName); var curtExceptionCnt = Interlocked.Increment(ref exceptionCount); @@ -147,15 +184,18 @@ sw.Stop(); Console.WriteLine("TotalMilliseconds = {0}", sw.Elapsed.TotalMilliseconds); - Console.WriteLine("Finished {0} archived, {1} skipped, {2} aborted.", archiveFileCount, totalFileCount - archiveFileCount, srceFilesQueue.Count); + Console.WriteLine("Finished {0} archived, {1} skipped, {2} aborted.", archiveFileCount, totalFileCount - archiveFileCount, totalFileCount); if (logws != null) { logws.Flush(); - logws.WriteLine("Finished {0} archived, {1} skipped, {2} aborted.", archiveFileCount, totalFileCount - archiveFileCount, srceFilesQueue.Count); + logws.WriteLine("Finished {0} archived, {1} skipped, {2} aborted.", archiveFileCount, totalFileCount - archiveFileCount, totalFileCount); - foreach (var fileName in srceFilesQueue) + for (int i = 0; i < maxThreadLimit; i++) { - logws.WriteLine("aborted, {0}", fileName); + foreach (var fileName in srceFilesQueue[i]) + { + logws.WriteLine("aborted, {0}", fileName); + } } logws.Close(); }