diff --git a/wbackup/CommandParser.cs b/wbackup/CommandParser.cs index af3f55d..032ea58 100644 --- a/wbackup/CommandParser.cs +++ b/wbackup/CommandParser.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Linq; using System.Text; @@ -19,6 +20,7 @@ public string sourceDir; public string destinationDir; public string logFileName; + public DateTime? abortedTime = null; public BackupMopde backupMode = BackupMopde.Full; @@ -27,6 +29,7 @@ bool isSource = false; bool isDestination = false; bool isLogFile = false; + bool isAbortTime = false; foreach (var arg in args) { @@ -45,7 +48,6 @@ { backupMode = BackupMopde.Differencial; } - if ("srce" == command.ToLower()) { isSource = true; @@ -58,6 +60,10 @@ { isLogFile = true; } + if ("aborttime" == command.ToLower()) + { + isAbortTime = true; + } } else { @@ -78,6 +84,15 @@ logFileName = arg; isLogFile = false; } + + if (isAbortTime) + { + abortedTime = DateTime.Parse(arg); + if (abortedTime <= DateTime.Now) + { + abortedTime = abortedTime.Value.AddDays(1); + } + } } } } diff --git a/wbackup/Program.cs b/wbackup/Program.cs index 858ba60..ea62a17 100644 --- a/wbackup/Program.cs +++ b/wbackup/Program.cs @@ -7,16 +7,18 @@ using System.Threading; using System.Security.Cryptography; using System.Security.Cryptography.X509Certificates; +using System.Numerics; using ICSharpCode.SharpZipLib.GZip; using ICSharpCode.SharpZipLib.Tar; using System.Threading.Tasks; +using ICSharpCode.SharpZipLib.BZip2; namespace wbackup { class Program { public static CommandParser param; - public static int maxThreadLimit = 4; + public static int maxThreadLimit = 6; class FileInformation { @@ -24,9 +26,34 @@ public uint hash; }; + static private uint CalclateHash(string fileName) + { + uint hashValue = 0; + byte[] readbuff = new byte [1024 * 4]; + Array.Clear(readbuff, 0, readbuff.Length); + using (var fs = File.OpenRead(fileName)) + { + int readLen = fs.Read(readbuff, 0, readbuff.Length); + fs.Close(); + Vector hashTemp1 = Vector.Zero; + Vector hashTemp2 = Vector.Zero; + ReadOnlySpan span = new ReadOnlySpan(readbuff); + for (int i = 0; i < readbuff.Length; i += Vector.Count * sizeof(int)) + { + hashTemp2 = new Vector(span.Slice(i, Vector.Count * sizeof(int))); + hashTemp1 ^= hashTemp2; + } + } + + return hashValue; + } + static void Main(string[] args) { var sw = new System.Diagnostics.Stopwatch(); + Mutex consoleLock = new Mutex(); + + Console.WriteLine("IsHardwareAccelerated = {0}.", Vector.IsHardwareAccelerated); param = new CommandParser(args); string postFixString = DateTime.Now.ToString("yyyyMMddHHmmss"); @@ -38,29 +65,67 @@ logws = File.CreateText(param.logFileName + "-" + postFixString + ".log"); } - List fileNameList = new List(Directory.GetFiles(param.sourceDir, "*.*", SearchOption.AllDirectories)); + Console.WriteLine("Loading file list."); + List fileNameList = new List(Directory.GetFiles(param.sourceDir, "*.*", SearchOption.AllDirectories )); - // 圧縮率を高めるためにファイルをソートする + int sourceFileCnt = fileNameList.Count; + int calclatedHashCnt = 0; var hashAlgorism = SHA256.Create(); List sortedTemp = new List(); - foreach (var fileName in fileNameList) + Parallel.ForEach(fileNameList, new ParallelOptions { MaxDegreeOfParallelism = maxThreadLimit }, fileName => { - byte []hashData = hashAlgorism.ComputeHash(File.OpenRead(fileName)); - uint hashTemp = (uint)hashData[0] ^ (((uint)hashData[1]) << 8) ^ (((uint)hashData[2]) << 16) ^ (((uint)hashData[3]) << 24); - sortedTemp.Add(new FileInformation() { fileName = fileName, hash = hashTemp }); - } + // バックアップ出来なかったファイルを次回処理するため、アーカイブフラグを一度設定する + try + { + var attribute = File.GetAttributes(fileName); + if (param.backupMode == BackupMopde.Full) + { + attribute |= FileAttributes.Archive; + File.SetAttributes(fileName, attribute); + } + + // 圧縮率を高めるためにハッシュ値を取得しファイルをソートする + if (true == attribute.HasFlag(FileAttributes.Archive) && false == attribute.HasFlag(FileAttributes.Hidden)) + { + uint hashTemp = CalclateHash(fileName); + lock (sortedTemp) + { + sortedTemp.Add(new FileInformation() { fileName = fileName, hash = hashTemp }); + } + } + } + catch (Exception exp) + { + consoleLock.WaitOne(); + Console.WriteLine("Skip calclate hash, {0}, {1}", fileName, exp.ToString()); + if (logws != null) + logws.WriteLine("Skip calclate hash, {0}, {1}", fileName, exp.ToString()); + consoleLock.ReleaseMutex(); + } + + int calclatedHashTemp = Interlocked.Increment(ref calclatedHashCnt); + Console.Write("\rCalclate Hash. {0}/{1}", calclatedHashTemp, sourceFileCnt); + }); + Console.WriteLine("Calclate Hash. {0}/{1}", calclatedHashCnt, sourceFileCnt); + + Console.WriteLine("Sorted file list..."); List sortedList = new List(sortedTemp.OrderBy((x) => x.hash)); + sortedTemp.Clear(); // ハッシュ化したデータを各Threadに振り分ける + int totalFileCount = sortedList.Count; + int archiveFileCount = 0; + int exceptionCount = 0; + ConcurrentQueue[] srceFilesQueue = new ConcurrentQueue[maxThreadLimit]; for (int i = 0;i < maxThreadLimit;i++) { srceFilesQueue[i] = new ConcurrentQueue(); - int beginIndex = (i * fileNameList.Count / maxThreadLimit); - int lastIndex = ((i + 1) * fileNameList.Count / maxThreadLimit); + int beginIndex = (i * sortedList.Count / maxThreadLimit); + int lastIndex = ((i + 1) * sortedList.Count / maxThreadLimit); if (maxThreadLimit == (i + 1)) { - lastIndex = fileNameList.Count; + lastIndex = sortedList.Count; } for (int j = beginIndex; j < lastIndex; j++) @@ -68,11 +133,7 @@ srceFilesQueue[i].Enqueue(sortedList[j].fileName); } } - - int totalFileCount = fileNameList.Count; - int archiveFileCount = 0; - int exceptionCount = 0; - Mutex consoleLock = new Mutex(); + sortedList.Clear(); sw.Start(); @@ -85,7 +146,6 @@ using (TarOutputStream zos = new TarOutputStream(bzo)) { bzo.IsStreamOwner = false; - bzo.SetLevel(9); int buffSelect = 0; int buffSize = zos.RecordSize; @@ -97,17 +157,23 @@ { try { - var attribute = File.GetAttributes(srceFileName); - if (attribute.HasFlag(FileAttributes.Archive) || param.backupMode == BackupMopde.Full) + if (param.abortedTime.HasValue) { - if (consoleLock.WaitOne()) + if (param.abortedTime < DateTime.Now) { - Console.WriteLine("Archive, {0}", srceFileName); - if (logws != null) - logws.WriteLine("Archive, {0}", srceFileName); - consoleLock.ReleaseMutex(); + break; } + } + var attribute = File.GetAttributes(srceFileName); + if (true == attribute.HasFlag(FileAttributes.Archive) && false == attribute.HasFlag(FileAttributes.Hidden)) + { + int archiveFileCountTemp = Interlocked.Increment(ref archiveFileCount); + consoleLock.WaitOne(); + Console.WriteLine("Archive[{1}, {2:0.000}%], {0}", srceFileName, i, 100.0 * archiveFileCountTemp / totalFileCount); + if (logws != null) + logws.WriteLine("Archive[{1}, {2:0.000}%], {0}", srceFileName, i, 100.0 * archiveFileCountTemp / totalFileCount); + consoleLock.ReleaseMutex(); // バックアップ対象を出力先に転記する using (var fis = new FileStream(srceFileName, FileMode.Open, FileAccess.Read)) @@ -142,19 +208,15 @@ { File.SetAttributes(srceFileName, (attribute & (~FileAttributes.Archive))); } - - Interlocked.Increment(ref archiveFileCount); } else { // アーカイブフラグがないファイルをスキップする - if (consoleLock.WaitOne()) - { - Console.WriteLine("Skip, {0}", srceFileName); - if (logws != null) - logws.WriteLine("Skip, {0}", srceFileName); - consoleLock.ReleaseMutex(); - } + consoleLock.WaitOne(); + Console.WriteLine("Skip, {0}", srceFileName); + if (logws != null) + logws.WriteLine("Skip, {0}", srceFileName); + consoleLock.ReleaseMutex(); } } catch (Exception exp) @@ -163,13 +225,11 @@ var curtExceptionCnt = Interlocked.Increment(ref exceptionCount); - if (consoleLock.WaitOne()) - { - Console.WriteLine("Exception {0}, {1}", curtExceptionCnt, exp.ToString()); - if (logws != null) - logws.WriteLine("Exception {0}, {1}", curtExceptionCnt, exp.ToString()); - consoleLock.ReleaseMutex(); - } + consoleLock.WaitOne(); + Console.WriteLine("Exception {0}, {1}", curtExceptionCnt, exp.ToString()); + if (logws != null) + logws.WriteLine("Exception {0}, {1}", curtExceptionCnt, exp.ToString()); + consoleLock.ReleaseMutex(); } if (exceptionCount > param.maxRetryCount) @@ -184,11 +244,18 @@ sw.Stop(); Console.WriteLine("TotalMilliseconds = {0}", sw.Elapsed.TotalMilliseconds); - Console.WriteLine("Finished {0} archived, {1} skipped, {2} aborted.", archiveFileCount, totalFileCount - archiveFileCount, totalFileCount); + // 未処理のファイル数を取得する + int abortedCount = 0; + for (int i = 0; i < maxThreadLimit; i++) + { + abortedCount += srceFilesQueue[i].Count(); + } + + Console.WriteLine("Finished {0} archived, {1} skipped, {2} aborted.", archiveFileCount, totalFileCount - archiveFileCount, abortedCount); if (logws != null) { logws.Flush(); - logws.WriteLine("Finished {0} archived, {1} skipped, {2} aborted.", archiveFileCount, totalFileCount - archiveFileCount, totalFileCount); + logws.WriteLine("Finished {0} archived, {1} skipped, {2} aborted.", archiveFileCount, totalFileCount - archiveFileCount, abortedCount); for (int i = 0; i < maxThreadLimit; i++) { diff --git a/wbackup/Properties/launchSettings.json b/wbackup/Properties/launchSettings.json index 496cd2d..7e3aa0f 100644 --- a/wbackup/Properties/launchSettings.json +++ b/wbackup/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "wbackup": { "commandName": "Project", - "commandLineArgs": "-srce \"D:\\Projects\\atozip\" -dest \"D:\\backup\\archive\" -log \"D:\\backup\\archive\" -full" + "commandLineArgs": "-srce \"D:\\Projects\\atozip\" -dest \"D:\\backup\\archive\" -log \"D:\\backup\\archive\" -full -aborttime 13:00" } } } \ No newline at end of file diff --git a/wbackup/wbackup.csproj b/wbackup/wbackup.csproj index d07cc14..1ae653b 100644 --- a/wbackup/wbackup.csproj +++ b/wbackup/wbackup.csproj @@ -5,8 +5,17 @@ netcoreapp3.1 + + x64 + + + + x64 + + +