diff --git a/ClearArchiveFlag.,cs b/ClearArchiveFlag.,cs new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/ClearArchiveFlag.,cs diff --git a/DuplicateCheck/DuplicateCheck.csproj b/DuplicateCheck/DuplicateCheck.csproj new file mode 100644 index 0000000..c73e0d1 --- /dev/null +++ b/DuplicateCheck/DuplicateCheck.csproj @@ -0,0 +1,8 @@ + + + + Exe + netcoreapp3.1 + + + diff --git a/DuplicateCheck/Program.cs b/DuplicateCheck/Program.cs new file mode 100644 index 0000000..737e126 --- /dev/null +++ b/DuplicateCheck/Program.cs @@ -0,0 +1,87 @@ +using System; +using System.IO; +using System.Collections; +using System.Collections.Generic; +using System.Security.Cryptography; + +namespace DuplicateCheck +{ + class DuplicateCheckInfo + { + public string fileName; + public byte[] hashValue; + public long fileSize; + + public ulong HashKey + { + get + { + ulong hashKey = 0; + for (int i = 0; i < sizeof(ulong); i++) + { + hashKey = (hashKey << 8) ^ hashValue[i]; + } + return hashKey; + } + } + + public DuplicateCheckInfo(string fileName) + { + this.fileName = fileName; + var hash = SHA256.Create(); + using (FileStream fs = File.OpenRead(fileName)) + { + fileSize = fs.Length; + if (fileSize > 4096) + { + hash.ComputeHash(fs); + hashValue = hash.Hash; + } + } + } + + public bool IsMatch(DuplicateCheckInfo srce) + { + if (this.fileSize != srce.fileSize) + return false; + + for (int i = 0;i < hashValue.Length;i ++) + { + if (hashValue[i] != srce.hashValue[i]) + return false; + } + + return true; + } + } + + class Program + { + static void Main(string[] args) + { + Console.WriteLine("Duplicate Check Test Logic."); + Dictionary hashData = new Dictionary(); + foreach (var item in Directory.GetFiles(@"D:\Projects\", "*", SearchOption.AllDirectories)) + { + DuplicateCheckInfo info = new DuplicateCheckInfo(item); + if (info.fileSize > 4096) + { + ulong hashValue = info.HashKey; + if (hashData.TryGetValue(hashValue, out DuplicateCheckInfo srceInfo)) + { + if (srceInfo.IsMatch(info)) + { + Console.WriteLine("Duplicate {0}, {1}", item, srceInfo.fileName); + Console.WriteLine("Copy-Item -Path \"{0}\" -Destination \"{1}\"", srceInfo.fileName, item); + Console.WriteLine("Set-ItemProperty -Path \"{0}\" -Name LastWriteTime -Value \"{1}\"", item, File.GetLastWriteTime(item).ToString("yyyy-MM-dd HH:mm:ss")); + } + } + else + { + hashData.Add(hashValue, info); + } + } + } + } + } +} diff --git a/wbackup.sln b/wbackup.sln index feda39d..6b7554f 100644 --- a/wbackup.sln +++ b/wbackup.sln @@ -3,7 +3,7 @@ # Visual Studio Version 16 VisualStudioVersion = 16.0.30413.136 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "wbackup", "wbackup\wbackup.csproj", "{1EA6ED4D-C97F-4CBA-B30D-786647B21198}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "wbackup", "wbackup\wbackup.csproj", "{1EA6ED4D-C97F-4CBA-B30D-786647B21198}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/wbackup/ArchiveTarFile.cs b/wbackup/ArchiveTarFile.cs new file mode 100644 index 0000000..c339e61 --- /dev/null +++ b/wbackup/ArchiveTarFile.cs @@ -0,0 +1,95 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using ICSharpCode.SharpZipLib.GZip; +using ICSharpCode.SharpZipLib.Tar; +using ICSharpCode.SharpZipLib.BZip2; + +using log4net; + +namespace wbackup +{ + class ArchiveTarFile : BackupStreamBase + { + protected FileStream fileOutputStream; + protected GZipOutputStream gzipOutputStream; + protected TarOutputStream tarOutputStream; + + protected int archiveCount; + protected int buffSelect; + protected int buffSize; + protected byte[][] readBuffer; + protected DateTime minLastWriteDate; + + public ArchiveTarFile(string exportPath, BackupStreamBase stream = null):base(stream) + { + string zipFileName = exportPath + ".tar.gz"; + + // 出力先ストリームのインスタンス生成 + fileOutputStream = new FileStream(zipFileName, FileMode.Create, FileAccess.Write); + gzipOutputStream = new GZipOutputStream(fileOutputStream); + tarOutputStream = new TarOutputStream(gzipOutputStream, Encoding.UTF8); + gzipOutputStream.IsStreamOwner = false; + + // ソース読込のためのバッファ生成 + buffSelect = 0; + buffSize = tarOutputStream.RecordSize; + readBuffer = new byte[2][]; + readBuffer[0] = new byte[buffSize]; + readBuffer[1] = new byte[buffSize]; + + archiveCount = 0; + minLastWriteDate = DateTime.Parse("1970-12-01"); + } + + protected override void DisposeManagedResource() + { + tarOutputStream.Flush(); + tarOutputStream.Close(); + } + + protected override void DoWork(string sourceFileName) + { + if (log.IsInfoEnabled) log.InfoFormat("Archive {0}, {1}", Interlocked.Increment(ref archiveCount), sourceFileName); + + using (var fis = new FileStream(sourceFileName, FileMode.Open, FileAccess.Read)) + { + // 最終書込日時が小さい場合には書き換えておく + var lastWriteTime = File.GetLastWriteTime(sourceFileName); + if (lastWriteTime < minLastWriteDate) + { + File.SetLastWriteTime(sourceFileName, minLastWriteDate); + } + + // ZIPファイルヘッダ作成 + TarEntry tarEntry = TarEntry.CreateEntryFromFile(sourceFileName); + tarOutputStream.PutNextEntry(tarEntry); + + Task readResult = fis.ReadAsync(readBuffer[buffSelect], 0, buffSize); + while (true) + { + readResult.Wait(); + int readSize = readResult.Result; + if (readSize <= 0) + { + break; + } + + Task writeResult = tarOutputStream.WriteAsync(readBuffer[buffSelect], 0, readSize); + + buffSelect ^= 1; + readResult = fis.ReadAsync(readBuffer[buffSelect], 0, buffSize); + + writeResult.Wait(); + } + + tarOutputStream.CloseEntry(); + + AddNextSource(sourceFileName); + } + } + } +} diff --git a/wbackup/BackupStreamBase.cs b/wbackup/BackupStreamBase.cs index 4c1c465..f32f5e5 100644 --- a/wbackup/BackupStreamBase.cs +++ b/wbackup/BackupStreamBase.cs @@ -4,38 +4,122 @@ using System.Threading; using System.Collections.Concurrent; +using log4net; + namespace wbackup { - class BackupStreamBase + class BackupStreamBase : IDisposable { + protected ILog log; + protected bool working; protected BackupStreamBase nextStream = null; protected ConcurrentQueue sourceFileQueue = null; protected Thread threadInstance = null; + + private bool _disposed = false; public BackupStreamBase(BackupStreamBase stream = null) { + log = LogManager.GetLogger(this.GetType().FullName); + nextStream = stream; sourceFileQueue = new ConcurrentQueue(); threadInstance = new Thread(this.DoWork); } + ~BackupStreamBase() + { + Dispose(false); + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (!_disposed) + { + if (disposing) + { + // TODO: Dispose managed resources here. + DisposeManagedResource(); + } + + // TODO: Free unmanaged resources here. + DisposeUnmanagedResource(); + + // Note disposing has been done. + _disposed = true; + } + } + + protected virtual void DisposeManagedResource() + { + + } + + protected virtual void DisposeUnmanagedResource() + { + + } + /// /// バックアップ対象フォルダを追加する /// /// - public void AddSource(string newFolder) + public void AddSource(string newSource) { - sourceFileQueue.Enqueue(newFolder); + sourceFileQueue.Enqueue(newSource); } - public void Start() + /// + /// 次のインスタンスのソースファイルに追加する + /// + /// + public void AddNextSource(string newSource) + { + if (null != nextStream) + { + nextStream.AddSource(newSource); + } + } + + /// + /// Threadを開始する + /// + public virtual void Start() { threadInstance.Start(); + + if (null != nextStream) + { + nextStream.Start(); + } } + public virtual void Wait() + { + while (threadInstance.IsAlive) + { + Thread.Sleep(1000); + } + + if (null != nextStream) + nextStream.Wait(); + } + + /// + /// ソースからデータを取得して処理する + /// + /// public void DoWork(object data) { + if (log.IsDebugEnabled) log.Debug("begin thread"); + working = true; while (true) { @@ -50,17 +134,37 @@ // これ以上処理するデータが無いことを示す if (sourceFileName == "") { - nextStream.AddSource(""); + if (log.IsDebugEnabled) log.Error("detect end of queue"); + AddNextSource(""); + OnEndStream(); break; } // ファイルの処理を行う - DoWork(sourceFileName); + try + { + DoWork(sourceFileName); + } + catch (Exception exp) + { + if (log.IsErrorEnabled) log.Error("Exception at DoWork", exp); + } } + working = false; + if (log.IsDebugEnabled) log.Debug("finish thread"); } - public virtual void DoWork(string sourceFileName) + /// + /// ソースから取得したファイルを処理する + /// + /// + protected virtual void DoWork(string sourceFileName) + { + + } + + protected virtual void OnEndStream() { } diff --git a/wbackup/ClearArchiveFlag.cs b/wbackup/ClearArchiveFlag.cs new file mode 100644 index 0000000..e11b7d3 --- /dev/null +++ b/wbackup/ClearArchiveFlag.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace wbackup +{ + class ClearArchiveFlag : BackupStreamBase + { + protected int archiveCount; + + public ClearArchiveFlag(BackupStreamBase stream = null) : base(stream) + { + archiveCount = 0; + } + + protected override void DoWork(string sourceFileName) + { + if (log.IsDebugEnabled) log.DebugFormat("Clear {0}, {1}", Interlocked.Increment(ref archiveCount), sourceFileName); + + var attributes = File.GetAttributes(sourceFileName); + if (attributes.HasFlag(FileAttributes.Archive)) + { + attributes = attributes & ~FileAttributes.Archive; + File.SetAttributes(sourceFileName, attributes); + } + } + } +} diff --git a/wbackup/CommandParser.cs b/wbackup/CommandParser.cs index 17c2944..02b6f92 100644 --- a/wbackup/CommandParser.cs +++ b/wbackup/CommandParser.cs @@ -22,6 +22,7 @@ class CommandParser { + public int maxThreadCount = 6; public int maxRetryCount = 1000; public string sourceDir; public string destinationDir; @@ -29,6 +30,7 @@ public DateTime? abortedTime = null; public string preFixText = ""; public string postFixText = ""; + public List ignoreFile = new List(); public BackupMopde backupMode = BackupMopde.Full; public ArchiveMode archiveMode = ArchiveMode.tgz; @@ -38,6 +40,7 @@ bool isSource = false; bool isDestination = false; bool isLogFile = false; + bool isIgnoreFile = false; bool isAbortTime = false; foreach (var arg in args) @@ -77,6 +80,10 @@ { isDestination = true; } + if ("ignore" == command.ToLower()) + { + isIgnoreFile = true; + } if ("log" == command.ToLower()) { isLogFile = true; @@ -106,6 +113,12 @@ isLogFile = false; } + if (isIgnoreFile) + { + ignoreFile.Add(arg); + isIgnoreFile = false; + } + if (isAbortTime) { abortedTime = DateTime.Parse(arg); @@ -113,6 +126,7 @@ { abortedTime = abortedTime.Value.AddDays(1); } + isAbortTime = false; } } } diff --git a/wbackup/ListingAllSourceFile.cs b/wbackup/ListingAllSourceFile.cs index c9ffbd6..8e35d74 100644 --- a/wbackup/ListingAllSourceFile.cs +++ b/wbackup/ListingAllSourceFile.cs @@ -7,28 +7,17 @@ using System.Threading.Tasks; using System.Text.RegularExpressions; +using log4net; + namespace wbackup { - class ListingAllSourceFile : BackupStreamBase + class ListingAllSourceFile : ListingSourceFiles { - protected List ignoreFolder = new List(); - public ListingAllSourceFile(BackupStreamBase stream = null):base(stream) { - } - - /// - /// バックアップ除外フォルダを追加する - /// - /// - public void AddIgnoreFolder(string newFolder) - { - ignoreFolder.Add(newFolder); - } - - public override void DoWork(string sourceFileName) + protected override void DoWork(string sourceFileName) { ListingSourceFiles(sourceFileName); } @@ -44,8 +33,22 @@ if (false == isIgnoreFile(fileName)) { var fileAttrib = File.GetAttributes(fileName); - File.SetAttributes(fileName, fileAttrib | FileAttributes.Archive); - nextStream.AddSource(fileName); + + if (false == fileAttrib.HasFlag(FileAttributes.Hidden)) + { + if (log.IsDebugEnabled) log.DebugFormat("Add File, {0}, {1:X}", fileName, fileAttrib); + + File.SetAttributes(fileName, fileAttrib | FileAttributes.Archive); + AddNextSource(fileName); + } + else + { + if (log.IsDebugEnabled) log.DebugFormat("Not Archive File, {0}, {1:X}", fileName, fileAttrib); + } + } + else + { + if (log.IsDebugEnabled) log.DebugFormat("Ignore File, {0}", fileName); } } @@ -54,27 +57,10 @@ var dirAttrib = File.GetAttributes(directory); if (false == dirAttrib.HasFlag(FileAttributes.Hidden)) { + if (log.IsDebugEnabled) log.DebugFormat("Open Directory, {0}", directory); ListingSourceFiles(directory); } } } - - /// - /// バックアップ対象外ファイルか判定する - /// - /// - /// - protected bool isIgnoreFile(string sourceFileName) - { - foreach (var ignoreInfo in ignoreFolder) - { - if (Regex.IsMatch(sourceFileName, ignoreInfo)) - { - return true; - } - } - - return false; - } } } diff --git a/wbackup/ListingArchiveSourceFile.cs b/wbackup/ListingArchiveSourceFile.cs new file mode 100644 index 0000000..041cfd2 --- /dev/null +++ b/wbackup/ListingArchiveSourceFile.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Collections.Concurrent; +using System.Numerics; +using System.IO; +using System.Threading.Tasks; +using System.Text.RegularExpressions; + +namespace wbackup +{ + class ListingArchiveSourceFile : ListingSourceFiles + { + public ListingArchiveSourceFile(BackupStreamBase stream = null) : base(stream) + { + + } + + protected override void DoWork(string sourceFileName) + { + ListingSourceFiles(sourceFileName); + } + + /// + /// ソースファイルのリストを作成する + /// + /// + protected void ListingSourceFiles(string rootFolder) + { + foreach (string fileName in Directory.GetFiles(rootFolder)) + { + if (false == isIgnoreFile(fileName)) + { + var fileAttrib = File.GetAttributes(fileName); + if (false == fileAttrib.HasFlag(FileAttributes.Hidden) && true == fileAttrib.HasFlag(FileAttributes.Archive)) + { + if (log.IsDebugEnabled) log.DebugFormat("Add File, {0}, {1:X}", fileName, fileAttrib); + + File.SetAttributes(fileName, fileAttrib | FileAttributes.Archive); + AddNextSource(fileName); + } + else + { + if (log.IsDebugEnabled) log.DebugFormat("Not Archive File, {0}, {1:X}", fileName, fileAttrib); + } + } + else + { + if (log.IsDebugEnabled) log.DebugFormat("Ignore File, {0}", fileName); + } + } + + foreach (string directory in Directory.GetDirectories(rootFolder)) + { + var dirAttrib = File.GetAttributes(directory); + if (false == dirAttrib.HasFlag(FileAttributes.Hidden)) + { + if (log.IsDebugEnabled) log.DebugFormat("Open Directory, {0}", directory); + ListingSourceFiles(directory); + } + } + } + } +} diff --git a/wbackup/ListingSourceFiles.cs b/wbackup/ListingSourceFiles.cs new file mode 100644 index 0000000..925195e --- /dev/null +++ b/wbackup/ListingSourceFiles.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace wbackup +{ + class ListingSourceFiles : BackupStreamBase + { + protected List ignoreFolder = new List(); + + public ListingSourceFiles(BackupStreamBase stream = null) : base(stream) + { + } + + /// + /// バックアップ除外フォルダを追加する + /// + /// + public void AddIgnoreFolder(string newFolder) + { + ignoreFolder.Add(newFolder); + } + + /// + /// バックアップ対象外ファイルか判定する + /// + /// + /// + protected bool isIgnoreFile(string sourceFileName) + { + foreach (var ignoreInfo in ignoreFolder) + { + if (0 <= sourceFileName.IndexOf(ignoreInfo)) + { + return true; + } + } + + return false; + } + } +} diff --git a/wbackup/ParallelBackup.cs b/wbackup/ParallelBackup.cs new file mode 100644 index 0000000..721cdcb --- /dev/null +++ b/wbackup/ParallelBackup.cs @@ -0,0 +1,67 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.IO; +using System.Threading; + +namespace wbackup +{ + class ParallelBackup : BackupStreamBase + { + int archiveCount = 0; + List nextParallelStream = new List(); + + public ParallelBackup() : base(null) + { + } + + public ParallelBackup(BackupStreamBase[] streamList) : base(null) + { + foreach(var stream in streamList) + { + AddNextStream(stream); + } + } + + public void AddNextStream(BackupStreamBase stream) + { + nextParallelStream.Add(stream); + } + + protected override void DoWork(string sourceFileName) + { + int archiveCountTemp = Interlocked.Increment(ref archiveCount); + if (log.IsDebugEnabled) log.DebugFormat("Clear {0}, {1}", archiveCountTemp, sourceFileName); + int parallelIndex = archiveCountTemp % nextParallelStream.Count; + nextParallelStream[parallelIndex].AddSource(sourceFileName); + } + + public override void Start() + { + base.Start(); + + foreach (var next in nextParallelStream) + { + next.Start(); + } + } + + public override void Wait() + { + base.Wait(); + + foreach (var next in nextParallelStream) + { + next.Wait(); + } + } + + protected override void OnEndStream() + { + foreach (var next in nextParallelStream) + { + next.AddSource(""); + } + } + } +} diff --git a/wbackup/Program.cs b/wbackup/Program.cs index cd15b7b..37bb285 100644 --- a/wbackup/Program.cs +++ b/wbackup/Program.cs @@ -9,271 +9,63 @@ using System.Security.Cryptography.X509Certificates; using System.Numerics; using System.Text; +using System.Threading.Tasks; + using ICSharpCode.SharpZipLib.GZip; using ICSharpCode.SharpZipLib.Tar; -using System.Threading.Tasks; using ICSharpCode.SharpZipLib.BZip2; +using log4net; namespace wbackup { class Program { - public static CommandParser param; - public static int maxThreadLimit = 6; - - class FileInformation - { - public string fileName; - public uint hash; - }; - - static private uint CalclateHash(string fileName) - { - byte[] readbuff = new byte [1024 * 4]; - Array.Clear(readbuff, 0, readbuff.Length); - Vector hashTemp1 = Vector.Zero; - Vector hashTemp2 = Vector.Zero; - using (var fs = File.OpenRead(fileName)) - { - int readLen = fs.Read(readbuff, 0, readbuff.Length); - fs.Close(); - ReadOnlySpan span = new ReadOnlySpan(readbuff); - for (int i = 0; i < readbuff.Length; i += Vector.Count * sizeof(int)) - { - hashTemp2 = new Vector(span.Slice(i, Vector.Count * sizeof(uint))); - hashTemp1 ^= hashTemp2; - } - } - - uint hashValue = 0; - for (int i = 0;i < Vector.Count;i++) - { - hashValue ^= hashTemp1[i]; - } - - return hashValue; - } - static void Main(string[] args) { - var sw = new System.Diagnostics.Stopwatch(); - Mutex consoleLock = new Mutex(); + log4net.Config.XmlConfigurator.Configure(new FileInfo(@"log4net.config")); + + ILog log = LogManager.GetLogger(typeof(Program)); + if (log.IsInfoEnabled) log.Info("Application [wbackup] Start"); - Console.WriteLine("IsHardwareAccelerated = {0}.", Vector.IsHardwareAccelerated); + CommandParser param = new CommandParser(args); - param = new CommandParser(args); + ListingSourceFiles backupEngine = null; + + // 出力先インスタンスを生成する + ParallelBackup parallel = new ParallelBackup(); string postFixString = DateTime.Now.ToString("yyyyMMddHHmmss"); - string[] storedTarget = { ".mov", ".mp4", ".mpg", ".mpeg", ".jpg", ".jpeg", ".png", ".gif", ".zip" }; - - StreamWriter logws = null; - if (param.logFileName != "") + for (int i = 0; i < param.maxThreadCount; i++) { - logws = File.CreateText(param.logFileName + "-" + postFixString + ".log"); + string exportFileName = param.destinationDir + "-" + postFixString + "-" + i.ToString("00"); + parallel.AddNextStream(new ArchiveTarFile(exportFileName, new ClearArchiveFlag())); } - Console.WriteLine("Loading file list."); - List fileNameList = new List(Directory.GetFiles(param.sourceDir, "*.*", SearchOption.AllDirectories )); - - int sourceFileCnt = fileNameList.Count; - int calclatedHashCnt = 0; - var hashAlgorism = SHA256.Create(); - List sortedTemp = new List(); - Parallel.ForEach(fileNameList, new ParallelOptions { MaxDegreeOfParallelism = maxThreadLimit }, fileName => + // バックアップエンジンのインスタンスを生成する + if (param.backupMode == BackupMopde.Full) { - // バックアップ出来なかったファイルを次回処理するため、アーカイブフラグを一度設定する - try - { - var attribute = File.GetAttributes(fileName); - if (param.backupMode == BackupMopde.Full) - { - attribute |= FileAttributes.Archive; - File.SetAttributes(fileName, attribute); - } - - // 圧縮率を高めるためにハッシュ値を取得しファイルをソートする - if (true == attribute.HasFlag(FileAttributes.Archive) && false == attribute.HasFlag(FileAttributes.Hidden)) - { - uint hashTemp = CalclateHash(fileName); - lock (sortedTemp) - { - sortedTemp.Add(new FileInformation() { fileName = fileName, hash = hashTemp }); - } - } - } - catch (Exception exp) - { - consoleLock.WaitOne(); - Console.WriteLine("Skip calclate hash, {0}, {1}", fileName, exp.ToString()); - if (logws != null) - logws.WriteLine("Skip calclate hash, {0}, {1}", fileName, exp.ToString()); - consoleLock.ReleaseMutex(); - } - - int calclatedHashTemp = Interlocked.Increment(ref calclatedHashCnt); - consoleLock.WaitOne(); - Console.Write("\rCalclate Hash. {0}/{1}", calclatedHashTemp, sourceFileCnt); - consoleLock.ReleaseMutex(); - }); - Console.WriteLine("Calclate Hash. {0}/{1}", calclatedHashCnt, sourceFileCnt); - - Console.WriteLine("Sorted file list..."); - List sortedList = new List(sortedTemp.OrderBy((x) => x.hash)); - sortedTemp.Clear(); - - // ハッシュ化したデータを各Threadに振り分ける - int totalFileCount = sortedList.Count; - int archiveFileCount = 0; - int exceptionCount = 0; - - ConcurrentQueue[] srceFilesQueue = new ConcurrentQueue[maxThreadLimit]; - for (int i = 0;i < maxThreadLimit;i++) - { - srceFilesQueue[i] = new ConcurrentQueue(); - int beginIndex = (i * sortedList.Count / maxThreadLimit); - int lastIndex = ((i + 1) * sortedList.Count / maxThreadLimit); - if (maxThreadLimit == (i + 1)) - { - lastIndex = sortedList.Count; - } - - for (int j = beginIndex; j < lastIndex; j++) - { - srceFilesQueue[i].Enqueue(sortedList[j].fileName); - } + //backupEngine = new ListingAllSourceFile(parallel); + backupEngine = new ListingAllSourceFile(new RejectDuplicateFile(param.destinationDir + "-" + postFixString + ".ps1", parallel)); } - sortedList.Clear(); - - sw.Start(); - - Parallel.For(0, maxThreadLimit, i => + if (param.backupMode == BackupMopde.Incremental) { - // アーカイブ先ファイルのインスタンスを作成 - string zipFileName = param.destinationDir + "-" + postFixString + "-" + i.ToString("00") + ".tar.gz"; - using (var fos = new FileStream(zipFileName, FileMode.Create, FileAccess.Write)) - using (GZipOutputStream bzo = new GZipOutputStream(fos)) - using (TarOutputStream zos = new TarOutputStream(bzo, Encoding.UTF8)) - { - bzo.IsStreamOwner = false; - - int buffSelect = 0; - int buffSize = zos.RecordSize; - byte[][] readBuffer = new byte[2][]; - readBuffer[0] = new byte[buffSize]; - readBuffer[1] = new byte[buffSize]; - - while (srceFilesQueue[i].TryDequeue(out string srceFileName)) - { - try - { - if (param.abortedTime.HasValue) - { - if (param.abortedTime < DateTime.Now) - { - break; - } - } - - var attribute = File.GetAttributes(srceFileName); - if (true == attribute.HasFlag(FileAttributes.Archive) && false == attribute.HasFlag(FileAttributes.Hidden)) - { - int archiveFileCountTemp = Interlocked.Increment(ref archiveFileCount); - consoleLock.WaitOne(); - Console.WriteLine("Archive[{1}, {2:0.000}%], {0}", srceFileName, i, 100.0 * archiveFileCountTemp / totalFileCount); - if (logws != null) - logws.WriteLine("Archive[{1}, {2:0.000}%], {0}", srceFileName, i, 100.0 * archiveFileCountTemp / totalFileCount); - consoleLock.ReleaseMutex(); - - // バックアップ対象を出力先に転記する - using (var fis = new FileStream(srceFileName, FileMode.Open, FileAccess.Read)) - { - // ZIPファイルヘッダ作成 - TarEntry tarEntry = TarEntry.CreateEntryFromFile(srceFileName); - zos.PutNextEntry(tarEntry); - - Task readResult = fis.ReadAsync(readBuffer[buffSelect], 0, buffSize); - while (true) - { - readResult.Wait(); - int readSize = readResult.Result; - if (readSize <= 0) - { - break; - } - - Task writeResult = zos.WriteAsync(readBuffer[buffSelect], 0, readSize); - - buffSelect ^= 1; - readResult = fis.ReadAsync(readBuffer[buffSelect], 0, buffSize); - - writeResult.Wait(); - } - - zos.CloseEntry(); - } - - // アーカイブフラグを解除する - if (param.backupMode == BackupMopde.Full || param.backupMode == BackupMopde.Differencial) - { - File.SetAttributes(srceFileName, (attribute & (~FileAttributes.Archive))); - } - } - else - { - // アーカイブフラグがないファイルをスキップする - consoleLock.WaitOne(); - Console.WriteLine("Skip, {0}", srceFileName); - if (logws != null) - logws.WriteLine("Skip, {0}", srceFileName); - consoleLock.ReleaseMutex(); - } - } - catch (Exception exp) - { - srceFilesQueue[i].Enqueue(srceFileName); - - var curtExceptionCnt = Interlocked.Increment(ref exceptionCount); - - consoleLock.WaitOne(); - Console.WriteLine("Exception {0}, {1}", curtExceptionCnt, exp.ToString()); - if (logws != null) - logws.WriteLine("Exception {0}, {1}", curtExceptionCnt, exp.ToString()); - consoleLock.ReleaseMutex(); - } - - if (exceptionCount > param.maxRetryCount) - break; - } - - zos.Flush(); - zos.Close(); - } - }); - - sw.Stop(); - Console.WriteLine("TotalMilliseconds = {0}", sw.Elapsed.TotalMilliseconds); - - // 未処理のファイル数を取得する - int abortedCount = 0; - for (int i = 0; i < maxThreadLimit; i++) + backupEngine = new ListingArchiveSourceFile(new RejectDuplicateFile(param.destinationDir + "-" + postFixString + ".ps1", parallel)); + } + if (param.backupMode == BackupMopde.Differencial) { - abortedCount += srceFilesQueue[i].Count(); + backupEngine = new ListingArchiveSourceFile(new RejectDuplicateFile(param.destinationDir + "-" + postFixString + ".ps1", parallel)); } - Console.WriteLine("Finished {0} archived, {1} skipped, {2} aborted.", archiveFileCount, totalFileCount - archiveFileCount, abortedCount); - if (logws != null) + foreach (var ignoreFile in param.ignoreFile) { - logws.Flush(); - logws.WriteLine("Finished {0} archived, {1} skipped, {2} aborted.", archiveFileCount, totalFileCount - archiveFileCount, abortedCount); - - for (int i = 0; i < maxThreadLimit; i++) - { - foreach (var fileName in srceFilesQueue[i]) - { - logws.WriteLine("aborted, {0}", fileName); - } - } - logws.Close(); + backupEngine.AddIgnoreFolder(ignoreFile); } + + backupEngine.AddSource(param.sourceDir); + backupEngine.AddSource(""); + backupEngine.Start(); + backupEngine.Wait(); + + if (log.IsInfoEnabled) log.Info("Application [wbackup] Stop"); } } } diff --git a/wbackup/Properties/AssemblyInfo.cs b/wbackup/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..12c7905 --- /dev/null +++ b/wbackup/Properties/AssemblyInfo.cs @@ -0,0 +1,63 @@ +#region Apache License +// +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to you under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#endregion + +using System.Reflection; +using System.Runtime.CompilerServices; + +// +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +// + +//[assembly: AssemblyTitle("log4net - ConsoleApp")] +//[assembly: AssemblyDescription("log4net ConsoleApp")] +//[assembly: AssemblyConfiguration("")] +//[assembly: AssemblyProduct("log4net - TestApp")] +//[assembly: AssemblyCulture("")] + +// +// In order to sign your assembly you must specify a key to use. Refer to the +// Microsoft .NET Framework documentation for more information on assembly signing. +// +// Use the attributes below to control which key is used for signing. +// +// Notes: +// (*) If no key is specified, the assembly is not signed. +// (*) KeyName refers to a key that has been installed in the Crypto Service +// Provider (CSP) on your machine. KeyFile refers to a file which contains +// a key. +// (*) If the KeyFile and the KeyName values are both specified, the +// following processing occurs: +// (1) If the KeyName can be found in the CSP, that key is used. +// (2) If the KeyName does not exist and the KeyFile does exist, the key +// in the KeyFile is installed into the CSP and used. +// (*) In order to create a KeyFile, you can use the sn.exe (Strong Name) utility. +// When specifying the KeyFile, the location of the KeyFile should be +// relative to the project output directory which is +// %Project Directory%\obj\. For example, if your KeyFile is +// located in the project directory, you would specify the AssemblyKeyFile +// attribute as [assembly: AssemblyKeyFile("..\\..\\mykey.snk")] +// (*) Delay Signing is an advanced option - see the Microsoft .NET Framework +// documentation for more information on this. +// + +//[assembly: AssemblyDelaySign(false)] +//[assembly: AssemblyKeyFile("")] +//[assembly: AssemblyKeyName("")] diff --git a/wbackup/Properties/launchSettings.json b/wbackup/Properties/launchSettings.json index 7e3aa0f..496cd2d 100644 --- a/wbackup/Properties/launchSettings.json +++ b/wbackup/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "wbackup": { "commandName": "Project", - "commandLineArgs": "-srce \"D:\\Projects\\atozip\" -dest \"D:\\backup\\archive\" -log \"D:\\backup\\archive\" -full -aborttime 13:00" + "commandLineArgs": "-srce \"D:\\Projects\\atozip\" -dest \"D:\\backup\\archive\" -log \"D:\\backup\\archive\" -full" } } } \ No newline at end of file diff --git a/wbackup/RejectDuplicateFile.cs b/wbackup/RejectDuplicateFile.cs new file mode 100644 index 0000000..f5fa511 --- /dev/null +++ b/wbackup/RejectDuplicateFile.cs @@ -0,0 +1,113 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using System.Security.Cryptography; + +namespace wbackup +{ + class DuplicateCheckInfo + { + public string fileName; + public byte[] hashValue; + public long fileSize; + + public ulong HashKey + { + get + { + ulong hashKey = 0; + for (int i = 0; i < sizeof(ulong); i++) + { + hashKey = (hashKey << 8) ^ hashValue[i]; + } + return hashKey; + } + } + + public DuplicateCheckInfo(string fileName) + { + this.fileName = fileName; + var hash = SHA256.Create(); + using (FileStream fs = File.OpenRead(fileName)) + { + fileSize = fs.Length; + if (fileSize > 4096) + { + hash.ComputeHash(fs); + hashValue = hash.Hash; + } + } + } + + public bool IsMatch(DuplicateCheckInfo srce) + { + if (this.fileSize != srce.fileSize) + return false; + + for (int i = 0; i < hashValue.Length; i++) + { + if (hashValue[i] != srce.hashValue[i]) + return false; + } + + return true; + } + } + + class RejectDuplicateFile : BackupStreamBase + { + protected string recoveryScriptName; + protected Dictionary hashData; + + public RejectDuplicateFile(string scriptName, BackupStreamBase stream = null) : base(stream) + { + recoveryScriptName = scriptName; + hashData = new Dictionary(); + } + + protected override void DoWork(string sourceFileName) + { + if (sourceFileName == "") + AddNextSource(sourceFileName); + + DuplicateCheckInfo info = new DuplicateCheckInfo(sourceFileName); + if (info.fileSize > 4096) + { + ulong hashValue = info.HashKey; + if (hashData.TryGetValue(hashValue, out DuplicateCheckInfo srceInfo)) + { + if (srceInfo.IsMatch(info)) + { + Console.WriteLine("Duplicate {0}, {1}", sourceFileName, srceInfo.fileName); + + StringBuilder script = new StringBuilder(); + string destDir = sourceFileName.Substring(0, sourceFileName.LastIndexOf("\\") + 1); + script.AppendFormat("New-Item -Path \"{0}\" -ItemType Directory", destDir); + script.AppendLine(); + script.AppendFormat("Copy-Item -Path \"{0}\" -Destination \"{1}\"", srceInfo.fileName, sourceFileName); + script.AppendLine(); + script.AppendFormat("Set-ItemProperty -Path \"{0}\" -Name LastWriteTime -Value \"{1}\"", sourceFileName, File.GetLastWriteTime(sourceFileName).ToString("yyyy-MM-dd HH:mm:ss")); + script.AppendLine(); + File.AppendAllText(recoveryScriptName, script.ToString()); + } + else + { + AddNextSource(sourceFileName); + } + } + else + { + hashData.Add(hashValue, info); + AddNextSource(sourceFileName); + } + } + else + { + AddNextSource(sourceFileName); + } + } + } +} diff --git a/wbackup/log4net.config b/wbackup/log4net.config new file mode 100644 index 0000000..5e004d6 --- /dev/null +++ b/wbackup/log4net.config @@ -0,0 +1,90 @@ + + + + + + + + + + + + + + + + +
+