diff --git a/wbackup/ArchiveTarFile.cs b/wbackup/ArchiveTarFile.cs new file mode 100644 index 0000000..b58c566 --- /dev/null +++ b/wbackup/ArchiveTarFile.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.IO; +using System.Threading.Tasks; +using ICSharpCode.SharpZipLib.GZip; +using ICSharpCode.SharpZipLib.Tar; +using ICSharpCode.SharpZipLib.BZip2; + +namespace wbackup +{ + class ArchiveTarFile : BackupStreamBase + { + protected FileStream fileOutputStream; + protected GZipOutputStream gzipOutputStream; + protected TarOutputStream tarOutputStream; + + protected int buffSelect; + protected int buffSize; + protected byte[][] readBuffer; + + public ArchiveTarFile(string exportPath, BackupStreamBase stream = null):base(stream) + { + string postFixString = DateTime.Now.ToString("yyyyMMddHHmmss"); + string zipFileName = exportPath + postFixString + ".tar.gz"; + + // 出力先ストリームのインスタンス生成 + fileOutputStream = new FileStream(zipFileName, FileMode.Create, FileAccess.Write); + gzipOutputStream = new GZipOutputStream(fileOutputStream); + tarOutputStream = new TarOutputStream(gzipOutputStream, Encoding.UTF8); + gzipOutputStream.IsStreamOwner = false; + + // ソース読込のためのバッファ生成 + buffSelect = 0; + buffSize = tarOutputStream.RecordSize; + readBuffer = new byte[2][]; + readBuffer[0] = new byte[buffSize]; + readBuffer[1] = new byte[buffSize]; + } + + protected override void DisposeManagedResource() + { + tarOutputStream.Flush(); + tarOutputStream.Close(); + } + + protected override void DoWork(string sourceFileName) + { + using (var fis = new FileStream(sourceFileName, FileMode.Open, FileAccess.Read)) + { + // ZIPファイルヘッダ作成 + TarEntry tarEntry = TarEntry.CreateEntryFromFile(sourceFileName); + tarOutputStream.PutNextEntry(tarEntry); + + Task readResult = fis.ReadAsync(readBuffer[buffSelect], 0, buffSize); + while (true) + { + readResult.Wait(); + int readSize = readResult.Result; + if (readSize <= 0) + { + break; + } + + Task writeResult = tarOutputStream.WriteAsync(readBuffer[buffSelect], 0, readSize); + + buffSelect ^= 1; + readResult = fis.ReadAsync(readBuffer[buffSelect], 0, buffSize); + + writeResult.Wait(); + } + + tarOutputStream.CloseEntry(); + } + } + } +} diff --git a/wbackup/BackupStreamBase.cs b/wbackup/BackupStreamBase.cs index 4c1c465..1e2c0e6 100644 --- a/wbackup/BackupStreamBase.cs +++ b/wbackup/BackupStreamBase.cs @@ -6,12 +6,14 @@ namespace wbackup { - class BackupStreamBase + class BackupStreamBase : IDisposable { protected bool working; protected BackupStreamBase nextStream = null; protected ConcurrentQueue sourceFileQueue = null; protected Thread threadInstance = null; + + private bool _disposed = false; public BackupStreamBase(BackupStreamBase stream = null) { @@ -20,20 +22,83 @@ threadInstance = new Thread(this.DoWork); } + ~BackupStreamBase() + { + Dispose(false); + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (!_disposed) + { + if (disposing) + { + // TODO: Dispose managed resources here. + DisposeManagedResource(); + } + + // TODO: Free unmanaged resources here. + DisposeUnmanagedResource(); + + // Note disposing has been done. + _disposed = true; + } + } + + protected virtual void DisposeManagedResource() + { + + } + + protected virtual void DisposeUnmanagedResource() + { + + } + /// /// バックアップ対象フォルダを追加する /// /// - public void AddSource(string newFolder) + public void AddSource(string newSource) { - sourceFileQueue.Enqueue(newFolder); + sourceFileQueue.Enqueue(newSource); } + /// + /// 次のインスタンスのソースファイルに追加する + /// + /// + public void AddNextSource(string newSource) + { + if (null != nextStream) + { + nextStream.AddSource(newSource); + } + } + + /// + /// Threadを開始する + /// public void Start() { threadInstance.Start(); + + if (null != nextStream) + { + nextStream.Start(); + } } + /// + /// ソースからデータを取得して処理する + /// + /// public void DoWork(object data) { working = true; @@ -50,7 +115,7 @@ // これ以上処理するデータが無いことを示す if (sourceFileName == "") { - nextStream.AddSource(""); + AddNextSource(""); break; } @@ -60,7 +125,11 @@ working = false; } - public virtual void DoWork(string sourceFileName) + /// + /// ソースから取得したファイルを処理する + /// + /// + protected virtual void DoWork(string sourceFileName) { } diff --git a/wbackup/ListingAllSourceFile.cs b/wbackup/ListingAllSourceFile.cs index c9ffbd6..2c2beb2 100644 --- a/wbackup/ListingAllSourceFile.cs +++ b/wbackup/ListingAllSourceFile.cs @@ -28,7 +28,7 @@ ignoreFolder.Add(newFolder); } - public override void DoWork(string sourceFileName) + protected override void DoWork(string sourceFileName) { ListingSourceFiles(sourceFileName); } @@ -44,8 +44,11 @@ if (false == isIgnoreFile(fileName)) { var fileAttrib = File.GetAttributes(fileName); - File.SetAttributes(fileName, fileAttrib | FileAttributes.Archive); - nextStream.AddSource(fileName); + if (false == fileAttrib.HasFlag(FileAttributes.Hidden)) + { + File.SetAttributes(fileName, fileAttrib | FileAttributes.Archive); + AddNextSource(fileName); + } } } diff --git a/wbackup/Program.cs b/wbackup/Program.cs index cd15b7b..4db4e9e 100644 --- a/wbackup/Program.cs +++ b/wbackup/Program.cs @@ -18,262 +18,16 @@ { class Program { - public static CommandParser param; - public static int maxThreadLimit = 6; - - class FileInformation - { - public string fileName; - public uint hash; - }; - - static private uint CalclateHash(string fileName) - { - byte[] readbuff = new byte [1024 * 4]; - Array.Clear(readbuff, 0, readbuff.Length); - Vector hashTemp1 = Vector.Zero; - Vector hashTemp2 = Vector.Zero; - using (var fs = File.OpenRead(fileName)) - { - int readLen = fs.Read(readbuff, 0, readbuff.Length); - fs.Close(); - ReadOnlySpan span = new ReadOnlySpan(readbuff); - for (int i = 0; i < readbuff.Length; i += Vector.Count * sizeof(int)) - { - hashTemp2 = new Vector(span.Slice(i, Vector.Count * sizeof(uint))); - hashTemp1 ^= hashTemp2; - } - } - - uint hashValue = 0; - for (int i = 0;i < Vector.Count;i++) - { - hashValue ^= hashTemp1[i]; - } - - return hashValue; - } - static void Main(string[] args) { - var sw = new System.Diagnostics.Stopwatch(); - Mutex consoleLock = new Mutex(); - Console.WriteLine("IsHardwareAccelerated = {0}.", Vector.IsHardwareAccelerated); - param = new CommandParser(args); - string postFixString = DateTime.Now.ToString("yyyyMMddHHmmss"); - string[] storedTarget = { ".mov", ".mp4", ".mpg", ".mpeg", ".jpg", ".jpeg", ".png", ".gif", ".zip" }; + CommandParser param = new CommandParser(args); - StreamWriter logws = null; - if (param.logFileName != "") - { - logws = File.CreateText(param.logFileName + "-" + postFixString + ".log"); - } - - Console.WriteLine("Loading file list."); - List fileNameList = new List(Directory.GetFiles(param.sourceDir, "*.*", SearchOption.AllDirectories )); - - int sourceFileCnt = fileNameList.Count; - int calclatedHashCnt = 0; - var hashAlgorism = SHA256.Create(); - List sortedTemp = new List(); - Parallel.ForEach(fileNameList, new ParallelOptions { MaxDegreeOfParallelism = maxThreadLimit }, fileName => - { - // バックアップ出来なかったファイルを次回処理するため、アーカイブフラグを一度設定する - try - { - var attribute = File.GetAttributes(fileName); - if (param.backupMode == BackupMopde.Full) - { - attribute |= FileAttributes.Archive; - File.SetAttributes(fileName, attribute); - } - - // 圧縮率を高めるためにハッシュ値を取得しファイルをソートする - if (true == attribute.HasFlag(FileAttributes.Archive) && false == attribute.HasFlag(FileAttributes.Hidden)) - { - uint hashTemp = CalclateHash(fileName); - lock (sortedTemp) - { - sortedTemp.Add(new FileInformation() { fileName = fileName, hash = hashTemp }); - } - } - } - catch (Exception exp) - { - consoleLock.WaitOne(); - Console.WriteLine("Skip calclate hash, {0}, {1}", fileName, exp.ToString()); - if (logws != null) - logws.WriteLine("Skip calclate hash, {0}, {1}", fileName, exp.ToString()); - consoleLock.ReleaseMutex(); - } - - int calclatedHashTemp = Interlocked.Increment(ref calclatedHashCnt); - consoleLock.WaitOne(); - Console.Write("\rCalclate Hash. {0}/{1}", calclatedHashTemp, sourceFileCnt); - consoleLock.ReleaseMutex(); - }); - Console.WriteLine("Calclate Hash. {0}/{1}", calclatedHashCnt, sourceFileCnt); - - Console.WriteLine("Sorted file list..."); - List sortedList = new List(sortedTemp.OrderBy((x) => x.hash)); - sortedTemp.Clear(); - - // ハッシュ化したデータを各Threadに振り分ける - int totalFileCount = sortedList.Count; - int archiveFileCount = 0; - int exceptionCount = 0; - - ConcurrentQueue[] srceFilesQueue = new ConcurrentQueue[maxThreadLimit]; - for (int i = 0;i < maxThreadLimit;i++) - { - srceFilesQueue[i] = new ConcurrentQueue(); - int beginIndex = (i * sortedList.Count / maxThreadLimit); - int lastIndex = ((i + 1) * sortedList.Count / maxThreadLimit); - if (maxThreadLimit == (i + 1)) - { - lastIndex = sortedList.Count; - } - - for (int j = beginIndex; j < lastIndex; j++) - { - srceFilesQueue[i].Enqueue(sortedList[j].fileName); - } - } - sortedList.Clear(); - - sw.Start(); - - Parallel.For(0, maxThreadLimit, i => - { - // アーカイブ先ファイルのインスタンスを作成 - string zipFileName = param.destinationDir + "-" + postFixString + "-" + i.ToString("00") + ".tar.gz"; - using (var fos = new FileStream(zipFileName, FileMode.Create, FileAccess.Write)) - using (GZipOutputStream bzo = new GZipOutputStream(fos)) - using (TarOutputStream zos = new TarOutputStream(bzo, Encoding.UTF8)) - { - bzo.IsStreamOwner = false; - - int buffSelect = 0; - int buffSize = zos.RecordSize; - byte[][] readBuffer = new byte[2][]; - readBuffer[0] = new byte[buffSize]; - readBuffer[1] = new byte[buffSize]; - - while (srceFilesQueue[i].TryDequeue(out string srceFileName)) - { - try - { - if (param.abortedTime.HasValue) - { - if (param.abortedTime < DateTime.Now) - { - break; - } - } - - var attribute = File.GetAttributes(srceFileName); - if (true == attribute.HasFlag(FileAttributes.Archive) && false == attribute.HasFlag(FileAttributes.Hidden)) - { - int archiveFileCountTemp = Interlocked.Increment(ref archiveFileCount); - consoleLock.WaitOne(); - Console.WriteLine("Archive[{1}, {2:0.000}%], {0}", srceFileName, i, 100.0 * archiveFileCountTemp / totalFileCount); - if (logws != null) - logws.WriteLine("Archive[{1}, {2:0.000}%], {0}", srceFileName, i, 100.0 * archiveFileCountTemp / totalFileCount); - consoleLock.ReleaseMutex(); - - // バックアップ対象を出力先に転記する - using (var fis = new FileStream(srceFileName, FileMode.Open, FileAccess.Read)) - { - // ZIPファイルヘッダ作成 - TarEntry tarEntry = TarEntry.CreateEntryFromFile(srceFileName); - zos.PutNextEntry(tarEntry); - - Task readResult = fis.ReadAsync(readBuffer[buffSelect], 0, buffSize); - while (true) - { - readResult.Wait(); - int readSize = readResult.Result; - if (readSize <= 0) - { - break; - } - - Task writeResult = zos.WriteAsync(readBuffer[buffSelect], 0, readSize); - - buffSelect ^= 1; - readResult = fis.ReadAsync(readBuffer[buffSelect], 0, buffSize); - - writeResult.Wait(); - } - - zos.CloseEntry(); - } - - // アーカイブフラグを解除する - if (param.backupMode == BackupMopde.Full || param.backupMode == BackupMopde.Differencial) - { - File.SetAttributes(srceFileName, (attribute & (~FileAttributes.Archive))); - } - } - else - { - // アーカイブフラグがないファイルをスキップする - consoleLock.WaitOne(); - Console.WriteLine("Skip, {0}", srceFileName); - if (logws != null) - logws.WriteLine("Skip, {0}", srceFileName); - consoleLock.ReleaseMutex(); - } - } - catch (Exception exp) - { - srceFilesQueue[i].Enqueue(srceFileName); - - var curtExceptionCnt = Interlocked.Increment(ref exceptionCount); - - consoleLock.WaitOne(); - Console.WriteLine("Exception {0}, {1}", curtExceptionCnt, exp.ToString()); - if (logws != null) - logws.WriteLine("Exception {0}, {1}", curtExceptionCnt, exp.ToString()); - consoleLock.ReleaseMutex(); - } - - if (exceptionCount > param.maxRetryCount) - break; - } - - zos.Flush(); - zos.Close(); - } - }); - - sw.Stop(); - Console.WriteLine("TotalMilliseconds = {0}", sw.Elapsed.TotalMilliseconds); - - // 未処理のファイル数を取得する - int abortedCount = 0; - for (int i = 0; i < maxThreadLimit; i++) - { - abortedCount += srceFilesQueue[i].Count(); - } - - Console.WriteLine("Finished {0} archived, {1} skipped, {2} aborted.", archiveFileCount, totalFileCount - archiveFileCount, abortedCount); - if (logws != null) - { - logws.Flush(); - logws.WriteLine("Finished {0} archived, {1} skipped, {2} aborted.", archiveFileCount, totalFileCount - archiveFileCount, abortedCount); - - for (int i = 0; i < maxThreadLimit; i++) - { - foreach (var fileName in srceFilesQueue[i]) - { - logws.WriteLine("aborted, {0}", fileName); - } - } - logws.Close(); - } + ListingAllSourceFile backupEngine = new ListingAllSourceFile(new ArchiveTarFile(param.destinationDir)); + backupEngine.AddSource(param.sourceDir); + backupEngine.AddSource(""); + backupEngine.Start();　 } } }