diff --git a/EternalArrowBackup.sln b/EternalArrowBackup.sln index a27fe40..d56fb93 100644 --- a/EternalArrowBackup.sln +++ b/EternalArrowBackup.sln @@ -47,6 +47,12 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ContentTransformer", "Conte EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Hasher", "Hasher", "{0059A121-D191-48FE-9215-F71CE1057EEB}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EternalArrowBackup.SourceStorage.InMemorySourceStorage", "Source\SourceStorage\InMemorySourceStorage\EternalArrowBackup.SourceStorage.InMemorySourceStorage.csproj", "{B8F0A6CF-6D26-4A38-B6AF-30BB4BE17380}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SourceStorage", "SourceStorage", "{97B894EA-9FAF-4C13-A20E-00A6CC6108C5}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EternalArrowBackup.SourceStorage.InMemorySourceStorage.Tests", "Tests\SourceStorage\InMemorySourceStorage\EternalArrowBackup.SourceStorage.InMemorySourceStorage.Tests.csproj", "{8B5C507A-C6FE-46BA-BB4F-4876B8230E26}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -97,6 +103,14 @@ Global {ACDA1912-B9A4-4C88-9C30-4C0ADB907FC7}.Debug|Any CPU.Build.0 = Debug|Any CPU {ACDA1912-B9A4-4C88-9C30-4C0ADB907FC7}.Release|Any CPU.ActiveCfg = Release|Any CPU {ACDA1912-B9A4-4C88-9C30-4C0ADB907FC7}.Release|Any CPU.Build.0 = Release|Any CPU + {B8F0A6CF-6D26-4A38-B6AF-30BB4BE17380}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B8F0A6CF-6D26-4A38-B6AF-30BB4BE17380}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B8F0A6CF-6D26-4A38-B6AF-30BB4BE17380}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B8F0A6CF-6D26-4A38-B6AF-30BB4BE17380}.Release|Any CPU.Build.0 = Release|Any CPU + {8B5C507A-C6FE-46BA-BB4F-4876B8230E26}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8B5C507A-C6FE-46BA-BB4F-4876B8230E26}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8B5C507A-C6FE-46BA-BB4F-4876B8230E26}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8B5C507A-C6FE-46BA-BB4F-4876B8230E26}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -122,5 +136,8 @@ Global {ACDA1912-B9A4-4C88-9C30-4C0ADB907FC7} = {FF7B2ECB-6EE0-4A60-8893-2F27CE9034A7} {8D284ABE-FDF1-4E01-832B-8918798E3FA5} = {884B8E01-303A-40CF-8884-D62115F98683} {0059A121-D191-48FE-9215-F71CE1057EEB} = {884B8E01-303A-40CF-8884-D62115F98683} + {B8F0A6CF-6D26-4A38-B6AF-30BB4BE17380} = {05426D05-AE66-4C5C-8B8E-403A46ADC49D} + {97B894EA-9FAF-4C13-A20E-00A6CC6108C5} = {884B8E01-303A-40CF-8884-D62115F98683} + {8B5C507A-C6FE-46BA-BB4F-4876B8230E26} = {97B894EA-9FAF-4C13-A20E-00A6CC6108C5} EndGlobalSection EndGlobal diff --git a/Source/SourceStorage/Contracts/EternalArrowBackup.SourceStorage.Contracts.csproj b/Source/SourceStorage/Contracts/EternalArrowBackup.SourceStorage.Contracts.csproj index 954020d..0a822af 100644 --- a/Source/SourceStorage/Contracts/EternalArrowBackup.SourceStorage.Contracts.csproj +++ b/Source/SourceStorage/Contracts/EternalArrowBackup.SourceStorage.Contracts.csproj @@ -1,7 +1,11 @@ - + netstandard1.4 + + + + \ No newline at end of file diff --git a/Source/SourceStorage/Contracts/ISourceDirectory.cs b/Source/SourceStorage/Contracts/ISourceDirectory.cs index 24bd7b1..63e7cc1 100644 --- a/Source/SourceStorage/Contracts/ISourceDirectory.cs +++ b/Source/SourceStorage/Contracts/ISourceDirectory.cs @@ -1,8 +1,8 @@ namespace EternalArrowBackup.SourceStorage.Contracts { - using System; using System.Threading; using System.Threading.Tasks; + using System.Threading.Tasks.Dataflow; public interface ISourceDirectory { @@ -10,6 +10,6 @@ Task GetFile(string filename); - IObservable GetAllFiles(CancellationToken ct); + Task GetAllFiles(ActionBlock actionBlock, CancellationToken ct); } } diff --git a/Source/SourceStorage/Contracts/ISourceStorage.cs b/Source/SourceStorage/Contracts/ISourceStorage.cs index 1888ccc..94fc536 100644 --- a/Source/SourceStorage/Contracts/ISourceStorage.cs +++ b/Source/SourceStorage/Contracts/ISourceStorage.cs @@ -1,13 +1,13 @@ namespace EternalArrowBackup.SourceStorage.Contracts { - using System; using System.Threading; using System.Threading.Tasks; + using System.Threading.Tasks.Dataflow; public interface ISourceStorage { Task GetDirectory(string normalizedRelativePath); - IObservable GetAllDirectories(CancellationToken ct); + Task GetAllDirectories(ActionBlock actionBlock, CancellationToken ct); } } diff --git a/Source/SourceStorage/InMemorySourceStorage/EternalArrowBackup.SourceStorage.InMemorySourceStorage.csproj b/Source/SourceStorage/InMemorySourceStorage/EternalArrowBackup.SourceStorage.InMemorySourceStorage.csproj new file mode 100644 index 0000000..04d2c2c --- /dev/null +++ b/Source/SourceStorage/InMemorySourceStorage/EternalArrowBackup.SourceStorage.InMemorySourceStorage.csproj @@ -0,0 +1,16 @@ + + + + netstandard1.4 + + + + + + + + + + + + \ No newline at end of file diff --git a/Source/SourceStorage/InMemorySourceStorage/SourceDirectory.cs b/Source/SourceStorage/InMemorySourceStorage/SourceDirectory.cs new file mode 100644 index 0000000..8c617bb --- /dev/null +++ b/Source/SourceStorage/InMemorySourceStorage/SourceDirectory.cs @@ -0,0 +1,44 @@ +namespace EternalArrowBackup.SourceStorage.InMemorySourceStorage +{ + using System.Collections.Immutable; + using System.Threading; + using System.Threading.Tasks; + using System.Threading.Tasks.Dataflow; + using EternalArrowBackup.SourceStorage.Contracts; + + internal class SourceDirectory : ISourceDirectory + { + public SourceDirectory(string path, ImmutableDictionary storageData) + { + this.NormalizedRelativePath = path; + this.StorageData = storageData; + } + + public string NormalizedRelativePath { get; } + + private ImmutableDictionary StorageData { get; } + + public Task GetAllFiles(ActionBlock actionBlock, CancellationToken ct) + { + return Task.Run(() => + { + foreach (var kvp in this.StorageData) + { + if (ct.IsCancellationRequested) + { + break; + } + + actionBlock.Post(new SourceFile(kvp.Key, kvp.Value)); + } + + actionBlock.Complete(); + }); + } + + public Task GetFile(string filename) + { + return Task.Run(() => (ISourceFile)new SourceFile(filename, this.StorageData[filename])); + } + } +} diff --git a/Source/SourceStorage/InMemorySourceStorage/SourceFile.cs b/Source/SourceStorage/InMemorySourceStorage/SourceFile.cs new file mode 100644 index 0000000..9e61c61 --- /dev/null +++ b/Source/SourceStorage/InMemorySourceStorage/SourceFile.cs @@ -0,0 +1,26 @@ +namespace EternalArrowBackup.SourceStorage.InMemorySourceStorage +{ + using System.Threading.Tasks; + using EternalArrowBackup.SourceStorage.Contracts; + + internal class SourceFile : ISourceFile + { + public SourceFile(string filename, byte[] contents) + { + this.Filename = filename; + this.Size = contents.Length; + this.Contents = contents; + } + + public string Filename { get; } + + public long Size { get; } + + private byte[] Contents { get; } + + public Task ReadContents() + { + return Task.Run(() => this.Contents); + } + } +} diff --git a/Source/SourceStorage/InMemorySourceStorage/SourceStorage.cs b/Source/SourceStorage/InMemorySourceStorage/SourceStorage.cs new file mode 100644 index 0000000..f6e2660 --- /dev/null +++ b/Source/SourceStorage/InMemorySourceStorage/SourceStorage.cs @@ -0,0 +1,42 @@ +namespace EternalArrowBackup.SourceStorage.InMemorySourceStorage +{ + using System; + using System.Collections.Immutable; + using System.Threading; + using System.Threading.Tasks; + using System.Threading.Tasks.Dataflow; + using EternalArrowBackup.SourceStorage.Contracts; + + public class SourceStorage : ISourceStorage + { + public SourceStorage(ImmutableDictionary> storageData) + { + this.StorageData = storageData; + } + + private ImmutableDictionary> StorageData { get; } + + public Task GetAllDirectories(ActionBlock actionBlock, CancellationToken ct) + { + return Task.Run(() => + { + foreach (var kvp in this.StorageData) + { + if (ct.IsCancellationRequested) + { + break; + } + + actionBlock.Post(new SourceDirectory(kvp.Key, kvp.Value)); + } + + actionBlock.Complete(); + }); + } + + public Task GetDirectory(string normalizedRelativePath) + { + return Task.Run(() => (ISourceDirectory)new SourceDirectory(normalizedRelativePath, this.StorageData[normalizedRelativePath])); + } + } +} diff --git a/Tests/SourceStorage/InMemorySourceStorage/EternalArrowBackup.SourceStorage.InMemorySourceStorage.Tests.csproj b/Tests/SourceStorage/InMemorySourceStorage/EternalArrowBackup.SourceStorage.InMemorySourceStorage.Tests.csproj new file mode 100644 index 0000000..60cee75 --- /dev/null +++ b/Tests/SourceStorage/InMemorySourceStorage/EternalArrowBackup.SourceStorage.InMemorySourceStorage.Tests.csproj @@ -0,0 +1,24 @@ + + + + netcoreapp1.1 + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Tests/SourceStorage/InMemorySourceStorage/StorageTests.cs b/Tests/SourceStorage/InMemorySourceStorage/StorageTests.cs new file mode 100644 index 0000000..ee9ee59 --- /dev/null +++ b/Tests/SourceStorage/InMemorySourceStorage/StorageTests.cs @@ -0,0 +1,110 @@ +namespace EternalArrowBackup.SourceStorage.InMemorySourceStorage.Tests +{ + using System; + using System.Collections.Generic; + using System.Collections.Immutable; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using System.Threading.Tasks.Dataflow; + using EternalArrowBackup.SourceStorage.Contracts; + using Xunit; + + public static class StorageTests + { + [Fact] + public static async Task CheckStorage() + { + var storageData = CreateStorageData(); + + var immutableStorageData = storageData + .Select(kvp => new KeyValuePair>(kvp.Key, kvp.Value.ToImmutableDictionary())) + .ToImmutableDictionary(); + var storage = new SourceStorage(immutableStorageData); + + var testDirectoryName = storageData.Keys.Skip(storageData.Count / 2).First(); + var testDirectoryInfo = await storage.GetDirectory(testDirectoryName); + Assert.Equal(testDirectoryName, testDirectoryInfo.NormalizedRelativePath); + await CheckDirectory(storageData[testDirectoryName].ToDictionary(kvp => kvp.Key, kvp => kvp.Value), testDirectoryInfo); + + var syncObject = new object(); + var actionBlock = new ActionBlock(async sourceDirectory => + { + Dictionary originalDirectoryData; + + lock (syncObject) + { + Assert.True(storageData.ContainsKey(sourceDirectory.NormalizedRelativePath), "Directory does not exist in original data"); + originalDirectoryData = storageData[sourceDirectory.NormalizedRelativePath]; + storageData.Remove(sourceDirectory.NormalizedRelativePath); + } + + await CheckDirectory(originalDirectoryData, sourceDirectory); + }); + + await storage.GetAllDirectories(actionBlock, CancellationToken.None); + await actionBlock.Completion; + + Assert.False(storageData.Any(), "Not all directories were enumerated"); + } + + private static async Task CheckDirectory(Dictionary originalDirectory, ISourceDirectory contractsDirectory) + { + if (originalDirectory.Count > 0) + { + var testFileName = originalDirectory.Keys.Skip(originalDirectory.Count / 2).First(); + var testFileInfo = await contractsDirectory.GetFile(testFileName); + Assert.Equal(testFileName, testFileInfo.Filename); + Assert.Equal(originalDirectory[testFileName].Length, testFileInfo.Size); + Assert.Equal(originalDirectory[testFileName], await testFileInfo.ReadContents()); + } + + var syncObject = new object(); + var actionBlock = new ActionBlock(async sourceFile => + { + byte[] originalFileContents; + lock (syncObject) + { + Assert.True(originalDirectory.ContainsKey(sourceFile.Filename), "File does not exist in original data"); + originalFileContents = originalDirectory[sourceFile.Filename]; + originalDirectory.Remove(sourceFile.Filename); + } + + Assert.Equal(originalFileContents.Length, sourceFile.Size); + Assert.Equal(originalFileContents, await sourceFile.ReadContents()); + }); + + await contractsDirectory.GetAllFiles(actionBlock, CancellationToken.None); + await actionBlock.Completion; + + Assert.False(originalDirectory.Any(), "Not all files were enumerated"); + } + + private static Dictionary> CreateStorageData() + { + var random = new Random(); + var result = new Dictionary>(); + for (var i = 0; i < 2000; i++) + { + var directoryName = Guid.NewGuid().ToString(); + var directoryContent = new Dictionary(); + var directorySize = random.Next(0, 500); + for (var j = 0; j < directorySize; j++) + { + var fileName = Guid.NewGuid().ToString(); + var fileContent = new byte[random.Next(0, 20)]; + for (var k = 0; k < fileContent.Length; k++) + { + fileContent[k] = (byte)random.Next(0, 255); + } + + directoryContent[fileName] = fileContent; + } + + result[directoryName] = directoryContent; + } + + return result; + } + } +}