Implemented InMemorySourceStorage

EternalArrowBackup.sln

@@ -47,6 +47,12 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ContentTransformer", "Conte
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Hasher", "Hasher", "{0059A121-D191-48FE-9215-F71CE1057EEB}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EternalArrowBackup.SourceStorage.InMemorySourceStorage", "Source\SourceStorage\InMemorySourceStorage\EternalArrowBackup.SourceStorage.InMemorySourceStorage.csproj", "{B8F0A6CF-6D26-4A38-B6AF-30BB4BE17380}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SourceStorage", "SourceStorage", "{97B894EA-9FAF-4C13-A20E-00A6CC6108C5}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EternalArrowBackup.SourceStorage.InMemorySourceStorage.Tests", "Tests\SourceStorage\InMemorySourceStorage\EternalArrowBackup.SourceStorage.InMemorySourceStorage.Tests.csproj", "{8B5C507A-C6FE-46BA-BB4F-4876B8230E26}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -97,6 +103,14 @@ Global
 		{ACDA1912-B9A4-4C88-9C30-4C0ADB907FC7}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{ACDA1912-B9A4-4C88-9C30-4C0ADB907FC7}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{ACDA1912-B9A4-4C88-9C30-4C0ADB907FC7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B8F0A6CF-6D26-4A38-B6AF-30BB4BE17380}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B8F0A6CF-6D26-4A38-B6AF-30BB4BE17380}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B8F0A6CF-6D26-4A38-B6AF-30BB4BE17380}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B8F0A6CF-6D26-4A38-B6AF-30BB4BE17380}.Release|Any CPU.Build.0 = Release|Any CPU
+		{8B5C507A-C6FE-46BA-BB4F-4876B8230E26}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{8B5C507A-C6FE-46BA-BB4F-4876B8230E26}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{8B5C507A-C6FE-46BA-BB4F-4876B8230E26}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{8B5C507A-C6FE-46BA-BB4F-4876B8230E26}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -122,5 +136,8 @@ Global
 		{ACDA1912-B9A4-4C88-9C30-4C0ADB907FC7} = {FF7B2ECB-6EE0-4A60-8893-2F27CE9034A7}
 		{8D284ABE-FDF1-4E01-832B-8918798E3FA5} = {884B8E01-303A-40CF-8884-D62115F98683}
 		{0059A121-D191-48FE-9215-F71CE1057EEB} = {884B8E01-303A-40CF-8884-D62115F98683}
+		{B8F0A6CF-6D26-4A38-B6AF-30BB4BE17380} = {05426D05-AE66-4C5C-8B8E-403A46ADC49D}
+		{97B894EA-9FAF-4C13-A20E-00A6CC6108C5} = {884B8E01-303A-40CF-8884-D62115F98683}
+		{8B5C507A-C6FE-46BA-BB4F-4876B8230E26} = {97B894EA-9FAF-4C13-A20E-00A6CC6108C5}
 	EndGlobalSection
 EndGlobal

Source/SourceStorage/Contracts/EternalArrowBackup.SourceStorage.Contracts.csproj

@@ -1,7 +1,11 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>netstandard1.4</TargetFramework>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="System.Threading.Tasks.Dataflow" Version="4.7.0" />
+  </ItemGroup>
+
 </Project>

Source/SourceStorage/Contracts/ISourceDirectory.cs

@@ -1,8 +1,8 @@
 namespace EternalArrowBackup.SourceStorage.Contracts
 {
-    using System;
     using System.Threading;
     using System.Threading.Tasks;
+    using System.Threading.Tasks.Dataflow;
 
     public interface ISourceDirectory
     {
@@ -10,6 +10,6 @@
 
         Task<ISourceFile> GetFile(string filename);
 
-        IObservable<ISourceFile> GetAllFiles(CancellationToken ct);
+        Task GetAllFiles(ActionBlock<ISourceFile> actionBlock, CancellationToken ct);
     }
 }

Source/SourceStorage/Contracts/ISourceStorage.cs

@@ -1,13 +1,13 @@
 namespace EternalArrowBackup.SourceStorage.Contracts
 {
-    using System;
     using System.Threading;
     using System.Threading.Tasks;
+    using System.Threading.Tasks.Dataflow;
 
     public interface ISourceStorage
     {
         Task<ISourceDirectory> GetDirectory(string normalizedRelativePath);
 
-        IObservable<ISourceDirectory> GetAllDirectories(CancellationToken ct);
+        Task GetAllDirectories(ActionBlock<ISourceDirectory> actionBlock, CancellationToken ct);
     }
 }

Source/SourceStorage/InMemorySourceStorage/EternalArrowBackup.SourceStorage.InMemorySourceStorage.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netstandard1.4</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="System.Collections.Immutable" Version="1.3.1" />
+    <PackageReference Include="System.Threading.Tasks.Dataflow" Version="4.7.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Contracts\EternalArrowBackup.SourceStorage.Contracts.csproj" />
+  </ItemGroup>
+
+</Project>

Source/SourceStorage/InMemorySourceStorage/SourceDirectory.cs

@@ -0,0 +1,44 @@
+namespace EternalArrowBackup.SourceStorage.InMemorySourceStorage
+{
+    using System.Collections.Immutable;
+    using System.Threading;
+    using System.Threading.Tasks;
+    using System.Threading.Tasks.Dataflow;
+    using EternalArrowBackup.SourceStorage.Contracts;
+
+    internal class SourceDirectory : ISourceDirectory
+    {
+        public SourceDirectory(string path, ImmutableDictionary<string, byte[]> storageData)
+        {
+            this.NormalizedRelativePath = path;
+            this.StorageData = storageData;
+        }
+
+        public string NormalizedRelativePath { get; }
+
+        private ImmutableDictionary<string, byte[]> StorageData { get; }
+
+        public Task GetAllFiles(ActionBlock<ISourceFile> actionBlock, CancellationToken ct)
+        {
+            return Task.Run(() =>
+            {
+                foreach (var kvp in this.StorageData)
+                {
+                    if (ct.IsCancellationRequested)
+                    {
+                        break;
+                    }
+
+                    actionBlock.Post(new SourceFile(kvp.Key, kvp.Value));
+                }
+
+                actionBlock.Complete();
+            });
+        }
+
+        public Task<ISourceFile> GetFile(string filename)
+        {
+            return Task.Run(() => (ISourceFile)new SourceFile(filename, this.StorageData[filename]));
+        }
+    }
+}

Source/SourceStorage/InMemorySourceStorage/SourceFile.cs

@@ -0,0 +1,26 @@
+namespace EternalArrowBackup.SourceStorage.InMemorySourceStorage
+{
+    using System.Threading.Tasks;
+    using EternalArrowBackup.SourceStorage.Contracts;
+
+    internal class SourceFile : ISourceFile
+    {
+        public SourceFile(string filename, byte[] contents)
+        {
+            this.Filename = filename;
+            this.Size = contents.Length;
+            this.Contents = contents;
+        }
+
+        public string Filename { get; }
+
+        public long Size { get; }
+
+        private byte[] Contents { get; }
+
+        public Task<byte[]> ReadContents()
+        {
+            return Task.Run(() => this.Contents);
+        }
+    }
+}

Source/SourceStorage/InMemorySourceStorage/SourceStorage.cs

@@ -0,0 +1,42 @@
+namespace EternalArrowBackup.SourceStorage.InMemorySourceStorage
+{
+    using System;
+    using System.Collections.Immutable;
+    using System.Threading;
+    using System.Threading.Tasks;
+    using System.Threading.Tasks.Dataflow;
+    using EternalArrowBackup.SourceStorage.Contracts;
+
+    public class SourceStorage : ISourceStorage
+    {
+        public SourceStorage(ImmutableDictionary<string, ImmutableDictionary<string, byte[]>> storageData)
+        {
+            this.StorageData = storageData;
+        }
+
+        private ImmutableDictionary<string, ImmutableDictionary<string, byte[]>> StorageData { get; }
+
+        public Task GetAllDirectories(ActionBlock<ISourceDirectory> actionBlock, CancellationToken ct)
+        {
+            return Task.Run(() =>
+            {
+                foreach (var kvp in this.StorageData)
+                {
+                    if (ct.IsCancellationRequested)
+                    {
+                        break;
+                    }
+
+                    actionBlock.Post(new SourceDirectory(kvp.Key, kvp.Value));
+                }
+
+                actionBlock.Complete();
+            });
+        }
+
+        public Task<ISourceDirectory> GetDirectory(string normalizedRelativePath)
+        {
+            return Task.Run(() => (ISourceDirectory)new SourceDirectory(normalizedRelativePath, this.StorageData[normalizedRelativePath]));
+        }
+    }
+}

Tests/SourceStorage/InMemorySourceStorage/EternalArrowBackup.SourceStorage.InMemorySourceStorage.Tests.csproj

@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netcoreapp1.1</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.0.0" />
+    <PackageReference Include="System.Collections.Immutable" Version="1.3.1" />
+    <PackageReference Include="System.Threading.Tasks.Dataflow" Version="4.7.0" />
+    <PackageReference Include="xunit" Version="2.2.0" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.2.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\..\Source\SourceStorage\Contracts\EternalArrowBackup.SourceStorage.Contracts.csproj" />
+    <ProjectReference Include="..\..\..\Source\SourceStorage\InMemorySourceStorage\EternalArrowBackup.SourceStorage.InMemorySourceStorage.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
+  </ItemGroup>
+
+</Project>

Tests/SourceStorage/InMemorySourceStorage/StorageTests.cs

@@ -0,0 +1,110 @@
+namespace EternalArrowBackup.SourceStorage.InMemorySourceStorage.Tests
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Collections.Immutable;
+    using System.Linq;
+    using System.Threading;
+    using System.Threading.Tasks;
+    using System.Threading.Tasks.Dataflow;
+    using EternalArrowBackup.SourceStorage.Contracts;
+    using Xunit;
+
+    public static class StorageTests
+    {
+        [Fact]
+        public static async Task CheckStorage()
+        {
+            var storageData = CreateStorageData();
+
+            var immutableStorageData = storageData
+                .Select(kvp => new KeyValuePair<string, ImmutableDictionary<string, byte[]>>(kvp.Key, kvp.Value.ToImmutableDictionary()))
+                .ToImmutableDictionary();
+            var storage = new SourceStorage(immutableStorageData);
+
+            var testDirectoryName = storageData.Keys.Skip(storageData.Count / 2).First();
+            var testDirectoryInfo = await storage.GetDirectory(testDirectoryName);
+            Assert.Equal(testDirectoryName, testDirectoryInfo.NormalizedRelativePath);
+            await CheckDirectory(storageData[testDirectoryName].ToDictionary(kvp => kvp.Key, kvp => kvp.Value), testDirectoryInfo);
+
+            var syncObject = new object();
+            var actionBlock = new ActionBlock<ISourceDirectory>(async sourceDirectory =>
+            {
+                Dictionary<string, byte[]> originalDirectoryData;
+
+                lock (syncObject)
+                {
+                    Assert.True(storageData.ContainsKey(sourceDirectory.NormalizedRelativePath), "Directory does not exist in original data");
+                    originalDirectoryData = storageData[sourceDirectory.NormalizedRelativePath];
+                    storageData.Remove(sourceDirectory.NormalizedRelativePath);
+                }
+
+                await CheckDirectory(originalDirectoryData, sourceDirectory);
+            });
+
+            await storage.GetAllDirectories(actionBlock, CancellationToken.None);
+            await actionBlock.Completion;
+
+            Assert.False(storageData.Any(), "Not all directories were enumerated");
+        }
+
+        private static async Task CheckDirectory(Dictionary<string, byte[]> originalDirectory, ISourceDirectory contractsDirectory)
+        {
+            if (originalDirectory.Count > 0)
+            {
+                var testFileName = originalDirectory.Keys.Skip(originalDirectory.Count / 2).First();
+                var testFileInfo = await contractsDirectory.GetFile(testFileName);
+                Assert.Equal(testFileName, testFileInfo.Filename);
+                Assert.Equal(originalDirectory[testFileName].Length, testFileInfo.Size);
+                Assert.Equal(originalDirectory[testFileName], await testFileInfo.ReadContents());
+            }
+
+            var syncObject = new object();
+            var actionBlock = new ActionBlock<ISourceFile>(async sourceFile =>
+            {
+                byte[] originalFileContents;
+                lock (syncObject)
+                {
+                    Assert.True(originalDirectory.ContainsKey(sourceFile.Filename), "File does not exist in original data");
+                    originalFileContents = originalDirectory[sourceFile.Filename];
+                    originalDirectory.Remove(sourceFile.Filename);
+                }
+
+                Assert.Equal(originalFileContents.Length, sourceFile.Size);
+                Assert.Equal(originalFileContents, await sourceFile.ReadContents());
+            });
+
+            await contractsDirectory.GetAllFiles(actionBlock, CancellationToken.None);
+            await actionBlock.Completion;
+
+            Assert.False(originalDirectory.Any(), "Not all files were enumerated");
+        }
+
+        private static Dictionary<string, Dictionary<string, byte[]>> CreateStorageData()
+        {
+            var random = new Random();
+            var result = new Dictionary<string, Dictionary<string, byte[]>>();
+            for (var i = 0; i < 2000; i++)
+            {
+                var directoryName = Guid.NewGuid().ToString();
+                var directoryContent = new Dictionary<string, byte[]>();
+                var directorySize = random.Next(0, 500);
+                for (var j = 0; j < directorySize; j++)
+                {
+                    var fileName = Guid.NewGuid().ToString();
+                    var fileContent = new byte[random.Next(0, 20)];
+                    for (var k = 0; k < fileContent.Length; k++)
+                    {
+                        fileContent[k] = (byte)random.Next(0, 255);
+                    }
+
+                    directoryContent[fileName] = fileContent;
+                }
+
+                result[directoryName] = directoryContent;
+            }
+
+            return result;
+        }
+    }
+}