From bd4eb3edc0142423ec7a150886b4bf73ae1603e5 Mon Sep 17 00:00:00 2001 From: Inga Lovinde <52715130+inga-lovinde@users.noreply.github.com> Date: Tue, 18 Jul 2017 17:57:36 +0300 Subject: [PATCH] Implemented InMemoryBinaryStorage --- EternalArrowBackup.sln | 21 ++++- .../Contracts/ITargetBinaryStorage.cs | 2 + .../InMemoryBinaryStorage/BinaryStorage.cs | 45 +++++++++++ .../InMemoryBinaryStorage/BlobInfo.cs | 21 +++++ .../InMemoryBinaryStorage/BlockId.cs | 15 ++++ .../InMemoryBinaryStorage/BlockInfo.cs | 21 +++++ ...BinaryStorage.InMemoryBinaryStorage.csproj | 11 +++ .../InMemorySourceStorage/StorageTests.cs | 1 + ...Storage.InMemoryBinaryStorage.Tests.csproj | 22 +++++ .../InMemoryBinaryStorage/StorageTests.cs | 81 +++++++++++++++++++ 10 files changed, 238 insertions(+), 2 deletions(-) create mode 100644 Source/TargetBinaryStorage/InMemoryBinaryStorage/BinaryStorage.cs create mode 100644 Source/TargetBinaryStorage/InMemoryBinaryStorage/BlobInfo.cs create mode 100644 Source/TargetBinaryStorage/InMemoryBinaryStorage/BlockId.cs create mode 100644 Source/TargetBinaryStorage/InMemoryBinaryStorage/BlockInfo.cs create mode 100644 Source/TargetBinaryStorage/InMemoryBinaryStorage/EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage.csproj create mode 100644 Tests/TargetBinaryStorage/InMemoryBinaryStorage/EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage.Tests.csproj create mode 100644 Tests/TargetBinaryStorage/InMemoryBinaryStorage/StorageTests.cs diff --git a/EternalArrowBackup.sln b/EternalArrowBackup.sln index d56fb93..2a41eaa 100644 --- a/EternalArrowBackup.sln +++ b/EternalArrowBackup.sln @@ -47,11 +47,17 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ContentTransformer", "Conte EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Hasher", "Hasher", "{0059A121-D191-48FE-9215-F71CE1057EEB}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EternalArrowBackup.SourceStorage.InMemorySourceStorage", "Source\SourceStorage\InMemorySourceStorage\EternalArrowBackup.SourceStorage.InMemorySourceStorage.csproj", "{B8F0A6CF-6D26-4A38-B6AF-30BB4BE17380}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EternalArrowBackup.SourceStorage.InMemorySourceStorage", "Source\SourceStorage\InMemorySourceStorage\EternalArrowBackup.SourceStorage.InMemorySourceStorage.csproj", "{B8F0A6CF-6D26-4A38-B6AF-30BB4BE17380}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SourceStorage", "SourceStorage", "{97B894EA-9FAF-4C13-A20E-00A6CC6108C5}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EternalArrowBackup.SourceStorage.InMemorySourceStorage.Tests", "Tests\SourceStorage\InMemorySourceStorage\EternalArrowBackup.SourceStorage.InMemorySourceStorage.Tests.csproj", "{8B5C507A-C6FE-46BA-BB4F-4876B8230E26}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EternalArrowBackup.SourceStorage.InMemorySourceStorage.Tests", "Tests\SourceStorage\InMemorySourceStorage\EternalArrowBackup.SourceStorage.InMemorySourceStorage.Tests.csproj", "{8B5C507A-C6FE-46BA-BB4F-4876B8230E26}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage", "Source\TargetBinaryStorage\InMemoryBinaryStorage\EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage.csproj", "{12102767-6F40-4328-9CB5-290EF484682D}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "TargetBinaryStorage", "TargetBinaryStorage", "{61CADBF4-3D45-4980-81E5-5268199E615A}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage.Tests", "Tests\TargetBinaryStorage\InMemoryBinaryStorage\EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage.Tests.csproj", "{3D95405C-C305-4D73-9B2C-938156FD9B2E}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -111,6 +117,14 @@ Global {8B5C507A-C6FE-46BA-BB4F-4876B8230E26}.Debug|Any CPU.Build.0 = Debug|Any CPU {8B5C507A-C6FE-46BA-BB4F-4876B8230E26}.Release|Any CPU.ActiveCfg = Release|Any CPU {8B5C507A-C6FE-46BA-BB4F-4876B8230E26}.Release|Any CPU.Build.0 = Release|Any CPU + {12102767-6F40-4328-9CB5-290EF484682D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {12102767-6F40-4328-9CB5-290EF484682D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {12102767-6F40-4328-9CB5-290EF484682D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {12102767-6F40-4328-9CB5-290EF484682D}.Release|Any CPU.Build.0 = Release|Any CPU + {3D95405C-C305-4D73-9B2C-938156FD9B2E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3D95405C-C305-4D73-9B2C-938156FD9B2E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3D95405C-C305-4D73-9B2C-938156FD9B2E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3D95405C-C305-4D73-9B2C-938156FD9B2E}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -139,5 +153,8 @@ Global {B8F0A6CF-6D26-4A38-B6AF-30BB4BE17380} = {05426D05-AE66-4C5C-8B8E-403A46ADC49D} {97B894EA-9FAF-4C13-A20E-00A6CC6108C5} = {884B8E01-303A-40CF-8884-D62115F98683} {8B5C507A-C6FE-46BA-BB4F-4876B8230E26} = {97B894EA-9FAF-4C13-A20E-00A6CC6108C5} + {12102767-6F40-4328-9CB5-290EF484682D} = {85879329-E36C-4585-8019-2825DEB4D0B8} + {61CADBF4-3D45-4980-81E5-5268199E615A} = {884B8E01-303A-40CF-8884-D62115F98683} + {3D95405C-C305-4D73-9B2C-938156FD9B2E} = {61CADBF4-3D45-4980-81E5-5268199E615A} EndGlobalSection EndGlobal diff --git a/Source/TargetBinaryStorage/Contracts/ITargetBinaryStorage.cs b/Source/TargetBinaryStorage/Contracts/ITargetBinaryStorage.cs index 4a0867c..af26562 100644 --- a/Source/TargetBinaryStorage/Contracts/ITargetBinaryStorage.cs +++ b/Source/TargetBinaryStorage/Contracts/ITargetBinaryStorage.cs @@ -9,5 +9,7 @@ Task WriteBlob(string blobId, long originalSize, string[] blockKeys); Task GetBlobIfExists(string blobId); + + Task RetrieveBlock(string blobId, string blockKey); } } diff --git a/Source/TargetBinaryStorage/InMemoryBinaryStorage/BinaryStorage.cs b/Source/TargetBinaryStorage/InMemoryBinaryStorage/BinaryStorage.cs new file mode 100644 index 0000000..af87e87 --- /dev/null +++ b/Source/TargetBinaryStorage/InMemoryBinaryStorage/BinaryStorage.cs @@ -0,0 +1,45 @@ +namespace EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage +{ + using System; + using System.Collections.Generic; + using System.Threading.Tasks; + using EternalArrowBackup.TargetBinaryStorage.Contracts; + + public class BinaryStorage : ITargetBinaryStorage + { + private Dictionary Blobs { get; } = new Dictionary(); + + private Dictionary Blocks { get; } = new Dictionary(); + + public Task GetBlobIfExists(string blobId) + { + if (!this.Blobs.ContainsKey(blobId)) + { + return Task.FromResult(default(IBlobInfo)); + } + + return Task.FromResult(this.Blobs[blobId]); + } + + public Task WriteBlob(string blobId, long originalSize, string[] blockKeys) + { + return Task.Run(() => + { + this.Blobs[blobId] = new BlobInfo(DateTime.UtcNow, originalSize, blockKeys); + }); + } + + public Task WriteBlock(byte[] block, string blobId, int partNumber, string blockKey) + { + return Task.Run(() => + { + this.Blocks[new BlockId(blobId, blockKey)] = new BlockInfo(blobId, partNumber, blockKey, block); + }); + } + + public Task RetrieveBlock(string blobId, string blockKey) + { + return Task.Run(() => this.Blocks[new BlockId(blobId, blockKey)].BlockData); + } + } +} diff --git a/Source/TargetBinaryStorage/InMemoryBinaryStorage/BlobInfo.cs b/Source/TargetBinaryStorage/InMemoryBinaryStorage/BlobInfo.cs new file mode 100644 index 0000000..ffb5d1d --- /dev/null +++ b/Source/TargetBinaryStorage/InMemoryBinaryStorage/BlobInfo.cs @@ -0,0 +1,21 @@ +namespace EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage +{ + using System; + using EternalArrowBackup.TargetBinaryStorage.Contracts; + + internal class BlobInfo : IBlobInfo + { + public BlobInfo(DateTime uploadDate, long originalSize, string[] blockKeys) + { + this.UploadDate = uploadDate; + this.OriginalSize = originalSize; + this.BlockKeys = blockKeys; + } + + public DateTime UploadDate { get; } + + public long OriginalSize { get; } + + public string[] BlockKeys { get; } + } +} diff --git a/Source/TargetBinaryStorage/InMemoryBinaryStorage/BlockId.cs b/Source/TargetBinaryStorage/InMemoryBinaryStorage/BlockId.cs new file mode 100644 index 0000000..8e90ba4 --- /dev/null +++ b/Source/TargetBinaryStorage/InMemoryBinaryStorage/BlockId.cs @@ -0,0 +1,15 @@ +namespace EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage +{ + internal struct BlockId + { + public readonly string BlobId; + + public readonly string BlockKey; + + public BlockId(string blobId, string blockKey) + { + this.BlobId = blobId; + this.BlockKey = blockKey; + } + } +} diff --git a/Source/TargetBinaryStorage/InMemoryBinaryStorage/BlockInfo.cs b/Source/TargetBinaryStorage/InMemoryBinaryStorage/BlockInfo.cs new file mode 100644 index 0000000..068e769 --- /dev/null +++ b/Source/TargetBinaryStorage/InMemoryBinaryStorage/BlockInfo.cs @@ -0,0 +1,21 @@ +namespace EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage +{ + internal class BlockInfo + { + public BlockInfo(string blobId, int partNumber, string blockKey, byte[] blockData) + { + this.BlobId = blobId; + this.PartNumber = partNumber; + this.BlockKey = blockKey; + this.BlockData = blockData; + } + + public string BlobId { get; } + + public int PartNumber { get; } + + public string BlockKey { get; } + + public byte[] BlockData { get; } + } +} diff --git a/Source/TargetBinaryStorage/InMemoryBinaryStorage/EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage.csproj b/Source/TargetBinaryStorage/InMemoryBinaryStorage/EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage.csproj new file mode 100644 index 0000000..9eee164 --- /dev/null +++ b/Source/TargetBinaryStorage/InMemoryBinaryStorage/EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage.csproj @@ -0,0 +1,11 @@ + + + + netstandard1.4 + + + + + + + \ No newline at end of file diff --git a/Tests/SourceStorage/InMemorySourceStorage/StorageTests.cs b/Tests/SourceStorage/InMemorySourceStorage/StorageTests.cs index ee9ee59..1a098a6 100644 --- a/Tests/SourceStorage/InMemorySourceStorage/StorageTests.cs +++ b/Tests/SourceStorage/InMemorySourceStorage/StorageTests.cs @@ -13,6 +13,7 @@ public static class StorageTests { [Fact] + [Trait("Category", "Simple")] public static async Task CheckStorage() { var storageData = CreateStorageData(); diff --git a/Tests/TargetBinaryStorage/InMemoryBinaryStorage/EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage.Tests.csproj b/Tests/TargetBinaryStorage/InMemoryBinaryStorage/EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage.Tests.csproj new file mode 100644 index 0000000..23c7647 --- /dev/null +++ b/Tests/TargetBinaryStorage/InMemoryBinaryStorage/EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage.Tests.csproj @@ -0,0 +1,22 @@ + + + + netcoreapp1.1 + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Tests/TargetBinaryStorage/InMemoryBinaryStorage/StorageTests.cs b/Tests/TargetBinaryStorage/InMemoryBinaryStorage/StorageTests.cs new file mode 100644 index 0000000..fefc458 --- /dev/null +++ b/Tests/TargetBinaryStorage/InMemoryBinaryStorage/StorageTests.cs @@ -0,0 +1,81 @@ +namespace EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage.Tests +{ + using System; + using System.Collections.Generic; + using System.Collections.Immutable; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using System.Threading.Tasks.Dataflow; + using Xunit; + + public static class StorageTests + { + [Fact] + [Trait("Category", "Simple")] + public static async Task CheckStorage() + { + var storageData = CreateStorageData(); + var storage = new BinaryStorage(); + + foreach (var kvp in storageData) + { + var blobId = kvp.Key; + Assert.Null(await storage.GetBlobIfExists(blobId)); + var blocks = kvp.Value.Item2; + for (var i = 0; i < blocks.Length; i++) + { + await storage.WriteBlock(blocks[i].Item2, blobId, i, blocks[i].Item1); + } + + await storage.WriteBlob(blobId, kvp.Value.Item1, blocks.Select(tuple => tuple.Item1).ToArray()); + } + + foreach (var kvp in storageData) + { + var blobId = kvp.Key; + var blobInfo = await storage.GetBlobIfExists(blobId); + Assert.NotNull(blobInfo); + Assert.Equal(kvp.Value.Item1, blobInfo.OriginalSize); + Assert.InRange(blobInfo.UploadDate, DateTime.UtcNow.AddMinutes(-2), DateTime.UtcNow); + Assert.Equal(kvp.Value.Item2.Select(tuple => tuple.Item1).ToArray(), blobInfo.BlockKeys); + + for (var i = 0; i < blobInfo.BlockKeys.Length; i++) + { + var blockKey = blobInfo.BlockKeys[i]; + Assert.Equal(kvp.Value.Item2[i].Item1, blockKey); + + var blockContent = await storage.RetrieveBlock(blobId, blockKey); + Assert.Equal(kvp.Value.Item2[i].Item2, blockContent); + } + } + } + + private static Dictionary[]>> CreateStorageData() + { + var random = new Random(); + var result = new Dictionary[]>>(); + for (var i = 0; i < 2000; i++) + { + var blobName = Guid.NewGuid().ToString(); + var blocksCount = random.Next(0, 20); + var blocks = new Tuple[blocksCount]; + for (var j = 0; j < blocksCount; j++) + { + var blockKey = Guid.NewGuid().ToString(); + var blockContent = new byte[random.Next(0, 20)]; + for (var k = 0; k < blockContent.Length; k++) + { + blockContent[k] = (byte)random.Next(0, 255); + } + + blocks[j] = Tuple.Create(blockKey, blockContent); + } + + result[blobName] = Tuple.Create(((long)random.Next()) * random.Next(), blocks); + } + + return result; + } + } +}