Implemented InMemoryBinaryStorage

master
Inga 🏳‍🌈 7 years ago
parent fe31ca1ecb
commit bd4eb3edc0
  1. 21
      EternalArrowBackup.sln
  2. 2
      Source/TargetBinaryStorage/Contracts/ITargetBinaryStorage.cs
  3. 45
      Source/TargetBinaryStorage/InMemoryBinaryStorage/BinaryStorage.cs
  4. 21
      Source/TargetBinaryStorage/InMemoryBinaryStorage/BlobInfo.cs
  5. 15
      Source/TargetBinaryStorage/InMemoryBinaryStorage/BlockId.cs
  6. 21
      Source/TargetBinaryStorage/InMemoryBinaryStorage/BlockInfo.cs
  7. 11
      Source/TargetBinaryStorage/InMemoryBinaryStorage/EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage.csproj
  8. 1
      Tests/SourceStorage/InMemorySourceStorage/StorageTests.cs
  9. 22
      Tests/TargetBinaryStorage/InMemoryBinaryStorage/EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage.Tests.csproj
  10. 81
      Tests/TargetBinaryStorage/InMemoryBinaryStorage/StorageTests.cs

@ -47,11 +47,17 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ContentTransformer", "Conte
EndProject EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Hasher", "Hasher", "{0059A121-D191-48FE-9215-F71CE1057EEB}" Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Hasher", "Hasher", "{0059A121-D191-48FE-9215-F71CE1057EEB}"
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EternalArrowBackup.SourceStorage.InMemorySourceStorage", "Source\SourceStorage\InMemorySourceStorage\EternalArrowBackup.SourceStorage.InMemorySourceStorage.csproj", "{B8F0A6CF-6D26-4A38-B6AF-30BB4BE17380}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EternalArrowBackup.SourceStorage.InMemorySourceStorage", "Source\SourceStorage\InMemorySourceStorage\EternalArrowBackup.SourceStorage.InMemorySourceStorage.csproj", "{B8F0A6CF-6D26-4A38-B6AF-30BB4BE17380}"
EndProject EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SourceStorage", "SourceStorage", "{97B894EA-9FAF-4C13-A20E-00A6CC6108C5}" Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SourceStorage", "SourceStorage", "{97B894EA-9FAF-4C13-A20E-00A6CC6108C5}"
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EternalArrowBackup.SourceStorage.InMemorySourceStorage.Tests", "Tests\SourceStorage\InMemorySourceStorage\EternalArrowBackup.SourceStorage.InMemorySourceStorage.Tests.csproj", "{8B5C507A-C6FE-46BA-BB4F-4876B8230E26}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EternalArrowBackup.SourceStorage.InMemorySourceStorage.Tests", "Tests\SourceStorage\InMemorySourceStorage\EternalArrowBackup.SourceStorage.InMemorySourceStorage.Tests.csproj", "{8B5C507A-C6FE-46BA-BB4F-4876B8230E26}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage", "Source\TargetBinaryStorage\InMemoryBinaryStorage\EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage.csproj", "{12102767-6F40-4328-9CB5-290EF484682D}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "TargetBinaryStorage", "TargetBinaryStorage", "{61CADBF4-3D45-4980-81E5-5268199E615A}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage.Tests", "Tests\TargetBinaryStorage\InMemoryBinaryStorage\EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage.Tests.csproj", "{3D95405C-C305-4D73-9B2C-938156FD9B2E}"
EndProject EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -111,6 +117,14 @@ Global
{8B5C507A-C6FE-46BA-BB4F-4876B8230E26}.Debug|Any CPU.Build.0 = Debug|Any CPU {8B5C507A-C6FE-46BA-BB4F-4876B8230E26}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8B5C507A-C6FE-46BA-BB4F-4876B8230E26}.Release|Any CPU.ActiveCfg = Release|Any CPU {8B5C507A-C6FE-46BA-BB4F-4876B8230E26}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8B5C507A-C6FE-46BA-BB4F-4876B8230E26}.Release|Any CPU.Build.0 = Release|Any CPU {8B5C507A-C6FE-46BA-BB4F-4876B8230E26}.Release|Any CPU.Build.0 = Release|Any CPU
{12102767-6F40-4328-9CB5-290EF484682D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{12102767-6F40-4328-9CB5-290EF484682D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{12102767-6F40-4328-9CB5-290EF484682D}.Release|Any CPU.ActiveCfg = Release|Any CPU
{12102767-6F40-4328-9CB5-290EF484682D}.Release|Any CPU.Build.0 = Release|Any CPU
{3D95405C-C305-4D73-9B2C-938156FD9B2E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{3D95405C-C305-4D73-9B2C-938156FD9B2E}.Debug|Any CPU.Build.0 = Debug|Any CPU
{3D95405C-C305-4D73-9B2C-938156FD9B2E}.Release|Any CPU.ActiveCfg = Release|Any CPU
{3D95405C-C305-4D73-9B2C-938156FD9B2E}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection EndGlobalSection
GlobalSection(SolutionProperties) = preSolution GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE HideSolutionNode = FALSE
@ -139,5 +153,8 @@ Global
{B8F0A6CF-6D26-4A38-B6AF-30BB4BE17380} = {05426D05-AE66-4C5C-8B8E-403A46ADC49D} {B8F0A6CF-6D26-4A38-B6AF-30BB4BE17380} = {05426D05-AE66-4C5C-8B8E-403A46ADC49D}
{97B894EA-9FAF-4C13-A20E-00A6CC6108C5} = {884B8E01-303A-40CF-8884-D62115F98683} {97B894EA-9FAF-4C13-A20E-00A6CC6108C5} = {884B8E01-303A-40CF-8884-D62115F98683}
{8B5C507A-C6FE-46BA-BB4F-4876B8230E26} = {97B894EA-9FAF-4C13-A20E-00A6CC6108C5} {8B5C507A-C6FE-46BA-BB4F-4876B8230E26} = {97B894EA-9FAF-4C13-A20E-00A6CC6108C5}
{12102767-6F40-4328-9CB5-290EF484682D} = {85879329-E36C-4585-8019-2825DEB4D0B8}
{61CADBF4-3D45-4980-81E5-5268199E615A} = {884B8E01-303A-40CF-8884-D62115F98683}
{3D95405C-C305-4D73-9B2C-938156FD9B2E} = {61CADBF4-3D45-4980-81E5-5268199E615A}
EndGlobalSection EndGlobalSection
EndGlobal EndGlobal

@ -9,5 +9,7 @@
Task WriteBlob(string blobId, long originalSize, string[] blockKeys); Task WriteBlob(string blobId, long originalSize, string[] blockKeys);
Task<IBlobInfo> GetBlobIfExists(string blobId); Task<IBlobInfo> GetBlobIfExists(string blobId);
Task<byte[]> RetrieveBlock(string blobId, string blockKey);
} }
} }

@ -0,0 +1,45 @@
namespace EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage
{
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using EternalArrowBackup.TargetBinaryStorage.Contracts;
public class BinaryStorage : ITargetBinaryStorage
{
private Dictionary<string, IBlobInfo> Blobs { get; } = new Dictionary<string, IBlobInfo>();
private Dictionary<BlockId, BlockInfo> Blocks { get; } = new Dictionary<BlockId, BlockInfo>();
public Task<IBlobInfo> GetBlobIfExists(string blobId)
{
if (!this.Blobs.ContainsKey(blobId))
{
return Task.FromResult(default(IBlobInfo));
}
return Task.FromResult(this.Blobs[blobId]);
}
public Task WriteBlob(string blobId, long originalSize, string[] blockKeys)
{
return Task.Run(() =>
{
this.Blobs[blobId] = new BlobInfo(DateTime.UtcNow, originalSize, blockKeys);
});
}
public Task WriteBlock(byte[] block, string blobId, int partNumber, string blockKey)
{
return Task.Run(() =>
{
this.Blocks[new BlockId(blobId, blockKey)] = new BlockInfo(blobId, partNumber, blockKey, block);
});
}
public Task<byte[]> RetrieveBlock(string blobId, string blockKey)
{
return Task.Run(() => this.Blocks[new BlockId(blobId, blockKey)].BlockData);
}
}
}

@ -0,0 +1,21 @@
namespace EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage
{
using System;
using EternalArrowBackup.TargetBinaryStorage.Contracts;
internal class BlobInfo : IBlobInfo
{
public BlobInfo(DateTime uploadDate, long originalSize, string[] blockKeys)
{
this.UploadDate = uploadDate;
this.OriginalSize = originalSize;
this.BlockKeys = blockKeys;
}
public DateTime UploadDate { get; }
public long OriginalSize { get; }
public string[] BlockKeys { get; }
}
}

@ -0,0 +1,15 @@
namespace EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage
{
internal struct BlockId
{
public readonly string BlobId;
public readonly string BlockKey;
public BlockId(string blobId, string blockKey)
{
this.BlobId = blobId;
this.BlockKey = blockKey;
}
}
}

@ -0,0 +1,21 @@
namespace EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage
{
internal class BlockInfo
{
public BlockInfo(string blobId, int partNumber, string blockKey, byte[] blockData)
{
this.BlobId = blobId;
this.PartNumber = partNumber;
this.BlockKey = blockKey;
this.BlockData = blockData;
}
public string BlobId { get; }
public int PartNumber { get; }
public string BlockKey { get; }
public byte[] BlockData { get; }
}
}

@ -0,0 +1,11 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard1.4</TargetFramework>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Contracts\EternalArrowBackup.TargetBinaryStorage.Contracts.csproj" />
</ItemGroup>
</Project>

@ -13,6 +13,7 @@
public static class StorageTests public static class StorageTests
{ {
[Fact] [Fact]
[Trait("Category", "Simple")]
public static async Task CheckStorage() public static async Task CheckStorage()
{ {
var storageData = CreateStorageData(); var storageData = CreateStorageData();

@ -0,0 +1,22 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netcoreapp1.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.0.0" />
<PackageReference Include="xunit" Version="2.2.0" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.2.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\Source\TargetBinaryStorage\Contracts\EternalArrowBackup.TargetBinaryStorage.Contracts.csproj" />
<ProjectReference Include="..\..\..\Source\TargetBinaryStorage\InMemoryBinaryStorage\EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage.csproj" />
</ItemGroup>
<ItemGroup>
<Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
</ItemGroup>
</Project>

@ -0,0 +1,81 @@
namespace EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage.Tests
{
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using Xunit;
public static class StorageTests
{
[Fact]
[Trait("Category", "Simple")]
public static async Task CheckStorage()
{
var storageData = CreateStorageData();
var storage = new BinaryStorage();
foreach (var kvp in storageData)
{
var blobId = kvp.Key;
Assert.Null(await storage.GetBlobIfExists(blobId));
var blocks = kvp.Value.Item2;
for (var i = 0; i < blocks.Length; i++)
{
await storage.WriteBlock(blocks[i].Item2, blobId, i, blocks[i].Item1);
}
await storage.WriteBlob(blobId, kvp.Value.Item1, blocks.Select(tuple => tuple.Item1).ToArray());
}
foreach (var kvp in storageData)
{
var blobId = kvp.Key;
var blobInfo = await storage.GetBlobIfExists(blobId);
Assert.NotNull(blobInfo);
Assert.Equal(kvp.Value.Item1, blobInfo.OriginalSize);
Assert.InRange(blobInfo.UploadDate, DateTime.UtcNow.AddMinutes(-2), DateTime.UtcNow);
Assert.Equal(kvp.Value.Item2.Select(tuple => tuple.Item1).ToArray(), blobInfo.BlockKeys);
for (var i = 0; i < blobInfo.BlockKeys.Length; i++)
{
var blockKey = blobInfo.BlockKeys[i];
Assert.Equal(kvp.Value.Item2[i].Item1, blockKey);
var blockContent = await storage.RetrieveBlock(blobId, blockKey);
Assert.Equal(kvp.Value.Item2[i].Item2, blockContent);
}
}
}
private static Dictionary<string, Tuple<long, Tuple<string, byte[]>[]>> CreateStorageData()
{
var random = new Random();
var result = new Dictionary<string, Tuple<long, Tuple<string, byte[]>[]>>();
for (var i = 0; i < 2000; i++)
{
var blobName = Guid.NewGuid().ToString();
var blocksCount = random.Next(0, 20);
var blocks = new Tuple<string, byte[]>[blocksCount];
for (var j = 0; j < blocksCount; j++)
{
var blockKey = Guid.NewGuid().ToString();
var blockContent = new byte[random.Next(0, 20)];
for (var k = 0; k < blockContent.Length; k++)
{
blockContent[k] = (byte)random.Next(0, 255);
}
blocks[j] = Tuple.Create(blockKey, blockContent);
}
result[blobName] = Tuple.Create(((long)random.Next()) * random.Next(), blocks);
}
return result;
}
}
}
Loading…
Cancel
Save