From 588027c76117455e828fc151ba9ac5391ae45bb8 Mon Sep 17 00:00:00 2001 From: Inga Lovinde <52715130+inga-lovinde@users.noreply.github.com> Date: Wed, 19 Jul 2017 16:50:29 +0300 Subject: [PATCH] Implemented InMemoryMetadataStorage --- EternalArrowBackup.sln | 7 ++ ...kup.TargetMetadataStorage.Contracts.csproj | 6 +- .../Contracts/ITargetDirectory.cs | 6 +- .../Contracts/ITargetFile.cs | 5 +- .../ITargetMetadataStorageForRecovery.cs | 8 +- ...dataStorage.InMemoryMetadataStorage.csproj | 15 +++ .../MetadataStorage.cs | 99 +++++++++++++++++++ .../TargetDirectory.cs | 62 ++++++++++++ .../InMemoryMetadataStorage/TargetFile.cs | 36 +++++++ .../TargetFileVersion.cs | 24 +++++ 10 files changed, 260 insertions(+), 8 deletions(-) create mode 100644 Source/TargetMetadataStorage/InMemoryMetadataStorage/EternalArrowBackup.TargetMetadataStorage.InMemoryMetadataStorage.csproj create mode 100644 Source/TargetMetadataStorage/InMemoryMetadataStorage/MetadataStorage.cs create mode 100644 Source/TargetMetadataStorage/InMemoryMetadataStorage/TargetDirectory.cs create mode 100644 Source/TargetMetadataStorage/InMemoryMetadataStorage/TargetFile.cs create mode 100644 Source/TargetMetadataStorage/InMemoryMetadataStorage/TargetFileVersion.cs diff --git a/EternalArrowBackup.sln b/EternalArrowBackup.sln index 2a41eaa..3013c6d 100644 --- a/EternalArrowBackup.sln +++ b/EternalArrowBackup.sln @@ -59,6 +59,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "TargetBinaryStorage", "Targ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage.Tests", "Tests\TargetBinaryStorage\InMemoryBinaryStorage\EternalArrowBackup.TargetBinaryStorage.InMemoryBinaryStorage.Tests.csproj", "{3D95405C-C305-4D73-9B2C-938156FD9B2E}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EternalArrowBackup.TargetMetadataStorage.InMemoryMetadataStorage", "Source\TargetMetadataStorage\InMemoryMetadataStorage\EternalArrowBackup.TargetMetadataStorage.InMemoryMetadataStorage.csproj", "{AC7EA3E8-6469-4102-940F-31EF831E823F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -125,6 +127,10 @@ Global {3D95405C-C305-4D73-9B2C-938156FD9B2E}.Debug|Any CPU.Build.0 = Debug|Any CPU {3D95405C-C305-4D73-9B2C-938156FD9B2E}.Release|Any CPU.ActiveCfg = Release|Any CPU {3D95405C-C305-4D73-9B2C-938156FD9B2E}.Release|Any CPU.Build.0 = Release|Any CPU + {AC7EA3E8-6469-4102-940F-31EF831E823F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AC7EA3E8-6469-4102-940F-31EF831E823F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AC7EA3E8-6469-4102-940F-31EF831E823F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AC7EA3E8-6469-4102-940F-31EF831E823F}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -156,5 +162,6 @@ Global {12102767-6F40-4328-9CB5-290EF484682D} = {85879329-E36C-4585-8019-2825DEB4D0B8} {61CADBF4-3D45-4980-81E5-5268199E615A} = {884B8E01-303A-40CF-8884-D62115F98683} {3D95405C-C305-4D73-9B2C-938156FD9B2E} = {61CADBF4-3D45-4980-81E5-5268199E615A} + {AC7EA3E8-6469-4102-940F-31EF831E823F} = {99C42B64-99CF-4FB3-B270-079476B43FF7} EndGlobalSection EndGlobal diff --git a/Source/TargetMetadataStorage/Contracts/EternalArrowBackup.TargetMetadataStorage.Contracts.csproj b/Source/TargetMetadataStorage/Contracts/EternalArrowBackup.TargetMetadataStorage.Contracts.csproj index 954020d..0a822af 100644 --- a/Source/TargetMetadataStorage/Contracts/EternalArrowBackup.TargetMetadataStorage.Contracts.csproj +++ b/Source/TargetMetadataStorage/Contracts/EternalArrowBackup.TargetMetadataStorage.Contracts.csproj @@ -1,7 +1,11 @@ - + netstandard1.4 + + + + \ No newline at end of file diff --git a/Source/TargetMetadataStorage/Contracts/ITargetDirectory.cs b/Source/TargetMetadataStorage/Contracts/ITargetDirectory.cs index 622edf2..1e9a2cd 100644 --- a/Source/TargetMetadataStorage/Contracts/ITargetDirectory.cs +++ b/Source/TargetMetadataStorage/Contracts/ITargetDirectory.cs @@ -1,13 +1,15 @@ namespace EternalArrowBackup.TargetMetadataStorage.Contracts { - using System; using System.Threading; using System.Threading.Tasks; + using System.Threading.Tasks.Dataflow; public interface ITargetDirectory { + string DirectoryName { get; } + Task UploadFile(string filename, string originalHash, long originalSize); - IObservable GetAllLatestFileVersions(CancellationToken ct); + Task GetAllLatestFileVersions(ActionBlock actionBlock, CancellationToken ct); } } diff --git a/Source/TargetMetadataStorage/Contracts/ITargetFile.cs b/Source/TargetMetadataStorage/Contracts/ITargetFile.cs index f75a62e..435b9e7 100644 --- a/Source/TargetMetadataStorage/Contracts/ITargetFile.cs +++ b/Source/TargetMetadataStorage/Contracts/ITargetFile.cs @@ -1,10 +1,11 @@ namespace EternalArrowBackup.TargetMetadataStorage.Contracts { - using System; using System.Threading; + using System.Threading.Tasks; + using System.Threading.Tasks.Dataflow; public interface ITargetFile { - IObservable GetAllVersions(CancellationToken ct); + Task GetAllVersions(ActionBlock actionBlock, CancellationToken ct); } } diff --git a/Source/TargetMetadataStorage/Contracts/ITargetMetadataStorageForRecovery.cs b/Source/TargetMetadataStorage/Contracts/ITargetMetadataStorageForRecovery.cs index 2aded10..038c200 100644 --- a/Source/TargetMetadataStorage/Contracts/ITargetMetadataStorageForRecovery.cs +++ b/Source/TargetMetadataStorage/Contracts/ITargetMetadataStorageForRecovery.cs @@ -1,13 +1,15 @@ namespace EternalArrowBackup.TargetMetadataStorage.Contracts { - using System; using System.Threading; using System.Threading.Tasks; + using System.Threading.Tasks.Dataflow; - public interface ITargetMetadataStorageForRecovery + public interface ITargetMetadataStorageForRecovery : ITargetMetadataStorage { - IObservable GetAllDirectories(CancellationToken ct); + Task GetAllDirectories(ActionBlock actionBlock, CancellationToken ct); Task GetFile(string normalizedRelativeDirectoryPath, string filename); + + Task GetBlobUsages(string hash, ActionBlock actionBlock, CancellationToken ct); } } diff --git a/Source/TargetMetadataStorage/InMemoryMetadataStorage/EternalArrowBackup.TargetMetadataStorage.InMemoryMetadataStorage.csproj b/Source/TargetMetadataStorage/InMemoryMetadataStorage/EternalArrowBackup.TargetMetadataStorage.InMemoryMetadataStorage.csproj new file mode 100644 index 0000000..0d553c0 --- /dev/null +++ b/Source/TargetMetadataStorage/InMemoryMetadataStorage/EternalArrowBackup.TargetMetadataStorage.InMemoryMetadataStorage.csproj @@ -0,0 +1,15 @@ + + + + netstandard1.4 + + + + + + + + + + + \ No newline at end of file diff --git a/Source/TargetMetadataStorage/InMemoryMetadataStorage/MetadataStorage.cs b/Source/TargetMetadataStorage/InMemoryMetadataStorage/MetadataStorage.cs new file mode 100644 index 0000000..1824e9f --- /dev/null +++ b/Source/TargetMetadataStorage/InMemoryMetadataStorage/MetadataStorage.cs @@ -0,0 +1,99 @@ +namespace EternalArrowBackup.TargetMetadataStorage.InMemoryMetadataStorage +{ + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using System.Threading.Tasks.Dataflow; + using EternalArrowBackup.TargetMetadataStorage.Contracts; + + public class MetadataStorage : ITargetMetadataStorageForRecovery + { + private Dictionary>> Data { get; } = new Dictionary>>(); + + private Dictionary> BlobsInfo { get; } = new Dictionary>(); + + public Task GetAllDirectories(ActionBlock actionBlock, CancellationToken ct) + { + return Task.Run(() => + { + foreach (var kvp in this.Data) + { + if (ct.IsCancellationRequested) + { + break; + } + + actionBlock.Post(new TargetDirectory(this, kvp.Key, kvp.Value)); + } + + actionBlock.Complete(); + }); + } + + public Task GetBlobUsages(string hash, ActionBlock actionBlock, CancellationToken ct) + { + return Task.Run(() => + { + if (!this.BlobsInfo.ContainsKey(hash)) + { + actionBlock.Complete(); + return; + } + + foreach (var fileVersion in this.BlobsInfo[hash]) + { + if (ct.IsCancellationRequested) + { + break; + } + + actionBlock.Post(fileVersion); + } + + actionBlock.Complete(); + }); + } + + public Task GetDirectory(string normalizedRelativeDirectoryPath) + { + return Task.Run(() => + { + if (!this.Data.ContainsKey(normalizedRelativeDirectoryPath)) + { + this.Data[normalizedRelativeDirectoryPath] = new Dictionary>(); + } + + return (ITargetDirectory)new TargetDirectory(this, normalizedRelativeDirectoryPath, this.Data[normalizedRelativeDirectoryPath]); + }); + } + + public Task GetFile(string normalizedRelativeDirectoryPath, string filename) + { + return Task.Run(() => + { + if (!this.Data.ContainsKey(normalizedRelativeDirectoryPath)) + { + this.Data[normalizedRelativeDirectoryPath] = new Dictionary>(); + } + + if (!this.Data[normalizedRelativeDirectoryPath].ContainsKey(filename)) + { + this.Data[normalizedRelativeDirectoryPath][filename] = new List(); + } + + return (ITargetFile)new TargetFile(this.Data[normalizedRelativeDirectoryPath][filename]); + }); + } + + public void AddBlob(ITargetFileVersion fileVersion) + { + if (!this.BlobsInfo.ContainsKey(fileVersion.Hash)) + { + this.BlobsInfo[fileVersion.Hash] = new List(); + } + + this.BlobsInfo[fileVersion.Hash].Add(fileVersion); + } + } +} diff --git a/Source/TargetMetadataStorage/InMemoryMetadataStorage/TargetDirectory.cs b/Source/TargetMetadataStorage/InMemoryMetadataStorage/TargetDirectory.cs new file mode 100644 index 0000000..67dceec --- /dev/null +++ b/Source/TargetMetadataStorage/InMemoryMetadataStorage/TargetDirectory.cs @@ -0,0 +1,62 @@ +namespace EternalArrowBackup.TargetMetadataStorage.InMemoryMetadataStorage +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using System.Threading.Tasks.Dataflow; + using EternalArrowBackup.TargetMetadataStorage.Contracts; + + internal class TargetDirectory : ITargetDirectory + { + public TargetDirectory(MetadataStorage metadataStorage, string directoryName, Dictionary> files) + { + this.MetadataStorage = metadataStorage; + this.DirectoryName = directoryName; + this.Files = files; + } + + public string DirectoryName { get; } + + private MetadataStorage MetadataStorage { get; } + + private Dictionary> Files { get; } + + public Task GetAllLatestFileVersions(ActionBlock actionBlock, CancellationToken ct) + { + return Task.Run(() => + { + foreach (var file in this.Files.Values) + { + if (ct.IsCancellationRequested) + { + break; + } + + if (file.Any()) + { + actionBlock.Post(file.Last()); + } + } + + actionBlock.Complete(); + }); + } + + public Task UploadFile(string filename, string originalHash, long originalSize) + { + return Task.Run(() => + { + if (!this.Files.ContainsKey(filename)) + { + this.Files[filename] = new List(); + } + + var fileVersion = new TargetFileVersion(filename, originalHash, DateTime.UtcNow, originalSize); + this.Files[filename].Add(fileVersion); + MetadataStorage.AddBlob(fileVersion); + }); + } + } +} diff --git a/Source/TargetMetadataStorage/InMemoryMetadataStorage/TargetFile.cs b/Source/TargetMetadataStorage/InMemoryMetadataStorage/TargetFile.cs new file mode 100644 index 0000000..ea0c48a --- /dev/null +++ b/Source/TargetMetadataStorage/InMemoryMetadataStorage/TargetFile.cs @@ -0,0 +1,36 @@ +namespace EternalArrowBackup.TargetMetadataStorage.InMemoryMetadataStorage +{ + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using System.Threading.Tasks.Dataflow; + using EternalArrowBackup.TargetMetadataStorage.Contracts; + + internal class TargetFile : ITargetFile + { + public TargetFile(List versions) + { + this.Versions = versions; + } + + private List Versions { get; } + + public Task GetAllVersions(ActionBlock actionBlock, CancellationToken ct) + { + return Task.Run(() => + { + foreach (var version in this.Versions) + { + if (ct.IsCancellationRequested) + { + break; + } + + actionBlock.Post(version); + } + + actionBlock.Complete(); + }); + } + } +} diff --git a/Source/TargetMetadataStorage/InMemoryMetadataStorage/TargetFileVersion.cs b/Source/TargetMetadataStorage/InMemoryMetadataStorage/TargetFileVersion.cs new file mode 100644 index 0000000..f008de1 --- /dev/null +++ b/Source/TargetMetadataStorage/InMemoryMetadataStorage/TargetFileVersion.cs @@ -0,0 +1,24 @@ +namespace EternalArrowBackup.TargetMetadataStorage.InMemoryMetadataStorage +{ + using System; + using EternalArrowBackup.TargetMetadataStorage.Contracts; + + internal class TargetFileVersion : ITargetFileVersion + { + public TargetFileVersion(string filename, string hash, DateTime uploadDate, long size) + { + this.Filename = filename; + this.Hash = hash; + this.UploadDate = uploadDate; + this.Size = size; + } + + public string Filename { get; } + + public string Hash { get; } + + public DateTime UploadDate { get; } + + public long Size { get; } + } +}