From 85dc08f5a85564d8fb9f0067abc5423c0bd8c3b2 Mon Sep 17 00:00:00 2001 From: inga-lovinde <52715130+inga-lovinde@users.noreply.github.com> Date: Thu, 9 Mar 2017 17:08:16 +0300 Subject: [PATCH] Improved dataflow integration --- WhiteRabbit/DataflowBlockHelpers.cs | 87 +++++++++++++++++++++++++++++ WhiteRabbit/Extensions.cs | 23 -------- WhiteRabbit/Program.cs | 57 +++++++++---------- WhiteRabbit/StringsProcessor.cs | 41 +++++--------- WhiteRabbit/VectorsProcessor.cs | 29 +++------- WhiteRabbit/WhiteRabbit.csproj | 2 +- 6 files changed, 140 insertions(+), 99 deletions(-) create mode 100644 WhiteRabbit/DataflowBlockHelpers.cs delete mode 100644 WhiteRabbit/Extensions.cs diff --git a/WhiteRabbit/DataflowBlockHelpers.cs b/WhiteRabbit/DataflowBlockHelpers.cs new file mode 100644 index 0000000..d9377a3 --- /dev/null +++ b/WhiteRabbit/DataflowBlockHelpers.cs @@ -0,0 +1,87 @@ +namespace WhiteRabbit +{ + using System; + using System.Collections.Generic; + using System.Threading.Tasks; + using System.Threading.Tasks.Dataflow; + + internal static class DataflowBlockHelpers + { + private static ExecutionDataflowBlockOptions ExecutionOptions { get; } = new ExecutionDataflowBlockOptions + { + BoundedCapacity = 100000, + }; + + public static IPropagatorBlock Id() + { + return new TransformBlock(element => element, ExecutionOptions); + } + + public static void WriteToTargetBlock(this IEnumerable enumerable, ITargetBlock target) + { + var block = new TransformBlock(line => line, ExecutionOptions); + block.LinkForever(target); + + WriteToTargetBlockAsync(enumerable, block).Wait(); + block.Complete(); + } + + public static IPropagatorBlock PipeMany(this IPropagatorBlock source, Func> mapper) + { + return source.Pipe(new TransformManyBlock(mapper, ExecutionOptions)); + } + + public static IPropagatorBlock Pipe(this IPropagatorBlock source, Func mapper) + { + return source.Pipe(new TransformBlock(mapper, ExecutionOptions)); + } + + public static IPropagatorBlock Pipe(this IPropagatorBlock source, IPropagatorBlock target) + { + source.LinkForever(target); + return DataflowBlock.Encapsulate(source, target); + } + + public static ISourceBlock Pipe(this ISourceBlock source, Func mapper) + { + return source.Pipe(new TransformBlock(mapper, ExecutionOptions)); + } + + public static ISourceBlock Pipe(this ISourceBlock source, IPropagatorBlock target) + { + source.LinkForever(target); + return target; + } + + public static Task LinkForever(this ISourceBlock source, Action action) + { + return source.LinkForever(new ActionBlock(action, ExecutionOptions)); + } + + public static Task LinkForever(this ISourceBlock source, ITargetBlock target) + { + source.LinkTo(target); + source.Completion.ContinueWith(t => + { + if (t.IsFaulted) + { + target.Fault(t.Exception); + } + else + { + target.Complete(); + } + }); + + return target.Completion; + } + + private static async Task WriteToTargetBlockAsync(IEnumerable enumerable, ITargetBlock target) + { + foreach (var element in enumerable) + { + await target.SendAsync(element); + } + } + } +} diff --git a/WhiteRabbit/Extensions.cs b/WhiteRabbit/Extensions.cs deleted file mode 100644 index 84c5f6b..0000000 --- a/WhiteRabbit/Extensions.cs +++ /dev/null @@ -1,23 +0,0 @@ -namespace WhiteRabbit -{ - using System.Threading.Tasks.Dataflow; - - internal static class Extensions - { - public static void LinkForever(this ISourceBlock source, ITargetBlock target) - { - source.LinkTo(target); - source.Completion.ContinueWith(t => - { - if (t.IsFaulted) - { - target.Fault(t.Exception); - } - else - { - target.Complete(); - } - }); - } - } -} diff --git a/WhiteRabbit/Program.cs b/WhiteRabbit/Program.cs index a5d0a6d..e3960b3 100644 --- a/WhiteRabbit/Program.cs +++ b/WhiteRabbit/Program.cs @@ -7,7 +7,7 @@ using System.Numerics; using System.Security.Cryptography; using System.Text; - using System.Threading.Tasks.Dataflow; + /// /// Main class /// @@ -35,39 +35,40 @@ var processor = new StringsProcessor("poultry outwits ants", 4, ReadInput()); - var unorderedSequencesToPhrases = processor.CreateUnorderedSequencesToPhrasesTransform(); - - var phrasesToPhrasesWithHash = new TransformBlock(phrase => - { - var hash = new Vector(hasher.ComputeHash(Encoding.ASCII.GetBytes(phrase))); - return new PhraseWithHash(phrase, hash); - }); + var startBlock = DataflowBlockHelpers.Id[]>(); - var phrasesWithHashFilter = new TransformManyBlock(phraseWithHash => - { - if (!expectedHashesAsVectors.Contains(phraseWithHash.Hash)) + var task = startBlock + .Pipe(processor.CreateUnorderedSequencesToPhrasesTransform()) + .Pipe(phrase => { - return Enumerable.Empty(); - } + //Console.WriteLine("Found phrase: " + phrase); - return new PhraseWithHash[] + var hash = new Vector(hasher.ComputeHash(Encoding.ASCII.GetBytes(phrase))); + return new PhraseWithHash(phrase, hash); + }) + .PipeMany(phraseWithHash => { - phraseWithHash, - }; - }); - - var printPhrases = new ActionBlock(phraseWithHash => - { - Console.WriteLine($"Found phrase for hash {phraseWithHash.Hash}: {phraseWithHash.Phrase} (spent {stopwatch.Elapsed})"); - }); - - unorderedSequencesToPhrases.LinkForever(phrasesToPhrasesWithHash); - phrasesToPhrasesWithHash.LinkForever(phrasesWithHashFilter); - phrasesWithHashFilter.LinkForever(printPhrases); + //Console.WriteLine($"Found phrase with hash: " + phraseWithHash.Phrase); + + if (!expectedHashesAsVectors.Contains(phraseWithHash.Hash)) + { + return Enumerable.Empty(); + } + + return new PhraseWithHash[] + { + phraseWithHash, + }; + }) + .LinkForever(phraseWithHash => + { + Console.WriteLine($"Found phrase for hash {phraseWithHash.Hash}: {phraseWithHash.Phrase} (spent {stopwatch.Elapsed})"); + }); - processor.PostUnorderedSequences(unorderedSequencesToPhrases); + Console.WriteLine($"Initialization complete: time spent: {stopwatch.Elapsed}"); + processor.PostUnorderedSequences(startBlock); - printPhrases.Completion.Wait(); + task.Wait(); Console.WriteLine($"Total time spent: {stopwatch.Elapsed}"); } } diff --git a/WhiteRabbit/StringsProcessor.cs b/WhiteRabbit/StringsProcessor.cs index 93638b5..a9c7e2a 100644 --- a/WhiteRabbit/StringsProcessor.cs +++ b/WhiteRabbit/StringsProcessor.cs @@ -34,20 +34,18 @@ private Dictionary, string[]> VectorsToWords { get; } - public void PostUnorderedSequences(ITargetBlock[]> target) => this.VectorsProcessor.PostUnorderedSequences(target); + public void PostUnorderedSequences(ITargetBlock[]> target) + { + this.VectorsProcessor.GenerateUnorderedSequences().WriteToTargetBlock(target); + } public IPropagatorBlock[], string> CreateUnorderedSequencesToPhrasesTransform() { - var unorderedSequencesToOrderedSequences = this.VectorsProcessor.CreateUnorderedSequencesToOrderedSequencesTransform(); - var orderedSequencesToWordVariants = this.CreateOrderedSequencesToWordVariantsTransform(); - var wordVariantsToFlatWords = this.CreateWordVariantsToFlatWordsTransform(); - var flatWordsToPhrases = this.CreateFlatWordsToPhrasesTransform(); - - unorderedSequencesToOrderedSequences.LinkForever(orderedSequencesToWordVariants); - orderedSequencesToWordVariants.LinkForever(wordVariantsToFlatWords); - wordVariantsToFlatWords.LinkForever(flatWordsToPhrases); - - return DataflowBlock.Encapsulate(unorderedSequencesToOrderedSequences, flatWordsToPhrases); + return DataflowBlockHelpers.Id[]>() + .PipeMany(this.VectorsProcessor.UnorderedSequenceToOrderedSequences) + .Pipe(this.OrderedSequenceToWordVariants) + .PipeMany(this.WordVariantsToFlatWords) + .Pipe(this.FlatWordsToPhrase); } // Converts e.g. pair of variants [[a, b, c], [d, e]] into all possible pairs: [[a, d], [a, e], [b, d], [b, e], [c, d], [c, e]] @@ -63,28 +61,19 @@ return this.Flatten(newStack).SelectMany(remainder => wordVariants.Select(word => remainder.Push(word))); } - private IPropagatorBlock[], ImmutableStack> CreateOrderedSequencesToWordVariantsTransform() + private ImmutableStack OrderedSequenceToWordVariants(Vector[] sum) { - return new TransformBlock[], ImmutableStack>(sum => - { - return ImmutableStack.CreateRange(sum.Select(vector => this.VectorsToWords[vector])); - }); + return ImmutableStack.CreateRange(sum.Select(vector => this.VectorsToWords[vector])); } - private IPropagatorBlock, ImmutableStack> CreateWordVariantsToFlatWordsTransform() + private IEnumerable> WordVariantsToFlatWords(ImmutableStack wordVariants) { - return new TransformManyBlock, ImmutableStack>(wordVariants => - { - return this.Flatten(wordVariants); - }); + return this.Flatten(wordVariants); } - private IPropagatorBlock, string> CreateFlatWordsToPhrasesTransform() + private string FlatWordsToPhrase(ImmutableStack words) { - return new TransformBlock, string>(words => - { - return string.Join(" ", words); - }); + return string.Join(" ", words); } } } diff --git a/WhiteRabbit/VectorsProcessor.cs b/WhiteRabbit/VectorsProcessor.cs index a33c16f..eeb0aa4 100644 --- a/WhiteRabbit/VectorsProcessor.cs +++ b/WhiteRabbit/VectorsProcessor.cs @@ -6,6 +6,7 @@ using System.Diagnostics; using System.Linq; using System.Numerics; + using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; internal class VectorsProcessor { @@ -37,23 +38,17 @@ private long Iterations { get; set; } = 0; - public IPropagatorBlock[], Vector[]> CreateUnorderedSequencesToOrderedSequencesTransform() + public IEnumerable[]> UnorderedSequenceToOrderedSequences(Vector[] sequence) { - return new TransformManyBlock[], Vector[]>(sequence => + foreach (var permutation in PrecomputedPermutationsGenerator.HamiltonianPermutations(sequence.Length)) { - return this.GeneratePermutations(sequence); - }); + yield return permutation.Select(i => sequence[i]).ToArray(); + } } - public void PostUnorderedSequences(ITargetBlock[]> target) + public IEnumerable[]> GenerateUnorderedSequences() { - var sequences = this.GenerateUnorderedSequences(this.Target, ImmutableStack.Create>(), this.Vectors); - foreach (var sequence in sequences) - { - target.Post(sequence); - } - - target.Complete(); + return this.GenerateUnorderedSequences(this.Target, ImmutableStack.Create>(), this.Vectors); } // We want words with more letters (and among these, words with more "rare" letters) to appear first, to reduce the searching time somewhat. @@ -79,7 +74,7 @@ .OrderBy(vector => GetVectorWeight(vector, target)); } - [Conditional("DEBUG")] + [Conditional("XDEBUG")] private void DebugState(ImmutableStack> partialSumStack, Vector currentVector) { this.Iterations++; @@ -140,13 +135,5 @@ } } } - - private IEnumerable GeneratePermutations(T[] original) - { - foreach (var permutation in PrecomputedPermutationsGenerator.HamiltonianPermutations(original.Length)) - { - yield return permutation.Select(i => original[i]).ToArray(); - } - } } } diff --git a/WhiteRabbit/WhiteRabbit.csproj b/WhiteRabbit/WhiteRabbit.csproj index a7fc331..8674654 100644 --- a/WhiteRabbit/WhiteRabbit.csproj +++ b/WhiteRabbit/WhiteRabbit.csproj @@ -57,7 +57,7 @@ - +