Improved dataflow integration

WhiteRabbit/DataflowBlockHelpers.cs

@@ -0,0 +1,87 @@
+namespace WhiteRabbit
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Threading.Tasks;
+    using System.Threading.Tasks.Dataflow;
+
+    internal static class DataflowBlockHelpers
+    {
+        private static ExecutionDataflowBlockOptions ExecutionOptions { get; } = new ExecutionDataflowBlockOptions
+        {
+            BoundedCapacity = 100000,
+        };
+
+        public static IPropagatorBlock<T, T> Id<T>()
+        {
+            return new TransformBlock<T, T>(element => element, ExecutionOptions);
+        }
+
+        public static void WriteToTargetBlock<T>(this IEnumerable<T> enumerable, ITargetBlock<T> target)
+        {
+            var block = new TransformBlock<T, T>(line => line, ExecutionOptions);
+            block.LinkForever(target);
+
+            WriteToTargetBlockAsync(enumerable, block).Wait();
+            block.Complete();
+        }
+
+        public static IPropagatorBlock<TInput, TOutput> PipeMany<TInput, TIntermediate, TOutput>(this IPropagatorBlock<TInput, TIntermediate> source, Func<TIntermediate, IEnumerable<TOutput>> mapper)
+        {
+            return source.Pipe(new TransformManyBlock<TIntermediate, TOutput>(mapper, ExecutionOptions));
+        }
+
+        public static IPropagatorBlock<TInput, TOutput> Pipe<TInput, TIntermediate, TOutput>(this IPropagatorBlock<TInput, TIntermediate> source, Func<TIntermediate, TOutput> mapper)
+        {
+            return source.Pipe(new TransformBlock<TIntermediate, TOutput>(mapper, ExecutionOptions));
+        }
+
+        public static IPropagatorBlock<TInput, TOutput> Pipe<TInput, TIntermediate, TOutput>(this IPropagatorBlock<TInput, TIntermediate> source, IPropagatorBlock<TIntermediate, TOutput> target)
+        {
+            source.LinkForever(target);
+            return DataflowBlock.Encapsulate(source, target);
+        }
+
+        public static ISourceBlock<TOutput> Pipe<TInput, TOutput>(this ISourceBlock<TInput> source, Func<TInput, TOutput> mapper)
+        {
+            return source.Pipe(new TransformBlock<TInput, TOutput>(mapper, ExecutionOptions));
+        }
+
+        public static ISourceBlock<TOutput> Pipe<TInput, TOutput>(this ISourceBlock<TInput> source, IPropagatorBlock<TInput, TOutput> target)
+        {
+            source.LinkForever(target);
+            return target;
+        }
+
+        public static Task LinkForever<TOutput>(this ISourceBlock<TOutput> source, Action<TOutput> action)
+        {
+            return source.LinkForever(new ActionBlock<TOutput>(action, ExecutionOptions));
+        }
+
+        public static Task LinkForever<TOutput>(this ISourceBlock<TOutput> source, ITargetBlock<TOutput> target)
+        {
+            source.LinkTo(target);
+            source.Completion.ContinueWith(t =>
+            {
+                if (t.IsFaulted)
+                {
+                    target.Fault(t.Exception);
+                }
+                else
+                {
+                    target.Complete();
+                }
+            });
+
+            return target.Completion;
+        }
+
+        private static async Task WriteToTargetBlockAsync<T>(IEnumerable<T> enumerable, ITargetBlock<T> target)
+        {
+            foreach (var element in enumerable)
+            {
+                await target.SendAsync(element);
+            }
+        }
+    }
+}

WhiteRabbit/Extensions.cs

@@ -1,23 +0,0 @@
-namespace WhiteRabbit
-{
-    using System.Threading.Tasks.Dataflow;
-
-    internal static class Extensions
-    {
-        public static void LinkForever<TOutput>(this ISourceBlock<TOutput> source, ITargetBlock<TOutput> target)
-        {
-            source.LinkTo(target);
-            source.Completion.ContinueWith(t =>
-            {
-                if (t.IsFaulted)
-                {
-                    target.Fault(t.Exception);
-                }
-                else
-                {
-                    target.Complete();
-                }
-            });
-        }
-    }
-}

WhiteRabbit/Program.cs

@@ -7,7 +7,7 @@
     using System.Numerics;
     using System.Security.Cryptography;
     using System.Text;
-    using System.Threading.Tasks.Dataflow;
+
     /// <summary>
     /// Main class
     /// </summary>
@@ -35,16 +35,21 @@
 
                 var processor = new StringsProcessor("poultry outwits ants", 4, ReadInput());
 
-                var unorderedSequencesToPhrases = processor.CreateUnorderedSequencesToPhrasesTransform();
+                var startBlock = DataflowBlockHelpers.Id<Vector<byte>[]>();
 
-                var phrasesToPhrasesWithHash = new TransformBlock<string, PhraseWithHash>(phrase =>
+                var task = startBlock
+                    .Pipe(processor.CreateUnorderedSequencesToPhrasesTransform())
+                    .Pipe(phrase =>
                     {
+                        //Console.WriteLine("Found phrase: " + phrase);
+
                         var hash = new Vector<byte>(hasher.ComputeHash(Encoding.ASCII.GetBytes(phrase)));
                         return new PhraseWithHash(phrase, hash);
-                });
-
-                var phrasesWithHashFilter = new TransformManyBlock<PhraseWithHash, PhraseWithHash>(phraseWithHash =>
+                    })
+                    .PipeMany(phraseWithHash =>
                     {
+                        //Console.WriteLine($"Found phrase with hash: " + phraseWithHash.Phrase);
+
                         if (!expectedHashesAsVectors.Contains(phraseWithHash.Hash))
                         {
                             return Enumerable.Empty<PhraseWithHash>();
@@ -54,20 +59,16 @@
                         {
                             phraseWithHash,
                         };
-                });
-
-                var printPhrases = new ActionBlock<PhraseWithHash>(phraseWithHash =>
+                    })
+                    .LinkForever(phraseWithHash =>
                     {
                         Console.WriteLine($"Found phrase for hash {phraseWithHash.Hash}: {phraseWithHash.Phrase} (spent {stopwatch.Elapsed})");
                     });
 
-                unorderedSequencesToPhrases.LinkForever(phrasesToPhrasesWithHash);
-                phrasesToPhrasesWithHash.LinkForever(phrasesWithHashFilter);
-                phrasesWithHashFilter.LinkForever(printPhrases);
-
-                processor.PostUnorderedSequences(unorderedSequencesToPhrases);
+                Console.WriteLine($"Initialization complete: time spent: {stopwatch.Elapsed}");
+                processor.PostUnorderedSequences(startBlock);
 
-                printPhrases.Completion.Wait();
+                task.Wait();
                 Console.WriteLine($"Total time spent: {stopwatch.Elapsed}");
             }
         }

WhiteRabbit/StringsProcessor.cs

@@ -34,20 +34,18 @@
 
         private Dictionary<Vector<byte>, string[]> VectorsToWords { get; }
 
-        public void PostUnorderedSequences(ITargetBlock<Vector<byte>[]> target) => this.VectorsProcessor.PostUnorderedSequences(target);
+        public void PostUnorderedSequences(ITargetBlock<Vector<byte>[]> target)
+        {
+            this.VectorsProcessor.GenerateUnorderedSequences().WriteToTargetBlock(target);
+        }
 
         public IPropagatorBlock<Vector<byte>[], string> CreateUnorderedSequencesToPhrasesTransform()
         {
-            var unorderedSequencesToOrderedSequences = this.VectorsProcessor.CreateUnorderedSequencesToOrderedSequencesTransform();
-            var orderedSequencesToWordVariants = this.CreateOrderedSequencesToWordVariantsTransform();
-            var wordVariantsToFlatWords = this.CreateWordVariantsToFlatWordsTransform();
-            var flatWordsToPhrases = this.CreateFlatWordsToPhrasesTransform();
-
-            unorderedSequencesToOrderedSequences.LinkForever(orderedSequencesToWordVariants);
-            orderedSequencesToWordVariants.LinkForever(wordVariantsToFlatWords);
-            wordVariantsToFlatWords.LinkForever(flatWordsToPhrases);
-
-            return DataflowBlock.Encapsulate(unorderedSequencesToOrderedSequences, flatWordsToPhrases);
+            return DataflowBlockHelpers.Id<Vector<byte>[]>()
+                .PipeMany(this.VectorsProcessor.UnorderedSequenceToOrderedSequences)
+                .Pipe(this.OrderedSequenceToWordVariants)
+                .PipeMany(this.WordVariantsToFlatWords)
+                .Pipe(this.FlatWordsToPhrase);
         }
 
         // Converts e.g. pair of variants [[a, b, c], [d, e]] into all possible pairs: [[a, d], [a, e], [b, d], [b, e], [c, d], [c, e]]
@@ -63,28 +61,19 @@
             return this.Flatten(newStack).SelectMany(remainder => wordVariants.Select(word => remainder.Push(word)));
         }
 
-        private IPropagatorBlock<Vector<byte>[], ImmutableStack<string[]>> CreateOrderedSequencesToWordVariantsTransform()
-        {
-            return new TransformBlock<Vector<byte>[], ImmutableStack<string[]>>(sum =>
+        private ImmutableStack<string[]> OrderedSequenceToWordVariants(Vector<byte>[] sum)
         {
             return ImmutableStack.CreateRange(sum.Select(vector => this.VectorsToWords[vector]));
-            });
         }
 
-        private IPropagatorBlock<ImmutableStack<string[]>, ImmutableStack<string>> CreateWordVariantsToFlatWordsTransform()
-        {
-            return new TransformManyBlock<ImmutableStack<string[]>, ImmutableStack<string>>(wordVariants =>
+        private IEnumerable<ImmutableStack<string>> WordVariantsToFlatWords(ImmutableStack<string[]> wordVariants)
         {
             return this.Flatten(wordVariants);
-            });
         }
 
-        private IPropagatorBlock<ImmutableStack<string>, string> CreateFlatWordsToPhrasesTransform()
-        {
-            return new TransformBlock<ImmutableStack<string>, string>(words =>
+        private string FlatWordsToPhrase(ImmutableStack<string> words)
         {
             return string.Join(" ", words);
-            });
         }
     }
 }

WhiteRabbit/VectorsProcessor.cs

@@ -6,6 +6,7 @@
     using System.Diagnostics;
     using System.Linq;
     using System.Numerics;
+    using System.Threading.Tasks;
     using System.Threading.Tasks.Dataflow;
     internal class VectorsProcessor
     {
@@ -37,23 +38,17 @@
 
         private long Iterations { get; set; } = 0;
 
-        public IPropagatorBlock<Vector<byte>[], Vector<byte>[]> CreateUnorderedSequencesToOrderedSequencesTransform()
+        public IEnumerable<Vector<byte>[]> UnorderedSequenceToOrderedSequences(Vector<byte>[] sequence)
         {
-            return new TransformManyBlock<Vector<byte>[], Vector<byte>[]>(sequence =>
+            foreach (var permutation in PrecomputedPermutationsGenerator.HamiltonianPermutations(sequence.Length))
             {
-                return this.GeneratePermutations(sequence);
-            });
+                yield return permutation.Select(i => sequence[i]).ToArray();
             }
-
-        public void PostUnorderedSequences(ITargetBlock<Vector<byte>[]> target)
-        {
-            var sequences = this.GenerateUnorderedSequences(this.Target, ImmutableStack.Create<Vector<byte>>(), this.Vectors);
-            foreach (var sequence in sequences)
-            {
-                target.Post(sequence);
         }
 
-            target.Complete();
+        public IEnumerable<Vector<byte>[]> GenerateUnorderedSequences()
+        {
+            return this.GenerateUnorderedSequences(this.Target, ImmutableStack.Create<Vector<byte>>(), this.Vectors);
         }
 
         // We want words with more letters (and among these, words with more "rare" letters) to appear first, to reduce the searching time somewhat.
@@ -79,7 +74,7 @@
                 .OrderBy(vector => GetVectorWeight(vector, target));
         }
 
-        [Conditional("DEBUG")]
+        [Conditional("XDEBUG")]
         private void DebugState(ImmutableStack<Vector<byte>> partialSumStack, Vector<byte> currentVector)
         {
             this.Iterations++;
@@ -140,13 +135,5 @@
                 }
             }
         }
-
-        private IEnumerable<T[]> GeneratePermutations<T>(T[] original)
-        {
-            foreach (var permutation in PrecomputedPermutationsGenerator.HamiltonianPermutations(original.Length))
-            {
-                yield return permutation.Select(i => original[i]).ToArray();
-            }
-        }
     }
 }

WhiteRabbit/WhiteRabbit.csproj

@@ -57,7 +57,7 @@
     <Reference Include="System.Xml" />
   </ItemGroup>
   <ItemGroup>
-    <Compile Include="Extensions.cs" />
+    <Compile Include="DataflowBlockHelpers.cs" />
     <Compile Include="PrecomputedPermutationsGenerator.cs" />
     <Compile Include="PermutationsGenerator.cs" />
     <Compile Include="StringsProcessor.cs" />