Improved dataflow integration

WhiteRabbit/DataflowBlockHelpers.cs

@@ -0,0 +1,87 @@
+namespace WhiteRabbit
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Threading.Tasks;
+    using System.Threading.Tasks.Dataflow;
+
+    internal static class DataflowBlockHelpers
+    {
+        private static ExecutionDataflowBlockOptions ExecutionOptions { get; } = new ExecutionDataflowBlockOptions
+        {
+            BoundedCapacity = 100000,
+        };
+
+        public static IPropagatorBlock<T, T> Id<T>()
+        {
+            return new TransformBlock<T, T>(element => element, ExecutionOptions);
+        }
+
+        public static void WriteToTargetBlock<T>(this IEnumerable<T> enumerable, ITargetBlock<T> target)
+        {
+            var block = new TransformBlock<T, T>(line => line, ExecutionOptions);
+            block.LinkForever(target);
+
+            WriteToTargetBlockAsync(enumerable, block).Wait();
+            block.Complete();
+        }
+
+        public static IPropagatorBlock<TInput, TOutput> PipeMany<TInput, TIntermediate, TOutput>(this IPropagatorBlock<TInput, TIntermediate> source, Func<TIntermediate, IEnumerable<TOutput>> mapper)
+        {
+            return source.Pipe(new TransformManyBlock<TIntermediate, TOutput>(mapper, ExecutionOptions));
+        }
+
+        public static IPropagatorBlock<TInput, TOutput> Pipe<TInput, TIntermediate, TOutput>(this IPropagatorBlock<TInput, TIntermediate> source, Func<TIntermediate, TOutput> mapper)
+        {
+            return source.Pipe(new TransformBlock<TIntermediate, TOutput>(mapper, ExecutionOptions));
+        }
+
+        public static IPropagatorBlock<TInput, TOutput> Pipe<TInput, TIntermediate, TOutput>(this IPropagatorBlock<TInput, TIntermediate> source, IPropagatorBlock<TIntermediate, TOutput> target)
+        {
+            source.LinkForever(target);
+            return DataflowBlock.Encapsulate(source, target);
+        }
+
+        public static ISourceBlock<TOutput> Pipe<TInput, TOutput>(this ISourceBlock<TInput> source, Func<TInput, TOutput> mapper)
+        {
+            return source.Pipe(new TransformBlock<TInput, TOutput>(mapper, ExecutionOptions));
+        }
+
+        public static ISourceBlock<TOutput> Pipe<TInput, TOutput>(this ISourceBlock<TInput> source, IPropagatorBlock<TInput, TOutput> target)
+        {
+            source.LinkForever(target);
+            return target;
+        }
+
+        public static Task LinkForever<TOutput>(this ISourceBlock<TOutput> source, Action<TOutput> action)
+        {
+            return source.LinkForever(new ActionBlock<TOutput>(action, ExecutionOptions));
+        }
+
+        public static Task LinkForever<TOutput>(this ISourceBlock<TOutput> source, ITargetBlock<TOutput> target)
+        {
+            source.LinkTo(target);
+            source.Completion.ContinueWith(t =>
+            {
+                if (t.IsFaulted)
+                {
+                    target.Fault(t.Exception);
+                }
+                else
+                {
+                    target.Complete();
+                }
+            });
+
+            return target.Completion;
+        }
+
+        private static async Task WriteToTargetBlockAsync<T>(IEnumerable<T> enumerable, ITargetBlock<T> target)
+        {
+            foreach (var element in enumerable)
+            {
+                await target.SendAsync(element);
+            }
+        }
+    }
+}

WhiteRabbit/Program.cs

@@ -18,10 +18,6 @@
         /// </summary>
         public static void Main()
         {
-            var stopwatch = new Stopwatch();
-            stopwatch.Start();
-
-            var processor = new StringsProcessor("poultry outwits ants", 4);
             var expectedHashes = new[]
             {
                 "e4820b45d2277f3844eac66c903e84be",
@@ -31,17 +27,51 @@
 
             var expectedHashesAsVectors = new HashSet<Vector<byte>>(expectedHashes.Select(hash => new Vector<byte>(StringToByteArray(hash))));
 
-            foreach (var result in AddHashes(processor.GeneratePhrases(ReadInput())))
+            var stopwatch = new Stopwatch();
+            stopwatch.Start();
+
+            using (var hasher = MD5.Create())
+            {
+
+                var processor = new StringsProcessor("poultry outwits ants", 4, ReadInput());
+
+                var startBlock = DataflowBlockHelpers.Id<Vector<byte>[]>();
+
+                var task = startBlock
+                    .Pipe(processor.CreateUnorderedSequencesToPhrasesTransform())
+                    .Pipe(phrase =>
                     {
-                if (expectedHashesAsVectors.Contains(result.Item2))
+                        //Console.WriteLine("Found phrase: " + phrase);
+
+                        var hash = new Vector<byte>(hasher.ComputeHash(Encoding.ASCII.GetBytes(phrase)));
+                        return new PhraseWithHash(phrase, hash);
+                    })
+                    .PipeMany(phraseWithHash =>
                     {
-                    Console.WriteLine($"Found phrase: {result.Item1} (spent {stopwatch.Elapsed})");
-                }
+                        //Console.WriteLine($"Found phrase with hash: " + phraseWithHash.Phrase);
+
+                        if (!expectedHashesAsVectors.Contains(phraseWithHash.Hash))
+                        {
+                            return Enumerable.Empty<PhraseWithHash>();
                         }
 
-            stopwatch.Stop();
+                        return new PhraseWithHash[]
+                        {
+                            phraseWithHash,
+                        };
+                    })
+                    .LinkForever(phraseWithHash =>
+                    {
+                        Console.WriteLine($"Found phrase for hash {phraseWithHash.Hash}: {phraseWithHash.Phrase} (spent {stopwatch.Elapsed})");
+                    });
+
+                Console.WriteLine($"Initialization complete: time spent: {stopwatch.Elapsed}");
+                processor.PostUnorderedSequences(startBlock);
+
+                task.Wait();
                 Console.WriteLine($"Total time spent: {stopwatch.Elapsed}");
             }
+        }
 
         // Code taken from http://stackoverflow.com/a/321404/831314
         private static byte[] StringToByteArray(string hex)
@@ -72,5 +102,18 @@
                 yield return line;
             }
         }
+
+        private class PhraseWithHash
+        {
+            public PhraseWithHash(string phrase, Vector<byte> hash)
+            {
+                this.Phrase = phrase;
+                this.Hash = hash;
+            }
+
+            public string Phrase { get; }
+
+            public Vector<byte> Hash { get; }
+        }
     }
 }

WhiteRabbit/StringsProcessor.cs

@@ -3,27 +3,17 @@
     using System.Collections.Generic;
     using System.Collections.Immutable;
     using System.Linq;
-
+    using System.Numerics;
+    using System.Threading.Tasks.Dataflow;
     internal class StringsProcessor
     {
-        public StringsProcessor(string sourceString, int maxWordsCount)
+        public StringsProcessor(string sourceString, int maxWordsCount, IEnumerable<string> words)
         {
             var filteredSource = new string(sourceString.Where(ch => ch != ' ').ToArray());
             this.VectorsConverter = new VectorsConverter(filteredSource);
-            this.VectorsProcessor = new VectorsProcessor(
-                this.VectorsConverter.GetVector(filteredSource).Value,
-                maxWordsCount,
-                this.VectorsConverter.GetString);
-        }
-
-        private VectorsConverter VectorsConverter { get; }
 
-        private VectorsProcessor VectorsProcessor { get; }
-
-        public IEnumerable<string> GeneratePhrases(IEnumerable<string> words)
-        {
             // Dictionary of vectors to array of words represented by this vector
-            var formattedWords = words
+            this.VectorsToWords = words
                 .Distinct()
                 .Select(word => new { word, vector = this.VectorsConverter.GetVector(word) })
                 .Where(tuple => tuple.vector != null)
@@ -31,16 +21,31 @@
                 .GroupBy(tuple => tuple.vector)
                 .ToDictionary(group => group.Key, group => group.Select(tuple => tuple.word).ToArray());
 
-            // task of finding anagrams could be reduced to the task of finding sequences of dictionary vectors with the target sum
-            var sums = this.VectorsProcessor.GenerateSequences(formattedWords.Keys);
+            this.VectorsProcessor = new VectorsProcessor(
+                this.VectorsConverter.GetVector(filteredSource).Value,
+                maxWordsCount,
+                this.VectorsToWords.Keys,
+                this.VectorsConverter.GetString);
+        }
 
-            // converting sequences of vectors to the sequences of words...
-            var anagramsWords = sums
-                .Select(sum => ImmutableStack.Create(sum.Select(vector => formattedWords[vector]).ToArray()))
-                .SelectMany(this.Flatten)
-                .Select(stack => stack.ToArray());
+        private VectorsProcessor VectorsProcessor { get; }
 
-            return anagramsWords.Select(list => string.Join(" ", list));
+        private VectorsConverter VectorsConverter { get; }
+
+        private Dictionary<Vector<byte>, string[]> VectorsToWords { get; }
+
+        public void PostUnorderedSequences(ITargetBlock<Vector<byte>[]> target)
+        {
+            this.VectorsProcessor.GenerateUnorderedSequences().WriteToTargetBlock(target);
+        }
+
+        public IPropagatorBlock<Vector<byte>[], string> CreateUnorderedSequencesToPhrasesTransform()
+        {
+            return DataflowBlockHelpers.Id<Vector<byte>[]>()
+                .PipeMany(this.VectorsProcessor.UnorderedSequenceToOrderedSequences)
+                .Pipe(this.OrderedSequenceToWordVariants)
+                .PipeMany(this.WordVariantsToFlatWords)
+                .Pipe(this.FlatWordsToPhrase);
         }
 
         // Converts e.g. pair of variants [[a, b, c], [d, e]] into all possible pairs: [[a, d], [a, e], [b, d], [b, e], [c, d], [c, e]]
@@ -55,5 +60,20 @@
             var newStack = phrase.Pop(out wordVariants);
             return this.Flatten(newStack).SelectMany(remainder => wordVariants.Select(word => remainder.Push(word)));
         }
+
+        private ImmutableStack<string[]> OrderedSequenceToWordVariants(Vector<byte>[] sum)
+        {
+            return ImmutableStack.CreateRange(sum.Select(vector => this.VectorsToWords[vector]));
+        }
+
+        private IEnumerable<ImmutableStack<string>> WordVariantsToFlatWords(ImmutableStack<string[]> wordVariants)
+        {
+            return this.Flatten(wordVariants);
+        }
+
+        private string FlatWordsToPhrase(ImmutableStack<string> words)
+        {
+            return string.Join(" ", words);
+        }
     }
 }

WhiteRabbit/VectorsProcessor.cs

@@ -6,14 +6,18 @@
     using System.Diagnostics;
     using System.Linq;
     using System.Numerics;
-
+    using System.Threading.Tasks;
+    using System.Threading.Tasks.Dataflow;
     internal class VectorsProcessor
     {
-        public VectorsProcessor(Vector<byte> target, int maxVectorsCount, Func<Vector<byte>, string> vectorToString)
+        public VectorsProcessor(Vector<byte> target, int maxVectorsCount, IEnumerable<Vector<byte>> vectors, Func<Vector<byte>, string> vectorToString)
         {
             this.Target = target;
             this.MaxVectorsCount = maxVectorsCount;
             this.VectorToString = vectorToString;
+
+            var filteredVectors = FilterVectors(vectors, target);
+            this.Vectors = ImmutableStack.Create(filteredVectors.ToArray());
         }
 
         /// <summary>
@@ -28,19 +32,23 @@
 
         private int MaxVectorsCount { get; }
 
+        private ImmutableStack<Vector<byte>> Vectors { get; }
+
         private Func<Vector<byte>, string> VectorToString { get; }
 
         private long Iterations { get; set; } = 0;
 
-        // Produces all sequences of vectors with the target sum
-        public IEnumerable<Vector<byte>[]> GenerateSequences(IEnumerable<Vector<byte>> vectors)
+        public IEnumerable<Vector<byte>[]> UnorderedSequenceToOrderedSequences(Vector<byte>[] sequence)
         {
-            var filteredVectors = this.FilterVectors(vectors);
-            var dictionary = ImmutableStack.Create(filteredVectors.ToArray());
-            var unorderedSequences = this.GenerateUnorderedSequences(this.Target, ImmutableStack.Create<Vector<byte>>(), dictionary);
-            var allSequences = unorderedSequences.SelectMany(this.GeneratePermutations);
+            foreach (var permutation in PrecomputedPermutationsGenerator.HamiltonianPermutations(sequence.Length))
+            {
+                yield return permutation.Select(i => sequence[i]).ToArray();
+            }
+        }
 
-            return allSequences;
+        public IEnumerable<Vector<byte>[]> GenerateUnorderedSequences()
+        {
+            return this.GenerateUnorderedSequences(this.Target, ImmutableStack.Create<Vector<byte>>(), this.Vectors);
         }
 
         // We want words with more letters (and among these, words with more "rare" letters) to appear first, to reduce the searching time somewhat.
@@ -48,25 +56,25 @@
         // Total number of quadruplets is reduced from 1468M to mere 311M.
         // Also, it produces the intended results faster (as these are more likely to contain longer words - e.g. "poultry outwits ants" is more likely than "p o u l t r y o u t w i t s a n t s").
         // This method basically gives us the 1-norm of the vector in the space rescaled so that the target is [1, 1, ..., 1].
-        private int GetVectorWeight(Vector<byte> vector)
+        private static int GetVectorWeight(Vector<byte> vector, Vector<byte> target)
         {
             var weight = 0;
-            for (var i = 0; this.Target[i] != 0; i++)
+            for (var i = 0; target[i] != 0; i++)
             {
-                weight += (720 * vector[i]) / this.Target[i]; // 720 = 6!, so that the result will be a whole number (unless Target[i] > 6)
+                weight += (720 * vector[i]) / target[i]; // 720 = 6!, so that the result will be a whole number (unless Target[i] > 6)
             }
 
             return weight;
         }
 
-        private IEnumerable<Vector<byte>> FilterVectors(IEnumerable<Vector<byte>> vectors)
+        private static IEnumerable<Vector<byte>> FilterVectors(IEnumerable<Vector<byte>> vectors, Vector<byte> target)
         {
             return vectors
-                .Where(vector => ((this.Target - vector) & Negative) == Vector<byte>.Zero)
-                .OrderBy(GetVectorWeight);
+                .Where(vector => ((target - vector) & Negative) == Vector<byte>.Zero)
+                .OrderBy(vector => GetVectorWeight(vector, target));
         }
 
-        [Conditional("DEBUG")]
+        [Conditional("XDEBUG")]
         private void DebugState(ImmutableStack<Vector<byte>> partialSumStack, Vector<byte> currentVector)
         {
             this.Iterations++;
@@ -127,13 +135,5 @@
                 }
             }
         }
-
-        private IEnumerable<T[]> GeneratePermutations<T>(T[] original)
-        {
-            foreach (var permutation in PrecomputedPermutationsGenerator.HamiltonianPermutations(original.Length))
-            {
-                yield return permutation.Select(i => original[i]).ToArray();
-            }
-        }
     }
 }

WhiteRabbit/WhiteRabbit.csproj

@@ -45,6 +45,10 @@
       <HintPath>..\packages\System.Numerics.Vectors.4.3.0\lib\net46\System.Numerics.Vectors.dll</HintPath>
       <Private>True</Private>
     </Reference>
+    <Reference Include="System.Threading.Tasks.Dataflow, Version=4.5.24.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
+      <HintPath>..\packages\Microsoft.Tpl.Dataflow.4.5.24\lib\portable-net45+win8+wpa81\System.Threading.Tasks.Dataflow.dll</HintPath>
+      <Private>True</Private>
+    </Reference>
     <Reference Include="System.Xml.Linq" />
     <Reference Include="System.Data.DataSetExtensions" />
     <Reference Include="Microsoft.CSharp" />
@@ -53,6 +57,7 @@
     <Reference Include="System.Xml" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="DataflowBlockHelpers.cs" />
     <Compile Include="PrecomputedPermutationsGenerator.cs" />
     <Compile Include="PermutationsGenerator.cs" />
     <Compile Include="StringsProcessor.cs" />

WhiteRabbit/packages.config

@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
+  <package id="Microsoft.Tpl.Dataflow" version="4.5.24" targetFramework="net46" />
   <package id="System.Collections.Immutable" version="1.3.1" targetFramework="net46" />
   <package id="System.Numerics.Vectors" version="4.3.0" targetFramework="net46" />
 </packages>