Code cleanup; additional information

README.md

@@ -13,9 +13,12 @@ WhiteRabbit.exe < wordlist
 Performance
 ===========
 
+Memory usage is minimal (for that kind of task), around 10-30MB.
+
 This solution is partially optimized for multi-threading.
 
 It is also somewhat optimized for likely intended phrases, as anagrams consisting of longer words are generated first.
+That's why the given hashes are solved much sooner than it takes to check all anagrams.
 
 Single-threaded performance on Sandy Bridge @2.8GHz is as follows:
 
@@ -29,6 +32,10 @@ Anagrams generation is not parallelized, as even single-threaded performance for
 
 Multi-threaded performance is as follows:
 
-* If only phrases of at most 4 words are allowed, then it takes 20 seconds to find and check all anagrams; all hashes are solved in first 1.5 seconds
+* If only phrases of at most 4 words are allowed, then it takes 20 seconds to find and check all anagrams; all hashes are solved in first 1 second.
 
 * If phrases of 5 words are allowed as well, then it takes around half an hour to find and check all anagrams; all hashes are solved in first 25 seconds. Around 50% of time is spent on MD5 computations for correct anagrams, so there is not a lot to optimize further.
+
+* If phrases of 6 words are allowed as well, then "more difficult" hash is solved in 50 seconds, "easiest" in 3.5 minutes, and "hard" in 6 minutes.
+
+* If phrases of 7 words are allowed as well, then "more difficult" hash is solved in 6 minutes.

WhiteRabbit/Program.cs

@@ -25,7 +25,6 @@
             var stopwatch = new Stopwatch();
             stopwatch.Start();
 
-            var processor = new StringsProcessor(Encoding.ASCII.GetBytes(SourcePhrase), MaxWordsInPhrase);
             var expectedHashes = new[]
             {
                 "e4820b45d2277f3844eac66c903e84be",
@@ -35,7 +34,11 @@
 
             var expectedHashesAsVectors = expectedHashes.Select(hash => new Vector<byte>(HexadecimalStringToByteArray(hash))).ToArray();
 
-            processor.GeneratePhrases(ReadInput())
+            var processor = new StringsProcessor(Encoding.ASCII.GetBytes(SourcePhrase), MaxWordsInPhrase, ReadInput());
+
+            Console.WriteLine($"Initialization complete; time spent: {stopwatch.Elapsed}");
+
+            processor.GeneratePhrases()
                 .Select(phraseBytes => new { phraseBytes, hashVector = ComputeHashVector(phraseBytes) })
                 .Where(tuple => expectedHashesAsVectors.Contains(tuple.hashVector))
                 .Select(tuple => new { phrase = Encoding.ASCII.GetString(tuple.phraseBytes), hash = VectorToHexadecimalString(tuple.hashVector) })
@@ -54,6 +57,7 @@
                              .ToArray();
         }
 
+        // Bouncy Castle is used instead of standard .NET methods for performance reasons
         private static Vector<byte> ComputeHashVector(byte[] input)
         {
             var digest = new MD5Digest();
@@ -75,8 +79,6 @@
             {
                 yield return Encoding.ASCII.GetBytes(line);
             }
-
-            //System.Threading.Thread.Sleep(10000);
         }
     }
 }

WhiteRabbit/StringsProcessor.cs

@@ -4,40 +4,45 @@
     using System.Collections.Generic;
     using System.Collections.Immutable;
     using System.Linq;
+    using System.Numerics;
 
     internal class StringsProcessor
     {
-        public StringsProcessor(byte[] sourceString, int maxWordsCount)
+        public StringsProcessor(byte[] sourceString, int maxWordsCount, IEnumerable<byte[]> words)
         {
             var filteredSource = sourceString.Where(ch => ch != 32).ToArray();
             this.VectorsConverter = new VectorsConverter(filteredSource);
+
+            // Dictionary of vectors to array of words represented by this vector
+            this.VectorsToWords = words
+                .Distinct(new ByteArrayEqualityComparer())
+                .Select(word => new { word, vector = this.VectorsConverter.GetVector(word) })
+                .Where(tuple => tuple.vector != null)
+                .Select(tuple => new { tuple.word, vector = tuple.vector.Value })
+                .GroupBy(tuple => tuple.vector)
+                .ToDictionary(group => group.Key, group => group.Select(tuple => tuple.word).ToArray());
+
             this.VectorsProcessor = new VectorsProcessor(
                 this.VectorsConverter.GetVector(filteredSource).Value,
                 maxWordsCount,
+                this.VectorsToWords.Keys,
                 this.VectorsConverter.GetString);
         }
 
         private VectorsConverter VectorsConverter { get; }
 
+        private Dictionary<Vector<byte>, byte[][]> VectorsToWords { get; }
+
         private VectorsProcessor VectorsProcessor { get; }
 
-        public ParallelQuery<byte[]> GeneratePhrases(IEnumerable<byte[]> words)
+        public ParallelQuery<byte[]> GeneratePhrases()
         {
-            // Dictionary of vectors to array of words represented by this vector
-            var formattedWords = words
-                .Distinct(new ByteArrayEqualityComparer())
-                .Select(word => new { word, vector = this.VectorsConverter.GetVector(word) })
-                .Where(tuple => tuple.vector != null)
-                .Select(tuple => new { tuple.word, vector = tuple.vector.Value })
-                .GroupBy(tuple => tuple.vector)
-                .ToDictionary(group => group.Key, group => group.Select(tuple => tuple.word).ToArray());
-
             // task of finding anagrams could be reduced to the task of finding sequences of dictionary vectors with the target sum
-            var sums = this.VectorsProcessor.GenerateSequences(formattedWords.Keys);
+            var sums = this.VectorsProcessor.GenerateSequences();
 
             // converting sequences of vectors to the sequences of words...
             var anagramsWords = sums
-                .Select(sum => ImmutableStack.Create(sum.Select(vector => formattedWords[vector]).ToArray()))
+                .Select(sum => ImmutableStack.Create(sum.Select(vector => this.VectorsToWords[vector]).ToArray()))
                 .SelectMany(this.Flatten)
                 .Select(stack => stack.ToArray());
 

WhiteRabbit/VectorsProcessor.cs

@@ -9,11 +9,18 @@
 
     internal class VectorsProcessor
     {
-        public VectorsProcessor(Vector<byte> target, int maxVectorsCount, Func<Vector<byte>, string> vectorToString)
+        // Ensure that permutations are precomputed prior to main run, so that processing times will be correct
+        static VectorsProcessor()
+        {
+            PrecomputedPermutationsGenerator.HamiltonianPermutations(0);
+        }
+
+        public VectorsProcessor(Vector<byte> target, int maxVectorsCount, IEnumerable<Vector<byte>> dictionary, Func<Vector<byte>, string> vectorToString)
         {
             this.Target = target;
             this.MaxVectorsCount = maxVectorsCount;
             this.VectorToString = vectorToString;
+            this.Dictionary = ImmutableStack.Create<Vector<byte>>(FilterVectors(dictionary, target).ToArray());
         }
 
         /// <summary>
@@ -28,16 +35,16 @@
 
         private int MaxVectorsCount { get; }
 
+        private ImmutableStack<Vector<byte>> Dictionary { get; }
+
         private Func<Vector<byte>, string> VectorToString { get; }
 
         private long Iterations { get; set; } = 0;
 
         // Produces all sequences of vectors with the target sum
-        public ParallelQuery<Vector<byte>[]> GenerateSequences(IEnumerable<Vector<byte>> vectors)
+        public ParallelQuery<Vector<byte>[]> GenerateSequences()
         {
-            var filteredVectors = this.FilterVectors(vectors);
-            var dictionary = ImmutableStack.Create(filteredVectors.ToArray());
-            var unorderedSequences = this.GenerateUnorderedSequences(this.Target, ImmutableStack.Create<Vector<byte>>(), dictionary)
+            var unorderedSequences = this.GenerateUnorderedSequences(this.Target, ImmutableStack.Create<Vector<byte>>(), this.Dictionary)
                 .AsParallel();
             var allSequences = unorderedSequences.SelectMany(this.GeneratePermutations);
 
@@ -50,22 +57,22 @@
         // And total number of quintuplets becomes reasonable 1412M.
         // Also, it produces the intended results faster (as these are more likely to contain longer words - e.g. "poultry outwits ants" is more likely than "p o u l t r y o u t w i t s a n t s").
         // This method basically gives us the 1-norm of the vector in the space rescaled so that the target is [1, 1, ..., 1].
-        private int GetVectorWeight(Vector<byte> vector)
+        private static int GetVectorWeight(Vector<byte> vector, Vector<byte> target)
         {
             var weight = 0;
-            for (var i = 0; this.Target[i] != 0; i++)
+            for (var i = 0; target[i] != 0; i++)
             {
-                weight += (720 * vector[i]) / this.Target[i]; // 720 = 6!, so that the result will be a whole number (unless Target[i] > 6)
+                weight += (720 * vector[i]) / target[i]; // 720 = 6!, so that the result will be a whole number (unless Target[i] > 6)
             }
 
             return weight;
         }
 
-        private IEnumerable<Vector<byte>> FilterVectors(IEnumerable<Vector<byte>> vectors)
+        private static IEnumerable<Vector<byte>> FilterVectors(IEnumerable<Vector<byte>> vectors, Vector<byte> target)
         {
             return vectors
-                .Where(vector => ((this.Target - vector) & Negative) == Vector<byte>.Zero)
-                .OrderBy(GetVectorWeight);
+                .Where(vector => ((target - vector) & Negative) == Vector<byte>.Zero)
+                .OrderBy(vector => GetVectorWeight(vector, target));
         }
 
         [Conditional("DEBUG")]