Optimized initialization, support for 10-word phrases, updated performance measurements

README.md

@@ -34,49 +34,36 @@ WhiteRabbit.exe < wordlist
 Performance
 ===========
 
-Memory usage is minimal (for that kind of task), less than 10MB.
+Memory usage is minimal (for that kind of task), less than 10MB (25MB for MaxNumberOfWords = 8).
 
 It is also somewhat optimized for likely intended phrases, as anagrams consisting of longer words are generated first.
 That's why the given hashes are solved much sooner than it takes to check all anagrams.
 
 Anagrams generation is not parallelized, as even single-threaded performance for 4-word anagrams is high enough; and 5-word (or larger) anagrams are frequent enough for most of the time being spent on computing hashes, with full CPU load.
 
-Multi-threaded performance with RyuJIT (.NET 4.6, 64-bit system) on laptop with dual-core Sandy Bridge @2.6GHz (without AVX2 support) is as follows (excluding initialization time of 0.2 seconds), for different maximum allowed words in an anagram:
+Multi-threaded performance with RyuJIT (.NET 4.6, 64-bit system) on i5-6500 is as follows (excluding initialization time of 0.2 seconds), for different maximum allowed words in an anagram:
 
-Number of words|Time to check all anagrams no longer than that|Time to solve "easy" hash|Time to solve "more difficult" hash|Time to solve "hard" hash|Number of anagrams no longer than that (see note below)
----------------|----------------------------------------------|-------------------------|-----------------------------------|-------------------------|-------------------------------------------------------
+Number of words|Time to check all anagrams no longer than that|Time to solve "easy" hash|Time to solve "more difficult" hash|Time to solve "hard" hash|Number of unique anagrams no longer than that
+---------------|----------------------------------------------|-------------------------|-----------------------------------|-------------------------|---------------------------------------------
 3|0.1s||||4560
-4|0.8s|||0.15s|7,433,016
-5|87s|1s|0.3s|2s|1,348,876,896
-6|1 hour (?)|10s|1.7s|25s|58,837,302,096
-7|11 hours (?)|78s|9s|3.5 minutes|1,108,328,708,976
-8||7.5 minutes|50s|20 minutes|12,089,249,231,856
-9|||||88,977,349,731,696
-10|||||482,627,715,786,096
-11|||||2,030,917,440,675,696
-12|||||6,813,402,098,518,896
-13|||||18,437,325,782,691,696
-14|||||40,367,286,468,925,296
-15|||||71,561,858,517,565,296
-16|||||103,280,807,987,773,296
-17|||||123,910,678,817,341,296
-18|||||130,313,052,523,069,296
+4|0.8s|||0.15s|7,431,984
+5|27s|0.4s|0.12s|0.75s|1,347,437,484
+6|16.5 minutes|3s|0.5s|7.3s|58,405,904,844
+7|5 hours|16s|2s|47s|1,070,307,744,114
+8|49 hours|61s|6.3s|3.2 minutes|10,893,594,396,594
+9||2.5 minutes|13s|9.5 minutes|70,596,864,409,954
+10||5 minutes|21s|17.5 minutes|314,972,701,475,754
 
 Note that all measurements were done on a Release build; Debug build is significantly slower.
 
 For comparison, certain other solutions available on GitHub seem to require 3 hours to find all 3-word anagrams. This solution is faster by 6-7 orders of magnitude (it finds and checks all 4-word anagrams in 1/10000th fraction of time required for other solution just to find all 3-word anagrams, with no MD5 calculations).
 
-Also, note that anagram counts are inflated for the sake of code simplicity.
-E.g. for phrase "aabbc" and dictionary [ab, ba, c] there are four possible set of words adding up to the source phrase: [ab, ab, c], [ab, ba, c], [ba, ab, c], [ba, ba, c].
-My implementation regards these sets as sets of different words, and applies all possible permutations to the every set, even if it will result in the same set.
-For the example above, my application would produce 24 anagrams (with six permutations for every of the four sets), although actually there are only 12 different anagrams.
-
 Conditional compilation symbols
 ===============================
 
 * Define `SINGLE_THREADED` to use standard enumerables instead of ParallelEnumerable (useful for profiling).
 
-* Define `DEBUG`, or build in debug mode, to get the total number of anagrams (not optimized, memory-hogging).
+* Define `DEBUG`, or build in debug mode, to get the total number of anagrams (not optimized).
 
 Implementation notes
 ====================

dotnet/WhiteRabbit.UnmanagedBridge/phraseset.cpp

@@ -40,75 +40,83 @@
 #define DONE_WORD(phraseNumber) \
     avx2buffer[phraseNumber] = phrase;
 
-#define REPEAT_WORDS3(phraseNumber) \
+#define REPEAT_WORDS_SIMPLE1(phraseNumber) \
     { \
-        INIT_WORD(phraseNumber); \
         PROCESS_WORD(phraseNumber, 0); \
-        PROCESS_WORD(phraseNumber, 1); \
-        PROCESS_WORD(phraseNumber, 2); \
-        DONE_WORD(phraseNumber); \
     }
 
-#define REPEAT_WORDS4(phraseNumber) \
+#define REPEAT_WORDS_SIMPLE2(phraseNumber) \
     { \
-        INIT_WORD(phraseNumber); \
-        PROCESS_WORD(phraseNumber, 0); \
+        REPEAT_WORDS_SIMPLE1(phraseNumber); \
         PROCESS_WORD(phraseNumber, 1); \
-        PROCESS_WORD(phraseNumber, 2); \
-        PROCESS_WORD(phraseNumber, 3); \
-        DONE_WORD(phraseNumber); \
     }
 
-#define REPEAT_WORDS5(phraseNumber) \
+#define REPEAT_WORDS_SIMPLE3(phraseNumber) \
     { \
-        INIT_WORD(phraseNumber); \
-        PROCESS_WORD(phraseNumber, 0); \
-        PROCESS_WORD(phraseNumber, 1); \
+        REPEAT_WORDS_SIMPLE2(phraseNumber); \
         PROCESS_WORD(phraseNumber, 2); \
-        PROCESS_WORD(phraseNumber, 3); \
-        PROCESS_WORD(phraseNumber, 4); \
-        DONE_WORD(phraseNumber); \
     }
 
-#define REPEAT_WORDS6(phraseNumber) \
+#define REPEAT_WORDS_SIMPLE4(phraseNumber) \
     { \
-        INIT_WORD(phraseNumber); \
-        PROCESS_WORD(phraseNumber, 0); \
-        PROCESS_WORD(phraseNumber, 1); \
-        PROCESS_WORD(phraseNumber, 2); \
+        REPEAT_WORDS_SIMPLE3(phraseNumber); \
         PROCESS_WORD(phraseNumber, 3); \
-        PROCESS_WORD(phraseNumber, 4); \
-        PROCESS_WORD(phraseNumber, 5); \
-        DONE_WORD(phraseNumber); \
     }
 
-#define REPEAT_WORDS7(phraseNumber) \
+#define REPEAT_WORDS_SIMPLE5(phraseNumber) \
     { \
-        INIT_WORD(phraseNumber); \
-        PROCESS_WORD(phraseNumber, 0); \
-        PROCESS_WORD(phraseNumber, 1); \
-        PROCESS_WORD(phraseNumber, 2); \
-        PROCESS_WORD(phraseNumber, 3); \
+        REPEAT_WORDS_SIMPLE4(phraseNumber); \
         PROCESS_WORD(phraseNumber, 4); \
-        PROCESS_WORD(phraseNumber, 5); \
-        PROCESS_WORD(phraseNumber, 6); \
-        DONE_WORD(phraseNumber); \
     }
 
-#define REPEAT_WORDS8(phraseNumber) \
+#define REPEAT_WORDS_SIMPLE6(phraseNumber) \
     { \
-        INIT_WORD(phraseNumber); \
-        PROCESS_WORD(phraseNumber, 0); \
-        PROCESS_WORD(phraseNumber, 1); \
-        PROCESS_WORD(phraseNumber, 2); \
-        PROCESS_WORD(phraseNumber, 3); \
-        PROCESS_WORD(phraseNumber, 4); \
+        REPEAT_WORDS_SIMPLE5(phraseNumber); \
         PROCESS_WORD(phraseNumber, 5); \
+    }
+
+#define REPEAT_WORDS_SIMPLE7(phraseNumber) \
+    { \
+        REPEAT_WORDS_SIMPLE6(phraseNumber); \
         PROCESS_WORD(phraseNumber, 6); \
+    }
+
+#define REPEAT_WORDS_SIMPLE8(phraseNumber) \
+    { \
+        REPEAT_WORDS_SIMPLE7(phraseNumber); \
         PROCESS_WORD(phraseNumber, 7); \
+    }
+
+#define REPEAT_WORDS_SIMPLE9(phraseNumber) \
+    { \
+        REPEAT_WORDS_SIMPLE8(phraseNumber); \
+        PROCESS_WORD(phraseNumber, 8); \
+    }
+
+#define REPEAT_WORDS_SIMPLE10(phraseNumber) \
+    { \
+        REPEAT_WORDS_SIMPLE9(phraseNumber); \
+        PROCESS_WORD(phraseNumber, 9); \
+    }
+
+#define REPEAT_WORDS(phraseNumber, repeater) \
+    { \
+        INIT_WORD(phraseNumber); \
+        repeater(phraseNumber); \
         DONE_WORD(phraseNumber); \
     }
 
+#define REPEAT_WORDS1(phraseNumber) REPEAT_WORDS(phraseNumber, REPEAT_WORDS_SIMPLE1)
+#define REPEAT_WORDS2(phraseNumber) REPEAT_WORDS(phraseNumber, REPEAT_WORDS_SIMPLE2)
+#define REPEAT_WORDS3(phraseNumber) REPEAT_WORDS(phraseNumber, REPEAT_WORDS_SIMPLE3)
+#define REPEAT_WORDS4(phraseNumber) REPEAT_WORDS(phraseNumber, REPEAT_WORDS_SIMPLE4)
+#define REPEAT_WORDS5(phraseNumber) REPEAT_WORDS(phraseNumber, REPEAT_WORDS_SIMPLE5)
+#define REPEAT_WORDS6(phraseNumber) REPEAT_WORDS(phraseNumber, REPEAT_WORDS_SIMPLE6)
+#define REPEAT_WORDS7(phraseNumber) REPEAT_WORDS(phraseNumber, REPEAT_WORDS_SIMPLE7)
+#define REPEAT_WORDS8(phraseNumber) REPEAT_WORDS(phraseNumber, REPEAT_WORDS_SIMPLE8)
+#define REPEAT_WORDS9(phraseNumber) REPEAT_WORDS(phraseNumber, REPEAT_WORDS_SIMPLE9)
+#define REPEAT_WORDS10(phraseNumber) REPEAT_WORDS(phraseNumber, REPEAT_WORDS_SIMPLE10)
+
 
 void fillPhraseSet(__int64* bufferPointer, unsigned __int64* allWordsPointer, __int32* wordIndexes, unsigned __int64* permutationsPointer, int permutationOffset, int numberOfCharacters, int numberOfWords)
 {
@@ -116,6 +124,12 @@ void fillPhraseSet(__int64* bufferPointer, unsigned __int64* allWordsPointer, __
 
     switch (numberOfWords)
     {
+    case 1:
+        REPEAT_PHRASES(REPEAT_WORDS1);
+        break;
+    case 2:
+        REPEAT_PHRASES(REPEAT_WORDS2);
+        break;
     case 3:
         REPEAT_PHRASES(REPEAT_WORDS3);
         break;
@@ -134,6 +148,12 @@ void fillPhraseSet(__int64* bufferPointer, unsigned __int64* allWordsPointer, __
     case 8:
         REPEAT_PHRASES(REPEAT_WORDS8);
         break;
+    case 9:
+        REPEAT_PHRASES(REPEAT_WORDS9);
+        break;
+    case 10:
+        REPEAT_PHRASES(REPEAT_WORDS10);
+        break;
     }
 
     auto length = numberOfCharacters + numberOfWords - 1;

dotnet/WhiteRabbit/Constants.cs

@@ -3,5 +3,7 @@
     internal class Constants
     {
         public const int PhrasesPerSet = WhiteRabbitUnmanagedBridge.MD5Unmanaged.PhrasesPerSet;
+
+        public const int MaxNumberOfWords = 8;
     }
 }

dotnet/WhiteRabbit/Flattener.cs

@@ -1,5 +1,6 @@
 namespace WhiteRabbit
 {
+    using System;
     using System.Collections.Generic;
     using System.Collections.Immutable;
     using System.Linq;
@@ -9,20 +10,6 @@
     /// </summary>
     internal static class Flattener
     {
-        // Slow universal implementation
-        private static IEnumerable<ImmutableStack<T>> FlattenAny<T>(ImmutableStack<T[]> phrase)
-        {
-            if (phrase.IsEmpty)
-            {
-                return new[] { ImmutableStack.Create<T>() };
-            }
-
-            T[] wordVariants;
-            var newStack = phrase.Pop(out wordVariants);
-            return FlattenAny(newStack).SelectMany(remainder => wordVariants.Select(word => remainder.Push(word)));
-        }
-
-        // Fast hard-coded implementation for 3 words
         private static IEnumerable<T[]> Flatten3<T>(T[][] phrase)
         {
             foreach (var item0 in phrase[0])
@@ -36,7 +23,6 @@
                 };
         }
 
-        // Fast hard-coded implementation for 4 words
         private static IEnumerable<T[]> Flatten4<T>(T[][] phrase)
         {
             foreach (var item0 in phrase[0])
@@ -52,7 +38,6 @@
                 };
         }
 
-        // Fast hard-coded implementation for 5 words
         private static IEnumerable<T[]> Flatten5<T>(T[][] phrase)
         {
             foreach (var item0 in phrase[0])
@@ -70,7 +55,6 @@
                 };
         }
 
-        // Fast hard-coded implementation for 6 words
         private static IEnumerable<T[]> Flatten6<T>(T[][] phrase)
         {
             foreach (var item0 in phrase[0])
@@ -90,7 +74,6 @@
                 };
         }
 
-        // Fast hard-coded implementation for 7 words
         private static IEnumerable<T[]> Flatten7<T>(T[][] phrase)
         {
             foreach (var item0 in phrase[0])
@@ -112,7 +95,6 @@
                 };
         }
 
-        // Fast hard-coded implementation for 7 words
         private static IEnumerable<T[]> Flatten8<T>(T[][] phrase)
         {
             foreach (var item0 in phrase[0])
@@ -136,6 +118,58 @@
                 };
         }
 
+        private static IEnumerable<T[]> Flatten9<T>(T[][] phrase)
+        {
+            foreach (var item0 in phrase[0])
+            foreach (var item1 in phrase[1])
+            foreach (var item2 in phrase[2])
+            foreach (var item3 in phrase[3])
+            foreach (var item4 in phrase[4])
+            foreach (var item5 in phrase[5])
+            foreach (var item6 in phrase[6])
+            foreach (var item7 in phrase[7])
+            foreach (var item8 in phrase[8])
+                yield return new T[]
+                {
+                    item0,
+                    item1,
+                    item2,
+                    item3,
+                    item4,
+                    item5,
+                    item6,
+                    item7,
+                    item8,
+                };
+        }
+
+        private static IEnumerable<T[]> Flatten10<T>(T[][] phrase)
+        {
+            foreach (var item0 in phrase[0])
+            foreach (var item1 in phrase[1])
+            foreach (var item2 in phrase[2])
+            foreach (var item3 in phrase[3])
+            foreach (var item4 in phrase[4])
+            foreach (var item5 in phrase[5])
+            foreach (var item6 in phrase[6])
+            foreach (var item7 in phrase[7])
+            foreach (var item8 in phrase[8])
+            foreach (var item9 in phrase[9])
+                yield return new T[]
+                {
+                    item0,
+                    item1,
+                    item2,
+                    item3,
+                    item4,
+                    item5,
+                    item6,
+                    item7,
+                    item8,
+                    item9,
+                };
+        }
+
         public static IEnumerable<T[]> Flatten<T>(T[][] wordVariants)
         {
             switch (wordVariants.Length)
@@ -152,8 +186,12 @@
                     return Flatten7(wordVariants);
                 case 8:
                     return Flatten8(wordVariants);
+                case 9:
+                    return Flatten9(wordVariants);
+                case 10:
+                    return Flatten10(wordVariants);
                 default:
-                    return FlattenAny(ImmutableStack.Create(wordVariants)).Select(words => words.ToArray());
+                    throw new ArgumentOutOfRangeException(nameof(wordVariants));
             }
         }
     }

dotnet/WhiteRabbit/PrecomputedPermutationsGenerator.cs

@@ -6,19 +6,31 @@
 
     internal static class PrecomputedPermutationsGenerator
     {
-        private static ulong[][][] Permutations { get; } = Enumerable.Range(0, 9).Select(GeneratePermutations).ToArray();
+        static PrecomputedPermutationsGenerator()
+        {
+            Permutations = new ulong[Constants.MaxNumberOfWords + 1][][];
+            PermutationsNumbers = new long[Constants.MaxNumberOfWords + 1][];
+            for (var i = 0; i <= Constants.MaxNumberOfWords; i++)
+            {
+                var permutationsInfo = GeneratePermutations(i);
+                Permutations[i] = permutationsInfo.Item1;
+                PermutationsNumbers[i] = permutationsInfo.Item2;
+            }
+        }
 
-        private static long[] PermutationsNumbers { get; } = GeneratePermutationsNumbers().Take(19).ToArray();
+        private static ulong[][][] Permutations { get; }
+
+        private static long[][] PermutationsNumbers { get; }
 
         public static ulong[] HamiltonianPermutations(int n, uint filter) => Permutations[n][filter];
 
-        public static long GetPermutationsNumber(int n) => PermutationsNumbers[n];
+        public static long GetPermutationsNumber(int n, uint filter) => PermutationsNumbers[n][filter];
 
-        private static ulong[][] GeneratePermutations(int n)
+        private static Tuple<ulong[][], long[]> GeneratePermutations(int n)
         {
             if (n == 0)
             {
-                return new ulong[0][];
+                return Tuple.Create(new ulong[0][], new long[0]);
             }
 
             var allPermutations = PermutationsGenerator.HamiltonianPermutations(n)
@@ -26,27 +38,41 @@
                 .ToArray();
 
             var statesCount = (uint)1 << (n - 1);
-            var result = new ulong[statesCount][];
-            for (uint i = 0; i < statesCount; i++)
-            {
-                result[i] = PadToWholeChunks(FilterPermutations(allPermutations, i).ToArray(), Constants.PhrasesPerSet);
-            }
-
-            return result;
-        }
+            var resultUnpadded = new PermutationInfo[statesCount][];
 
-        private static IEnumerable<ulong> FilterPermutations(IEnumerable<ulong> permutations, uint state)
-        {
-            for (int position = 0; position < 16; position++)
+            resultUnpadded[0] = allPermutations;
+            for (uint i = 1; i < statesCount; i++)
             {
-                if (((state >> position) & 1) != 0)
+                var mask = i;
+                mask |= mask >> 1;
+                mask |= mask >> 2;
+                mask |= mask >> 4;
+                mask |= mask >> 8;
+                mask |= mask >> 16;
+                mask = mask >> 1;
+                var existing = i & mask;
+                var seniorBit = i ^ existing;
+                var position = 0;
+                while (seniorBit != 0)
                 {
-                    var innerPosition = (uint)position;
-                    permutations = permutations.Where(permutation => IsOrderPreserved(permutation, innerPosition));
+                    seniorBit = seniorBit >> 1;
+                    position++;
                 }
+
+                resultUnpadded[i] = resultUnpadded[existing]
+                    .Where(info => ((info.PermutationInverse >> (4 * (position - 1))) % 16 < (info.PermutationInverse >> (4 * position)) % 16))
+                    .ToArray();
             }
 
-            return permutations;
+            var result = new ulong[statesCount][];
+            var numbers = new long[statesCount];
+            for (uint i = 0; i < statesCount; i++)
+            {
+                result[i] = PadToWholeChunks(resultUnpadded[i], Constants.PhrasesPerSet);
+                numbers[i] = resultUnpadded[i].LongLength;
+            }
+
+            return Tuple.Create(result, numbers);
         }
 
         public static bool IsOrderPreserved(ulong permutation, uint position)
@@ -71,27 +97,41 @@
             throw new ApplicationException("Malformed permutation " + permutation + " for position " + position);
         }
 
-        private static T[] PadToWholeChunks<T>(T[] original, int chunkSize)
+        private static ulong[] PadToWholeChunks(PermutationInfo[] original, int chunkSize)
         {
+            ulong[] result;
             if (original.Length % chunkSize == 0)
             {
-                return original;
+                result = new ulong[original.Length];
+            }
+            else
+            {
+                result = new ulong[original.Length + chunkSize - (original.Length % chunkSize)];
+            }
+
+            for (var i = 0; i < original.Length; i++)
+            {
+                result[i] = original[i].Permutation;
             }
 
-            return original.Concat(Enumerable.Repeat(default(T), chunkSize - (original.Length % chunkSize))).ToArray();
+            return result;
         }
 
-        private static ulong FormatPermutation(PermutationsGenerator.Permutation permutation)
+        private static PermutationInfo FormatPermutation(PermutationsGenerator.Permutation permutation)
         {
             System.Diagnostics.Debug.Assert(permutation.PermutationData.Length <= 16);
 
             ulong result = 0;
+            ulong resultInverse = 0;
             for (var i = 0; i < permutation.PermutationData.Length; i++)
             {
-                result |= (ulong)(permutation.PermutationData[i]) << (4 * i);
+                var source = i;
+                var target = permutation.PermutationData[i];
+                result |= (ulong)(target) << (4 * source);
+                resultInverse |= (ulong)(source) << (4 * target);
             }
 
-            return result;
+            return new PermutationInfo { Permutation = result, PermutationInverse = resultInverse };
         }
 
         private static IEnumerable<long> GeneratePermutationsNumbers()
@@ -107,5 +147,11 @@
                 i++;
             }
         }
+
+        private struct PermutationInfo
+        {
+            public ulong Permutation;
+            public ulong PermutationInverse;
+        }
     }
 }

dotnet/WhiteRabbit/Program.cs

@@ -1,12 +1,10 @@
 namespace WhiteRabbit
 {
     using System;
-    using System.Collections.Concurrent;
     using System.Collections.Generic;
     using System.Configuration;
     using System.Diagnostics;
     using System.Linq;
-    using System.Numerics;
     using System.Security.Cryptography;
     using System.Text;
 
@@ -28,9 +26,15 @@
 
             var maxWordsInPhrase = int.Parse(ConfigurationManager.AppSettings["MaxWordsInPhrase"]);
 
-            if (sourceChars.Length + maxWordsInPhrase > 27)
+            if (sourceChars.Length + maxWordsInPhrase > 28)
             {
-                Console.WriteLine("Only anagrams of up to 27 characters are allowed");
+                Console.WriteLine("Only anagrams of up to 27 characters (including whitespace) are allowed");
+                return;
+            }
+
+            if (maxWordsInPhrase > Constants.MaxNumberOfWords)
+            {
+                Console.WriteLine($"Only anagrams of up to {Constants.MaxNumberOfWords} words are allowed");
                 return;
             }
 

dotnet/WhiteRabbit/StringsProcessor.cs

@@ -79,9 +79,14 @@
 
         public long GetPhrasesCount()
         {
-            return this.VectorsProcessor.GenerateSequences()
-                .Select(this.ConvertVectorsToWordsNumber)
-                .Sum(tuple => tuple.Item2 * PrecomputedPermutationsGenerator.GetPermutationsNumber(tuple.Item1));
+            var sums = this.VectorsProcessor.GenerateSequences();
+            return (from sum in sums
+                    let filter = ComputeFilter(sum)
+                    let wordsVariantsNumber = this.ConvertVectorsToWordsNumber(sum)
+                    let permutationsNumber = PrecomputedPermutationsGenerator.GetPermutationsNumber(sum.Length, filter)
+                    let total = wordsVariantsNumber * permutationsNumber
+                    select total)
+                    .Sum();
         }
 
         private static uint ComputeFilter(int[] vectors)
@@ -110,7 +115,7 @@
             return words;
         }
 
-        private Tuple<int, long> ConvertVectorsToWordsNumber(int[] vectors)
+        private long ConvertVectorsToWordsNumber(int[] vectors)
         {
             long result = 1;
             for (var i = 0; i < vectors.Length; i++)
@@ -118,7 +123,7 @@
                 result *= this.WordsDictionary[vectors[i]].Length;
             }
 
-            return Tuple.Create(vectors.Length, result);
+            return result;
         }
 
         private IEnumerable<PhraseSet> ConvertWordsToPhrases(int[] wordIndexes, uint filter)