From 705baf969c285e5ee36cec5ffa20e1c7012ea497 Mon Sep 17 00:00:00 2001 From: inga-lovinde <52715130+inga-lovinde@users.noreply.github.com> Date: Wed, 3 May 2017 11:20:17 +0300 Subject: [PATCH] Optimized initialization, support for 10-word phrases, updated performance measurements --- README.md | 37 ++----- .../WhiteRabbit.UnmanagedBridge/phraseset.cpp | 104 +++++++++++------- dotnet/WhiteRabbit/Constants.cs | 2 + dotnet/WhiteRabbit/Flattener.cs | 78 +++++++++---- .../PrecomputedPermutationsGenerator.cs | 98 ++++++++++++----- dotnet/WhiteRabbit/Program.cs | 12 +- dotnet/WhiteRabbit/StringsProcessor.cs | 15 ++- 7 files changed, 224 insertions(+), 122 deletions(-) diff --git a/README.md b/README.md index 99c7052..9ddfc94 100644 --- a/README.md +++ b/README.md @@ -34,49 +34,36 @@ WhiteRabbit.exe < wordlist Performance =========== -Memory usage is minimal (for that kind of task), less than 10MB. +Memory usage is minimal (for that kind of task), less than 10MB (25MB for MaxNumberOfWords = 8). It is also somewhat optimized for likely intended phrases, as anagrams consisting of longer words are generated first. That's why the given hashes are solved much sooner than it takes to check all anagrams. Anagrams generation is not parallelized, as even single-threaded performance for 4-word anagrams is high enough; and 5-word (or larger) anagrams are frequent enough for most of the time being spent on computing hashes, with full CPU load. -Multi-threaded performance with RyuJIT (.NET 4.6, 64-bit system) on laptop with dual-core Sandy Bridge @2.6GHz (without AVX2 support) is as follows (excluding initialization time of 0.2 seconds), for different maximum allowed words in an anagram: +Multi-threaded performance with RyuJIT (.NET 4.6, 64-bit system) on i5-6500 is as follows (excluding initialization time of 0.2 seconds), for different maximum allowed words in an anagram: -Number of words|Time to check all anagrams no longer than that|Time to solve "easy" hash|Time to solve "more difficult" hash|Time to solve "hard" hash|Number of anagrams no longer than that (see note below) ----------------|----------------------------------------------|-------------------------|-----------------------------------|-------------------------|------------------------------------------------------- +Number of words|Time to check all anagrams no longer than that|Time to solve "easy" hash|Time to solve "more difficult" hash|Time to solve "hard" hash|Number of unique anagrams no longer than that +---------------|----------------------------------------------|-------------------------|-----------------------------------|-------------------------|--------------------------------------------- 3|0.1s||||4560 -4|0.8s|||0.15s|7,433,016 -5|87s|1s|0.3s|2s|1,348,876,896 -6|1 hour (?)|10s|1.7s|25s|58,837,302,096 -7|11 hours (?)|78s|9s|3.5 minutes|1,108,328,708,976 -8||7.5 minutes|50s|20 minutes|12,089,249,231,856 -9|||||88,977,349,731,696 -10|||||482,627,715,786,096 -11|||||2,030,917,440,675,696 -12|||||6,813,402,098,518,896 -13|||||18,437,325,782,691,696 -14|||||40,367,286,468,925,296 -15|||||71,561,858,517,565,296 -16|||||103,280,807,987,773,296 -17|||||123,910,678,817,341,296 -18|||||130,313,052,523,069,296 +4|0.8s|||0.15s|7,431,984 +5|27s|0.4s|0.12s|0.75s|1,347,437,484 +6|16.5 minutes|3s|0.5s|7.3s|58,405,904,844 +7|5 hours|16s|2s|47s|1,070,307,744,114 +8|49 hours|61s|6.3s|3.2 minutes|10,893,594,396,594 +9||2.5 minutes|13s|9.5 minutes|70,596,864,409,954 +10||5 minutes|21s|17.5 minutes|314,972,701,475,754 Note that all measurements were done on a Release build; Debug build is significantly slower. For comparison, certain other solutions available on GitHub seem to require 3 hours to find all 3-word anagrams. This solution is faster by 6-7 orders of magnitude (it finds and checks all 4-word anagrams in 1/10000th fraction of time required for other solution just to find all 3-word anagrams, with no MD5 calculations). -Also, note that anagram counts are inflated for the sake of code simplicity. -E.g. for phrase "aabbc" and dictionary [ab, ba, c] there are four possible set of words adding up to the source phrase: [ab, ab, c], [ab, ba, c], [ba, ab, c], [ba, ba, c]. -My implementation regards these sets as sets of different words, and applies all possible permutations to the every set, even if it will result in the same set. -For the example above, my application would produce 24 anagrams (with six permutations for every of the four sets), although actually there are only 12 different anagrams. - Conditional compilation symbols =============================== * Define `SINGLE_THREADED` to use standard enumerables instead of ParallelEnumerable (useful for profiling). -* Define `DEBUG`, or build in debug mode, to get the total number of anagrams (not optimized, memory-hogging). +* Define `DEBUG`, or build in debug mode, to get the total number of anagrams (not optimized). Implementation notes ==================== diff --git a/dotnet/WhiteRabbit.UnmanagedBridge/phraseset.cpp b/dotnet/WhiteRabbit.UnmanagedBridge/phraseset.cpp index ca454ac..88e41aa 100644 --- a/dotnet/WhiteRabbit.UnmanagedBridge/phraseset.cpp +++ b/dotnet/WhiteRabbit.UnmanagedBridge/phraseset.cpp @@ -40,75 +40,83 @@ #define DONE_WORD(phraseNumber) \ avx2buffer[phraseNumber] = phrase; -#define REPEAT_WORDS3(phraseNumber) \ +#define REPEAT_WORDS_SIMPLE1(phraseNumber) \ { \ - INIT_WORD(phraseNumber); \ PROCESS_WORD(phraseNumber, 0); \ - PROCESS_WORD(phraseNumber, 1); \ - PROCESS_WORD(phraseNumber, 2); \ - DONE_WORD(phraseNumber); \ } -#define REPEAT_WORDS4(phraseNumber) \ +#define REPEAT_WORDS_SIMPLE2(phraseNumber) \ { \ - INIT_WORD(phraseNumber); \ - PROCESS_WORD(phraseNumber, 0); \ + REPEAT_WORDS_SIMPLE1(phraseNumber); \ PROCESS_WORD(phraseNumber, 1); \ - PROCESS_WORD(phraseNumber, 2); \ - PROCESS_WORD(phraseNumber, 3); \ - DONE_WORD(phraseNumber); \ } -#define REPEAT_WORDS5(phraseNumber) \ +#define REPEAT_WORDS_SIMPLE3(phraseNumber) \ { \ - INIT_WORD(phraseNumber); \ - PROCESS_WORD(phraseNumber, 0); \ - PROCESS_WORD(phraseNumber, 1); \ + REPEAT_WORDS_SIMPLE2(phraseNumber); \ PROCESS_WORD(phraseNumber, 2); \ - PROCESS_WORD(phraseNumber, 3); \ - PROCESS_WORD(phraseNumber, 4); \ - DONE_WORD(phraseNumber); \ } -#define REPEAT_WORDS6(phraseNumber) \ +#define REPEAT_WORDS_SIMPLE4(phraseNumber) \ { \ - INIT_WORD(phraseNumber); \ - PROCESS_WORD(phraseNumber, 0); \ - PROCESS_WORD(phraseNumber, 1); \ - PROCESS_WORD(phraseNumber, 2); \ + REPEAT_WORDS_SIMPLE3(phraseNumber); \ PROCESS_WORD(phraseNumber, 3); \ - PROCESS_WORD(phraseNumber, 4); \ - PROCESS_WORD(phraseNumber, 5); \ - DONE_WORD(phraseNumber); \ } -#define REPEAT_WORDS7(phraseNumber) \ +#define REPEAT_WORDS_SIMPLE5(phraseNumber) \ { \ - INIT_WORD(phraseNumber); \ - PROCESS_WORD(phraseNumber, 0); \ - PROCESS_WORD(phraseNumber, 1); \ - PROCESS_WORD(phraseNumber, 2); \ - PROCESS_WORD(phraseNumber, 3); \ + REPEAT_WORDS_SIMPLE4(phraseNumber); \ PROCESS_WORD(phraseNumber, 4); \ - PROCESS_WORD(phraseNumber, 5); \ - PROCESS_WORD(phraseNumber, 6); \ - DONE_WORD(phraseNumber); \ } -#define REPEAT_WORDS8(phraseNumber) \ +#define REPEAT_WORDS_SIMPLE6(phraseNumber) \ { \ - INIT_WORD(phraseNumber); \ - PROCESS_WORD(phraseNumber, 0); \ - PROCESS_WORD(phraseNumber, 1); \ - PROCESS_WORD(phraseNumber, 2); \ - PROCESS_WORD(phraseNumber, 3); \ - PROCESS_WORD(phraseNumber, 4); \ + REPEAT_WORDS_SIMPLE5(phraseNumber); \ PROCESS_WORD(phraseNumber, 5); \ + } + +#define REPEAT_WORDS_SIMPLE7(phraseNumber) \ + { \ + REPEAT_WORDS_SIMPLE6(phraseNumber); \ PROCESS_WORD(phraseNumber, 6); \ + } + +#define REPEAT_WORDS_SIMPLE8(phraseNumber) \ + { \ + REPEAT_WORDS_SIMPLE7(phraseNumber); \ PROCESS_WORD(phraseNumber, 7); \ + } + +#define REPEAT_WORDS_SIMPLE9(phraseNumber) \ + { \ + REPEAT_WORDS_SIMPLE8(phraseNumber); \ + PROCESS_WORD(phraseNumber, 8); \ + } + +#define REPEAT_WORDS_SIMPLE10(phraseNumber) \ + { \ + REPEAT_WORDS_SIMPLE9(phraseNumber); \ + PROCESS_WORD(phraseNumber, 9); \ + } + +#define REPEAT_WORDS(phraseNumber, repeater) \ + { \ + INIT_WORD(phraseNumber); \ + repeater(phraseNumber); \ DONE_WORD(phraseNumber); \ } +#define REPEAT_WORDS1(phraseNumber) REPEAT_WORDS(phraseNumber, REPEAT_WORDS_SIMPLE1) +#define REPEAT_WORDS2(phraseNumber) REPEAT_WORDS(phraseNumber, REPEAT_WORDS_SIMPLE2) +#define REPEAT_WORDS3(phraseNumber) REPEAT_WORDS(phraseNumber, REPEAT_WORDS_SIMPLE3) +#define REPEAT_WORDS4(phraseNumber) REPEAT_WORDS(phraseNumber, REPEAT_WORDS_SIMPLE4) +#define REPEAT_WORDS5(phraseNumber) REPEAT_WORDS(phraseNumber, REPEAT_WORDS_SIMPLE5) +#define REPEAT_WORDS6(phraseNumber) REPEAT_WORDS(phraseNumber, REPEAT_WORDS_SIMPLE6) +#define REPEAT_WORDS7(phraseNumber) REPEAT_WORDS(phraseNumber, REPEAT_WORDS_SIMPLE7) +#define REPEAT_WORDS8(phraseNumber) REPEAT_WORDS(phraseNumber, REPEAT_WORDS_SIMPLE8) +#define REPEAT_WORDS9(phraseNumber) REPEAT_WORDS(phraseNumber, REPEAT_WORDS_SIMPLE9) +#define REPEAT_WORDS10(phraseNumber) REPEAT_WORDS(phraseNumber, REPEAT_WORDS_SIMPLE10) + void fillPhraseSet(__int64* bufferPointer, unsigned __int64* allWordsPointer, __int32* wordIndexes, unsigned __int64* permutationsPointer, int permutationOffset, int numberOfCharacters, int numberOfWords) { @@ -116,6 +124,12 @@ void fillPhraseSet(__int64* bufferPointer, unsigned __int64* allWordsPointer, __ switch (numberOfWords) { + case 1: + REPEAT_PHRASES(REPEAT_WORDS1); + break; + case 2: + REPEAT_PHRASES(REPEAT_WORDS2); + break; case 3: REPEAT_PHRASES(REPEAT_WORDS3); break; @@ -134,6 +148,12 @@ void fillPhraseSet(__int64* bufferPointer, unsigned __int64* allWordsPointer, __ case 8: REPEAT_PHRASES(REPEAT_WORDS8); break; + case 9: + REPEAT_PHRASES(REPEAT_WORDS9); + break; + case 10: + REPEAT_PHRASES(REPEAT_WORDS10); + break; } auto length = numberOfCharacters + numberOfWords - 1; diff --git a/dotnet/WhiteRabbit/Constants.cs b/dotnet/WhiteRabbit/Constants.cs index 13a4d21..016338e 100644 --- a/dotnet/WhiteRabbit/Constants.cs +++ b/dotnet/WhiteRabbit/Constants.cs @@ -3,5 +3,7 @@ internal class Constants { public const int PhrasesPerSet = WhiteRabbitUnmanagedBridge.MD5Unmanaged.PhrasesPerSet; + + public const int MaxNumberOfWords = 8; } } diff --git a/dotnet/WhiteRabbit/Flattener.cs b/dotnet/WhiteRabbit/Flattener.cs index 6c17811..c79ac2a 100644 --- a/dotnet/WhiteRabbit/Flattener.cs +++ b/dotnet/WhiteRabbit/Flattener.cs @@ -1,5 +1,6 @@ namespace WhiteRabbit { + using System; using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; @@ -9,20 +10,6 @@ /// internal static class Flattener { - // Slow universal implementation - private static IEnumerable> FlattenAny(ImmutableStack phrase) - { - if (phrase.IsEmpty) - { - return new[] { ImmutableStack.Create() }; - } - - T[] wordVariants; - var newStack = phrase.Pop(out wordVariants); - return FlattenAny(newStack).SelectMany(remainder => wordVariants.Select(word => remainder.Push(word))); - } - - // Fast hard-coded implementation for 3 words private static IEnumerable Flatten3(T[][] phrase) { foreach (var item0 in phrase[0]) @@ -36,7 +23,6 @@ }; } - // Fast hard-coded implementation for 4 words private static IEnumerable Flatten4(T[][] phrase) { foreach (var item0 in phrase[0]) @@ -52,7 +38,6 @@ }; } - // Fast hard-coded implementation for 5 words private static IEnumerable Flatten5(T[][] phrase) { foreach (var item0 in phrase[0]) @@ -70,7 +55,6 @@ }; } - // Fast hard-coded implementation for 6 words private static IEnumerable Flatten6(T[][] phrase) { foreach (var item0 in phrase[0]) @@ -90,7 +74,6 @@ }; } - // Fast hard-coded implementation for 7 words private static IEnumerable Flatten7(T[][] phrase) { foreach (var item0 in phrase[0]) @@ -112,7 +95,6 @@ }; } - // Fast hard-coded implementation for 7 words private static IEnumerable Flatten8(T[][] phrase) { foreach (var item0 in phrase[0]) @@ -136,6 +118,58 @@ }; } + private static IEnumerable Flatten9(T[][] phrase) + { + foreach (var item0 in phrase[0]) + foreach (var item1 in phrase[1]) + foreach (var item2 in phrase[2]) + foreach (var item3 in phrase[3]) + foreach (var item4 in phrase[4]) + foreach (var item5 in phrase[5]) + foreach (var item6 in phrase[6]) + foreach (var item7 in phrase[7]) + foreach (var item8 in phrase[8]) + yield return new T[] + { + item0, + item1, + item2, + item3, + item4, + item5, + item6, + item7, + item8, + }; + } + + private static IEnumerable Flatten10(T[][] phrase) + { + foreach (var item0 in phrase[0]) + foreach (var item1 in phrase[1]) + foreach (var item2 in phrase[2]) + foreach (var item3 in phrase[3]) + foreach (var item4 in phrase[4]) + foreach (var item5 in phrase[5]) + foreach (var item6 in phrase[6]) + foreach (var item7 in phrase[7]) + foreach (var item8 in phrase[8]) + foreach (var item9 in phrase[9]) + yield return new T[] + { + item0, + item1, + item2, + item3, + item4, + item5, + item6, + item7, + item8, + item9, + }; + } + public static IEnumerable Flatten(T[][] wordVariants) { switch (wordVariants.Length) @@ -152,8 +186,12 @@ return Flatten7(wordVariants); case 8: return Flatten8(wordVariants); + case 9: + return Flatten9(wordVariants); + case 10: + return Flatten10(wordVariants); default: - return FlattenAny(ImmutableStack.Create(wordVariants)).Select(words => words.ToArray()); + throw new ArgumentOutOfRangeException(nameof(wordVariants)); } } } diff --git a/dotnet/WhiteRabbit/PrecomputedPermutationsGenerator.cs b/dotnet/WhiteRabbit/PrecomputedPermutationsGenerator.cs index 3968252..43a29a4 100644 --- a/dotnet/WhiteRabbit/PrecomputedPermutationsGenerator.cs +++ b/dotnet/WhiteRabbit/PrecomputedPermutationsGenerator.cs @@ -6,19 +6,31 @@ internal static class PrecomputedPermutationsGenerator { - private static ulong[][][] Permutations { get; } = Enumerable.Range(0, 9).Select(GeneratePermutations).ToArray(); + static PrecomputedPermutationsGenerator() + { + Permutations = new ulong[Constants.MaxNumberOfWords + 1][][]; + PermutationsNumbers = new long[Constants.MaxNumberOfWords + 1][]; + for (var i = 0; i <= Constants.MaxNumberOfWords; i++) + { + var permutationsInfo = GeneratePermutations(i); + Permutations[i] = permutationsInfo.Item1; + PermutationsNumbers[i] = permutationsInfo.Item2; + } + } - private static long[] PermutationsNumbers { get; } = GeneratePermutationsNumbers().Take(19).ToArray(); + private static ulong[][][] Permutations { get; } + + private static long[][] PermutationsNumbers { get; } public static ulong[] HamiltonianPermutations(int n, uint filter) => Permutations[n][filter]; - public static long GetPermutationsNumber(int n) => PermutationsNumbers[n]; + public static long GetPermutationsNumber(int n, uint filter) => PermutationsNumbers[n][filter]; - private static ulong[][] GeneratePermutations(int n) + private static Tuple GeneratePermutations(int n) { if (n == 0) { - return new ulong[0][]; + return Tuple.Create(new ulong[0][], new long[0]); } var allPermutations = PermutationsGenerator.HamiltonianPermutations(n) @@ -26,27 +38,41 @@ .ToArray(); var statesCount = (uint)1 << (n - 1); - var result = new ulong[statesCount][]; - for (uint i = 0; i < statesCount; i++) - { - result[i] = PadToWholeChunks(FilterPermutations(allPermutations, i).ToArray(), Constants.PhrasesPerSet); - } - - return result; - } + var resultUnpadded = new PermutationInfo[statesCount][]; - private static IEnumerable FilterPermutations(IEnumerable permutations, uint state) - { - for (int position = 0; position < 16; position++) + resultUnpadded[0] = allPermutations; + for (uint i = 1; i < statesCount; i++) { - if (((state >> position) & 1) != 0) + var mask = i; + mask |= mask >> 1; + mask |= mask >> 2; + mask |= mask >> 4; + mask |= mask >> 8; + mask |= mask >> 16; + mask = mask >> 1; + var existing = i & mask; + var seniorBit = i ^ existing; + var position = 0; + while (seniorBit != 0) { - var innerPosition = (uint)position; - permutations = permutations.Where(permutation => IsOrderPreserved(permutation, innerPosition)); + seniorBit = seniorBit >> 1; + position++; } + + resultUnpadded[i] = resultUnpadded[existing] + .Where(info => ((info.PermutationInverse >> (4 * (position - 1))) % 16 < (info.PermutationInverse >> (4 * position)) % 16)) + .ToArray(); } - return permutations; + var result = new ulong[statesCount][]; + var numbers = new long[statesCount]; + for (uint i = 0; i < statesCount; i++) + { + result[i] = PadToWholeChunks(resultUnpadded[i], Constants.PhrasesPerSet); + numbers[i] = resultUnpadded[i].LongLength; + } + + return Tuple.Create(result, numbers); } public static bool IsOrderPreserved(ulong permutation, uint position) @@ -71,27 +97,41 @@ throw new ApplicationException("Malformed permutation " + permutation + " for position " + position); } - private static T[] PadToWholeChunks(T[] original, int chunkSize) + private static ulong[] PadToWholeChunks(PermutationInfo[] original, int chunkSize) { + ulong[] result; if (original.Length % chunkSize == 0) { - return original; + result = new ulong[original.Length]; + } + else + { + result = new ulong[original.Length + chunkSize - (original.Length % chunkSize)]; + } + + for (var i = 0; i < original.Length; i++) + { + result[i] = original[i].Permutation; } - return original.Concat(Enumerable.Repeat(default(T), chunkSize - (original.Length % chunkSize))).ToArray(); + return result; } - private static ulong FormatPermutation(PermutationsGenerator.Permutation permutation) + private static PermutationInfo FormatPermutation(PermutationsGenerator.Permutation permutation) { System.Diagnostics.Debug.Assert(permutation.PermutationData.Length <= 16); ulong result = 0; + ulong resultInverse = 0; for (var i = 0; i < permutation.PermutationData.Length; i++) { - result |= (ulong)(permutation.PermutationData[i]) << (4 * i); + var source = i; + var target = permutation.PermutationData[i]; + result |= (ulong)(target) << (4 * source); + resultInverse |= (ulong)(source) << (4 * target); } - return result; + return new PermutationInfo { Permutation = result, PermutationInverse = resultInverse }; } private static IEnumerable GeneratePermutationsNumbers() @@ -107,5 +147,11 @@ i++; } } + + private struct PermutationInfo + { + public ulong Permutation; + public ulong PermutationInverse; + } } } diff --git a/dotnet/WhiteRabbit/Program.cs b/dotnet/WhiteRabbit/Program.cs index df951b3..97dae1e 100644 --- a/dotnet/WhiteRabbit/Program.cs +++ b/dotnet/WhiteRabbit/Program.cs @@ -1,12 +1,10 @@ namespace WhiteRabbit { using System; - using System.Collections.Concurrent; using System.Collections.Generic; using System.Configuration; using System.Diagnostics; using System.Linq; - using System.Numerics; using System.Security.Cryptography; using System.Text; @@ -28,9 +26,15 @@ var maxWordsInPhrase = int.Parse(ConfigurationManager.AppSettings["MaxWordsInPhrase"]); - if (sourceChars.Length + maxWordsInPhrase > 27) + if (sourceChars.Length + maxWordsInPhrase > 28) { - Console.WriteLine("Only anagrams of up to 27 characters are allowed"); + Console.WriteLine("Only anagrams of up to 27 characters (including whitespace) are allowed"); + return; + } + + if (maxWordsInPhrase > Constants.MaxNumberOfWords) + { + Console.WriteLine($"Only anagrams of up to {Constants.MaxNumberOfWords} words are allowed"); return; } diff --git a/dotnet/WhiteRabbit/StringsProcessor.cs b/dotnet/WhiteRabbit/StringsProcessor.cs index fd82e10..d48671e 100644 --- a/dotnet/WhiteRabbit/StringsProcessor.cs +++ b/dotnet/WhiteRabbit/StringsProcessor.cs @@ -79,9 +79,14 @@ public long GetPhrasesCount() { - return this.VectorsProcessor.GenerateSequences() - .Select(this.ConvertVectorsToWordsNumber) - .Sum(tuple => tuple.Item2 * PrecomputedPermutationsGenerator.GetPermutationsNumber(tuple.Item1)); + var sums = this.VectorsProcessor.GenerateSequences(); + return (from sum in sums + let filter = ComputeFilter(sum) + let wordsVariantsNumber = this.ConvertVectorsToWordsNumber(sum) + let permutationsNumber = PrecomputedPermutationsGenerator.GetPermutationsNumber(sum.Length, filter) + let total = wordsVariantsNumber * permutationsNumber + select total) + .Sum(); } private static uint ComputeFilter(int[] vectors) @@ -110,7 +115,7 @@ return words; } - private Tuple ConvertVectorsToWordsNumber(int[] vectors) + private long ConvertVectorsToWordsNumber(int[] vectors) { long result = 1; for (var i = 0; i < vectors.Length; i++) @@ -118,7 +123,7 @@ result *= this.WordsDictionary[vectors[i]].Length; } - return Tuple.Create(vectors.Length, result); + return result; } private IEnumerable ConvertWordsToPhrases(int[] wordIndexes, uint filter)