From 760c1b5b13bebaa39fe401eddec6c83abb9091fe Mon Sep 17 00:00:00 2001 From: inga-lovinde <52715130+inga-lovinde@users.noreply.github.com> Date: Mon, 13 Mar 2017 11:14:42 +0300 Subject: [PATCH] Further optimizations --- README.md | 10 ++-- .../PrecomputedPermutationsGenerator.cs | 2 - WhiteRabbit/Program.cs | 2 +- WhiteRabbit/StringsProcessor.cs | 52 +++++++++++++------ WhiteRabbit/VectorsConverter.cs | 5 -- WhiteRabbit/VectorsProcessor.cs | 18 ++++--- 6 files changed, 51 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index eb68131..d1b8832 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ WhiteRabbit.exe < wordlist Performance =========== -Memory usage is minimal (for that kind of task), around 10-30MB. +Memory usage is minimal (for that kind of task), around 10-20MB. It is also somewhat optimized for likely intended phrases, as anagrams consisting of longer words are generated first. That's why the given hashes are solved much sooner than it takes to check all anagrams. @@ -22,13 +22,13 @@ Anagrams generation is not parallelized, as even single-threaded performance for Multi-threaded performance with RyuJIT (.NET 4.6, 64-bit system) on quad-core Sandy Bridge @2.8GHz is as follows: -* If only phrases of at most 4 words are allowed, then it takes less than 5.5 seconds to find and check all 7433016 anagrams; all hashes are solved in first 0.7 seconds. +* If only phrases of at most 4 words are allowed, then it takes less than 4.5 seconds to find and check all 7433016 anagrams; all hashes are solved in first 0.6 seconds. -* If phrases of 5 words are allowed as well, then it takes around 17 minutes to find and check all anagrams; all hashes are solved in first 25 seconds. Most of time is spent on MD5 computations for correct anagrams, so there is not a lot to optimize further. +* If phrases of 5 words are allowed as well, then it takes around 13 minutes to find and check all 1348876896 anagrams; all hashes are solved in first 20 seconds. Most of time is spent on MD5 computations for correct anagrams, so there is not a lot to optimize further. -* If phrases of 6 words are allowed as well, then "more difficult" hash is solved in 30 seconds, "easiest" in 3 minutes, and "hard" in 6 minutes. +* If phrases of 6 words are allowed as well, then "more difficult" hash is solved in 20 seconds, "easiest" in 2.5 minutes, and "hard" in 6 minutes. -* If phrases of 7 words are allowed as well, then "more difficult" hash is solved in 3 minutes. +* If phrases of 7 words are allowed as well, then "more difficult" hash is solved in 2.5 minutes. Note that all measurements were done on a Release build; Debug build is significantly slower. diff --git a/WhiteRabbit/PrecomputedPermutationsGenerator.cs b/WhiteRabbit/PrecomputedPermutationsGenerator.cs index c118b70..78e7b93 100644 --- a/WhiteRabbit/PrecomputedPermutationsGenerator.cs +++ b/WhiteRabbit/PrecomputedPermutationsGenerator.cs @@ -15,8 +15,6 @@ PermutationsGenerator.HamiltonianPermutations(5).ToArray(), PermutationsGenerator.HamiltonianPermutations(6).ToArray(), PermutationsGenerator.HamiltonianPermutations(7).ToArray(), - PermutationsGenerator.HamiltonianPermutations(8).ToArray(), - PermutationsGenerator.HamiltonianPermutations(9).ToArray(), }; public static IEnumerable HamiltonianPermutations(int n) diff --git a/WhiteRabbit/Program.cs b/WhiteRabbit/Program.cs index 47ea3b9..05c95a5 100644 --- a/WhiteRabbit/Program.cs +++ b/WhiteRabbit/Program.cs @@ -46,7 +46,7 @@ processor.GeneratePhrases() .Select(phraseBytes => new { phraseBytes, hashVector = ComputeHashVector(phraseBytes) }) - .Where(tuple => expectedHashesAsVectors.Contains(tuple.hashVector)) + .Where(tuple => Array.IndexOf(expectedHashesAsVectors, tuple.hashVector) >= 0) .Select(tuple => new { phrase = Encoding.ASCII.GetString(tuple.phraseBytes), hash = VectorToHexadecimalString(tuple.hashVector) }) .ForAll(phraseInfo => Console.WriteLine($"Found phrase for {phraseInfo.hash}: {phraseInfo.phrase}; time from start is {stopwatch.Elapsed}")); diff --git a/WhiteRabbit/StringsProcessor.cs b/WhiteRabbit/StringsProcessor.cs index 8b7f06f..b18038e 100644 --- a/WhiteRabbit/StringsProcessor.cs +++ b/WhiteRabbit/StringsProcessor.cs @@ -16,18 +16,16 @@ // Dictionary of vectors to array of words represented by this vector this.VectorsToWords = words - .Distinct(new ByteArrayEqualityComparer()) .Select(word => new { word, vector = this.VectorsConverter.GetVector(word) }) .Where(tuple => tuple.vector != null) .Select(tuple => new { tuple.word, vector = tuple.vector.Value }) .GroupBy(tuple => tuple.vector) - .ToDictionary(group => group.Key, group => group.Select(tuple => tuple.word).ToArray()); + .ToDictionary(group => group.Key, group => group.Select(tuple => tuple.word).Distinct(new ByteArrayEqualityComparer()).ToArray()); this.VectorsProcessor = new VectorsProcessor( this.VectorsConverter.GetVector(filteredSource).Value, maxWordsCount, - this.VectorsToWords.Keys, - this.VectorsConverter.GetString); + this.VectorsToWords.Keys); } private VectorsConverter VectorsConverter { get; } @@ -44,12 +42,10 @@ var sums = this.VectorsProcessor.GenerateSequences(); // converting sequences of vectors to the sequences of words... - var anagramsWords = sums - .Select(sum => ImmutableStack.Create(sum.Select(vector => this.VectorsToWords[vector]).ToArray())) - .SelectMany(Flatten) - .Select(stack => stack.ToArray()); - - return anagramsWords.Select(WordsToPhrase); + return sums + .Select(ConvertVectorsToWords) + .SelectMany(FlattenWords) + .Select(ConvertWordsToPhrase); } // Converts e.g. pair of variants [[a, b, c], [d, e]] into all possible pairs: [[a, d], [a, e], [b, d], [b, e], [c, d], [c, e]] @@ -65,19 +61,41 @@ return Flatten(newStack).SelectMany(remainder => wordVariants.Select(word => remainder.Push(word))); } - private byte[] WordsToPhrase(byte[][] words) + private Tuple> ConvertVectorsToWords(Vector[] vectors) { - var result = new byte[this.NumberOfCharacters + words.Length - 1]; + var length = vectors.Length; + var words = new byte[length][][]; + for (var i = 0; i < length; i++) + { + words[i] = this.VectorsToWords[vectors[i]]; + } + + return Tuple.Create(length, ImmutableStack.Create(words)); + } - Buffer.BlockCopy(words[0], 0, result, 0, words[0].Length); - var position = words[0].Length; - for (var i = 1; i < words.Length; i++) + private IEnumerable>> FlattenWords(Tuple> wordVariants) + { + var item1 = wordVariants.Item1; + return Flatten(wordVariants.Item2).Select(words => Tuple.Create(item1, words)); + } + + private byte[] ConvertWordsToPhrase(Tuple> words) + { + var wordCount = words.Item1; + var result = new byte[this.NumberOfCharacters + wordCount - 1]; + + byte[] currentWord; + var currentStack = words.Item2.Pop(out currentWord); + Buffer.BlockCopy(currentWord, 0, result, 0, currentWord.Length); + var position = currentWord.Length; + while (!currentStack.IsEmpty) { result[position] = 32; position++; - Buffer.BlockCopy(words[i], 0, result, position, words[i].Length); - position += words[i].Length; + currentStack = currentStack.Pop(out currentWord); + Buffer.BlockCopy(currentWord, 0, result, position, currentWord.Length); + position += currentWord.Length; } return result; diff --git a/WhiteRabbit/VectorsConverter.cs b/WhiteRabbit/VectorsConverter.cs index a420108..3bc0721 100644 --- a/WhiteRabbit/VectorsConverter.cs +++ b/WhiteRabbit/VectorsConverter.cs @@ -43,10 +43,5 @@ return new Vector(arr); } - - public string GetString(Vector vector) - { - return new string(Enumerable.Range(0, this.IntToChar.Length).SelectMany(i => Enumerable.Repeat((char)this.IntToChar[i], vector[i])).ToArray()); - } } } diff --git a/WhiteRabbit/VectorsProcessor.cs b/WhiteRabbit/VectorsProcessor.cs index b3cb5ab..81613ce 100644 --- a/WhiteRabbit/VectorsProcessor.cs +++ b/WhiteRabbit/VectorsProcessor.cs @@ -17,7 +17,7 @@ PrecomputedPermutationsGenerator.HamiltonianPermutations(0); } - public VectorsProcessor(Vector target, int maxVectorsCount, IEnumerable> dictionary, Func, string> vectorToString) + public VectorsProcessor(Vector target, int maxVectorsCount, IEnumerable> dictionary) { if (Enumerable.Range(0, Vector.Count).Any(i => target[i] > MaxComponentValue)) { @@ -27,7 +27,6 @@ this.Target = target; this.MaxVectorsCount = maxVectorsCount; - this.VectorToString = vectorToString; this.Dictionary = ImmutableArray.Create(FilterVectors(dictionary, target).ToArray()); } @@ -37,10 +36,6 @@ private ImmutableArray Dictionary { get; } - private Func, string> VectorToString { get; } - - private long Iterations { get; set; } = 0; - // Produces all sequences of vectors with the target sum public ParallelQuery[]> GenerateSequences() { @@ -165,9 +160,16 @@ private static IEnumerable GeneratePermutations(T[] original) { - foreach (var permutation in PrecomputedPermutationsGenerator.HamiltonianPermutations(original.Length)) + var length = original.Length; + foreach (var permutation in PrecomputedPermutationsGenerator.HamiltonianPermutations(length)) { - yield return permutation.Select(i => original[i]).ToArray(); + var result = new T[length]; + for (var i = 0; i < length; i++) + { + result[i] = original[permutation[i]]; + } + + yield return result; } }