From 325ae0b3140f07fce0fdc255d17870e03c87d397 Mon Sep 17 00:00:00 2001
From: inga-lovinde <52715130+inga-lovinde@users.noreply.github.com>
Date: Tue, 21 Mar 2017 11:43:56 +0300
Subject: [PATCH] Another 2x speedup by hardcoding flattening for fixed arrays
---
README.md | 9 +-
WhiteRabbit/Flattener.cs | 181 ++++++++++++++++++++++++++++++++
WhiteRabbit/StringsProcessor.cs | 37 ++-----
WhiteRabbit/WhiteRabbit.csproj | 1 +
4 files changed, 195 insertions(+), 33 deletions(-)
create mode 100644 WhiteRabbit/Flattener.cs
diff --git a/README.md b/README.md
index 2b864cc..de57773 100644
--- a/README.md
+++ b/README.md
@@ -22,13 +22,14 @@ Anagrams generation is not parallelized, as even single-threaded performance for
Multi-threaded performance with RyuJIT (.NET 4.6, 64-bit system) on quad-core Sandy Bridge @2.8GHz is as follows:
-* If only phrases of at most 4 words are allowed, then it takes around 4 seconds to find and check all 7433016 anagrams; all hashes are solved in first 0.5 seconds.
+* If only phrases of at most 4 words are allowed, then it takes **less than 3 seconds** to find and check all 7433016 anagrams; all hashes are solved in first 0.4 seconds.
-* If phrases of 5 words are allowed as well, then it takes around 12 minutes to find and check all 1348876896 anagrams; all hashes are solved in first 18 seconds. Most of time is spent on MD5 computations for correct anagrams, so there is not a lot to optimize further.
+* If phrases of 5 words are allowed as well, then it takes around 7.5 minutes to find and check all 1348876896 anagrams; all hashes are solved in first 10 seconds.
+Most of time is spent on MD5 computations for correct anagrams, so there is not a lot to optimize further.
-* If phrases of 6 words are allowed as well, then "more difficult" hash is solved in 19 seconds, "easiest" in 2 minutes, and "hard" in less than 5 minutes.
+* If phrases of 6 words are allowed as well, then "more difficult" hash is solved in 10 seconds, "easiest" in 1 minute, and "hard" in 2.5 minutes.
-* If phrases of 7 words are allowed as well, then "more difficult" hash is solved in ~2 minutes.
+* If phrases of 7 words are allowed as well, then "more difficult" hash is solved in 58 seconds.
Note that all measurements were done on a Release build; Debug build is significantly slower.
diff --git a/WhiteRabbit/Flattener.cs b/WhiteRabbit/Flattener.cs
new file mode 100644
index 0000000..43b46dc
--- /dev/null
+++ b/WhiteRabbit/Flattener.cs
@@ -0,0 +1,181 @@
+namespace WhiteRabbit
+{
+ using System.Collections.Generic;
+ using System.Collections.Immutable;
+ using System.Linq;
+
+ ///
+ /// Converts e.g. pair of variants [[a, b, c], [d, e]] into all possible pairs: [[a, d], [a, e], [b, d], [b, e], [c, d], [c, e]]
+ ///
+ internal static class Flattener
+ {
+ // Slow universal implementation
+ private static IEnumerable> FlattenAny(ImmutableStack phrase)
+ {
+ if (phrase.IsEmpty)
+ {
+ return new[] { ImmutableStack.Create() };
+ }
+
+ T[] wordVariants;
+ var newStack = phrase.Pop(out wordVariants);
+ return FlattenAny(newStack).SelectMany(remainder => wordVariants.Select(word => remainder.Push(word)));
+ }
+
+ // Fast hard-coded implementation for 3 words
+ private static IEnumerable Flatten3(T[][] phrase)
+ {
+ for (var i0 = 0; i0 < phrase[0].Length; i0++)
+ {
+ for (var i1 = 0; i1 < phrase[1].Length; i1++)
+ {
+ for (var i2 = 0; i2 < phrase[2].Length; i2++)
+ {
+ yield return new T[]
+ {
+ phrase[0][i0],
+ phrase[1][i1],
+ phrase[2][i2],
+ };
+ }
+ }
+ }
+ }
+
+ // Fast hard-coded implementation for 4 words
+ private static IEnumerable Flatten4(T[][] phrase)
+ {
+ for (var i0 = 0; i0 < phrase[0].Length; i0++)
+ {
+ for (var i1 = 0; i1 < phrase[1].Length; i1++)
+ {
+ for (var i2 = 0; i2 < phrase[2].Length; i2++)
+ {
+ for (var i3 = 0; i3 < phrase[3].Length; i3++)
+ {
+ yield return new T[]
+ {
+ phrase[0][i0],
+ phrase[1][i1],
+ phrase[2][i2],
+ phrase[3][i3],
+ };
+ }
+ }
+ }
+ }
+ }
+
+ // Fast hard-coded implementation for 5 words
+ private static IEnumerable Flatten5(T[][] phrase)
+ {
+ for (var i0 = 0; i0 < phrase[0].Length; i0++)
+ {
+ for (var i1 = 0; i1 < phrase[1].Length; i1++)
+ {
+ for (var i2 = 0; i2 < phrase[2].Length; i2++)
+ {
+ for (var i3 = 0; i3 < phrase[3].Length; i3++)
+ {
+ for (var i4 = 0; i4 < phrase[4].Length; i4++)
+ {
+ yield return new T[]
+ {
+ phrase[0][i0],
+ phrase[1][i1],
+ phrase[2][i2],
+ phrase[3][i3],
+ phrase[4][i4],
+ };
+ }
+ }
+ }
+ }
+ }
+ }
+
+ // Fast hard-coded implementation for 6 words
+ private static IEnumerable Flatten6(T[][] phrase)
+ {
+ for (var i0 = 0; i0 < phrase[0].Length; i0++)
+ {
+ for (var i1 = 0; i1 < phrase[1].Length; i1++)
+ {
+ for (var i2 = 0; i2 < phrase[2].Length; i2++)
+ {
+ for (var i3 = 0; i3 < phrase[3].Length; i3++)
+ {
+ for (var i4 = 0; i4 < phrase[4].Length; i4++)
+ {
+ for (var i5 = 0; i5 < phrase[5].Length; i5++)
+ {
+ yield return new T[]
+ {
+ phrase[0][i0],
+ phrase[1][i1],
+ phrase[2][i2],
+ phrase[3][i3],
+ phrase[4][i4],
+ };
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ // Fast hard-coded implementation for 7 words
+ private static IEnumerable Flatten7(T[][] phrase)
+ {
+ for (var i0 = 0; i0 < phrase[0].Length; i0++)
+ {
+ for (var i1 = 0; i1 < phrase[1].Length; i1++)
+ {
+ for (var i2 = 0; i2 < phrase[2].Length; i2++)
+ {
+ for (var i3 = 0; i3 < phrase[3].Length; i3++)
+ {
+ for (var i4 = 0; i4 < phrase[4].Length; i4++)
+ {
+ for (var i5 = 0; i5 < phrase[5].Length; i5++)
+ {
+ for (var i6 = 0; i6 < phrase[6].Length; i6++)
+ {
+ yield return new T[]
+ {
+ phrase[0][i0],
+ phrase[1][i1],
+ phrase[2][i2],
+ phrase[3][i3],
+ phrase[4][i4],
+ };
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ public static IEnumerable Flatten(T[][] wordVariants)
+ {
+ switch (wordVariants.Length)
+ {
+ case 3:
+ return Flatten3(wordVariants);
+ case 4:
+ return Flatten4(wordVariants);
+ case 5:
+ return Flatten5(wordVariants);
+ case 6:
+ return Flatten6(wordVariants);
+ case 7:
+ return Flatten7(wordVariants);
+ default:
+ return FlattenAny(ImmutableStack.Create(wordVariants)).Select(words => words.ToArray());
+ }
+ }
+ }
+}
diff --git a/WhiteRabbit/StringsProcessor.cs b/WhiteRabbit/StringsProcessor.cs
index 1a3b6fb..a870da2 100644
--- a/WhiteRabbit/StringsProcessor.cs
+++ b/WhiteRabbit/StringsProcessor.cs
@@ -54,30 +54,11 @@
// converting sequences of vectors to the sequences of words...
return sums
.Select(this.ConvertVectorsToWords)
- .SelectMany(FlattenWords)
+ .SelectMany(Flattener.Flatten)
.Select(this.ConvertWordsToPhrase);
}
- // Converts e.g. pair of variants [[a, b, c], [d, e]] into all possible pairs: [[a, d], [a, e], [b, d], [b, e], [c, d], [c, e]]
- private static IEnumerable> Flatten(ImmutableStack phrase)
- {
- if (phrase.IsEmpty)
- {
- return new[] { ImmutableStack.Create() };
- }
-
- T[] wordVariants;
- var newStack = phrase.Pop(out wordVariants);
- return Flatten(newStack).SelectMany(remainder => wordVariants.Select(word => remainder.Push(word)));
- }
-
- private static IEnumerable>> FlattenWords(Tuple> wordVariants)
- {
- var item1 = wordVariants.Item1;
- return Flatten(wordVariants.Item2).Select(words => Tuple.Create(item1, words));
- }
-
- private Tuple> ConvertVectorsToWords(int[] vectors)
+ private byte[][][] ConvertVectorsToWords(int[] vectors)
{
var length = vectors.Length;
var words = new byte[length][][];
@@ -86,24 +67,22 @@
words[i] = this.WordsDictionary[vectors[i]];
}
- return Tuple.Create(length, ImmutableStack.Create(words));
+ return words;
}
- private byte[] ConvertWordsToPhrase(Tuple> words)
+ private byte[] ConvertWordsToPhrase(byte[][] words)
{
- var wordCount = words.Item1;
- var result = new byte[this.NumberOfCharacters + wordCount - 1];
+ var result = new byte[this.NumberOfCharacters + words.Length - 1];
- byte[] currentWord;
- var currentStack = words.Item2.Pop(out currentWord);
+ byte[] currentWord = words[0];
Buffer.BlockCopy(currentWord, 0, result, 0, currentWord.Length);
var position = currentWord.Length;
- while (!currentStack.IsEmpty)
+ for (var i = 1; i < words.Length; i++)
{
result[position] = 32;
position++;
- currentStack = currentStack.Pop(out currentWord);
+ currentWord = words[i];
Buffer.BlockCopy(currentWord, 0, result, position, currentWord.Length);
position += currentWord.Length;
}
diff --git a/WhiteRabbit/WhiteRabbit.csproj b/WhiteRabbit/WhiteRabbit.csproj
index 0d1a9cb..144e295 100644
--- a/WhiteRabbit/WhiteRabbit.csproj
+++ b/WhiteRabbit/WhiteRabbit.csproj
@@ -60,6 +60,7 @@
+