Permutation filters implemented (to avoid duplicate phrases)

dotnet/WhiteRabbit/PhraseSet.cs

@@ -7,7 +7,7 @@
     {
         public long[] Buffer;
 
-        public unsafe PhraseSet(Word[] words, long[] permutations, int offset, int numberOfCharacters)
+        public unsafe PhraseSet(Word[] words, ulong[] permutations, int offset, int numberOfCharacters)
         {
             Debug.Assert(numberOfCharacters + words.Length - 1 < 27);
 
@@ -18,7 +18,7 @@
                 long* longBuffer = (long*)bufferPointer;
                 int numberOfWords = words.Length;
 
-                fixed (long* permutationsPointer = permutations)
+                fixed (ulong* permutationsPointer = permutations)
                 {
                     var currentPermutationPointer = permutationsPointer + offset;
                     for (var i = 0; i < Constants.PhrasesPerSet; i++, currentPermutationPointer++)

dotnet/WhiteRabbit/PrecomputedPermutationsGenerator.cs

@@ -1,25 +1,74 @@
 namespace WhiteRabbit
 {
+    using System;
     using System.Collections.Generic;
     using System.Linq;
 
     internal static class PrecomputedPermutationsGenerator
     {
-        private static long[][] Permutations { get; } = Enumerable.Range(0, 9).Select(GeneratePermutations).ToArray();
+        private static ulong[][][] Permutations { get; } = Enumerable.Range(0, 9).Select(GeneratePermutations).ToArray();
 
         private static long[] PermutationsNumbers { get; } = GeneratePermutationsNumbers().Take(19).ToArray();
 
-        public static long[] HamiltonianPermutations(int n) => Permutations[n];
+        public static ulong[] HamiltonianPermutations(int n, uint filter) => Permutations[n][filter];
 
         public static long GetPermutationsNumber(int n) => PermutationsNumbers[n];
 
-        private static long[] GeneratePermutations(int n)
+        private static ulong[][] GeneratePermutations(int n)
         {
-            var result = PermutationsGenerator.HamiltonianPermutations(n)
+            if (n == 0)
+            {
+                return new ulong[0][];
+            }
+
+            var allPermutations = PermutationsGenerator.HamiltonianPermutations(n)
                 .Select(FormatPermutation)
                 .ToArray();
 
-            return PadToWholeChunks(result, Constants.PhrasesPerSet);
+            var statesCount = (uint)1 << (n - 1);
+            var result = new ulong[statesCount][];
+            for (uint i = 0; i < statesCount; i++)
+            {
+                result[i] = PadToWholeChunks(FilterPermutations(allPermutations, i).ToArray(), Constants.PhrasesPerSet);
+            }
+
+            return result;
+        }
+
+        private static IEnumerable<ulong> FilterPermutations(IEnumerable<ulong> permutations, uint state)
+        {
+            for (int position = 0; position < 16; position++)
+            {
+                if (((state >> position) & 1) != 0)
+                {
+                    var innerPosition = (uint)position;
+                    permutations = permutations.Where(permutation => IsOrderPreserved(permutation, innerPosition));
+                }
+            }
+
+            return permutations;
+        }
+
+        public static bool IsOrderPreserved(ulong permutation, uint position)
+        {
+            var currentPermutation = permutation;
+
+            while (currentPermutation != 0)
+            {
+                if ((currentPermutation & 15) == position)
+                {
+                    return true;
+                }
+
+                if ((currentPermutation & 15) == (position + 1))
+                {
+                    return false;
+                }
+
+                currentPermutation = currentPermutation >> 4;
+            }
+
+            throw new ApplicationException("Malformed permutation " + permutation + " for position " + position);
         }
 
         private static T[] PadToWholeChunks<T>(T[] original, int chunkSize)
@@ -32,14 +81,14 @@
             return original.Concat(Enumerable.Repeat(default(T), chunkSize - (original.Length % chunkSize))).ToArray();
         }
 
-        private static long FormatPermutation(PermutationsGenerator.Permutation permutation)
+        private static ulong FormatPermutation(PermutationsGenerator.Permutation permutation)
         {
             System.Diagnostics.Debug.Assert(permutation.PermutationData.Length <= 16);
 
-            long result = 0;
+            ulong result = 0;
             for (var i = 0; i < permutation.PermutationData.Length; i++)
             {
-                result |= (long)(permutation.PermutationData[i]) << (4 * i);
+                result |= (ulong)(permutation.PermutationData[i]) << (4 * i);
             }
 
             return result;

dotnet/WhiteRabbit/Program.cs

@@ -78,7 +78,7 @@
                         {
                             var phrase = ToString(phraseSet, i);
                             var hash = ComputeFullMD5(phrase);
-                            Console.WriteLine($"Found phrase for {hash}: {phrase}; time from start is {stopwatch.Elapsed}");
+                            Console.WriteLine($"Found phrase for {hash} ({hashesFirstComponents[i]:x8}): {phrase}; time from start is {stopwatch.Elapsed}");
                         }
                     }
                 });

dotnet/WhiteRabbit/StringsProcessor.cs

@@ -11,7 +11,7 @@
         // Ensure that permutations are precomputed prior to main run, so that processing times will be correct
         static StringsProcessor()
         {
-            PrecomputedPermutationsGenerator.HamiltonianPermutations(0);
+            PrecomputedPermutationsGenerator.HamiltonianPermutations(1, 0);
         }
 
         public StringsProcessor(byte[] sourceString, int maxWordsCount, IEnumerable<byte[]> words)
@@ -59,10 +59,12 @@
             var sums = this.VectorsProcessor.GenerateSequences();
 
             // converting sequences of vectors to the sequences of words...
-            return sums
+            return from sum in sums
-                .Select(this.ConvertVectorsToWords)
+                   let filter = ComputeFilter(sum)
-                .SelectMany(Flattener.Flatten)
+                   let wordsVariants = this.ConvertVectorsToWords(sum)
-                .SelectMany(this.ConvertWordsToPhrases);
+                   from wordsArray in Flattener.Flatten(wordsVariants)
+                   from phraseSet in this.ConvertWordsToPhrases(wordsArray, filter)
+                   select phraseSet;
         }
 
         public long GetPhrasesCount()
@@ -72,6 +74,20 @@
                 .Sum(tuple => tuple.Item2 * PrecomputedPermutationsGenerator.GetPermutationsNumber(tuple.Item1));
         }
 
+        private static uint ComputeFilter(int[] vectors)
+        {
+            uint result = 0;
+            for (var i = 1; i < vectors.Length; i++)
+            {
+                if (vectors[i] == vectors[i - 1])
+                {
+                    result |= (uint)1 << (i - 1);
+                }
+            }
+
+            return result;
+        }
+
         private Word[][] ConvertVectorsToWords(int[] vectors)
         {
             var length = vectors.Length;
@@ -95,9 +111,9 @@
             return Tuple.Create(vectors.Length, result);
         }
 
-        private IEnumerable<PhraseSet> ConvertWordsToPhrases(Word[] words)
+        private IEnumerable<PhraseSet> ConvertWordsToPhrases(Word[] words, uint filter)
         {
-            var permutations = PrecomputedPermutationsGenerator.HamiltonianPermutations(words.Length);
+            var permutations = PrecomputedPermutationsGenerator.HamiltonianPermutations(words.Length, filter);
             var permutationsLength = permutations.Length;
             for (var i = 0; i < permutationsLength; i += Constants.PhrasesPerSet)
             {