Optimized MD5 hash computation

README.md

@@ -41,14 +41,14 @@ Anagrams generation is not parallelized, as even single-threaded performance for
 
 Multi-threaded performance with RyuJIT (.NET 4.6, 64-bit system) on quad-core Sandy Bridge @2.8GHz is as follows:
 
-* If only phrases of at most 4 words are allowed, then it takes **less than 3 seconds** to find and check all 7433016 anagrams; all hashes are solved in first 0.4 seconds.
+* If only phrases of at most 4 words are allowed, then it takes **around 2 seconds** to find and check all 7433016 anagrams; all hashes are solved in first 0.4 seconds.
 
-* If phrases of 5 words are allowed as well, then it takes around 7.5 minutes to find and check all 1348876896 anagrams; all hashes are solved in first 10 seconds.
+* If phrases of 5 words are allowed as well, then it takes around 5.5 minutes to find and check all 1348876896 anagrams; all hashes are solved in first 8.5 seconds.
 Most of time is spent on MD5 computations for correct anagrams, so there is not a lot to optimize further.
 
-* If phrases of 6 words are allowed as well, then "more difficult" hash is solved in 10 seconds, "easiest" in 1 minute, and "hard" in 2.5 minutes.
+* If phrases of 6 words are allowed as well, then "more difficult" hash is solved in 9 seconds, "easiest" in 50 seconds, and "hard" in 2 minutes.
 
-* If phrases of 7 words are allowed as well, then "more difficult" hash is solved in 58 seconds.
+* If phrases of 7 words are allowed as well, then "more difficult" hash is solved in 56 seconds, "easiest" in 7 minutes, and "hard" in 18 minutes.
 
 Note that all measurements were done on a Release build; Debug build is significantly slower.
 

WhiteRabbit/MD5Digest.cs

@@ -0,0 +1,133 @@
+namespace WhiteRabbit
+{
+    using System;
+
+    /**
+     * Code taken from BouncyCastle and optimized for specific constraints (e.g. input is always larger than 4 bytes and smaller than 52 bytes).
+     * base implementation of MD4 family style digest as outlined in
+     * "Handbook of Applied Cryptography", pages 344 - 347.
+     * implementation of MD5 as outlined in "Handbook of Applied Cryptography", pages 346 - 347.
+     */
+    internal static class MD5Digest
+    {
+        public static uint[] Compute(byte[] input)
+        {
+            var length = input.Length;
+
+            var xBytes = new byte[4 * 14];
+            Buffer.BlockCopy(input, 0, xBytes, 0, length);
+            xBytes[length] = 128;
+
+            var x0 = LE_To_UInt32(xBytes, 4 * 0);
+            var x1 = LE_To_UInt32(xBytes, 4 * 1);
+            var x2 = LE_To_UInt32(xBytes, 4 * 2);
+            var x3 = LE_To_UInt32(xBytes, 4 * 3);
+            var x4 = LE_To_UInt32(xBytes, 4 * 4);
+            var x5 = LE_To_UInt32(xBytes, 4 * 5);
+            var x6 = LE_To_UInt32(xBytes, 4 * 6);
+            var x7 = LE_To_UInt32(xBytes, 4 * 7);
+            var x8 = LE_To_UInt32(xBytes, 4 * 8);
+            var x9 = LE_To_UInt32(xBytes, 4 * 9);
+            var x10 = LE_To_UInt32(xBytes, 4 * 10);
+            var x11 = LE_To_UInt32(xBytes, 4 * 11);
+            var x12 = LE_To_UInt32(xBytes, 4 * 12);
+            var x13 = LE_To_UInt32(xBytes, 4 * 13);
+            var x14 = (uint)(length << 3);
+            uint x15 = 0;
+
+            uint a = 0x67452301;
+            uint b = 0xefcdab89;
+            uint c = 0x98badcfe;
+            uint d = 0x10325476;
+
+            a = LeftRotate(x0 + 0xd76aa478 + a + ((b & c) | (~b & d)), 7, 32 - 7) + b;
+            d = LeftRotate(x1 + 0xe8c7b756 + d + ((a & b) | (~a & c)), 12, 32 - 12) + a;
+            c = LeftRotate(x2 + 0x242070db + c + ((d & a) | (~d & b)), 17, 32 - 17) + d;
+            b = LeftRotate(x3 + 0xc1bdceee + b + ((c & d) | (~c & a)), 22, 32 - 22) + c;
+            a = LeftRotate(x4 + 0xf57c0faf + a + ((b & c) | (~b & d)), 7, 32 - 7) + b;
+            d = LeftRotate(x5 + 0x4787c62a + d + ((a & b) | (~a & c)), 12, 32 - 12) + a;
+            c = LeftRotate(x6 + 0xa8304613 + c + ((d & a) | (~d & b)), 17, 32 - 17) + d;
+            b = LeftRotate(x7 + 0xfd469501 + b + ((c & d) | (~c & a)), 22, 32 - 22) + c;
+            a = LeftRotate(x8 + 0x698098d8 + a + ((b & c) | (~b & d)), 7, 32 - 7) + b;
+            d = LeftRotate(x9 + 0x8b44f7af + d + ((a & b) | (~a & c)), 12, 32 - 12) + a;
+            c = LeftRotate(x10 + 0xffff5bb1 + c + ((d & a) | (~d & b)), 17, 32 - 17) + d;
+            b = LeftRotate(x11 + 0x895cd7be + b + ((c & d) | (~c & a)), 22, 32 - 22) + c;
+            a = LeftRotate(x12 + 0x6b901122 + a + ((b & c) | (~b & d)), 7, 32 - 7) + b;
+            d = LeftRotate(x13 + 0xfd987193 + d + ((a & b) | (~a & c)), 12, 32 - 12) + a;
+            c = LeftRotate(x14 + 0xa679438e + c + ((d & a) | (~d & b)), 17, 32 - 17) + d;
+            b = LeftRotate(x15 + 0x49b40821 + b + ((c & d) | (~c & a)), 22, 32 - 22) + c;
+
+            a = LeftRotate(x1 + 0xf61e2562 + a + ((b & d) | (c & ~d)), 5, 32 - 5) + b;
+            d = LeftRotate(x6 + 0xc040b340 + d + ((a & c) | (b & ~c)), 9, 32 - 9) + a;
+            c = LeftRotate(x11 + 0x265e5a51 + c + ((d & b) | (a & ~b)), 14, 32 - 14) + d;
+            b = LeftRotate(x0 + 0xe9b6c7aa + b + ((c & a) | (d & ~a)), 20, 32 - 20) + c;
+            a = LeftRotate(x5 + 0xd62f105d + a + ((b & d) | (c & ~d)), 5, 32 - 5) + b;
+            d = LeftRotate(x10 + 0x2441453 + d + ((a & c) | (b & ~c)), 9, 32 - 9) + a;
+            c = LeftRotate(x15 + 0xd8a1e681 + c + ((d & b) | (a & ~b)), 14, 32 - 14) + d;
+            b = LeftRotate(x4 + 0xe7d3fbc8 + b + ((c & a) | (d & ~a)), 20, 32 - 20) + c;
+            a = LeftRotate(x9 + 0x21e1cde6 + a + ((b & d) | (c & ~d)), 5, 32 - 5) + b;
+            d = LeftRotate(x14 + 0xc33707d6 + d + ((a & c) | (b & ~c)), 9, 32 - 9) + a;
+            c = LeftRotate(x3 + 0xf4d50d87 + c + ((d & b) | (a & ~b)), 14, 32 - 14) + d;
+            b = LeftRotate(x8 + 0x455a14ed + b + ((c & a) | (d & ~a)), 20, 32 - 20) + c;
+            a = LeftRotate(x13 + 0xa9e3e905 + a + ((b & d) | (c & ~d)), 5, 32 - 5) + b;
+            d = LeftRotate(x2 + 0xfcefa3f8 + d + ((a & c) | (b & ~c)), 9, 32 - 9) + a;
+            c = LeftRotate(x7 + 0x676f02d9 + c + ((d & b) | (a & ~b)), 14, 32 - 14) + d;
+            b = LeftRotate(x12 + 0x8d2a4c8a + b + ((c & a) | (d & ~a)), 20, 32 - 20) + c;
+
+            a = LeftRotate(x5 + 0xfffa3942 + a + (b ^ c ^ d), 4, 32 - 4) + b;
+            d = LeftRotate(x8 + 0x8771f681 + d + (a ^ b ^ c), 11, 32 - 11) + a;
+            c = LeftRotate(x11 + 0x6d9d6122 + c + (d ^ a ^ b), 16, 32 - 16) + d;
+            b = LeftRotate(x14 + 0xfde5380c + b + (c ^ d ^ a), 23, 32 - 23) + c;
+            a = LeftRotate(x1 + 0xa4beea44 + a + (b ^ c ^ d), 4, 32 - 4) + b;
+            d = LeftRotate(x4 + 0x4bdecfa9 + d + (a ^ b ^ c), 11, 32 - 11) + a;
+            c = LeftRotate(x7 + 0xf6bb4b60 + c + (d ^ a ^ b), 16, 32 - 16) + d;
+            b = LeftRotate(x10 + 0xbebfbc70 + b + (c ^ d ^ a), 23, 32 - 23) + c;
+            a = LeftRotate(x13 + 0x289b7ec6 + a + (b ^ c ^ d), 4, 32 - 4) + b;
+            d = LeftRotate(x0 + 0xeaa127fa + d + (a ^ b ^ c), 11, 32 - 11) + a;
+            c = LeftRotate(x3 + 0xd4ef3085 + c + (d ^ a ^ b), 16, 32 - 16) + d;
+            b = LeftRotate(x6 + 0x4881d05 + b + (c ^ d ^ a), 23, 32 - 23) + c;
+            a = LeftRotate(x9 + 0xd9d4d039 + a + (b ^ c ^ d), 4, 32 - 4) + b;
+            d = LeftRotate(x12 + 0xe6db99e5 + d + (a ^ b ^ c), 11, 32 - 11) + a;
+            c = LeftRotate(x15 + 0x1fa27cf8 + c + (d ^ a ^ b), 16, 32 - 16) + d;
+            b = LeftRotate(x2 + 0xc4ac5665 + b + (c ^ d ^ a), 23, 32 - 23) + c;
+
+            a = LeftRotate(x0 + 0xf4292244 + a + (c ^ (b | ~d)), 6, 32 - 6) + b;
+            d = LeftRotate(x7 + 0x432aff97 + d + (b ^ (a | ~c)), 10, 32 - 10) + a;
+            c = LeftRotate(x14 + 0xab9423a7 + c + (a ^ (d | ~b)), 15, 32 - 15) + d;
+            b = LeftRotate(x5 + 0xfc93a039 + b + (d ^ (c | ~a)), 21, 32 - 21) + c;
+            a = LeftRotate(x12 + 0x655b59c3 + a + (c ^ (b | ~d)), 6, 32 - 6) + b;
+            d = LeftRotate(x3 + 0x8f0ccc92 + d + (b ^ (a | ~c)), 10, 32 - 10) + a;
+            c = LeftRotate(x10 + 0xffeff47d + c + (a ^ (d | ~b)), 15, 32 - 15) + d;
+            b = LeftRotate(x1 + 0x85845dd1 + b + (d ^ (c | ~a)), 21, 32 - 21) + c;
+            a = LeftRotate(x8 + 0x6fa87e4f + a + (c ^ (b | ~d)), 6, 32 - 6) + b;
+            d = LeftRotate(x15 + 0xfe2ce6e0 + d + (b ^ (a | ~c)), 10, 32 - 10) + a;
+            c = LeftRotate(x6 + 0xa3014314 + c + (a ^ (d | ~b)), 15, 32 - 15) + d;
+            b = LeftRotate(x13 + 0x4e0811a1 + b + (d ^ (c | ~a)), 21, 32 - 21) + c;
+            a = LeftRotate(x4 + 0xf7537e82 + a + (c ^ (b | ~d)), 6, 32 - 6) + b;
+            d = LeftRotate(x11 + 0xbd3af235 + d + (b ^ (a | ~c)), 10, 32 - 10) + a;
+            c = LeftRotate(x2 + 0x2ad7d2bb + c + (a ^ (d | ~b)), 15, 32 - 15) + d;
+            b = LeftRotate(x9 + 0xeb86d391 + b + (d ^ (c | ~a)), 21, 32 - 21) + c;
+
+            return new[]
+            {
+                0x67452301 + a,
+                0xefcdab89 + b,
+                0x98badcfe + c,
+                0x10325476 + d,
+            };
+        }
+
+        private static uint LE_To_UInt32(byte[] bs, int off)
+        {
+            return (uint)bs[off]
+                | (uint)bs[off + 1] << 8
+                | (uint)bs[off + 2] << 16
+                | (uint)bs[off + 3] << 24;
+        }
+
+        private static uint LeftRotate(uint x, int left, int right)
+        {
+            return (x << left) | (x >> right);
+        }
+    }
+}

WhiteRabbit/Program.cs

@@ -8,7 +8,6 @@
     using System.Linq;
     using System.Numerics;
     using System.Text;
-    using Org.BouncyCastle.Crypto.Digests;
 
     /// <summary>
     /// Main class
@@ -30,7 +29,7 @@
 
             var expectedHashesAsVectors = ConfigurationManager.AppSettings["ExpectedHashes"]
                 .Split(',')
-                .Select(hash => new Vector<byte>(HexadecimalStringToByteArray(hash)))
+                .Select(hash => new Vector<uint>(HexadecimalStringToUnsignedIntArray(hash)))
                 .ToArray();
 
 #if DEBUG
@@ -85,27 +84,33 @@
         }
 
         // Code taken from http://stackoverflow.com/a/321404/831314
-        private static byte[] HexadecimalStringToByteArray(string hex)
+        private static uint[] HexadecimalStringToUnsignedIntArray(string hex)
         {
             return Enumerable.Range(0, hex.Length)
-                             .Where(x => x % 2 == 0)
-                             .Select(x => Convert.ToByte(hex.Substring(x, 2), 16))
+                             .Where(x => x % 8 == 0)
+                             .Select(x => ChangeEndianness(hex.Substring(x, 8)))
+                             .Select(hexLe => Convert.ToUInt32(hexLe, 16))
                              .ToArray();
         }
 
         // Bouncy Castle is used instead of standard .NET methods for performance reasons
-        private static Vector<byte> ComputeHashVector(byte[] input)
+        private static Vector<uint> ComputeHashVector(byte[] input)
         {
-            var digest = new MD5Digest();
-            digest.BlockUpdate(input, 0, input.Length);
-            byte[] hash = new byte[16];
-            digest.DoFinal(hash, 0);
-            return new Vector<byte>(hash);
+            return new Vector<uint>(MD5Digest.Compute(input));
         }
 
-        private static string VectorToHexadecimalString(Vector<byte> hash)
+        private static string VectorToHexadecimalString(Vector<uint> hash)
         {
-            return string.Concat(Enumerable.Range(0, 16).Select(i => hash[i].ToString("x2")));
+            var components = Enumerable.Range(0, 4)
+                .Select(i => hash[i].ToString("x8"))
+                .Select(ChangeEndianness);
+
+            return string.Concat(components);
+        }
+
+        private static string ChangeEndianness(string hex)
+        {
+            return hex.Substring(6, 2) + hex.Substring(4, 2) + hex.Substring(2, 2) + hex.Substring(0, 2);
         }
 
         private static IEnumerable<byte[]> ReadInput()

WhiteRabbit/WhiteRabbit.csproj

@@ -36,10 +36,6 @@
     <Prefer32Bit>false</Prefer32Bit>
   </PropertyGroup>
   <ItemGroup>
-    <Reference Include="BouncyCastle.Crypto, Version=1.8.1.0, Culture=neutral, PublicKeyToken=0e99375e54769942">
-      <HintPath>..\packages\BouncyCastle.1.8.1\lib\BouncyCastle.Crypto.dll</HintPath>
-      <Private>True</Private>
-    </Reference>
     <Reference Include="System" />
     <Reference Include="System.Collections.Immutable, Version=1.2.1.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
       <HintPath>..\packages\System.Collections.Immutable.1.3.1\lib\portable-net45+win8+wp8+wpa81\System.Collections.Immutable.dll</HintPath>
@@ -62,6 +58,7 @@
   <ItemGroup>
     <Compile Include="ByteArrayEqualityComparer.cs" />
     <Compile Include="Flattener.cs" />
+    <Compile Include="MD5Digest.cs" />
     <Compile Include="PrecomputedPermutationsGenerator.cs" />
     <Compile Include="PermutationsGenerator.cs" />
     <Compile Include="StringsProcessor.cs" />

WhiteRabbit/packages.config

@@ -1,6 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="BouncyCastle" version="1.8.1" targetFramework="net46" />
   <package id="System.Collections.Immutable" version="1.3.1" targetFramework="net46" />
   <package id="System.Numerics.Vectors" version="4.3.0" targetFramework="net46" />
 </packages>