Improved dataflow integration

dataflow
Inga 🏳‍🌈 7 years ago
parent 08ef75d4cf
commit 85dc08f5a8
  1. 87
      WhiteRabbit/DataflowBlockHelpers.cs
  2. 23
      WhiteRabbit/Extensions.cs
  3. 57
      WhiteRabbit/Program.cs
  4. 41
      WhiteRabbit/StringsProcessor.cs
  5. 29
      WhiteRabbit/VectorsProcessor.cs
  6. 2
      WhiteRabbit/WhiteRabbit.csproj

@ -0,0 +1,87 @@
namespace WhiteRabbit
{
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
internal static class DataflowBlockHelpers
{
private static ExecutionDataflowBlockOptions ExecutionOptions { get; } = new ExecutionDataflowBlockOptions
{
BoundedCapacity = 100000,
};
public static IPropagatorBlock<T, T> Id<T>()
{
return new TransformBlock<T, T>(element => element, ExecutionOptions);
}
public static void WriteToTargetBlock<T>(this IEnumerable<T> enumerable, ITargetBlock<T> target)
{
var block = new TransformBlock<T, T>(line => line, ExecutionOptions);
block.LinkForever(target);
WriteToTargetBlockAsync(enumerable, block).Wait();
block.Complete();
}
public static IPropagatorBlock<TInput, TOutput> PipeMany<TInput, TIntermediate, TOutput>(this IPropagatorBlock<TInput, TIntermediate> source, Func<TIntermediate, IEnumerable<TOutput>> mapper)
{
return source.Pipe(new TransformManyBlock<TIntermediate, TOutput>(mapper, ExecutionOptions));
}
public static IPropagatorBlock<TInput, TOutput> Pipe<TInput, TIntermediate, TOutput>(this IPropagatorBlock<TInput, TIntermediate> source, Func<TIntermediate, TOutput> mapper)
{
return source.Pipe(new TransformBlock<TIntermediate, TOutput>(mapper, ExecutionOptions));
}
public static IPropagatorBlock<TInput, TOutput> Pipe<TInput, TIntermediate, TOutput>(this IPropagatorBlock<TInput, TIntermediate> source, IPropagatorBlock<TIntermediate, TOutput> target)
{
source.LinkForever(target);
return DataflowBlock.Encapsulate(source, target);
}
public static ISourceBlock<TOutput> Pipe<TInput, TOutput>(this ISourceBlock<TInput> source, Func<TInput, TOutput> mapper)
{
return source.Pipe(new TransformBlock<TInput, TOutput>(mapper, ExecutionOptions));
}
public static ISourceBlock<TOutput> Pipe<TInput, TOutput>(this ISourceBlock<TInput> source, IPropagatorBlock<TInput, TOutput> target)
{
source.LinkForever(target);
return target;
}
public static Task LinkForever<TOutput>(this ISourceBlock<TOutput> source, Action<TOutput> action)
{
return source.LinkForever(new ActionBlock<TOutput>(action, ExecutionOptions));
}
public static Task LinkForever<TOutput>(this ISourceBlock<TOutput> source, ITargetBlock<TOutput> target)
{
source.LinkTo(target);
source.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
target.Fault(t.Exception);
}
else
{
target.Complete();
}
});
return target.Completion;
}
private static async Task WriteToTargetBlockAsync<T>(IEnumerable<T> enumerable, ITargetBlock<T> target)
{
foreach (var element in enumerable)
{
await target.SendAsync(element);
}
}
}
}

@ -1,23 +0,0 @@
namespace WhiteRabbit
{
using System.Threading.Tasks.Dataflow;
internal static class Extensions
{
public static void LinkForever<TOutput>(this ISourceBlock<TOutput> source, ITargetBlock<TOutput> target)
{
source.LinkTo(target);
source.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
target.Fault(t.Exception);
}
else
{
target.Complete();
}
});
}
}
}

@ -7,7 +7,7 @@
using System.Numerics;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks.Dataflow;
/// <summary>
/// Main class
/// </summary>
@ -35,39 +35,40 @@
var processor = new StringsProcessor("poultry outwits ants", 4, ReadInput());
var unorderedSequencesToPhrases = processor.CreateUnorderedSequencesToPhrasesTransform();
var phrasesToPhrasesWithHash = new TransformBlock<string, PhraseWithHash>(phrase =>
{
var hash = new Vector<byte>(hasher.ComputeHash(Encoding.ASCII.GetBytes(phrase)));
return new PhraseWithHash(phrase, hash);
});
var startBlock = DataflowBlockHelpers.Id<Vector<byte>[]>();
var phrasesWithHashFilter = new TransformManyBlock<PhraseWithHash, PhraseWithHash>(phraseWithHash =>
{
if (!expectedHashesAsVectors.Contains(phraseWithHash.Hash))
var task = startBlock
.Pipe(processor.CreateUnorderedSequencesToPhrasesTransform())
.Pipe(phrase =>
{
return Enumerable.Empty<PhraseWithHash>();
}
//Console.WriteLine("Found phrase: " + phrase);
return new PhraseWithHash[]
var hash = new Vector<byte>(hasher.ComputeHash(Encoding.ASCII.GetBytes(phrase)));
return new PhraseWithHash(phrase, hash);
})
.PipeMany(phraseWithHash =>
{
phraseWithHash,
};
});
var printPhrases = new ActionBlock<PhraseWithHash>(phraseWithHash =>
{
Console.WriteLine($"Found phrase for hash {phraseWithHash.Hash}: {phraseWithHash.Phrase} (spent {stopwatch.Elapsed})");
});
unorderedSequencesToPhrases.LinkForever(phrasesToPhrasesWithHash);
phrasesToPhrasesWithHash.LinkForever(phrasesWithHashFilter);
phrasesWithHashFilter.LinkForever(printPhrases);
//Console.WriteLine($"Found phrase with hash: " + phraseWithHash.Phrase);
if (!expectedHashesAsVectors.Contains(phraseWithHash.Hash))
{
return Enumerable.Empty<PhraseWithHash>();
}
return new PhraseWithHash[]
{
phraseWithHash,
};
})
.LinkForever(phraseWithHash =>
{
Console.WriteLine($"Found phrase for hash {phraseWithHash.Hash}: {phraseWithHash.Phrase} (spent {stopwatch.Elapsed})");
});
processor.PostUnorderedSequences(unorderedSequencesToPhrases);
Console.WriteLine($"Initialization complete: time spent: {stopwatch.Elapsed}");
processor.PostUnorderedSequences(startBlock);
printPhrases.Completion.Wait();
task.Wait();
Console.WriteLine($"Total time spent: {stopwatch.Elapsed}");
}
}

@ -34,20 +34,18 @@
private Dictionary<Vector<byte>, string[]> VectorsToWords { get; }
public void PostUnorderedSequences(ITargetBlock<Vector<byte>[]> target) => this.VectorsProcessor.PostUnorderedSequences(target);
public void PostUnorderedSequences(ITargetBlock<Vector<byte>[]> target)
{
this.VectorsProcessor.GenerateUnorderedSequences().WriteToTargetBlock(target);
}
public IPropagatorBlock<Vector<byte>[], string> CreateUnorderedSequencesToPhrasesTransform()
{
var unorderedSequencesToOrderedSequences = this.VectorsProcessor.CreateUnorderedSequencesToOrderedSequencesTransform();
var orderedSequencesToWordVariants = this.CreateOrderedSequencesToWordVariantsTransform();
var wordVariantsToFlatWords = this.CreateWordVariantsToFlatWordsTransform();
var flatWordsToPhrases = this.CreateFlatWordsToPhrasesTransform();
unorderedSequencesToOrderedSequences.LinkForever(orderedSequencesToWordVariants);
orderedSequencesToWordVariants.LinkForever(wordVariantsToFlatWords);
wordVariantsToFlatWords.LinkForever(flatWordsToPhrases);
return DataflowBlock.Encapsulate(unorderedSequencesToOrderedSequences, flatWordsToPhrases);
return DataflowBlockHelpers.Id<Vector<byte>[]>()
.PipeMany(this.VectorsProcessor.UnorderedSequenceToOrderedSequences)
.Pipe(this.OrderedSequenceToWordVariants)
.PipeMany(this.WordVariantsToFlatWords)
.Pipe(this.FlatWordsToPhrase);
}
// Converts e.g. pair of variants [[a, b, c], [d, e]] into all possible pairs: [[a, d], [a, e], [b, d], [b, e], [c, d], [c, e]]
@ -63,28 +61,19 @@
return this.Flatten(newStack).SelectMany(remainder => wordVariants.Select(word => remainder.Push(word)));
}
private IPropagatorBlock<Vector<byte>[], ImmutableStack<string[]>> CreateOrderedSequencesToWordVariantsTransform()
private ImmutableStack<string[]> OrderedSequenceToWordVariants(Vector<byte>[] sum)
{
return new TransformBlock<Vector<byte>[], ImmutableStack<string[]>>(sum =>
{
return ImmutableStack.CreateRange(sum.Select(vector => this.VectorsToWords[vector]));
});
return ImmutableStack.CreateRange(sum.Select(vector => this.VectorsToWords[vector]));
}
private IPropagatorBlock<ImmutableStack<string[]>, ImmutableStack<string>> CreateWordVariantsToFlatWordsTransform()
private IEnumerable<ImmutableStack<string>> WordVariantsToFlatWords(ImmutableStack<string[]> wordVariants)
{
return new TransformManyBlock<ImmutableStack<string[]>, ImmutableStack<string>>(wordVariants =>
{
return this.Flatten(wordVariants);
});
return this.Flatten(wordVariants);
}
private IPropagatorBlock<ImmutableStack<string>, string> CreateFlatWordsToPhrasesTransform()
private string FlatWordsToPhrase(ImmutableStack<string> words)
{
return new TransformBlock<ImmutableStack<string>, string>(words =>
{
return string.Join(" ", words);
});
return string.Join(" ", words);
}
}
}

@ -6,6 +6,7 @@
using System.Diagnostics;
using System.Linq;
using System.Numerics;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
internal class VectorsProcessor
{
@ -37,23 +38,17 @@
private long Iterations { get; set; } = 0;
public IPropagatorBlock<Vector<byte>[], Vector<byte>[]> CreateUnorderedSequencesToOrderedSequencesTransform()
public IEnumerable<Vector<byte>[]> UnorderedSequenceToOrderedSequences(Vector<byte>[] sequence)
{
return new TransformManyBlock<Vector<byte>[], Vector<byte>[]>(sequence =>
foreach (var permutation in PrecomputedPermutationsGenerator.HamiltonianPermutations(sequence.Length))
{
return this.GeneratePermutations(sequence);
});
yield return permutation.Select(i => sequence[i]).ToArray();
}
}
public void PostUnorderedSequences(ITargetBlock<Vector<byte>[]> target)
public IEnumerable<Vector<byte>[]> GenerateUnorderedSequences()
{
var sequences = this.GenerateUnorderedSequences(this.Target, ImmutableStack.Create<Vector<byte>>(), this.Vectors);
foreach (var sequence in sequences)
{
target.Post(sequence);
}
target.Complete();
return this.GenerateUnorderedSequences(this.Target, ImmutableStack.Create<Vector<byte>>(), this.Vectors);
}
// We want words with more letters (and among these, words with more "rare" letters) to appear first, to reduce the searching time somewhat.
@ -79,7 +74,7 @@
.OrderBy(vector => GetVectorWeight(vector, target));
}
[Conditional("DEBUG")]
[Conditional("XDEBUG")]
private void DebugState(ImmutableStack<Vector<byte>> partialSumStack, Vector<byte> currentVector)
{
this.Iterations++;
@ -140,13 +135,5 @@
}
}
}
private IEnumerable<T[]> GeneratePermutations<T>(T[] original)
{
foreach (var permutation in PrecomputedPermutationsGenerator.HamiltonianPermutations(original.Length))
{
yield return permutation.Select(i => original[i]).ToArray();
}
}
}
}

@ -57,7 +57,7 @@
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="Extensions.cs" />
<Compile Include="DataflowBlockHelpers.cs" />
<Compile Include="PrecomputedPermutationsGenerator.cs" />
<Compile Include="PermutationsGenerator.cs" />
<Compile Include="StringsProcessor.cs" />

Loading…
Cancel
Save