How to use C#8 IAsyncEnumerable to async-enumerate tasks run in parallel

Question

If possible I want to create an async-enumerator for tasks launched in parallel. So first to complete is first element of the enumeration, second to finish is second element of

Answer 1

In case you wanna take an async stream (IAsyncEnumerable) and run Select in parallel so the first to finish is the first to come out:

/// 

/// Runs the selectors in parallel and yields in completion order
/// 
public static async IAsyncEnumerable SelectParallel(
    this IAsyncEnumerable source,
    Func> selector)
{
    if (source == null)
    {
        throw new InvalidOperationException("Source is null");
    }

    var enumerator = source.GetAsyncEnumerator();

    var sourceFinished = false;
    var tasks = new HashSet>();

    Task sourceMoveTask = null;
    Task> pipeCompletionTask = null;

    try
    {
        while (!sourceFinished || tasks.Any())
        {
            if (sourceMoveTask == null && !sourceFinished)
            {
                sourceMoveTask = enumerator.MoveNextAsync().AsTask();
            }

            if (pipeCompletionTask == null && tasks.Any())
            {
                pipeCompletionTask = Task.WhenAny(tasks);
            }

            var coreTasks = new Task[] { pipeCompletionTask, sourceMoveTask }
                .Where(t => t != null)
                .ToList();

            if (!coreTasks.Any())
            {
                break;
            }

            await Task.WhenAny(coreTasks);

            if (sourceMoveTask != null && sourceMoveTask.IsCompleted)
            {
                sourceFinished = !sourceMoveTask.Result;

                if (!sourceFinished)
                {
                    try
                    {
                        tasks.Add(selector(enumerator.Current));
                    }
                    catch { }
                }

                sourceMoveTask = null;
            }
            
            if (pipeCompletionTask != null && pipeCompletionTask.IsCompleted)
            {
                var completedTask = pipeCompletionTask.Result;

                if (completedTask.IsCompletedSuccessfully)
                {
                    yield return completedTask.Result;
                }

                tasks.Remove(completedTask);
                pipeCompletionTask = null;
            }
        }
    }
    finally
    {
        await enumerator.DisposeAsync();
    }
}

Can be used like the following:

    static async Task Main(string[] args)
    {
        var source = GetIds();
        var strs = source.SelectParallel(Map);

        await foreach (var str in strs)
        {
            Console.WriteLine(str);
        }
    }

    static async IAsyncEnumerable GetIds()
    {
        foreach (var i in Enumerable.Range(1, 20))
        {
            await Task.Delay(200);
            yield return i;
        }
    }

    static async Task Map(int id)
    {
        await Task.Delay(rnd.Next(1000, 2000));
        return $"{id}_{Thread.CurrentThread.ManagedThreadId}";
    }

Possible output:

[6:31:03 PM] 1_5
[6:31:03 PM] 2_6
[6:31:04 PM] 3_6
[6:31:04 PM] 6_4
[6:31:04 PM] 5_4
[6:31:04 PM] 4_5
[6:31:05 PM] 8_6
[6:31:05 PM] 7_6
[6:31:05 PM] 11_6
[6:31:05 PM] 10_4
[6:31:05 PM] 9_6
[6:31:06 PM] 14_6
[6:31:06 PM] 12_4
[6:31:06 PM] 13_4
[6:31:06 PM] 15_4
[6:31:07 PM] 17_4
[6:31:07 PM] 20_4
[6:31:07 PM] 16_6
[6:31:07 PM] 18_6
[6:31:08 PM] 19_6