How to limit the amount of concurrent async I/O operations?

Question

// let\'s say there is a list of 1000+ URLs
string[] urls = { \"http://google.com\", \"http://yahoo.com\", ... };

// now let\'s send HTTP requests to each of these          


        

Answer 1

SemaphoreSlim can be very helpful here. Here's the extension method I've created.

    /// 

    /// Concurrently Executes async actions for each item of 
    /// 
    /// Type of IEnumerable
    /// instance of "/>
    /// an async  to execute
    /// Optional, max numbers of the actions to run in parallel,
    /// Must be grater than 0
    /// A Task representing an async operation
    /// If the maxActionsToRunInParallel is less than 1
    public static async Task ForEachAsyncConcurrent(
        this IEnumerable enumerable,
        Func action,
        int? maxActionsToRunInParallel = null)
    {
        if (maxActionsToRunInParallel.HasValue)
        {
            using (var semaphoreSlim = new SemaphoreSlim(
                maxActionsToRunInParallel.Value, maxActionsToRunInParallel.Value))
            {
                var tasksWithThrottler = new List();

                foreach (var item in enumerable)
                {
                    // Increment the number of currently running tasks and wait if they are more than limit.
                    await semaphoreSlim.WaitAsync();

                    tasksWithThrottler.Add(Task.Run(async () =>
                    {
                        await action(item).ContinueWith(res =>
                        {
                            // action is completed, so decrement the number of currently running tasks
                            semaphoreSlim.Release();
                        });
                    }));
                }

                // Wait for all of the provided tasks to complete.
                await Task.WhenAll(tasksWithThrottler.ToArray());
            }
        }
        else
        {
            await Task.WhenAll(enumerable.Select(item => action(item)));
        }
    }

Sample Usage:

await enumerable.ForEachAsyncConcurrent(
    async item =>
    {
        await SomeAsyncMethod(item);
    },
    5);