Improve configuration of RemoteExecutor

RemoteExec.Client/Exceptions/MaxRetriesException.cs

@@ -0,0 +1,5 @@
+namespace RemoteExec.Client.Exceptions;
+
+public class MaxRetriesException(string message) : Exception(message)
+{
+}

RemoteExec.Client/Exceptions/RemoteExecutionException.cs

@@ -1,4 +1,4 @@
-namespace RemoteExec.Client;
+namespace RemoteExec.Client.Exceptions;
 
 public class RemoteExecutionException(string message) : Exception(message)
 {

RemoteExec.Client/ILoadBalancingStrategy.cs

@@ -0,0 +1,6 @@
+namespace RemoteExec.Client;
+
+public interface ILoadBalancingStrategy
+{
+    ServerConnection? SelectServer(IEnumerable<ServerConnection> availableServers);
+}

RemoteExec.Client/LoadBalancingStrategies/LeastBacklogStrategy.cs

@@ -0,0 +1,9 @@
+namespace RemoteExec.Client.LoadBalancingStrategies;
+
+public class LeastBacklogStrategy : ILoadBalancingStrategy
+{
+    public ServerConnection? SelectServer(IEnumerable<ServerConnection> availableServers)
+    {
+        return availableServers.MinBy(s => s.TaskChannel.Reader.Count);
+    }
+}

RemoteExec.Client/LoadBalancingStrategies/ResourceAwareStrategy.cs

@@ -0,0 +1,21 @@
+namespace RemoteExec.Client.LoadBalancingStrategies;
+
+internal class ResourceAwareStrategy : ILoadBalancingStrategy
+{
+    public ServerConnection? SelectServer(IEnumerable<ServerConnection> availableServers)
+    {
+        return availableServers.MinBy(s =>
+        {
+            if (s.Metrics == null)
+            {
+                return double.MaxValue;
+            }
+
+            double cpuScore = s.Metrics.CpuUsage;
+            double activeTaskScore = s.Metrics.ActiveTasks * 10;
+            double backlogScore = s.TaskChannel.Reader.Count * 50;
+
+            return cpuScore + activeTaskScore + backlogScore;
+        });
+    }
+}

RemoteExec.Client/LoadBalancingStrategy.cs

@@ -1,7 +0,0 @@
-namespace RemoteExec.Client;
-
-public enum LoadBalancingStrategy
-{
-    ResourceAware,
-    LeastBacklog
-}

RemoteExec.Client/RemoteExecutor.ConnectionManagement.cs

@@ -1,6 +1,7 @@
 using Microsoft.AspNetCore.SignalR.Client;
 using Microsoft.Extensions.Logging;
 
+using RemoteExec.Client.Exceptions;
 using RemoteExec.Shared;
 
 using System.Collections.Concurrent;
@@ -92,8 +93,8 @@ public partial class RemoteExecutor
 
     private async Task AttemptReconnectionAsync(ServerConnection server, CancellationToken cancellationToken)
     {
-        int retryDelay = 5000; // Start with 5 seconds
-        int maxRetryDelay = 60000; // Max 1 minute
+        int retryDelay = (int)options.ServerReconnectInitialDelay.TotalMilliseconds;
+        int maxRetryDelay = (int)options.ServerReconnectMaxDelay.TotalMilliseconds;
 
         while (!cancellationToken.IsCancellationRequested && server.Connection.State == HubConnectionState.Disconnected)
         {
@@ -151,8 +152,17 @@ public partial class RemoteExecutor
         {
             if (taskDict.TryRemove(id, out PendingTask? task) && pendingResults.ContainsKey(id))
             {
-                globalQueue.Add(task);
-                count++;
+                task.RetryCount++;
+
+                if (task.RetryCount <= options.MaxTaskRetries)
+                {
+                    globalQueue.Add(task);
+                    count++;
+                }
+                else if (pendingResults.TryRemove(id, out TaskCompletionSource<RemoteExecutionResult>? tcs))
+                {
+                    tcs.SetException(new MaxRetriesException("Max retry attempts reached."));
+                }
             }
         }
 

RemoteExec.Client/RemoteExecutor.InternalTypes.cs

@@ -1,26 +1,15 @@
-using Microsoft.AspNetCore.SignalR.Client;
-
 using RemoteExec.Shared;
 
-using System.Threading.Channels;
-
 namespace RemoteExec.Client;
 
 public partial class RemoteExecutor
 {
-    private sealed class ServerConnection(HubConnection connection, HttpClient httpClient)
-    {
-        public HubConnection Connection { get; } = connection;
-        public HttpClient HttpClient { get; } = httpClient;
-        public Channel<TaskItem> TaskChannel { get; } = Channel.CreateUnbounded<TaskItem>();
-        public ServerMetrics? Metrics { get; set; }
-        public Func<Exception?, Task>? ClosedEventHandler { get; set; }
-    }
 
     private sealed class PendingTask
     {
         public required Guid TaskId { get; init; }
         public required RemoteExecutionRequest Request { get; init; }
         public required DateTime EnqueuedAt { get; init; }
+        public int RetryCount { get; set; } = 0;
     }
 }

RemoteExec.Client/RemoteExecutor.TaskDistribution.cs

@@ -79,24 +79,6 @@ public partial class RemoteExecutor
             return null;
         }
 
-        return loadBalancingStrategy switch
-        {
-            LoadBalancingStrategy.ResourceAware => connectedServers.MinBy(s =>
-            {
-                if (s.Metrics == null)
-                {
-                    return double.MaxValue;
-                }
-
-                double cpuScore = s.Metrics.CpuUsage;
-                double activeTaskScore = s.Metrics.ActiveTasks * 10;
-                double backlogScore = s.TaskChannel.Reader.Count * 50;
-
-                return cpuScore + activeTaskScore + backlogScore;
-            }),
-            LoadBalancingStrategy.LeastBacklog => connectedServers.MinBy(s => s.TaskChannel.Reader.Count),
-
-            _ => connectedServers.FirstOrDefault()
-        };
+        return options.Strategy.SelectServer(connectedServers);
     }
 }

RemoteExec.Client/RemoteExecutor.cs

@@ -2,6 +2,7 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
 
+using RemoteExec.Client.Exceptions;
 using RemoteExec.Shared;
 
 using System.Collections.Concurrent;
@@ -12,8 +13,8 @@ namespace RemoteExec.Client;
 
 public partial class RemoteExecutor : IAsyncDisposable
 {
+    private readonly BlockingCollection<PendingTask> globalQueue;
     private readonly List<ServerConnection> servers = [];
-    private readonly BlockingCollection<PendingTask> globalQueue = [];
     private readonly ConcurrentDictionary<Guid, TaskCompletionSource<RemoteExecutionResult>> pendingResults = new();
     private readonly ConcurrentDictionary<ServerConnection, ConcurrentDictionary<Guid, PendingTask>> serverAssignedTasks = new();
 
@@ -21,38 +22,65 @@ public partial class RemoteExecutor : IAsyncDisposable
     private CancellationTokenSource distributorCts = new();
     private Task? distributorTask;
 
-    private readonly LoadBalancingStrategy loadBalancingStrategy;
+    private readonly RemoteExecutorOptions options = new();
     private readonly ILogger logger;
 
     private bool disposedValue;
 
     public event EventHandler<ServerMetricsUpdatedEventArgs>? MetricsUpdated;
 
-    public RemoteExecutor(string url) : this([url], LoadBalancingStrategy.ResourceAware, NullLogger.Instance)
+    public RemoteExecutor(string url) : this([url], new RemoteExecutorOptions(), NullLogger.Instance)
     {
     }
 
-    public RemoteExecutor(string url, ILogger logger) : this([url], LoadBalancingStrategy.ResourceAware, logger)
+    public RemoteExecutor(string url, ILogger logger) : this([url], new RemoteExecutorOptions(), logger)
     {
     }
 
-    public RemoteExecutor(string url, LoadBalancingStrategy loadBalancingStrategy) : this([url], loadBalancingStrategy, NullLogger.Instance)
+    public RemoteExecutor(string url, Action<RemoteExecutorOptions> configure) : this([url], NullLogger.Instance, configure)
     {
     }
 
-    public RemoteExecutor(string[] urls) : this(urls, LoadBalancingStrategy.ResourceAware, NullLogger.Instance)
+    public RemoteExecutor(string url, ILogger logger, Action<RemoteExecutorOptions> configure) : this([url], logger, configure)
     {
     }
 
-    public RemoteExecutor(string[] urls, LoadBalancingStrategy loadBalancingStrategy) : this(urls, loadBalancingStrategy, NullLogger.Instance)
+    public RemoteExecutor(string[] urls) : this(urls, new RemoteExecutorOptions(), NullLogger.Instance)
     {
     }
 
-    public RemoteExecutor(string[] urls, LoadBalancingStrategy loadBalancingStrategy, ILogger logger)
+    public RemoteExecutor(string[] urls, ILogger logger) : this(urls, new RemoteExecutorOptions(), logger)
+    {
+    }
+
+    public RemoteExecutor(string[] urls, Action<RemoteExecutorOptions> configure) : this(urls, NullLogger.Instance, configure)
+    {
+    }
+
+    public RemoteExecutor(string[] urls, ILogger logger, Action<RemoteExecutorOptions> configure)
     {
-        this.loadBalancingStrategy = loadBalancingStrategy;
         this.logger = logger;
 
+        options = new RemoteExecutorOptions();
+        globalQueue = new BlockingCollection<PendingTask>(options.GlobalQueueCapacity);
+
+        configure(options);
+
+        InitializeServers(urls);
+    }
+
+    public RemoteExecutor(string[] urls, RemoteExecutorOptions options, ILogger logger)
+    {
+        this.logger = logger;
+        this.options = options;
+
+        globalQueue = new BlockingCollection<PendingTask>(this.options.GlobalQueueCapacity);
+
+        InitializeServers(urls);
+    }
+
+    private void InitializeServers(string[] urls)
+    {
         foreach (string url in urls)
         {
             Uri baseUri = new(url);
@@ -85,9 +113,9 @@ public partial class RemoteExecutor : IAsyncDisposable
             .ToDictionary(metrics => metrics!.ServerId, metrics => metrics!);
     }
 
-    public async Task<TResult> Execute<TDelegate, TResult>(TDelegate del, params object[] args) where TDelegate : Delegate
+    public async Task<TResult> ExecuteAsync<TDelegate, TResult>(TDelegate @delegate, CancellationToken cancellationToken, params object[] args) where TDelegate : Delegate
     {
-        object? execResult = await Execute(del, args);
+        object? execResult = await ExecuteAsync(@delegate, cancellationToken, args);
 
         if (execResult is TResult typedResult)
         {
@@ -103,12 +131,21 @@ public partial class RemoteExecutor : IAsyncDisposable
         }
     }
 
-    public async Task<object?> Execute<T>(T del, params object[] args) where T : Delegate
+    public async Task<TResult> ExecuteAsync<TDelegate, TResult>(TDelegate @delegate, params object[] args) where TDelegate : Delegate
     {
-        MethodInfo method = del.Method;
+        return await ExecuteAsync<TDelegate, TResult>(@delegate, CancellationToken.None, args);
+    }
+
+    public async Task<object?> ExecuteAsync<T>(T @delegate, CancellationToken cancellationToken, params object[] args) where T : Delegate
+    {
+        MethodInfo method = @delegate.Method;
         Type declaringType = method.DeclaringType!;
         Assembly assembly = declaringType.Assembly;
 
+        using CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, distributorCts.Token);
+        using CancellationTokenSource timeoutCts = new CancellationTokenSource(options.ExecutionTimeout);
+        using CancellationTokenSource finalCts = CancellationTokenSource.CreateLinkedTokenSource(cts.Token, timeoutCts.Token);
+
         if (!method.IsStatic)
         {
             throw new InvalidOperationException("Only static methods supported");
@@ -141,14 +178,26 @@ public partial class RemoteExecutor : IAsyncDisposable
 
         globalQueue.Add(pendingTask);
 
-        RemoteExecutionResult result = await tcs.Task;
-
-        if (result.Exception != null)
+        try
         {
-            throw new RemoteExecutionException(result.Exception);
-        }
+            RemoteExecutionResult result = await tcs.Task.WaitAsync(finalCts.Token);
 
-        return result.Result;
+            if (result.Exception != null)
+            {
+                throw new RemoteExecutionException(result.Exception);
+            }
+
+            return result.Result;
+        }
+        finally
+        {
+            _ = pendingResults.TryRemove(taskId, out _);
+        }
+    }
+
+    public Task<object?> ExecuteAsync<T>(T @delegate, params object[] args) where T : Delegate
+    {
+        return ExecuteAsync(@delegate, CancellationToken.None, args);
     }
 
     protected virtual async Task DisposeAsync(bool disposing)

RemoteExec.Client/RemoteExecutorOptions.cs

@@ -0,0 +1,21 @@
+using RemoteExec.Client.LoadBalancingStrategies;
+
+namespace RemoteExec.Client;
+
+public class RemoteExecutorOptions
+{
+    public ILoadBalancingStrategy Strategy { get; set; } = new ResourceAwareStrategy();
+
+    // How long to wait for a result before throwing a TimeoutException
+    public TimeSpan ExecutionTimeout { get; set; } = TimeSpan.FromMinutes(5);
+
+    // Backoff settings for server reconnections
+    public TimeSpan ServerReconnectInitialDelay { get; set; } = TimeSpan.FromSeconds(5);
+    public TimeSpan ServerReconnectMaxDelay { get; set; } = TimeSpan.FromSeconds(60);
+
+    // Reliability: How many times a task can be requeued before failing
+    public int MaxTaskRetries { get; set; } = 3;
+
+    // Capacity: Stop accepting Execute calls if the global queue is too full
+    public int GlobalQueueCapacity { get; set; } = 1000;
+}

RemoteExec.Client/ServerConnection.cs

@@ -0,0 +1,16 @@
+using Microsoft.AspNetCore.SignalR.Client;
+
+using RemoteExec.Shared;
+
+using System.Threading.Channels;
+
+namespace RemoteExec.Client;
+
+public class ServerConnection(HubConnection connection, HttpClient httpClient)
+{
+    public HubConnection Connection { get; } = connection;
+    public HttpClient HttpClient { get; } = httpClient;
+    public Channel<TaskItem> TaskChannel { get; } = Channel.CreateUnbounded<TaskItem>();
+    public ServerMetrics? Metrics { get; set; }
+    public Func<Exception?, Task>? ClosedEventHandler { get; set; }
+}