From bf1b5506c079fd4a3244414cbae7386c629c87ea Mon Sep 17 00:00:00 2001 From: Stone_Red <56473591+Stone-Red-Code@users.noreply.github.com> Date: Sat, 27 Dec 2025 00:08:01 +0100 Subject: [PATCH] Add docker execution environment --- .../RemoteExecutor.ConnectionManagement.cs | 2 +- .../RemoteExecutor.EventHandlers.cs | 2 +- .../RemoteExecutor.InternalTypes.cs | 2 +- .../RemoteExecutor.TaskDistribution.cs | 2 +- RemoteExec.Client/RemoteExecutor.cs | 2 +- RemoteExec.Client/ServerConnection.cs | 2 +- .../ServerMetricsUpdatedEventArgs.cs | 2 +- .../Configuration/ExecutionConfiguration.cs | 39 ++ RemoteExec.Server/GlobalSuppressions.cs | 2 + RemoteExec.Server/Hubs/RemoteExecutionHub.cs | 11 +- RemoteExec.Server/Program.cs | 3 + RemoteExec.Server/RemoteExec.Server.csproj | 1 + ...AssemblyLoadContextExecutionEnvironment.cs | 4 +- .../DockerContainerExecutionEnvironment.cs | 385 ++++++++++++++++++ .../Services/ExecutionEnvironment.cs | 2 +- .../appsettings.Development.json | 12 + RemoteExec.Server/appsettings.json | 47 ++- .../Docker/ContainerExecutionRequest.cs | 10 + .../Docker/ContainerExecutionResponse.cs | 8 + .../{ => Models}/RemoteExecutionRequest.cs | 2 +- .../{ => Models}/RemoteExecutionResult.cs | 2 +- .../{ => Models}/ServerMetrics.cs | 2 +- RemoteExec.Shared/{ => Models}/TaskItem.cs | 2 +- .../Utilities/AssemblyUtilities.cs | 41 ++ .../Utilities/CompletableEventArgs.cs | 57 +++ RemoteExec.Worker/Dockerfile | 32 ++ RemoteExec.Worker/GlobalSuppressions.cs | 8 + RemoteExec.Worker/Program.cs | 157 +++++++ RemoteExec.Worker/RemoteExec.Worker.csproj | 15 + RemoteExec.slnx | 1 + RemoteExec/Program.cs | 16 +- 31 files changed, 848 insertions(+), 25 deletions(-) create mode 100644 RemoteExec.Server/Services/DockerContainerExecutionEnvironment.cs create mode 100644 RemoteExec.Shared/Models/Docker/ContainerExecutionRequest.cs create mode 100644 RemoteExec.Shared/Models/Docker/ContainerExecutionResponse.cs rename RemoteExec.Shared/{ => Models}/RemoteExecutionRequest.cs (96%) rename RemoteExec.Shared/{ => Models}/RemoteExecutionResult.cs (92%) rename RemoteExec.Shared/{ => Models}/ServerMetrics.cs (97%) rename RemoteExec.Shared/{ => Models}/TaskItem.cs (91%) create mode 100644 RemoteExec.Shared/Utilities/AssemblyUtilities.cs create mode 100644 RemoteExec.Shared/Utilities/CompletableEventArgs.cs create mode 100644 RemoteExec.Worker/Dockerfile create mode 100644 RemoteExec.Worker/GlobalSuppressions.cs create mode 100644 RemoteExec.Worker/Program.cs create mode 100644 RemoteExec.Worker/RemoteExec.Worker.csproj diff --git a/RemoteExec.Client/RemoteExecutor.ConnectionManagement.cs b/RemoteExec.Client/RemoteExecutor.ConnectionManagement.cs index 9e4ac1d..a43742b 100644 --- a/RemoteExec.Client/RemoteExecutor.ConnectionManagement.cs +++ b/RemoteExec.Client/RemoteExecutor.ConnectionManagement.cs @@ -2,7 +2,7 @@ using Microsoft.AspNetCore.SignalR.Client; using Microsoft.Extensions.Logging; using RemoteExec.Client.Exceptions; -using RemoteExec.Shared; +using RemoteExec.Shared.Models; using System.Collections.Concurrent; diff --git a/RemoteExec.Client/RemoteExecutor.EventHandlers.cs b/RemoteExec.Client/RemoteExecutor.EventHandlers.cs index 6311098..e045b72 100644 --- a/RemoteExec.Client/RemoteExecutor.EventHandlers.cs +++ b/RemoteExec.Client/RemoteExecutor.EventHandlers.cs @@ -1,7 +1,7 @@ using Microsoft.AspNetCore.SignalR.Client; using Microsoft.Extensions.Logging; -using RemoteExec.Shared; +using RemoteExec.Shared.Models; using System.Collections.Concurrent; using System.Reflection; diff --git a/RemoteExec.Client/RemoteExecutor.InternalTypes.cs b/RemoteExec.Client/RemoteExecutor.InternalTypes.cs index 4daec93..35f6e9f 100644 --- a/RemoteExec.Client/RemoteExecutor.InternalTypes.cs +++ b/RemoteExec.Client/RemoteExecutor.InternalTypes.cs @@ -1,4 +1,4 @@ -using RemoteExec.Shared; +using RemoteExec.Shared.Models; namespace RemoteExec.Client; diff --git a/RemoteExec.Client/RemoteExecutor.TaskDistribution.cs b/RemoteExec.Client/RemoteExecutor.TaskDistribution.cs index 7b72906..517654b 100644 --- a/RemoteExec.Client/RemoteExecutor.TaskDistribution.cs +++ b/RemoteExec.Client/RemoteExecutor.TaskDistribution.cs @@ -1,7 +1,7 @@ using Microsoft.AspNetCore.SignalR.Client; using Microsoft.Extensions.Logging; -using RemoteExec.Shared; +using RemoteExec.Shared.Models; using System.Collections.Concurrent; diff --git a/RemoteExec.Client/RemoteExecutor.cs b/RemoteExec.Client/RemoteExecutor.cs index db4fafe..7b626ce 100644 --- a/RemoteExec.Client/RemoteExecutor.cs +++ b/RemoteExec.Client/RemoteExecutor.cs @@ -3,7 +3,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using RemoteExec.Client.Exceptions; -using RemoteExec.Shared; +using RemoteExec.Shared.Models; using System.Collections.Concurrent; using System.Reflection; diff --git a/RemoteExec.Client/ServerConnection.cs b/RemoteExec.Client/ServerConnection.cs index ffe479c..46ba6a0 100644 --- a/RemoteExec.Client/ServerConnection.cs +++ b/RemoteExec.Client/ServerConnection.cs @@ -1,6 +1,6 @@ using Microsoft.AspNetCore.SignalR.Client; -using RemoteExec.Shared; +using RemoteExec.Shared.Models; using System.Threading.Channels; diff --git a/RemoteExec.Client/ServerMetricsUpdatedEventArgs.cs b/RemoteExec.Client/ServerMetricsUpdatedEventArgs.cs index 7e859f9..f0b5c07 100644 --- a/RemoteExec.Client/ServerMetricsUpdatedEventArgs.cs +++ b/RemoteExec.Client/ServerMetricsUpdatedEventArgs.cs @@ -1,6 +1,6 @@ using Microsoft.AspNetCore.SignalR.Client; -using RemoteExec.Shared; +using RemoteExec.Shared.Models; namespace RemoteExec.Client; diff --git a/RemoteExec.Server/Configuration/ExecutionConfiguration.cs b/RemoteExec.Server/Configuration/ExecutionConfiguration.cs index f7a9e3e..bc3847e 100644 --- a/RemoteExec.Server/Configuration/ExecutionConfiguration.cs +++ b/RemoteExec.Server/Configuration/ExecutionConfiguration.cs @@ -14,6 +14,45 @@ public class ExecutionConfiguration /// /// Type of execution environment to use. Default is "AssemblyLoadContext". + /// Options: "AssemblyLoadContext", "DockerContainer" /// public string ExecutionEnvironment { get; set; } = "AssemblyLoadContext"; +} + +public class DockerExecutionConfiguration +{ + /// + /// Docker host URL. Default is unix:///var/run/docker.sock (Linux) or npipe://./pipe/docker_engine (Windows). + /// + public string DockerHost { get; set; } = string.Empty; + + /// + /// Docker worker image name. Default is remoteexec-worker:latest. + /// + public string WorkerImageName { get; set; } = "remoteexec-worker:latest"; + + /// + /// Container timeout in seconds. Default is 300 (5 minutes). + /// + public int ContainerTimeoutSeconds { get; set; } = 300; + + /// + /// Memory limit per container in MB. Default is 512 MB. + /// + public long ContainerMemoryLimitMb { get; set; } = 512; + + /// + /// CPU shares per container. Default is 1024. + /// + public long ContainerCpuShares { get; set; } = 1024; + + /// + /// Disable network access in containers. Default is true. + /// + public bool DisableNetwork { get; set; } = true; + + /// + /// Use read-only filesystem in containers. Default is true. + /// + public bool ReadOnlyFilesystem { get; set; } = true; } \ No newline at end of file diff --git a/RemoteExec.Server/GlobalSuppressions.cs b/RemoteExec.Server/GlobalSuppressions.cs index 5f3407a..e908d7c 100644 --- a/RemoteExec.Server/GlobalSuppressions.cs +++ b/RemoteExec.Server/GlobalSuppressions.cs @@ -8,3 +8,5 @@ using System.Diagnostics.CodeAnalysis; [assembly: SuppressMessage("Minor Code Smell", "S2325:Methods and properties that don't access instance data should be static", Justification = "", Scope = "type", Target = "~T:RemoteExec.Server.Hubs.RemoteExecutionHub")] [assembly: SuppressMessage("Major Code Smell", "S3011:Reflection should not be used to increase accessibility of classes, methods, or fields", Justification = "", Scope = "member", Target = "~M:RemoteExec.Server.Hubs.RemoteExecutionHub.Execute(RemoteExec.Shared.RemoteExecutionRequest)~System.Threading.Tasks.Task{RemoteExec.Shared.RemoteExecutionResult}")] [assembly: SuppressMessage("Major Code Smell", "S3010:Static fields should not be updated in constructors", Justification = "")] +[assembly: SuppressMessage("Performance", "CA1873:Avoid potentially expensive logging", Justification = "Annoying")] +[assembly: SuppressMessage("Major Code Smell", "S3011:Reflection should not be used to increase accessibility of classes, methods, or fields", Justification = "", Scope = "member", Target = "~M:RemoteExec.Server.Services.AssemblyLoadContextExecutionEnvironment.ExecuteTaskAsync(RemoteExec.Shared.Models.RemoteExecutionRequest)~System.Threading.Tasks.Task{RemoteExec.Shared.Models.RemoteExecutionResult}")] diff --git a/RemoteExec.Server/Hubs/RemoteExecutionHub.cs b/RemoteExec.Server/Hubs/RemoteExecutionHub.cs index 6555e1d..47bf9bb 100644 --- a/RemoteExec.Server/Hubs/RemoteExecutionHub.cs +++ b/RemoteExec.Server/Hubs/RemoteExecutionHub.cs @@ -3,7 +3,7 @@ using Microsoft.Extensions.Options; using RemoteExec.Server.Configuration; using RemoteExec.Server.Services; -using RemoteExec.Shared; +using RemoteExec.Shared.Models; using System.Collections.Concurrent; using System.Diagnostics; @@ -126,6 +126,10 @@ public class RemoteExecutionHub : Hub RemoteExecutionResult result = await ExecuteTask(taskItem.Request); await Clients.Caller.SendAsync("TaskResult", taskItem.TaskId, result); } + catch (ObjectDisposedException ex) + { + logger.LogWarning(ex, "Connection {ConnectionId} disposed while processing task {TaskId}", Context.ConnectionId, taskItem.TaskId); + } catch (Exception ex) { logger.LogError(ex, "Error processing task {TaskId}", taskItem.TaskId); @@ -144,7 +148,7 @@ public class RemoteExecutionHub : Hub } catch (OperationCanceledException ex) { - logger.LogError(ex, "Task stream for connection {ConnectionId} was canceled", Context.ConnectionId); + logger.LogInformation(ex, "Task stream for connection {ConnectionId} was canceled", Context.ConnectionId); } catch (Exception ex) { @@ -300,7 +304,8 @@ public class RemoteExecutionHub : Hub catch (Exception ex) { logger.LogError(ex, "Error loading assembly {Assembly}", assemblyName); - throw; } + + return []; } } \ No newline at end of file diff --git a/RemoteExec.Server/Program.cs b/RemoteExec.Server/Program.cs index a90f54f..71c9713 100644 --- a/RemoteExec.Server/Program.cs +++ b/RemoteExec.Server/Program.cs @@ -13,7 +13,10 @@ builder.Services.AddHealthChecks(); builder.Services.Configure(builder.Configuration.GetSection("Authentication")); builder.Services.Configure(builder.Configuration.GetSection("Execution")); +builder.Services.Configure(builder.Configuration.GetSection("DockerExecution")); builder.Services.Configure(builder.Configuration.GetSection("Metrics")); + +builder.Services.AddScoped(); builder.Services.AddScoped(); builder.Services.AddHostedService(); diff --git a/RemoteExec.Server/RemoteExec.Server.csproj b/RemoteExec.Server/RemoteExec.Server.csproj index d0da686..50319bd 100644 --- a/RemoteExec.Server/RemoteExec.Server.csproj +++ b/RemoteExec.Server/RemoteExec.Server.csproj @@ -9,6 +9,7 @@ + diff --git a/RemoteExec.Server/Services/AssemblyLoadContextExecutionEnvironment.cs b/RemoteExec.Server/Services/AssemblyLoadContextExecutionEnvironment.cs index 97eb921..2020885 100644 --- a/RemoteExec.Server/Services/AssemblyLoadContextExecutionEnvironment.cs +++ b/RemoteExec.Server/Services/AssemblyLoadContextExecutionEnvironment.cs @@ -1,5 +1,5 @@ -using RemoteExec.Server.Utilities; -using RemoteExec.Shared; +using RemoteExec.Shared.Models; +using RemoteExec.Shared.Utilities; using System.Reflection; using System.Text.Json; diff --git a/RemoteExec.Server/Services/DockerContainerExecutionEnvironment.cs b/RemoteExec.Server/Services/DockerContainerExecutionEnvironment.cs new file mode 100644 index 0000000..6803a60 --- /dev/null +++ b/RemoteExec.Server/Services/DockerContainerExecutionEnvironment.cs @@ -0,0 +1,385 @@ +using Docker.DotNet; +using Docker.DotNet.Models; + +using Microsoft.Extensions.Options; + +using RemoteExec.Server.Configuration; +using RemoteExec.Shared.Models; +using RemoteExec.Shared.Models.Docker; + +using System.Collections.Concurrent; +using System.Formats.Tar; +using System.Text; +using System.Text.Json; + +namespace RemoteExec.Server.Services; + +/// +/// Executes tasks in ephemeral Docker containers for maximum isolation. +/// +public class DockerContainerExecutionEnvironment : ExecutionEnvironment +{ + public override string Name => "DockerContainer"; + + private readonly DockerClient dockerClient; + private readonly ILogger logger; + private readonly string workerImageName; + private readonly TimeSpan containerTimeout; + private readonly long memoryLimit; + private readonly long cpuLimit; + private readonly bool networkDisabled; + private readonly bool readOnlyFilesystem; + + private readonly ConcurrentDictionary assemblyCache = []; + private readonly ConcurrentDictionary runningContainers = []; + + public DockerContainerExecutionEnvironment(ILogger logger, IOptions dockerConfig) + { + this.logger = logger; + + DockerExecutionConfiguration config = dockerConfig.Value; + + workerImageName = config.WorkerImageName; + containerTimeout = TimeSpan.FromSeconds(config.ContainerTimeoutSeconds); + memoryLimit = config.ContainerMemoryLimitMb * 1024 * 1024; + cpuLimit = config.ContainerCpuShares; + networkDisabled = config.DisableNetwork; + readOnlyFilesystem = config.ReadOnlyFilesystem; + + DockerClientConfiguration dockerClientConfig; + + if (string.IsNullOrEmpty(config.DockerHost)) + { + dockerClientConfig = new DockerClientConfiguration(); + } + else + { + dockerClientConfig = new DockerClientConfiguration(new Uri(config.DockerHost)); + } + + dockerClient = dockerClientConfig.CreateClient(); + } + + public override Task PrepareEnvironmentAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + + public override async Task ExecuteTaskAsync(RemoteExecutionRequest request) + { + string containerId = string.Empty; + + try + { + // Get assembly bytes from cache + if (!assemblyCache.TryGetValue(request.AssemblyName, out byte[]? assemblyBytes)) + { + assemblyBytes = await RequestAssemblyAsync(request.AssemblyName); + assemblyCache[request.AssemblyName] = assemblyBytes; + } + + // Prepare execution request + ContainerExecutionRequest containerRequest = new() + { + AssemblyBytes = Convert.ToBase64String(assemblyBytes), + TypeName = request.TypeName, + MethodName = request.MethodName, + ArgumentTypes = request.ArgumentTypes, + Arguments = request.Arguments + }; + + string requestJson = JsonSerializer.Serialize(containerRequest); + + // Create and start container + containerId = await CreateAndStartContainerAsync(requestJson, CancellationToken.None); + + // Start monitoring logs for assembly requests in background + using CancellationTokenSource timeoutCts = new(containerTimeout); + Task logMonitorTask = MonitorContainerLogsAsync(containerId, timeoutCts.Token); + + // Wait for container to complete with timeout + ContainerWaitResponse waitResponse = await dockerClient.Containers.WaitContainerAsync(containerId, timeoutCts.Token); + + // Cancel log monitoring + await timeoutCts.CancelAsync(); + + try + { + await logMonitorTask; + } + catch (OperationCanceledException) + { + // Expected + } + + // Get container logs (stdout contains JSON result) + string stdout = await GetContainerLogsAsync(containerId); + + if (waitResponse.StatusCode != 0) + { + logger.LogError("Container {ContainerId} exited with code {ExitCode}", containerId, waitResponse.StatusCode); + return new RemoteExecutionResult + { + Exception = $"Container exited with code {waitResponse.StatusCode}\nOutput: {stdout}" + }; + } + + // Parse result from stdout - get last JSON line (filter out assembly protocol lines) + string[] lines = stdout.Split('\n', StringSplitOptions.RemoveEmptyEntries); + string? resultLine = lines.LastOrDefault(l => + { + string trimmed = l.TrimStart(); + return trimmed.StartsWith('{') && !trimmed.Contains("#REQUEST_ASSEMBLY") && !trimmed.Contains("#PROVIDE_ASSEMBLY"); + }); + + if (resultLine == null) + { + return new RemoteExecutionResult + { + Exception = $"No valid JSON result found in output: {stdout}" + }; + } + + ContainerExecutionResponse? response = JsonSerializer.Deserialize(resultLine); + + return new RemoteExecutionResult + { + Result = response?.Result, + Exception = response?.Exception + }; + } + catch (Exception ex) + { + logger.LogError(ex, "Error executing task in container {ContainerId}", containerId); + return new RemoteExecutionResult + { + Exception = ex.ToString() + }; + } + finally + { + if (!string.IsNullOrEmpty(containerId)) + { + await CleanupContainerAsync(containerId); + } + } + } + + public override async Task CleanupEnvironmentAsync(CancellationToken cancellationToken) + { + foreach (string containerId in runningContainers.Keys) + { + await CleanupContainerAsync(containerId); + } + + assemblyCache.Clear(); + + logger.LogInformation("Docker container execution environment cleaned up"); + } + + private async Task MonitorContainerLogsAsync(string containerId, CancellationToken cancellationToken) + { + try + { + MultiplexedStream logStream = await dockerClient.Containers.GetContainerLogsAsync( + containerId, + false, + new ContainerLogsParameters + { + ShowStdout = true, + ShowStderr = false, + Follow = true + }, + cancellationToken); + + byte[] buffer = new byte[4096]; + StringBuilder lineBuffer = new(); + + while (!cancellationToken.IsCancellationRequested) + { + MultiplexedStream.ReadResult result = await logStream.ReadOutputAsync(buffer, 0, buffer.Length, cancellationToken); + + if (result.Count == 0) + { + break; + } + + string text = Encoding.UTF8.GetString(buffer, 0, result.Count); + _ = lineBuffer.Append(text); + + // Process complete lines + string bufferContent = lineBuffer.ToString(); + int lastNewline = bufferContent.LastIndexOf('\n'); + + if (lastNewline == -1) + { + continue; + } + + string completeLines = bufferContent[..lastNewline]; + string remaining = bufferContent[(lastNewline + 1)..]; + + _ = lineBuffer.Clear(); + _ = lineBuffer.Append(remaining); + + string[] lines = completeLines.Split('\n', StringSplitOptions.RemoveEmptyEntries); + + foreach (string line in lines) + { + string trimmedLine = line.Trim(); + + if (trimmedLine.StartsWith("#REQUEST_ASSEMBLY ") && trimmedLine.EndsWith('#')) + { + string assemblyName = trimmedLine.Substring("#REQUEST_ASSEMBLY ".Length, trimmedLine.Length - "#REQUEST_ASSEMBLY ".Length - 1); + _ = Task.Run(() => HandleAssemblyRequestAsync(containerId, assemblyName, cancellationToken), cancellationToken); + } + } + } + } + catch (OperationCanceledException) + { + // Expected when container completes or timeout occurs + } + catch (Exception ex) + { + logger.LogError(ex, "Error monitoring container {ContainerId} logs", containerId); + } + } + + private async Task HandleAssemblyRequestAsync(string containerId, string assemblyName, CancellationToken cancellationToken) + { + try + { + if (!assemblyCache.TryGetValue(assemblyName, out byte[]? assemblyBytes)) + { + assemblyBytes = await RequestAssemblyAsync(assemblyName); + assemblyCache[assemblyName] = assemblyBytes; + } + + using MemoryStream tarStream = new(); + using (TarWriter tarWriter = new(tarStream, TarEntryFormat.Ustar, leaveOpen: true)) + { + string fileName = $"{assemblyName}.dll"; + + UstarTarEntry dllEntry = new(TarEntryType.RegularFile, fileName) + { + DataStream = new MemoryStream(assemblyBytes) + }; + await tarWriter.WriteEntryAsync(dllEntry, cancellationToken); + + UstarTarEntry sentinelEntry = new(TarEntryType.RegularFile, fileName + ".ready") + { + DataStream = new MemoryStream() + }; + + await tarWriter.WriteEntryAsync(sentinelEntry, cancellationToken); + } + + tarStream.Position = 0; + + ContainerPathStatParameters pathParams = new() + { + Path = "/tmp", + AllowOverwriteDirWithFile = false, + }; + + await dockerClient.Containers.ExtractArchiveToContainerAsync(containerId, pathParams, tarStream, cancellationToken); + } + catch (Exception ex) + { + logger.LogError(ex, "Error transferring assembly {AssemblyName} to container {ContainerId}", assemblyName, containerId); + } + } + + private async Task CreateAndStartContainerAsync(string requestJson, CancellationToken cancellationToken) + { + CreateContainerParameters parameters = new() + { + Image = workerImageName, + Name = $"remoteexec-task-{Guid.NewGuid()}", + HostConfig = new HostConfig + { + Memory = memoryLimit, + CPUShares = cpuLimit, + NetworkMode = networkDisabled ? "none" : "bridge", + ReadonlyRootfs = readOnlyFilesystem, + AutoRemove = false, + CapDrop = ["ALL"], + SecurityOpt = ["no-new-privileges"], + Tmpfs = new Dictionary + { + ["/tmp/assemblies"] = "rw,noexec,nosuid,size=100m" + } + }, + Env = + [ + $"EXECUTION_REQUEST={Convert.ToBase64String(Encoding.UTF8.GetBytes(requestJson))}" + ], + WorkingDir = "/app", + AttachStdout = true, + AttachStderr = true + }; + + CreateContainerResponse container = await dockerClient.Containers.CreateContainerAsync(parameters, cancellationToken); + + bool started = await dockerClient.Containers.StartContainerAsync(container.ID, new ContainerStartParameters(), cancellationToken); + + if (!started) + { + throw new InvalidOperationException($"Failed to start container {container.ID}"); + } + + runningContainers[container.ID] = string.Empty; + + return container.ID; + } + + private async Task GetContainerLogsAsync(string containerId) + { + MultiplexedStream logStream = await dockerClient.Containers.GetContainerLogsAsync( + containerId, + false, + new ContainerLogsParameters + { + ShowStdout = true, + ShowStderr = true + }); + + StringBuilder output = new(); + byte[] buffer = new byte[4096]; + + MultiplexedStream.ReadResult result = await logStream.ReadOutputAsync(buffer, 0, buffer.Length, CancellationToken.None); + + while (result.Count > 0) + { + logger.LogDebug("Read {ByteCount} bytes from container {ContainerId} logs", result.Count, containerId); + + _ = output.Append(Encoding.UTF8.GetString(buffer, 0, result.Count)); + result = await logStream.ReadOutputAsync(buffer, 0, buffer.Length, CancellationToken.None); + } + + return output.ToString(); + } + + private async Task CleanupContainerAsync(string containerId) + { + try + { + // Stop container if still running + _ = await dockerClient.Containers.StopContainerAsync( + containerId, + new ContainerStopParameters { WaitBeforeKillSeconds = 5 }); + + // Remove container + await dockerClient.Containers.RemoveContainerAsync( + containerId, + new ContainerRemoveParameters { Force = true, RemoveVolumes = true }); + + _ = runningContainers.TryRemove(containerId, out _); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to cleanup container {ContainerId}", containerId); + } + } +} \ No newline at end of file diff --git a/RemoteExec.Server/Services/ExecutionEnvironment.cs b/RemoteExec.Server/Services/ExecutionEnvironment.cs index 41899df..7040397 100644 --- a/RemoteExec.Server/Services/ExecutionEnvironment.cs +++ b/RemoteExec.Server/Services/ExecutionEnvironment.cs @@ -1,5 +1,5 @@ using RemoteExec.Server.Utilities; -using RemoteExec.Shared; +using RemoteExec.Shared.Models; namespace RemoteExec.Server.Services; diff --git a/RemoteExec.Server/appsettings.Development.json b/RemoteExec.Server/appsettings.Development.json index a9a287b..65b7966 100644 --- a/RemoteExec.Server/appsettings.Development.json +++ b/RemoteExec.Server/appsettings.Development.json @@ -16,6 +16,18 @@ } }, "AllowedHosts": "*", + "Execution": { + "ExecutionEnvironment": "DockerContainer" + }, + "DockerExecution": { + "DockerHost": null, + "WorkerImageName": "remoteexec-worker:latest", + "ContainerTimeoutSeconds": 300, + "ContainerMemoryLimitMb": 512, + "ContainerCpuShares": 1024, + "DisableNetwork": true, + "ReadOnlyFilesystem": false + }, "Authentication": { "ApiKeys": [ { diff --git a/RemoteExec.Server/appsettings.json b/RemoteExec.Server/appsettings.json index 673daed..19d318e 100644 --- a/RemoteExec.Server/appsettings.json +++ b/RemoteExec.Server/appsettings.json @@ -31,7 +31,52 @@ // Timeout in seconds for loading assemblies from clients // If a client doesn't respond with assembly bytes within this time, the request fails // Default: 30 seconds - "AssemblyLoadTimeoutSeconds": 30 + "AssemblyLoadTimeoutSeconds": 30, + + // Type of execution environment to use + // Options: "AssemblyLoadContext", "DockerContainer" + // - AssemblyLoadContext: Lighter weight, per-connection assembly isolation + // - DockerContainer: Maximum isolation, ephemeral containers per task + // Default: "AssemblyLoadContext" + "ExecutionEnvironment": "AssemblyLoadContext" + }, + + // Docker execution configuration (only used when ExecutionEnvironment = "DockerContainer") + "DockerExecution": { + // Docker host URL + // Linux: "unix:///var/run/docker.sock" + // Windows: "npipe://./pipe/docker_engine" + "DockerHost": "unix:///var/run/docker.sock", + + // Docker worker image name to use for task execution + // Must be built and available on the Docker host + // Default: "remoteexec-worker:latest" + "WorkerImageName": "remoteexec-worker:latest", + + // Maximum execution time per container in seconds + // Containers exceeding this time will be forcefully terminated + // Default: 300 (5 minutes) + "ContainerTimeoutSeconds": 300, + + // Memory limit per container in MB + // Prevents containers from consuming excessive memory + // Default: 512 MB + "ContainerMemoryLimitMb": 512, + + // CPU shares allocated to each container (relative weight) + // Higher values = more CPU priority + // Default: 1024 + "ContainerCpuShares": 1024, + + // Disable network access in containers for security + // Set to false if tasks require network connectivity + // Default: true + "DisableNetwork": true, + + // Make container filesystem read-only + // Enhances security by preventing file modifications + // Default: true + "ReadOnlyFilesystem": true }, // Server metrics broadcasting configuration diff --git a/RemoteExec.Shared/Models/Docker/ContainerExecutionRequest.cs b/RemoteExec.Shared/Models/Docker/ContainerExecutionRequest.cs new file mode 100644 index 0000000..ae57c35 --- /dev/null +++ b/RemoteExec.Shared/Models/Docker/ContainerExecutionRequest.cs @@ -0,0 +1,10 @@ +namespace RemoteExec.Shared.Models.Docker; + +public class ContainerExecutionRequest +{ + public required string AssemblyBytes { get; set; } + public required string TypeName { get; set; } + public required string MethodName { get; set; } + public required string[] ArgumentTypes { get; set; } + public required object[] Arguments { get; set; } +} diff --git a/RemoteExec.Shared/Models/Docker/ContainerExecutionResponse.cs b/RemoteExec.Shared/Models/Docker/ContainerExecutionResponse.cs new file mode 100644 index 0000000..377778b --- /dev/null +++ b/RemoteExec.Shared/Models/Docker/ContainerExecutionResponse.cs @@ -0,0 +1,8 @@ +namespace RemoteExec.Shared.Models.Docker; + +public class ContainerExecutionResponse +{ + public bool Success { get; set; } + public object? Result { get; set; } + public string? Exception { get; set; } +} diff --git a/RemoteExec.Shared/RemoteExecutionRequest.cs b/RemoteExec.Shared/Models/RemoteExecutionRequest.cs similarity index 96% rename from RemoteExec.Shared/RemoteExecutionRequest.cs rename to RemoteExec.Shared/Models/RemoteExecutionRequest.cs index 704f1d5..ce66ba5 100644 --- a/RemoteExec.Shared/RemoteExecutionRequest.cs +++ b/RemoteExec.Shared/Models/RemoteExecutionRequest.cs @@ -1,4 +1,4 @@ -namespace RemoteExec.Shared; +namespace RemoteExec.Shared.Models; /// /// Represents a request to execute a static method on a remote server. diff --git a/RemoteExec.Shared/RemoteExecutionResult.cs b/RemoteExec.Shared/Models/RemoteExecutionResult.cs similarity index 92% rename from RemoteExec.Shared/RemoteExecutionResult.cs rename to RemoteExec.Shared/Models/RemoteExecutionResult.cs index a6f04ee..27263b1 100644 --- a/RemoteExec.Shared/RemoteExecutionResult.cs +++ b/RemoteExec.Shared/Models/RemoteExecutionResult.cs @@ -1,4 +1,4 @@ -namespace RemoteExec.Shared; +namespace RemoteExec.Shared.Models; /// /// Represents the result of a remote method execution. diff --git a/RemoteExec.Shared/ServerMetrics.cs b/RemoteExec.Shared/Models/ServerMetrics.cs similarity index 97% rename from RemoteExec.Shared/ServerMetrics.cs rename to RemoteExec.Shared/Models/ServerMetrics.cs index dec4ffa..00c3c67 100644 --- a/RemoteExec.Shared/ServerMetrics.cs +++ b/RemoteExec.Shared/Models/ServerMetrics.cs @@ -1,4 +1,4 @@ -namespace RemoteExec.Shared; +namespace RemoteExec.Shared.Models; /// /// Represents performance and status metrics for a remote execution server. diff --git a/RemoteExec.Shared/TaskItem.cs b/RemoteExec.Shared/Models/TaskItem.cs similarity index 91% rename from RemoteExec.Shared/TaskItem.cs rename to RemoteExec.Shared/Models/TaskItem.cs index 3535751..4d42d4b 100644 --- a/RemoteExec.Shared/TaskItem.cs +++ b/RemoteExec.Shared/Models/TaskItem.cs @@ -1,4 +1,4 @@ -namespace RemoteExec.Shared; +namespace RemoteExec.Shared.Models; /// /// Represents a task item in the execution stream, combining a task ID with its execution request. diff --git a/RemoteExec.Shared/Utilities/AssemblyUtilities.cs b/RemoteExec.Shared/Utilities/AssemblyUtilities.cs new file mode 100644 index 0000000..96d08bb --- /dev/null +++ b/RemoteExec.Shared/Utilities/AssemblyUtilities.cs @@ -0,0 +1,41 @@ +using System.Reflection; +using System.Runtime.Loader; + +namespace RemoteExec.Shared.Utilities; + +public static class AssemblyUtilities +{ + public static async Task PreLoadReferencedAssembliesAsync(AssemblyLoadContext assemblyLoadContext, Assembly assembly, Func> requestAssembly) + { + AssemblyName[] referencedAssemblies = assembly.GetReferencedAssemblies(); + + foreach (AssemblyName referencedAssembly in referencedAssemblies) + { + // Try to load from the assembly load context first + Assembly? loadedAssembly = assemblyLoadContext.Assemblies.FirstOrDefault(a => a.GetName().FullName == referencedAssembly.FullName); + + if (loadedAssembly != null) + { + continue; // Already loaded in the context + } + + // Try to load from default context (BCL assemblies) + try + { + _ = assemblyLoadContext.LoadFromAssemblyName(referencedAssembly); + continue; // Successfully loaded from default context + } + catch + { + byte[] assemblyBytes = await requestAssembly(referencedAssembly.FullName!); + _ = assemblyLoadContext.LoadFromBytes(assemblyBytes); + } + } + } + + public static Assembly LoadFromBytes(this AssemblyLoadContext assemblyLoadContext, byte[] assemblyBytes) + { + using MemoryStream ms = new(assemblyBytes); + return assemblyLoadContext.LoadFromStream(ms); + } +} diff --git a/RemoteExec.Shared/Utilities/CompletableEventArgs.cs b/RemoteExec.Shared/Utilities/CompletableEventArgs.cs new file mode 100644 index 0000000..d232257 --- /dev/null +++ b/RemoteExec.Shared/Utilities/CompletableEventArgs.cs @@ -0,0 +1,57 @@ +namespace RemoteExec.Shared.Utilities; + +public class CompletableEventArgs : EventArgs +{ + private readonly TaskCompletionSource tcs = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public void SetCompleted() + { + _ = tcs.TrySetResult(true); + } + + public Task WaitAsync(CancellationToken cancellationToken = default) + { + if (cancellationToken.CanBeCanceled) + { + _ = cancellationToken.Register(() => tcs.TrySetCanceled(cancellationToken)); + } + return tcs.Task; + } +} + +public class CompletableEventArgs : EventArgs +{ + private readonly TaskCompletionSource tcs = new(TaskCreationOptions.RunContinuationsAsynchronously); + public void SetCompleted(T result) + { + _ = tcs.TrySetResult(result); + } + public Task WaitAsync(CancellationToken cancellationToken = default) + { + if (cancellationToken.CanBeCanceled) + { + _ = cancellationToken.Register(() => tcs.TrySetCanceled(cancellationToken)); + } + return tcs.Task; + } +} + +public class CompletableEventArgs(TValue value) +{ + public TValue Value { get; } = value; + + private readonly TaskCompletionSource tcs = new(TaskCreationOptions.RunContinuationsAsynchronously); + public void SetCompleted(TResult result) + { + _ = tcs.TrySetResult(result); + } + public Task WaitAsync(CancellationToken cancellationToken = default) + { + if (cancellationToken.CanBeCanceled) + { + _ = cancellationToken.Register(() => tcs.TrySetCanceled(cancellationToken)); + } + return tcs.Task; + } +} + diff --git a/RemoteExec.Worker/Dockerfile b/RemoteExec.Worker/Dockerfile new file mode 100644 index 0000000..deacfe8 --- /dev/null +++ b/RemoteExec.Worker/Dockerfile @@ -0,0 +1,32 @@ +# Worker Docker image for isolated task execution +FROM mcr.microsoft.com/dotnet/runtime:10.0 AS base +WORKDIR /app + +FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build +WORKDIR /src +COPY ["RemoteExec.Worker/RemoteExec.Worker.csproj", "RemoteExec.Worker/"] +COPY ["RemoteExec.Shared/RemoteExec.Shared.csproj", "RemoteExec.Shared/"] +RUN dotnet restore "RemoteExec.Worker/RemoteExec.Worker.csproj" +COPY . . +WORKDIR "/src/RemoteExec.Worker" +RUN dotnet build "RemoteExec.Worker.csproj" -c Release -o /app/build + +FROM build AS publish +RUN dotnet publish "RemoteExec.Worker.csproj" -c Release -o /app/publish /p:UseAppHost=false + +FROM base AS final +WORKDIR /app +COPY --from=publish /app/publish . + +# Create assembly cache directory with proper permissions +RUN mkdir -p /tmp/assemblies && chmod 777 /tmp/assemblies + +# Create non-root user for security (using ID that doesn't conflict) +RUN groupadd -g 10000 worker && \ + useradd -r -u 10000 -g worker worker + +# Run as non-root user +# USER worker + +# Security: Minimal runtime image with no network access by default +ENTRYPOINT ["dotnet", "RemoteExec.Worker.dll"] diff --git a/RemoteExec.Worker/GlobalSuppressions.cs b/RemoteExec.Worker/GlobalSuppressions.cs new file mode 100644 index 0000000..678e64b --- /dev/null +++ b/RemoteExec.Worker/GlobalSuppressions.cs @@ -0,0 +1,8 @@ +// This file is used by Code Analysis to maintain SuppressMessage +// attributes that are applied to this project. +// Project-level suppressions either have no target or are given +// a specific target and scoped to a namespace, type, member, etc. + +using System.Diagnostics.CodeAnalysis; + +[assembly: SuppressMessage("Major Code Smell", "S3011:Reflection should not be used to increase accessibility of classes, methods, or fields", Justification = "", Scope = "member", Target = "~M:RemoteExec.Worker.Program.Main~System.Threading.Tasks.Task{System.Int32}")] diff --git a/RemoteExec.Worker/Program.cs b/RemoteExec.Worker/Program.cs new file mode 100644 index 0000000..794617c --- /dev/null +++ b/RemoteExec.Worker/Program.cs @@ -0,0 +1,157 @@ +using RemoteExec.Shared.Models.Docker; +using RemoteExec.Shared.Utilities; + +using System.Reflection; +using System.Runtime.Loader; +using System.Text; +using System.Text.Json; + +namespace RemoteExec.Worker; + +public static class Program +{ + private const string AssemblyCachePath = "/tmp"; + + public static async Task Main() + { + try + { + string? requestBase64 = Environment.GetEnvironmentVariable("EXECUTION_REQUEST"); + + if (string.IsNullOrEmpty(requestBase64)) + { + await Console.Error.WriteLineAsync("EXECUTION_REQUEST environment variable not set"); + return 1; + } + + byte[] requestBytes = Convert.FromBase64String(requestBase64); + string requestJson = Encoding.UTF8.GetString(requestBytes); + + ContainerExecutionRequest? request = JsonSerializer.Deserialize(requestJson); + + if (request == null) + { + await Console.Error.WriteLineAsync("Failed to deserialize execution request"); + return 1; + } + + byte[] assemblyBytes = Convert.FromBase64String(request.AssemblyBytes); + Assembly assembly = Assembly.Load(assemblyBytes); + + await AssemblyUtilities.PreLoadReferencedAssembliesAsync(AssemblyLoadContext.Default, assembly, RequestAssemblyAsync); + + Type? type = assembly.GetType(request.TypeName) ?? throw new TypeLoadException($"Type {request.TypeName} not found in assembly"); + Type[] argTypes = request.ArgumentTypes + .Select(Type.GetType) + .ToArray()!; + + MethodInfo? method = type.GetMethod( + request.MethodName, + BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic, + binder: null, + argTypes, + modifiers: null); + + if (method == null) + { + throw new MissingMethodException($"Method {request.MethodName} not found in type {request.TypeName}"); + } + + ParameterInfo[] parameters = method.GetParameters(); + object?[] invokeArgs = new object?[request.Arguments.Length]; + + for (int i = 0; i < invokeArgs.Length; i++) + { + Type targetType = parameters[i].ParameterType; + object arg = request.Arguments[i]; + + if (arg is JsonElement je) + { + invokeArgs[i] = JsonSerializer.Deserialize(je.GetRawText(), targetType); + } + else if (arg == null) + { + invokeArgs[i] = null; + } + else if (!targetType.IsInstanceOfType(arg)) + { + invokeArgs[i] = Convert.ChangeType(arg, targetType); + } + else + { + invokeArgs[i] = arg; + } + } + + object? result = method.Invoke(null, invokeArgs); + + // Handle async methods + if (result is Task taskResult) + { + await taskResult.ConfigureAwait(false); + Type returnType = method.ReturnType; + if (returnType.IsGenericType && returnType.GetGenericTypeDefinition() == typeof(Task<>)) + { + PropertyInfo resultProperty = returnType.GetProperty("Result")!; + result = resultProperty.GetValue(taskResult); + } + else + { + result = null; + } + } + + ContainerExecutionResponse response = new() + { + Success = true, + Result = result + }; + + string responseJson = JsonSerializer.Serialize(response); + await Console.Out.WriteLineAsync(responseJson); + + return 0; + } + catch (Exception ex) + { + ContainerExecutionResponse errorResponse = new() + { + Success = false, + Exception = ex.ToString() + }; + + string errorJson = JsonSerializer.Serialize(errorResponse); + await Console.Out.WriteLineAsync(errorJson); + + return 1; + } + } + + private static async Task RequestAssemblyAsync(string name) + { + string simpleName = new AssemblyName(name).Name!; + await Console.Out.WriteLineAsync($"#REQUEST_ASSEMBLY {simpleName}#"); + await Console.Out.FlushAsync(); + + string assemblyPath = Path.Combine(AssemblyCachePath, $"{simpleName}.dll"); + string sentinelPath = assemblyPath + ".ready"; + + int maxAttempts = 100; + for (int attempt = 0; attempt < maxAttempts; attempt++) + { + if (File.Exists(sentinelPath)) + { + byte[] assemblyBytes = await File.ReadAllBytesAsync(assemblyPath); + + File.Delete(sentinelPath); + + await Console.Out.WriteLineAsync($"#LOADED_ASSEMBLY {simpleName}#"); + return assemblyBytes; + } + + await Task.Delay(10); + } + + throw new FileNotFoundException($"Assembly {simpleName} timed out."); + } +} diff --git a/RemoteExec.Worker/RemoteExec.Worker.csproj b/RemoteExec.Worker/RemoteExec.Worker.csproj new file mode 100644 index 0000000..e5d4639 --- /dev/null +++ b/RemoteExec.Worker/RemoteExec.Worker.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + enable + enable + preview + + + + + + + diff --git a/RemoteExec.slnx b/RemoteExec.slnx index a2ab5cc..f003ff0 100644 --- a/RemoteExec.slnx +++ b/RemoteExec.slnx @@ -4,5 +4,6 @@ + diff --git a/RemoteExec/Program.cs b/RemoteExec/Program.cs index 4c99952..7d75618 100644 --- a/RemoteExec/Program.cs +++ b/RemoteExec/Program.cs @@ -20,13 +20,15 @@ singleHostExecutor.MetricsUpdated += (sender, e) => await singleHostExecutor.StartAsync(); +//int r = await singleHostExecutor.ExecuteAsync(Multiply, 4, 2); +//Console.WriteLine($"Multiply 4 * 2 = {r}"); +// +//return; + await Parallel.ForAsync(0, 1000, async (i, cancellationToken) => { int r = await singleHostExecutor.ExecuteAsync(Multiply, i, i + 1, cancellationToken); Console.WriteLine($"Multiply {i} * {i + 1} = {r}"); - - int s = await singleHostExecutor.ExecuteAsync(Add, i, i + 1, cancellationToken); - Console.WriteLine($"Add {i} + {i + 1} = {s}"); }); await singleHostExecutor.StopAsync(); @@ -37,7 +39,7 @@ static async Task Multiply(int x, int y) return x.Multiply(y); } -static int Add(int x, int y) -{ - return x + y; -} \ No newline at end of file +//static int Add(int x, int y) +//{ +// return x + y; +//} \ No newline at end of file