using Docker.DotNet;
using Docker.DotNet.Models;
using Microsoft.Extensions.Options;
using RemoteExec.Server.Configuration;
using RemoteExec.Shared.Models;
using RemoteExec.Shared.Models.Docker;
using System.Collections.Concurrent;
using System.Formats.Tar;
using System.Text;
using System.Text.Json;
namespace RemoteExec.Server.Services;
///
/// Executes tasks in ephemeral Docker containers for maximum isolation.
///
public class DockerContainerExecutionEnvironment : ExecutionEnvironment
{
public override string Name => "DockerContainer";
private readonly DockerClient dockerClient;
private readonly ILogger logger;
private readonly string workerImageName;
private readonly TimeSpan containerTimeout;
private readonly long memoryLimit;
private readonly long cpuLimit;
private readonly bool networkDisabled;
private readonly bool readOnlyFilesystem;
private readonly ConcurrentDictionary assemblyCache = [];
private readonly ConcurrentDictionary runningContainers = [];
public DockerContainerExecutionEnvironment(ILogger logger, IOptions dockerConfig)
{
this.logger = logger;
DockerExecutionConfiguration config = dockerConfig.Value;
workerImageName = config.WorkerImageName;
containerTimeout = TimeSpan.FromSeconds(config.ContainerTimeoutSeconds);
memoryLimit = config.ContainerMemoryLimitMb * 1024 * 1024;
cpuLimit = config.ContainerCpuShares;
networkDisabled = config.DisableNetwork;
readOnlyFilesystem = config.ReadOnlyFilesystem;
DockerClientConfiguration dockerClientConfig;
if (string.IsNullOrEmpty(config.DockerHost))
{
dockerClientConfig = new DockerClientConfiguration();
}
else
{
dockerClientConfig = new DockerClientConfiguration(new Uri(config.DockerHost));
}
dockerClient = dockerClientConfig.CreateClient();
}
public override Task PrepareEnvironmentAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
public override async Task ExecuteTaskAsync(RemoteExecutionRequest request)
{
string containerId = string.Empty;
try
{
if (!assemblyCache.TryGetValue(request.AssemblyName, out byte[]? assemblyBytes))
{
assemblyBytes = await RequestAssemblyAsync(request.AssemblyName);
assemblyCache[request.AssemblyName] = assemblyBytes;
}
ContainerExecutionRequest containerRequest = new()
{
AssemblyBytes = Convert.ToBase64String(assemblyBytes),
TypeName = request.TypeName,
MethodName = request.MethodName,
ArgumentTypes = request.ArgumentTypes,
Arguments = request.Arguments
};
string requestJson = JsonSerializer.Serialize(containerRequest);
containerId = await CreateAndStartContainerAsync(requestJson, CancellationToken.None);
using CancellationTokenSource timeoutCts = new(containerTimeout);
Task logMonitorTask = MonitorContainerLogsAsync(containerId, timeoutCts.Token);
ContainerWaitResponse waitResponse = await dockerClient.Containers.WaitContainerAsync(containerId, timeoutCts.Token);
await timeoutCts.CancelAsync();
try
{
await logMonitorTask;
}
catch (OperationCanceledException)
{
// Expected
}
// Get container logs (stdout contains JSON result)
string stdout = await GetContainerLogsAsync(containerId);
if (waitResponse.StatusCode != 0)
{
logger.LogError("Container {ContainerId} exited with code {ExitCode}", containerId, waitResponse.StatusCode);
return new RemoteExecutionResult
{
Exception = $"Container exited with code {waitResponse.StatusCode}\nOutput: {stdout}"
};
}
string[] lines = stdout.Split('\n', StringSplitOptions.RemoveEmptyEntries);
string? resultLine = lines.LastOrDefault(l =>
{
string trimmed = l.TrimStart();
return trimmed.StartsWith('{') && !trimmed.Contains("#REQUEST_ASSEMBLY") && !trimmed.Contains("#PROVIDE_ASSEMBLY");
});
if (resultLine == null)
{
return new RemoteExecutionResult
{
Exception = $"No valid JSON result found in output: {stdout}"
};
}
ContainerExecutionResponse? response = JsonSerializer.Deserialize(resultLine);
return new RemoteExecutionResult
{
Result = response?.Result,
Exception = response?.Exception
};
}
catch (Exception ex)
{
logger.LogError(ex, "Error executing task in container {ContainerId}", containerId);
return new RemoteExecutionResult
{
Exception = ex.ToString()
};
}
finally
{
if (!string.IsNullOrEmpty(containerId))
{
await CleanupContainerAsync(containerId);
}
}
}
public override async Task CleanupEnvironmentAsync(CancellationToken cancellationToken)
{
foreach (string containerId in runningContainers.Keys)
{
await CleanupContainerAsync(containerId);
}
assemblyCache.Clear();
logger.LogInformation("Docker container execution environment cleaned up");
}
private async Task MonitorContainerLogsAsync(string containerId, CancellationToken cancellationToken)
{
try
{
MultiplexedStream logStream = await dockerClient.Containers.GetContainerLogsAsync(containerId, false, new ContainerLogsParameters
{
ShowStdout = true,
ShowStderr = false,
Follow = true
}, cancellationToken);
byte[] buffer = new byte[4096];
StringBuilder lineBuffer = new();
while (!cancellationToken.IsCancellationRequested)
{
MultiplexedStream.ReadResult result = await logStream.ReadOutputAsync(buffer, 0, buffer.Length, cancellationToken);
if (result.Count == 0)
{
break;
}
string text = Encoding.UTF8.GetString(buffer, 0, result.Count);
_ = lineBuffer.Append(text);
string bufferContent = lineBuffer.ToString();
int lastNewline = bufferContent.LastIndexOf('\n');
if (lastNewline == -1)
{
continue;
}
string completeLines = bufferContent[..lastNewline];
string remaining = bufferContent[(lastNewline + 1)..];
_ = lineBuffer.Clear();
_ = lineBuffer.Append(remaining);
string[] lines = completeLines.Split('\n', StringSplitOptions.RemoveEmptyEntries);
foreach (string line in lines)
{
string trimmedLine = line.Trim();
if (trimmedLine.StartsWith("#REQUEST_ASSEMBLY ") && trimmedLine.EndsWith('#'))
{
string assemblyName = trimmedLine.Substring("#REQUEST_ASSEMBLY ".Length, trimmedLine.Length - "#REQUEST_ASSEMBLY ".Length - 1);
_ = Task.Run(() => HandleAssemblyRequestAsync(containerId, assemblyName, cancellationToken), cancellationToken);
}
}
}
}
catch (OperationCanceledException)
{
// Expected when container completes or timeout occurs
}
catch (Exception ex)
{
logger.LogError(ex, "Error monitoring container {ContainerId} logs", containerId);
}
}
private async Task HandleAssemblyRequestAsync(string containerId, string assemblyName, CancellationToken cancellationToken)
{
try
{
if (!assemblyCache.TryGetValue(assemblyName, out byte[]? assemblyBytes))
{
assemblyBytes = await RequestAssemblyAsync(assemblyName);
assemblyCache[assemblyName] = assemblyBytes;
}
using MemoryStream tarStream = new();
using (TarWriter tarWriter = new(tarStream, TarEntryFormat.Ustar, leaveOpen: true))
{
string fileName = $"{assemblyName}.dll";
UstarTarEntry dllEntry = new(TarEntryType.RegularFile, fileName)
{
DataStream = new MemoryStream(assemblyBytes)
};
await tarWriter.WriteEntryAsync(dllEntry, cancellationToken);
UstarTarEntry sentinelEntry = new(TarEntryType.RegularFile, fileName + ".ready")
{
DataStream = new MemoryStream()
};
await tarWriter.WriteEntryAsync(sentinelEntry, cancellationToken);
}
tarStream.Position = 0;
ContainerPathStatParameters pathParams = new()
{
Path = "/tmp",
AllowOverwriteDirWithFile = false,
};
await dockerClient.Containers.ExtractArchiveToContainerAsync(containerId, pathParams, tarStream, cancellationToken);
}
catch (Exception ex)
{
logger.LogError(ex, "Error transferring assembly {AssemblyName} to container {ContainerId}", assemblyName, containerId);
}
}
private async Task CreateAndStartContainerAsync(string requestJson, CancellationToken cancellationToken)
{
CreateContainerParameters parameters = new()
{
Image = workerImageName,
Name = $"remoteexec-task-{Guid.NewGuid()}",
HostConfig = new HostConfig
{
Memory = memoryLimit,
CPUShares = cpuLimit,
NetworkMode = networkDisabled ? "none" : "bridge",
ReadonlyRootfs = readOnlyFilesystem,
AutoRemove = false,
CapDrop = ["ALL"],
SecurityOpt = ["no-new-privileges"],
Tmpfs = new Dictionary
{
["/tmp/assemblies"] = "rw,noexec,nosuid,size=100m"
}
},
Env =
[
$"EXECUTION_REQUEST={Convert.ToBase64String(Encoding.UTF8.GetBytes(requestJson))}"
],
WorkingDir = "/app",
AttachStdout = true,
AttachStderr = true
};
CreateContainerResponse container = await dockerClient.Containers.CreateContainerAsync(parameters, cancellationToken);
bool started = await dockerClient.Containers.StartContainerAsync(container.ID, new ContainerStartParameters(), cancellationToken);
if (!started)
{
throw new InvalidOperationException($"Failed to start container {container.ID}");
}
runningContainers[container.ID] = string.Empty;
return container.ID;
}
private async Task GetContainerLogsAsync(string containerId)
{
MultiplexedStream logStream = await dockerClient.Containers.GetContainerLogsAsync(containerId, false, new ContainerLogsParameters
{
ShowStdout = true,
ShowStderr = true
});
StringBuilder output = new();
byte[] buffer = new byte[4096];
MultiplexedStream.ReadResult result = await logStream.ReadOutputAsync(buffer, 0, buffer.Length, CancellationToken.None);
while (result.Count > 0)
{
logger.LogDebug("Read {ByteCount} bytes from container {ContainerId} logs", result.Count, containerId);
_ = output.Append(Encoding.UTF8.GetString(buffer, 0, result.Count));
result = await logStream.ReadOutputAsync(buffer, 0, buffer.Length, CancellationToken.None);
}
return output.ToString();
}
private async Task CleanupContainerAsync(string containerId)
{
try
{
_ = await dockerClient.Containers.StopContainerAsync(containerId, new ContainerStopParameters { WaitBeforeKillSeconds = 5 });
await dockerClient.Containers.RemoveContainerAsync(containerId, new ContainerRemoveParameters { Force = true, RemoveVolumes = true });
_ = runningContainers.TryRemove(containerId, out _);
}
catch (Exception ex)
{
logger.LogWarning(ex, "Failed to cleanup container {ContainerId}", containerId);
}
}
}