Class CudaMemoryManager

Namespace

DotCompute.Backends.CUDA.Memory

Assembly

DotCompute.Backends.CUDA.dll

High-performance CUDA device memory manager with automatic pooling and unified memory support.

public sealed class CudaMemoryManager : BaseMemoryManager, IUnifiedMemoryManager, IAsyncDisposable, IDisposable

Inheritance

object

BaseMemoryManager

CudaMemoryManager

Implements

IUnifiedMemoryManager

IAsyncDisposable

IDisposable

Inherited Members

BaseMemoryManager.CurrentStatistics

BaseMemoryManager.TotalAllocatedBytes

BaseMemoryManager.PeakAllocatedBytes

BaseMemoryManager.AllocationCount

BaseMemoryManager.IsDisposed

BaseMemoryManager.AllocateAsync<T>(int, MemoryOptions, CancellationToken)

BaseMemoryManager.AllocateAndCopyAsync<T>(ReadOnlyMemory<T>, MemoryOptions, CancellationToken)

BaseMemoryManager.AllocateRawAsync(long, MemoryOptions, CancellationToken)

BaseMemoryManager.AllocateAsync(long, MemoryOptions, CancellationToken)

BaseMemoryManager.CreateView(IUnifiedMemoryBuffer, long, long)

BaseMemoryManager.AllocateAsync<T>(int)

BaseMemoryManager.CopyToDevice<T>(IUnifiedMemoryBuffer, ReadOnlySpan<T>)

BaseMemoryManager.CopyFromDevice<T>(Span<T>, IUnifiedMemoryBuffer)

BaseMemoryManager.Free(IUnifiedMemoryBuffer)

BaseMemoryManager.AllocateDevice(long)

BaseMemoryManager.FreeDevice(DeviceMemory)

BaseMemoryManager.MemsetDevice(DeviceMemory, byte, long)

BaseMemoryManager.MemsetDeviceAsync(DeviceMemory, byte, long, CancellationToken)

BaseMemoryManager.CopyHostToDevice(nint, DeviceMemory, long)

BaseMemoryManager.CopyDeviceToHost(DeviceMemory, nint, long)

BaseMemoryManager.CopyHostToDeviceAsync(nint, DeviceMemory, long, CancellationToken)

BaseMemoryManager.CopyDeviceToHostAsync(DeviceMemory, nint, long, CancellationToken)

BaseMemoryManager.CopyDeviceToDevice(DeviceMemory, DeviceMemory, long)

BaseMemoryManager.GetRefreshedStatistics()

object.Equals(object)

object.Equals(object, object)

object.GetHashCode()

object.GetType()

object.ReferenceEquals(object, object)

object.ToString()

Extension Methods

BufferExtensions.IsSafeToDispose(IDisposable)

BufferExtensions.SafeDispose(IDisposable)

Examples

var manager = new CudaMemoryManager(context, device, logger);

// Allocate device memory for GPU-only operations
var deviceBuffer = await manager.AllocateAsync<float>(1024);

// Allocate unified memory for CPU/GPU sharing
var unifiedBuffer = await manager.AllocateAsync<float>(1024, MemoryOptions.Unified);

// Copy data to device
await manager.CopyToDeviceAsync(hostData, deviceBuffer);

Remarks

The memory manager provides efficient GPU memory allocation with:

• Zero-copy unified memory (cudaMallocManaged) for CPU/GPU shared access
• Pinned host memory (cudaHostAlloc) for faster PCIe transfers
• Standard device memory (cudaMalloc) for GPU-only allocations
• Automatic memory pooling to reduce allocation overhead by ~90%
• Thread-safe allocation tracking with detailed statistics

Memory Types:

Device Memory	GPU-only, highest performance (MemoryOptions.None)
Unified Memory	CPU/GPU accessible, automatic migration (MemoryOptions.Unified)
Pinned Memory	Host memory locked for faster DMA (MemoryOptions.Pinned)

Constructors

CudaMemoryManager(CudaContext, CudaDevice?, ILogger)

Initializes a new instance of the CudaMemoryManager class.

public CudaMemoryManager(CudaContext context, CudaDevice? device, ILogger logger)

Parameters

context CudaContext

The context.

device CudaDevice

The device.

logger ILogger

The logger.

CudaMemoryManager(CudaContext, ILogger)

Initializes a new instance of the CudaMemoryManager class.

public CudaMemoryManager(CudaContext context, ILogger logger)

Parameters

context CudaContext

The context.

logger ILogger

The logger.

Properties

Accelerator

Gets the accelerator this memory manager is associated with.

public override IAccelerator Accelerator { get; }

Property Value

IAccelerator

CurrentAllocatedMemory

Gets the current allocated memory in bytes.

public override long CurrentAllocatedMemory { get; }

Property Value

long

MaxAllocationSize

Gets the maximum allocation size supported.

public override long MaxAllocationSize { get; }

Property Value

long

Statistics

Gets memory usage statistics.

public override MemoryStatistics Statistics { get; }

Property Value

MemoryStatistics

TotalAllocated

Gets the total amount of memory currently allocated.

public long TotalAllocated { get; }

Property Value

long

TotalAvailableMemory

Gets the total available memory on the device for interface compatibility.

public override long TotalAvailableMemory { get; }

Property Value

long

TotalMemory

Gets the total available memory on the device.

public long TotalMemory { get; }

Property Value

long

UsedMemory

Gets the currently used memory on the device.

public long UsedMemory { get; }

Property Value

long

Methods

AllocateAsync(long, CancellationToken)

Allocates device memory asynchronously.

public Task<IUnifiedMemoryBuffer> AllocateAsync(long sizeInBytes, CancellationToken cancellationToken = default)

Parameters

sizeInBytes long

cancellationToken CancellationToken

Returns

Task<IUnifiedMemoryBuffer>

AllocateAsync<T>(long, MemoryOptions, CancellationToken)

Allocates memory asynchronously with specific type and options.

public ValueTask<IUnifiedMemoryBuffer<T>> AllocateAsync<T>(long count, MemoryOptions options, CancellationToken cancellationToken = default) where T : unmanaged

Parameters

count long

options MemoryOptions

cancellationToken CancellationToken

Returns

ValueTask<IUnifiedMemoryBuffer<T>>

Type Parameters

T

AllocateAsync<T>(long, CancellationToken)

Allocates device memory asynchronously with specific type.

public ValueTask<IUnifiedMemoryBuffer<T>> AllocateAsync<T>(long count, CancellationToken cancellationToken = default) where T : unmanaged

Parameters

count long

cancellationToken CancellationToken

Returns

ValueTask<IUnifiedMemoryBuffer<T>>

Type Parameters

T

AllocateInternalAsync(long, MemoryOptions, CancellationToken)

Allocates memory using backend-specific implementation.

protected override ValueTask<IUnifiedMemoryBuffer> AllocateInternalAsync(long sizeInBytes, MemoryOptions options, CancellationToken cancellationToken)

Parameters

sizeInBytes long

options MemoryOptions

cancellationToken CancellationToken

Returns

ValueTask<IUnifiedMemoryBuffer>

Allocate<T>(long)

Allocates device memory synchronously.

public IUnifiedMemoryBuffer<T> Allocate<T>(long count) where T : unmanaged

Parameters

count long

Returns

IUnifiedMemoryBuffer<T>

Type Parameters

T

Clear()

Clears all allocated memory and resets the manager.

public override void Clear()

CopyAsync<T>(IUnifiedMemoryBuffer<T>, IUnifiedMemoryBuffer<T>, CancellationToken)

Copies data between buffers.

public override ValueTask CopyAsync<T>(IUnifiedMemoryBuffer<T> source, IUnifiedMemoryBuffer<T> destination, CancellationToken cancellationToken = default) where T : unmanaged

Parameters

source IUnifiedMemoryBuffer<T>

The source buffer.

destination IUnifiedMemoryBuffer<T>

The destination buffer.

cancellationToken CancellationToken

Cancellation token.

Returns

ValueTask

A task representing the copy operation.

Type Parameters

T

The element type.

CopyAsync<T>(IUnifiedMemoryBuffer<T>, int, IUnifiedMemoryBuffer<T>, int, int, CancellationToken)

Copies data between buffers with specified ranges.

public override ValueTask CopyAsync<T>(IUnifiedMemoryBuffer<T> source, int sourceOffset, IUnifiedMemoryBuffer<T> destination, int destinationOffset, int count, CancellationToken cancellationToken = default) where T : unmanaged

Parameters

source IUnifiedMemoryBuffer<T>

The source buffer.

sourceOffset int

The offset in the source buffer.

destination IUnifiedMemoryBuffer<T>

The destination buffer.

destinationOffset int

The offset in the destination buffer.

count int

The number of elements to copy.

cancellationToken CancellationToken

Cancellation token.

Returns

ValueTask

A task representing the copy operation.

Type Parameters

T

The element type.

CopyFromDeviceAsync<T>(IUnifiedMemoryBuffer<T>, Memory<T>, CancellationToken)

Copies data from a device buffer to host memory.

public override ValueTask CopyFromDeviceAsync<T>(IUnifiedMemoryBuffer<T> source, Memory<T> destination, CancellationToken cancellationToken = default) where T : unmanaged

Parameters

source IUnifiedMemoryBuffer<T>

The source buffer.

destination Memory<T>

The destination memory.

cancellationToken CancellationToken

Cancellation token.

Returns

ValueTask

A task representing the copy operation.

Type Parameters

T

The element type.

CopyToDeviceAsync<T>(ReadOnlyMemory<T>, IUnifiedMemoryBuffer<T>, CancellationToken)

Copies data from host memory to a device buffer.

public override ValueTask CopyToDeviceAsync<T>(ReadOnlyMemory<T> source, IUnifiedMemoryBuffer<T> destination, CancellationToken cancellationToken = default) where T : unmanaged

Parameters

source ReadOnlyMemory<T>

The source data.

destination IUnifiedMemoryBuffer<T>

The destination buffer.

cancellationToken CancellationToken

Cancellation token.

Returns

ValueTask

A task representing the copy operation.

Type Parameters

T

The element type.

CreateViewCore(IUnifiedMemoryBuffer, long, long)

Backend-specific view creation implementation.

protected override IUnifiedMemoryBuffer CreateViewCore(IUnifiedMemoryBuffer buffer, long offset, long length)

Parameters

buffer IUnifiedMemoryBuffer

offset long

length long

Returns

IUnifiedMemoryBuffer

CreateView<T>(IUnifiedMemoryBuffer<T>, int, int)

Creates a view over existing memory.

public override IUnifiedMemoryBuffer<T> CreateView<T>(IUnifiedMemoryBuffer<T> buffer, int offset, int length) where T : unmanaged

Parameters

buffer IUnifiedMemoryBuffer<T>

The source buffer.

offset int

The offset in elements.

length int

The length of the view in elements.

Returns

IUnifiedMemoryBuffer<T>

A view over the existing buffer.

Type Parameters

T

The element type.

Dispose()

Performs dispose.

public void Dispose()

DisposeAsync()

Asynchronously disposes the memory manager.

public override ValueTask DisposeAsync()

Returns

ValueTask

A ValueTask representing the asynchronous dispose operation.

Free(nint)

Frees device memory.

public void Free(nint devicePtr)

Parameters

devicePtr nint

FreeAsync(IUnifiedMemoryBuffer, CancellationToken)

Asynchronously frees a memory buffer.

public override ValueTask FreeAsync(IUnifiedMemoryBuffer buffer, CancellationToken cancellationToken = default)

Parameters

buffer IUnifiedMemoryBuffer

The buffer to free.

cancellationToken CancellationToken

Cancellation token.

Returns

ValueTask

A task representing the free operation.

FreeAsync(nint, CancellationToken)

Frees device memory asynchronously.

public Task FreeAsync(nint devicePtr, CancellationToken cancellationToken = default)

Parameters

devicePtr nint

cancellationToken CancellationToken

Returns

Task

GetStatisticsAsync(CancellationToken)

Gets accurate CUDA memory statistics asynchronously.

public ValueTask<MemoryStatistics> GetStatisticsAsync(CancellationToken cancellationToken = default)

Parameters

cancellationToken CancellationToken

Cancellation token.

Returns

ValueTask<MemoryStatistics>

A task containing accurate memory statistics from the device.

Remarks

This method queries the CUDA device using cudaMemGetInfo() which may take 1-10ms but provides the most accurate view of current device memory state. For fast synchronous access, use the Statistics property.

OptimizeAsync(CancellationToken)

Optimizes memory by defragmenting and releasing unused memory.

public override ValueTask OptimizeAsync(CancellationToken cancellationToken = default)

Parameters

cancellationToken CancellationToken

Cancellation token.

Returns

ValueTask

A task representing the optimization operation.

Reset()

Resets the memory manager by clearing all tracking and reinitializing.

public void Reset()