UCX  1.14
Unified Communication X

Data Structures

struct  uct_md_attr
 Memory domain attributes. More...
 
struct  uct_md_attr.cap
 
struct  uct_md_mem_attr
 Memory domain attributes. More...
 
struct  uct_allocated_memory
 Describes a memory allocated by UCT. More...
 
struct  uct_rkey_bundle
 Remote key with its type. More...
 
struct  uct_mem_alloc_params_t
 Parameters for allocating memory using uct_mem_alloc. More...
 
struct  uct_mem_alloc_params_t.mds
 

Typedefs

typedef enum uct_md_mem_attr_field uct_md_mem_attr_field_t
 UCT MD memory attributes field mask. More...
 
typedef struct uct_md_mem_attr uct_md_mem_attr_t
 Memory domain attributes. More...
 
typedef struct uct_allocated_memory uct_allocated_memory_t
 Describes a memory allocated by UCT. More...
 
typedef struct uct_rkey_bundle uct_rkey_bundle_t
 Remote key with its type. More...
 

Enumerations

enum  uct_sockaddr_accessibility_t { UCT_SOCKADDR_ACC_LOCAL, UCT_SOCKADDR_ACC_REMOTE }
 Socket address accessibility type. More...
 
enum  {
  UCT_MD_FLAG_ALLOC = UCS_BIT(0), UCT_MD_FLAG_REG = UCS_BIT(1), UCT_MD_FLAG_NEED_MEMH = UCS_BIT(2), UCT_MD_FLAG_NEED_RKEY = UCS_BIT(3),
  UCT_MD_FLAG_ADVISE = UCS_BIT(4), UCT_MD_FLAG_FIXED = UCS_BIT(5), UCT_MD_FLAG_RKEY_PTR = UCS_BIT(6), UCT_MD_FLAG_SOCKADDR = UCS_BIT(7),
  UCT_MD_FLAG_INVALIDATE = UCS_BIT(8), UCT_MD_FLAG_EXPORTED_MKEY = UCS_BIT(9), UCT_MD_FLAG_REG_DMABUF = UCS_BIT(10)
}
 Memory domain capability flags. More...
 
enum  uct_md_mem_flags {
  UCT_MD_MEM_FLAG_NONBLOCK = UCS_BIT(0), UCT_MD_MEM_FLAG_FIXED = UCS_BIT(1), UCT_MD_MEM_FLAG_LOCK = UCS_BIT(2), UCT_MD_MEM_FLAG_HIDE_ERRORS = UCS_BIT(3),
  UCT_MD_MEM_ACCESS_REMOTE_PUT = UCS_BIT(5), UCT_MD_MEM_ACCESS_REMOTE_GET = UCS_BIT(6), UCT_MD_MEM_ACCESS_REMOTE_ATOMIC = UCS_BIT(7), UCT_MD_MEM_ACCESS_LOCAL_READ = UCS_BIT(8),
  UCT_MD_MEM_ACCESS_LOCAL_WRITE = UCS_BIT(9), UCT_MD_MEM_ACCESS_ALL, UCT_MD_MEM_ACCESS_RMA
}
 Memory allocation/registration flags. More...
 
enum  uct_mem_advice_t { UCT_MADV_NORMAL = 0, UCT_MADV_WILLNEED }
 list of UCT memory use advice More...
 
enum  uct_md_mem_attr_field {
  UCT_MD_MEM_ATTR_FIELD_MEM_TYPE = UCS_BIT(0), UCT_MD_MEM_ATTR_FIELD_SYS_DEV = UCS_BIT(1), UCT_MD_MEM_ATTR_FIELD_BASE_ADDRESS = UCS_BIT(2), UCT_MD_MEM_ATTR_FIELD_ALLOC_LENGTH = UCS_BIT(3),
  UCT_MD_MEM_ATTR_FIELD_DMABUF_FD = UCS_BIT(4), UCT_MD_MEM_ATTR_FIELD_DMABUF_OFFSET = UCS_BIT(5)
}
 UCT MD memory attributes field mask. More...
 
enum  uct_mem_alloc_params_field_t {
  UCT_MEM_ALLOC_PARAM_FIELD_FLAGS = UCS_BIT(0), UCT_MEM_ALLOC_PARAM_FIELD_ADDRESS = UCS_BIT(1), UCT_MEM_ALLOC_PARAM_FIELD_MEM_TYPE = UCS_BIT(2), UCT_MEM_ALLOC_PARAM_FIELD_MDS = UCS_BIT(3),
  UCT_MEM_ALLOC_PARAM_FIELD_NAME = UCS_BIT(4)
}
 UCT allocation parameters specification field mask. More...
 

Functions

ucs_status_t uct_md_mem_query (uct_md_h md, const void *address, size_t length, uct_md_mem_attr_t *mem_attr)
 Query attributes of a given pointer. More...
 
ucs_status_t uct_md_query (uct_md_h md, uct_md_attr_t *md_attr)
 Query for memory domain attributes. More...
 
ucs_status_t uct_md_mem_advise (uct_md_h md, uct_mem_h memh, void *addr, size_t length, uct_mem_advice_t advice)
 Give advice about the use of memory. More...
 
ucs_status_t uct_md_mem_reg (uct_md_h md, void *address, size_t length, unsigned flags, uct_mem_h *memh_p)
 Register memory for zero-copy sends and remote access. More...
 
ucs_status_t uct_md_mem_dereg (uct_md_h md, uct_mem_h memh)
 Undo the operation of uct_md_mem_reg(). More...
 
ucs_status_t uct_md_detect_memory_type (uct_md_h md, const void *addr, size_t length, ucs_memory_type_t *mem_type_p)
 Detect memory type. More...
 
ucs_status_t uct_mem_alloc (size_t length, const uct_alloc_method_t *methods, unsigned num_methods, const uct_mem_alloc_params_t *params, uct_allocated_memory_t *mem)
 Allocate memory for zero-copy communications and remote access. More...
 
ucs_status_t uct_mem_free (const uct_allocated_memory_t *mem)
 Release allocated memory. More...
 
ucs_status_t uct_md_config_read (uct_component_h component, const char *env_prefix, const char *filename, uct_md_config_t **config_p)
 Read the configuration for a memory domain. More...
 
int uct_md_is_sockaddr_accessible (uct_md_h md, const ucs_sock_addr_t *sockaddr, uct_sockaddr_accessibility_t mode)
 Check if remote sock address is accessible from the memory domain. More...
 
ucs_status_t uct_md_mkey_pack (uct_md_h md, uct_mem_h memh, void *rkey_buffer)
 Pack a remote key. More...
 
ucs_status_t uct_rkey_unpack (uct_component_h component, const void *rkey_buffer, uct_rkey_bundle_t *rkey_ob)
 Unpack a remote key. More...
 
ucs_status_t uct_rkey_ptr (uct_component_h component, uct_rkey_bundle_t *rkey_ob, uint64_t remote_addr, void **addr_p)
 Get a local pointer to remote memory. More...
 
ucs_status_t uct_rkey_release (uct_component_h component, const uct_rkey_bundle_t *rkey_ob)
 Release a remote key. More...
 

Detailed Description

The Memory Domain abstracts resources required for network communication, which typically includes memory, transport mechanisms, compute and network resources. It is an isolation mechanism that can be employed by the applications for isolating resources between multiple programming models. The attributes of the Memory Domain are defined by the structure uct_md_attr(). The communication and memory operations are defined in the context of Memory Domain.


Data Structure Documentation

struct uct_md_attr

This structure defines the attributes of a Memory Domain which includes maximum memory that can be allocated, credentials required for accessing the memory, CPU mask indicating the proximity of CPUs, and bitmaps indicating the types of memory (CPU/CUDA/ROCM) that can be detected, allocated and accessed.

Examples:
uct_hello_world.c.
Data Fields
struct uct_md_attr cap
ucs_linear_func_t reg_cost

Memory registration cost estimation (time,seconds) as a linear function of the buffer size.

char component_name[UCT_COMPONENT_NAME_MAX]

Component name

size_t rkey_packed_size

Size of buffer needed for packed rkey

ucs_cpu_set_t local_cpus

Mask of CPUs near the resource

struct uct_md_attr.cap
Data Fields
uint64_t max_alloc

Maximal allocation size

size_t max_reg

Maximal registration size

uint64_t flags

UCT_MD_FLAG_xx

uint64_t reg_mem_types

Bitmap of memory types that Memory Domain can be registered with

uint64_t detect_mem_types

Bitmap of memory types that Memory Domain can detect if address belongs to it

uint64_t alloc_mem_types

Bitmap of memory types that Memory Domain can allocate memory on

uint64_t access_mem_types

Memory types that Memory Domain can access

struct uct_md_mem_attr

This structure defines the attributes of a memory pointer which may include the memory type of the pointer, and the system device that backs the pointer depending on the bit fields populated in field_mask.

Data Fields
uint64_t field_mask

Mask of valid fields in this structure, using bits from uct_md_mem_attr_field_t.

ucs_memory_type_t mem_type

The type of memory. E.g. CPU/GPU memory or some other valid type. If the md does not support sys_dev query, then UCS_MEMORY_TYPE_UNKNOWN is returned.

ucs_sys_device_t sys_dev

Index of the system device on which the buffer resides. eg: NUMA/GPU If the md does not support sys_dev query, then UCS_SYS_DEVICE_ID_UNKNOWN is returned.

void * base_address

Base address of the allocation to which the provided buffer belongs to. If the md not support base address query, then the pointer passed to uct_md_mem_query is returned as is.

size_t alloc_length

Length of the whole allocation to which the provided buffer belongs to. If the md not support querying allocation length, then the length passed to uct_md_mem_query is returned as is.

int dmabuf_fd

Dmabuf file descriptor to expose memory regions across devices. Refer (https://01.org/linuxgraphics/gfx-docs/drm/driver-api/dma-buf.html). If the md does not support querying the fd object associated with the region, then dmabuf_fd is set to UCT_DMABUF_FD_INVALID by uct_md_mem_query(). It is the responsibility of the user to close the returned fd using close (2) when it's no longer needed.

size_t dmabuf_offset

Offset of the given address from the start of the memory region (identified by dmabuf_fd) backing the memory region being queried.

struct uct_allocated_memory

This structure describes the memory block which includes the address, size, and Memory Domain used for allocation. This structure is passed to interface and the memory is allocated by memory allocation functions uct_mem_alloc.

Data Fields
void * address

Address of allocated memory

size_t length

Real size of allocated memory

uct_alloc_method_t method

Method used to allocate the memory

ucs_memory_type_t mem_type

type of allocated memory

uct_md_h md

if method==MD: MD used to allocate the memory

uct_mem_h memh

if method==MD: MD memory handle

struct uct_rkey_bundle

This structure describes the credentials (typically key) and information required to access the remote memory by the communication interfaces.

Data Fields
uct_rkey_t rkey

Remote key descriptor, passed to RMA functions

void * handle

Handle, used internally for releasing the key

void * type

Remote key type

struct uct_mem_alloc_params_t
Data Fields
uint64_t field_mask

Mask of valid fields in this structure, using bits from uct_mem_alloc_params_field_t. Fields not specified in this mask will be ignored.

unsigned flags

Memory allocation flags, see uct_md_mem_flags If UCT_MEM_ALLOC_PARAM_FIELD_FLAGS is not specified in field_mask, then (UCT_MD_MEM_ACCESS_LOCAL_READ | UCT_MD_MEM_ACCESS_LOCAL_WRITE) is used by default.

void * address

If address is NULL, the underlying allocation routine will choose the address at which to create the mapping. If address is non-NULL and UCT_MD_MEM_FLAG_FIXED is not set, the address will be interpreted as a hint as to where to establish the mapping. If address is non-NULL and UCT_MD_MEM_FLAG_FIXED is set, then the specified address is interpreted as a requirement. In this case, if the mapping to the exact address cannot be made, the allocation request fails.

ucs_memory_type_t mem_type

Type of memory to be allocated.

struct uct_mem_alloc_params_t mds
const char * name

Name of the allocated region, used to track memory usage for debugging and profiling. If UCT_MEM_ALLOC_PARAM_FIELD_NAME is not specified in field_mask, then "anonymous-uct_mem_alloc" is used by default.

struct uct_mem_alloc_params_t.mds
Data Fields
const uct_md_h * mds

Array of memory domains to attempt to allocate the memory with, for MD allocation method.

unsigned count

Length of 'mds' array. May be empty, in such case 'mds' may be NULL, and MD allocation method will be skipped.

Typedef Documentation

The enumeration allows specifying which fields in uct_md_mem_attr_t are present.

This structure defines the attributes of a memory pointer which may include the memory type of the pointer, and the system device that backs the pointer depending on the bit fields populated in field_mask.

This structure describes the memory block which includes the address, size, and Memory Domain used for allocation. This structure is passed to interface and the memory is allocated by memory allocation functions uct_mem_alloc.

This structure describes the credentials (typically key) and information required to access the remote memory by the communication interfaces.

Enumeration Type Documentation

Enumerator
UCT_SOCKADDR_ACC_LOCAL 

Check if local address exists. Address should belong to a local network interface

UCT_SOCKADDR_ACC_REMOTE 

Check if remote address can be reached. Address is routable from one of the local network interfaces

anonymous enum
Enumerator
UCT_MD_FLAG_ALLOC 

MD supports memory allocation

UCT_MD_FLAG_REG 

MD supports memory registration

UCT_MD_FLAG_NEED_MEMH 

The transport needs a valid local memory handle for zero-copy operations

UCT_MD_FLAG_NEED_RKEY 

The transport needs a valid remote memory key for remote memory operations

UCT_MD_FLAG_ADVISE 

MD supports memory advice

UCT_MD_FLAG_FIXED 

MD supports memory allocation with fixed address

UCT_MD_FLAG_RKEY_PTR 

MD supports direct access to remote memory via a pointer that is returned by uct_rkey_ptr.

Note
This flag is deprecated and replaced by UCT_COMPONENT_FLAG_RKEY_PTR.
UCT_MD_FLAG_SOCKADDR 

MD support for client-server connection establishment via sockaddr

UCT_MD_FLAG_INVALIDATE 

MD supports memory invalidation

UCT_MD_FLAG_EXPORTED_MKEY 

MD supports exporting memory keys with another process using the same device or attaching to an exported memory key.

UCT_MD_FLAG_REG_DMABUF 

MD supports registering a dmabuf file descriptor.

Enumerator
UCT_MD_MEM_FLAG_NONBLOCK 

Hint to perform non-blocking allocation/registration: page mapping may be deferred until it is accessed by the CPU or a transport.

UCT_MD_MEM_FLAG_FIXED 

Place the mapping at exactly defined address.

UCT_MD_MEM_FLAG_LOCK 

Registered memory should be locked. May incur extra cost for registration, but memory access is usually faster.

UCT_MD_MEM_FLAG_HIDE_ERRORS 

Hide errors on memory registration. In some cases registration failure is not an error (e. g. for merged memory regions).

UCT_MD_MEM_ACCESS_REMOTE_PUT 

Enable remote put access.

UCT_MD_MEM_ACCESS_REMOTE_GET 

Enable remote get access.

UCT_MD_MEM_ACCESS_REMOTE_ATOMIC 

Enable remote atomic access.

UCT_MD_MEM_ACCESS_LOCAL_READ 

Enable local read access.

UCT_MD_MEM_ACCESS_LOCAL_WRITE 

Enable local write access.

UCT_MD_MEM_ACCESS_ALL 

Enable local and remote access for all operations.

UCT_MD_MEM_ACCESS_RMA 

Enable local and remote access for put and get operations.

Enumerator
UCT_MADV_NORMAL 

No special treatment

UCT_MADV_WILLNEED 

can be used on the memory mapped with UCT_MD_MEM_FLAG_NONBLOCK to speed up memory mapping and to avoid page faults when the memory is accessed for the first time.

The enumeration allows specifying which fields in uct_md_mem_attr_t are present.

Enumerator
UCT_MD_MEM_ATTR_FIELD_MEM_TYPE 

Indicate if memory type is populated. E.g. CPU/GPU

UCT_MD_MEM_ATTR_FIELD_SYS_DEV 

Indicate if details of system device backing the pointer are populated. For example: GPU device, NUMA domain, etc.

UCT_MD_MEM_ATTR_FIELD_BASE_ADDRESS 

Request base address of the allocation to which the buffer belongs.

UCT_MD_MEM_ATTR_FIELD_ALLOC_LENGTH 

Request the whole length of the allocation to which the buffer belongs.

UCT_MD_MEM_ATTR_FIELD_DMABUF_FD 

Request a cross-device dmabuf file descriptor that represents a memory region, and can be used to register the region with another memory domain.

UCT_MD_MEM_ATTR_FIELD_DMABUF_OFFSET 

Request the offset of the provided virtual address relative to the beginning of its backing dmabuf region.

The enumeration allows specifying which fields in uct_mem_alloc_params_t are present.

Enumerator
UCT_MEM_ALLOC_PARAM_FIELD_FLAGS 

Enables uct_mem_alloc_params_t::flags

UCT_MEM_ALLOC_PARAM_FIELD_ADDRESS 

Enables uct_mem_alloc_params_t::address

UCT_MEM_ALLOC_PARAM_FIELD_MEM_TYPE 

Enables uct_mem_alloc_params_t::mem_type

UCT_MEM_ALLOC_PARAM_FIELD_MDS 

Enables uct_mem_alloc_params_t::mds

UCT_MEM_ALLOC_PARAM_FIELD_NAME 

Enables uct_mem_alloc_params_t::name

Function Documentation

ucs_status_t uct_md_mem_query ( uct_md_h  md,
const void *  address,
size_t  length,
uct_md_mem_attr_t mem_attr 
)

Return attributes such as memory type, base address, allocation length, and system device for the given pointer of specific length.

Parameters
[in]mdMemory domain to run the query on. This function returns an error if the md does not recognize the pointer.
[in]addressThe address of the pointer. Must be non-NULL else UCS_ERR_INVALID_PARAM error is returned.
[in]lengthLength of the memory region to examine. Must be nonzero else UCS_ERR_INVALID_PARAM error is returned.
[in,out]mem_attrIf successful, filled with ptr attributes.
Returns
UCS_OK if at least one attribute is successfully queried otherwise an error code as defined by ucs_status_t is returned.
ucs_status_t uct_md_query ( uct_md_h  md,
uct_md_attr_t md_attr 
)
Parameters
[in]mdMemory domain to query.
[out]md_attrFilled with memory domain attributes.
Examples:
uct_hello_world.c.
ucs_status_t uct_md_mem_advise ( uct_md_h  md,
uct_mem_h  memh,
void *  addr,
size_t  length,
uct_mem_advice_t  advice 
)

This routine advises the UCT about how to handle memory range beginning at address and size of length bytes. This call does not influence the semantics of the application, but may influence its performance. The advice may be ignored.

Parameters
[in]mdMemory domain memory was allocated or registered on.
[in]memhMemory handle, as returned from uct_mem_alloc
[in]addrMemory base address. Memory range must belong to the memh
[in]lengthLength of memory to advise. Must be >0.
[in]adviceMemory use advice as defined in the uct_mem_advice_t list
ucs_status_t uct_md_mem_reg ( uct_md_h  md,
void *  address,
size_t  length,
unsigned  flags,
uct_mem_h memh_p 
)

Register memory on the memory domain. In order to use this function, MD must support UCT_MD_FLAG_REG flag.

Parameters
[in]mdMemory domain to register memory on.
[in]addressMemory to register.
[in]lengthSize of memory to register. Must be >0.
[in]flagsMemory allocation flags, see uct_md_mem_flags.
[out]memh_pFilled with handle for allocated region.
Examples:
uct_hello_world.c.
ucs_status_t uct_md_mem_dereg ( uct_md_h  md,
uct_mem_h  memh 
)
Parameters
[in]mdMemory domain which was used to register the memory.
[in]memhLocal access key to memory region.
Examples:
uct_hello_world.c.
ucs_status_t uct_md_detect_memory_type ( uct_md_h  md,
const void *  addr,
size_t  length,
ucs_memory_type_t mem_type_p 
)
Parameters
[in]mdMemory domain to detect memory type
[in]addrMemory address to detect.
[in]lengthSize of memory
[out]mem_type_pFilled with memory type of the address range if function succeeds
Returns
UCS_OK If memory type is successfully detected UCS_ERR_INVALID_ADDR If failed to detect memory type
ucs_status_t uct_mem_alloc ( size_t  length,
const uct_alloc_method_t methods,
unsigned  num_methods,
const uct_mem_alloc_params_t params,
uct_allocated_memory_t mem 
)

Allocate potentially registered memory.

Parameters
[in]lengthThe minimal size to allocate. The actual size may be larger, for example because of alignment restrictions. Must be >0.
[in]methodsArray of memory allocation methods to attempt. Each of the provided allocation methods will be tried in array order, to perform the allocation, until one succeeds. Whenever the MD method is encountered, each of the provided MDs will be tried in array order, to allocate the memory, until one succeeds, or they are exhausted. In this case the next allocation method from the initial list will be attempted.
[in]num_methodsLength of 'methods' array.
[in]paramsMemory allocation characteristics, see uct_mem_alloc_params_t.
[out]memIn case of success, filled with information about the allocated memory. uct_allocated_memory_t
ucs_status_t uct_mem_free ( const uct_allocated_memory_t mem)

Release the memory allocated by uct_mem_alloc.

Parameters
[in]memDescription of allocated memory, as returned from uct_mem_alloc.
ucs_status_t uct_md_config_read ( uct_component_h  component,
const char *  env_prefix,
const char *  filename,
uct_md_config_t **  config_p 
)
Parameters
[in]componentRead the configuration of this component.
[in]env_prefixIf non-NULL, search for environment variables starting with this UCT_<prefix>_. Otherwise, search for environment variables starting with just UCT_.
[in]filenameIf non-NULL, read configuration from this file. If the file does not exist, it will be ignored.
[out]config_pFilled with a pointer to the configuration.
Returns
Error code.
Examples:
uct_hello_world.c.
int uct_md_is_sockaddr_accessible ( uct_md_h  md,
const ucs_sock_addr_t sockaddr,
uct_sockaddr_accessibility_t  mode 
)

This function checks if a remote sock address can be accessed from a local memory domain. Accessibility can be checked in local or remote mode.

Parameters
[in]mdMemory domain to check accessibility from. This memory domain must support the UCT_MD_FLAG_SOCKADDR flag.
[in]sockaddrSocket address to check accessibility to.
[in]modeMode for checking accessibility, as defined in uct_sockaddr_accessibility_t. Indicates if accessibility is tested on the server side - for binding to the given sockaddr, or on the client side - for connecting to the given remote peer's sockaddr.
Returns
Nonzero if accessible, 0 if inaccessible.
ucs_status_t uct_md_mkey_pack ( uct_md_h  md,
uct_mem_h  memh,
void *  rkey_buffer 
)
Parameters
[in]mdHandle to memory domain.
[in]memhLocal key, whose remote key should be packed.
[out]rkey_bufferFilled with packed remote key.
Returns
Error code.
ucs_status_t uct_rkey_unpack ( uct_component_h  component,
const void *  rkey_buffer,
uct_rkey_bundle_t rkey_ob 
)
Parameters
[in]componentComponent on which to unpack the remote key.
[in]rkey_bufferPacked remote key buffer.
[out]rkey_obFilled with the unpacked remote key and its type.
Note
The remote key must be unpacked with the same component that was used to pack it. For example, if a remote device address on the remote memory domain which was used to pack the key is reachable by a transport on a local component, then that component is eligible to unpack the key. If the remote key buffer cannot be unpacked with the given component, UCS_ERR_INVALID_PARAM will be returned.
Returns
Error code.
ucs_status_t uct_rkey_ptr ( uct_component_h  component,
uct_rkey_bundle_t rkey_ob,
uint64_t  remote_addr,
void **  addr_p 
)

This routine returns a local pointer to the remote memory described by the rkey bundle. The component must support UCT_COMPONENT_FLAG_RKEY_PTR flag.

Parameters
[in]componentComponent on which to obtain the pointer to the remote key.
[in]rkey_obA remote key bundle as returned by the uct_rkey_unpack function.
[in]remote_addrA remote address within the memory area described by the rkey_ob.
[out]addr_pA pointer that can be used for direct access to the remote memory.
Note
The component used to obtain a local pointer to the remote memory must be the same component that was used to pack the remote key. See notes section for uct_rkey_unpack.
Returns
Error code if the remote memory cannot be accessed directly or the remote address is not valid.
ucs_status_t uct_rkey_release ( uct_component_h  component,
const uct_rkey_bundle_t rkey_ob 
)
Parameters
[in]componentComponent which was used to unpack the remote key.
[in]rkey_obRemote key to release.