UCP Communication routines

Data Structures
struct	ucp_request_attr_t
	Attributes of a particular request. More...
struct	ucp_err_handler
	UCP endpoint error handling context. More...

Typedefs
typedef uint64_t	ucp_tag_t
	UCP Tag Identifier.
typedef struct ucp_recv_desc *	ucp_tag_message_h
	UCP Message descriptor.
typedef uint64_t	ucp_datatype_t
	UCP Datatype Identifier.
typedef void(*	ucp_send_callback_t) (void *request, ucs_status_t status)
	Completion callback for non-blocking sends.
typedef void(*	ucp_send_nbx_callback_t) (void *request, ucs_status_t status, void *user_data)
	Completion callback for non-blocking sends.
typedef void(*	ucp_err_handler_cb_t) (void *arg, ucp_ep_h ep, ucs_status_t status)
	Callback to process peer failure.
typedef struct ucp_err_handler	ucp_err_handler_t
	UCP endpoint error handling context.
typedef void(*	ucp_stream_recv_callback_t) (void *request, ucs_status_t status, size_t length)
	Completion callback for non-blocking stream oriented receives.
typedef void(*	ucp_stream_recv_nbx_callback_t) (void *request, ucs_status_t status, size_t length, void *user_data)
	Completion callback for non-blocking stream receives ucp_stream_recv_nbx call.
typedef void(*	ucp_tag_recv_callback_t) (void *request, ucs_status_t status, ucp_tag_recv_info_t *info)
	Completion callback for non-blocking tag receives.
typedef void(*	ucp_tag_recv_nbx_callback_t) (void *request, ucs_status_t status, const ucp_tag_recv_info_t *tag_info, void *user_data)
	Completion callback for non-blocking tag receives ucp_tag_recv_nbx call.
typedef void(*	ucp_am_recv_data_nbx_callback_t) (void *request, ucs_status_t status, size_t length, void *user_data)
	Completion callback for non-blocking Active Message receives.

Enumerations
enum	ucp_atomic_op_t {   UCP_ATOMIC_OP_ADD , UCP_ATOMIC_OP_SWAP , UCP_ATOMIC_OP_CSWAP , UCP_ATOMIC_OP_AND ,   UCP_ATOMIC_OP_OR , UCP_ATOMIC_OP_XOR , UCP_ATOMIC_OP_LAST }
	Atomic operation requested for ucp_atomic_op_nbx. More...
enum	ucp_stream_recv_flags_t { UCP_STREAM_RECV_FLAG_WAITALL = UCS_BIT(0) }
	Flags to define behavior of ucp_stream_recv_nb function. More...
enum	ucp_op_attr_t {   UCP_OP_ATTR_FIELD_REQUEST = UCS_BIT(0) , UCP_OP_ATTR_FIELD_CALLBACK = UCS_BIT(1) , UCP_OP_ATTR_FIELD_USER_DATA = UCS_BIT(2) , UCP_OP_ATTR_FIELD_DATATYPE = UCS_BIT(3) ,   UCP_OP_ATTR_FIELD_FLAGS = UCS_BIT(4) , UCP_OP_ATTR_FIELD_REPLY_BUFFER = UCS_BIT(5) , UCP_OP_ATTR_FIELD_MEMORY_TYPE = UCS_BIT(6) , UCP_OP_ATTR_FIELD_RECV_INFO = UCS_BIT(7) ,   UCP_OP_ATTR_FIELD_MEMH = UCS_BIT(8) , UCP_OP_ATTR_FLAG_NO_IMM_CMPL = UCS_BIT(16) , UCP_OP_ATTR_FLAG_FAST_CMPL = UCS_BIT(17) , UCP_OP_ATTR_FLAG_FORCE_IMM_CMPL = UCS_BIT(18) ,   UCP_OP_ATTR_FLAG_MULTI_SEND = UCS_BIT(19) }
	UCP operation fields and flags. More...
enum	ucp_req_attr_field { UCP_REQUEST_ATTR_FIELD_INFO_STRING = UCS_BIT(0) , UCP_REQUEST_ATTR_FIELD_INFO_STRING_SIZE = UCS_BIT(1) , UCP_REQUEST_ATTR_FIELD_STATUS = UCS_BIT(2) , UCP_REQUEST_ATTR_FIELD_MEM_TYPE = UCS_BIT(3) }
	UCP request query attributes. More...
enum	ucp_am_recv_attr_t { UCP_AM_RECV_ATTR_FIELD_REPLY_EP = UCS_BIT(0) , UCP_AM_RECV_ATTR_FLAG_DATA = UCS_BIT(16) , UCP_AM_RECV_ATTR_FLAG_RNDV = UCS_BIT(17) }
	UCP AM receive data parameter fields and flags. More...
enum	ucp_am_handler_param_field { UCP_AM_HANDLER_PARAM_FIELD_ID = UCS_BIT(0) , UCP_AM_HANDLER_PARAM_FIELD_FLAGS = UCS_BIT(1) , UCP_AM_HANDLER_PARAM_FIELD_CB = UCS_BIT(2) , UCP_AM_HANDLER_PARAM_FIELD_ARG = UCS_BIT(3) }
	UCP AM receive data parameters fields and flags. More...
enum	ucp_atomic_post_op_t {   UCP_ATOMIC_POST_OP_ADD , UCP_ATOMIC_POST_OP_AND , UCP_ATOMIC_POST_OP_OR , UCP_ATOMIC_POST_OP_XOR ,   UCP_ATOMIC_POST_OP_LAST }
	Atomic operation requested for ucp_atomic_post. More...
enum	ucp_atomic_fetch_op_t {   UCP_ATOMIC_FETCH_OP_FADD , UCP_ATOMIC_FETCH_OP_SWAP , UCP_ATOMIC_FETCH_OP_CSWAP , UCP_ATOMIC_FETCH_OP_FAND ,   UCP_ATOMIC_FETCH_OP_FOR , UCP_ATOMIC_FETCH_OP_FXOR , UCP_ATOMIC_FETCH_OP_LAST }
	Atomic operation requested for ucp_atomic_fetch. More...

Functions
ucs_status_t	ucp_request_query (void *request, ucp_request_attr_t *attr)
	Get information about ucp_request.
ucs_status_ptr_t	ucp_am_send_nbx (ucp_ep_h ep, unsigned id, const void *header, size_t header_length, const void *buffer, size_t count, const ucp_request_param_t *param)
	Send Active Message.
ucs_status_ptr_t	ucp_am_recv_data_nbx (ucp_worker_h worker, void *data_desc, void *buffer, size_t count, const ucp_request_param_t *param)
	Receive Active Message as defined by provided data descriptor.
void	ucp_am_data_release (ucp_worker_h worker, void *data)
	Releases Active Message data.
ucs_status_ptr_t	ucp_stream_send_nbx (ucp_ep_h ep, const void *buffer, size_t count, const ucp_request_param_t *param)
	Non-blocking stream send operation.
ucs_status_ptr_t	ucp_tag_send_nbx (ucp_ep_h ep, const void *buffer, size_t count, ucp_tag_t tag, const ucp_request_param_t *param)
	Non-blocking tagged-send operation.
ucs_status_ptr_t	ucp_tag_send_sync_nbx (ucp_ep_h ep, const void *buffer, size_t count, ucp_tag_t tag, const ucp_request_param_t *param)
	Non-blocking synchronous tagged-send operation.
ucs_status_ptr_t	ucp_stream_recv_nbx (ucp_ep_h ep, void *buffer, size_t count, size_t *length, const ucp_request_param_t *param)
	Non-blocking stream receive operation of structured data into a user-supplied buffer.
ucs_status_ptr_t	ucp_stream_recv_data_nb (ucp_ep_h ep, size_t *length)
	Non-blocking stream receive operation of unstructured data into a UCP-supplied buffer.
ucs_status_ptr_t	ucp_tag_recv_nbx (ucp_worker_h worker, void *buffer, size_t count, ucp_tag_t tag, ucp_tag_t tag_mask, const ucp_request_param_t *param)
	Non-blocking tagged-receive operation.
ucp_tag_message_h	ucp_tag_probe_nb (ucp_worker_h worker, ucp_tag_t tag, ucp_tag_t tag_mask, int remove, ucp_tag_recv_info_t *info)
	Non-blocking probe and return a message.
ucs_status_ptr_t	ucp_tag_msg_recv_nbx (ucp_worker_h worker, void *buffer, size_t count, ucp_tag_message_h message, const ucp_request_param_t *param)
	Non-blocking receive operation for a probed message.
ucs_status_ptr_t	ucp_put_nbx (ucp_ep_h ep, const void *buffer, size_t count, uint64_t remote_addr, ucp_rkey_h rkey, const ucp_request_param_t *param)
	Non-blocking remote memory put operation.
ucs_status_ptr_t	ucp_get_nbx (ucp_ep_h ep, void *buffer, size_t count, uint64_t remote_addr, ucp_rkey_h rkey, const ucp_request_param_t *param)
	Non-blocking remote memory get operation.
ucs_status_ptr_t	ucp_atomic_op_nbx (ucp_ep_h ep, ucp_atomic_op_t opcode, const void *buffer, size_t count, uint64_t remote_addr, ucp_rkey_h rkey, const ucp_request_param_t *param)
	Post an atomic memory operation.
ucs_status_t	ucp_request_check_status (void *request)
	Check the status of non-blocking request.
ucs_status_t	ucp_tag_recv_request_test (void *request, ucp_tag_recv_info_t *info)
	Check the status and currently available state of non-blocking request returned from ucp_tag_recv_nb routine.
ucs_status_t	ucp_stream_recv_request_test (void *request, size_t *length_p)
	Check the status and currently available state of non-blocking request returned from ucp_stream_recv_nb routine.
void	ucp_request_cancel (ucp_worker_h worker, void *request)
	Cancel an outstanding communications request.
void	ucp_stream_data_release (ucp_ep_h ep, void *data)
	Release UCP data buffer returned by ucp_stream_recv_data_nb.
void	ucp_request_free (void *request)
	Release a communications request.
void *	ucp_request_alloc (ucp_worker_h worker)
	Create an empty communications request.
int	ucp_request_is_completed (void *request)
ucs_status_t	ucp_put (ucp_ep_h ep, const void *buffer, size_t length, uint64_t remote_addr, ucp_rkey_h rkey)
	Blocking remote memory put operation.
ucs_status_t	ucp_get (ucp_ep_h ep, void *buffer, size_t length, uint64_t remote_addr, ucp_rkey_h rkey)
	Blocking remote memory get operation.
ucs_status_t	ucp_atomic_add32 (ucp_ep_h ep, uint32_t add, uint64_t remote_addr, ucp_rkey_h rkey)
	Blocking atomic add operation for 32 bit integers.
ucs_status_t	ucp_atomic_add64 (ucp_ep_h ep, uint64_t add, uint64_t remote_addr, ucp_rkey_h rkey)
	Blocking atomic add operation for 64 bit integers.
ucs_status_t	ucp_atomic_fadd32 (ucp_ep_h ep, uint32_t add, uint64_t remote_addr, ucp_rkey_h rkey, uint32_t *result)
	Blocking atomic fetch and add operation for 32 bit integers.
ucs_status_t	ucp_atomic_fadd64 (ucp_ep_h ep, uint64_t add, uint64_t remote_addr, ucp_rkey_h rkey, uint64_t *result)
	Blocking atomic fetch and add operation for 64 bit integers.
ucs_status_t	ucp_atomic_swap32 (ucp_ep_h ep, uint32_t swap, uint64_t remote_addr, ucp_rkey_h rkey, uint32_t *result)
	Blocking atomic swap operation for 32 bit values.
ucs_status_t	ucp_atomic_swap64 (ucp_ep_h ep, uint64_t swap, uint64_t remote_addr, ucp_rkey_h rkey, uint64_t *result)
	Blocking atomic swap operation for 64 bit values.
ucs_status_t	ucp_atomic_cswap32 (ucp_ep_h ep, uint32_t compare, uint32_t swap, uint64_t remote_addr, ucp_rkey_h rkey, uint32_t *result)
	Blocking atomic conditional swap (cswap) operation for 32 bit values.
ucs_status_t	ucp_atomic_cswap64 (ucp_ep_h ep, uint64_t compare, uint64_t swap, uint64_t remote_addr, ucp_rkey_h rkey, uint64_t *result)
	Blocking atomic conditional swap (cswap) operation for 64 bit values.
ucs_status_ptr_t	ucp_am_send_nb (ucp_ep_h ep, uint16_t id, const void *buffer, size_t count, ucp_datatype_t datatype, ucp_send_callback_t cb, unsigned flags)
	Send Active Message.
ucs_status_ptr_t	ucp_stream_send_nb (ucp_ep_h ep, const void *buffer, size_t count, ucp_datatype_t datatype, ucp_send_callback_t cb, unsigned flags)
	Non-blocking stream send operation.
ucs_status_ptr_t	ucp_stream_recv_nb (ucp_ep_h ep, void *buffer, size_t count, ucp_datatype_t datatype, ucp_stream_recv_callback_t cb, size_t *length, unsigned flags)
	Non-blocking stream receive operation of structured data into a user-supplied buffer.
ucs_status_ptr_t	ucp_tag_send_nb (ucp_ep_h ep, const void *buffer, size_t count, ucp_datatype_t datatype, ucp_tag_t tag, ucp_send_callback_t cb)
	Non-blocking tagged-send operations.
ucs_status_t	ucp_tag_send_nbr (ucp_ep_h ep, const void *buffer, size_t count, ucp_datatype_t datatype, ucp_tag_t tag, void *req)
	Non-blocking tagged-send operations with user provided request.
ucs_status_ptr_t	ucp_tag_send_sync_nb (ucp_ep_h ep, const void *buffer, size_t count, ucp_datatype_t datatype, ucp_tag_t tag, ucp_send_callback_t cb)
	Non-blocking synchronous tagged-send operation.
ucs_status_ptr_t	ucp_tag_recv_nb (ucp_worker_h worker, void *buffer, size_t count, ucp_datatype_t datatype, ucp_tag_t tag, ucp_tag_t tag_mask, ucp_tag_recv_callback_t cb)
	Non-blocking tagged-receive operation.
ucs_status_t	ucp_tag_recv_nbr (ucp_worker_h worker, void *buffer, size_t count, ucp_datatype_t datatype, ucp_tag_t tag, ucp_tag_t tag_mask, void *req)
	Non-blocking tagged-receive operation.
ucs_status_ptr_t	ucp_tag_msg_recv_nb (ucp_worker_h worker, void *buffer, size_t count, ucp_datatype_t datatype, ucp_tag_message_h message, ucp_tag_recv_callback_t cb)
	Non-blocking receive operation for a probed message.
ucs_status_t	ucp_put_nbi (ucp_ep_h ep, const void *buffer, size_t length, uint64_t remote_addr, ucp_rkey_h rkey)
	Non-blocking implicit remote memory put operation.
ucs_status_ptr_t	ucp_put_nb (ucp_ep_h ep, const void *buffer, size_t length, uint64_t remote_addr, ucp_rkey_h rkey, ucp_send_callback_t cb)
	Non-blocking remote memory put operation.
ucs_status_t	ucp_get_nbi (ucp_ep_h ep, void *buffer, size_t length, uint64_t remote_addr, ucp_rkey_h rkey)
	Non-blocking implicit remote memory get operation.
ucs_status_ptr_t	ucp_get_nb (ucp_ep_h ep, void *buffer, size_t length, uint64_t remote_addr, ucp_rkey_h rkey, ucp_send_callback_t cb)
	Non-blocking remote memory get operation.
ucs_status_t	ucp_atomic_post (ucp_ep_h ep, ucp_atomic_post_op_t opcode, uint64_t value, size_t op_size, uint64_t remote_addr, ucp_rkey_h rkey)
	Post an atomic memory operation.
ucs_status_ptr_t	ucp_atomic_fetch_nb (ucp_ep_h ep, ucp_atomic_fetch_op_t opcode, uint64_t value, void *result, size_t op_size, uint64_t remote_addr, ucp_rkey_h rkey, ucp_send_callback_t cb)
	Post an atomic fetch operation.

Detailed Description

UCP Communication routines

---

Data Structure Documentation

ucp_request_attr_t

struct ucp_request_attr_t

Data Fields
uint64_t	field_mask	Mask of valid fields in this structure, using bits from ucp_req_attr_field. Fields not specified in this mask will be ignored. Provides ABI compatibility with respect to adding new fields.
char *	debug_string	Pointer to allocated string of size debug_string_size that will be filled with debug information about transports and protocols that were selected to complete the request.
size_t	debug_string_size	Size of the debug_string. String will be filled up to this size. Maximum possible size debug string can be obtained by querying the worker via ucp_worker_query.
ucs_status_t	status	Status of the request. The same as ucp_request_check_status.
ucs_memory_type_t	mem_type	Detected memory type of the buffer passed to the operation.

ucp_err_handler

struct ucp_err_handler

This structure should be initialized in ucp_ep_params_t to handle peer failure

Data Fields
ucp_err_handler_cb_t	cb	Error handler callback, if NULL, will not be called.
void *	arg	User defined argument associated with an endpoint, it will be overridden by ucp_ep_params_t::user_data if both are set.

Typedef Documentation

ucp_tag_t

typedef uint64_t ucp_tag_t

UCP tag identifier is a 64bit object used for message identification. UCP tag send and receive operations use the object for an implementation tag matching semantics (derivative of MPI tag matching semantics).

ucp_tag_message_h

typedef struct ucp_recv_desc* ucp_tag_message_h

UCP Message descriptor is an opaque handle for a message returned by ucp_tag_probe_nb. This handle can be passed to ucp_tag_msg_recv_nb in order to receive the message data to a specific buffer.

ucp_datatype_t

typedef uint64_t ucp_datatype_t

UCP datatype identifier is a 64bit object used for datatype identification. Predefined UCP identifiers are defined by ucp_dt_type.

ucp_send_callback_t

typedef void(* ucp_send_callback_t) (void *request, ucs_status_t status)

This callback routine is invoked whenever the send operation is completed. It is important to note that the call-back is only invoked in a case when the operation cannot be completed in place.

Parameters

[in]	request	The completed send request.
[in]	status	Completion status. If the send operation was completed successfully UCS_OK is returned. If send operation was canceled UCS_ERR_CANCELED is returned. Otherwise, an error status is returned.

ucp_send_nbx_callback_t

typedef void(* ucp_send_nbx_callback_t) (void *request, ucs_status_t status, void *user_data)

This callback routine is invoked whenever the ucp_tag_send_nbx, ucp_am_send_nbx, ucp_stream_send_nbx, ucp_put_nbx, ucp_get_nbx, ucp_atomic_op_nbx or any other "send operation" is completed.

Parameters

[in]	request	The completed send request.
[in]	status	Completion status. If the send operation was completed successfully UCS_OK is returned. If send operation was canceled UCS_ERR_CANCELED is returned. Otherwise, an error status is returned.
[in]	user_data	User data passed to "user_data" value, see ucp_request_param_t

Examples

ucp_client_server.c.

ucp_err_handler_cb_t

typedef void(* ucp_err_handler_cb_t) (void *arg, ucp_ep_h ep, ucs_status_t status)

This callback routine is invoked when transport level error detected.

Parameters

[in]	arg	User argument to be passed to the callback.
[in]	ep	Endpoint to handle transport level error. Upon return from the callback, this ep is no longer usable and all subsequent operations on this ep will fail with the error code passed in status.
[in]	status	error status.

ucp_err_handler_t

typedef struct ucp_err_handler ucp_err_handler_t

This structure should be initialized in ucp_ep_params_t to handle peer failure

ucp_stream_recv_callback_t

typedef void(* ucp_stream_recv_callback_t) (void *request, ucs_status_t status, size_t length)

This callback routine is invoked whenever the receive operation is completed and the data is ready in the receive buffer.

Parameters

[in]	request	The completed receive request.
[in]	status	Completion status. If the send operation was completed successfully UCS_OK is returned. Otherwise, an error status is returned.
[in]	length	The size of the received data in bytes, always boundary of base datatype size. The value is valid only if the status is UCS_OK.

ucp_stream_recv_nbx_callback_t

typedef void(* ucp_stream_recv_nbx_callback_t) (void *request, ucs_status_t status, size_t length, void *user_data)

This callback routine is invoked whenever the receive operation is completed and the data is ready in the receive buffer.

Parameters

[in]	request	The completed receive request.
[in]	status	Completion status. If the send operation was completed successfully UCS_OK is returned. Otherwise, an error status is returned.
[in]	length	The size of the received data in bytes, always on the boundary of base datatype size. The value is valid only if the status is UCS_OK.
[in]	user_data	User data passed to "user_data" value, see ucp_request_param_t.

ucp_tag_recv_callback_t

typedef void(* ucp_tag_recv_callback_t) (void *request, ucs_status_t status, ucp_tag_recv_info_t *info)

This callback routine is invoked whenever the receive operation is completed and the data is ready in the receive buffer.

Parameters

[in]	request	The completed receive request.
[in]	status	Completion status. If the send operation was completed successfully UCS_OK is returned. If send operation was canceled UCS_ERR_CANCELED is returned. If the data can not fit into the receive buffer the UCS_ERR_MESSAGE_TRUNCATED error code is returned. Otherwise, an error status is returned.
[in]	info	Completion information The info descriptor is Valid only if the status is UCS_OK.

ucp_tag_recv_nbx_callback_t

typedef void(* ucp_tag_recv_nbx_callback_t) (void *request, ucs_status_t status, const ucp_tag_recv_info_t *tag_info, void *user_data)

This callback routine is invoked whenever the receive operation is completed and the data is ready in the receive buffer.

Parameters

[in]	request	The completed receive request.
[in]	status	Completion status. If the receive operation was completed successfully UCS_OK is returned. If send operation was canceled, UCS_ERR_CANCELED is returned. If the data can not fit into the receive buffer the UCS_ERR_MESSAGE_TRUNCATED error code is returned. Otherwise, an error status is returned.
[in]	tag_info	Completion information The info descriptor is Valid only if the status is UCS_OK.
[in]	user_data	User data passed to "user_data" value, see ucp_request_param_t

ucp_am_recv_data_nbx_callback_t

typedef void(* ucp_am_recv_data_nbx_callback_t) (void *request, ucs_status_t status, size_t length, void *user_data)

This callback routine is invoked whenever the receive operation is completed and the data is ready in the receive buffer.

Parameters

[in]	request	The completed receive request.
[in]	status	Completion status. If the receive operation was completed successfully UCS_OK is returned. Otherwise, an error status is returned.
[in]	length	The size of the received data in bytes, always boundary of base datatype size. The value is valid only if the status is UCS_OK.
[in]	user_data	User data passed to "user_data" value, see ucp_request_param_t

Enumeration Type Documentation

ucp_atomic_op_t

enum ucp_atomic_op_t

This enumeration defines which atomic memory operation should be performed by the ucp_atomic_op_nbx routine.

Enumerator
UCP_ATOMIC_OP_ADD	Atomic add
UCP_ATOMIC_OP_SWAP	Atomic swap
UCP_ATOMIC_OP_CSWAP	Atomic conditional swap
UCP_ATOMIC_OP_AND	Atomic and
UCP_ATOMIC_OP_OR	Atomic or
UCP_ATOMIC_OP_XOR	Atomic xor
UCP_ATOMIC_OP_LAST

ucp_stream_recv_flags_t

enum ucp_stream_recv_flags_t

This enumeration defines behavior of ucp_stream_recv_nb function.

Enumerator
UCP_STREAM_RECV_FLAG_WAITALL	This flag requests that the operation will not be completed until all requested data is received and placed in the user buffer.

ucp_op_attr_t

enum ucp_op_attr_t

The enumeration allows specifying which fields in ucp_request_param_t are present and operation flags are used. It is used to enable backward compatibility support.

Enumerator
UCP_OP_ATTR_FIELD_REQUEST	request field
UCP_OP_ATTR_FIELD_CALLBACK	cb field
UCP_OP_ATTR_FIELD_USER_DATA	user_data field
UCP_OP_ATTR_FIELD_DATATYPE	datatype field
UCP_OP_ATTR_FIELD_FLAGS	operation-specific flags
UCP_OP_ATTR_FIELD_REPLY_BUFFER	reply_buffer field
UCP_OP_ATTR_FIELD_MEMORY_TYPE	memory type field
UCP_OP_ATTR_FIELD_RECV_INFO	recv_info field
UCP_OP_ATTR_FIELD_MEMH	memory handle field
UCP_OP_ATTR_FLAG_NO_IMM_CMPL	Deny immediate completion, i.e NULL cannot be returned. If a completion callback is provided, it can be called before the function returns.
UCP_OP_ATTR_FLAG_FAST_CMPL	expedite local completion, even if it delays remote data delivery. Note for implementer: this option can disable zero copy and/or rendezvous protocols which require synchronization with the remote peer before releasing the local send buffer
UCP_OP_ATTR_FLAG_FORCE_IMM_CMPL	force immediate complete operation, fail if the operation cannot be completed immediately
UCP_OP_ATTR_FLAG_MULTI_SEND	optimize for bandwidth of multiple in-flight operations, rather than for the latency of a single operation. This flag and UCP_OP_ATTR_FLAG_FAST_CMPL are mutually exclusive.

ucp_req_attr_field

enum ucp_req_attr_field

The enumeration allows specifying which fields in ucp_request_attr_t are present. It is used to enable backward compatibility support.

Enumerator
UCP_REQUEST_ATTR_FIELD_INFO_STRING
UCP_REQUEST_ATTR_FIELD_INFO_STRING_SIZE
UCP_REQUEST_ATTR_FIELD_STATUS
UCP_REQUEST_ATTR_FIELD_MEM_TYPE

ucp_am_recv_attr_t

enum ucp_am_recv_attr_t

The enumeration allows specifying which fields in ucp_am_recv_param_t are present and receive operation flags are used. It is used to enable backward compatibility support.

Enumerator
UCP_AM_RECV_ATTR_FIELD_REPLY_EP	reply_ep field
UCP_AM_RECV_ATTR_FLAG_DATA	Indicates that the data provided in ucp_am_recv_callback_t callback can be held by the user. If UCS_INPROGRESS is returned from the callback, the data parameter will persist and the user has to call ucp_am_data_release when data is no longer needed. This flag is mutually exclusive with UCP_AM_RECV_ATTR_FLAG_RNDV.
UCP_AM_RECV_ATTR_FLAG_RNDV	Indicates that the arriving data was sent using rendezvous protocol. In this case data parameter of the ucp_am_recv_callback_t points to the internal UCP descriptor, which can be used for obtaining the actual data by calling ucp_am_recv_data_nbx routine. This flag is mutually exclusive with UCP_AM_RECV_ATTR_FLAG_DATA.

ucp_am_handler_param_field

enum ucp_am_handler_param_field

The enumeration allows specifying which fields in ucp_am_handler_param_t are present. It is used to enable backward compatibility support.

Enumerator
UCP_AM_HANDLER_PARAM_FIELD_ID	Indicates that ucp_am_handler_param_t::id field is valid.
UCP_AM_HANDLER_PARAM_FIELD_FLAGS	Indicates that ucp_am_handler_param_t::flags field is valid.
UCP_AM_HANDLER_PARAM_FIELD_CB	Indicates that ucp_am_handler_param_t::cb field is valid.
UCP_AM_HANDLER_PARAM_FIELD_ARG	Indicates that ucp_am_handler_param_t::arg field is valid.

ucp_atomic_post_op_t

enum ucp_atomic_post_op_t

Deprecated

Use ucp_atomic_op_nbx and ucp_atomic_op_t instead.

This enumeration defines which atomic memory operation should be performed by the ucp_atomic_post family of functions. All of these are non-fetching atomics and will not result in a request handle.

Enumerator
UCP_ATOMIC_POST_OP_ADD	Atomic add
UCP_ATOMIC_POST_OP_AND	Atomic and
UCP_ATOMIC_POST_OP_OR	Atomic or
UCP_ATOMIC_POST_OP_XOR	Atomic xor
UCP_ATOMIC_POST_OP_LAST

ucp_atomic_fetch_op_t

enum ucp_atomic_fetch_op_t

Deprecated

Use ucp_atomic_op_nbx and ucp_atomic_op_t instead.

This enumeration defines which atomic memory operation should be performed by the ucp_atomic_fetch family of functions. All of these functions will fetch data from the remote node.

Enumerator
UCP_ATOMIC_FETCH_OP_FADD	Atomic Fetch and add
UCP_ATOMIC_FETCH_OP_SWAP	Atomic swap
UCP_ATOMIC_FETCH_OP_CSWAP	Atomic conditional swap
UCP_ATOMIC_FETCH_OP_FAND	Atomic Fetch and and
UCP_ATOMIC_FETCH_OP_FOR	Atomic Fetch and or
UCP_ATOMIC_FETCH_OP_FXOR	Atomic Fetch and xor
UCP_ATOMIC_FETCH_OP_LAST

Function Documentation

ucp_request_query()

ucs_status_t ucp_request_query	(	void *	request,
		ucp_request_attr_t *	attr )

Parameters

[in]	request	Non-blocking request to query.
[out]	attr	Filled with attributes of the request.

Returns

Error code as defined by ucs_status_t

ucp_am_send_nbx()

ucs_status_ptr_t ucp_am_send_nbx	(	ucp_ep_h	ep,
		unsigned	id,
		const void *	header,
		size_t	header_length,
		const void *	buffer,
		size_t	count,
		const ucp_request_param_t *	param )

This routine sends an Active Message to an ep. If the operation completes immediately, then the routine returns NULL and the callback function is ignored, even if specified. Otherwise, if no error is reported and a callback is requested (i.e. the UCP_OP_ATTR_FIELD_CALLBACK flag is set in the op_attr_mask field of param), then the UCP library will schedule invocation of the callback routine param->cb.send upon completion of the operation.

Note

If UCP_OP_ATTR_FLAG_NO_IMM_CMPL flag is set in the op_attr_mask field of param, then the operation will return a request handle, even if it completes immediately.

This operation supports specific flags, which can be passed in param by ucp_request_param_t::flags. The exact set of flags is defined by ucp_send_am_flags.

Parameters

[in]	ep	UCP endpoint where the Active Message will be run.
[in]	id	Active Message id. Specifies which registered callback to run.
[in]	header	User defined Active Message header. NULL value is allowed if no header needed. In this case header_length must be set to 0. By default the header must be valid until the active message send operation completes. If the flag UCP_AM_SEND_FLAG_COPY_HEADER is specified, the header is only required to be valid until this function call returns.
[in]	header_length	Active message header length in bytes.
[in]	buffer	Pointer to the data to be sent to the target node of the Active Message.
[in]	count	Number of elements to send.
[in]	param	Operation parameters, see ucp_request_param_t.

Note

Sending only header without actual data is allowed and is recommended for transferring a latency-critical amount of data.

The maximum allowed header size can be obtained by querying worker attributes by the ucp_worker_query routine.

Returns

NULL - Active Message was sent immediately.

UCS_PTR_IS_ERR(_ptr) - Error sending Active Message.

otherwise - Operation was scheduled for send and can be completed at any point in time. The request handle is returned to the application in order to track progress of the message. If user request was not provided in param->request, the application is responsible for releasing the handle using ucp_request_free routine.

Examples

ucp_client_server.c.

ucp_am_recv_data_nbx()

ucs_status_ptr_t ucp_am_recv_data_nbx	(	ucp_worker_h	worker,
		void *	data_desc,
		void *	buffer,
		size_t	count,
		const ucp_request_param_t *	param )

This routine receives a message that is described by the data descriptor data_desc, local address buffer, size count and param parameters on the worker. The routine is non-blocking and therefore returns immediately. The receive operation is considered completed when the message is delivered to the buffer. If the receive operation cannot be started the routine returns an error.

Note

This routine can be performed on any valid data descriptor delivered in ucp_am_recv_callback_t. Data descriptor is considered to be valid if:

• It is a rendezvous request (UCP_AM_RECV_ATTR_FLAG_RNDV is set in ucp_am_recv_param_t::recv_attr) or
• It is a persistent data pointer (UCP_AM_RECV_ATTR_FLAG_DATA is set in ucp_am_recv_param_t::recv_attr). In this case receive operation may be needed to unpack data to device memory (for example GPU device) or some specific datatype.

After this call UCP takes ownership of data_desc descriptor, so there is no need to release it even if the operation fails. The routine returns a request handle instead, which can be used for tracking operation progress.

Parameters

[in]	worker	Worker that is used for the receive operation.
[in]	data_desc	Data descriptor, provided in ucp_am_recv_callback_t routine.
[in]	buffer	Pointer to the buffer to receive the data.
[in]	count	Number of elements to receive into buffer.
[in]	param	Operation parameters, see ucp_request_param_t.

Returns

NULL - The receive operation was completed immediately. In this case, if param->recv_info.length is specified in the param, the value to which it points is updated with the size of the received message.

UCS_PTR_IS_ERR(_ptr) - The receive operation failed.

otherwise - Receive operation was scheduled and can be completed at any point in time. The request handle is returned to the application in order to track operation progress. If user request was not provided in param->request, the application is responsible for releasing the handle using ucp_request_free routine.

Examples

ucp_client_server.c.

ucp_am_data_release()

void ucp_am_data_release	(	ucp_worker_h	worker,
		void *	data )

This routine releases data that persisted through an Active Message callback because that callback returned UCS_INPROGRESS.

Parameters

[in]	worker	Worker which received the Active Message.
[in]	data	Pointer to data that was passed into the Active Message callback as the data parameter.

ucp_stream_send_nbx()

ucs_status_ptr_t ucp_stream_send_nbx	(	ucp_ep_h	ep,
		const void *	buffer,
		size_t	count,
		const ucp_request_param_t *	param )

This routine sends data that is described by the local address buffer, size count object to the destination endpoint ep. The routine is non-blocking and therefore returns immediately, however the actual send operation may be delayed. The send operation is considered completed when it is safe to reuse the source buffer. If the send operation is completed immediately the routine returns UCS_OK.

Note

The user should not modify any part of the buffer after this operation is called, until the operation completes.

Parameters

[in]	ep	Destination endpoint handle.
[in]	buffer	Pointer to the message buffer (payload).
[in]	count	Number of elements to send.
[in]	param	Operation parameters, see ucp_request_param_t.

Returns

NULL - The send operation was completed immediately.

UCS_PTR_IS_ERR(_ptr) - The send operation failed.

otherwise - Operation was scheduled for send and can be completed at any point in time. The request handle is returned to the application in order to track progress of the message.

Examples

ucp_client_server.c.

ucp_tag_send_nbx()

ucs_status_ptr_t ucp_tag_send_nbx	(	ucp_ep_h	ep,
		const void *	buffer,
		size_t	count,
		ucp_tag_t	tag,
		const ucp_request_param_t *	param )

This routine sends a messages that is described by the local address buffer, size count object to the destination endpoint ep. Each message is associated with a tag value that is used for message matching on the ucp_tag_recv_nb or receiver. The routine is non-blocking and therefore returns immediately, however the actual send operation may be delayed. The send operation is considered completed when it is safe to reuse the source buffer. If the send operation is completed immediately the routine returns UCS_OK and the call-back function is not invoked. If the operation is not completed immediately and no error reported then the UCP library will schedule to invoke the call-back whenever the send operation is completed. In other words, the completion of a message can be signaled by the return code or the call-back. Immediate completion signals can be fine-tuned via the ucp_request_param_t::op_attr_mask field in the ucp_request_param_t structure. The values of this field are a bit-wise OR of the ucp_op_attr_t enumeration.

Note

The user should not modify any part of the buffer after this operation is called, until the operation completes.

Parameters

[in]	ep	Destination endpoint handle.
[in]	buffer	Pointer to the message buffer (payload).
[in]	count	Number of elements to send
[in]	tag	Message tag.
[in]	param	Operation parameters, see ucp_request_param_t

Returns

UCS_OK - The send operation was completed immediately.

UCS_PTR_IS_ERR(_ptr) - The send operation failed.

otherwise - Operation was scheduled for send and can be completed in any point in time. The request handle is returned to the application in order to track progress of the message.

Examples

ucp_client_server.c, and ucp_hello_world.c.

ucp_tag_send_sync_nbx()

ucs_status_ptr_t ucp_tag_send_sync_nbx	(	ucp_ep_h	ep,
		const void *	buffer,
		size_t	count,
		ucp_tag_t	tag,
		const ucp_request_param_t *	param )

Same as ucp_tag_send_nbx, except the request completes only after there is a remote tag match on the message (which does not always mean the remote receive has been completed). This function never completes "in-place", and always returns a request handle.

Note

The user should not modify any part of the buffer after this operation is called, until the operation completes.

Returns UCS_ERR_UNSUPPORTED if UCP_ERR_HANDLING_MODE_PEER is enabled. This is a temporary implementation-related constraint that will be addressed in future releases.

Parameters

[in]	ep	Destination endpoint handle.
[in]	buffer	Pointer to the message buffer (payload).
[in]	count	Number of elements to send
[in]	tag	Message tag.
[in]	param	Operation parameters, see ucp_request_param_t

Returns

UCS_OK - The send operation was completed immediately.

UCS_PTR_IS_ERR(_ptr) - The send operation failed.

otherwise - Operation was scheduled for send and can be completed in any point in time. The request handle is returned to the application in order to track progress of the message.

ucp_stream_recv_nbx()

ucs_status_ptr_t ucp_stream_recv_nbx	(	ucp_ep_h	ep,
		void *	buffer,
		size_t	count,
		size_t *	length,
		const ucp_request_param_t *	param )

This routine receives data that is described by the local address buffer, size count object on the endpoint ep. The routine is non-blocking and therefore returns immediately. The receive operation is considered complete when the message is delivered to the buffer. If the receive operation cannot be started, then the routine returns an error.

Parameters

[in]	ep	UCP endpoint that is used for the receive operation.
[in]	buffer	Pointer to the buffer that will receive the data.
[in]	count	Number of elements to receive into buffer.
[out]	length	Size of the received data in bytes. The value is valid only if return code is NULL.
[in]	param	Operation parameters, see ucp_request_param_t. This operation supports specific flags, which can be passed in param by ucp_request_param_t::flags. The exact set of flags is defined by ucp_stream_recv_flags_t.

Returns

NULL - The receive operation was completed immediately. In this case the value pointed by length is updated by the size of received data. Note param->recv_info is not relevant for this function.

UCS_PTR_IS_ERR(_ptr) - The receive operation failed.

otherwise - Operation was scheduled for receive. A request handle is returned to the application in order to track progress of the operation.

Note

The amount of data received, in bytes, is always an integral multiple of the datatype size.

Examples

ucp_client_server.c.

ucp_stream_recv_data_nb()

ucs_status_ptr_t ucp_stream_recv_data_nb	(	ucp_ep_h	ep,
		size_t *	length )

This routine receives any available data from endpoint ep. Unlike ucp_stream_recv_nb, the returned data is unstructured and is treated as an array of bytes. If data is immediately available, UCS_STATUS_PTR(_ptr) is returned as a pointer to the data, and length is set to the size of the returned data buffer. The routine is non-blocking and therefore returns immediately.

Parameters

[in]	ep	UCP endpoint that is used for the receive operation.
[out]	length	Length of received data.

Returns

NULL - No received data available on the ep.

UCS_PTR_IS_ERR(_ptr) - the receive operation failed and UCS_PTR_STATUS(_ptr) indicates an error.

otherwise - The pointer to the data UCS_STATUS_PTR(_ptr) is returned to the application. After the data is processed, the application is responsible for releasing the data buffer by calling the ucp_stream_data_release routine.

Note

This function returns packed data (equivalent to ucp_dt_make_contig(1)).

This function returns a pointer to a UCP-supplied buffer, whereas ucp_stream_recv_nb places the data into a user-provided buffer. In some cases, receiving data directly into a UCP-supplied buffer can be more optimal, for example by processing the incoming data in-place and thus avoiding extra memory copy operations.

ucp_tag_recv_nbx()

ucs_status_ptr_t ucp_tag_recv_nbx	(	ucp_worker_h	worker,
		void *	buffer,
		size_t	count,
		ucp_tag_t	tag,
		ucp_tag_t	tag_mask,
		const ucp_request_param_t *	param )

This routine receives a message that is described by the local address buffer, size count, and info object on the worker. The tag value of the receive message has to match the tag and tag_mask values, where the tag_mask indicates what bits of the tag have to be matched. The routine is a non-blocking and therefore returns immediately. The receive operation is considered completed when the message is delivered to the buffer. In order to notify the application about completion of the receive operation the UCP library will invoke the call-back cb when the received message is in the receive buffer and ready for application access. If the receive operation cannot be started, then the routine returns an error.

Parameters

[in]	worker	UCP worker that is used for the receive operation.
[in]	buffer	Pointer to the buffer to receive the data.
[in]	count	Number of elements to receive
[in]	tag	Message tag to expect.
[in]	tag_mask	Bit mask that indicates the bits that are used for the matching of the incoming tag against the expected tag.
[in]	param	Operation parameters, see ucp_request_param_t

Returns

NULL - The receive operation was completed immediately. In this case, if param->recv_info.tag_info is specified in the param, the value to which it points is updated with the information about the received message.

UCS_PTR_IS_ERR(_ptr) - The receive operation failed.

otherwise - Operation was scheduled for receive. The request handle is returned to the application in order to track progress of the operation. The application is responsible for releasing the handle using ucp_request_free() routine.

Examples

ucp_client_server.c.

ucp_tag_probe_nb()

ucp_tag_message_h ucp_tag_probe_nb	(	ucp_worker_h	worker,
		ucp_tag_t	tag,
		ucp_tag_t	tag_mask,
		int	remove,
		ucp_tag_recv_info_t *	info )

This routine probes (checks) if a messages described by the tag and tag_mask was received (fully or partially) on the worker. The tag value of the received message has to match the tag and tag_mask values, where the tag_mask indicates what bits of the tag have to be matched. The function returns immediately and if the message is matched it returns a handle for the message.

Parameters

[in]	worker	UCP worker that is used for the probe operation.
[in]	tag	Message tag to probe for.
[in]	tag_mask	Bit mask that indicates the bits that are used for the matching of the incoming tag against the expected tag.
[in]	remove	The flag indicates if the matched message has to be removed from UCP library. If true (1), the message handle is removed from the UCP library and the application is responsible to call ucp_tag_msg_recv_nb() in order to receive the data and release the resources associated with the message handle. If false (0), the return value is merely an indication to whether a matching message is present, and it cannot be used in any other way, and in particular it cannot be passed to ucp_tag_msg_recv_nb().
[out]	info	If the matching message is found the descriptor is filled with the details about the message.

Returns

NULL - No match found.

Message handle (not NULL) - If message is matched the message handle is returned.

Note

This function does not advance the communication state of the network. If this routine is used in busy-poll mode, need to make sure ucp_worker_progress() is called periodically to extract messages from the transport.

Examples

ucp_hello_world.c.

ucp_tag_msg_recv_nbx()

ucs_status_ptr_t ucp_tag_msg_recv_nbx	(	ucp_worker_h	worker,
		void *	buffer,
		size_t	count,
		ucp_tag_message_h	message,
		const ucp_request_param_t *	param )

This routine receives a message that is described by the local address buffer, size count, and message handle on the worker. The message handle can be obtained by calling the ucp_tag_probe_nb() routine. The ucp_tag_msg_recv_nbx() routine is non-blocking and therefore returns immediately. The receive operation is considered completed when the message is delivered to the buffer. In order to notify the application about completion of the receive operation the UCP library will invoke the call-back cb when the received message is in the receive buffer and ready for application access. If the receive operation cannot be started, then the routine returns an error.

Parameters

[in]	worker	UCP worker that is used for the receive operation.
[in]	buffer	Pointer to the buffer that will receive the data.
[in]	count	Number of elements to receive
[in]	message	Message handle.
[in]	param	Operation parameters, see ucp_request_param_t

Returns

UCS_PTR_IS_ERR(_ptr) - The receive operation failed.

otherwise - Operation was scheduled for receive. The request handle is returned to the application in order to track progress of the operation. The application is responsible for releasing the handle using ucp_request_free() routine.

Examples

ucp_hello_world.c.

ucp_put_nbx()

ucs_status_ptr_t ucp_put_nbx	(	ucp_ep_h	ep,
		const void *	buffer,
		size_t	count,
		uint64_t	remote_addr,
		ucp_rkey_h	rkey,
		const ucp_request_param_t *	param )

This routine initiates a storage of contiguous block of data that is described by the local address buffer in the remote contiguous memory region described by remote_addr address and the memory handle rkey. The routine returns immediately and does not guarantee re-usability of the source address buffer. If the operation is completed immediately the routine return UCS_OK, otherwise UCS_INPROGRESS or an error is returned to user. If the put operation completes immediately, the routine returns UCS_OK and the call-back routine param.cb.send is not invoked. If the operation is not completed immediately and no error is reported, then the UCP library will schedule invocation of the call-back routine param.cb.send upon completion of the put operation. In other words, the completion of a put operation can be signaled by the return code or execution of the call-back. Immediate completion signals can be fine-tuned via the ucp_request_param_t::op_attr_mask field in the ucp_request_param_t structure. The values of this field are a bit-wise OR of the ucp_op_attr_t enumeration.

Note

The completion of a put operation signals the local buffer can be reused. The completion of the operation on the remote address requires use of ucp_worker_flush_nbx() or ucp_ep_flush_nbx(), after completion of which the data in remote_addr is guaranteed to be available.

Parameters

[in]	ep	Remote endpoint handle.
[in]	buffer	Pointer to the local source address.
[in]	count	Number of elements of type ucp_request_param_t::datatype to put. If ucp_request_param_t::datatype is not specified, the type defaults to ucp_dt_make_contig(1), which corresponds to byte elements.
[in]	remote_addr	Pointer to the destination remote memory address to write to.
[in]	rkey	Remote memory key associated with the remote memory address.
[in]	param	Operation parameters, see ucp_request_param_t

Returns

UCS_OK - The operation was completed immediately.

UCS_PTR_IS_ERR(_ptr) - The operation failed.

otherwise - Operation was scheduled and can be completed at any point in time. The request handle is returned to the application in order to track progress of the operation. The application is responsible for releasing the handle using ucp_request_free() routine.

Note

Only the datatype ucp_dt_make_contig(1) is supported for param->datatype, see ucp_dt_make_contig.

ucp_get_nbx()

ucs_status_ptr_t ucp_get_nbx	(	ucp_ep_h	ep,
		void *	buffer,
		size_t	count,
		uint64_t	remote_addr,
		ucp_rkey_h	rkey,
		const ucp_request_param_t *	param )

This routine initiates a load of a contiguous block of data that is described by the remote memory address remote_addr and the memory handle rkey in the local contiguous memory region described by buffer address. The routine returns immediately and does not guarantee that remote data is loaded and stored under the local address buffer. If the operation is completed immediately the routine return UCS_OK, otherwise UCS_INPROGRESS or an error is returned to user. If the get operation completes immediately, the routine returns UCS_OK and the call-back routine param.cb.send is not invoked. If the operation is not completed immediately and no error is reported, then the UCP library will schedule invocation of the call-back routine param.cb.send upon completion of the get operation. In other words, the completion of a get operation can be signaled by the return code or execution of the call-back.

Note

A user can use ucp_worker_flush_nb() in order to guarantee re-usability of the source address buffer.

The completion of a get operation signals the local buffer holds the the expected data and that both local buffer and remote remote_addr are safe to be reused, unlike with ucp_put_nbx where the use of ucp_worker_flush_nbx() or ucp_ep_flush_nbx() is required before the remote data is available.

Parameters

[in]	ep	Remote endpoint handle.
[in]	buffer	Pointer to the local destination address.
[in]	count	Number of elements of type ucp_request_param_t::datatype to put. If ucp_request_param_t::datatype is not specified, the type defaults to ucp_dt_make_contig(1), which corresponds to byte elements.
[in]	remote_addr	Pointer to the source remote memory address to read from.
[in]	rkey	Remote memory key associated with the remote memory address.
[in]	param	Operation parameters, see ucp_request_param_t.

Returns

UCS_OK - The operation was completed immediately.

UCS_PTR_IS_ERR(_ptr) - The operation failed.

otherwise - Operation was scheduled and can be completed at any point in time. The request handle is returned to the application in order to track progress of the operation. The application is responsible for releasing the handle using ucp_request_free() routine.

Note

Only the datatype ucp_dt_make_contig(1) is supported for param->datatype, see ucp_dt_make_contig.

ucp_atomic_op_nbx()

ucs_status_ptr_t ucp_atomic_op_nbx	(	ucp_ep_h	ep,
		ucp_atomic_op_t	opcode,
		const void *	buffer,
		size_t	count,
		uint64_t	remote_addr,
		ucp_rkey_h	rkey,
		const ucp_request_param_t *	param )

This routine will post an atomic operation to remote memory. The remote value is described by the combination of the remote memory address remote_addr and the remote memory handle rkey. The routine is non-blocking and therefore returns immediately. However, the actual atomic operation may be delayed. In order to enable fetching semantics for atomic operations user has to specify param.reply_buffer. Please see table below for more details.

Note

The user should not modify any part of the buffer (or also param->reply_buffer for fetch operations), until the operation completes.

Only ucp_dt_make_config(4) and ucp_dt_make_contig(8) are supported in param->datatype, see ucp_dt_make_contig. Also, currently atomic operations can handle one element only. Thus, count argument must be set to 1.

Atomic Operations Semantic

Atomic Operation	Pseudo code	X	Y	Z	Result
UCP_ATOMIC_OP_ADD	Result=Y; Y+=X	buffer	remote_addr	-	param.reply_buffer(optional)
UCP_ATOMIC_OP_SWAP	Result=Y; Y=X	buffer	remote_addr	-	param.reply_buffer
UCP_ATOMIC_OP_CSWAP	Result=Y; if (X==Y) then Y=Z	buffer	remote_addr	param.reply_buffer	param.reply_buffer
UCP_ATOMIC_OP_AND	Result=Y; Y&=X	buffer	remote_addr	-	param.reply_buffer(optional)
UCP_ATOMIC_OP_OR	Result=Y; Y|=X	buffer	remote_addr	-	param.reply_buffer(optional)
UCP_ATOMIC_OP_XOR	Result=Y; Y^=X	buffer	remote_addr	-	param.reply_buffer(optional)

Parameters

[in]	ep	UCP endpoint.
[in]	opcode	One of ucp_atomic_op_t.
[in]	buffer	Address of operand for the atomic operation. See Atomic Operations Semantic table for exact usage by different atomic operations.
[in]	count	Number of elements in buffer and result. The size of each element is specified by ucp_request_param_t::datatype
[in]	remote_addr	Remote address to operate on.
[in]	rkey	Remote key handle for the remote memory address.
[in]	param	Operation parameters, see ucp_request_param_t.

Returns

NULL - The operation completed immediately.

UCS_PTR_IS_ERR(_ptr) - The operation failed.

otherwise - Operation was scheduled and can be completed at some time in the future. The request handle is returned to the application in order to track progress of the operation.

ucp_request_check_status()

ucs_status_t ucp_request_check_status

(

void *

request

)

This routine checks the state of the request and returns its current status. Any value different from UCS_INPROGRESS means that request is in a completed state.

Parameters

[in]

request

Non-blocking request to check.

Returns

Error code as defined by ucs_status_t

Examples

ucp_client_server.c, and ucp_hello_world.c.

ucp_tag_recv_request_test()

ucs_status_t ucp_tag_recv_request_test	(	void *	request,
		ucp_tag_recv_info_t *	info )

This routine checks the state and returns current status of the request returned from ucp_tag_recv_nb routine or the user allocated request for ucp_tag_recv_nbr. Any value different from UCS_INPROGRESS means that the request is in a completed state.

Parameters

[in]	request	Non-blocking request to check.
[out]	info	It is filled with the details about the message available at the moment of calling.

Returns

Error code as defined by ucs_status_t

ucp_stream_recv_request_test()

ucs_status_t ucp_stream_recv_request_test	(	void *	request,
		size_t *	length_p )

This routine checks the state and returns current status of the request returned from ucp_stream_recv_nb routine. Any value different from UCS_INPROGRESS means that the request is in a completed state.

Parameters

[in]	request	Non-blocking request to check.
[out]	length_p	The size of the received data in bytes. This value is only valid if the status is UCS_OK. If valid, it is always an integral multiple of the datatype size associated with the request.

Returns

Error code as defined by ucs_status_t

ucp_request_cancel()

void ucp_request_cancel	(	ucp_worker_h	worker,
		void *	request )

Parameters

[in]	worker	UCP worker.
[in]	request	Non-blocking request to cancel.

This routine tries to cancels an outstanding communication request. After calling this routine, the request will be in completed or canceled (but not both) state regardless of the status of the target endpoint associated with the communication request. If the request is completed successfully, the send or receive completion callbacks (based on the type of the request) will be called with the status argument of the callback set to UCS_OK, and in a case it is canceled the status argument is set to UCS_ERR_CANCELED. It is important to note that in order to release the request back to the library the application is responsible for calling ucp_request_free().

ucp_stream_data_release()

void ucp_stream_data_release	(	ucp_ep_h	ep,
		void *	data )

Parameters

[in]	ep	Endpoint data received from.
[in]	data	Data pointer to release, which was returned from ucp_stream_recv_data_nb.

This routine releases internal UCP data buffer returned by ucp_stream_recv_data_nb when data is processed, the application can't use this buffer after calling this function.

ucp_request_free()

void ucp_request_free

(

void *

request

)

Parameters

[in]

request

Non-blocking request to release.

This routine releases the non-blocking request back to the library, regardless of its current state. Communications operations associated with this request will make progress internally, however no further notifications or callbacks will be invoked for this request.

Examples

ucp_client_server.c, and ucp_hello_world.c.

ucp_request_alloc()

void * ucp_request_alloc

(

ucp_worker_h

worker

)

Parameters

[in]

worker

UCP worker.

Returns

Error code as defined by ucs_status_t

This routine creates request which may be used in functions ucp_tag_send_nbx, ucp_tag_recv_nbx, etc. The application is responsible for releasing the handle using the ucp_request_free routine

ucp_request_is_completed()

int ucp_request_is_completed

(

void *

request

)

Deprecated

Replaced by ucp_request_test.

ucp_put()

ucs_status_t ucp_put	(	ucp_ep_h	ep,
		const void *	buffer,
		size_t	length,
		uint64_t	remote_addr,
		ucp_rkey_h	rkey )

Deprecated

Replaced by ucp_put_nb. The following example implements the same functionality using ucp_put_nb :

void empty_callback(void *request, ucs_status_t status)
{
}
 
ucs_status_t put(ucp_ep_h ep, const void *buffer, size_t length,
                  uint64_t remote_addr, ucp_rkey_h rkey)
{
    void *request = ucp_put_nb(ep, buffer, length, remote_addr, rkey,
                               empty_callback),
    if (request == NULL) {
        return UCS_OK;
    } else if (UCS_PTR_IS_ERR(request)) {
        return UCS_PTR_STATUS(request);
    } else {
        ucs_status_t status;
        do {
            ucp_worker_progress(worker);
            status = ucp_request_check_status(request);
        } while (status == UCS_INPROGRESS);
        ucp_request_release(request);
        return status;
    }
}

This routine stores contiguous block of data that is described by the local address buffer in the remote contiguous memory region described by remote_addr address and the memory handle rkey. The routine returns when it is safe to reuse the source address buffer.

Parameters

[in]	ep	Remote endpoint handle.
[in]	buffer	Pointer to the local source address.
[in]	length	Length of the data (in bytes) stored under the source address.
[in]	remote_addr	Pointer to the destination remote address to write to.
[in]	rkey	Remote memory key associated with the remote address.

Returns

Error code as defined by ucs_status_t

ucp_get()

ucs_status_t ucp_get	(	ucp_ep_h	ep,
		void *	buffer,
		size_t	length,
		uint64_t	remote_addr,
		ucp_rkey_h	rkey )

Deprecated

Replaced by ucp_get_nb.

See also

ucp_put.

This routine loads contiguous block of data that is described by the remote address remote_addr and the memory handle rkey in the local contiguous memory region described by buffer address. The routine returns when remote data is loaded and stored under the local address buffer.

Parameters

[in]	ep	Remote endpoint handle.
[in]	buffer	Pointer to the local source address.
[in]	length	Length of the data (in bytes) stored under the source address.
[in]	remote_addr	Pointer to the destination remote address to write to.
[in]	rkey	Remote memory key associated with the remote address.

Returns

Error code as defined by ucs_status_t

ucp_atomic_add32()

ucs_status_t ucp_atomic_add32	(	ucp_ep_h	ep,
		uint32_t	add,
		uint64_t	remote_addr,
		ucp_rkey_h	rkey )

Deprecated

Replaced by ucp_atomic_post with opcode UCP_ATOMIC_POST_OP_ADD.

See also

ucp_put.

This routine performs an add operation on a 32 bit integer value atomically. The remote integer value is described by the combination of the remote memory address remote_addr and the remote memory handle rkey. The add value is the value that is used for the add operation. When the operation completes the sum of the original remote value and the operand value (add) is stored in remote memory. The call to the routine returns immediately, independent of operation completion.

Note

The remote address must be aligned to 32 bit.

Parameters

[in]	ep	Remote endpoint handle.
[in]	add	Value to add.
[in]	remote_addr	Pointer to the destination remote address of the atomic variable.
[in]	rkey	Remote memory key associated with the remote address.

Returns

Error code as defined by ucs_status_t

ucp_atomic_add64()

ucs_status_t ucp_atomic_add64	(	ucp_ep_h	ep,
		uint64_t	add,
		uint64_t	remote_addr,
		ucp_rkey_h	rkey )

Deprecated

Replaced by ucp_atomic_post with opcode UCP_ATOMIC_POST_OP_ADD.

See also

ucp_put.

This routine performs an add operation on a 64 bit integer value atomically. The remote integer value is described by the combination of the remote memory address remote_addr and the remote memory handle rkey. The add value is the value that is used for the add operation. When the operation completes the sum of the original remote value and the operand value (add) is stored in remote memory. The call to the routine returns immediately, independent of operation completion.

Note

The remote address must be aligned to 64 bit.

Parameters

[in]	ep	Remote endpoint handle.
[in]	add	Value to add.
[in]	remote_addr	Pointer to the destination remote address of the atomic variable.
[in]	rkey	Remote memory key associated with the remote address.

Returns

Error code as defined by ucs_status_t

ucp_atomic_fadd32()

ucs_status_t ucp_atomic_fadd32	(	ucp_ep_h	ep,
		uint32_t	add,
		uint64_t	remote_addr,
		ucp_rkey_h	rkey,
		uint32_t *	result )

Deprecated

Replaced by ucp_atomic_fetch_nb with opcode UCP_ATOMIC_FETCH_OP_FADD.

See also

ucp_put.

This routine performs an add operation on a 32 bit integer value atomically. The remote integer value is described by the combination of the remote memory address remote_addr and the remote memory handle rkey. The add value is the value that is used for the add operation. When the operation completes, the original remote value is stored in the local memory result, and the sum of the original remote value and the operand value is stored in remote memory. The call to the routine returns when the operation is completed and the result value is updated.

Note

The remote address must be aligned to 32 bit.

Parameters

[in]	ep	Remote endpoint handle.
[in]	add	Value to add.
[in]	remote_addr	Pointer to the destination remote address of the atomic variable.
[in]	rkey	Remote memory key associated with the remote address.
[out]	result	Pointer to the address that is used to store the previous value of the atomic variable described by the remote_addr

Returns

Error code as defined by ucs_status_t

ucp_atomic_fadd64()

ucs_status_t ucp_atomic_fadd64	(	ucp_ep_h	ep,
		uint64_t	add,
		uint64_t	remote_addr,
		ucp_rkey_h	rkey,
		uint64_t *	result )

Deprecated

Replaced by ucp_atomic_fetch_nb with opcode UCP_ATOMIC_FETCH_OP_FADD.

See also

ucp_put.

This routine performs an add operation on a 64 bit integer value atomically. The remote integer value is described by the combination of the remote memory address remote_addr and the remote memory handle rkey. The add value is the value that is used for the add operation. When the operation completes, the original remote value is stored in the local memory result, and the sum of the original remote value and the operand value is stored in remote memory. The call to the routine returns when the operation is completed and the result value is updated.

Note

The remote address must be aligned to 64 bit.

Parameters

[in]	ep	Remote endpoint handle.
[in]	add	Value to add.
[in]	remote_addr	Pointer to the destination remote address of the atomic variable.
[in]	rkey	Remote memory key associated with the remote address.
[out]	result	Pointer to the address that is used to store the previous value of the atomic variable described by the remote_addr

Returns

Error code as defined by ucs_status_t

ucp_atomic_swap32()

ucs_status_t ucp_atomic_swap32	(	ucp_ep_h	ep,
		uint32_t	swap,
		uint64_t	remote_addr,
		ucp_rkey_h	rkey,
		uint32_t *	result )

Deprecated

Replaced by ucp_atomic_fetch_nb with opcode UCP_ATOMIC_FETCH_OP_SWAP.

See also

ucp_put.

This routine swaps a 32 bit value between local and remote memory. The remote value is described by the combination of the remote memory address remote_addr and the remote memory handle rkey. The swap value is the value that is used for the swap operation. When the operation completes, the remote value is stored in the local memory result, and the operand value (swap) is stored in remote memory. The call to the routine returns when the operation is completed and the result value is updated.

Note

The remote address must be aligned to 32 bit.

Parameters

[in]	ep	Remote endpoint handle.
[in]	swap	Value to swap.
[in]	remote_addr	Pointer to the destination remote address of the atomic variable.
[in]	rkey	Remote memory key associated with the remote address.
[out]	result	Pointer to the address that is used to store the previous value of the atomic variable described by the remote_addr

Returns

Error code as defined by ucs_status_t

ucp_atomic_swap64()

ucs_status_t ucp_atomic_swap64	(	ucp_ep_h	ep,
		uint64_t	swap,
		uint64_t	remote_addr,
		ucp_rkey_h	rkey,
		uint64_t *	result )

Deprecated

Replaced by ucp_atomic_fetch_nb with opcode UCP_ATOMIC_FETCH_OP_SWAP.

See also

ucp_put.

This routine swaps a 64 bit value between local and remote memory. The remote value is described by the combination of the remote memory address remote_addr and the remote memory handle rkey. The swap value is the value that is used for the swap operation. When the operation completes, the remote value is stored in the local memory result, and the operand value (swap) is stored in remote memory. The call to the routine returns when the operation is completed and the result value is updated.

Note

The remote address must be aligned to 64 bit.

Parameters

[in]	ep	Remote endpoint handle.
[in]	swap	Value to swap.
[in]	remote_addr	Pointer to the destination remote address of the atomic variable.
[in]	rkey	Remote memory key associated with the remote address.
[out]	result	Pointer to the address that is used to store the previous value of the atomic variable described by the remote_addr

Returns

Error code as defined by ucs_status_t

ucp_atomic_cswap32()

ucs_status_t ucp_atomic_cswap32	(	ucp_ep_h	ep,
		uint32_t	compare,
		uint32_t	swap,
		uint64_t	remote_addr,
		ucp_rkey_h	rkey,
		uint32_t *	result )

Deprecated

Replaced by ucp_atomic_fetch_nb with opcode UCP_ATOMIC_FETCH_OP_CSWAP.

See also

ucp_put.

This routine conditionally swaps a 32 bit value between local and remote memory. The swap occurs only if the condition value (continue) is equal to the remote value, otherwise the remote memory is not modified. The remote value is described by the combination of the remote memory address remote_addr and the remote memory handle rkey. The swap value is the value that is used to update the remote memory if the condition is true. The call to the routine returns when the operation is completed and the result value is updated.

Note

The remote address must be aligned to 32 bit.

Parameters

[in]	ep	Remote endpoint handle.
[in]	compare	Value to compare to.
[in]	swap	Value to swap.
[in]	remote_addr	Pointer to the destination remote address of the atomic variable.
[in]	rkey	Remote memory key associated with the remote address.
[out]	result	Pointer to the address that is used to store the previous value of the atomic variable described by the remote_addr

Returns

Error code as defined by ucs_status_t

ucp_atomic_cswap64()

ucs_status_t ucp_atomic_cswap64	(	ucp_ep_h	ep,
		uint64_t	compare,
		uint64_t	swap,
		uint64_t	remote_addr,
		ucp_rkey_h	rkey,
		uint64_t *	result )

Deprecated

Replaced by ucp_atomic_fetch_nb with opcode UCP_ATOMIC_FETCH_OP_CSWAP.

See also

ucp_put.

This routine conditionally swaps a 64 bit value between local and remote memory. The swap occurs only if the condition value (continue) is equal to the remote value, otherwise the remote memory is not modified. The remote value is described by the combination of the remote memory address remote_addr and the remote memory handle rkey. The swap value is the value that is used to update the remote memory if the condition is true. The call to the routine returns when the operation is completed and the result value is updated.

Note

The remote address must be aligned to 64 bit.

Parameters

[in]	ep	Remote endpoint handle.
[in]	compare	Value to compare to.
[in]	swap	Value to swap.
[in]	remote_addr	Pointer to the destination remote address of the atomic variable.
[in]	rkey	Remote memory key associated with the remote address.
[out]	result	Pointer to the address that is used to store the previous value of the atomic variable described by the remote_addr

Returns

Error code as defined by ucs_status_t

ucp_am_send_nb()

ucs_status_ptr_t ucp_am_send_nb	(	ucp_ep_h	ep,
		uint16_t	id,
		const void *	buffer,
		size_t	count,
		ucp_datatype_t	datatype,
		ucp_send_callback_t	cb,
		unsigned	flags )

Deprecated

Use ucp_am_send_nbx instead.

This routine sends an Active Message to an ep. It does not support CUDA memory.

Parameters

[in]	ep	UCP endpoint where the Active Message will be run.
[in]	id	Active Message id. Specifies which registered callback to run.
[in]	buffer	Pointer to the data to be sent to the target node of the Active Message.
[in]	count	Number of elements to send.
[in]	datatype	Datatype descriptor for the elements in the buffer.
[in]	cb	Callback that is invoked upon completion of the data transfer if it is not completed immediately.
[in]	flags	Operation flags as defined by ucp_send_am_flags.

Returns

NULL Active Message was sent immediately.

UCS_PTR_IS_ERR(_ptr) Error sending Active Message.

otherwise Pointer to request, and Active Message is known to be completed after cb is run.

ucp_stream_send_nb()

ucs_status_ptr_t ucp_stream_send_nb	(	ucp_ep_h	ep,
		const void *	buffer,
		size_t	count,
		ucp_datatype_t	datatype,
		ucp_send_callback_t	cb,
		unsigned	flags )

Deprecated

Use ucp_stream_send_nbx instead.

This routine sends data that is described by the local address buffer, size count, and datatype object to the destination endpoint ep. The routine is non-blocking and therefore returns immediately, however the actual send operation may be delayed. The send operation is considered completed when it is safe to reuse the source buffer. If the send operation is completed immediately the routine returns UCS_OK and the callback function cb is not invoked. If the operation is not completed immediately and no error reported, then the UCP library will schedule invocation of the callback cb upon completion of the send operation. In other words, the completion of the operation will be signaled either by the return code or by the callback.

Note

The user should not modify any part of the buffer after this operation is called, until the operation completes.

Parameters

[in]	ep	Destination endpoint handle.
[in]	buffer	Pointer to the message buffer (payload).
[in]	count	Number of elements to send.
[in]	datatype	Datatype descriptor for the elements in the buffer.
[in]	cb	Callback function that is invoked whenever the send operation is completed. It is important to note that the callback is only invoked in the event that the operation cannot be completed in place.
[in]	flags	Reserved for future use.

Returns

NULL - The send operation was completed immediately.

UCS_PTR_IS_ERR(_ptr) - The send operation failed.

otherwise - Operation was scheduled for send and can be completed in any point in time. The request handle is returned to the application in order to track progress of the message. The application is responsible for releasing the handle using ucp_request_free routine.

ucp_stream_recv_nb()

ucs_status_ptr_t ucp_stream_recv_nb	(	ucp_ep_h	ep,
		void *	buffer,
		size_t	count,
		ucp_datatype_t	datatype,
		ucp_stream_recv_callback_t	cb,
		size_t *	length,
		unsigned	flags )

Deprecated

Use ucp_stream_recv_nbx instead.

This routine receives data that is described by the local address buffer, size count, and datatype object on the endpoint ep. The routine is non-blocking and therefore returns immediately. The receive operation is considered complete when the message is delivered to the buffer. If data is not immediately available, the operation will be scheduled for receive and a request handle will be returned. In order to notify the application about completion of a scheduled receive operation, the UCP library will invoke the call-back cb when data is in the receive buffer and ready for application access. If the receive operation cannot be started, the routine returns an error.

Parameters

[in]	ep	UCP endpoint that is used for the receive operation.
[in]	buffer	Pointer to the buffer to receive the data.
[in]	count	Number of elements to receive into buffer.
[in]	datatype	Datatype descriptor for the elements in the buffer.
[in]	cb	Callback function that is invoked whenever the receive operation is completed and the data is ready in the receive buffer. It is important to note that the call-back is only invoked in a case when the operation cannot be completed immediately.
[out]	length	Size of the received data in bytes. The value is valid only if return code is UCS_OK.

Note

The amount of data received, in bytes, is always an integral multiple of the datatype size.

Parameters

[in]

flags

Flags defined in ucp_stream_recv_flags_t.

Returns

NULL - The receive operation was completed immediately.

UCS_PTR_IS_ERR(_ptr) - The receive operation failed.

otherwise - Operation was scheduled for receive. A request handle is returned to the application in order to track progress of the operation. The application is responsible for releasing the handle by calling the ucp_request_free routine.

ucp_tag_send_nb()

ucs_status_ptr_t ucp_tag_send_nb	(	ucp_ep_h	ep,
		const void *	buffer,
		size_t	count,
		ucp_datatype_t	datatype,
		ucp_tag_t	tag,
		ucp_send_callback_t	cb )

Deprecated

Use ucp_tag_send_nbx instead.

This routine sends a messages that is described by the local address buffer, size count, and datatype object to the destination endpoint ep. Each message is associated with a tag value that is used for message matching on the receiver. The routine is non-blocking and therefore returns immediately, however the actual send operation may be delayed. The send operation is considered completed when it is safe to reuse the source buffer. If the send operation is completed immediately the routine return UCS_OK and the call-back function cb is not invoked. If the operation is not completed immediately and no error reported then the UCP library will schedule to invoke the call-back cb whenever the send operation will be completed. In other words, the completion of a message can be signaled by the return code or the call-back.

Note

The user should not modify any part of the buffer after this operation is called, until the operation completes.

Parameters

[in]	ep	Destination endpoint handle.
[in]	buffer	Pointer to the message buffer (payload).
[in]	count	Number of elements to send
[in]	datatype	Datatype descriptor for the elements in the buffer.
[in]	tag	Message tag.
[in]	cb	Callback function that is invoked whenever the send operation is completed. It is important to note that the call-back is only invoked in a case when the operation cannot be completed in place.

Returns

NULL - The send operation was completed immediately.

UCS_PTR_IS_ERR(_ptr) - The send operation failed.

otherwise - Operation was scheduled for send and can be completed in any point in time. The request handle is returned to the application in order to track progress of the message. The application is responsible for releasing the handle using ucp_request_free() routine.

ucp_tag_send_nbr()

ucs_status_t ucp_tag_send_nbr	(	ucp_ep_h	ep,
		const void *	buffer,
		size_t	count,
		ucp_datatype_t	datatype,
		ucp_tag_t	tag,
		void *	req )

Deprecated

Use ucp_tag_send_nbx with the flag UCP_OP_ATTR_FIELD_REQUEST instead.

This routine provides a convenient and efficient way to implement a blocking send pattern. It also completes requests faster than ucp_tag_send_nb() because:

• it always uses eager protocol to send data up to the rendezvous threshold.
• its rendezvous threshold is higher than the one used by the ucp_tag_send_nb(). The threshold is controlled by the UCX_SEND_NBR_RNDV_THRESH environment variable.
• its request handling is simpler. There is no callback and no need to allocate and free requests. In fact request can be allocated by caller on the stack.

This routine sends a messages that is described by the local address buffer, size count, and datatype object to the destination endpoint ep. Each message is associated with a tag value that is used for message matching on the receiver.

The routine is non-blocking and therefore returns immediately, however the actual send operation may be delayed. The send operation is considered completed when it is safe to reuse the source buffer. If the send operation is completed immediately the routine returns UCS_OK.

If the operation is not completed immediately and no error reported then the UCP library will fill a user provided req and return UCS_INPROGRESS status. In order to monitor completion of the operation ucp_request_check_status() should be used.

Following pseudo code implements a blocking send function:

MPI_send(...)
{
    char *request;
    ucs_status_t status;
 
    // allocate request on the stack
    // ucp_context_query() was used to get ucp_request_size
    request = alloca(ucp_request_size);
 
    // note: make sure that there is enough memory before the
    // request handle
    status = ucp_tag_send_nbr(ep, ..., request + ucp_request_size);
    if (status != UCS_INPROGRESS) {
        return status;
    }
 
    do {
        ucp_worker_progress(worker);
        status = ucp_request_check_status(request + ucp_request_size);
    } while (status == UCS_INPROGRESS);
 
    return status;
}

Note

The user should not modify any part of the buffer after this operation is called, until the operation completes.

Parameters

[in]	ep	Destination endpoint handle.
[in]	buffer	Pointer to the message buffer (payload).
[in]	count	Number of elements to send
[in]	datatype	Datatype descriptor for the elements in the buffer.
[in]	tag	Message tag.
[in]	req	Request handle allocated by the user. There should be at least UCP request size bytes of available space before the req. The size of UCP request can be obtained by ucp_context_query function.

Returns

UCS_OK - The send operation was completed immediately.

UCS_INPROGRESS - The send was not completed and is in progress. ucp_request_check_status() should be used to monitor req status.

Error code as defined by ucs_status_t

ucp_tag_send_sync_nb()

ucs_status_ptr_t ucp_tag_send_sync_nb	(	ucp_ep_h	ep,
		const void *	buffer,
		size_t	count,
		ucp_datatype_t	datatype,
		ucp_tag_t	tag,
		ucp_send_callback_t	cb )

Deprecated

Use ucp_tag_send_sync_nbx instead.

Same as ucp_tag_send_nb, except the request completes only after there is a remote tag match on the message (which does not always mean the remote receive has been completed). This function never completes "in-place", and always returns a request handle.

Note

The user should not modify any part of the buffer after this operation is called, until the operation completes.

Returns UCS_ERR_UNSUPPORTED if UCP_ERR_HANDLING_MODE_PEER is enabled. This is a temporary implementation-related constraint that will be addressed in future releases.

Parameters

[in]	ep	Destination endpoint handle.
[in]	buffer	Pointer to the message buffer (payload).
[in]	count	Number of elements to send
[in]	datatype	Datatype descriptor for the elements in the buffer.
[in]	tag	Message tag.
[in]	cb	Callback function that is invoked whenever the send operation is completed.

Returns

UCS_PTR_IS_ERR(_ptr) - The send operation failed.

otherwise - Operation was scheduled for send and can be completed in any point in time. The request handle is returned to the application in order to track progress of the message. The application is responsible for releasing the handle using ucp_request_free() routine.

ucp_tag_recv_nb()

ucs_status_ptr_t ucp_tag_recv_nb	(	ucp_worker_h	worker,
		void *	buffer,
		size_t	count,
		ucp_datatype_t	datatype,
		ucp_tag_t	tag,
		ucp_tag_t	tag_mask,
		ucp_tag_recv_callback_t	cb )

Deprecated

Use ucp_tag_recv_nbx instead.

This routine receives a message that is described by the local address buffer, size count, and datatype object on the worker. The tag value of the receive message has to match the tag and tag_mask values, where the tag_mask indicates which bits of the tag have to be matched. The routine is non-blocking and therefore returns immediately. The receive operation is considered completed when the message is delivered to the buffer. In order to notify the application about completion of the receive operation the UCP library will invoke the call-back cb when the received message is in the receive buffer and ready for application access. If the receive operation cannot be stated the routine returns an error.

Note

This routine cannot return UCS_OK. It always returns a request handle or an error.

Parameters

[in]	worker	UCP worker that is used for the receive operation.
[in]	buffer	Pointer to the buffer to receive the data.
[in]	count	Number of elements to receive
[in]	datatype	Datatype descriptor for the elements in the buffer.
[in]	tag	Message tag to expect.
[in]	tag_mask	Bit mask that indicates the bits that are used for the matching of the incoming tag against the expected tag.
[in]	cb	Callback function that is invoked whenever the receive operation is completed and the data is ready in the receive buffer.

Returns

UCS_PTR_IS_ERR(_ptr) - The receive operation failed.

otherwise - Operation was scheduled for receive. The request handle is returned to the application in order to track progress of the operation. The application is responsible for releasing the handle using ucp_request_free() routine.

ucp_tag_recv_nbr()

ucs_status_t ucp_tag_recv_nbr	(	ucp_worker_h	worker,
		void *	buffer,
		size_t	count,
		ucp_datatype_t	datatype,
		ucp_tag_t	tag,
		ucp_tag_t	tag_mask,
		void *	req )

Deprecated

Use ucp_tag_recv_nbx with the flag UCP_OP_ATTR_FIELD_REQUEST instead.

This routine receives a message that is described by the local address buffer, size count, and datatype object on the worker. The tag value of the receive message has to match the tag and tag_mask values, where the tag_mask indicates which bits of the tag have to be matched. The routine is non-blocking and therefore returns immediately. The receive operation is considered completed when the message is delivered to the buffer. In order to monitor completion of the operation ucp_request_check_status or ucp_tag_recv_request_test should be used.

Parameters

[in]	worker	UCP worker that is used for the receive operation.
[in]	buffer	Pointer to the buffer to receive the data.
[in]	count	Number of elements to receive
[in]	datatype	Datatype descriptor for the elements in the buffer.
[in]	tag	Message tag to expect.
[in]	tag_mask	Bit mask that indicates the bits that are used for the matching of the incoming tag against the expected tag.
[in]	req	Request handle allocated by the user. There should be at least UCP request size bytes of available space before the req. The size of UCP request can be obtained by ucp_context_query function.

Returns

Error code as defined by ucs_status_t

ucp_tag_msg_recv_nb()

ucs_status_ptr_t ucp_tag_msg_recv_nb	(	ucp_worker_h	worker,
		void *	buffer,
		size_t	count,
		ucp_datatype_t	datatype,
		ucp_tag_message_h	message,
		ucp_tag_recv_callback_t	cb )

Deprecated

Use ucp_tag_recv_nbx instead.

This routine receives a message that is described by the local address buffer, size count, message handle, and datatype object on the worker. The message handle can be obtained by calling the ucp_tag_probe_nb() routine. The ucp_tag_msg_recv_nb() routine is non-blocking and therefore returns immediately. The receive operation is considered completed when the message is delivered to the buffer. In order to notify the application about completion of the receive operation the UCP library will invoke the call-back cb when the received message is in the receive buffer and ready for application access. If the receive operation cannot be started the routine returns an error.

Parameters

[in]	worker	UCP worker that is used for the receive operation.
[in]	buffer	Pointer to the buffer that will receive the data.
[in]	count	Number of elements to receive
[in]	datatype	Datatype descriptor for the elements in the buffer.
[in]	message	Message handle.
[in]	cb	Callback function that is invoked whenever the receive operation is completed and the data is ready in the receive buffer.

Returns

UCS_PTR_IS_ERR(_ptr) - The receive operation failed.

otherwise - Operation was scheduled for receive. The request handle is returned to the application in order to track progress of the operation. The application is responsible for releasing the handle using ucp_request_free() routine.

ucp_put_nbi()

ucs_status_t ucp_put_nbi	(	ucp_ep_h	ep,
		const void *	buffer,
		size_t	length,
		uint64_t	remote_addr,
		ucp_rkey_h	rkey )

Deprecated

Use ucp_put_nbx without passing the flag UCP_OP_ATTR_FIELD_CALLBACK instead. If a request pointer is returned, release it immediately by ucp_request_free.

This routine initiates a storage of contiguous block of data that is described by the local address buffer in the remote contiguous memory region described by remote_addr address and the memoryhandle" @a rkey. The routine returns immediately and @b does @b not guarantee re-usability of the source address @e buffer. If the operation is completed immediately the routine return UCS_OK, otherwise UCS_INPROGRESS or an error is returned to user. @note A user can use @ref ucp_worker_flush_nb "ucp_worker_flush_nb()" in order to guarantee re-usability of the source address buffer.

Parameters

[in]	ep	Remote endpoint handle.
[in]	buffer	Pointer to the local source address.
[in]	length	Length of the data (in bytes) stored under the source address.
[in]	remote_addr	Pointer to the destination remote memory address to write to.
[in]	rkey	Remote memory key associated with the remote memory address.

Returns

Error code as defined by ucs_status_t

ucp_put_nb()

ucs_status_ptr_t ucp_put_nb	(	ucp_ep_h	ep,
		const void *	buffer,
		size_t	length,
		uint64_t	remote_addr,
		ucp_rkey_h	rkey,
		ucp_send_callback_t	cb )

Deprecated

Use ucp_put_nbx instead.

This routine initiates a storage of contiguous block of data that is described by the local address buffer in the remote contiguous memory region described by remote_addr address and the memoryhandle" @a rkey. The routine returns immediately and @b does @b not guarantee re-usability of the source address @e buffer. If the operation is completed immediately the routine return UCS_OK, otherwise UCS_INPROGRESS or an error is returned to user. If the put operation completes immediately, the routine returns UCS_OK and the call-back routine @a cb is @b not invoked. If the operation is @b not completed immediately and no error is reported, then the UCP library will schedule invocation of the call-back routine @a cb upon completion of the put operation. In other words, the completion of a put operation can be signaled by the return code or execution of the call-back. @note A user can use @ref ucp_worker_flush_nb "ucp_worker_flush_nb()" in order to guarantee re-usability of the source address @e buffer. @param [in] ep Remote endpoint handle. @param [in] buffer Pointer to the local source address. @param [in] length Length of the data (in bytes) stored under the source address. @param [in] remote_addr Pointer to the destination remote memory address to write to. @param [in] rkey Remote memory key associated with the remote memory address. @param [in] cb Call-back function that is invoked whenever the put operation is completed and the local buffer can be modified. Does not guarantee remote completion. @return NULL - The operation was completed immediately. @return UCS_PTR_IS_ERR(_ptr) - The operation failed. @return otherwise - Operation was scheduled and can be completed at any point in time. The request handle is returned to the application in order to track progress of the operation. The application is responsible for releasing the handle using @ref ucp_request_free "ucp_request_free()" routine.

ucp_get_nbi()

ucs_status_t ucp_get_nbi	(	ucp_ep_h	ep,
		void *	buffer,
		size_t	length,
		uint64_t	remote_addr,
		ucp_rkey_h	rkey )

Deprecated

Use ucp_get_nbx without passing the flag UCP_OP_ATTR_FIELD_CALLBACK instead. If a request pointer is returned, release it immediately by ucp_request_free.

This routine initiate a load of contiguous block of data that is described by the remote memory address remote_addr and the memory handle rkey in the local contiguous memory region described by buffer address. The routine returns immediately and does not guarantee that remote data is loaded and stored under the local address buffer.

Note

A user can use ucp_worker_flush_nb() in order guarantee that remote data is loaded and stored under the local address buffer.

Parameters

[in]	ep	Remote endpoint handle.
[in]	buffer	Pointer to the local destination address.
[in]	length	Length of the data (in bytes) stored under the destination address.
[in]	remote_addr	Pointer to the source remote memory address to read from.
[in]	rkey	Remote memory key associated with the remote memory address.

Returns

Error code as defined by ucs_status_t

ucp_get_nb()

ucs_status_ptr_t ucp_get_nb	(	ucp_ep_h	ep,
		void *	buffer,
		size_t	length,
		uint64_t	remote_addr,
		ucp_rkey_h	rkey,
		ucp_send_callback_t	cb )

Deprecated

Use ucp_get_nbx instead.

This routine initiates a load of a contiguous block of data that is described by the remote memory address remote_addr and the memory handle rkey in the local contiguous memory region described by buffer address. The routine returns immediately and does not guarantee that remote data is loaded and stored under the local address buffer. If the operation is completed immediately the routine return UCS_OK, otherwise UCS_INPROGRESS or an error is returned to user. If the get operation completes immediately, the routine returns UCS_OK and the call-back routine cb is not invoked. If the operation is not completed immediately and no error is reported, then the UCP library will schedule invocation of the call-back routine cb upon completion of the get operation. In other words, the completion of a get operation can be signaled by the return code or execution of the call-back.

Note

A user can use ucp_worker_flush_nb() in order to guarantee re-usability of the source address buffer.

Parameters

[in]	ep	Remote endpoint handle.
[in]	buffer	Pointer to the local destination address.
[in]	length	Length of the data (in bytes) stored under the destination address.
[in]	remote_addr	Pointer to the source remote memory address to read from.
[in]	rkey	Remote memory key associated with the remote memory address.
[in]	cb	Call-back function that is invoked whenever the get operation is completed and the data is visible to the local process.

Returns

NULL - The operation was completed immediately.

UCS_PTR_IS_ERR(_ptr) - The operation failed.

otherwise - Operation was scheduled and can be completed at any point in time. The request handle is returned to the application in order to track progress of the operation. The application is responsible for releasing the handle using ucp_request_free() routine.

ucp_atomic_post()

ucs_status_t ucp_atomic_post	(	ucp_ep_h	ep,
		ucp_atomic_post_op_t	opcode,
		uint64_t	value,
		size_t	op_size,
		uint64_t	remote_addr,
		ucp_rkey_h	rkey )

Deprecated

Use ucp_atomic_op_nbx without the flag UCP_OP_ATTR_FIELD_REPLY_BUFFER instead.

This routine posts an atomic memory operation to a remote value. The remote value is described by the combination of the remote memory address remote_addr and the remote memory handle rkey. Return from the function does not guarantee completion. A user must call ucp_ep_flush_nb or ucp_worker_flush_nb to guarantee that the remote value has been updated.

Parameters

[in]	ep	UCP endpoint.
[in]	opcode	One of ucp_atomic_post_op_t.
[in]	value	Source operand for the atomic operation.
[in]	op_size	Size of value in bytes
[in]	remote_addr	Remote address to operate on.
[in]	rkey	Remote key handle for the remote memory address.

Returns

Error code as defined by ucs_status_t

ucp_atomic_fetch_nb()

ucs_status_ptr_t ucp_atomic_fetch_nb	(	ucp_ep_h	ep,
		ucp_atomic_fetch_op_t	opcode,
		uint64_t	value,
		void *	result,
		size_t	op_size,
		uint64_t	remote_addr,
		ucp_rkey_h	rkey,
		ucp_send_callback_t	cb )

Deprecated

Use ucp_atomic_op_nbx with the flag UCP_OP_ATTR_FIELD_REPLY_BUFFER instead.

This routine will post an atomic fetch operation to remote memory. The remote value is described by the combination of the remote memory address remote_addr and the remote memory handle rkey. The routine is non-blocking and therefore returns immediately. However the actual atomic operation may be delayed. The atomic operation is not considered complete until the values in remote and local memory are completed. If the atomic operation completes immediately, the routine returns UCS_OK and the call-back routine cb is not invoked. If the operation is not completed immediately and no error is reported, then the UCP library will schedule invocation of the call-back routine cb upon completion of the atomic operation. In other words, the completion of an atomic operation can be signaled by the return code or execution of the call-back.

Note

The user should not modify any part of the result after this operation is called, until the operation completes.

Parameters

[in]	ep	UCP endpoint.
[in]	opcode	One of ucp_atomic_fetch_op_t.
[in]	value	Source operand for atomic operation. In the case of CSWAP this is the conditional for the swap. For SWAP this is the value to be placed in remote memory.
[in,out]	result	Local memory address to store resulting fetch to. In the case of CSWAP the value in result will be swapped into the remote_addr if the condition is true.
[in]	op_size	Size of value in bytes and pointer type for result
[in]	remote_addr	Remote address to operate on.
[in]	rkey	Remote key handle for the remote memory address.
[in]	cb	Call-back function that is invoked whenever the send operation is completed. It is important to note that the call-back function is only invoked in a case when the operation cannot be completed in place.

Returns

NULL - The operation was completed immediately.

UCS_PTR_IS_ERR(_ptr) - The operation failed.

otherwise - Operation was scheduled and can be completed at any point in time. The request handle is returned to the application in order to track progress of the operation. The application is responsible for releasing the handle using ucp_request_free() routine.