UCP Worker

Data Structures
struct	ucp_worker_attr
	UCP worker attributes. More...
struct	ucp_worker_params
	Tuning parameters for the UCP worker. More...
struct	ucp_worker_address_attr
	UCP worker address attributes. More...
struct	ucp_listener_attr
	UCP listener attributes. More...
struct	ucp_conn_request_attr
	UCP listener's connection request attributes. More...
struct	ucp_listener_params
	Parameters for a UCP listener object. More...
struct	ucp_am_handler_param
	Active Message handler parameters passed to ucp_worker_set_am_recv_handler routine. More...
struct	ucp_am_recv_param
	Operation parameters provided in ucp_am_recv_callback_t callback. More...
struct	ucp_listener_accept_handler
struct	ucp_listener_conn_handler
	UCP callback to handle the connection request in a client-server connection establishment flow. More...

Typedefs
typedef struct ucp_worker_attr	ucp_worker_attr_t
	UCP worker attributes.
typedef struct ucp_worker_params	ucp_worker_params_t
	Tuning parameters for the UCP worker.
typedef struct ucp_worker_address_attr	ucp_worker_address_attr_t
	UCP worker address attributes.
typedef struct ucp_listener_attr	ucp_listener_attr_t
	UCP listener attributes.
typedef struct ucp_conn_request_attr	ucp_conn_request_attr_t
	UCP listener's connection request attributes.
typedef struct ucp_listener_params	ucp_listener_params_t
	Parameters for a UCP listener object.
typedef struct ucp_am_handler_param	ucp_am_handler_param_t
	Active Message handler parameters passed to ucp_worker_set_am_recv_handler routine.
typedef struct ucp_listener_accept_handler	ucp_listener_accept_handler_t
typedef struct ucp_am_recv_param	ucp_am_recv_param_t
	Operation parameters provided in ucp_am_recv_callback_t callback.
typedef struct ucp_address	ucp_address_t
	UCP worker address.
typedef struct ucp_listener *	ucp_listener_h
	UCP listen handle.
typedef struct ucp_worker *	ucp_worker_h
	UCP Worker.
typedef void(*	ucp_listener_accept_callback_t) (ucp_ep_h ep, void *arg)
	A callback for accepting client/server connections on a listener ucp_listener_h.
typedef void(*	ucp_listener_conn_callback_t) (ucp_conn_request_h conn_request, void *arg)
	A callback for handling of incoming connection request conn_request from a client.
typedef struct ucp_listener_conn_handler	ucp_listener_conn_handler_t
	UCP callback to handle the connection request in a client-server connection establishment flow.
typedef enum ucp_wakeup_event_types	ucp_wakeup_event_t
	UCP worker wakeup events mask.

Enumerations
enum	ucp_worker_params_field {   UCP_WORKER_PARAM_FIELD_THREAD_MODE = UCS_BIT(0) , UCP_WORKER_PARAM_FIELD_CPU_MASK = UCS_BIT(1) , UCP_WORKER_PARAM_FIELD_EVENTS = UCS_BIT(2) , UCP_WORKER_PARAM_FIELD_USER_DATA = UCS_BIT(3) ,   UCP_WORKER_PARAM_FIELD_EVENT_FD = UCS_BIT(4) , UCP_WORKER_PARAM_FIELD_FLAGS = UCS_BIT(5) , UCP_WORKER_PARAM_FIELD_NAME = UCS_BIT(6) , UCP_WORKER_PARAM_FIELD_AM_ALIGNMENT = UCS_BIT(7) ,   UCP_WORKER_PARAM_FIELD_CLIENT_ID = UCS_BIT(8) }
	UCP worker parameters field mask. More...
enum	ucp_worker_flags_t { UCP_WORKER_FLAG_IGNORE_REQUEST_LEAK = UCS_BIT(0) }
	UCP worker flags. More...
enum	ucp_listener_params_field { UCP_LISTENER_PARAM_FIELD_SOCK_ADDR = UCS_BIT(0) , UCP_LISTENER_PARAM_FIELD_ACCEPT_HANDLER = UCS_BIT(1) , UCP_LISTENER_PARAM_FIELD_CONN_HANDLER = UCS_BIT(2) }
	UCP listener parameters field mask. More...
enum	ucp_worker_address_flags_t { UCP_WORKER_ADDRESS_FLAG_NET_ONLY = UCS_BIT(0) }
	UCP worker address flags. More...
enum	ucp_worker_attr_field {   UCP_WORKER_ATTR_FIELD_THREAD_MODE = UCS_BIT(0) , UCP_WORKER_ATTR_FIELD_ADDRESS = UCS_BIT(1) , UCP_WORKER_ATTR_FIELD_ADDRESS_FLAGS = UCS_BIT(2) , UCP_WORKER_ATTR_FIELD_MAX_AM_HEADER = UCS_BIT(3) ,   UCP_WORKER_ATTR_FIELD_NAME = UCS_BIT(4) , UCP_WORKER_ATTR_FIELD_MAX_INFO_STRING = UCS_BIT(5) }
	UCP worker attributes field mask. More...
enum	ucp_worker_address_attr_field { UCP_WORKER_ADDRESS_ATTR_FIELD_UID = UCS_BIT(0) }
	UCP worker address attributes field mask. More...
enum	ucp_listener_attr_field { UCP_LISTENER_ATTR_FIELD_SOCKADDR = UCS_BIT(0) }
	UCP listener attributes field mask. More...
enum	ucp_conn_request_attr_field { UCP_CONN_REQUEST_ATTR_FIELD_CLIENT_ADDR = UCS_BIT(0) , UCP_CONN_REQUEST_ATTR_FIELD_CLIENT_ID = UCS_BIT(1) }
	UCP listener's connection request attributes field mask. More...
enum	ucp_am_cb_flags { UCP_AM_FLAG_WHOLE_MSG = UCS_BIT(0) , UCP_AM_FLAG_PERSISTENT_DATA = UCS_BIT(1) }
	Flags for a UCP Active Message callback. More...
enum	ucp_send_am_flags {   UCP_AM_SEND_FLAG_REPLY = UCS_BIT(0) , UCP_AM_SEND_FLAG_EAGER = UCS_BIT(1) , UCP_AM_SEND_FLAG_RNDV = UCS_BIT(2) , UCP_AM_SEND_FLAG_COPY_HEADER = UCS_BIT(3) ,   UCP_AM_SEND_REPLY = UCP_AM_SEND_FLAG_REPLY }
	Flags for sending a UCP Active Message. More...
enum	ucp_wakeup_event_types {   UCP_WAKEUP_RMA = UCS_BIT(0) , UCP_WAKEUP_AMO = UCS_BIT(1) , UCP_WAKEUP_TAG_SEND = UCS_BIT(2) , UCP_WAKEUP_TAG_RECV = UCS_BIT(3) ,   UCP_WAKEUP_TX = UCS_BIT(10) , UCP_WAKEUP_RX = UCS_BIT(11) , UCP_WAKEUP_EDGE = UCS_BIT(16) }
	UCP worker wakeup events mask. More...

Functions
ucs_status_t	ucp_worker_create (ucp_context_h context, const ucp_worker_params_t *params, ucp_worker_h *worker_p)
	Create a worker object.
void	ucp_worker_destroy (ucp_worker_h worker)
	Destroy a worker object.
ucs_status_t	ucp_worker_query (ucp_worker_h worker, ucp_worker_attr_t *attr)
	Get attributes specific to a particular worker.
void	ucp_worker_print_info (ucp_worker_h worker, FILE *stream)
	Print information about the worker.
void	ucp_worker_release_address (ucp_worker_h worker, ucp_address_t *address)
	Release an address of the worker object.
ucs_status_t	ucp_worker_address_query (ucp_address_t *address, ucp_worker_address_attr_t *attr)
	Get attributes of the particular worker address.
unsigned	ucp_worker_progress (ucp_worker_h worker)
	Progress all communications on a specific worker.
ssize_t	ucp_stream_worker_poll (ucp_worker_h worker, ucp_stream_poll_ep_t *poll_eps, size_t max_eps, unsigned flags)
	Poll for endpoints that are ready to consume streaming data.
ucs_status_t	ucp_listener_create (ucp_worker_h worker, const ucp_listener_params_t *params, ucp_listener_h *listener_p)
	Create a listener to accept connections on. Connection requests on the listener will arrive at a local address specified by the user.
void	ucp_listener_destroy (ucp_listener_h listener)
	Stop accepting connections on a local address of the worker object.
ucs_status_t	ucp_listener_query (ucp_listener_h listener, ucp_listener_attr_t *attr)
	Get attributes specific to a particular listener.
ucs_status_t	ucp_conn_request_query (ucp_conn_request_h conn_request, ucp_conn_request_attr_t *attr)
	Get attributes specific to a particular connection request received on the server side.
ucs_status_t	ucp_listener_reject (ucp_listener_h listener, ucp_conn_request_h conn_request)
	Reject an incoming connection request.
ucs_status_t	ucp_worker_set_am_recv_handler (ucp_worker_h worker, const ucp_am_handler_param_t *param)
	Add user defined callback for Active Message.
ucs_status_t	ucp_worker_fence (ucp_worker_h worker)
	Assures ordering between non-blocking operations.
ucs_status_ptr_t	ucp_worker_flush_nbx (ucp_worker_h worker, const ucp_request_param_t *param)
	Flush outstanding AMO and RMA operations on the worker.
ucs_status_t	ucp_worker_flush (ucp_worker_h worker)
	Flush outstanding AMO and RMA operations on the worker.
ucs_status_t	ucp_worker_get_address (ucp_worker_h worker, ucp_address_t **address_p, size_t *address_length_p)
	Get the address of the worker object.
ucs_status_t	ucp_worker_set_am_handler (ucp_worker_h worker, uint16_t id, ucp_am_callback_t cb, void *arg, uint32_t flags)
	Add user defined callback for Active Message.
ucs_status_ptr_t	ucp_worker_flush_nb (ucp_worker_h worker, unsigned flags, ucp_send_callback_t cb)
	Flush outstanding AMO and RMA operations on the worker.

Detailed Description

UCP Worker routines

---

Data Structure Documentation

ucp_worker_attr

struct ucp_worker_attr

The structure defines the attributes which characterize the particular worker.

Examples

ucp_hello_world.c.

Data Fields
uint64_t	field_mask	Mask of valid fields in this structure, using bits from ucp_worker_attr_field. Fields not specified in this mask will be ignored. Provides ABI compatibility with respect to adding new fields.
ucs_thread_mode_t	thread_mode	Thread safe level of the worker.
uint32_t	address_flags	Flags indicating requested details of the worker address. If UCP_WORKER_ATTR_FIELD_ADDRESS_FLAGS bit is set in the field_mask, this value should be set as well. Possible flags are specified in ucp_worker_address_flags_t. Note This is an input attribute.
ucp_address_t *	address	Worker address, which can be passed to remote instances of the UCP library in order to connect to this worker. The memory for the address handle is allocated by ucp_worker_query() routine, and must be released by using ucp_worker_release_address() routine.
size_t	address_length	Size of worker address in bytes.
size_t	max_am_header	Maximum allowed header size for ucp_am_send_nbx routine.
char	name[UCP_ENTITY_NAME_MAX]	Tracing and analysis tools can identify the worker using this name.
size_t	max_debug_string	Maximum debug string size that can be filled with ucp_request_query.

ucp_worker_params

struct ucp_worker_params

The structure defines the parameters that are used for the UCP worker tuning during the UCP worker creation.

Examples

ucp_client_server.c, and ucp_hello_world.c.

Data Fields
uint64_t	field_mask	Mask of valid fields in this structure, using bits from ucp_worker_params_field. Fields not specified in this mask will be ignored. Provides ABI compatibility with respect to adding new fields.
ucs_thread_mode_t	thread_mode	The parameter thread_mode suggests the thread safety mode which worker and the associated resources should be created with. This is an optional parameter. The default value is UCS_THREAD_MODE_SINGLE and it is used when the value of the parameter is not set. When this parameter along with its corresponding bit in the field_mask - UCP_WORKER_PARAM_FIELD_THREAD_MODE is set, the ucp_worker_create attempts to create worker with this thread mode. The thread mode with which worker is created can differ from the suggested mode. The actual thread mode of the worker should be obtained using the query interface ucp_worker_query.
ucs_cpu_set_t	cpu_mask	Mask of which CPUs worker resources should preferably be allocated on. This value is optional. If it's not set (along with its corresponding bit in the field_mask - UCP_WORKER_PARAM_FIELD_CPU_MASK), resources are allocated according to system's default policy.
unsigned	events	Mask of events (ucp_wakeup_event_t) which are expected on wakeup. This value is optional. If it's not set (along with its corresponding bit in the field_mask - UCP_WORKER_PARAM_FIELD_EVENTS), all types of events will trigger on wakeup.
void *	user_data	User data associated with the current worker. This value is optional. If it's not set (along with its corresponding bit in the field_mask - UCP_WORKER_PARAM_FIELD_USER_DATA), it will default to NULL.
int	event_fd	External event file descriptor. This value is optional. If UCP_WORKER_PARAM_FIELD_EVENT_FD is set in the field_mask, events on the worker will be reported on the provided event file descriptor. In this case, calling ucp_worker_get_efd will result in an error. The provided file descriptor must be capable of aggregating notifications for arbitrary events, for example epoll(7) on Linux systems. user_data will be used as the event user-data on systems which support it. For example, on Linux, it will be placed in epoll_data_t::ptr, when returned from epoll_wait(2). Otherwise, events will be reported to the event file descriptor returned from ucp_worker_get_efd().
uint64_t	flags	Worker flags. This value is optional. If UCP_WORKER_PARAM_FIELD_FLAGS is not set in the field_mask, the value of this field will default to 0.
const char *	name	Tracing and analysis tools can identify the worker using this name. To retrieve the worker's name, use ucp_worker_query, as the name you supply may be changed by UCX under some circumstances, e.g. a name conflict. This field is only assigned if you set UCP_WORKER_PARAM_FIELD_NAME in the field mask. If not, then a default unique name will be created for you.
size_t	am_alignment	Minimal address alignment of the active message data pointer as passed in argument data to the active message handler, defined as ucp_am_recv_callback_t.
uint64_t	client_id	Client id that is sent as part of the connection request payload when connecting to a remote socket address. On the remote side, this value can be obtained from ucp_conn_request_h using ucp_conn_request_query.

ucp_worker_address_attr

struct ucp_worker_address_attr

The structure defines the attributes of the particular worker address.

Data Fields
uint64_t	field_mask	Mask of valid fields in this structure, using bits from ucp_worker_address_attr_field. Fields not specified in this mask will be ignored. Provides ABI compatibility with respect to adding new fields.
uint64_t	worker_uid	Unique id of the worker this address belongs to.

ucp_listener_attr

struct ucp_listener_attr

The structure defines the attributes which characterize the particular listener.

Examples

ucp_client_server.c.

Data Fields
uint64_t	field_mask	Mask of valid fields in this structure, using bits from ucp_listener_attr_field. Fields not specified in this mask will be ignored. Provides ABI compatibility with respect to adding new fields.
struct sockaddr_storage	sockaddr	Sockaddr on which this listener is listening for incoming connection requests.

ucp_conn_request_attr

struct ucp_conn_request_attr

The structure defines the attributes that characterize the particular connection request received on the server side.

Examples

ucp_client_server.c.

Data Fields
uint64_t	field_mask	Mask of valid fields in this structure, using bits from ucp_conn_request_attr_field. Fields not specified in this mask will be ignored. Provides ABI compatibility with respect to adding new fields.
struct sockaddr_storage	client_address	The address of the remote client that sent the connection request to the server.
uint64_t	client_id	Remote client id if remote endpoint's flag UCP_EP_PARAMS_FLAGS_SEND_CLIENT_ID is set.

ucp_listener_params

struct ucp_listener_params

This structure defines parameters for ucp_listener_create, which is used to listen for incoming client/server connections.

Examples

ucp_client_server.c.

Data Fields
uint64_t	field_mask	Mask of valid fields in this structure, using bits from ucp_listener_params_field. Fields not specified in this mask will be ignored. Provides ABI compatibility with respect to adding new fields.
ucs_sock_addr_t	sockaddr	An address in the form of a sockaddr. This field is mandatory for filling (along with its corresponding bit in the field_mask - UCP_LISTENER_PARAM_FIELD_SOCK_ADDR). The ucp_listener_create routine will return with an error if sockaddr is not specified.
ucp_listener_accept_handler_t	accept_handler	Handler to endpoint creation in a client-server connection flow. In order for the callback inside this handler to be invoked, the UCP_LISTENER_PARAM_FIELD_ACCEPT_HANDLER needs to be set in the field_mask.
ucp_listener_conn_handler_t	conn_handler	Handler of an incoming connection request in a client-server connection flow. In order for the callback inside this handler to be invoked, the UCP_LISTENER_PARAM_FIELD_CONN_HANDLER needs to be set in the field_mask. Note User is expected to call ucp_ep_create with set UCP_EP_PARAM_FIELD_CONN_REQUEST flag to ucp_ep_params_t::field_mask and ucp_ep_params_t::conn_request in order to be able to receive communications.

ucp_am_handler_param

struct ucp_am_handler_param

Examples

ucp_client_server.c.

Data Fields
uint64_t	field_mask	Mask of valid fields in this structure, using bits from ucp_am_handler_param_field. Fields not specified in this mask will be ignored. Provides ABI compatibility with respect to adding new fields.
unsigned	id	Active Message id.
uint32_t	flags	Handler flags as defined by ucp_am_cb_flags.
ucp_am_recv_callback_t	cb	Active Message callback. To clear the already set callback, this value should be set to NULL.
void *	arg	Active Message argument, which will be passed in to every invocation of ucp_am_recv_callback_t function as the arg argument.

ucp_am_recv_param

struct ucp_am_recv_param

Examples

ucp_client_server.c.

Data Fields
uint64_t	recv_attr	Mask of valid fields in this structure and receive operation flags, using bits from ucp_am_recv_attr_t. Fields not specified in this mask will be ignored. Provides ABI compatibility with respect to adding new fields.
ucp_ep_h	reply_ep	Endpoint, which can be used for the reply to this message.

ucp_listener_accept_handler

struct ucp_listener_accept_handler

Deprecated

Replaced by ucp_listener_conn_handler_t.

Data Fields
ucp_listener_accept_callback_t	cb	Endpoint creation callback
void *	arg	User defined argument for the callback

ucp_listener_conn_handler

struct ucp_listener_conn_handler

This structure is used for handling an incoming connection request on the listener. Setting this type of handler allows creating an endpoint on any other worker and not limited to the worker on which the listener was created.

Note

• Other than communication progress routines, it is allowed to call all other communication routines from the callback in the struct.
• The callback is thread safe with respect to the worker it is invoked on.
• It is the user's responsibility to avoid potential dead lock accessing different worker.

Data Fields
ucp_listener_conn_callback_t	cb	Connection request callback
void *	arg	User defined argument for the callback

Typedef Documentation

ucp_worker_attr_t

typedef struct ucp_worker_attr ucp_worker_attr_t

The structure defines the attributes which characterize the particular worker.

ucp_worker_params_t

typedef struct ucp_worker_params ucp_worker_params_t

The structure defines the parameters that are used for the UCP worker tuning during the UCP worker creation.

ucp_worker_address_attr_t

typedef struct ucp_worker_address_attr ucp_worker_address_attr_t

The structure defines the attributes of the particular worker address.

ucp_listener_attr_t

typedef struct ucp_listener_attr ucp_listener_attr_t

The structure defines the attributes which characterize the particular listener.

ucp_conn_request_attr_t

typedef struct ucp_conn_request_attr ucp_conn_request_attr_t

The structure defines the attributes that characterize the particular connection request received on the server side.

ucp_listener_params_t

typedef struct ucp_listener_params ucp_listener_params_t

This structure defines parameters for ucp_listener_create, which is used to listen for incoming client/server connections.

ucp_am_handler_param_t

typedef struct ucp_am_handler_param ucp_am_handler_param_t

ucp_listener_accept_handler_t

typedef struct ucp_listener_accept_handler ucp_listener_accept_handler_t

Deprecated

Replaced by ucp_listener_conn_handler_t.

ucp_am_recv_param_t

typedef struct ucp_am_recv_param ucp_am_recv_param_t

ucp_address_t

typedef struct ucp_address ucp_address_t

The address handle is an opaque object that is used as an identifier for a worker instance.

ucp_listener_h

typedef struct ucp_listener* ucp_listener_h

The listener handle is an opaque object that is used for listening on a specific address and accepting connections from clients.

ucp_worker_h

typedef struct ucp_worker* ucp_worker_h

UCP worker is an opaque object representing the communication context. The worker represents an instance of a local communication resource and the progress engine associated with it. The progress engine is a construct that is responsible for asynchronous and independent progress of communication directives. The progress engine could be implemented in hardware or software. The worker object abstracts an instance of network resources such as a host channel adapter port, network interface, or multiple resources such as multiple network interfaces or communication ports. It could also represent virtual communication resources that are defined across multiple devices. Although the worker can represent multiple network resources, it is associated with a single UCX application context. All communication functions require a context to perform the operation on the dedicated hardware resource(s) and an endpoint to address the destination.

Note

Worker are parallel "threading points" that an upper layer may use to optimize concurrent communications.

ucp_listener_accept_callback_t

typedef void(* ucp_listener_accept_callback_t) (ucp_ep_h ep, void *arg)

This callback routine is invoked on the server side upon creating a connection to a remote client. The user can pass an argument to this callback. The user is responsible for releasing the ep handle using the ucp_ep_destroy() routine.

Parameters

[in]	ep	Handle to a newly created endpoint which is connected to the remote peer which has initiated the connection.
[in]	arg	User's argument for the callback.

ucp_listener_conn_callback_t

typedef void(* ucp_listener_conn_callback_t) (ucp_conn_request_h conn_request, void *arg)

This callback routine is invoked on the server side to handle incoming connections from remote clients. The user can pass an argument to this callback. The conn_request handle has to be released, either by ucp_ep_create or ucp_listener_reject routine.

Parameters

[in]	conn_request	Connection request handle.
[in]	arg	User's argument for the callback.

ucp_listener_conn_handler_t

typedef struct ucp_listener_conn_handler ucp_listener_conn_handler_t

This structure is used for handling an incoming connection request on the listener. Setting this type of handler allows creating an endpoint on any other worker and not limited to the worker on which the listener was created.

Note

• Other than communication progress routines, it is allowed to call all other communication routines from the callback in the struct.
• The callback is thread safe with respect to the worker it is invoked on.
• It is the user's responsibility to avoid potential dead lock accessing different worker.

ucp_wakeup_event_t

typedef enum ucp_wakeup_event_types ucp_wakeup_event_t

The enumeration allows specifying which events are expected on wakeup. Empty events are possible for any type of event except for UCP_WAKEUP_TX and UCP_WAKEUP_RX.

Note

Send completions are reported by POLLIN-like events (see poll man page). Since outgoing operations can be initiated at any time, UCP does not generate POLLOUT-like events, although it must be noted that outgoing operations may be queued depending upon resource availability.

Enumeration Type Documentation

ucp_worker_params_field

enum ucp_worker_params_field

The enumeration allows specifying which fields in ucp_worker_params_t are present. It is used to enable backward compatibility support.

Enumerator
UCP_WORKER_PARAM_FIELD_THREAD_MODE	UCP thread mode
UCP_WORKER_PARAM_FIELD_CPU_MASK	Worker's CPU bitmap
UCP_WORKER_PARAM_FIELD_EVENTS	Worker's events bitmap
UCP_WORKER_PARAM_FIELD_USER_DATA	User data
UCP_WORKER_PARAM_FIELD_EVENT_FD	External event file descriptor
UCP_WORKER_PARAM_FIELD_FLAGS	Worker flags
UCP_WORKER_PARAM_FIELD_NAME	Worker name
UCP_WORKER_PARAM_FIELD_AM_ALIGNMENT	Alignment of active messages on the receiver
UCP_WORKER_PARAM_FIELD_CLIENT_ID	Client id

ucp_worker_flags_t

enum ucp_worker_flags_t

This enumeration allows specifying flags for ucp_worker_params_t::flags, which is used as parameter for ucp_worker_create.

Enumerator
UCP_WORKER_FLAG_IGNORE_REQUEST_LEAK	Do not print warnings about request leaks

ucp_listener_params_field

enum ucp_listener_params_field

The enumeration allows specifying which fields in ucp_listener_params_t are present. It is used to enable backward compatibility support.

Enumerator
UCP_LISTENER_PARAM_FIELD_SOCK_ADDR	Sock address and length.
UCP_LISTENER_PARAM_FIELD_ACCEPT_HANDLER	User's callback and argument for handling the creation of an endpoint. User's callback and argument for handling the incoming connection request.
UCP_LISTENER_PARAM_FIELD_CONN_HANDLER

ucp_worker_address_flags_t

enum ucp_worker_address_flags_t

The enumeration list describes possible UCP worker address flags, indicating what needs to be included to the worker address returned by ucp_worker_query() routine.

Enumerator
UCP_WORKER_ADDRESS_FLAG_NET_ONLY	Pack addresses of network devices only. Using such shortened addresses for the remote node peers will reduce the amount of wireup data being exchanged during connection establishment phase.

ucp_worker_attr_field

enum ucp_worker_attr_field

The enumeration allows specifying which fields in ucp_worker_attr_t are present. It is used to enable backward compatibility support.

Enumerator
UCP_WORKER_ATTR_FIELD_THREAD_MODE	UCP thread mode
UCP_WORKER_ATTR_FIELD_ADDRESS	UCP address
UCP_WORKER_ATTR_FIELD_ADDRESS_FLAGS	UCP address flags
UCP_WORKER_ATTR_FIELD_MAX_AM_HEADER	Maximum header size used by UCP AM API
UCP_WORKER_ATTR_FIELD_NAME	UCP worker name
UCP_WORKER_ATTR_FIELD_MAX_INFO_STRING	Maximum size of info string

ucp_worker_address_attr_field

enum ucp_worker_address_attr_field

The enumeration allows specifying which fields in ucp_worker_address_attr_t are present. It is used to enable backward compatibility support.

Enumerator
UCP_WORKER_ADDRESS_ATTR_FIELD_UID	Unique id of the worker

ucp_listener_attr_field

enum ucp_listener_attr_field

The enumeration allows specifying which fields in ucp_listener_attr_t are present. It is used to enable backward compatibility support.

Enumerator
UCP_LISTENER_ATTR_FIELD_SOCKADDR	Sockaddr used for listening

ucp_conn_request_attr_field

enum ucp_conn_request_attr_field

The enumeration allows specifying which fields in ucp_conn_request_attr_t are present. It is used to enable backward compatibility support.

Enumerator
UCP_CONN_REQUEST_ATTR_FIELD_CLIENT_ADDR	Client's address
UCP_CONN_REQUEST_ATTR_FIELD_CLIENT_ID	Remote client id

ucp_am_cb_flags

enum ucp_am_cb_flags

Flags that indicate how to handle UCP Active Messages.

Enumerator
UCP_AM_FLAG_WHOLE_MSG	Indicates that the entire message will be handled in one callback.
UCP_AM_FLAG_PERSISTENT_DATA	Guarantees that the specified ucp_am_recv_callback_t callback, will always be called with UCP_AM_RECV_ATTR_FLAG_DATA flag set, so the data will be accessible outside the callback, until ucp_am_data_release is called.

ucp_send_am_flags

enum ucp_send_am_flags

Flags dictate the behavior of ucp_am_send_nb and ucp_am_send_nbx routines.

Enumerator
UCP_AM_SEND_FLAG_REPLY	Force relevant reply endpoint to be passed to the data callback on the receiver.
UCP_AM_SEND_FLAG_EAGER	Force UCP to use only eager protocol for AM sends.
UCP_AM_SEND_FLAG_RNDV	Force UCP to use only rendezvous protocol for AM sends.
UCP_AM_SEND_FLAG_COPY_HEADER	The flag indicates that the header should be copied to an internal buffer in case it's needed after the send function returns. If this flag is specified, the header can be released immediately after the send function returns, even if the non-blocking send request is not completed.
UCP_AM_SEND_REPLY	Backward compatibility.

ucp_wakeup_event_types

enum ucp_wakeup_event_types

The enumeration allows specifying which events are expected on wakeup. Empty events are possible for any type of event except for UCP_WAKEUP_TX and UCP_WAKEUP_RX.

Note

Send completions are reported by POLLIN-like events (see poll man page). Since outgoing operations can be initiated at any time, UCP does not generate POLLOUT-like events, although it must be noted that outgoing operations may be queued depending upon resource availability.

Enumerator
UCP_WAKEUP_RMA	Remote memory access send completion
UCP_WAKEUP_AMO	Atomic operation send completion
UCP_WAKEUP_TAG_SEND	Tag send completion
UCP_WAKEUP_TAG_RECV	Tag receive completion
UCP_WAKEUP_TX	This event type will generate an event on completion of any outgoing operation (complete or partial, according to the underlying protocol) for any type of transfer (send, atomic, or RMA).
UCP_WAKEUP_RX	This event type will generate an event on completion of any receive operation (complete or partial, according to the underlying protocol).
UCP_WAKEUP_EDGE	Use edge-triggered wakeup. The event file descriptor will be signaled only for new events, rather than existing ones.

Function Documentation

ucp_worker_create()

ucs_status_t ucp_worker_create	(	ucp_context_h	context,
		const ucp_worker_params_t *	params,
		ucp_worker_h *	worker_p )

This routine allocates and initializes a worker object. Each worker is associated with one and only one application context. In the same time, an application context can create multiple workers in order to enable concurrent access to communication resources. For example, application can allocate a dedicated worker for each application thread, where every worker can be progressed independently of others.

Note

The worker object is allocated within context of the calling thread

Parameters

[in]	context	Handle to UCP application context.
[in]	params	User defined ucp_worker_params_t configurations for the UCP worker.
[out]	worker_p	A pointer to the worker object allocated by the UCP library

Returns

Error code as defined by ucs_status_t

Examples

ucp_client_server.c, and ucp_hello_world.c.

ucp_worker_destroy()

void ucp_worker_destroy

(

ucp_worker_h

worker

)

This routine releases the resources associated with a UCP worker.

Warning

Once the UCP worker destroy the worker handle cannot be used with any UCP routine.

The destroy process releases and shuts down all resources associated with the worker.

Parameters

[in]

worker

Worker object to destroy.

Examples

ucp_client_server.c, and ucp_hello_world.c.

ucp_worker_query()

ucs_status_t ucp_worker_query	(	ucp_worker_h	worker,
		ucp_worker_attr_t *	attr )

This routine fetches information about the worker.

Parameters

[in]	worker	Worker object to query.
[out]	attr	Filled with attributes of worker.

Returns

Error code as defined by ucs_status_t

Examples

ucp_hello_world.c.

ucp_worker_print_info()

void ucp_worker_print_info	(	ucp_worker_h	worker,
		FILE *	stream )

This routine prints information about the protocols being used, thresholds, UCT transport methods, and other useful information associated with the worker.

Parameters

[in]	worker	Worker object to print information for.
[in]	stream	Output stream to print the information to.

ucp_worker_release_address()

void ucp_worker_release_address	(	ucp_worker_h	worker,
		ucp_address_t *	address )

This routine release an address handle associated within the worker object.

Warning

Once the address released the address handle cannot be used with any UCP routine.

Parameters

[in]	worker	Worker object that is associated with the address object.
[in]	address	Address to release; the address object has to be allocated using ucp_worker_query() routine.

Examples

ucp_hello_world.c.

ucp_worker_address_query()

ucs_status_t ucp_worker_address_query	(	ucp_address_t *	address,
		ucp_worker_address_attr_t *	attr )

This routine fetches information about the worker address. The address can be either of local or remote worker.

Parameters

[in]	address	Worker address to query.
[out]	attr	Filled with attributes of the worker address.

Returns

Error code as defined by ucs_status_t.

ucp_worker_progress()

unsigned ucp_worker_progress

(

ucp_worker_h

worker

)

This routine explicitly progresses all communication operations on a worker.

Note

• Typically, request wait and test routines call this routine to progress any outstanding operations.
• Transport layers, implementing asynchronous progress using threads, require callbacks and other user code to be thread safe.
• The state of communication can be advanced (progressed) by blocking routines. Nevertheless, the non-blocking routines can not be used for communication progress.

Parameters

[in]

worker

Worker to progress.

Returns

Non-zero if any communication was progressed, zero otherwise.

Examples

ucp_client_server.c, and ucp_hello_world.c.

ucp_stream_worker_poll()

ssize_t ucp_stream_worker_poll	(	ucp_worker_h	worker,
		ucp_stream_poll_ep_t *	poll_eps,
		size_t	max_eps,
		unsigned	flags )

This non-blocking routine returns endpoints on a worker which are ready to consume streaming data. The ready endpoints are placed in poll_eps array, and the function return value indicates how many are there.

Parameters

[in]	worker	Worker to poll.
[out]	poll_eps	Pointer to array of endpoints, should be allocated by user.
[in]	max_eps	Maximum number of endpoints that should be filled in poll_eps.
[in]	flags	Reserved for future use.

Returns

Negative value indicates an error according to ucs_status_t. On success, non-negative value (less or equal max_eps) indicates actual number of endpoints filled in poll_eps array.

ucp_listener_create()

ucs_status_t ucp_listener_create	(	ucp_worker_h	worker,
		const ucp_listener_params_t *	params,
		ucp_listener_h *	listener_p )

This routine creates a new listener object that is bound to a specific local address. The listener will listen to incoming connection requests. After receiving a request from the remote peer, an endpoint to this peer will be created - either right away or by calling ucp_ep_create, as specified by the callback type in ucp_listener_params_t. The user's callback will be invoked once the endpoint is created.

Parameters

[in]	worker	Worker object to create the listener on.
[in]	params	User defined ucp_listener_params_t configurations for the ucp_listener_h.
[out]	listener_p	A handle to the created listener, can be released by calling ucp_listener_destroy

Returns

Error code as defined by ucs_status_t

Note

ucp_listener_params_t::conn_handler or ucp_listener_params_t::accept_handler must be provided to be able to handle incoming connections.

Examples

ucp_client_server.c.

ucp_listener_destroy()

void ucp_listener_destroy

(

ucp_listener_h

listener

)

This routine unbinds the worker from the given handle and stops listening for incoming connection requests on it.

Parameters

[in]

listener

A handle to the listener to stop listening on.

Examples

ucp_client_server.c.

ucp_listener_query()

ucs_status_t ucp_listener_query	(	ucp_listener_h	listener,
		ucp_listener_attr_t *	attr )

This routine fetches information about the listener.

Parameters

[in]	listener	listener object to query.
[out]	attr	Filled with attributes of the listener.

Returns

Error code as defined by ucs_status_t

Examples

ucp_client_server.c.

ucp_conn_request_query()

ucs_status_t ucp_conn_request_query	(	ucp_conn_request_h	conn_request,
		ucp_conn_request_attr_t *	attr )

This routine fetches information about the connection request.

Parameters

[in]	conn_request	connection request object to query.
[out]	attr	Filled with attributes of the connection request.

Returns

Error code as defined by ucs_status_t

Examples

ucp_client_server.c.

ucp_listener_reject()

ucs_status_t ucp_listener_reject	(	ucp_listener_h	listener,
		ucp_conn_request_h	conn_request )

Reject the incoming connection request and release associated resources. If the remote initiator endpoint has set an ucp_ep_params_t::err_handler, it will be invoked with status UCS_ERR_REJECTED.

Parameters

[in]	listener	Handle to the listener on which the connection request was received.
[in]	conn_request	Handle to the connection request to reject.

Returns

Error code as defined by ucs_status_t

Examples

ucp_client_server.c.

ucp_worker_set_am_recv_handler()

ucs_status_t ucp_worker_set_am_recv_handler	(	ucp_worker_h	worker,
		const ucp_am_handler_param_t *	param )

This routine installs a user defined callback to handle incoming Active Messages with a specific id. This callback is called whenever an Active Message that was sent from the remote peer by ucp_am_send_nbx is received on this worker.

Warning

Handlers set by this function are not compatible with ucp_am_send_nb routine.

Parameters

[in]	worker	UCP worker on which to set the Active Message handler.
[in]	param	Active Message handler parameters, as defined by ucp_am_handler_param_t.

Returns

error code if the worker does not support Active Messages or requested callback flags.

Examples

ucp_client_server.c.

ucp_worker_fence()

ucs_status_t ucp_worker_fence

(

ucp_worker_h

worker

)

This routine ensures ordering of non-blocking communication operations on the UCP worker. Communication operations issued on a particular endpoint created on the worker prior to this call are guaranteed to be completed before any communication operations issued on the same endpoint after this call.

Note

The primary difference between ucp_worker_fence() and the ucp_worker_flush_nb() is the fact the fence routine does not guarantee completion of the operations on the call return but only ensures the order between communication operations. The flush operation on return guarantees that all operations are completed and corresponding memory regions were updated.

Parameters

[in]

worker

UCP worker.

Returns

Error code as defined by ucs_status_t

ucp_worker_flush_nbx()

ucs_status_ptr_t ucp_worker_flush_nbx	(	ucp_worker_h	worker,
		const ucp_request_param_t *	param )

This routine flushes all outstanding AMO and RMA communications on the worker. All the AMO and RMA operations issued on the worker prior to this call are completed both at the origin and at the target when this call returns.

Note

For description of the differences between flush and fence operations please see ucp_worker_fence()

Parameters

[in]	worker	UCP worker.
[in]	param	Operation parameters, see ucp_request_param_t

Returns

NULL - The flush operation was completed immediately.

UCS_PTR_IS_ERR(_ptr) - The flush operation failed.

otherwise - Flush operation was scheduled and can be completed in any point in time. The request handle is returned to the application in order to track progress.

ucp_worker_flush()

ucs_status_t ucp_worker_flush

(

ucp_worker_h

worker

)

Deprecated

Replaced by ucp_worker_flush_nb. The following example implements the same functionality using ucp_worker_flush_nb :

ucs_status_t worker_flush(ucp_worker_h worker)
{
    void *request = ucp_worker_flush_nb(worker);
    if (request == NULL) {
        return UCS_OK;
    } else if (UCS_PTR_IS_ERR(request)) {
        return UCS_PTR_STATUS(request);
    } else {
        ucs_status_t status;
        do {
            ucp_worker_progress(worker);
            status = ucp_request_check_status(request);
        } while (status == UCS_INPROGRESS);
        ucp_request_release(request);
        return status;
    }
}

This routine flushes all outstanding AMO and RMA communications on the worker. All the AMO and RMA operations issued on the worker prior to this call are completed both at the origin and at the target when this call returns.

Note

For description of the differences between flush and fence operations please see ucp_worker_fence()

Parameters

[in]

worker

UCP worker.

Returns

Error code as defined by ucs_status_t

ucp_worker_get_address()

ucs_status_t ucp_worker_get_address	(	ucp_worker_h	worker,
		ucp_address_t **	address_p,
		size_t *	address_length_p )

Deprecated

Use ucp_worker_query with the flag UCP_WORKER_ATTR_FIELD_ADDRESS in order to obtain the worker address.

This routine returns the address of the worker object. This address can be passed to remote instances of the UCP library in order to connect to this worker. The memory for the address handle is allocated by this function, and must be released by using ucp_worker_release_address() routine.

Parameters

[in]	worker	Worker object whose address to return.
[out]	address_p	A pointer to the worker address.
[out]	address_length_p	The size in bytes of the address.

Returns

Error code as defined by ucs_status_t

ucp_worker_set_am_handler()

ucs_status_t ucp_worker_set_am_handler	(	ucp_worker_h	worker,
		uint16_t	id,
		ucp_am_callback_t	cb,
		void *	arg,
		uint32_t	flags )

Deprecated

Use ucp_worker_set_am_recv_handler instead.

This routine installs a user defined callback to handle incoming Active Messages with a specific id. This callback is called whenever an Active Message that was sent from the remote peer by ucp_am_send_nb is received on this worker.

Parameters

[in]	worker	UCP worker on which to set the Active Message handler.
[in]	id	Active Message id.
[in]	cb	Active Message callback. NULL to clear.
[in]	arg	Active Message argument, which will be passed in to every invocation of the callback as the arg argument.
[in]	flags	Dictates how an Active Message is handled on the remote endpoint. Currently only UCP_AM_FLAG_WHOLE_MSG is supported, which indicates the callback will not be invoked until all data has arrived.

Returns

error code if the worker does not support Active Messages or requested callback flags.

ucp_worker_flush_nb()

ucs_status_ptr_t ucp_worker_flush_nb	(	ucp_worker_h	worker,
		unsigned	flags,
		ucp_send_callback_t	cb )

Deprecated

Use ucp_worker_flush_nbx instead.

This routine flushes all outstanding AMO and RMA communications on the worker. All the AMO and RMA operations issued on the worker prior to this call are completed both at the origin and at the target when this call returns.

Note

For description of the differences between flush and fence operations please see ucp_worker_fence()

Parameters

[in]	worker	UCP worker.
[in]	flags	Flags for flush operation. Reserved for future use.
[in]	cb	Callback which will be called when the flush operation completes.

Returns

NULL - The flush operation was completed immediately.

UCS_PTR_IS_ERR(_ptr) - The flush operation failed.

otherwise - Flush operation was scheduled and can be completed in any point in time. The request handle is returned to the application in order to track progress. The application is responsible for releasing the handle using ucp_request_free() routine.