uct.h

 
#ifndef UCT_H_
#define UCT_H_
 
#include <uct/api/uct_def.h>
#include <uct/api/tl.h>
#include <uct/api/version.h>
#include <ucs/async/async_fwd.h>
#include <ucs/datastruct/callbackq.h>
#include <ucs/datastruct/callbackq_compat.h>
#include <ucs/datastruct/linear_func.h>
#include <ucs/memory/memory_type.h>
#include <ucs/type/status.h>
#include <ucs/type/thread_mode.h>
#include <ucs/type/cpu_set.h>
#include <ucs/stats/stats_fwd.h>
#include <ucs/sys/compiler_def.h>
#include <ucs/sys/topo/base/topo.h>
 
#include <sys/socket.h>
#include <stdio.h>
#include <sched.h>
 
BEGIN_C_DECLS
 
typedef struct uct_md_resource_desc {
    char                     md_name[UCT_MD_NAME_MAX]; 
} uct_md_resource_desc_t;
 
 
enum uct_component_attr_field {
    UCT_COMPONENT_ATTR_FIELD_NAME              = UCS_BIT(0), 
    UCT_COMPONENT_ATTR_FIELD_MD_RESOURCE_COUNT = UCS_BIT(1), 
    UCT_COMPONENT_ATTR_FIELD_MD_RESOURCES      = UCS_BIT(2), 
    UCT_COMPONENT_ATTR_FIELD_FLAGS             = UCS_BIT(3)  
};
 
 
typedef struct uct_component_attr {
    uint64_t               field_mask;
 
    char                   name[UCT_COMPONENT_NAME_MAX];
 
    unsigned               md_resource_count;
 
    uct_md_resource_desc_t *md_resources;
 
    uint64_t               flags;
} uct_component_attr_t;
 
 
enum {
    UCT_COMPONENT_FLAG_CM       = UCS_BIT(0),
 
    UCT_COMPONENT_FLAG_RKEY_PTR = UCS_BIT(1)
};
 
 
typedef enum {
    UCT_DEVICE_TYPE_NET,     
    UCT_DEVICE_TYPE_SHM,     
    UCT_DEVICE_TYPE_ACC,     
    UCT_DEVICE_TYPE_SELF,    
    UCT_DEVICE_TYPE_LAST
} uct_device_type_t;
 
 
typedef struct uct_tl_resource_desc {
    char                     tl_name[UCT_TL_NAME_MAX];   
    char                     dev_name[UCT_DEVICE_NAME_MAX]; 
    uct_device_type_t        dev_type;     
    ucs_sys_device_t         sys_device;   
} uct_tl_resource_desc_t;
 
#define UCT_TL_RESOURCE_DESC_FMT              "%s/%s"
#define UCT_TL_RESOURCE_DESC_ARG(_resource)   (_resource)->tl_name, (_resource)->dev_name
 
 
typedef enum uct_atomic_op {
    UCT_ATOMIC_OP_ADD,   
    UCT_ATOMIC_OP_AND,   
    UCT_ATOMIC_OP_OR,    
    UCT_ATOMIC_OP_XOR,   
    UCT_ATOMIC_OP_SWAP,  
    UCT_ATOMIC_OP_CSWAP, 
    UCT_ATOMIC_OP_LAST
} uct_atomic_op_t;
 
 
        /* Active message capabilities */
#define UCT_IFACE_FLAG_AM_SHORT       UCS_BIT(0)  
#define UCT_IFACE_FLAG_AM_BCOPY       UCS_BIT(1)  
#define UCT_IFACE_FLAG_AM_ZCOPY       UCS_BIT(2)  
#define UCT_IFACE_FLAG_PENDING        UCS_BIT(3)  
        /* PUT capabilities */
#define UCT_IFACE_FLAG_PUT_SHORT      UCS_BIT(4)  
#define UCT_IFACE_FLAG_PUT_BCOPY      UCS_BIT(5)  
#define UCT_IFACE_FLAG_PUT_ZCOPY      UCS_BIT(6)  
        /* GET capabilities */
#define UCT_IFACE_FLAG_GET_SHORT      UCS_BIT(8)  
#define UCT_IFACE_FLAG_GET_BCOPY      UCS_BIT(9)  
#define UCT_IFACE_FLAG_GET_ZCOPY      UCS_BIT(10) 
        /* Atomic operations domain */
#define UCT_IFACE_FLAG_ATOMIC_CPU     UCS_BIT(30) 
#define UCT_IFACE_FLAG_ATOMIC_DEVICE  UCS_BIT(31) 
        /* Error handling capabilities */
#define UCT_IFACE_FLAG_ERRHANDLE_SHORT_BUF    UCS_BIT(32) 
#define UCT_IFACE_FLAG_ERRHANDLE_BCOPY_BUF    UCS_BIT(33) 
#define UCT_IFACE_FLAG_ERRHANDLE_ZCOPY_BUF    UCS_BIT(34) 
#define UCT_IFACE_FLAG_ERRHANDLE_AM_ID        UCS_BIT(35) 
#define UCT_IFACE_FLAG_ERRHANDLE_REMOTE_MEM   UCS_BIT(36) 
#define UCT_IFACE_FLAG_ERRHANDLE_BCOPY_LEN    UCS_BIT(37) 
#define UCT_IFACE_FLAG_ERRHANDLE_PEER_FAILURE UCS_BIT(38) 
#define UCT_IFACE_FLAG_EP_CHECK               UCS_BIT(39) 
        /* Connection establishment */
#define UCT_IFACE_FLAG_CONNECT_TO_IFACE       UCS_BIT(40) 
#define UCT_IFACE_FLAG_CONNECT_TO_EP          UCS_BIT(41) 
#define UCT_IFACE_FLAG_CONNECT_TO_SOCKADDR    UCS_BIT(42) 
        /* Special transport flags */
#define UCT_IFACE_FLAG_AM_DUP         UCS_BIT(43) 
        /* Callback invocation */
#define UCT_IFACE_FLAG_CB_SYNC        UCS_BIT(44) 
#define UCT_IFACE_FLAG_CB_ASYNC       UCS_BIT(45) 
        /* Keepalive */
#define UCT_IFACE_FLAG_EP_KEEPALIVE   UCS_BIT(46) 
        /* Tag matching operations */
#define UCT_IFACE_FLAG_TAG_EAGER_SHORT UCS_BIT(50) 
#define UCT_IFACE_FLAG_TAG_EAGER_BCOPY UCS_BIT(51) 
#define UCT_IFACE_FLAG_TAG_EAGER_ZCOPY UCS_BIT(52) 
#define UCT_IFACE_FLAG_TAG_RNDV_ZCOPY  UCS_BIT(53) 
        /* Interface capability */
#define UCT_IFACE_FLAG_INTER_NODE      UCS_BIT(54) 
        /* Event types */
#define UCT_IFACE_FLAG_EVENT_SEND_COMP UCS_BIT(0) 
#define UCT_IFACE_FLAG_EVENT_RECV      UCS_BIT(1) 
#define UCT_IFACE_FLAG_EVENT_RECV_SIG  UCS_BIT(2) 
        /* Event notification mechanisms */
#define UCT_IFACE_FLAG_EVENT_FD        UCS_BIT(3) 
#define UCT_IFACE_FLAG_EVENT_ASYNC_CB  UCS_BIT(4) 
typedef enum {
    UCT_ALLOC_METHOD_THP,  
    UCT_ALLOC_METHOD_MD,   
    UCT_ALLOC_METHOD_HEAP, 
    UCT_ALLOC_METHOD_MMAP, 
    UCT_ALLOC_METHOD_HUGE, 
    UCT_ALLOC_METHOD_LAST,
    UCT_ALLOC_METHOD_DEFAULT = UCT_ALLOC_METHOD_LAST 
} uct_alloc_method_t;
 
 
enum uct_iface_event_types {
    UCT_EVENT_SEND_COMP     = UCS_BIT(0), 
    UCT_EVENT_RECV          = UCS_BIT(1), 
    UCT_EVENT_RECV_SIG      = UCS_BIT(2)  
};
 
 
enum uct_flush_flags {
    UCT_FLUSH_FLAG_LOCAL    = 0,            
    UCT_FLUSH_FLAG_CANCEL   = UCS_BIT(0),   
    UCT_FLUSH_FLAG_REMOTE   = UCS_BIT(1)    
};
 
 
enum uct_progress_types {
    UCT_PROGRESS_SEND        = UCS_BIT(0),  
    UCT_PROGRESS_RECV        = UCS_BIT(1),  
    UCT_PROGRESS_THREAD_SAFE = UCS_BIT(7)   
};
 
 
enum uct_msg_flags {
    UCT_SEND_FLAG_SIGNALED   = UCS_BIT(0), 
    UCT_SEND_FLAG_PEER_CHECK = UCS_BIT(1)  
};
 
 
enum uct_cb_flags {
    UCT_CB_FLAG_RESERVED = UCS_BIT(1), 
    UCT_CB_FLAG_ASYNC    = UCS_BIT(2)  
};
 
 
enum uct_iface_open_mode {
   UCT_IFACE_OPEN_MODE_DEVICE          = UCS_BIT(0),
 
   UCT_IFACE_OPEN_MODE_SOCKADDR_SERVER = UCS_BIT(1),
 
   UCT_IFACE_OPEN_MODE_SOCKADDR_CLIENT = UCS_BIT(2)
};
 
 
enum uct_iface_params_field {
    UCT_IFACE_PARAM_FIELD_CPU_MASK           = UCS_BIT(0),
 
    UCT_IFACE_PARAM_FIELD_OPEN_MODE          = UCS_BIT(1),
 
    UCT_IFACE_PARAM_FIELD_DEVICE             = UCS_BIT(2),
 
    UCT_IFACE_PARAM_FIELD_SOCKADDR           = UCS_BIT(3),
 
    UCT_IFACE_PARAM_FIELD_STATS_ROOT         = UCS_BIT(4),
 
    UCT_IFACE_PARAM_FIELD_RX_HEADROOM        = UCS_BIT(5),
 
    UCT_IFACE_PARAM_FIELD_ERR_HANDLER_ARG    = UCS_BIT(6),
 
    UCT_IFACE_PARAM_FIELD_ERR_HANDLER        = UCS_BIT(7),
 
    UCT_IFACE_PARAM_FIELD_ERR_HANDLER_FLAGS  = UCS_BIT(8),
 
    UCT_IFACE_PARAM_FIELD_HW_TM_EAGER_ARG    = UCS_BIT(9),
 
    UCT_IFACE_PARAM_FIELD_HW_TM_EAGER_CB     = UCS_BIT(10),
 
    UCT_IFACE_PARAM_FIELD_HW_TM_RNDV_ARG     = UCS_BIT(11),
 
    UCT_IFACE_PARAM_FIELD_HW_TM_RNDV_CB      = UCS_BIT(12),
 
    UCT_IFACE_PARAM_FIELD_ASYNC_EVENT_ARG    = UCS_BIT(13),
 
    UCT_IFACE_PARAM_FIELD_ASYNC_EVENT_CB     = UCS_BIT(14),
 
    UCT_IFACE_PARAM_FIELD_KEEPALIVE_INTERVAL = UCS_BIT(15),
 
    UCT_IFACE_PARAM_FIELD_AM_ALIGNMENT       = UCS_BIT(16),
 
    UCT_IFACE_PARAM_FIELD_AM_ALIGN_OFFSET    = UCS_BIT(17),
 
    UCT_IFACE_PARAM_FIELD_FEATURES           = UCS_BIT(18)
};
 
typedef enum {
   UCT_SOCKADDR_ACC_LOCAL,  
   UCT_SOCKADDR_ACC_REMOTE  
} uct_sockaddr_accessibility_t;
 
 
enum {
    UCT_MD_FLAG_ALLOC         = UCS_BIT(0),
 
    UCT_MD_FLAG_REG           = UCS_BIT(1),
 
    UCT_MD_FLAG_NEED_MEMH     = UCS_BIT(2),
 
    UCT_MD_FLAG_NEED_RKEY     = UCS_BIT(3),
 
    UCT_MD_FLAG_ADVISE        = UCS_BIT(4),
 
    UCT_MD_FLAG_FIXED         = UCS_BIT(5),
 
    UCT_MD_FLAG_RKEY_PTR      = UCS_BIT(6),
 
    UCT_MD_FLAG_SOCKADDR      = UCS_BIT(7),
 
    UCT_MD_FLAG_INVALIDATE    = UCS_BIT(8),
 
    UCT_MD_FLAG_EXPORTED_MKEY = UCS_BIT(9),
 
    UCT_MD_FLAG_REG_DMABUF    = UCS_BIT(10),
 
    UCT_MD_FLAG_LAST          = UCS_BIT(11)
};
 
enum uct_md_mem_flags {
    UCT_MD_MEM_FLAG_NONBLOCK        = UCS_BIT(0),
 
    UCT_MD_MEM_FLAG_FIXED           = UCS_BIT(1),
 
    UCT_MD_MEM_FLAG_LOCK            = UCS_BIT(2),
 
    UCT_MD_MEM_FLAG_HIDE_ERRORS     = UCS_BIT(3),
 
    /* Memory access flags */
    UCT_MD_MEM_ACCESS_REMOTE_PUT    = UCS_BIT(5),
 
    UCT_MD_MEM_ACCESS_REMOTE_GET    = UCS_BIT(6),
 
    UCT_MD_MEM_ACCESS_REMOTE_ATOMIC = UCS_BIT(7),
 
    UCT_MD_MEM_ACCESS_LOCAL_READ    = UCS_BIT(8),
 
    UCT_MD_MEM_ACCESS_LOCAL_WRITE   = UCS_BIT(9),
 
    UCT_MD_MEM_SYMMETRIC_RKEY       = UCS_BIT(10),
 
    UCT_MD_MEM_GVA                  = UCS_BIT(11),
 
    UCT_MD_MEM_ACCESS_ALL           = (UCT_MD_MEM_ACCESS_REMOTE_PUT |
                                       UCT_MD_MEM_ACCESS_REMOTE_GET |
                                       UCT_MD_MEM_ACCESS_REMOTE_ATOMIC |
                                       UCT_MD_MEM_ACCESS_LOCAL_READ |
                                       UCT_MD_MEM_ACCESS_LOCAL_WRITE),
 
    UCT_MD_MEM_ACCESS_RMA           = (UCT_MD_MEM_ACCESS_REMOTE_PUT |
                                       UCT_MD_MEM_ACCESS_REMOTE_GET |
                                       UCT_MD_MEM_ACCESS_LOCAL_READ |
                                       UCT_MD_MEM_ACCESS_LOCAL_WRITE)
};
 
 
typedef enum {
    UCT_MADV_NORMAL  = 0,  
    UCT_MADV_WILLNEED      
} uct_mem_advice_t;
 
 
enum uct_cm_attr_field {
    UCT_CM_ATTR_FIELD_MAX_CONN_PRIV = UCS_BIT(0)
};
 
 
enum uct_listener_attr_field {
    UCT_LISTENER_ATTR_FIELD_SOCKADDR = UCS_BIT(0)
};
 
 
enum uct_listener_params_field {
    UCT_LISTENER_PARAM_FIELD_BACKLOG         = UCS_BIT(0),
 
    UCT_LISTENER_PARAM_FIELD_CONN_REQUEST_CB = UCS_BIT(1),
 
    UCT_LISTENER_PARAM_FIELD_USER_DATA       = UCS_BIT(2)
};
 
 
enum uct_ep_params_field {
    UCT_EP_PARAM_FIELD_IFACE                      = UCS_BIT(0),
 
    UCT_EP_PARAM_FIELD_USER_DATA                  = UCS_BIT(1),
 
    UCT_EP_PARAM_FIELD_DEV_ADDR                   = UCS_BIT(2),
 
    UCT_EP_PARAM_FIELD_IFACE_ADDR                 = UCS_BIT(3),
 
    UCT_EP_PARAM_FIELD_SOCKADDR                   = UCS_BIT(4),
 
    UCT_EP_PARAM_FIELD_SOCKADDR_CB_FLAGS          = UCS_BIT(5),
 
    UCT_EP_PARAM_FIELD_SOCKADDR_PACK_CB           = UCS_BIT(6),
 
    UCT_EP_PARAM_FIELD_CM                         = UCS_BIT(7),
 
    UCT_EP_PARAM_FIELD_CONN_REQUEST               = UCS_BIT(8),
 
    UCT_EP_PARAM_FIELD_SOCKADDR_CONNECT_CB_CLIENT = UCS_BIT(9),
 
    UCT_EP_PARAM_FIELD_SOCKADDR_NOTIFY_CB_SERVER  = UCS_BIT(10),
 
    UCT_EP_PARAM_FIELD_SOCKADDR_DISCONNECT_CB     = UCS_BIT(11),
 
    UCT_EP_PARAM_FIELD_PATH_INDEX                 = UCS_BIT(12),
 
    UCT_EP_PARAM_FIELD_CM_RESOLVE_CB              = UCS_BIT(13),
 
    UCT_EP_PARAM_FIELD_PRIV_DATA                  = UCS_BIT(14),
 
    UCT_EP_PARAM_FIELD_PRIV_DATA_LENGTH           = UCS_BIT(15),
 
    UCT_EP_PARAM_FIELD_LOCAL_SOCKADDR             = UCS_BIT(16),
 
    UCT_EP_PARAM_FIELD_DEV_ADDR_LENGTH            = UCS_BIT(17),
 
    UCT_EP_PARAM_FIELD_IFACE_ADDR_LENGTH          = UCS_BIT(18)
};
 
 
enum uct_ep_connect_params_field {
    UCT_EP_CONNECT_PARAM_FIELD_PRIVATE_DATA         = UCS_BIT(0),
 
    UCT_EP_CONNECT_PARAM_FIELD_PRIVATE_DATA_LENGTH  = UCS_BIT(1)
};
 
 
enum uct_iface_feature {
    UCT_IFACE_FEATURE_AM           = UCS_BIT(0),
 
    UCT_IFACE_FEATURE_PUT          = UCS_BIT(1),
 
    UCT_IFACE_FEATURE_GET          = UCS_BIT(2),
 
    UCT_IFACE_FEATURE_AMO32        = UCS_BIT(3),
 
    UCT_IFACE_FEATURE_AMO64        = UCS_BIT(4),
 
    UCT_IFACE_FEATURE_TAG          = UCS_BIT(5),
 
    UCT_IFACE_FEATURE_FLUSH_REMOTE = UCS_BIT(6),
 
    UCT_IFACE_FEATURE_LAST         = UCS_BIT(7)
};
 
/*
 * @ingroup UCT_RESOURCE
 * @brief Process Per Node (PPN) bandwidth specification: f(ppn) = dedicated + shared / ppn
 *
 *  This structure specifies a function which is used as basis for bandwidth
 * estimation of various UCT operations. This information can be used to select
 * the best performing combination of UCT operations.
 */
typedef struct uct_ppn_bandwidth {
    double                   dedicated; 
    double                   shared;    
} uct_ppn_bandwidth_t;
 
 
struct uct_iface_attr {
    struct {
        struct {
            size_t           max_short;  
            size_t           max_bcopy;  
            size_t           min_zcopy;  
            size_t           max_zcopy;  
            size_t           opt_zcopy_align; 
            size_t           align_mtu;       
            size_t           max_iov;    
        } put;                           
        struct {
            size_t           max_short;  
            size_t           max_bcopy;  
            size_t           min_zcopy;  
            size_t           max_zcopy;  
            size_t           opt_zcopy_align; 
            size_t           align_mtu;       
            size_t           max_iov;    
        } get;                           
        struct {
            size_t           max_short;  
            size_t           max_bcopy;  
            size_t           min_zcopy;  
            size_t           max_zcopy;  
            size_t           opt_zcopy_align; 
            size_t           align_mtu;       
            size_t           max_hdr;    
            size_t           max_iov;    
        } am;                            
        struct {
            struct {
                size_t       min_recv;   
                size_t       max_zcopy;  
                size_t       max_iov;    
                size_t       max_outstanding; 
            } recv;
 
            struct {
                  size_t     max_short;  
                  size_t     max_bcopy;  
                  size_t     max_zcopy;  
                  size_t     max_iov;    
            } eager;                     
            struct {
                  size_t     max_zcopy;  
                  size_t     max_hdr;    
                  size_t     max_iov;    
            } rndv;                      
        } tag;                           
        struct {
            uint64_t         op_flags;   
            uint64_t         fop_flags;  
        } atomic32, atomic64;            
        uint64_t             flags;      
        uint64_t             event_flags;
    } cap;                               
    size_t                   device_addr_len;
    size_t                   iface_addr_len; 
    size_t                   ep_addr_len;    
    size_t                   max_conn_priv;  
    struct sockaddr_storage  listen_sockaddr; 
    /*
     * The following fields define expected performance of the communication
     * interface, this would usually be a combination of device and system
     * characteristics and determined at run time.
     */
    double                   overhead;     
    uct_ppn_bandwidth_t      bandwidth;    
    ucs_linear_func_t        latency;      
    uint8_t                  priority;     
    size_t                   max_num_eps;  
    unsigned                 dev_num_paths;
};
 
 
struct uct_iface_params {
    uint64_t                                     field_mask;
    ucs_cpu_set_t                                cpu_mask;
    uint64_t                                     open_mode;
    union {
        struct {
            const char                           *tl_name;  
            const char                           *dev_name; 
        } device;
        struct {
            ucs_sock_addr_t                      listen_sockaddr;
            void                                 *conn_request_arg;
            uct_sockaddr_conn_request_callback_t conn_request_cb;
            uint32_t                             cb_flags;
        } sockaddr;
    } mode;
 
    ucs_stats_node_t                             *stats_root;
    size_t                                       rx_headroom;
 
    void                                         *err_handler_arg;
    uct_error_handler_t                          err_handler;
    uint32_t                                     err_handler_flags;
 
    void                                         *eager_arg;
    uct_tag_unexp_eager_cb_t                     eager_cb;
    void                                         *rndv_arg;
    uct_tag_unexp_rndv_cb_t                      rndv_cb;
 
    void                                         *async_event_arg;
    uct_async_event_cb_t                         async_event_cb;
 
    /* Time period between keepalive rounds */
    ucs_time_t                                   keepalive_interval;
 
    size_t                                       am_alignment;
 
    size_t                                       am_align_offset;
 
    uint64_t                                     features;
};
 
 
struct uct_ep_params {
    uint64_t                          field_mask;
 
    uct_iface_h                       iface;
 
    void                              *user_data;
 
    const uct_device_addr_t           *dev_addr;
 
    const uct_iface_addr_t            *iface_addr;
 
    const ucs_sock_addr_t             *sockaddr;
 
    uint32_t                          sockaddr_cb_flags;
 
    uct_cm_ep_priv_data_pack_callback_t sockaddr_pack_cb;
 
    uct_cm_h                          cm;
 
    uct_conn_request_h                conn_request;
 
    uct_cm_ep_client_connect_callback_t      sockaddr_cb_client;
 
    uct_cm_ep_server_conn_notify_callback_t  sockaddr_cb_server;
 
    uct_ep_disconnect_cb_t              disconnect_cb;
 
    unsigned                            path_index;
 
    uct_cm_ep_resolve_callback_t        cm_resolve_cb;
 
    const void                          *private_data;
 
    size_t                              private_data_length;
 
    const ucs_sock_addr_t               *local_sockaddr;
 
    size_t                              dev_addr_length;
 
    size_t                              iface_addr_length;
};
 
 
struct uct_ep_connect_params {
    uint64_t                            field_mask;
 
    const void                          *private_data;
 
    size_t                              private_data_length;
};
 
struct uct_cm_attr {
    uint64_t    field_mask;
 
    size_t      max_conn_priv;
};
 
 
struct uct_listener_attr {
    uint64_t                field_mask;
 
    struct sockaddr_storage sockaddr;
};
 
 
struct uct_listener_params {
    uint64_t                                field_mask;
 
    int                                     backlog;
 
    uct_cm_listener_conn_request_callback_t conn_request_cb;
 
    void                                    *user_data;
};
 
 
struct uct_md_attr {
    struct {
        uint64_t             max_alloc; 
        size_t               max_reg;   
        uint64_t             flags;     
        uint64_t             reg_mem_types; 
        uint64_t             detect_mem_types; 
        uint64_t             alloc_mem_types;  
        uint64_t             access_mem_types; 
    } cap;
 
    ucs_linear_func_t        reg_cost;  
    char                     component_name[UCT_COMPONENT_NAME_MAX]; 
    size_t                   rkey_packed_size; 
    ucs_cpu_set_t            local_cpus;    
};
 
 
typedef enum uct_md_mem_attr_field {
    UCT_MD_MEM_ATTR_FIELD_MEM_TYPE      = UCS_BIT(0),
 
    UCT_MD_MEM_ATTR_FIELD_SYS_DEV       = UCS_BIT(1),
 
    UCT_MD_MEM_ATTR_FIELD_BASE_ADDRESS  = UCS_BIT(2),
 
    UCT_MD_MEM_ATTR_FIELD_ALLOC_LENGTH  = UCS_BIT(3),
 
    UCT_MD_MEM_ATTR_FIELD_DMABUF_FD     = UCS_BIT(4),
 
    UCT_MD_MEM_ATTR_FIELD_DMABUF_OFFSET = UCS_BIT(5)
} uct_md_mem_attr_field_t;
 
 
typedef struct uct_md_mem_attr {
    uint64_t          field_mask;
 
    ucs_memory_type_t mem_type;
 
    ucs_sys_device_t  sys_dev;
 
    void              *base_address;
 
    size_t            alloc_length;
 
    int               dmabuf_fd;
 
    size_t            dmabuf_offset;
} uct_md_mem_attr_t;
 
 
ucs_status_t uct_md_mem_query(uct_md_h md, const void *address, size_t length,
                              uct_md_mem_attr_t *mem_attr);
 
 
typedef struct uct_allocated_memory {
    void                     *address;   
    size_t                   length;     
    uct_alloc_method_t       method;     
    ucs_memory_type_t        mem_type;   
    uct_md_h                 md;         
    uct_mem_h                memh;       
    ucs_sys_device_t         sys_device; 
} uct_allocated_memory_t;
 
 
typedef struct uct_rkey_bundle {
    uct_rkey_t               rkey;    
    void                     *handle; 
    void                     *type;   
} uct_rkey_bundle_t;
 
 
struct uct_completion {
    uct_completion_callback_t func;    
    int                       count;   
    ucs_status_t              status;  
};
 
 
struct uct_pending_req {
    uct_pending_callback_t    func;   
    char                      priv[UCT_PENDING_REQ_PRIV_LEN]; 
};
 
 
struct uct_tag_context {
    void (*tag_consumed_cb)(uct_tag_context_t *self);
 
     void (*completed_cb)(uct_tag_context_t *self, uct_tag_t stag, uint64_t imm,
                          size_t length, void *inline_data, ucs_status_t status);
 
     void (*rndv_cb)(uct_tag_context_t *self, uct_tag_t stag, const void *header,
                     unsigned header_length, ucs_status_t status, unsigned flags);
 
     char priv[UCT_TAG_PRIV_LEN];
};
 
 
enum {
    /* If set, header points to inline data, otherwise it is user buffer. */
    UCT_TAG_RECV_CB_INLINE_DATA = UCS_BIT(0)
};
 
 
extern const char *uct_alloc_method_names[];
extern const char *uct_device_type_names[];
 
 
ucs_status_t uct_query_components(uct_component_h **components_p,
                                  unsigned *num_components_p);
 
void uct_release_component_list(uct_component_h *components);
 
 
ucs_status_t uct_component_query(uct_component_h component,
                                 uct_component_attr_t *component_attr);
 
 
ucs_status_t uct_md_open(uct_component_h component, const char *md_name,
                         const uct_md_config_t *config, uct_md_h *md_p);
 
void uct_md_close(uct_md_h md);
 
 
ucs_status_t uct_md_query_tl_resources(uct_md_h md,
                                       uct_tl_resource_desc_t **resources_p,
                                       unsigned *num_resources_p);
 
 
void uct_release_tl_resource_list(uct_tl_resource_desc_t *resources);
 
 
ucs_status_t uct_worker_create(ucs_async_context_t *async,
                               ucs_thread_mode_t thread_mode,
                               uct_worker_h *worker_p);
 
 
void uct_worker_destroy(uct_worker_h worker);
 
 
void uct_worker_progress_register_safe(uct_worker_h worker, ucs_callback_t func,
                                       void *arg, unsigned flags,
                                       uct_worker_cb_id_t *id_p);
 
 
void uct_worker_progress_unregister_safe(uct_worker_h worker,
                                         uct_worker_cb_id_t *id_p);
 
 
ucs_status_t uct_md_iface_config_read(uct_md_h md, const char *tl_name,
                                      const char *env_prefix, const char *filename,
                                      uct_iface_config_t **config_p);
 
 
void uct_config_release(void *config);
 
 
ucs_status_t uct_config_get(void *config, const char *name, char *value,
                            size_t max);
 
 
ucs_status_t uct_config_modify(void *config, const char *name, const char *value);
 
 
ucs_status_t uct_iface_open(uct_md_h md, uct_worker_h worker,
                            const uct_iface_params_t *params,
                            const uct_iface_config_t *config,
                            uct_iface_h *iface_p);
 
 
void uct_iface_close(uct_iface_h iface);
 
 
ucs_status_t uct_iface_query(uct_iface_h iface, uct_iface_attr_t *iface_attr);
 
 
ucs_status_t uct_iface_get_device_address(uct_iface_h iface, uct_device_addr_t *addr);
 
 
ucs_status_t uct_iface_get_address(uct_iface_h iface, uct_iface_addr_t *addr);
 
 
int uct_iface_is_reachable(const uct_iface_h iface, const uct_device_addr_t *dev_addr,
                           const uct_iface_addr_t *iface_addr);
 
 
ucs_status_t uct_ep_check(const uct_ep_h ep, unsigned flags,
                          uct_completion_t *comp);
 
 
ucs_status_t uct_iface_event_fd_get(uct_iface_h iface, int *fd_p);
 
 
ucs_status_t uct_iface_event_arm(uct_iface_h iface, unsigned events);
 
 
ucs_status_t uct_iface_mem_alloc(uct_iface_h iface, size_t length, unsigned flags,
                                 const char *name, uct_allocated_memory_t *mem);
 
 
void uct_iface_mem_free(const uct_allocated_memory_t *mem);
 
 
ucs_status_t uct_iface_set_am_handler(uct_iface_h iface, uint8_t id,
                                      uct_am_callback_t cb, void *arg, uint32_t flags);
 
 
ucs_status_t uct_iface_set_am_tracer(uct_iface_h iface, uct_am_tracer_t tracer,
                                     void *arg);
 
 
ucs_status_t uct_iface_accept(uct_iface_h iface,
                              uct_conn_request_h conn_request);
 
 
ucs_status_t uct_iface_reject(uct_iface_h iface,
                              uct_conn_request_h conn_request);
 
 
ucs_status_t uct_ep_create(const uct_ep_params_t *params, uct_ep_h *ep_p);
 
 
ucs_status_t uct_ep_connect(uct_ep_h ep, const uct_ep_connect_params_t *params);
 
 
ucs_status_t uct_ep_disconnect(uct_ep_h ep, unsigned flags);
 
 
void uct_ep_destroy(uct_ep_h ep);
 
 
ucs_status_t uct_ep_get_address(uct_ep_h ep, uct_ep_addr_t *addr);
 
 
ucs_status_t uct_ep_connect_to_ep(uct_ep_h ep, const uct_device_addr_t *dev_addr,
                                  const uct_ep_addr_t *ep_addr);
 
 
ucs_status_t uct_md_query(uct_md_h md, uct_md_attr_t *md_attr);
 
 
typedef enum {
    UCT_MEM_ALLOC_PARAM_FIELD_FLAGS          = UCS_BIT(0),
 
    UCT_MEM_ALLOC_PARAM_FIELD_ADDRESS        = UCS_BIT(1),
 
    UCT_MEM_ALLOC_PARAM_FIELD_MEM_TYPE       = UCS_BIT(2),
 
    UCT_MEM_ALLOC_PARAM_FIELD_MDS            = UCS_BIT(3),
 
    UCT_MEM_ALLOC_PARAM_FIELD_NAME           = UCS_BIT(4),
 
    UCT_MEM_ALLOC_PARAM_FIELD_SYS_DEVICE     = UCS_BIT(5)
} uct_mem_alloc_params_field_t;
 
 
typedef struct {
    uint64_t                     field_mask;
 
    unsigned                     flags;
 
    void                         *address;
 
    ucs_memory_type_t            mem_type;
 
    struct {
        const uct_md_h           *mds;
 
        unsigned                 count;
    } mds;
 
    const char                   *name;
 
    ucs_sys_device_t             sys_device;
} uct_mem_alloc_params_t;
 
 
ucs_status_t uct_md_mem_advise(uct_md_h md, uct_mem_h memh, void *addr,
                               size_t length, uct_mem_advice_t advice);
 
 
ucs_status_t uct_md_mem_reg(uct_md_h md, void *address, size_t length,
                            unsigned flags, uct_mem_h *memh_p);
 
 
ucs_status_t uct_md_mem_dereg(uct_md_h md, uct_mem_h memh);
 
 
ucs_status_t uct_md_detect_memory_type(uct_md_h md, const void *addr,
                                       size_t length,
                                       ucs_memory_type_t *mem_type_p);
 
 
ucs_status_t uct_mem_alloc(size_t length, const uct_alloc_method_t *methods,
                           unsigned num_methods,
                           const uct_mem_alloc_params_t *params,
                           uct_allocated_memory_t *mem);
 
 
ucs_status_t uct_mem_free(const uct_allocated_memory_t *mem);
 
ucs_status_t uct_md_config_read(uct_component_h component,
                                const char *env_prefix, const char *filename,
                                uct_md_config_t **config_p);
 
 
int uct_md_is_sockaddr_accessible(uct_md_h md, const ucs_sock_addr_t *sockaddr,
                                  uct_sockaddr_accessibility_t mode);
 
 
ucs_status_t uct_md_mkey_pack(uct_md_h md, uct_mem_h memh, void *rkey_buffer);
 
 
ucs_status_t uct_rkey_unpack(uct_component_h component, const void *rkey_buffer,
                             uct_rkey_bundle_t *rkey_ob);
 
 
ucs_status_t uct_rkey_ptr(uct_component_h component, uct_rkey_bundle_t *rkey_ob,
                          uint64_t remote_addr, void **addr_p);
 
 
ucs_status_t uct_rkey_release(uct_component_h component,
                              const uct_rkey_bundle_t *rkey_ob);
 
 
UCT_INLINE_API unsigned uct_worker_progress(uct_worker_h worker)
{
    return ucs_callbackq_dispatch(&worker->progress_q);
}
 
 
UCT_INLINE_API ucs_status_t uct_iface_flush(uct_iface_h iface, unsigned flags,
                                            uct_completion_t *comp)
{
    return iface->ops.iface_flush(iface, flags, comp);
}
 
UCT_INLINE_API ucs_status_t uct_iface_fence(uct_iface_h iface, unsigned flags)
{
    return iface->ops.iface_fence(iface, flags);
}
 
UCT_INLINE_API void uct_iface_release_desc(void *desc)
{
    uct_recv_desc_t *release_desc = uct_recv_desc(desc);
    release_desc->cb(release_desc, desc);
}
 
 
UCT_INLINE_API ucs_status_t uct_ep_put_short(uct_ep_h ep, const void *buffer, unsigned length,
                                             uint64_t remote_addr, uct_rkey_t rkey)
{
    return ep->iface->ops.ep_put_short(ep, buffer, length, remote_addr, rkey);
}
 
 
UCT_INLINE_API ssize_t uct_ep_put_bcopy(uct_ep_h ep, uct_pack_callback_t pack_cb,
                                        void *arg, uint64_t remote_addr,
                                        uct_rkey_t rkey)
{
    return ep->iface->ops.ep_put_bcopy(ep, pack_cb, arg, remote_addr, rkey);
}
 
 
UCT_INLINE_API ucs_status_t uct_ep_put_zcopy(uct_ep_h ep,
                                             const uct_iov_t *iov, size_t iovcnt,
                                             uint64_t remote_addr, uct_rkey_t rkey,
                                             uct_completion_t *comp)
{
    return ep->iface->ops.ep_put_zcopy(ep, iov, iovcnt, remote_addr, rkey, comp);
}
 
 
UCT_INLINE_API ucs_status_t uct_ep_get_short(uct_ep_h ep, void *buffer, unsigned length,
                                             uint64_t remote_addr, uct_rkey_t rkey)
{
    return ep->iface->ops.ep_get_short(ep, buffer, length, remote_addr, rkey);
}
 
 
UCT_INLINE_API ucs_status_t uct_ep_get_bcopy(uct_ep_h ep, uct_unpack_callback_t unpack_cb,
                                             void *arg, size_t length,
                                             uint64_t remote_addr, uct_rkey_t rkey,
                                             uct_completion_t *comp)
{
    return ep->iface->ops.ep_get_bcopy(ep, unpack_cb, arg, length, remote_addr,
                                       rkey, comp);
}
 
 
UCT_INLINE_API ucs_status_t uct_ep_get_zcopy(uct_ep_h ep,
                                             const uct_iov_t *iov, size_t iovcnt,
                                             uint64_t remote_addr, uct_rkey_t rkey,
                                             uct_completion_t *comp)
{
    return ep->iface->ops.ep_get_zcopy(ep, iov, iovcnt, remote_addr, rkey, comp);
}
 
 
UCT_INLINE_API ucs_status_t uct_ep_am_short(uct_ep_h ep, uint8_t id, uint64_t header,
                                            const void *payload, unsigned length)
{
    return ep->iface->ops.ep_am_short(ep, id, header, payload, length);
}
 
 
UCT_INLINE_API ucs_status_t uct_ep_am_short_iov(uct_ep_h ep, uint8_t id,
                                                const uct_iov_t *iov, size_t iovcnt)
{
    return ep->iface->ops.ep_am_short_iov(ep, id, iov, iovcnt);
}
 
 
UCT_INLINE_API ssize_t uct_ep_am_bcopy(uct_ep_h ep, uint8_t id,
                                       uct_pack_callback_t pack_cb, void *arg,
                                       unsigned flags)
{
    return ep->iface->ops.ep_am_bcopy(ep, id, pack_cb, arg, flags);
}
 
 
UCT_INLINE_API ucs_status_t uct_ep_am_zcopy(uct_ep_h ep, uint8_t id,
                                            const void *header,
                                            unsigned header_length,
                                            const uct_iov_t *iov, size_t iovcnt,
                                            unsigned flags,
                                            uct_completion_t *comp)
{
    return ep->iface->ops.ep_am_zcopy(ep, id, header, header_length, iov, iovcnt,
                                      flags, comp);
}
 
UCT_INLINE_API ucs_status_t uct_ep_atomic_cswap64(uct_ep_h ep, uint64_t compare, uint64_t swap,
                                                  uint64_t remote_addr, uct_rkey_t rkey,
                                                  uint64_t *result, uct_completion_t *comp)
{
    return ep->iface->ops.ep_atomic_cswap64(ep, compare, swap, remote_addr, rkey, result, comp);
}
 
 
UCT_INLINE_API ucs_status_t uct_ep_atomic_cswap32(uct_ep_h ep, uint32_t compare, uint32_t swap,
                                                  uint64_t remote_addr, uct_rkey_t rkey,
                                                  uint32_t *result, uct_completion_t *comp)
{
    return ep->iface->ops.ep_atomic_cswap32(ep, compare, swap, remote_addr, rkey, result, comp);
}
 
 
UCT_INLINE_API ucs_status_t uct_ep_atomic32_post(uct_ep_h ep, uct_atomic_op_t opcode,
                                                 uint32_t value, uint64_t remote_addr,
                                                 uct_rkey_t rkey)
{
    return ep->iface->ops.ep_atomic32_post(ep, opcode, value, remote_addr, rkey);
}
 
 
UCT_INLINE_API ucs_status_t uct_ep_atomic64_post(uct_ep_h ep, uct_atomic_op_t opcode,
                                                 uint64_t value, uint64_t remote_addr,
                                                 uct_rkey_t rkey)
{
    return ep->iface->ops.ep_atomic64_post(ep, opcode, value, remote_addr, rkey);
}
 
 
UCT_INLINE_API ucs_status_t uct_ep_atomic32_fetch(uct_ep_h ep, uct_atomic_op_t opcode,
                                                  uint32_t value, uint32_t *result,
                                                  uint64_t remote_addr, uct_rkey_t rkey,
                                                  uct_completion_t *comp)
{
    return ep->iface->ops.ep_atomic32_fetch(ep, opcode, value, result,
                                            remote_addr, rkey, comp);
}
 
 
UCT_INLINE_API ucs_status_t uct_ep_atomic64_fetch(uct_ep_h ep, uct_atomic_op_t opcode,
                                                  uint64_t value, uint64_t *result,
                                                  uint64_t remote_addr, uct_rkey_t rkey,
                                                  uct_completion_t *comp)
{
    return ep->iface->ops.ep_atomic64_fetch(ep, opcode, value, result,
                                            remote_addr, rkey, comp);
}
 
 
UCT_INLINE_API ucs_status_t uct_ep_pending_add(uct_ep_h ep,
                                               uct_pending_req_t *req,
                                               unsigned flags)
{
    return ep->iface->ops.ep_pending_add(ep, req, flags);
}
 
 
UCT_INLINE_API void uct_ep_pending_purge(uct_ep_h ep,
                                         uct_pending_purge_callback_t cb,
                                         void *arg)
{
    ep->iface->ops.ep_pending_purge(ep, cb, arg);
}
 
 
UCT_INLINE_API ucs_status_t uct_ep_flush(uct_ep_h ep, unsigned flags,
                                         uct_completion_t *comp)
{
    return ep->iface->ops.ep_flush(ep, flags, comp);
}
 
 
UCT_INLINE_API ucs_status_t uct_ep_fence(uct_ep_h ep, unsigned flags)
{
    return ep->iface->ops.ep_fence(ep, flags);
}
 
 
UCT_INLINE_API ucs_status_t uct_ep_tag_eager_short(uct_ep_h ep, uct_tag_t tag,
                                                   const void *data, size_t length)
{
    return ep->iface->ops.ep_tag_eager_short(ep, tag, data, length);
}
 
 
UCT_INLINE_API ssize_t uct_ep_tag_eager_bcopy(uct_ep_h ep, uct_tag_t tag,
                                              uint64_t imm,
                                              uct_pack_callback_t pack_cb,
                                              void *arg, unsigned flags)
{
    return ep->iface->ops.ep_tag_eager_bcopy(ep, tag, imm, pack_cb, arg, flags);
}
 
 
UCT_INLINE_API ucs_status_t uct_ep_tag_eager_zcopy(uct_ep_h ep, uct_tag_t tag,
                                                   uint64_t imm,
                                                   const uct_iov_t *iov,
                                                   size_t iovcnt,
                                                   unsigned flags,
                                                   uct_completion_t *comp)
{
    return ep->iface->ops.ep_tag_eager_zcopy(ep, tag, imm, iov, iovcnt, flags,
                                             comp);
}
 
 
UCT_INLINE_API ucs_status_ptr_t uct_ep_tag_rndv_zcopy(uct_ep_h ep, uct_tag_t tag,
                                                      const void *header,
                                                      unsigned header_length,
                                                      const uct_iov_t *iov,
                                                      size_t iovcnt,
                                                      unsigned flags,
                                                      uct_completion_t *comp)
{
    return ep->iface->ops.ep_tag_rndv_zcopy(ep, tag, header, header_length,
                                            iov, iovcnt, flags, comp);
}
 
 
UCT_INLINE_API ucs_status_t uct_ep_tag_rndv_cancel(uct_ep_h ep, void *op)
{
    return ep->iface->ops.ep_tag_rndv_cancel(ep, op);
}
 
 
UCT_INLINE_API ucs_status_t uct_ep_tag_rndv_request(uct_ep_h ep, uct_tag_t tag,
                                                    const void* header,
                                                    unsigned header_length,
                                                    unsigned flags)
{
    return ep->iface->ops.ep_tag_rndv_request(ep, tag, header, header_length,
                                              flags);
}
 
 
UCT_INLINE_API ucs_status_t uct_iface_tag_recv_zcopy(uct_iface_h iface,
                                                     uct_tag_t tag,
                                                     uct_tag_t tag_mask,
                                                     const uct_iov_t *iov,
                                                     size_t iovcnt,
                                                     uct_tag_context_t *ctx)
{
    return iface->ops.iface_tag_recv_zcopy(iface, tag, tag_mask, iov, iovcnt, ctx);
}
 
 
UCT_INLINE_API ucs_status_t uct_iface_tag_recv_cancel(uct_iface_h iface,
                                                      uct_tag_context_t *ctx,
                                                      int force)
{
    return iface->ops.iface_tag_recv_cancel(iface, ctx, force);
}
 
 
UCT_INLINE_API void uct_iface_progress_enable(uct_iface_h iface, unsigned flags)
{
    iface->ops.iface_progress_enable(iface, flags);
}
 
 
UCT_INLINE_API void uct_iface_progress_disable(uct_iface_h iface, unsigned flags)
{
    iface->ops.iface_progress_disable(iface, flags);
}
 
 
UCT_INLINE_API unsigned uct_iface_progress(uct_iface_h iface)
{
    return iface->ops.iface_progress(iface);
}
 
 
ucs_status_t uct_cm_open(uct_component_h component, uct_worker_h worker,
                         const uct_cm_config_t *config, uct_cm_h *cm_p);
 
 
void uct_cm_close(uct_cm_h cm);
 
 
ucs_status_t uct_cm_query(uct_cm_h cm, uct_cm_attr_t *cm_attr);
 
 
ucs_status_t uct_cm_config_read(uct_component_h component,
                                const char *env_prefix, const char *filename,
                                uct_cm_config_t **config_p);
 
 
ucs_status_t uct_cm_client_ep_conn_notify(uct_ep_h ep);
 
 
ucs_status_t uct_listener_create(uct_cm_h cm, const struct sockaddr *saddr,
                                 socklen_t socklen,
                                 const uct_listener_params_t *params,
                                 uct_listener_h *listener_p);
 
 
void uct_listener_destroy(uct_listener_h listener);
 
 
ucs_status_t uct_listener_reject(uct_listener_h listener,
                                 uct_conn_request_h conn_request);
 
 
ucs_status_t uct_listener_query(uct_listener_h listener,
                                uct_listener_attr_t *listener_attr);
 
 
static UCS_F_ALWAYS_INLINE
void uct_completion_update_status(uct_completion_t *comp, ucs_status_t status)
{
    if (ucs_unlikely(status != UCS_OK) && (comp->status == UCS_OK)) {
        /* store first failure status */
        comp->status = status;
    }
}
 
 
END_C_DECLS
 
#endif