ucp.h

/*
* Copyright (C) Mellanox Technologies Ltd. 2001-2014.  ALL RIGHTS RESERVED.
* Copyright (C) UT-Battelle, LLC. 2014-2017. ALL RIGHTS RESERVED.
* Copyright (C) ARM Ltd. 2016-2017.  ALL RIGHTS RESERVED.
* Copyright (C) Los Alamos National Security, LLC. 2018 ALL RIGHTS RESERVED.
* See file LICENSE for terms.
*/

#ifndef UCP_H_
#define UCP_H_

#include <ucp/api/ucp_def.h>
#include <ucp/api/ucp_compat.h>
#include <ucp/api/ucp_version.h>
#include <ucs/type/thread_mode.h>
#include <ucs/type/cpu_set.h>
#include <ucs/config/types.h>
#include <ucs/sys/compiler_def.h>
#include <stdio.h>
#include <sys/types.h>

BEGIN_C_DECLS

enum ucp_params_field {
    UCP_PARAM_FIELD_FEATURES          = UCS_BIT(0), 
    UCP_PARAM_FIELD_REQUEST_SIZE      = UCS_BIT(1), 
    UCP_PARAM_FIELD_REQUEST_INIT      = UCS_BIT(2), 
    UCP_PARAM_FIELD_REQUEST_CLEANUP   = UCS_BIT(3), 
    UCP_PARAM_FIELD_TAG_SENDER_MASK   = UCS_BIT(4), 
    UCP_PARAM_FIELD_MT_WORKERS_SHARED = UCS_BIT(5), 
    UCP_PARAM_FIELD_ESTIMATED_NUM_EPS = UCS_BIT(6), 
    UCP_PARAM_FIELD_ESTIMATED_NUM_PPN = UCS_BIT(7)  
};


enum ucp_feature {
    UCP_FEATURE_TAG          = UCS_BIT(0),  
    UCP_FEATURE_RMA          = UCS_BIT(1),  
    UCP_FEATURE_AMO32        = UCS_BIT(2),  
    UCP_FEATURE_AMO64        = UCS_BIT(3),  
    UCP_FEATURE_WAKEUP       = UCS_BIT(4),  
    UCP_FEATURE_STREAM       = UCS_BIT(5),  
    UCP_FEATURE_AM           = UCS_BIT(6)   
};


enum ucp_worker_params_field {
    UCP_WORKER_PARAM_FIELD_THREAD_MODE  = UCS_BIT(0), 
    UCP_WORKER_PARAM_FIELD_CPU_MASK     = UCS_BIT(1), 
    UCP_WORKER_PARAM_FIELD_EVENTS       = UCS_BIT(2), 
    UCP_WORKER_PARAM_FIELD_USER_DATA    = UCS_BIT(3), 
    UCP_WORKER_PARAM_FIELD_EVENT_FD     = UCS_BIT(4)  
};


enum ucp_listener_params_field {
    UCP_LISTENER_PARAM_FIELD_SOCK_ADDR           = UCS_BIT(0),

    UCP_LISTENER_PARAM_FIELD_ACCEPT_HANDLER      = UCS_BIT(1),

    UCP_LISTENER_PARAM_FIELD_CONN_HANDLER        = UCS_BIT(2)
};


typedef enum {

    UCP_WORKER_ADDRESS_FLAG_NET_ONLY = UCS_BIT(0)
} ucp_worker_address_flags_t;


enum ucp_ep_params_field {
    UCP_EP_PARAM_FIELD_REMOTE_ADDRESS    = UCS_BIT(0), 
    UCP_EP_PARAM_FIELD_ERR_HANDLING_MODE = UCS_BIT(1), 
    UCP_EP_PARAM_FIELD_ERR_HANDLER       = UCS_BIT(2), 
    UCP_EP_PARAM_FIELD_USER_DATA         = UCS_BIT(3), 
    UCP_EP_PARAM_FIELD_SOCK_ADDR         = UCS_BIT(4), 
    UCP_EP_PARAM_FIELD_FLAGS             = UCS_BIT(5), 
    UCP_EP_PARAM_FIELD_CONN_REQUEST      = UCS_BIT(6)  
};


enum ucp_ep_params_flags_field {
    UCP_EP_PARAMS_FLAGS_CLIENT_SERVER  = UCS_BIT(0),  
    UCP_EP_PARAMS_FLAGS_NO_LOOPBACK    = UCS_BIT(1)   
};


typedef enum {
    UCP_EP_CLOSE_FLAG_FORCE = UCS_BIT(0) 
} ucp_ep_close_flags_t;


enum ucp_ep_close_mode {
    UCP_EP_CLOSE_MODE_FORCE         = 0, 
    UCP_EP_CLOSE_MODE_FLUSH         = 1  
};


enum ucp_mem_map_params_field {
    UCP_MEM_MAP_PARAM_FIELD_ADDRESS = UCS_BIT(0), 
    UCP_MEM_MAP_PARAM_FIELD_LENGTH  = UCS_BIT(1), 
    UCP_MEM_MAP_PARAM_FIELD_FLAGS   = UCS_BIT(2)  
};

enum ucp_mem_advise_params_field {
    UCP_MEM_ADVISE_PARAM_FIELD_ADDRESS = UCS_BIT(0), 
    UCP_MEM_ADVISE_PARAM_FIELD_LENGTH  = UCS_BIT(1), 
    UCP_MEM_ADVISE_PARAM_FIELD_ADVICE  = UCS_BIT(2)  
};


enum ucp_context_attr_field {
    UCP_ATTR_FIELD_REQUEST_SIZE = UCS_BIT(0), 
    UCP_ATTR_FIELD_THREAD_MODE  = UCS_BIT(1)  
};


enum ucp_worker_attr_field {
    UCP_WORKER_ATTR_FIELD_THREAD_MODE   = UCS_BIT(0), 
    UCP_WORKER_ATTR_FIELD_ADDRESS       = UCS_BIT(1), 
    UCP_WORKER_ATTR_FIELD_ADDRESS_FLAGS = UCS_BIT(2)  
};


enum ucp_listener_attr_field {
    UCP_LISTENER_ATTR_FIELD_SOCKADDR = UCS_BIT(0) 
};


enum ucp_conn_request_attr_field {
    UCP_CONN_REQUEST_ATTR_FIELD_CLIENT_ADDR = UCS_BIT(0) 
};


enum ucp_dt_type {
    UCP_DATATYPE_CONTIG  = 0,      
    UCP_DATATYPE_STRIDED = 1,      
    UCP_DATATYPE_IOV     = 2,      
    UCP_DATATYPE_GENERIC = 7,      
    UCP_DATATYPE_SHIFT   = 3,      
    UCP_DATATYPE_CLASS_MASK = UCS_MASK(UCP_DATATYPE_SHIFT) 
};


enum {
    UCP_MEM_MAP_NONBLOCK = UCS_BIT(0), 
    UCP_MEM_MAP_ALLOCATE = UCS_BIT(1), 
    UCP_MEM_MAP_FIXED    = UCS_BIT(2)  
};


enum ucp_am_cb_flags {
    UCP_AM_FLAG_WHOLE_MSG = UCS_BIT(0)
};


enum ucp_send_am_flags {
    UCP_AM_SEND_REPLY = UCS_BIT(0)
};


enum ucp_cb_param_flags {
    UCP_CB_PARAM_FLAG_DATA = UCS_BIT(0)
};


typedef enum {
    UCP_ATOMIC_POST_OP_ADD, 
    UCP_ATOMIC_POST_OP_AND, 
    UCP_ATOMIC_POST_OP_OR,  
    UCP_ATOMIC_POST_OP_XOR, 
    UCP_ATOMIC_POST_OP_LAST
} ucp_atomic_post_op_t;


typedef enum {
    UCP_ATOMIC_FETCH_OP_FADD,  
    UCP_ATOMIC_FETCH_OP_SWAP,  
    UCP_ATOMIC_FETCH_OP_CSWAP, 
    UCP_ATOMIC_FETCH_OP_FAND,  
    UCP_ATOMIC_FETCH_OP_FOR,   
    UCP_ATOMIC_FETCH_OP_FXOR,  
    UCP_ATOMIC_FETCH_OP_LAST
} ucp_atomic_fetch_op_t;


typedef enum {
    UCP_ATOMIC_OP_ADD,   
    UCP_ATOMIC_OP_SWAP,  
    UCP_ATOMIC_OP_CSWAP, 
    UCP_ATOMIC_OP_AND,   
    UCP_ATOMIC_OP_OR,    
    UCP_ATOMIC_OP_XOR,   
    UCP_ATOMIC_OP_LAST
} ucp_atomic_op_t;


typedef enum {
    UCP_STREAM_RECV_FLAG_WAITALL = UCS_BIT(0)  
} ucp_stream_recv_flags_t;


typedef enum {
    UCP_OP_ATTR_FIELD_REQUEST       = UCS_BIT(0),  
    UCP_OP_ATTR_FIELD_CALLBACK      = UCS_BIT(1),  
    UCP_OP_ATTR_FIELD_USER_DATA     = UCS_BIT(2),  
    UCP_OP_ATTR_FIELD_DATATYPE      = UCS_BIT(3),  
    UCP_OP_ATTR_FIELD_FLAGS         = UCS_BIT(4),  
    UCP_OP_ATTR_FIELD_REPLY_BUFFER  = UCS_BIT(5),  
    UCP_OP_ATTR_FLAG_NO_IMM_CMPL    = UCS_BIT(16), 
    UCP_OP_ATTR_FLAG_FAST_CMPL      = UCS_BIT(17), 
    UCP_OP_ATTR_FLAG_FORCE_IMM_CMPL = UCS_BIT(18)  
} ucp_op_attr_t;


#define ucp_dt_make_contig(_elem_size) \
    (((ucp_datatype_t)(_elem_size) << UCP_DATATYPE_SHIFT) | UCP_DATATYPE_CONTIG)


#define ucp_dt_make_iov() (UCP_DATATYPE_IOV)


typedef struct ucp_dt_iov {
    void   *buffer;   
    size_t  length;   
} ucp_dt_iov_t;


typedef struct ucp_generic_dt_ops {

    void* (*start_pack)(void *context, const void *buffer, size_t count);

    void* (*start_unpack)(void *context, void *buffer, size_t count);

    size_t (*packed_size)(void *state);

    size_t (*pack) (void *state, size_t offset, void *dest, size_t max_length);

    ucs_status_t (*unpack)(void *state, size_t offset, const void *src, size_t length);

    void (*finish)(void *state);
} ucp_generic_dt_ops_t;


typedef struct ucp_params {
    uint64_t                           field_mask;

    uint64_t                           features;

    size_t                             request_size;

    ucp_request_init_callback_t        request_init;

    ucp_request_cleanup_callback_t     request_cleanup;

    uint64_t                           tag_sender_mask;

    int                                mt_workers_shared;

    size_t                             estimated_num_eps;

    size_t                             estimated_num_ppn;
} ucp_params_t;


typedef struct ucp_context_attr {
    uint64_t              field_mask;

    size_t                request_size;

    ucs_thread_mode_t     thread_mode;
} ucp_context_attr_t;


typedef struct ucp_worker_attr {
    uint64_t              field_mask;

    ucs_thread_mode_t     thread_mode;

    uint32_t              address_flags;

    ucp_address_t         *address;

    size_t                address_length;
} ucp_worker_attr_t;


typedef struct ucp_worker_params {
    uint64_t                field_mask;

    ucs_thread_mode_t       thread_mode;

    ucs_cpu_set_t           cpu_mask;

    unsigned                events;

    void                    *user_data;

    int                     event_fd;

} ucp_worker_params_t;


typedef struct ucp_listener_attr {
    uint64_t                field_mask;

    struct sockaddr_storage sockaddr;
} ucp_listener_attr_t;


typedef struct ucp_conn_request_attr {
    uint64_t                field_mask;

    struct sockaddr_storage client_address;
} ucp_conn_request_attr_t;


typedef struct ucp_listener_params {
    uint64_t                            field_mask;

    ucs_sock_addr_t                     sockaddr;

    ucp_listener_accept_handler_t       accept_handler;

    ucp_listener_conn_handler_t         conn_handler;
} ucp_listener_params_t;


typedef struct ucp_stream_poll_ep {
    ucp_ep_h    ep;

    void        *user_data;

    unsigned    flags;

    uint8_t     reserved[16];
} ucp_stream_poll_ep_t;


typedef struct ucp_mem_map_params {
    uint64_t                field_mask;

     void                   *address;

     size_t                 length;

     unsigned               flags;
} ucp_mem_map_params_t;


struct ucp_tag_recv_info {
    ucp_tag_t                              sender_tag;
    size_t                                 length;
};


typedef struct {
    uint32_t       op_attr_mask;

    /* Operation specific flags. */
    uint32_t       flags;

    void          *request;

    union {
        ucp_send_nbx_callback_t        send;
        ucp_tag_recv_nbx_callback_t    recv;
        ucp_stream_recv_nbx_callback_t recv_stream;
    }              cb;

    ucp_datatype_t datatype;

    void          *user_data;

    void          *reply_buffer;
} ucp_request_param_t;


ucs_status_t ucp_config_read(const char *env_prefix, const char *filename,
                             ucp_config_t **config_p);


void ucp_config_release(ucp_config_t *config);


ucs_status_t ucp_config_modify(ucp_config_t *config, const char *name,
                               const char *value);


void ucp_config_print(const ucp_config_t *config, FILE *stream,
                      const char *title, ucs_config_print_flags_t print_flags);


void ucp_get_version(unsigned *major_version, unsigned *minor_version,
                     unsigned *release_number);


const char *ucp_get_version_string(void);


ucs_status_t ucp_init_version(unsigned api_major_version, unsigned api_minor_version,
                              const ucp_params_t *params, const ucp_config_t *config,
                              ucp_context_h *context_p);
static inline ucs_status_t ucp_init(const ucp_params_t *params,
                                    const ucp_config_t *config,
                                    ucp_context_h *context_p)
{
    return ucp_init_version(UCP_API_MAJOR, UCP_API_MINOR, params, config,
                            context_p);
}


void ucp_cleanup(ucp_context_h context_p);


ucs_status_t ucp_context_query(ucp_context_h context_p,
                               ucp_context_attr_t *attr);


void ucp_context_print_info(const ucp_context_h context, FILE *stream);


ucs_status_t ucp_worker_create(ucp_context_h context,
                               const ucp_worker_params_t *params,
                               ucp_worker_h *worker_p);


void ucp_worker_destroy(ucp_worker_h worker);


ucs_status_t ucp_worker_query(ucp_worker_h worker,
                              ucp_worker_attr_t *attr);


void ucp_worker_print_info(ucp_worker_h worker, FILE *stream);


ucs_status_t ucp_worker_get_address(ucp_worker_h worker,
                                    ucp_address_t **address_p,
                                    size_t *address_length_p);


void ucp_worker_release_address(ucp_worker_h worker, ucp_address_t *address);


unsigned ucp_worker_progress(ucp_worker_h worker);


ssize_t ucp_stream_worker_poll(ucp_worker_h worker,
                               ucp_stream_poll_ep_t *poll_eps, size_t max_eps,
                               unsigned flags);


ucs_status_t ucp_worker_get_efd(ucp_worker_h worker, int *fd);


ucs_status_t ucp_worker_wait(ucp_worker_h worker);


void ucp_worker_wait_mem(ucp_worker_h worker, void *address);


ucs_status_t ucp_worker_arm(ucp_worker_h worker);


ucs_status_t ucp_worker_signal(ucp_worker_h worker);


ucs_status_t ucp_listener_create(ucp_worker_h worker,
                                 const ucp_listener_params_t *params,
                                 ucp_listener_h *listener_p);


void ucp_listener_destroy(ucp_listener_h listener);


ucs_status_t ucp_listener_query(ucp_listener_h listener, ucp_listener_attr_t *attr);


ucs_status_t ucp_conn_request_query(ucp_conn_request_h conn_request,
                                    ucp_conn_request_attr_t *attr);


ucs_status_t ucp_ep_create(ucp_worker_h worker, const ucp_ep_params_t *params,
                           ucp_ep_h *ep_p);


ucs_status_ptr_t ucp_ep_close_nb(ucp_ep_h ep, unsigned mode);


ucs_status_ptr_t ucp_ep_close_nbx(ucp_ep_h ep,
                                  const ucp_request_param_t *param);


ucs_status_t ucp_listener_reject(ucp_listener_h listener,
                                 ucp_conn_request_h conn_request);


void ucp_ep_print_info(ucp_ep_h ep, FILE *stream);


ucs_status_ptr_t ucp_ep_flush_nb(ucp_ep_h ep, unsigned flags,
                                 ucp_send_callback_t cb);


ucs_status_ptr_t ucp_ep_flush_nbx(ucp_ep_h ep, const ucp_request_param_t *param);


ucs_status_t ucp_mem_map(ucp_context_h context, const ucp_mem_map_params_t *params,
                         ucp_mem_h *memh_p);


ucs_status_t ucp_mem_unmap(ucp_context_h context, ucp_mem_h memh);


ucs_status_t ucp_mem_query(const ucp_mem_h memh, ucp_mem_attr_t *attr);


void ucp_mem_print_info(const char *mem_size, ucp_context_h context, FILE *stream);


typedef enum ucp_mem_advice {
    UCP_MADV_NORMAL   = 0,  
    UCP_MADV_WILLNEED       
} ucp_mem_advice_t;


typedef struct ucp_mem_advise_params {
    uint64_t                field_mask;

     void                   *address;

     size_t                 length;

     ucp_mem_advice_t       advice;
} ucp_mem_advise_params_t;


ucs_status_t ucp_mem_advise(ucp_context_h context, ucp_mem_h memh,
                            ucp_mem_advise_params_t *params);


ucs_status_t ucp_rkey_pack(ucp_context_h context, ucp_mem_h memh,
                           void **rkey_buffer_p, size_t *size_p);


void ucp_rkey_buffer_release(void *rkey_buffer);


ucs_status_t ucp_ep_rkey_unpack(ucp_ep_h ep, const void *rkey_buffer,
                                ucp_rkey_h *rkey_p);


ucs_status_t ucp_rkey_ptr(ucp_rkey_h rkey, uint64_t raddr, void **addr_p);


void ucp_rkey_destroy(ucp_rkey_h rkey);


ucs_status_t ucp_worker_set_am_handler(ucp_worker_h worker, uint16_t id,
                                       ucp_am_callback_t cb, void *arg,
                                       uint32_t flags);


ucs_status_ptr_t ucp_am_send_nb(ucp_ep_h ep, uint16_t id,
                                const void *buffer, size_t count,
                                ucp_datatype_t datatype,
                                ucp_send_callback_t cb, unsigned flags);


void ucp_am_data_release(ucp_worker_h worker, void *data);


ucs_status_ptr_t ucp_stream_send_nb(ucp_ep_h ep, const void *buffer, size_t count,
                                    ucp_datatype_t datatype, ucp_send_callback_t cb,
                                    unsigned flags);


ucs_status_ptr_t ucp_stream_send_nbx(ucp_ep_h ep, const void *buffer, size_t count,
                                     const ucp_request_param_t *param);


ucs_status_ptr_t ucp_tag_send_nb(ucp_ep_h ep, const void *buffer, size_t count,
                                 ucp_datatype_t datatype, ucp_tag_t tag,
                                 ucp_send_callback_t cb);

ucs_status_t ucp_tag_send_nbr(ucp_ep_h ep, const void *buffer, size_t count,
                              ucp_datatype_t datatype, ucp_tag_t tag, void *req);

ucs_status_ptr_t ucp_tag_send_sync_nb(ucp_ep_h ep, const void *buffer, size_t count,
                                      ucp_datatype_t datatype, ucp_tag_t tag,
                                      ucp_send_callback_t cb);


ucs_status_ptr_t ucp_tag_send_nbx(ucp_ep_h ep, const void *buffer, size_t count,
                                  ucp_tag_t tag, const ucp_request_param_t *param);


ucs_status_ptr_t ucp_tag_send_sync_nbx(ucp_ep_h ep, const void *buffer,
                                       size_t count, ucp_tag_t tag,
                                       const ucp_request_param_t *param);


ucs_status_ptr_t ucp_stream_recv_nb(ucp_ep_h ep, void *buffer, size_t count,
                                    ucp_datatype_t datatype,
                                    ucp_stream_recv_callback_t cb,
                                    size_t *length, unsigned flags);


ucs_status_ptr_t ucp_stream_recv_nbx(ucp_ep_h ep, void *buffer, size_t count,
                                     size_t *length,
                                     const ucp_request_param_t *param);


ucs_status_ptr_t ucp_stream_recv_data_nb(ucp_ep_h ep, size_t *length);


ucs_status_ptr_t ucp_tag_recv_nb(ucp_worker_h worker, void *buffer, size_t count,
                                 ucp_datatype_t datatype, ucp_tag_t tag,
                                 ucp_tag_t tag_mask, ucp_tag_recv_callback_t cb);


ucs_status_t ucp_tag_recv_nbr(ucp_worker_h worker, void *buffer, size_t count,
                              ucp_datatype_t datatype, ucp_tag_t tag,
                              ucp_tag_t tag_mask, void *req);


ucs_status_ptr_t ucp_tag_recv_nbx(ucp_worker_h worker, void *buffer, size_t count,
                                  ucp_tag_t tag, ucp_tag_t tag_mask,
                                  const ucp_request_param_t *param);


ucp_tag_message_h ucp_tag_probe_nb(ucp_worker_h worker, ucp_tag_t tag,
                                   ucp_tag_t tag_mask, int remove,
                                   ucp_tag_recv_info_t *info);


ucs_status_ptr_t ucp_tag_msg_recv_nb(ucp_worker_h worker, void *buffer,
                                     size_t count, ucp_datatype_t datatype,
                                     ucp_tag_message_h message,
                                     ucp_tag_recv_callback_t cb);


ucs_status_t ucp_put_nbi(ucp_ep_h ep, const void *buffer, size_t length,
                         uint64_t remote_addr, ucp_rkey_h rkey);

ucs_status_ptr_t ucp_put_nb(ucp_ep_h ep, const void *buffer, size_t length,
                            uint64_t remote_addr, ucp_rkey_h rkey,
                            ucp_send_callback_t cb);


ucs_status_ptr_t ucp_put_nbx(ucp_ep_h ep, const void *buffer, size_t count,
                             uint64_t remote_addr, ucp_rkey_h rkey,
                             const ucp_request_param_t *param);


ucs_status_t ucp_get_nbi(ucp_ep_h ep, void *buffer, size_t length,
                         uint64_t remote_addr, ucp_rkey_h rkey);

ucs_status_ptr_t ucp_get_nb(ucp_ep_h ep, void *buffer, size_t length,
                            uint64_t remote_addr, ucp_rkey_h rkey,
                            ucp_send_callback_t cb);


ucs_status_ptr_t ucp_get_nbx(ucp_ep_h ep, void *buffer, size_t count,
                             uint64_t remote_addr, ucp_rkey_h rkey,
                             const ucp_request_param_t *param);


ucs_status_t ucp_atomic_post(ucp_ep_h ep, ucp_atomic_post_op_t opcode, uint64_t value,
                             size_t op_size, uint64_t remote_addr, ucp_rkey_h rkey);


ucs_status_ptr_t
ucp_atomic_fetch_nb(ucp_ep_h ep, ucp_atomic_fetch_op_t opcode,
                    uint64_t value, void *result, size_t op_size,
                    uint64_t remote_addr, ucp_rkey_h rkey,
                    ucp_send_callback_t cb);


ucs_status_ptr_t
ucp_atomic_op_nbx(ucp_ep_h ep, ucp_atomic_op_t opcode, const void *buffer,
                  size_t count, uint64_t remote_addr, ucp_rkey_h rkey,
                  const ucp_request_param_t *param);


ucs_status_t ucp_request_check_status(void *request);


ucs_status_t ucp_tag_recv_request_test(void *request, ucp_tag_recv_info_t *info);


ucs_status_t ucp_stream_recv_request_test(void *request, size_t *length_p);

void ucp_request_cancel(ucp_worker_h worker, void *request);


void ucp_stream_data_release(ucp_ep_h ep, void *data);


void ucp_request_free(void *request);


void *ucp_request_alloc(ucp_worker_h worker);


ucs_status_t ucp_dt_create_generic(const ucp_generic_dt_ops_t *ops, void *context,
                                   ucp_datatype_t *datatype_p);


void ucp_dt_destroy(ucp_datatype_t datatype);


ucs_status_t ucp_worker_fence(ucp_worker_h worker);


ucs_status_ptr_t ucp_worker_flush_nb(ucp_worker_h worker, unsigned flags,
                                     ucp_send_callback_t cb);


ucs_status_ptr_t ucp_worker_flush_nbx(ucp_worker_h worker,
                                      const ucp_request_param_t *param);


END_C_DECLS

#endif