Skip to content

Commit

Permalink
UCP/RMA: Report the error if doing RMA/AMO for non-HOST memory
Browse files Browse the repository at this point in the history
  • Loading branch information
@dmitrygx
dmitrygx committed Dec 24, 2019
1 parent 4f844d9 commit ee55de8
Show file tree
Hide file tree
Showing 20 changed files with 758 additions and 216 deletions.
24 changes: 21 additions & 3 deletions src/tools/perf/lib/libperf.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -393,6 +393,22 @@ static ucs_status_t ucx_perf_test_check_params(ucx_perf_params_t *params)
return UCS_ERR_INVALID_PARAM;
}

if ((params->api == UCX_PERF_API_UCP) &&
(params->mem_type != UCS_MEMORY_TYPE_HOST) &&
((params->command == UCX_PERF_CMD_PUT) ||
(params->command == UCX_PERF_CMD_GET) ||
(params->command == UCX_PERF_CMD_ADD) ||
(params->command == UCX_PERF_CMD_FADD) ||
(params->command == UCX_PERF_CMD_SWAP) ||
(params->command == UCX_PERF_CMD_CSWAP))) {
/* TODO: remove when support for non-HOST memory types will be added */
if (params->flags & UCX_PERF_TEST_FLAG_VERBOSE) {
ucs_error("UCP doesn't support RMA/AMO for \"%s\" memory type",
ucs_memory_type_names[params->mem_type]);
}
return UCS_ERR_INVALID_PARAM;
}

if (params->max_outstanding < 1) {
if (params->flags & UCX_PERF_TEST_FLAG_VERBOSE) {
ucs_error("max_outstanding, need to be at least 1");
Expand Down Expand Up @@ -650,9 +666,11 @@ static ucs_status_t uct_perf_test_check_capabilities(ucx_perf_params_t *params,

if (!(md_attr.cap.access_mem_type == params->mem_type) &&
!(md_attr.cap.reg_mem_types & UCS_BIT(params->mem_type))) {
ucs_error("Unsupported memory type %s by %s/%s",
ucs_memory_type_names[params->mem_type],
params->uct.tl_name, params->uct.dev_name);
if (params->flags & UCX_PERF_TEST_FLAG_VERBOSE) {
ucs_error("Unsupported memory type %s by %s/%s",
ucs_memory_type_names[params->mem_type],
params->uct.tl_name, params->uct.dev_name);
}
return UCS_ERR_INVALID_PARAM;
}

Expand Down
2 changes: 1 addition & 1 deletion src/ucp/core/ucp_context.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@

ucp_am_handler_t ucp_am_handlers[UCP_AM_ID_LAST] = {{0, NULL, NULL}};

static const char *ucp_atomic_modes[] = {
const char *ucp_atomic_modes[] = {
[UCP_ATOMIC_MODE_CPU] = "cpu",
[UCP_ATOMIC_MODE_DEVICE] = "device",
[UCP_ATOMIC_MODE_GUESS] = "guess",
Expand Down
1 change: 1 addition & 0 deletions src/ucp/core/ucp_context.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -329,6 +329,7 @@ typedef struct ucp_tl_iface_atomic_flags {
ucs_assert(ucp_memory_type_detect(_context, _buffer, _length) == (_mem_type))


extern const char *ucp_atomic_modes[];
extern ucp_am_handler_t ucp_am_handlers[];
extern const char *ucp_feature_str[];

Expand Down
153 changes: 93 additions & 60 deletions src/ucp/core/ucp_ep.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,13 @@ static ucs_stats_class_t ucp_ep_stats_class = {
};
#endif

static const ucp_ep_rma_proto_t ucp_ep_rma_proto_dummy = {
.max_short = -1,
.max_bcopy = SIZE_MAX,
.zcopy_thresh = { SIZE_MAX, SIZE_MAX, SIZE_MAX, SIZE_MAX, SIZE_MAX },
.max_zcopy = SIZE_MAX
};


void ucp_ep_config_key_reset(ucp_ep_config_key_t *key)
{
Expand Down Expand Up @@ -1171,6 +1178,60 @@ static void ucp_ep_config_set_memtype_thresh(ucp_memtype_thresh_t *max_eager_sho
max_eager_short->memtype_on = max_short;
}

static void
ucp_ep_config_set_memtype_zcopy_thresh(const uct_md_attr_t *md_attr,
size_t zcopy_thresh[UCS_MEMORY_TYPE_LAST],
size_t zcopy_thresh_val)
{
ucs_memory_type_t mem_type;

UCS_STATIC_ASSERT(UCS_MEMORY_TYPE_HOST == 0);
for (mem_type = UCS_MEMORY_TYPE_HOST; mem_type < UCS_MEMORY_TYPE_LAST; mem_type++) {
if (UCP_MEM_IS_ACCESSIBLE_FROM_CPU(mem_type)) {
zcopy_thresh[mem_type] = zcopy_thresh_val;
} else if (md_attr->cap.reg_mem_types & UCS_BIT(mem_type)) {
zcopy_thresh[mem_type] = 1;
}
}
}

static void
ucp_ep_config_set_rma(const ucp_context_h context, const uct_md_attr_t *md_attr,
ucp_ep_rma_proto_t *rma_proto, uint64_t flags,
uint64_t short_flag, size_t max_short,
uint64_t bcopy_flag, size_t max_bcopy,
uint64_t zcopy_flag, size_t min_zcopy, size_t max_zcopy)
{
size_t zcopy_thresh;

/* Zcopy */
if (flags & zcopy_flag) {
rma_proto->max_zcopy = max_zcopy;
/* TODO: formula */
if (context->config.ext.zcopy_thresh == UCS_MEMUNITS_AUTO) {
zcopy_thresh = 16384;
} else {
zcopy_thresh = context->config.ext.zcopy_thresh;
}
zcopy_thresh = ucs_max(zcopy_thresh, min_zcopy);
ucp_ep_config_set_memtype_zcopy_thresh(md_attr, rma_proto->zcopy_thresh,
zcopy_thresh);
}

/* Bcopy */
if (flags & bcopy_flag) {
rma_proto->max_bcopy = ucs_min(max_bcopy,
rma_proto->zcopy_thresh[UCS_MEMORY_TYPE_HOST]);
}

/* Short */
if (flags & short_flag) {
ucs_assert(max_short <= SSIZE_MAX);
rma_proto->max_short = (context->num_mem_type_detect_mds ? -1 :
(ssize_t)ucs_min(max_short, rma_proto->max_bcopy));
}
}

static void ucp_ep_config_init_attrs(ucp_worker_t *worker, ucp_rsc_index_t rsc_index,
ucp_ep_msg_config_t *config, size_t max_short,
size_t max_bcopy, size_t max_zcopy,
Expand All @@ -1183,7 +1244,6 @@ static void ucp_ep_config_init_attrs(ucp_worker_t *worker, ucp_rsc_index_t rsc_i
uct_iface_attr_t *iface_attr;
size_t it;
size_t zcopy_thresh;
int mem_type;

iface_attr = ucp_worker_iface_get_attr(worker, rsc_index);

Expand Down Expand Up @@ -1231,13 +1291,9 @@ static void ucp_ep_config_init_attrs(ucp_worker_t *worker, ucp_rsc_index_t rsc_i
config->zcopy_thresh[0]);
}

for (mem_type = 0; mem_type < UCS_MEMORY_TYPE_LAST; mem_type++) {
if (UCP_MEM_IS_ACCESSIBLE_FROM_CPU(mem_type)) {
config->mem_type_zcopy_thresh[mem_type] = config->zcopy_thresh[0];
} else if (md_attr->cap.reg_mem_types & UCS_BIT(mem_type)) {
config->mem_type_zcopy_thresh[mem_type] = 1;
}
}
ucp_ep_config_set_memtype_zcopy_thresh(md_attr,
config->mem_type_zcopy_thresh,
config->zcopy_thresh[0]);
}

static ucs_status_t ucp_ep_config_key_copy(ucp_ep_config_key_t *dst,
Expand Down Expand Up @@ -1472,13 +1528,9 @@ ucs_status_t ucp_ep_config_init(ucp_worker_h worker, ucp_ep_config_t *config,

/* Configuration for remote memory access */
for (lane = 0; lane < config->key.num_lanes; ++lane) {
rma_config = &config->rma[lane];
rma_config->put_zcopy_thresh = SIZE_MAX;
rma_config->get_zcopy_thresh = SIZE_MAX;
rma_config->max_put_short = SIZE_MAX;
rma_config->max_get_short = SIZE_MAX;
rma_config->max_put_bcopy = SIZE_MAX;
rma_config->max_get_bcopy = SIZE_MAX;
rma_config = &config->rma[lane];
rma_config->put = ucp_ep_rma_proto_dummy;
rma_config->get = ucp_ep_rma_proto_dummy;

if (ucp_ep_config_get_multi_lane_prio(config->key.rma_lanes, lane) == -1) {
continue;
Expand All @@ -1488,49 +1540,28 @@ ucs_status_t ucp_ep_config_init(ucp_worker_h worker, ucp_ep_config_t *config,

if (rsc_index != UCP_NULL_RESOURCE) {
iface_attr = ucp_worker_iface_get_attr(worker, rsc_index);
/* PUT */
if (iface_attr->cap.flags & UCT_IFACE_FLAG_PUT_ZCOPY) {
rma_config->max_put_zcopy = iface_attr->cap.put.max_zcopy;
/* TODO: formula */
if (context->config.ext.zcopy_thresh == UCS_MEMUNITS_AUTO) {
rma_config->put_zcopy_thresh = 16384;
} else {
rma_config->put_zcopy_thresh = context->config.ext.zcopy_thresh;
}
rma_config->put_zcopy_thresh = ucs_max(rma_config->put_zcopy_thresh,
iface_attr->cap.put.min_zcopy);
}
if (iface_attr->cap.flags & UCT_IFACE_FLAG_PUT_BCOPY) {
rma_config->max_put_bcopy = ucs_min(iface_attr->cap.put.max_bcopy,
rma_config->put_zcopy_thresh);
}
if (iface_attr->cap.flags & UCT_IFACE_FLAG_PUT_SHORT) {
rma_config->max_put_short = ucs_min(iface_attr->cap.put.max_short,
rma_config->max_put_bcopy);
}
md_attr = &context->tl_mds[context->tl_rscs[rsc_index].md_index].attr;

/* GET */
if (iface_attr->cap.flags & UCT_IFACE_FLAG_GET_ZCOPY) {
/* TODO: formula */
rma_config->max_get_zcopy = iface_attr->cap.get.max_zcopy;
if (context->config.ext.zcopy_thresh == UCS_MEMUNITS_AUTO) {
rma_config->get_zcopy_thresh = 16384;
} else {
rma_config->get_zcopy_thresh = context->config.ext.zcopy_thresh;
}
rma_config->get_zcopy_thresh = ucs_max(rma_config->get_zcopy_thresh,
iface_attr->cap.get.min_zcopy);
}
if (iface_attr->cap.flags & UCT_IFACE_FLAG_GET_BCOPY) {
rma_config->max_get_bcopy = ucs_min(iface_attr->cap.get.max_bcopy,
rma_config->get_zcopy_thresh);
}
if (iface_attr->cap.flags & UCT_IFACE_FLAG_GET_SHORT) {
rma_config->max_get_short = ucs_min(iface_attr->cap.get.max_short,
rma_config->max_get_bcopy);
}
ucp_ep_config_set_rma(context, md_attr, &rma_config->put,
iface_attr->cap.flags,
UCT_IFACE_FLAG_PUT_SHORT,
iface_attr->cap.put.max_short,
UCT_IFACE_FLAG_PUT_BCOPY,
iface_attr->cap.put.max_bcopy,
UCT_IFACE_FLAG_PUT_ZCOPY,
iface_attr->cap.put.min_zcopy,
iface_attr->cap.put.max_zcopy); /* PUT */
ucp_ep_config_set_rma(context, md_attr, &rma_config->get,
iface_attr->cap.flags,
UCT_IFACE_FLAG_GET_SHORT,
iface_attr->cap.get.max_short,
UCT_IFACE_FLAG_GET_BCOPY,
iface_attr->cap.get.max_bcopy,
UCT_IFACE_FLAG_GET_ZCOPY,
iface_attr->cap.get.min_zcopy,
iface_attr->cap.get.max_zcopy); /* GET */
} else {
rma_config->max_put_bcopy = UCP_MIN_BCOPY; /* Stub endpoint */
rma_config->put.max_bcopy = UCP_MIN_BCOPY; /* Stub endpoint */
}
}

Expand Down Expand Up @@ -1767,12 +1798,14 @@ static void ucp_ep_config_print(FILE *stream, ucp_worker_h worker,
if (ucp_ep_config_get_multi_lane_prio(config->key.rma_lanes, lane) == -1) {
continue;
}
/* TODO: print for all supported memory types */
ucp_ep_config_print_rma_proto(stream, "put", lane,
ucs_max(config->rma[lane].max_put_short + 1,
ucs_max(config->rma[lane].put.max_short + 1,
config->bcopy_thresh),
config->rma[lane].put_zcopy_thresh);
ucp_ep_config_print_rma_proto(stream, "get", lane, 0,
config->rma[lane].get_zcopy_thresh);
config->rma[lane].put.zcopy_thresh[UCS_MEMORY_TYPE_HOST]);
ucp_ep_config_print_rma_proto(stream, "get", lane,
/* UCP RMA doesn't use UCT GET Short*/
0, config->rma[lane].get.zcopy_thresh[UCS_MEMORY_TYPE_HOST]);
}
}

Expand Down
22 changes: 14 additions & 8 deletions src/ucp/core/ucp_ep.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,18 +151,24 @@ typedef struct ucp_ep_config_key {
} ucp_ep_config_key_t;


/*
* Protocol limits for RMA
*/
typedef struct ucp_ep_rma_proto {
ssize_t max_short; /* Maximal payload of short */
size_t max_bcopy; /* Maximal total size of bcopy */
/* Minimal total size of zcopy for memory types */
size_t zcopy_thresh[UCS_MEMORY_TYPE_LAST];
size_t max_zcopy; /* Maximal total size of zcopy */
} ucp_ep_rma_proto_t;


/*
* Configuration for RMA protocols
*/
typedef struct ucp_ep_rma_config {
size_t max_put_short; /* Maximal payload of put short */
size_t max_put_bcopy; /* Maximal total size of put_bcopy */
size_t max_put_zcopy;
size_t max_get_short; /* Maximal payload of get short */
size_t max_get_bcopy; /* Maximal total size of get_bcopy */
size_t max_get_zcopy;
size_t put_zcopy_thresh;
size_t get_zcopy_thresh;
ucp_ep_rma_proto_t put; /* Protocol limits for PUT */
ucp_ep_rma_proto_t get; /* Protocol limits for GET */
} ucp_ep_rma_config_t;


Expand Down
30 changes: 18 additions & 12 deletions src/ucp/core/ucp_mm.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,22 +50,22 @@ enum {
typedef struct ucp_rkey {
/* cached values for the most recent endpoint configuration */
struct {
ucp_ep_cfg_index_t ep_cfg_index; /* EP configuration relevant for the cache */
ucp_lane_index_t rma_lane; /* Lane to use for RMAs */
ucp_lane_index_t amo_lane; /* Lane to use for AMOs */
unsigned max_put_short;/* Cached value of max_put_short */
uct_rkey_t rma_rkey; /* Key to use for RMAs */
uct_rkey_t amo_rkey; /* Key to use for AMOs */
ucp_amo_proto_t *amo_proto; /* Protocol for AMOs */
ucp_rma_proto_t *rma_proto; /* Protocol for RMAs */
ucp_ep_cfg_index_t ep_cfg_index; /* EP configuration relevant for the cache */
ucp_lane_index_t rma_lane; /* Lane to use for RMAs */
ucp_lane_index_t amo_lane; /* Lane to use for AMOs */
ssize_t max_put_short; /* Cached value of max_put_short */
uct_rkey_t rma_rkey; /* Key to use for RMAs */
uct_rkey_t amo_rkey; /* Key to use for AMOs */
ucp_amo_proto_t *amo_proto; /* Protocol for AMOs */
ucp_rma_proto_t *rma_proto; /* Protocol for RMAs */
} cache;
ucp_md_map_t md_map; /* Which *remote* MDs have valid memory handles */
ucs_memory_type_t mem_type; /* Memory type of remote key memory */
uint8_t flags; /* Rkey flags */
ucp_md_map_t md_map; /* Which *remote* MDs have valid memory handles */
ucs_memory_type_t mem_type; /* Memory type of remote key memory */
uint8_t flags; /* Rkey flags */
#if ENABLE_PARAMS_CHECK
ucp_ep_h ep;
#endif
ucp_tl_rkey_t tl_rkey[0]; /* UCT rkey for every remote MD */
ucp_tl_rkey_t tl_rkey[0]; /* UCT rkey for every remote MD */
} ucp_rkey_t;


Expand Down Expand Up @@ -154,6 +154,12 @@ ssize_t ucp_rkey_pack_uct(ucp_context_h context, ucp_md_map_t md_map,

void ucp_rkey_dump_packed(const void *rkey_buffer, char *buffer, size_t max);

void ucp_rkey_select_rma_lane(ucp_rkey_h rkey, ucp_ep_h ep,
ucs_memory_type_t local_mem_type);

void ucp_rkey_select_amo_lane(ucp_rkey_h rkey, ucp_ep_h ep,
ucs_memory_type_t local_mem_type);

ucs_status_t ucp_mem_type_reg_buffers(ucp_worker_h worker, void *remote_addr,
size_t length, ucs_memory_type_t mem_type,
ucp_md_index_t md_index, uct_mem_h *memh,
Expand Down
9 changes: 8 additions & 1 deletion src/ucp/core/ucp_request.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -417,4 +417,11 @@ ucs_status_t ucp_request_recv_msg_truncated(ucp_request_t *req, size_t length,
return UCS_ERR_MESSAGE_TRUNCATED;
}


void ucp_request_unsupported_mem_type_error(const ucp_request_t *req,
const char *op_type_str)
{
/* TODO: remove when support for non-HOST memory types will be added */
ucs_error("UCP doesn't support %s for \"%s\"<->\"%s\" memory types",
op_type_str, ucs_memory_type_names[req->send.mem_type],
ucs_memory_type_names[req->send.rma.rkey->mem_type]);
}
3 changes: 3 additions & 0 deletions src/ucp/core/ucp_request.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -354,4 +354,7 @@ void ucp_request_send_state_ff(ucp_request_t *req, ucs_status_t status);
ucs_status_t ucp_request_recv_msg_truncated(ucp_request_t *req, size_t length,
size_t offset);

void ucp_request_unsupported_mem_type_error(const ucp_request_t *req,
const char *op_type_str);

#endif
Loading

0 comments on commit ee55de8

Please sign in to comment.