diff --git a/opal/mca/hwloc/base/hwloc_base_frame.c b/opal/mca/hwloc/base/hwloc_base_frame.c index da48d4833ec..af85a642dcd 100644 --- a/opal/mca/hwloc/base/hwloc_base_frame.c +++ b/opal/mca/hwloc/base/hwloc_base_frame.c @@ -3,6 +3,7 @@ * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. * Copyright (c) 2016-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2019 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -217,7 +218,14 @@ static int opal_hwloc_base_open(mca_base_open_flag_t flags) * we do bind to the given cpus if provided, otherwise this would be * ignored if someone didn't also specify a binding policy */ - OPAL_SET_BINDING_POLICY(opal_hwloc_binding_policy, OPAL_BIND_TO_CPUSET); +// Restoring pre ef86707fbe3392c8ed15f79cc4892f0313b409af behavior. +// Formerly -cpu-set #,#,# along with -use_hwthread-cpus resulted +// in the binding policy staying OPAL_BIND_TO_HWTHREAD +// I think that should be right because I thought -cpu-set was a contraint you put +// on another binding policy, not a binding policy in itself. + if (!OPAL_BINDING_POLICY_IS_SET(opal_hwloc_binding_policy)) { + OPAL_SET_BINDING_POLICY(opal_hwloc_binding_policy, OPAL_BIND_TO_CPUSET); + } } /* if we are binding to hwthreads, then we must use hwthreads as cpus */ diff --git a/opal/mca/hwloc/base/hwloc_base_util.c b/opal/mca/hwloc/base/hwloc_base_util.c index d0cf6f03b1b..94c2740c50e 100644 --- a/opal/mca/hwloc/base/hwloc_base_util.c +++ b/opal/mca/hwloc/base/hwloc_base_util.c @@ -775,15 +775,34 @@ static hwloc_obj_t df_search(hwloc_topology_t topo, return found; } if (OPAL_HWLOC_AVAILABLE == rtype) { +// The previous (3.x) code included a check for +// available = opal_hwloc_base_get_available_cpus(topo, start) +// and skipped objs that had hwloc_bitmap_iszero(available) + hwloc_obj_t root; + opal_hwloc_topo_data_t *rdata; + root = hwloc_get_root_obj(topo); + rdata = (opal_hwloc_topo_data_t*)root->userdata; + hwloc_cpuset_t constrained_cpuset; + + constrained_cpuset = hwloc_bitmap_alloc(); + if (rdata && rdata->available) { + hwloc_bitmap_and(constrained_cpuset, start->cpuset, rdata->available); + } else { + hwloc_bitmap_copy(constrained_cpuset, start->cpuset); + } + unsigned idx = 0; if (num_objs) - *num_objs = hwloc_get_nbobjs_inside_cpuset_by_depth(topo, start->cpuset, search_depth); + *num_objs = hwloc_get_nbobjs_inside_cpuset_by_depth(topo, constrained_cpuset, search_depth); obj = NULL; - while ((obj = hwloc_get_next_obj_inside_cpuset_by_depth(topo, start->cpuset, search_depth, obj)) != NULL) { - if (idx == nobj) + while ((obj = hwloc_get_next_obj_inside_cpuset_by_depth(topo, constrained_cpuset, search_depth, obj)) != NULL) { + if (idx == nobj) { + hwloc_bitmap_free(constrained_cpuset); return obj; + } idx++; } + hwloc_bitmap_free(constrained_cpuset); return NULL; } return NULL; diff --git a/orte/mca/rmaps/base/rmaps_base_binding.c b/orte/mca/rmaps/base/rmaps_base_binding.c index 0715a1fe2d8..0e80db6d483 100644 --- a/orte/mca/rmaps/base/rmaps_base_binding.c +++ b/orte/mca/rmaps/base/rmaps_base_binding.c @@ -16,6 +16,7 @@ * Copyright (c) 2015-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2018 Inria. All rights reserved. + * Copyright (c) 2019 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -168,8 +169,19 @@ static int bind_generic(orte_job_t *jdata, trg_obj = NULL; min_bound = UINT_MAX; while (NULL != (tmp_obj = hwloc_get_next_obj_by_depth(node->topology->topo, target_depth, tmp_obj))) { + hwloc_obj_t root; + opal_hwloc_topo_data_t *rdata; + root = hwloc_get_root_obj(node->topology->topo); + rdata = (opal_hwloc_topo_data_t*)root->userdata; + if (!hwloc_bitmap_intersects(locale->cpuset, tmp_obj->cpuset)) continue; +// From the old 3.x code trg_obj was picked via a call to +// opal_hwloc_base_find_min_bound_target_under_obj() which +// skiped over unavailable objects (via opal_hwloc_base_get_npus). + if (rdata && rdata->available && !hwloc_bitmap_intersects(rdata->available, tmp_obj->cpuset)) + continue; + data = (opal_hwloc_obj_data_t*)tmp_obj->userdata; if (NULL == data) { data = OBJ_NEW(opal_hwloc_obj_data_t); diff --git a/orte/mca/rmaps/base/rmaps_base_ranking.c b/orte/mca/rmaps/base/rmaps_base_ranking.c index 9c0960b93b3..175590ebb69 100644 --- a/orte/mca/rmaps/base/rmaps_base_ranking.c +++ b/orte/mca/rmaps/base/rmaps_base_ranking.c @@ -13,6 +13,7 @@ * Copyright (c) 2014-2020 Intel, Inc. All rights reserved. * Copyright (c) 2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2019 IBM Corporation. All rights reserved. * Copyright (c) 2020 Huawei Technologies Co., Ltd. All rights reserved. * $COPYRIGHT$ * diff --git a/orte/mca/rmaps/rank_file/rmaps_rank_file_component.c b/orte/mca/rmaps/rank_file/rmaps_rank_file_component.c index 5f7f5d66419..a3338ed09f4 100644 --- a/orte/mca/rmaps/rank_file/rmaps_rank_file_component.c +++ b/orte/mca/rmaps/rank_file/rmaps_rank_file_component.c @@ -15,6 +15,7 @@ * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. + * Copyright (c) 2019 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -106,7 +107,8 @@ static int orte_rmaps_rank_file_register(void) static int orte_rmaps_rank_file_open(void) { /* ensure we flag mapping by user */ - if ((NULL != opal_hwloc_base_cpu_list && !OPAL_BIND_ORDERED_REQUESTED(opal_hwloc_binding_policy)) || + if ((OPAL_BIND_TO_CPUSET == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy) && + !OPAL_BIND_ORDERED_REQUESTED(opal_hwloc_binding_policy)) || NULL != orte_rankfile) { if (ORTE_MAPPING_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping)) { /* if a non-default mapping is already specified, then we