Merge pull request #921 from rhc54/topic/pmix

Sync to PMIx master

opal/mca/pmix/pmix1xx/pmix/Makefile.am

@@ -37,7 +37,8 @@ endif
 
 man_MANS = \
         man/man3/pmix_init.3 \
-        man/man7/pmix.7
+        man/man7/pmix.7 \
+        man/man7/pmix_constants.7
 
 include config/Makefile.am
 include include/Makefile.am

opal/mca/pmix/pmix1xx/pmix/VERSION

@@ -30,7 +30,7 @@ greek=a1
 # command, or with the date (if "git describe" fails) in the form of
 # "date<date>".
 
-repo_rev=git89680d6
+repo_rev=git4dec948
 
 # If tarball_version is not empty, it is used as the version string in
 # the tarball filename, regardless of all other versions listed in
@@ -44,7 +44,7 @@ tarball_version=
 
 # The date when this release was created
 
-date="Sep 10, 2015"
+date="Sep 21, 2015"
 
 # The shared library version of each of PMIx's public libraries.
 # These versions are maintained in accordance with the "Library

opal/mca/pmix/pmix1xx/pmix/include/pmix/pmix_common.h.in

@@ -80,7 +80,6 @@ BEGIN_C_DECLS
 
 /* define maximum value and key sizes */
 #define PMIX_MAX_NSLEN     255
-#define PMIX_MAX_VALLEN   1023
 #define PMIX_MAX_KEYLEN    511
 
 /* define a *wildcard* value for requests involving rank */
@@ -175,6 +174,7 @@ BEGIN_C_DECLS
 #define PMIX_PROC_DATA             "pmix.pdata"        // (pmix_value_array_t) starts with rank, then contains more data
 #define PMIX_NODE_MAP              "pmix.nmap"         // (char*) regex of nodes containing procs for this job
 #define PMIX_PROC_MAP              "pmix.pmap"         // (char*) regex describing procs on each node within this job
+#define PMIX_ANL_MAP               "pmix.anlmap"       // (char*) process mapping in ANL notation (used in PMI-1/PMI-2)
 
 /* attributes used internally to communicate data from the server to the client */
 #define PMIX_PROC_BLOB             "pmix.pblob"        // (pmix_byte_object_t) packed blob of process data
@@ -307,6 +307,7 @@ typedef enum {
 /* define a "persistence" policy for data published by clients */
 typedef enum {
     PMIX_PERSIST_INDEF = 0,   // retain until specifically deleted
+    PMIX_PERSIST_FIRST_READ,  // delete upon first access
     PMIX_PERSIST_PROC,        // retain until publishing process terminates
     PMIX_PERSIST_APP,         // retain until application terminates
     PMIX_PERSIST_SESSION      // retain until session/allocation terminates
@@ -468,13 +469,11 @@ typedef struct {
         }                                       \
     } while(0);
 
-#define PMIX_INFO_LOAD(m, d, k, v, t)                   \
-    do {                                                \
+#define PMIX_INFO_LOAD(m, k, v, t)                      \
     if (NULL != (m)) {                                  \
-        (m)->mandatory = (d);                           \
         (void)strncpy((m)->key, (k), PMIX_MAX_KEYLEN);  \
         pmix_value_load(&((m)->value), (v), (t));       \
-    } while(0);
+    }
 
 
 /****    PMIX LOOKUP RETURN STRUCT    ****/
@@ -827,7 +826,7 @@ const char* PMIx_Get_version(void);
     PMIX_VAL_SET_ ## _field(_v, _field, _val)
 
 #define PMIX_VAL_cmp_val(_val1, _val2)      ((_val1) != (_val2))
-#define PMIX_VAL_cmp_float(_val1, _val2)    (abs((_val1) - (_val2)) > 0.000001)
+#define PMIX_VAL_cmp_float(_val1, _val2)    (((_val1)>(_val2))?(((_val1)-(_val2))>0.000001):(((_val2)-(_val1))>0.000001))
 #define PMIX_VAL_cmp_ptr(_val1, _val2)      strncmp(_val1, _val2, strlen(_val1)+1)
 
 #define PMIX_VAL_CMP_int        PMIX_VAL_cmp_val

opal/mca/pmix/pmix1xx/pmix/man/man7/pmix_constants.7

@@ -0,0 +1,94 @@
+.TH "pmix_constants" "7" "2015\-09\-16" "PMIx Programmer\[aq]s Manual" "\@VERSION\@"
+.SH NAME
+.PP
+PMIx Constants
+.SH SYNOPSIS
+.IP
+.nf
+\f[C]
+#include\ <pmix_common.h>
+\f[]
+.fi
+.SH OVERVIEW
+.PP
+PMIx relies on the following types of constants:
+.PP
+\f[I]Maximum Sizes\f[] : In order to minimize malloc performance
+penalties, PMIx utilizes constant\-sized arrays wherever possible.
+These constants provide the user with the maximum size of the various
+array types.
+.PP
+\f[I]Attributes\f[] : .
+.PP
+\f[I]Errors\f[] : PMIx uses negative error constants, with 0 indicating
+"success".
+.SH MAXIMUM SIZES
+.PP
+The .
+.PP
+\f[I]PMIX_MAX_NSLEN\f[] : The maximum length of a namespace.
+Note that any declaration of an array to hold a key string must include
+one extra space for the terminating \f[I]NULL\f[].
+.PP
+\f[I]PMIX_MAX_KEYLEN\f[] : Maximum length of the key string used in
+structures such as the \f[I]pmix\f[]info_t_.
+Note that any declaration of an array to hold a key string must include
+one extra space for the terminating \f[I]NULL\f[].
+.SH ATTRIBUTES
+.PP
+Define a set of "standard" PMIx attributes that can be queried using the
+PMIx_Get function.
+Implementations (and users) are free to extend as desired \- thus,
+functions calling PMIx_Get must be capable of handling the "not found"
+condition.
+Note that these are attributes of the system and the job as opposed to
+values the application (or underlying programming library) might choose
+to expose \- i.e., they are values provided by the resource manager as
+opposed to the application.
+Thus, these keys are RESERVED for use by PMIx, and users should avoid
+defining any attribute starting with the keyword \f[I]PMIX\f[].
+.PP
+A list of the current PMIx attributes, and the type of their associated
+data value, is provided here.
+.PP
+\f[I]PMIX_ATTR_UNDEF (NULL)\f[] : Used to initialize an attribute field,
+indicating that the attribute has not yet been assigned.
+.PP
+\f[I]PMIX_USERID (uint32_t)\f[] : .
+.PP
+\f[I]PMIX_GRPID (uint32_t)\f[] : An access domain represents a single
+logical connection into a fabric.
+It may map to a single physical or virtual NIC or a port.
+An access domain defines the boundary across which fabric resources may
+be associated.
+Each access domain belongs to a single fabric domain.
+.PP
+\f[I]PMIX_CPUSET (char*)\f[] : .
+.SH ERROR CONSTANTS
+.PP
+\&.
+.PP
+\f[I]PMIX_SUCCESS\f[] : Indicates that the operation was successful.
+.PP
+\f[I]PMIX_ERROR\f[] : A general error code \- an error occurred, but no
+specific reason can be provided.
+.PP
+\f[I]fi_rma \- Remote Memory Access\f[] : RMA transfers are one\-sided
+operations that read or write data directly to a remote memory region.
+Other than defining the appropriate memory region, RMA operations do not
+require interaction at the target side for the data transfer to
+complete.
+.PP
+\f[I]fi_atomic \- Atomic\f[] : Atomic operations can perform one of
+several operations on a remote memory region.
+Atomic operations include well\-known functionality, such as atomic\-add
+and compare\-and\-swap, plus several other pre\-defined calls.
+Unlike other data transfer interfaces, atomic operations are aware of
+the data formatting at the target memory region.
+.SH SEE ALSO
+.PP
+\f[C]pmix\f[](7), \f[C]fi_getinfo\f[](3), \f[C]fi_endpoint\f[](3),
+\f[C]fi_domain\f[](3), \f[C]fi_av\f[](3), \f[C]fi_eq\f[](3),
+\f[C]fi_cq\f[](3), \f[C]fi_cntr\f[](3), \f[C]fi_mr\f[](3)
+.SH AUTHORS
+PMIx.

opal/mca/pmix/pmix1xx/pmix/src/client/pmi1.c

@@ -30,6 +30,8 @@
 #endif
 #include PMIX_EVENT_HEADER
 
+#define ANL_MAPPING "PMI_process_mapping"
+
 #include "src/buffer_ops/buffer_ops.h"
 #include "src/util/argv.h"
 #include "src/util/error.h"
@@ -39,6 +41,7 @@
 static pmix_status_t convert_int(int *value, pmix_value_t *kv);
 static int convert_err(pmix_status_t rc);
 static pmix_proc_t myproc;
+static bool data_commited = false;
 
 int PMI_Init( int *spawned )
 {
@@ -93,6 +96,10 @@ int PMI_KVS_Put(const char kvsname[], const char key[], const char value[])
     pmix_status_t rc;
     pmix_value_t val;
 
+    pmix_output_verbose(2, pmix_globals.debug_output,
+            "PMI_KVS_Put: KVS=%s, key=%s value=%s",
+            kvsname, key, value);
+
     val.type = PMIX_STRING;
     val.data.string = (char*)value;
     rc = PMIx_Put(PMIX_GLOBAL, key, &val);
@@ -104,15 +111,106 @@ int PMI_KVS_Commit(const char kvsname[])
 {
     pmix_status_t rc;
 
+    pmix_output_verbose(2, pmix_globals.debug_output,
+            "PMI_KVS_Commit: KVS=%s", kvsname);
+
     rc = PMIx_Commit();
+    /* PMIx permits only one data commit! */
+    data_commited = true;
     return convert_err(rc);
 }
 
+int PMI_KVS_Get( const char kvsname[], const char key[], char value[], int length)
+{
+    pmix_value_t *val;
+    uint32_t i;
+    static pmix_proc_t proc;
+    uint32_t procnum;
+    proc = myproc;
+    int rc;
+
+    pmix_output_verbose(2, pmix_globals.debug_output,
+                        "PMI_KVS_Get: KVS=%s, key=%s value=%s",
+                        kvsname, key, value);
+
+    /* PMI-1 expects resource manager to set
+     * process mapping in ANL notation. */
+    if( !strcmp(key, ANL_MAPPING) ) {
+        /* we are looking in the job-data. If there is nothing there
+         * we don't want to look in rank's data, thus set rank to widcard */
+        proc.rank = PMIX_RANK_WILDCARD;
+        if( PMIX_SUCCESS == PMIx_Get(&proc, PMIX_ANL_MAP, NULL, 0, &val) &&
+                (NULL != val) && (PMIX_STRING == val->type) ){
+            strncpy(value,val->data.string,length);
+            PMIX_VALUE_FREE(val,1);
+            return PMI_SUCCESS;
+        } else {
+            /* artpol:
+             * Some RM's (i.e. SLURM) already have ANL precomputed. The export it
+             * through PMIX_ANL_MAP variable.
+             * If we haven't found it we want to have our own packing functionality
+             * since it's common.
+             * Somebody else has to write it since I've already done that for
+             * GPL'ed SLURM :) */
+            return PMI_FAIL;
+        }
+    }
+
+    /* We don't know what process keeps this data. So it looks like we need to
+     * check each process.
+     * TODO: Is there any beter way?
+     * WARNING: this may lead to the VERY long HANG's if we ask for the unknown key
+     * before we've done Commit on all nodes. We need a workaround for that.
+     *
+     * SOLUTION: perhaps rovide "OK if nothing" info flag to tell PMIx that
+     * the key supposed to already be there and if nothing there - gave up with
+     * an error and don't try to use direct modex.
+     */
+
+    if (PMIX_SUCCESS != (rc = PMIx_Get(&myproc,PMIX_JOB_SIZE, NULL, 0,&val))) {
+        pmix_output_verbose(2, pmix_globals.debug_output,
+                            "pmi1: executing put for KVS %s, key %s value %s",
+                            kvsname, key, value);
+        return  convert_err(rc);
+    }
+    procnum = val->data.uint32;
+    PMIX_VALUE_FREE(val,1);
+
+    for( i=0; i < procnum; i++){
+        proc.rank = i;
+        if( PMIX_SUCCESS == PMIx_Get(&proc, key, NULL, 0, &val) &&
+                (NULL != val) && (PMIX_STRING == val->type) ){
+            strncpy(value,val->data.string,length);
+            PMIX_VALUE_FREE(val,1);
+            return PMI_SUCCESS;
+        }
+        PMIX_VALUE_FREE(val,1);
+    }
+    return PMI_FAIL;
+}
+
+
 /* Barrier only applies to our own nspace, and we want all
  * data to be collected upon completion */
 int PMI_Barrier(void)
 {
-    return PMIx_Fence(NULL, 0, NULL, 0);
+    pmix_info_t buf;
+    int rc, ninfo = 0;
+    pmix_info_t *info = NULL;
+
+    if( data_commited ){
+        bool  val = 1;
+        info = &buf;
+        PMIX_INFO_CONSTRUCT(info);
+        PMIX_INFO_LOAD(info, PMIX_COLLECT_DATA, &val, PMIX_BOOL );
+        ninfo = 1;
+    }
+    rc = PMIx_Fence(NULL, 0, info, ninfo);
+
+    if( NULL != info ){
+        PMIX_INFO_DESTRUCT(info);
+    }
+    return rc;
 }
 
 int PMI_Get_size(int *size)
@@ -229,8 +327,12 @@ int PMI_Lookup_name(const char service_name[], char port[])
         return convert_err(PMIX_ERR_NOT_FOUND);
     }
 
-    /* return the port */
-    (void)strncpy(port, pdata.value.data.string, PMIX_MAX_VALLEN);
+    /* return the port - sadly, this API doesn't tell us
+     * the size of the port array, and so there is a
+     * potential we could overrun it. As this feature
+     * isn't widely supported in PMI-1, try being
+     * conservative */
+    (void)strncpy(port, pdata.value.data.string, PMIX_MAX_KEYLEN);
     PMIX_PDATA_DESTRUCT(&pdata);
 
     return PMIX_SUCCESS;
@@ -260,7 +362,7 @@ int PMI_Get_id_length_max(int *length)
     if (NULL == length) {
         return PMI_ERR_INVALID_VAL_LENGTH;
     }
-    *length = PMIX_MAX_VALLEN;
+    *length = PMIX_MAX_KEYLEN;
     return PMI_SUCCESS;
 }
 
@@ -327,7 +429,9 @@ int PMI_KVS_Get_value_length_max(int *length)
     if (NULL == length) {
         return PMI_ERR_INVALID_VAL_LENGTH;
     }
-    *length = PMIX_MAX_VALLEN;
+    /* don't give them an enormous size of some implementations
+     * immediately malloc a data block for their use */
+    *length = 4096;
     return PMI_SUCCESS;
 }
 
@@ -484,6 +588,9 @@ static pmix_status_t convert_int(int *value, pmix_value_t *kv)
     case PMIX_SIZE:
         *value = kv->data.size;
         break;
+    case PMIX_BOOL:
+        *value = kv->data.flag;
+        break;
     default:
         /* not an integer type */
         return PMIX_ERR_BAD_PARAM;

opal/mca/pmix/pmix1xx/pmix/src/client/pmi2.c

@@ -304,7 +304,7 @@ int PMI2_Nameserv_lookup(const char service_name[], const PMI_keyval_t *info_ptr
     }
 
     /* return the port */
-    (void)strncpy(port, pdata[0].value.data.string, PMIX_MAX_VALLEN);
+    (void)strncpy(port, pdata[0].value.data.string, portLen);
     PMIX_PDATA_DESTRUCT(&pdata[0]);
 
     if (NULL != info_ptr) {