Skip to content

Commit

Permalink
binding: generate embiggened functions
Browse files Browse the repository at this point in the history
We will call dump_mpi_c twice for functions with POLY types. First we
call with map_type="SMALL-poly", then call with map_type="BIG".
Functions without POLY types will just call dump_mpi_c once with
map_type="SMALL".

First of all, if there is a custom "large_count" block, we'll always
use that in large version.

If a POLY function only has scalar POLY type, we assume the
implementation use MPI_Aint -- need fix any exceptions.

If a POLY function has scalar output or inout POLY type, we assume
internally use MPI_Aint, and python generrates temporary MPI_Aint
variable to copy in and out value in the *small* version.

If the function uses POLY arrays, we currently always assume there is a
large mpir impl version with `_c_impl` suffix, and will call that. The
compilation will fail if that (_c_impl) routine is missing.

We will also rely on the -Wconversion warnings to find all cases when we
assume internal version use MPI_Aint but it doesn't.
  • Loading branch information
hzhou committed Dec 26, 2020
1 parent 58f9b46 commit 2f8c1fa
Show file tree
Hide file tree
Showing 3 changed files with 157 additions and 26 deletions.
7 changes: 5 additions & 2 deletions maint/gen_binding_c.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,13 @@ def main():
else:
G.out = []
G.err_codes = {}
mapping = G.MAPS['SMALL_C_KIND_MAP']

# dumps the code to G.out array
dump_mpi_c(func, mapping)
if function_has_POLY_parameters(func):
dump_mpi_c(func, "SMALL-poly")
dump_mpi_c(func, "BIG")
else:
dump_mpi_c(func, "SMALL")

file_path = get_func_file_path(func, binding_dir)
dump_c_file(file_path, G.out)
Expand Down
170 changes: 146 additions & 24 deletions maint/local_python/binding_c.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,28 @@

# ---- top-level routines ----

def dump_mpi_c(func, mapping):
def dump_mpi_c(func, map_type="SMALL"):
"""Dumps the function's C source code to G.out array"""
if map_type == "SMALL":
# single version, business as usual
mapping = G.MAPS['SMALL_C_KIND_MAP']
elif map_type == "SMALL-poly":
# small version of poly type, difference in man page and body of routines
mapping = G.MAPS['SMALL_C_KIND_MAP']
elif map_type == "BIG":
# big version of poly type, difference in man page and body of routines
mapping = G.MAPS['BIG_C_KIND_MAP']
else:
raise Exception("dump_mpi_c is called with map_type = " + map_type)

# save as attribute so we don't have pass it around
func['_map_type'] = map_type

check_func_directives(func)
filter_c_parameters(func)

check_large_parameters(func)

G.mpi_declares.append(get_declare_function(func, mapping))

check_params_with_large_only(func, mapping)
Expand Down Expand Up @@ -48,6 +65,11 @@ def dump_mpi_c(func, mapping):
else:
dump_function(func, mapping, kind="normal")

if "decl" in func:
push_impl_decl(func, func['decl'])
elif 'body' not in func and 'impl' not in func:
push_impl_decl(func)

def get_func_file_path(func, root_dir):
file_path = None
dir_path = root_dir + '/' + func['dir']
Expand Down Expand Up @@ -247,6 +269,9 @@ def process_func_parameters(func, mapping):
# Note: we'll attach the lists to func at the end
validation_list, handle_ptr_list, impl_arg_list, impl_param_list = [], [], [], []

# init to empty list or we will have double entries due to being called twice (small and large)
func['_has_handle_out'] = []

func_name = func['name']
n = len(func['c_parameters'])
i = 0
Expand Down Expand Up @@ -296,7 +321,7 @@ def process_func_parameters(func, mapping):
t += ","
t += temp_p['name']
impl_arg_list.append(temp_p['name'])
impl_param_list.append(get_C_param(temp_p, mapping))
impl_param_list.append(get_impl_param(temp_p, mapping))
validation_list.append({'kind': group_kind, 'name': t})
i += group_count
continue
Expand Down Expand Up @@ -437,11 +462,9 @@ def process_func_parameters(func, mapping):
impl_arg_list.append(name + "_ptr")
impl_param_list.append("%s *%s_ptr" % (G.handle_mpir_types[kind], name))
elif do_handle_ptr == 2:
if '_has_handle_out' in func:
# Just MPI_Comm_idup[_with_info] have 2 handle output, but anyway...
func['_has_handle_out'].append(p)
else:
func['_has_handle_out'] = [p]
# Just MPI_Comm_idup[_with_info] have 2 handle output, but use a list anyway
func['_has_handle_out'].append(p)

impl_arg_list.append('&' + name + "_ptr")
impl_param_list.append("%s **%s_ptr" % (G.handle_mpir_types[kind], name))
elif do_handle_ptr == 3:
Expand All @@ -460,7 +483,7 @@ def process_func_parameters(func, mapping):
impl_param_list.append("%s *%s_ptr" % (G.handle_mpir_types[kind], name))
else:
impl_arg_list.append(name)
impl_param_list.append(get_C_param(p, mapping))
impl_param_list.append(get_impl_param(p, mapping))
i += 1

if RE.match(r'MPI_(Wait|Test)', func_name):
Expand Down Expand Up @@ -641,10 +664,35 @@ def dump_function(func, mapping, kind):
G.out.append("DEDENT")
G.out.append("}")

if "decl" in func:
push_impl_decl(func, func['decl'])
elif 'body' not in func and 'impl' not in func:
push_impl_decl(func)
def check_large_parameters(func):
# whether we need *separate* large version
# whether we always use large version internally
func['_need_large_impl'] = False
func['_impl_uses_large'] = False
# for large parameters that needs copy in & out, e.g. output, inout, array
func['_poly_in_list'] = []
func['_poly_out_list'] = []
func['_poly_inout_list'] = []
for p in func['c_parameters']:
if RE.match(r'POLY', p['kind']):
if p['param_direction'] == 'out':
func['_poly_out_list'].append(p)
elif p['param_direction'] == 'inout':
# MPI_{Pack,Unpack}[_external]
func['_poly_inout_list'].append(p)
elif p['length']:
# For perforance reason, default internal code will use small
# counts array to avoid extra array copy for the common case
func['_need_large_impl'] = True
else:
func['_poly_in_list'].append(p)

if 'code-large_count' in func:
func['_need_large_impl'] = True

if not func['_need_large_impl']:
if func['_poly_in_list'] or func['_poly_out_list'] or func['_poly_inout_list']:
func['_impl_uses_large'] = True

def dump_function_normal(func, state_name, mapping):
func['exit_routines'] = []
Expand Down Expand Up @@ -724,10 +772,20 @@ def dump_function_normal(func, state_name, mapping):

check_early_returns(func)
G.out.append("")

# out or inout POLY scalar may need temporary variable to copy in and out values
dump_poly_pre_filter(func)

G.out.append("/* ... body of routine ... */")
if 'body' in func:
for l in func['body']:
G.out.append(l)
if func['_map_type'] == "BIG" and func['_need_large_impl']:
if 'code-large_count' not in func:
raise Exception("%s missing large count code block." % func['name'])
for l in func['code-large_count']:
G.out.append(l)
else:
for l in func['body']:
G.out.append(l)
elif 'impl' in func and RE.match(r'mpid', func['impl'], re.IGNORECASE):
dump_body_impl(func, "mpid")
elif func['dir'] == 'coll':
Expand All @@ -736,6 +794,9 @@ def dump_function_normal(func, state_name, mapping):
dump_body_impl(func, "mpir")

G.out.append("/* ... end of body of routine ... */")

dump_poly_post_filter(func)

G.out.append("")
G.out.append("fn_exit:")
for l in func['exit_routines']:
Expand All @@ -757,15 +818,61 @@ def dump_function_normal(func, state_name, mapping):
dump_mpi_fn_fail(func, mapping)
G.out.append("goto fn_exit;")

def dump_poly_pre_filter(func):
if not func['_need_large_impl']:
if func['_map_type'] != "BIG":
# internal code will use MPI_Aint
for p in func['_poly_out_list']:
G.out.append("MPI_Aint %s_c;" % p['name'])
for p in func['_poly_inout_list']:
G.out.append("MPI_Aint %s_c = *%s;" % (p['name'], p['name']))

for p in func['_poly_out_list'] + func['_poly_inout_list']:
for i in range(len(func['_impl_arg_list'])):
if func['_impl_arg_list'][i] == p['name']:
func['_impl_arg_list'][i] = "&%s_c" % p['name']

def dump_poly_post_filter(func):
if not func['_need_large_impl']:
if func['_map_type'] != "BIG":
for p in func['_poly_out_list'] + func['_poly_inout_list']:
# check if value fits
dump_if_open("%s_c > INT_MAX" % p['name'])
G.out.append("*%s = MPI_UNDEFINED;" % p['name'])
errname = "\"**too_big_for_output\", \"**too_big_for_output %s\""
G.out.append("mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,")
G.out.append(" __func__, __LINE__, MPI_ERR_ARG,")
G.out.append(" %s, \"%s\");" % (errname, p['name']))
G.out.append("goto fn_fail;")
dump_if_close()
# copy the value
G.out.append("*%s = %s_c;" % (p['name'], p['name']))

def push_impl_decl(func, impl_name=None):
if not impl_name:
impl_name = re.sub(r'^MPIX?_', 'MPIR_', func['name']) + "_impl"

if func['_map_type'] != "BIG":
# SMALL
if func['_impl_uses_large']:
# skip since large interface will be used
return
else:
# BIG
if func['_need_large_impl']:
# add suffix to differentiate
impl_name = re.sub(r'_impl$', '_c_impl', impl_name)
elif not func['_impl_uses_large']:
# skip since small interface will be used
return

if func['_impl_param_list']:
params = ', '.join(func['_impl_param_list'])
if func['dir'] == 'coll' and not RE.match(r'MPI_I', func['name']):
params = params + ", MPIR_Errflag_t *errflag"
else:
params="void"

G.impl_declares.append("int %s(%s);" % (impl_name, params))

def dump_CHECKENUM(var, errname, t, type="ENUM"):
Expand Down Expand Up @@ -798,11 +905,17 @@ def dump_body_coll(func):
if not name.startswith('I'):
G.out.append("MPIR_Errflag_t errflag = MPIR_ERR_NONE;")
args = args + ", " + "&errflag"
G.out.append("if (%s || (%s && %s)) {" % (cond_a, cond_b1, cond_b2))
dump_line_with_break(" mpi_errno = MPID_%s(%s);" % (name, args))
G.out.append("} else {")
dump_line_with_break(" mpi_errno = MPIR_%s_impl(%s);" % (name, args))
G.out.append("}")

if func['_map_type'] == "BIG" and func['_need_large_impl']:
# BIG v-count only do MPIR_Xxx_c_impl for now
dump_line_with_break(" mpi_errno = MPIR_%s_c_impl(%s);" % (name, args))
else:
G.out.append("if (%s || (%s && %s)) {" % (cond_a, cond_b1, cond_b2))
dump_line_with_break(" mpi_errno = MPID_%s(%s);" % (name, args))
G.out.append("} else {")
dump_line_with_break(" mpi_errno = MPIR_%s_impl(%s);" % (name, args))
G.out.append("}")

dump_error_check("")
if name.startswith('I'):
G.out.append("if (!request_ptr) {")
Expand All @@ -812,7 +925,7 @@ def dump_body_coll(func):

def dump_body_impl(func, prefix='mpir'):
# mpi_errno = MPIR_Xxx_impl(...);
if '_has_handle_out' in func:
if func['_has_handle_out']:
for p in func['_has_handle_out']:
(name, kind) = (p['name'], p['kind'])
mpir_type = G.handle_mpir_types[kind]
Expand All @@ -824,17 +937,19 @@ def dump_body_impl(func, prefix='mpir'):
if p['kind'] == "DATATYPE" and p['param_direction'] == 'out':
G.out.append("*%s = MPI_DATATYPE_NULL;" % p['name'])

impl = func['name']
if func['_map_type'] == "BIG" and func['_need_large_impl']:
impl += '_c'
if prefix == 'mpid':
impl = func['name']
impl = re.sub(r'^MPIX?_', 'MPID_', impl)
else:
impl = func['name'] + "_impl"
impl = re.sub(r'^MPIX?_', 'MPIR_', impl)
impl = re.sub(r'^MPIX?_', 'MPIR_', impl) + "_impl"

args = ", ".join(func['_impl_arg_list'])
dump_line_with_break("mpi_errno = %s(%s);" % (impl, args))
dump_error_check("")

if '_has_handle_out' in func:
if func['_has_handle_out']:
# assuming we are creating new handle(s)
for p in func['_has_handle_out']:
(name, kind) = (p['name'], p['kind'])
Expand Down Expand Up @@ -1633,6 +1748,13 @@ def get_C_param(param, mapping):

return s

def get_impl_param(param, mapping):
s = get_C_param(param, mapping)
if RE.match(r'POLY', param['kind']):
# internally we always use MPI_Aint for large poly type
s = re.sub(r'MPI_Count', 'MPI_Aint', s)
return s

def get_polymorph_param_and_arg(s):
# s is a polymorph spec, e.g. "MPIR_Attr_type attr_type=MPIR_ATTR_PTR"
# Note: we assum limit of single extra param for now (which is sufficient)
Expand Down
6 changes: 6 additions & 0 deletions maint/local_python/mpi_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -175,3 +175,9 @@ def parse_param_attributes(p):
p['constant'] = True
else:
p['constant'] = False

def function_has_POLY_parameters(func):
for p in func['parameters']:
if p['kind'].startswith('POLY'):
return True
return False

0 comments on commit 2f8c1fa

Please sign in to comment.