From a37ca49d25ec17158e23226bc8a9c5e0124b1d80 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 22 Jul 2023 20:40:53 -0600 Subject: [PATCH 1/2] Modify PR https://github.com/Unidata/netcdf-c/pull/2655 to ensure transient types have names. re: PR https://github.com/Unidata/netcdf-c/pull/2655 This PR modifies the transient types PR so that all created transient types are given a created unique name (within a group). The form of the name is "_AnonymousNN". The class is the user-defined type class: Enum, Compound, Opaque, or Vlen. NN is an integer identifier to ensure uniqueness. Additionally, this was applied to DAP/4 anonymous dimensions. This also required some test baseline data changes. The transient test case is modified to verify that the name exists. --- RELEASE_NOTES.md | 1 + dap4_test/baseline/test_vlen4.nc.d4d | 2 +- dap4_test/baseline/test_vlen4.nc.d4m | 2 +- dap4_test/baseline/test_vlen4.nc.d4p | 4 +-- .../baselinehyrax/amsre_20060131v5.dat.hyrax | 6 ++-- dap4_test/baselineraw/test_vlen4.nc.ncdump | 2 +- .../test_atomic_array.1.nc.ncdump | 12 +++---- .../test_atomic_array.2.nc.ncdump | 4 +-- .../test_atomic_array.3.nc.ncdump | 4 +-- .../test_enum_array.6.nc.ncdump | 4 +-- .../test_one_vararray.4.nc.ncdump | 4 +-- .../test_one_vararray.5.nc.ncdump | 4 +-- .../test_opaque_array.7.nc.ncdump | 6 ++-- .../test_struct_array.8.nc.ncdump | 4 +-- dap4_test/baselineremote/test_vlen4.nc.ncdump | 2 +- .../2004050300_eta_211.nc.thredds | 8 ++--- include/hdf5internal.h | 1 + include/netcdf_filter_hdf5_build.h | 6 +++- libdap4/d4parser.c | 2 +- libhdf5/hdf5open.c | 34 +++++++++++++++---- nc_test4/tst_h_transient_types.c | 19 +++++++---- plugins/H5Zblosc.h | 3 ++ plugins/H5Zzstd.h | 2 ++ 23 files changed, 86 insertions(+), 50 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 1ab194fc0b..1a30288031 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.9.3 - TBD +* Support HDF5 transient types when reading an HDF5 file. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????). * Suppress filters on variables with non-fixed-size types. See [Github #2716](https://github.com/Unidata/netcdf-c/pull/2716). * Provide a single option to disable all network access and testing. See [Github #2708](https://github.com/Unidata/netcdf-c/pull/2708). * Fix some problems with earthdata authorization and data access. See [Github #2709](https://github.com/Unidata/netcdf-c/pull/2709). diff --git a/dap4_test/baseline/test_vlen4.nc.d4d b/dap4_test/baseline/test_vlen4.nc.d4d index a6e1b4656f..fbab1e8fd1 100644 --- a/dap4_test/baseline/test_vlen4.nc.d4d +++ b/dap4_test/baseline/test_vlen4.nc.d4d @@ -8,7 +8,7 @@ types: v1_f1_t f1(2) ; }; // v1_t dimensions: - _Anonymous2 = 2 ; + _AnonymousDim2 = 2 ; variables: v1_t v1 ; data: diff --git a/dap4_test/baseline/test_vlen4.nc.d4m b/dap4_test/baseline/test_vlen4.nc.d4m index 2b25ec8894..9eba21f8e7 100644 --- a/dap4_test/baseline/test_vlen4.nc.d4m +++ b/dap4_test/baseline/test_vlen4.nc.d4m @@ -8,7 +8,7 @@ types: v1_f1_t f1(2) ; }; // v1_t dimensions: - _Anonymous2 = 2 ; + _AnonymousDim2 = 2 ; variables: v1_t v1 ; } diff --git a/dap4_test/baseline/test_vlen4.nc.d4p b/dap4_test/baseline/test_vlen4.nc.d4p index 153672af63..f8d9db7c28 100644 --- a/dap4_test/baseline/test_vlen4.nc.d4p +++ b/dap4_test/baseline/test_vlen4.nc.d4p @@ -5,12 +5,12 @@ xmlns="http://xml.opendap.org/ns/DAP/4.0#" xmlns:dap="http://xml.opendap.org/ns/DAP/4.0#"> - + - + diff --git a/dap4_test/baselinehyrax/amsre_20060131v5.dat.hyrax b/dap4_test/baselinehyrax/amsre_20060131v5.dat.hyrax index 9b908a4d70..5e25c524e3 100644 --- a/dap4_test/baselinehyrax/amsre_20060131v5.dat.hyrax +++ b/dap4_test/baselinehyrax/amsre_20060131v5.dat.hyrax @@ -1,9 +1,9 @@ netcdf amsre_20060131v5 { dimensions: - _Anonymous3 = 3 ; - _Anonymous6 = 6 ; + _AnonymousDim3 = 3 ; + _AnonymousDim6 = 6 ; variables: - byte time_a(_Anonymous3, _Anonymous6) ; + byte time_a(_AnonymousDim3, _AnonymousDim6) ; string time_a:Equator_Crossing_Time = "1:30 PM" ; data: diff --git a/dap4_test/baselineraw/test_vlen4.nc.ncdump b/dap4_test/baselineraw/test_vlen4.nc.ncdump index a6e1b4656f..fbab1e8fd1 100644 --- a/dap4_test/baselineraw/test_vlen4.nc.ncdump +++ b/dap4_test/baselineraw/test_vlen4.nc.ncdump @@ -8,7 +8,7 @@ types: v1_f1_t f1(2) ; }; // v1_t dimensions: - _Anonymous2 = 2 ; + _AnonymousDim2 = 2 ; variables: v1_t v1 ; data: diff --git a/dap4_test/baselineremote/test_atomic_array.1.nc.ncdump b/dap4_test/baselineremote/test_atomic_array.1.nc.ncdump index 927b9df93a..024e4533ce 100644 --- a/dap4_test/baselineremote/test_atomic_array.1.nc.ncdump +++ b/dap4_test/baselineremote/test_atomic_array.1.nc.ncdump @@ -2,13 +2,13 @@ netcdf test_atomic_array { types: opaque(16) opaque16_t ; dimensions: - _Anonymous1 = 1 ; - _Anonymous2 = 2 ; + _AnonymousDim1 = 1 ; + _AnonymousDim2 = 2 ; variables: - ubyte vu8(_Anonymous1, _Anonymous2) ; - double vd(_Anonymous1) ; - string vs(_Anonymous1, _Anonymous1) ; - opaque16_t vo(_Anonymous1, _Anonymous1) ; + ubyte vu8(_AnonymousDim1, _AnonymousDim2) ; + double vd(_AnonymousDim1) ; + string vs(_AnonymousDim1, _AnonymousDim1) ; + opaque16_t vo(_AnonymousDim1, _AnonymousDim1) ; // global attributes: string :_dap4.ce = "/vu8[1][0:2:2];/vd[1];/vs[1][0];/vo[0][1]" ; diff --git a/dap4_test/baselineremote/test_atomic_array.2.nc.ncdump b/dap4_test/baselineremote/test_atomic_array.2.nc.ncdump index fa6a70dabd..5efb61a484 100644 --- a/dap4_test/baselineremote/test_atomic_array.2.nc.ncdump +++ b/dap4_test/baselineremote/test_atomic_array.2.nc.ncdump @@ -1,8 +1,8 @@ netcdf test_atomic_array { dimensions: - _Anonymous3 = 3 ; + _AnonymousDim3 = 3 ; variables: - short v16(_Anonymous3) ; + short v16(_AnonymousDim3) ; // global attributes: string :_dap4.ce = "/v16[0:1,3]" ; diff --git a/dap4_test/baselineremote/test_atomic_array.3.nc.ncdump b/dap4_test/baselineremote/test_atomic_array.3.nc.ncdump index 459e17e551..c7fd968ab1 100644 --- a/dap4_test/baselineremote/test_atomic_array.3.nc.ncdump +++ b/dap4_test/baselineremote/test_atomic_array.3.nc.ncdump @@ -1,8 +1,8 @@ netcdf test_atomic_array { dimensions: - _Anonymous3 = 3 ; + _AnonymousDim3 = 3 ; variables: - short v16(_Anonymous3) ; + short v16(_AnonymousDim3) ; // global attributes: string :_dap4.ce = "/v16[3,0:1]" ; diff --git a/dap4_test/baselineremote/test_enum_array.6.nc.ncdump b/dap4_test/baselineremote/test_enum_array.6.nc.ncdump index 3761f3d765..833f9bbdcd 100644 --- a/dap4_test/baselineremote/test_enum_array.6.nc.ncdump +++ b/dap4_test/baselineremote/test_enum_array.6.nc.ncdump @@ -5,9 +5,9 @@ types: Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10, Missing = 127} ; dimensions: - _Anonymous2 = 2 ; + _AnonymousDim2 = 2 ; variables: - cloud_class_t primary_cloud(_Anonymous2) ; + cloud_class_t primary_cloud(_AnonymousDim2) ; cloud_class_t primary_cloud:_FillValue = Missing ; // global attributes: diff --git a/dap4_test/baselineremote/test_one_vararray.4.nc.ncdump b/dap4_test/baselineremote/test_one_vararray.4.nc.ncdump index 90afd65b35..6a7731d58a 100644 --- a/dap4_test/baselineremote/test_one_vararray.4.nc.ncdump +++ b/dap4_test/baselineremote/test_one_vararray.4.nc.ncdump @@ -1,8 +1,8 @@ netcdf test_one_vararray { dimensions: - _Anonymous1 = 1 ; + _AnonymousDim1 = 1 ; variables: - int t(_Anonymous1) ; + int t(_AnonymousDim1) ; // global attributes: string :_dap4.ce = "/t[1]" ; diff --git a/dap4_test/baselineremote/test_one_vararray.5.nc.ncdump b/dap4_test/baselineremote/test_one_vararray.5.nc.ncdump index 780252c938..133fd050f7 100644 --- a/dap4_test/baselineremote/test_one_vararray.5.nc.ncdump +++ b/dap4_test/baselineremote/test_one_vararray.5.nc.ncdump @@ -1,8 +1,8 @@ netcdf test_one_vararray { dimensions: - _Anonymous2 = 2 ; + _AnonymousDim2 = 2 ; variables: - int t(_Anonymous2) ; + int t(_AnonymousDim2) ; // global attributes: string :_dap4.ce = "/t[0:1]" ; diff --git a/dap4_test/baselineremote/test_opaque_array.7.nc.ncdump b/dap4_test/baselineremote/test_opaque_array.7.nc.ncdump index f9f74fc5df..f7dfccd836 100644 --- a/dap4_test/baselineremote/test_opaque_array.7.nc.ncdump +++ b/dap4_test/baselineremote/test_opaque_array.7.nc.ncdump @@ -2,10 +2,10 @@ netcdf test_opaque_array { types: opaque(16) opaque16_t ; dimensions: - _Anonymous1 = 1 ; - _Anonymous2 = 2 ; + _AnonymousDim1 = 1 ; + _AnonymousDim2 = 2 ; variables: - opaque16_t vo2(_Anonymous1, _Anonymous2) ; + opaque16_t vo2(_AnonymousDim1, _AnonymousDim2) ; // global attributes: string :_dap4.ce = "/vo2[1][0:1]" ; diff --git a/dap4_test/baselineremote/test_struct_array.8.nc.ncdump b/dap4_test/baselineremote/test_struct_array.8.nc.ncdump index c8d53da311..fc2536e442 100644 --- a/dap4_test/baselineremote/test_struct_array.8.nc.ncdump +++ b/dap4_test/baselineremote/test_struct_array.8.nc.ncdump @@ -5,9 +5,9 @@ types: int y ; }; // s_t dimensions: - _Anonymous2 = 2 ; + _AnonymousDim2 = 2 ; variables: - s_t s(_Anonymous2, _Anonymous2) ; + s_t s(_AnonymousDim2, _AnonymousDim2) ; // global attributes: string :_dap4.ce = "/s[0:2:3][0:1]" ; diff --git a/dap4_test/baselineremote/test_vlen4.nc.ncdump b/dap4_test/baselineremote/test_vlen4.nc.ncdump index a6e1b4656f..fbab1e8fd1 100644 --- a/dap4_test/baselineremote/test_vlen4.nc.ncdump +++ b/dap4_test/baselineremote/test_vlen4.nc.ncdump @@ -8,7 +8,7 @@ types: v1_f1_t f1(2) ; }; // v1_t dimensions: - _Anonymous2 = 2 ; + _AnonymousDim2 = 2 ; variables: v1_t v1 ; data: diff --git a/dap4_test/baselinethredds/2004050300_eta_211.nc.thredds b/dap4_test/baselinethredds/2004050300_eta_211.nc.thredds index cdfb7d2cfc..5f4ca16075 100644 --- a/dap4_test/baselinethredds/2004050300_eta_211.nc.thredds +++ b/dap4_test/baselinethredds/2004050300_eta_211.nc.thredds @@ -1,9 +1,9 @@ netcdf \2004050300_eta_211 { dimensions: record = UNLIMITED ; // (1 currently) - _Anonymous1 = 1 ; - _Anonymous4 = 4 ; - _Anonymous15 = 15 ; + _AnonymousDim1 = 1 ; + _AnonymousDim4 = 4 ; + _AnonymousDim15 = 15 ; variables: double reftime(record) ; string reftime:units = "hours since 1992-1-1" ; @@ -11,7 +11,7 @@ variables: double valtime(record) ; string valtime:units = "hours since 1992-1-1" ; string valtime:long_name = "valid time" ; - float Z_sfc(_Anonymous1, _Anonymous4, _Anonymous15) ; + float Z_sfc(_AnonymousDim1, _AnonymousDim4, _AnonymousDim15) ; string Z_sfc:navigation = "nav" ; float Z_sfc:_FillValue = -9999.006f ; string Z_sfc:units = "gp m" ; diff --git a/include/hdf5internal.h b/include/hdf5internal.h index 58d076567b..a83edad320 100644 --- a/include/hdf5internal.h +++ b/include/hdf5internal.h @@ -60,6 +60,7 @@ struct NCauth; /** Struct to hold HDF5-specific info for the file. */ typedef struct NC_HDF5_FILE_INFO { hid_t hdfid; + unsigned transientid; /* counter for transient ids */ NCURI* uri; /* Parse of the incoming path, if url */ #if defined(ENABLE_BYTERANGE) int byterange; diff --git a/include/netcdf_filter_hdf5_build.h b/include/netcdf_filter_hdf5_build.h index 24000c7b8c..63d2857eef 100644 --- a/include/netcdf_filter_hdf5_build.h +++ b/include/netcdf_filter_hdf5_build.h @@ -27,6 +27,10 @@ #include "config.h" #endif +#include +#include +#include + /* Support headers */ #include #include @@ -142,7 +146,7 @@ typedef const void* (*H5PL_get_plugin_info_proto)(void); #define H5MM_realloc realloc #endif #ifndef H5MM_xfree -#define H5MM_xfree nullfree +#define H5MM_xfree(x) do{if((x)!=NULL) free(x);}while(0) #endif #ifndef H5_ATTR_UNUSED #define H5_ATTR_UNUSED diff --git a/libdap4/d4parser.c b/libdap4/d4parser.c index c9fd8e3c7e..013e320928 100644 --- a/libdap4/d4parser.c +++ b/libdap4/d4parser.c @@ -1357,7 +1357,7 @@ makeAnonDim(NCD4parser* parser, const char* sizestr) ret = parseLL(sizestr,&size); if(ret) return NULL; - snprintf(name,NC_MAX_NAME,"/_Anonymous%lld",size); + snprintf(name,NC_MAX_NAME,"/_AnonymousDim%lld",size); /* See if it exists already */ dim = lookupFQN(parser,name,NCD4_DIM); if(dim == NULL) {/* create it */ diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 7eaa9a8ef8..8d3f4fce3a 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -245,9 +245,8 @@ get_type_info2(NC_GRP_INFO_T *h5_grp, hid_t datasetid, NC_TYPE_INFO_T **type_inf if (type == NULL) { /* If we still can't read the type, ignore it, it probably * means this object is a reference */ - if (read_type(h5_grp, native_typeid, "")) + if (read_type(h5_grp, native_typeid, NULL)) return NC_EBADTYPID; - if((type = nc4_rec_find_hdf_type(h5, native_typeid))) *type_info = type; } @@ -1979,7 +1978,7 @@ hdf5free(void* memory) * * @param grp Pointer to group info struct. * @param hdf_typeid HDF5 type ID. - * @param type_name Pointer that gets the type name. + * @param type_name Pointer that gets the type name; NULL => anonymous * * @return ::NC_NOERR No error. * @return ::NC_EBADID Bad ncid. @@ -1991,19 +1990,26 @@ hdf5free(void* memory) static int read_type(NC_GRP_INFO_T *grp, hid_t hdf_typeid, char *type_name) { - NC_TYPE_INFO_T *type; - NC_HDF5_TYPE_INFO_T *hdf5_type; + NC_FILE_INFO_T *h5 = NULL; + NC_TYPE_INFO_T *type = NULL; + NC_HDF5_FILE_INFO_T *hdf5_info = NULL; + NC_HDF5_TYPE_INFO_T *hdf5_type = NULL; H5T_class_t class; hid_t native_typeid; size_t type_size; int nmembers; int retval; + char transientname[NC_MAX_NAME]; - assert(grp && type_name); + assert(grp); - LOG((4, "%s: type_name %s grp->hdr.name %s", __func__, type_name, + LOG((4, "%s: type_name %s grp->hdr.name %s", __func__, (type_name?type_name:"NULL"), grp->hdr.name)); + /* Get HDF5-specific object info. */ + h5 = (NC_FILE_INFO_T *)grp->nc4_info; + hdf5_info = (NC_HDF5_FILE_INFO_T*)h5->format_file_info; + /* What is the class of this type, compound, vlen, etc. */ if ((class = H5Tget_class(hdf_typeid)) < 0) return NC_EHDFERR; @@ -2021,6 +2027,20 @@ read_type(NC_GRP_INFO_T *grp, hid_t hdf_typeid, char *type_name) return NC_EHDFERR; LOG((5, "type_size %d", type_size)); + if(type_name == NULL) { + /* This is a transient/anonymous type, so generate a name for it */ + const char* suffix = NULL; /* class */ + switch (class) { + case H5T_ENUM: suffix = "Enum"; break; + case H5T_COMPOUND: suffix = "Compound"; break; + case H5T_VLEN: suffix = "Vlen"; break; + case H5T_OPAQUE: suffix = "Opaque"; break; + default: suffix = "Unknown"; break; + } + snprintf(transientname,sizeof(transientname),"_Anonymous%s%u",suffix,++hdf5_info->transientid); + type_name = transientname; + } + /* Add to the list for this new type, and get a local pointer to it. */ if ((retval = nc4_type_list_add(grp, type_size, type_name, &type))) return retval; diff --git a/nc_test4/tst_h_transient_types.c b/nc_test4/tst_h_transient_types.c index 33025f5625..83a4f0129b 100644 --- a/nc_test4/tst_h_transient_types.c +++ b/nc_test4/tst_h_transient_types.c @@ -76,17 +76,14 @@ main() printf("*** Checking accessing file through netCDF-4 API..."); { - int ncid, varid, enumid; + int i, ncid, varid, enumid; complex read_z; - int num_types, class; int *typeids; - nc_type base_nc_type; char name[NC_MAX_NAME]; - size_t size, nfields; bool read_b; + int num_types; - - nc_set_log_level(4); + nc_set_log_level(0); /* Report HDF5 errors */ if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR; /* Read known types */ @@ -97,7 +94,15 @@ main() typeids = (int*)malloc((size_t)num_types * sizeof(int)); if (nc_inq_typeids(ncid, NULL, typeids)) ERR; - if (nc_inq_user_type(ncid, typeids[0], name, &size, &base_nc_type, &nfields, &class)) ERR; + /* Verify transient names */ + for(i=0;i Date: Sun, 23 Jul 2023 12:44:00 -0600 Subject: [PATCH 2/2] Update release notes --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 1a30288031..bbb0b6c9c2 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,7 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.9.3 - TBD -* Support HDF5 transient types when reading an HDF5 file. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????). +* Support HDF5 transient types when reading an HDF5 file. See [Github #2724](https://github.com/Unidata/netcdf-c/pull/2724). * Suppress filters on variables with non-fixed-size types. See [Github #2716](https://github.com/Unidata/netcdf-c/pull/2716). * Provide a single option to disable all network access and testing. See [Github #2708](https://github.com/Unidata/netcdf-c/pull/2708). * Fix some problems with earthdata authorization and data access. See [Github #2709](https://github.com/Unidata/netcdf-c/pull/2709).