diff --git a/.github/workflows/run_tests_s3.yml b/.github/workflows/run_tests_s3.yml new file mode 100644 index 0000000000..0a1c942460 --- /dev/null +++ b/.github/workflows/run_tests_s3.yml @@ -0,0 +1,153 @@ +### +# Test S3 Support +# -- derived from run_tests_ubuntu.yml +### + +### +# Build hdf5 dependencies and cache them in a combined directory. +### + +name: Run S3 netCDF Tests (under Ubuntu Linux) + +on: [workflow_dispatch] + +jobs: + + build-deps-serial: + + runs-on: ubuntu-latest + + strategy: + matrix: + hdf5: [ 1.10.8, 1.12.2, 1.14.0 ] + + steps: + - uses: actions/checkout@v3 + + - name: Install System dependencies + shell: bash -l {0} + run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev + + ### + # Installing libhdf5 + ### + - name: Cache libhdf5-${{ matrix.hdf5 }} + id: cache-hdf5 + uses: actions/cache@v3 + with: + path: ~/environments/${{ matrix.hdf5 }} + key: hdf5-${{ runner.os }}-${{ matrix.hdf5 }} + + + - name: Build libhdf5-${{ matrix.hdf5 }} + if: steps.cache-hdf5.outputs.cache-hit != 'true' + run: | + set -x + + wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-$(echo ${{ matrix.hdf5 }} | cut -d. -f 1,2)/hdf5-${{ matrix.hdf5 }}/src/hdf5-${{ matrix.hdf5 }}.tar.bz2 + tar -jxf hdf5-${{ matrix.hdf5 }}.tar.bz2 + pushd hdf5-${{ matrix.hdf5 }} + ./configure --disable-static --enable-shared --prefix=${HOME}/environments/${{ matrix.hdf5 }} --enable-hl --with-szlib + make -j + make install -j + popd + + + ##### + # S3 Autotools-based tests. + ##### + ## + # Serial + ## + nc-ac-tests-s3-serial: + + needs: build-deps-serial + runs-on: ubuntu-latest + + strategy: + matrix: + hdf5: [ 1.14.0 ] + steps: + + - uses: actions/checkout@v3 + + - name: Install System dependencies + shell: bash -l {0} + run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev openssl libssl-dev + + ### + # Set Environmental Variables + ### + + - run: echo "CFLAGS=-I${HOME}/environments/${{ matrix.hdf5 }}/include" >> $GITHUB_ENV + - run: echo "LDFLAGS=-L${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV + - run: echo "LD_LIBRARY_PATH=${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV + + + ### + # Fetch Cache + ### + + - name: Fetch HDF Cache + id: cache-hdf + uses: actions/cache@v3 + with: + path: ~/environments/${{ matrix.hdf5 }} + key: hdf5-${{ runner.os }}-${{ matrix.hdf5 }} + + - name: Check Cache + shell: bash -l {0} + run: ls ${HOME}/environments && ls ${HOME}/environments/${{ matrix.hdf5 }} && ls ${HOME}/environments/${{ matrix.hdf5}}/lib + + ### + # Configure and build + ### + + - name: Run autoconf + shell: bash -l {0} + run: autoreconf -if + + - name: Configure + shell: bash -l {0} + run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ./configure --enable-hdf5 --disable-dap --enable-external-server-tests --enable-s3 --enable-s3-internal --with-s3-testing=public + if: ${{ success() }} + + - name: Look at config.log if error + shell: bash -l {0} + run: cat config.log + if: ${{ failure() }} + + - name: Print Summary + shell: bash -l {0} + run: cat libnetcdf.settings + + - name: Build Library and Utilities + shell: bash -l {0} + run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make -j + if: ${{ success() }} + + - name: Build Tests + shell: bash -l {0} + run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make check TESTS="" -j + if: ${{ success() }} + + - name: Run Tests + shell: bash -l {0} + env: + AWS_PROFILE: ${{ secrets.DEFAULT_PROFILE }} + run: | + mkdir -p ~/.aws + echo "" > ~/.aws/config + chmod go-x ~/.aws/config + echo "${AWS_PROFILE}" >> ~/.aws/config + LD_LIBRARY_PATH="/home/runner/work/netcdf-c/netcdf-c/liblib/.libs:${LD_LIBRARY_PATH}" + CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make check -j + if: ${{ success() }} + + ##### + # S3 CMake-based tests. + ##### + ## + # Serial + ## + # T.B.D. nc-cmake-tests-s3-serial: diff --git a/.github/workflows/run_tests_ubuntu.yml b/.github/workflows/run_tests_ubuntu.yml index 923c520ab5..6784c7a7de 100644 --- a/.github/workflows/run_tests_ubuntu.yml +++ b/.github/workflows/run_tests_ubuntu.yml @@ -17,12 +17,11 @@ jobs: hdf5: [ 1.10.8, 1.12.2, 1.14.0 ] steps: - - uses: actions/checkout@v3 - name: Install System dependencies shell: bash -l {0} - run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev doxygen + run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev doxygen openssl ### # Installing libhdf5 @@ -132,7 +131,7 @@ jobs: - name: Install System dependencies shell: bash -l {0} - run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev doxygen + run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev doxygen valgrind ### # Set Environmental Variables @@ -192,7 +191,9 @@ jobs: - name: Run Tests shell: bash -l {0} - run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make check -j + run: | + LD_LIBRARY_PATH="/home/runner/work/netcdf-c/netcdf-c/liblib/.libs:${LD_LIBRARY_PATH}" + CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make check -j if: ${{ success() }} ## @@ -627,4 +628,3 @@ jobs: cd build LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ctest -j 12 --rerun-failed --output-on-failure -VV if: ${{ failure() }} - diff --git a/.github/workflows/run_tests_win_cygwin.yml b/.github/workflows/run_tests_win_cygwin.yml index bfd642f504..5a8a2eb0dd 100644 --- a/.github/workflows/run_tests_win_cygwin.yml +++ b/.github/workflows/run_tests_win_cygwin.yml @@ -30,6 +30,7 @@ jobs: libhdf4-devel zipinfo libxml2-devel perl zlib-devel libzstd-devel libbz2-devel libaec-devel libzip-devel libdeflate-devel gcc-core libcurl-devel libiconv-devel + libssl-devel libcrypt-devel - name: (Autotools) Run autoconf and friends run: | @@ -40,10 +41,9 @@ jobs: - name: (Autotools) Configure in-tree build run: >- - /bin/dash ./configure --enable-hdf5 --enable-shared - --disable-static --enable-dap --disable-dap-remote-tests - --enable-plugins --disable-nczarr-filters - --disable-nczarr-s3 --disable-nczarr-s3-tests --disable-nczarr + /bin/dash ./configure --enable-hdf5 --enable-shared + --disable-static --enable-dap --disable-dap-remote-tests + --enable-plugins --enable-nczarr - name: Look at config.log if error if: ${{ failure() }} @@ -66,4 +66,5 @@ jobs: - name: (Autotools) Build and run tests timeout-minutes: 30 - run: make check -j8 SHELL=/bin/dash + run: | + make check -j8 SHELL=/bin/dash diff --git a/.github/workflows/run_tests_win_mingw.yml b/.github/workflows/run_tests_win_mingw.yml index 3d1e2ef5bb..7b8bfef208 100644 --- a/.github/workflows/run_tests_win_mingw.yml +++ b/.github/workflows/run_tests_win_mingw.yml @@ -37,7 +37,7 @@ jobs: run: autoreconf -if - name: (Autotools) Configure Build - run: ./configure --enable-hdf5 --enable-dap --disable-dap-remote-tests --disable-static --disable-byterange --disable-dap-remote-tests --disable-logging --enable-plugins --disable-nczarr-filters --disable-nczarr-s3 --disable-nczarr-s3-tests + run: ./configure --enable-hdf5 --enable-dap --disable-dap-remote-tests --disable-static --disable-byterange --disable-dap-remote-tests --disable-logging --enable-plugins --disable-nczarr-filters --disable-s3 if: ${{ success() }} - name: (Autotools) Look at config.log if error @@ -54,7 +54,7 @@ jobs: - name: Check for plugins run: | dir ./plugins - if test -e ./plugins/.libs ; then dir ./plugins/.libs ; fi + if test -f ./plugins/.libs ; then dir ./plugins/.libs; fi - name: (Autotools) Build and Run Tests run: make check -j 8 LDFLAGS="-Wl,--export-all-symbols" diff --git a/CMakeLists.txt b/CMakeLists.txt index 9da330daa2..1572a7acc7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1270,8 +1270,19 @@ ENDIF() # Options for S3 Support OPTION(ENABLE_S3 "Enable S3 support." OFF) +OPTION(ENABLE_S3_INTERNAL "Enable S3 Internal support." OFF) OPTION(ENABLE_NCZARR_S3 "Enable NCZarr S3 support; Deprecated in favor of ENABLE_S3" OFF) -OPTION(ENABLE_NCZARR_S3_TESTS "Enable NCZarr S3 tests." OFF) + +# Control S3 Testing: Multi-valued option +SET(WITH_S3_TESTING OFF CACHE STRING "Control S3 Testing: ON (i.e. all) OFF (i.e. none) PUBLIC") + SET_PROPERTY(CACHE WITH_S3_TESTING PROPERTY STRINGS ON OFF PUBLIC) # +IF(WITH_S3_TESTING STREQUAL "") + SET(WITH_S3_TESTING OFF CACHE STRING "") # Default +ENDIF() + +IF(WITH_S3_TESTING) +message(WARNING "**** DO NOT USE WITH_S3_TESTING=ON UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***") +ENDIF() # ENABLE_NCZARR_S3 is now an alias for ENABLE_S3 (but...) if (NOT ENABLE_S3 AND ENABLE_NCZARR_S3) @@ -1279,38 +1290,46 @@ if (NOT ENABLE_S3 AND ENABLE_NCZARR_S3) ENDIF() UNSET(ENABLE_NCZARR_S3) -IF(ENABLE_NCZARR_S3_TESTS AND NOT ENABLE_S3) - message(FATAL_ERROR "S3 support is disabled; please specify option -DENABLE_NCZARR_S3_TESTS=NO") - SET(ENABLE_NCZARR_S3_TESTS OFF CACHE BOOL "NCZARR S3 TESTS" FORCE) -ENDIF() - # Note we check for the library after checking for enable_s3 # because for some reason this screws up if we unconditionally test for sdk # and it is not available. Fix someday IF(ENABLE_S3) - # See if aws-s3-sdk is available - find_package(AWSSDK REQUIRED COMPONENTS s3;core) - IF(AWSSDK_FOUND) - SET(service s3;core) - AWSSDK_DETERMINE_LIBS_TO_LINK(service AWS_LINK_LIBRARIES) - SET(ENABLE_S3_SDK ON CACHE BOOL "S3 SDK" FORCE) - ELSE() - SET(ENABLE_S3_SDK OFF CACHE BOOL "S3 SDK" FORCE) + IF(NOT ENABLE_S3_INTERNAL) + # See if aws-s3-sdk is available + find_package(AWSSDK REQUIRED COMPONENTS s3;core) + IF(AWSSDK_FOUND) + SET(service s3;core) + AWSSDK_DETERMINE_LIBS_TO_LINK(service AWS_LINK_LIBRARIES) + SET(ENABLE_S3_AWS ON CACHE BOOL "S3 AWS" FORCE) + ELSE() + SET(ENABLE_S3_AWS OFF CACHE BOOL "S3 AWS" FORCE) + ENDIF() ENDIF() ELSE() - SET(ENABLE_S3_SDK OFF CACHE BOOL "S3 SDK" FORCE) + SET(ENABLE_S3_AWS OFF CACHE BOOL "S3 AWS" FORCE) ENDIF() -IF(NOT ENABLE_S3_SDK) - IF(ENABLE_S3) +# Unless/until we get aws-sdk-cpp working for Windows, force use of internal +IF(ENABLE_S3 AND MSVC) + SET(ENABLE_S3_INTERNAL ON CACHE BOOL "S3 Intern" FORCE) +ENDIF() + +IF(ENABLE_S3) + IF(NOT ENABLE_S3_AWS AND NOT ENABLE_S3_INTERNAL) message(FATAL_ERROR "S3 support library not found; please specify option -DENABLE_S3=NO") - SET(ENABLE_S3 OFF CACHE BOOL "S3 support" FORCE) - SET(ENABLE_NCZARR_S3_TESTS OFF CACHE BOOL "S3 tests" FORCE) + SET(ENABLE_S3 OFF CACHE BOOL "S3 support" FORCE) + ENDIF() + IF(ENABLE_S3_AWS AND ENABLE_S3_INTERNAL) + message(WARNING "Both aws-sdk-cpp and s3-internal enabled => use s3-internal") + SET(ENABLE_S3_AWS OFF CACHE BOOL "S3 AWS" FORCE) ENDIF() ENDIF() -IF(ENABLE_NCZARR_S3_TESTS) - message(WARNING "**** DO NOT ENABLE_NCZARR_S3_TESTS UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***") +IF(NOT ENABLE_S3) +IF(WITH_S3_TESTING STREQUAL "PUBLIC" OR WITH_S3_TESTING) + message(WARNING "S3 support is disabled => WITH_S3_TESTING=OFF") + SET(WITH_S3_TESTING OFF CACHE STRING "" FORCE) +ENDIF() ENDIF() # Start disabling if curl not found @@ -1890,6 +1909,7 @@ ENDIF() # Check for various functions. CHECK_FUNCTION_EXISTS(fsync HAVE_FSYNC) CHECK_FUNCTION_EXISTS(strlcat HAVE_STRLCAT) +CHECK_FUNCTION_EXISTS(strlcpy HAVE_STRLCPY) CHECK_FUNCTION_EXISTS(strdup HAVE_STRDUP) CHECK_FUNCTION_EXISTS(strndup HAVE_STRNDUP) CHECK_FUNCTION_EXISTS(strtoll HAVE_STRTOLL) @@ -2298,6 +2318,10 @@ ENDIF() IF(ENABLE_DAP4) ADD_SUBDIRECTORY(libdap4) ADD_SUBDIRECTORY(libncxml) +ELSE() + IF(ENABLE_S3_INTERNAL) + ADD_SUBDIRECTORY(libncxml) + ENDIF() ENDIF() IF(ENABLE_PLUGINS) @@ -2540,9 +2564,10 @@ is_enabled(USE_CDF5 HAS_CDF5) is_enabled(ENABLE_ERANGE_FILL HAS_ERANGE_FILL) is_enabled(HDF5_HAS_PAR_FILTERS HAS_PAR_FILTERS) is_enabled(ENABLE_S3 HAS_S3) -is_enabled(ENABLE_S3_SDK HAS_S3_SDK) +is_enabled(ENABLE_S3_AWS HAS_S3_AWS) +is_enabled(ENABLE_S3_INTERNAL HAS_S3_INTERNAL) +is_enabled(HAS_HDF5_ROS3 HAS_HDF5_ROS3) is_enabled(ENABLE_NCZARR HAS_NCZARR) -is_enabled(ENABLE_NCZARR_S3_TESTS DO_NCZARR_S3_TESTS) is_enabled(ENABLE_NCZARR_ZIP HAS_NCZARR_ZIP) is_enabled(ENABLE_MULTIFILTERS HAS_MULTIFILTERS) is_enabled(ENABLE_NCZARR_ZIP DO_NCZARR_ZIP_TESTS) @@ -2555,6 +2580,26 @@ is_enabled(HAVE_ZSTD HAS_ZSTD) is_enabled(HAVE_BLOSC HAS_BLOSC) is_enabled(HAVE_BZ2 HAS_BZ2) +if(ENABLE_S3_INTERNAL) +SET(WHICH_S3_SDK "internal") +SET(NC_WHICH_S3_SDK "internal") +elseif(ENABLE_S3_AWS) +SET(WHICH_S3_SDK "aws-sdk-cpp") +SET(NC_WHICH_S3_SDK "aws-sdk-cpp") +else() +SET(WHICH_S3_SDK "none") +SET(NC_WHICH_S3_SDK "none") +endif() + +if(WITH_S3_TESTING STREQUAL PUBLIC) +SET(DO_S3_TESTING "public") +elseif(WITH_S3_TESTING) +SET(DO_S3_TESTING "yes") +elseif(NOT WITH_S3_TESTING) +SET(DO_S3_TESTING "no") +else() +SET(DO_S3_TESTING "no") +endif() # Generate file from template. CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/libnetcdf.settings.in" diff --git a/Makefile.am b/Makefile.am index d6e7da0255..c24f2c19d0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -7,9 +7,19 @@ # Ed Hartnett, Ward Fisher +# Put together AM_CPPFLAGS and AM_LDFLAGS. +include $(top_srcdir)/lib_flags.am + # This directory stores libtool macros, put there by aclocal. ACLOCAL_AMFLAGS = -I m4 +TESTS_ENVIRONMENT = +TEST_EXTENSIONS = .sh +#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#TESTS_ENVIRONMENT += export SETX=1; + ## # Turn off plugin directory during distcheck, see # comment thread at https://github.com/Unidata/netcdf-c/pull/2348 @@ -74,10 +84,14 @@ endif if ENABLE_DAP4 DAP4 = libdap4 -XML = libncxml NCDAP4TESTDIR = dap4_test +XML = libncxml endif #DAP4 +if ENABLE_S3_INTERNAL +XML = libncxml # Internal S3 requires XML +endif #ENABLE_S3_INTERNAL + # Build PnetCDF if USE_PNETCDF LIBSRCP = libsrcp diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 5648b024ce..7b87710836 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,18 +7,20 @@ This file contains a high-level description of this package's evolution. Release ## 4.9.3 - TBD + +* Improve S3 documentation, testing, and support See [Github #2686](https://github.com/Unidata/netcdf-c/pull/2686). +* Remove obsolete code. See [Github #2680](https://github.com/Unidata/netcdf-c/pull/2680). * [Bug Fix] Add a crude test to see if an NCZarr path looks like a valid NCZarr/Zarr file. See [Github #2658](https://github.com/Unidata/netcdf-c/pull/2658). ## 4.9.2 - March 14, 2023 This is the maintenance release which adds support for HDF5 version 1.14.0, in addition to a handful of other changes and bugfixes. +* Fix 'make distcheck' error in run_interop.sh. See [Github #2631](https://github.com/Unidata/netcdf-c/pull/2631). * Fix a minor bug in reporting the use of szip. See [Github #2679](https://github.com/Unidata/netcdf-c/pull/2679). * Simplify the handling of XGetopt. See [Github #2678](https://github.com/Unidata/netcdf-c/pull/2678). -* Fix 'make distcheck' error in run_interop.sh. See [Github #2631](https://github.com/Unidata/netcdf-c/pull/2631). * Update `nc-config` to remove inclusion from automatically-detected `nf-config` and `ncxx-config` files, as the wrong files could be included in the output. This is in support of [GitHub #2274](https://github.com/Unidata/netcdf-c/issues/2274). * Update H5FDhttp.[ch] to work with HDF5 version 1.13.2 and later. See [Github #2635](https://github.com/Unidata/netcdf-c/pull/2635). -* Fix 'make distcheck' error in run_interop.sh. See [Github #2631](https://github.com/Unidata/netcdf-c/pull/2631). * [Bug Fix] Update DAP code to enable CURLOPT_ACCEPT_ENCODING by default. See [Github #2630](https://github.com/Unidata/netcdf-c/pull/2630). * [Bug Fix] Fix byterange failures for certain URLs. See [Github #2649](https://github.com/Unidata/netcdf-c/pull/2649). * [Bug Fix] Fix 'make distcheck' error in run_interop.sh. See [Github #2631](https://github.com/Unidata/netcdf-c/pull/2631). diff --git a/config.h.cmake.in b/config.h.cmake.in index 80dc01ee77..ef1384ae82 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -136,27 +136,30 @@ are set when opening a binary file on Windows. */ /* if true, do remote tests */ #cmakedefine ENABLE_DAP_REMOTE_TESTS 1 -/* if true, enable S3 support */ -#cmakedefine ENABLE_S3 1 - -/* if true, S3 SDK is available */ -#cmakedefine ENABLE_S3_SDK 1 - /* if true, enable NCZARR */ #cmakedefine ENABLE_NCZARR 1 /* if true, enable nczarr filter support */ #cmakedefine ENABLE_NCZARR_FILTERS 1 -/* if true, enable S3 testing*/ -#cmakedefine ENABLE_NCZARR_S3_TESTS 1 - /* if true, enable nczarr zip support */ #cmakedefine ENABLE_NCZARR_ZIP 1 /* if true, Allow dynamically loaded plugins */ #cmakedefine ENABLE_PLUGINS 1 +/* if true, enable S3 support */ +#cmakedefine ENABLE_S3 1 + +/* if true, AWS S3 SDK is available */ +#cmakedefine ENABLE_S3_AWS 1 + +/* if true, Force use of S3 internal library */ +#cmakedefine ENABLE_S3_INTERNAL 1 + +/* if true, enable S3 testing*/ +#cmakedefine WITH_S3_TESTING "PUBLIC" + /* if true, run extra tests which may not work yet */ #cmakedefine EXTRA_TESTS 1 @@ -379,6 +382,9 @@ are set when opening a binary file on Windows. */ /* Define to 1 if you have the `strlcat' function. */ #cmakedefine HAVE_STRLCAT 1 +/* Define to 1 if you have the `strlcpy' function. */ +#cmakedefine HAVE_STRLCPY 1 + /* Define to 1 if you have the `strtoll' function. */ #cmakedefine HAVE_STRTOLL 1 diff --git a/configure.ac b/configure.ac index 31d74712ab..63fdb9fe42 100644 --- a/configure.ac +++ b/configure.ac @@ -135,9 +135,9 @@ AC_MSG_NOTICE([checking supported formats]) # An explicit disable of netcdf-4 | netcdf4 is treated as if it was disable-hdf5 AC_MSG_CHECKING([whether we should build with netcdf4 (alias for HDF5)]) AC_ARG_ENABLE([netcdf4], [AS_HELP_STRING([--disable-netcdf4], - [(deprecated synonym for --enable-hdf5)])]) + [(Deprecated) Synonym for --enable-hdf5)])]) test "x$enable_netcdf4" = xno || enable_netcdf4=yes -AC_MSG_RESULT([$enable_netcdf4 (deprecated; Please use with --disable-hdf5)]) +AC_MSG_RESULT([$enable_netcdf4 (Deprecated) Please use --disable-hdf5)]) AC_MSG_CHECKING([whether we should build with netcdf-4 (alias for HDF5)]) AC_ARG_ENABLE([netcdf-4], [AS_HELP_STRING([--disable-netcdf-4], [(synonym for --disable-netcdf4)])]) @@ -565,7 +565,7 @@ if test "x$enable_libxml2" = xyes; then if test -z "$NC_XML2_CONFIG"; then AC_MSG_ERROR([Cannot find xml2-config utility. Either install the libxml2 development package, or re-run configure with --disable-libxml2 to use the bundled xml2 parser]) fi - # We can optionally use libxml2 for DAP4 + # We can optionally use libxml2 for DAP4 and nch5comms, if enabled AC_CHECK_LIB([xml2],[xmlReadMemory],[have_libxml2=yes],[have_libxml2=no]) if test "x$have_libxml2" = "xyes" ; then AC_SEARCH_LIBS([xmlReadMemory],[xml2 xml2.dll cygxml2.dll], [],[]) @@ -803,13 +803,14 @@ AC_MSG_RESULT($enable_s3) AC_MSG_CHECKING([whether netcdf NCZarr S3 support should be enabled]) AC_ARG_ENABLE([nczarr-s3], [AS_HELP_STRING([--enable-nczarr-s3], - [enable netcdf NCZarr S3 support; Deprecated in favor of --enable-s3])]) + [(Deprecated) enable netcdf NCZarr S3 support; Deprecated in favor of --enable-s3])]) +AC_MSG_RESULT([$enable_nczarr_s3 (Deprecated) Please use --enable-s3)]) + # Set enable_s3 instead of enable_nczarr_s3 if test "x$enable_s3" = xno && test "x$enable_nczarr_s3" = xyes ; then enable_s3=yes # back compatibility fi unset enable_nczarr_s3 -AC_MSG_RESULT($enable_s3) # Note we check for the library after checking for enable_s3 # because for some reason this fails if we unconditionally test for sdk @@ -818,48 +819,73 @@ if test "x$enable_s3" = xyes ; then # See if we have the s3 aws library # Check for the AWS S3 SDK library AC_LANG_PUSH([C++]) -AC_SEARCH_LIBS([aws_allocator_is_valid],[aws-c-common aws-cpp-sdk-s3 aws-cpp-sdk-core], [enable_s3_sdk=yes],[enable_s3_sdk=no]) +AC_SEARCH_LIBS([aws_allocator_is_valid],[aws-c-common aws-cpp-sdk-s3 aws-cpp-sdk-core], [enable_s3_aws=yes],[enable_s3_aws=no]) AC_LANG_POP else -enable_s3_sdk=no +enable_s3_aws=no fi AC_MSG_CHECKING([whether AWS S3 SDK library is available]) -AC_MSG_RESULT([$enable_s3_sdk]) +AC_MSG_RESULT([$enable_s3_aws]) + +# Check for enabling forced use of Internal S3 library +AC_MSG_CHECKING([whether internal S3 support should be used]) +AC_ARG_ENABLE([s3-internal], + [AS_HELP_STRING([--enable-s3-internal], + [enable internal S3 support])]) +test "x$enable_s3_internal" = xyes || enable_s3_internal=no +AC_MSG_RESULT($enable_s3_internal) -if test "x$enable_s3_sdk" = xno ; then +if test "x$enable_s3_aws" = xno && test "x$enable_s3_internal" = xno ; then AC_MSG_WARN([No S3 library available => S3 support disabled]) enable_S3=no fi +if test "x$enable_s3_aws" = xyes && test "x$enable_s3_internal" = xyes ; then +AC_MSG_WARN([Both aws-sdk-cpp and s3-internal enabled => use s3-internal.]) +enable_s3_aws=no +fi + +if test "x$enable_s3_internal" = xyes ; then +if test "x$ISOSX" != xyes && test "x$ISMINGW" != xyes && test "x$ISMSVC" != xyes ; then +# Find crypto libraries if using ssl +AC_CHECK_LIB([ssl],[ssl_create_cipher_list]) +AC_CHECK_LIB([crypto],[SHA256]) +fi +fi + # Check for enabling S3 testing -AC_MSG_CHECKING([whether netcdf zarr S3 testing should be enabled]) -AC_ARG_ENABLE([nczarr-s3-tests], - [AS_HELP_STRING([--enable-nczarr-s3-tests], - [enable netcdf zarr S3 testing])]) -test "x$enable_nczarr_s3_tests" = xyes || enable_nczarr_s3_tests=no -AC_MSG_RESULT($enable_nczarr_s3_tests) +AC_MSG_CHECKING([what level of netcdf S3 testing should be enabled]) +AC_ARG_WITH([s3-testing], + [AS_HELP_STRING([--with-s3-testing=yes|no|public], + [control netcdf S3 testing])], + [], [with_s3_testing=public]) +AC_MSG_RESULT($with_s3_testing) # Disable S3 tests if S3 support is disabled -if test "x$enable_nczarr_s3_tests" = xyes ; then - if test "x$enable_s3" = xno ; then - AC_MSG_ERROR([S3 support is disabled => no testing]) - enable_nczarr_s3_tests=no +if test "x$enable_s3" = xno ; then + if test "x$with_s3_testing" != xno ; then + AC_MSG_WARN([S3 support is disabled => no testing]) + with_s3_testing=no fi fi if test "x$enable_s3" = xyes ; then AC_DEFINE([ENABLE_S3], [1], [if true, build netcdf-c with S3 support enabled]) fi -if test "x$enable_nczarr_s3_tests" = xyes ; then - AC_DEFINE([ENABLE_NCZARR_S3_TESTS], [1], [if true, build libnczarr with S3 tests enabled]) + +if test "x$enable_s3_aws" = xyes ; then +AC_DEFINE([ENABLE_S3_AWS], [1], [If true, then use aws S3 library]) fi -if test "x$enable_s3_sdk" = xyes ; then -AC_DEFINE([ENABLE_S3_SDK], [1], [If true, then S3 sdk was found]) + +if test "x$enable_s3_internal" = xyes ; then +AC_DEFINE([ENABLE_S3_INTERNAL], [1], [If true, then use internal S3 library]) fi -if test "x$enable_nczarr_s3_tests" = xyes ; then - AC_MSG_WARN([*** DO NOT ENABLE_NCZARR_S3_TESTS UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***]) +AC_DEFINE_UNQUOTED([WITH_S3_TESTING], [$with_s3_testing], [control S3 testing.]) + +if test "x$with_s3_testing" = xyes ; then + AC_MSG_WARN([*** DO NOT SPECIFY WITH_S3_TESTING=YES UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***]) fi # Set default @@ -1209,7 +1235,7 @@ AC_CHECK_FUNCS([strlcat snprintf strcasecmp fileno \ strdup strtoll strtoull \ mkstemp mktemp random \ getrlimit gettimeofday fsync MPI_Comm_f2c MPI_Info_f2c \ - strncasecmp]) + strncasecmp strlcpy]) # See if clock_gettime is available and its arg types. AC_CHECK_FUNCS([clock_gettime]) @@ -1480,7 +1506,7 @@ fi hdf5_parallel=no hdf5_supports_par_filters=no enable_hdf5_szip=no -has_ross3=no +has_hdf5_ros3=no if test "x$enable_hdf5" = xyes; then @@ -1524,8 +1550,8 @@ if test "x$enable_hdf5" = xyes; then AC_SEARCH_LIBS([H5Dread_chunk],[hdf5_hldll hdf5_hl], [has_readchunks=yes], [has_readdhunks=no]) # See if hdf5 library supports Read-Only S3 (byte-range) driver - AC_SEARCH_LIBS([H5Pset_fapl_ros3],[hdf5_hldll hdf5_hl], [has_ros3=yes], [has_ros3=no]) - if test "x$has_ros3" = xyes && test "x$enable_byterange" = xyes; then + AC_SEARCH_LIBS([H5Pset_fapl_ros3],[hdf5_hldll hdf5_hl], [has_hdf5_ros3=yes], [has_hdf5_ros3=no]) + if test "x$has_hdf5_ros3" = xyes && test "x$enable_byterange" = xyes; then AC_DEFINE([ENABLE_HDF5_ROS3], [1], [if true, support byte-range using hdf5 virtual file driver.]) fi @@ -1824,9 +1850,11 @@ AM_CONDITIONAL(RELAX_COORD_BOUND, [test "xyes" = xyes]) AM_CONDITIONAL(HAS_PAR_FILTERS, [test x$hdf5_supports_par_filters = xyes ]) # We need to simplify the set of S3 and Zarr flag combinations AM_CONDITIONAL(ENABLE_S3, [test "x$enable_s3" = xyes]) -AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$enable_s3_sdk" = xyes]) +AM_CONDITIONAL(ENABLE_S3_AWS, [test "x$enable_s3_aws" = xyes]) +AM_CONDITIONAL(ENABLE_S3_INTERNAL, [test "x$enable_s3_internal" = xyes]) AM_CONDITIONAL(ENABLE_NCZARR, [test "x$enable_nczarr" = xyes]) -AM_CONDITIONAL(ENABLE_NCZARR_S3_TESTS, [test "x$enable_nczarr_s3_tests" = xyes]) +AM_CONDITIONAL(ENABLE_S3_TESTPUB, [test "x$with_s3_testing" != xno]) # all => public +AM_CONDITIONAL(ENABLE_S3_TESTALL, [test "x$with_s3_testing" = xyes]) AM_CONDITIONAL(ENABLE_NCZARR_ZIP, [test "x$enable_nczarr_zip" = xyes]) AM_CONDITIONAL(HAS_MULTIFILTERS, [test "x$has_multifilters" = xyes]) AM_CONDITIONAL(HAVE_SZ, [test "x$have_sz" = xyes]) @@ -1932,8 +1960,11 @@ AC_SUBST(HAS_BYTERANGE,[$enable_byterange]) AC_SUBST(RELAX_COORD_BOUND,[yes]) AC_SUBST([HAS_PAR_FILTERS], [$hdf5_supports_par_filters]) AC_SUBST(HAS_S3,[$enable_s3]) +AC_SUBST(HAS_S3_AWS,[$enable_s3_aws]) +AC_SUBST(HAS_S3_INTERNAL,[$enable_s3_internal]) +AC_SUBST(HAS_HDF5_ROS3,[$has_hdf5_ros3]) AC_SUBST(HAS_NCZARR,[$enable_nczarr]) -AC_SUBST(DO_NCZARR_S3_TESTS,[$enable_nczarr_s3_tests]) +AC_SUBST(DO_S3_TESTING,[$with_s3_testing]) AC_SUBST(HAS_NCZARR_ZIP,[$enable_nczarr_zip]) AC_SUBST(HAS_MULTIFILTERS,[$has_multifilters]) AC_SUBST(DO_NCZARR_ZIP_TESTS,[$enable_nczarr_zip]) @@ -1943,6 +1974,17 @@ AC_SUBST(DO_FILTER_TESTS,[$enable_filter_testing]) AC_SUBST(HAS_SZLIB,[$have_sz]) AC_SUBST(HAS_SZLIB_WRITE, [$have_sz]) AC_SUBST(HAS_ZSTD,[$have_zstd]) +AC_SUBST(DO_LARGE_TESTS,[$enable_large_file_tests]) + +if test "x$enable_s3_aws" = xyes ; then +AC_SUBST(WHICH_S3_SDK,[aws-sdk-cpp]) +fi +if test "x$enable_s3_internal" = xyes; then +AC_SUBST(WHICH_S3_SDK,[internal]) +fi +if test "x$enable_s3_aws" = xno && test "x$enable_s3_internal" = xno; then +AC_SUBST(WHICH_S3_SDK,[none]) +fi # Always available std_filters="deflate bz2" @@ -2032,7 +2074,7 @@ AC_SUBST([NOUNDEFINED]) # example: AX_SET_META([NC_HAS_NC2],[$nc_build_v2],[]) # Because it checks for no. # AX_SET_META([NC_HAS_HDF4],[$enable_hdf4],[yes]) AC_DEFUN([AX_SET_META],[ - if [ test "x$2" = x$3 ]; then + if [ test "x$2" = "x$3" ]; then AC_SUBST([$1]) $1=1 else AC_SUBST([$1]) $1=0 @@ -2060,8 +2102,11 @@ AX_SET_META([NC_HAS_CDF5],[$enable_cdf5],[yes]) AX_SET_META([NC_HAS_ERANGE_FILL], [$enable_erange_fill],[yes]) AX_SET_META([NC_HAS_PAR_FILTERS], [$hdf5_supports_par_filters],[yes]) AX_SET_META([NC_HAS_BYTERANGE],[$enable_byterange],[yes]) -AX_SET_META([NC_HAS_NCZARR],[$enable_nczarr],[yes]) AX_SET_META([NC_HAS_S3],[$enable_s3],[yes]) +AX_SET_META([NC_HAS_S3_AWS],[$enable_s3_aws],[yes]) +AX_SET_META([NC_HAS_S3_INTERNAL],[$enable_s3_internal],[yes]) +AX_SET_META([NC_HAS_HDF5_ROS3],[$has_hdf5_ros3],[yes]) +AX_SET_META([NC_HAS_NCZARR],[$enable_nczarr],[yes]) AX_SET_META([NC_HAS_MULTIFILTERS],[$has_multifilters],[yes]) AX_SET_META([NC_HAS_LOGGING],[$enable_logging],[yes]) AX_SET_META([NC_HAS_QUANTIZE],[$enable_quantize],[yes]) diff --git a/dap4_test/maketests.sh b/dap4_test/maketests.sh index 4d5ae2d542..aa3cbbc669 100755 --- a/dap4_test/maketests.sh +++ b/dap4_test/maketests.sh @@ -40,7 +40,7 @@ if test "x$DEBUG" = x1 ; then F=`ls -1 *.dap | sed -e 's/[.]dap//' |tr '\r\n' ' '` popd for f in ${F} ; do - if ! diff -wBb ${SRC}/dmrtestfiles/${f}.dmr ./dmrtestfiles/${f}.dmr > /dev/null 2>&1 ; then + if ! diff -wBb ${SRC}/dmrtestfiles/${f}.dmr ./dmrtestfiles/${f}.dmr &> /dev/null ; then echo diff -wBb ${SRC}/dmrtestfiles/${f}.dmr ./dmrtestfiles/${f}.dmr diff -wBb ${SRC}/dmrtestfiles/${f}.dmr ./dmrtestfiles/${f}.dmr fi diff --git a/dap4_test/test_dap4url.sh b/dap4_test/test_dap4url.sh index 7180ecd8ba..674e9f8aac 100755 --- a/dap4_test/test_dap4url.sh +++ b/dap4_test/test_dap4url.sh @@ -5,7 +5,7 @@ export srcdir; . ../test_common.sh -set -x +set -e . ${srcdir}/d4test_common.sh diff --git a/dap4_test/test_hyrax.sh b/dap4_test/test_hyrax.sh index ff1239cd03..c370e41150 100755 --- a/dap4_test/test_hyrax.sh +++ b/dap4_test/test_hyrax.sh @@ -11,8 +11,10 @@ set -e echo "test_hyrax.sh:" +# Some test baseline paths are too long for tar. +TOOLONG="AIRS/AIRH3STM.003/2002.12.01/AIRS.2002.12.01.L3.RetStd_H031.v4.0.21.0.G06101132853.hdf?/TotalCounts_A" + F="\ -AIRS/AIRH3STM.003/2002.12.01/AIRS.2002.12.01.L3.RetStd_H031.v4.0.21.0.G06101132853.hdf?/TotalCounts_A \ RSS/amsre/bmaps_v05/y2006/m01/amsre_20060131v5.dat?/time_a[0:2][0:5] \ nc4_test_files/nc4_nc_classic_no_comp.nc \ nc4_test_files/nc4_nc_classic_comp.nc \ @@ -35,6 +37,13 @@ echo "***XFAIL: Cannot find test.opendap.org testserver; test skipped" exit 0 fi +gethyraxbaseline() { + ARG1="$1" + BASELINEPART=`echo $ARG1 | cut -d':' -f1` + URLPART=`echo $ARG1 | cut -d':' -f2` + if test "x$BASELINEPART" = "x$URLPART" ; then BASELINEPART=""; fi +} + makehyraxurl() { if test "x$QUERY" != x ; then QUERY="&dap4.ce=$QUERY"; fi QUERY="?dap4.checksum=true${QUERY}" diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index 119d23eece..b9521ad75c 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -95,6 +95,7 @@ obsolete/fan_utils.html bestpractices.md filters.md indexing.md inmemory.md DAP2.dox FAQ.md known_problems.md COPYRIGHT.dox user_defined_formats.md DAP4.md DAP4.dox -testserver.dox byterange.md filters.md nczarr.md auth.md quantize.md) +testserver.dox byterange.md filters.md nczarr.md auth.md quantize.md +quickstart_paths.md cloud.md) ADD_EXTRA_DIST("${CUR_EXTRA_DIST}") diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index ccfcd90fee..84a227cebf 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -812,6 +812,7 @@ INPUT = @abs_top_srcdir@/docs/mainpage.dox \ @abs_top_srcdir@/docs/inmemory.md \ @abs_top_srcdir@/docs/byterange.md \ @abs_top_srcdir@/docs/nczarr.md \ + @abs_top_srcdir@/docs/cloud.md \ @abs_top_srcdir@/docs/notes.md \ @abs_top_srcdir@/docs/building-with-cmake.md \ @abs_top_srcdir@/docs/FAQ.md \ @@ -829,6 +830,7 @@ INPUT = @abs_top_srcdir@/docs/mainpage.dox \ @abs_top_srcdir@/docs/all-error-codes.md \ @abs_top_srcdir@/docs/filters.md \ @abs_top_srcdir@/docs/quickstart_filters.md \ + @abs_top_srcdir@/docs/quickstart_paths.md \ @abs_top_srcdir@/include/netcdf.h \ @abs_top_srcdir@/include/netcdf_mem.h \ @abs_top_srcdir@/include/netcdf_par.h \ diff --git a/docs/Makefile.am b/docs/Makefile.am index e1ef94e67f..a167e0d869 100644 --- a/docs/Makefile.am +++ b/docs/Makefile.am @@ -13,7 +13,8 @@ windows-binaries.md dispatch.md building-with-cmake.md CMakeLists.txt groups.dox notes.md install-fortran.md credits.md auth.md filters.md \ obsolete/fan_utils.html indexing.dox inmemory.md FAQ.md \ known_problems.md COPYRIGHT.md inmeminternal.dox testserver.dox \ -byterange.md nczarr.md quantize.md all-error-codes.md +byterange.md nczarr.md quantize.md all-error-codes.md \ +quickstart_paths.md cloud.md # Turn off parallel builds in this directory. .NOTPARALLEL: diff --git a/docs/byterange.md b/docs/byterange.md index 7e67407742..352cafdafa 100644 --- a/docs/byterange.md +++ b/docs/byterange.md @@ -16,10 +16,9 @@ the remote server supports byte-range access. Two examples: -1. An Amazon S3 object containing a netcdf classic file. - - location: "https://remotetest.unidata.ucar.edu/thredds/fileServer/testdata/2004050300_eta_211.nc#mode=bytes" -2. A Thredds Server dataset supporting the Thredds HTTPServer protocol. - and containing a netcdf enhanced file. +1. A Thredds server supporting the Thredds "fileserver" Thredds protocol, and containing a netcdf classic file. + - location: "https://remotetest.unidata.ucar.edu/thredds/fileserver/testdata/2004050300_eta_211.nc#mode=bytes" +2. An Amazon S3 dataset containing a netcdf enhanced file. - location: "http://noaa-goes16.s3.amazonaws.com/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes" Other remote servers may also provide byte-range access in a similar form. @@ -36,11 +35,9 @@ to the *./configure* command for Automake. For Cmake, the option flag is *-DENABLE_BYTERANGE=true*. This capability requires access to *libcurl*, and an error will occur -if byterange is enabled, but no *libcurl* could not be located. +if byterange is enabled, but *libcurl* could not be located. In this, it is similar to the DAP2 and DAP4 capabilities. -Note also that here, the term "http" is often used as a synonym for *byterange*. - # Run-time Usage {#byterange_url} In order to use this capability at run-time, with *ncdump* for @@ -59,15 +56,17 @@ classic, enhanced, cdf5, etc. # Architecture {#byterange_arch} -Internally, this capability is implemented with three files: +Internally, this capability is implemented with the following drivers: 1. libdispatch/dhttp.c -- wrap libcurl operations. 2. libsrc/httpio.c -- provide byte-range reading to the netcdf-3 dispatcher. -3. libhdf5/H5FDhttp.c -- provide byte-range reading to the netcdf-4 dispatcher. +3. libhdf5/H5FDhttp.c -- provide byte-range reading to the netcdf-4 dispatcher for non-cloud storage. +4. H5FDros3.c -- provide byte-range reading to the netcdf-4 dispatcher for cloud storage (Amazon S3 currently). Both *httpio.c* and *H5FDhttp.c* are adapters that use *dhttp.c* to do the work. Testing for the magic number is also carried out by using the *dhttp.c* code. +*H5FDros3* is also an adapter, but specialized for cloud storage access. ## NetCDF Classic Access @@ -77,12 +76,12 @@ netcdf-3 code be independent of the lowest level IO access mechanisms. This is how in-memory and mmap based access is implemented. The file *httpio.c* is the dispatcher used to provide byte-range IO for the netcdf-3 code. - Note that *httpio.c* is mostly just an adapter between the *ncio* API and the *dhttp.c* code. ## NetCDF Enhanced Access +### Non-Cloud Access Similar to the netcdf-3 code, the HDF5 library provides a secondary dispatch mechanism *H5FD*. This allows the HDF5 code to be independent of the lowest level IO access mechanisms. @@ -90,13 +89,14 @@ The netcdf-4 code in libhdf5 is built on the HDF5 library, so it indirectly inherits the H5FD mechanism. The file *H5FDhttp.c* implements the H5FD dispatcher API -and provides byte-range IO for the netcdf-4 code +and provides byte-range IO for the netcdf-4 code (and for the HDF5 library as a side effect). +It only works for non-cloud servers such as the Unidata Thredds server. Note that *H5FDhttp.c* is mostly just an adapter between the *H5FD* API and the *dhttp.c* code. -# The dhttp.c Code {#byterange_dhttp} +#### The dhttp.c Code {#byterange_dhttp} The core of all this is *dhttp.c* (and its header *include/nchttp.c*). It is a wrapper over *libcurl* @@ -112,7 +112,7 @@ The type *fileoffset_t* is used to avoid use of *off_t* or *off64_t* which are too volatile. It is intended to be represent file lengths and offsets. -## nc_http_open +##### nc_http_open The *nc_http_open* procedure creates a *Curl* handle and returns it in the *curlp* argument. It also obtains and searches the headers looking for two headers: @@ -122,7 +122,7 @@ looking for two headers: The dataset length is returned in the *filelenp* argument. -## nc_http_read +#### nc_http_read The *nc_http_read* procedure reads a specified set of contiguous bytes as specified by the *start* and *count* arguments. It takes the *Curl* @@ -134,18 +134,28 @@ is a dynamically expandable byte vector (see the file *include/ncbytes.h*). This procedure reads *count* bytes from the remote dataset starting at the offset *start* position. The bytes are stored in *buf*. -## nc_http_close +#### nc_http_close The *nc_http_close* function closes the *Curl* handle and does any necessary cleanup. +### Cloud Access + +The HDF5 library code-base also provides a Virtual File Drive (VFD) +capable of providing byte-range access to cloud storage +(Amazon S3 specifically). + +This VFD is called *H5FDros3*. In order for the netcdf library +to make use of it, the HDF5 library must be built using the +*--enable-ros3-vfd* option. +Netcdf can discover that this capability was enabled and can +then make use of it to provide byte-range access to the cloud. + # Point of Contact {#byterange_poc} __Author__: Dennis Heimbigner
__Email__: dmh at ucar dot edu
__Initial Version__: 12/30/2018
-__Last Revised__: 12/30/2018 +__Last Revised__: 3/11/2023 - -*/ diff --git a/docs/cloud.md b/docs/cloud.md new file mode 100644 index 0000000000..fb459080df --- /dev/null +++ b/docs/cloud.md @@ -0,0 +1,350 @@ + +Cloud Storage Access Using The NetCDF-C Library +============================ + + +# Cloud Storage Access Using The NetCDF-C Library {#nccloud_head} + +\tableofcontents + +# Introduction {#nccloud_introduction} + +The NetCDF-C library supports limited access to cloud storage. +Currently, that access is restricted to the Amazon S3 cloud storage, +so this document is S3-centric. +It is expected that over time, access to additional cloud stores will be added, +and this document will be expanded to cover those additional cases. + +# S3 Use-Cases in NetCDF-C + +At the moment, the NetCDF-C library provides access to S3 for the following purposes: +* Byte-Range access to Netcdf-4/HDF5 datasets stored as single object in S3. +* Zarr dataset access as represented as a "tree" of objects in S3. + +# Amazon S3 Storage Access Mechanisms {#nccloud_s3_sdks} + +Three S3 storage drivers are available for accessing Amazon S3. +1. *H5FDros3* -- This is an HDF5 Virtual File Driver provided as part of the HDF5 library. It is specifically used by NetCDF-C to perform byte-range access to netcdf-4 files. In order for this functionality to be available to NetCDF-C, +it is necessary to build the HDF5 library with the *--enable-ros3-vfd* option. +2. *aws-sdk-cpp* -- This is the standard Amazon AWS S3 SDK. It is written in C++. It is used specifically for NCZarr access to Zarr formatted datasets on S3. +3. *nch5s3comms* -- This is an experimental SDK, written in C, that provides the minimum functionality necessary to access S3. As with *aws-sdk-cpp*, it is used specifically for NCZarr access to Zarr formatted datasets on S3. + +All three S3 drivers use the AWS profile mechanism to provide configuration information, and especially to provide authorization information. +Specifically, the ''~/.aws/credentials'' file should contain something like this. + +``` +[default] +output = json +aws_access_key_id=XXXX... +aws_secret_access_key=YYYY... +``` + +## Byte-Range Access {#nccloud_byterange} + +The NetCDF-C library contains a mechanism for accessing traditional netcdf-4 files stored on remote computers. +The idea is to treat the remote data as if it was one big file, +and to use the HTTP protocol to read a contiguous sequence of bytes +from that remote "file". +This is performed using the "byte-range" header in an HTTP request. + +In the Amazon S3 context, a copy of a dataset, a netcdf-3 or netdf-4 file, is uploaded into a single object in some bucket. +Then using the key to this object, it is possible to tell the netcdf-c library to treat the object as a remote file and to use the HTTP Byte-Range protocol to access the contents of the object. +The dataset object is referenced using a URL with the trailing fragment containing the string ````#mode=bytes````. + +An examination of the test program _nc_test/test_byterange.sh_ shows simple examples using the _ncdump_ program. +One such test is specified as follows: +```` +https://s3.us-east-1.amazonaws.com/noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes +```` +Note that for S3 access, it is expected that the URL is in what is called "path" format where the bucket, _noaa-goes16_ in this case, is part of the URL path instead of the host. + +The _#mode=bytes_ mechanism generalizes to work with most servers that support byte-range access. + +Specifically, Thredds servers support such access using the HttpServer access method as can be seen from this URL taken from the above test program. +```` +https://thredds-test.unidata.ucar.edu/thredds/fileServer/irma/metar/files/METAR_20170910_0000.nc#bytes +```` + +# References {#nccloud_bib} + +[1] [Amazon Simple Storage Service Documentation](https://docs.aws.amazon.com/s3/index.html)
+[2] [Amazon Simple Storage Service Library](https://github.com/aws/aws-sdk-cpp)
+[11] [Conda-forge / packages / aws-sdk-cpp](https://anaconda.org/conda-forge/aws-sdk-cpp)
+ +# Appendix A. S3 Build Support {#nccloud_s3build} + +Currently the following build cases are known to work. + + +
Operating SystemBuild SystemSDKS3 Support +
Linux Automake aws-s3-sdk yes +
Linux Automake nch5s3comms yes +
Linux CMake aws-s3-sdk yes +
Linux CMake nch5s3comms yes +
OSX Automake aws-s3-sdk unknown +
OSX Automake nch5s3comms unknown +
OSX CMake aws-s3-sdk unknown +
OSX CMake nch5s3comms unknown +
Visual Studio CMake aws-s3-sdk no (tests-fail) +
Visual Studio CMake nch5s3comms yes +
Cygwin Automake aws-s3-sdk unknown +
Cygwin Automake nch5s3comms yes +
Cygwin CMake aws-s3-sdk unknown +
Cygwin CMake nch5s3comms unknown +
Mingw Automake aws-s3-sdk unknown +
Mingw Automake nch5s3comms unknown +
Mingw CMake aws-s3-sdk unknown +
Mingw CMake nch5s3comms unknown +
+ +## Automake + +There are several options relevant to Amazon S3 support. +These are as follows. + +1. _--enable-s3_ -- Enable S3 support. +2. _--enable-s3-internal_ -- Force use of the *nch5s3comms* SDK instead of the *aws-cpp-sdk* (assuming it is available). +3. _--with-s3-testing_=yes|no|public -- "yes" means do all S3 tests, "no" means do no S3 testing, "public" means do only those tests that involve publically accessible S3 data. + +__A note about using S3 with Automake.__ +If S3 support is desired, and using the Amazon "aws-sdk-cpp" SDK, and using Automake, then LDFLAGS must be properly set, namely to this. +```` +LDFLAGS="$LDFLAGS -L/usr/local/lib -laws-cpp-sdk-s3" +```` +The above assumes that these libraries were installed in '/usr/local/lib', so the above requires modification if they were installed elsewhere. + +Note also that if S3 support is enabled, then you need to have a C++ compiler installed because the "aws-sdk-cpp" S3 support code is written in C++. + +## CMake {#nccloud_cmake} + +The necessary CMake flags are as follows (with defaults) + +1. *-DENABLE_S3* -- Controll S3 support +2. *-DENABLE_S3_INTERNAL* -- Force use of the *nch5s3comms* SDK instead of the *aws-cpp-sdk*. +3. *-DWITH-S3-TESTING_=ON|OFF|PUBLIC -- "ON" means do all S3 tests, "OFF" means do no S3 testing, "PUBLIC" means do only those tests that involve publically accessible S3 data. + +Note that unlike Automake, CMake can properly locate C++ libraries, so it should not be necessary to specify _-laws-cpp-sdk-s3_ assuming that the aws s3 libraries are installed in the default location. +For CMake with Visual Studio, the default location is here: + +```` +C:/Program Files (x86)/aws-cpp-sdk-all +```` + +It is possible to install the sdk library in another location. +In this case, one must add the following flag to the cmake command. +```` +cmake ... -DAWSSDK_DIR=\ +```` +where "awssdkdir" is the path to the sdk installation. +For example, this might be as follows. +```` +cmake ... -DAWSSDK_DIR="c:\tools\aws-cpp-sdk-all" +```` +This can be useful if blanks in path names cause problems in your build environment. + +# Appendix B. Building the S3 SDKs {#nccloud_s3sdk} + +As mentioned, three S3 storage drivers are available for accessing Amazon S3. +1. *H5FDros3* +2. *aws-sdk-cpp* +2. *nch5s3comms* + +## Building *H5FDros3* + +This driver is part of the HDF5 library codebase. +It must be enabled at the time that the HDF5 library is built +by using the *--enable-ros3=vfd* option. +If built, then the NetCDF-C build process should detect it and make use of it. + +## Building *aws-sdk-cpp* + +Amazon provides (thru AWS-labs) an SDK for accessing the Amazon S3 cloud. +This library, [aws-sdk-cpp library](https://github.com/aws/aws-sdk-cpp.git), +has a number of properties of interest: +* It is written in C++ +* It is available on [GitHub](https://github.com/aws/aws-sdk-cpp.git), +* It uses CMake + ninja as its primary build system. + +### *\*nix\** Build + +For linux, the following context works. Of course your mileage may vary. +* OS: ubuntu 21 +* aws-sdk-cpp version 1.9.96 (or later) +* Dependencies: openssl, libcurl, cmake, ninja (ninja-build using *apt-get*) + +#### AWS-SDK-CPP CMake Build Recipe +```` +git clone --recurse-submodules https://www.github.com/aws/aws-sdk-cpp +pushd aws-sdk-cpp +mkdir build +cd build +PREFIX=/usr/local +FLAGS="-DCMAKE_INSTALL_PREFIX=${PREFIX} \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_MODULE_PATH=${PREFIX}/lib/cmake \ + -DCMAKE_POLICY_DEFAULT_CMP0075=NEW \ + -DBUILD_ONLY=s3 \ + -DENABLE_UNITY_BUILD=ON \ + -DENABLE_TESTING=OFF \ + -DCMAKE_BUILD_TYPE=$CFG \ + -DSIMPLE_INSTALL=ON" +cmake -GNinja $FLAGS .. +ninja all +ninja install +cd .. +popd +```` + +### Windows build +It is possible to build and install aws-sdk-cpp on Windows using CMake. +Unfortunately, testing currently fails. + +For Windows, the following context work. Of course your mileage may vary. +* OS: Windows 10 64-bit with Visual Studio community edition 2019. +* aws-sdk-cpp version 1.9.96 (or later) +* Dependencies: openssl, libcurl, cmake + +#### AWS-SDK-CPP Build Recipe + +This command-line build assumes one is using Cygwin or Mingw to provide +tools such as bash. + +```` +git clone --recurse-submodules https://www.github.com/aws/aws-sdk-cpp +pushd aws-sdk-cpp +mkdir build +cd build +CFG="Release" +PREFIX="c:/tools/aws-sdk-cpp" + +FLAGS="-DCMAKE_INSTALL_PREFIX=${PREFIX} \ + -DCMAKE_INSTALL_LIBDIR=lib" \ + -DCMAKE_MODULE_PATH=${PREFIX}/cmake \ + -DCMAKE_POLICY_DEFAULT_CMP0075=NEW \ + -DBUILD_ONLY=s3 \ + -DENABLE_UNITY_BUILD=ON \ + -DCMAKE_BUILD_TYPE=$CFG \ + -DSIMPLE_INSTALL=ON" + +rm -fr build +mkdir -p build +cd build +cmake -DCMAKE_BUILD_TYPE=${CFG} $FLAGS .. +cmake --build . --config ${CFG} +cmake --install . --config ${CFG} +cd .. +popd +```` +Notice that the sdk is being installed in the directory "c:\tools\aws-sdk-cpp" +rather than the default location "c:\Program Files (x86)/aws-sdk-cpp-all" +This is because when using a command line, an install path that contains +blanks may not work. + +In order for CMake to find the aws sdk libraries, +the following environment variables must be set: +```` +AWSSDK_ROOT_DIR="c:/tools/aws-sdk-cpp" +AWSSDKBIN="/cygdrive/c/tools/aws-sdk-cpp/bin" +PATH="$PATH:${AWSSDKBIN}" +```` +Then the following options must be specified for cmake. +```` +-DAWSSDK_ROOT_DIR=${AWSSDK_ROOT_DIR} +-DAWSSDK_DIR=${AWSSDK_ROOT_DIR}/lib/cmake/AWSSDK" +```` +## Building ``nch5s3comms'' + +This is an experimental SDK provided internally in the netcdf-c library. + +* It is written in C +* It provides the minimum functionality necessary to read/write/search an Amazon S3 bucket. +* It was constructed by heavily modifying the HDF5 *H5FDros3* Virtual File Driver and combining it with crypto code wrappers provided by libcurl. The resulting file was then modified to fit into the netcdf coding style. +* The resulting code is rather ugly, but appears to work under at least Linux and under Windows (using Visual C++). + +### Dependencies + +* *\*nix\**: the following packages need to be installed: openssl, libcurl, (optionally) libxml2. +* *Windows (Visual C++)*: the following packages need to be installed: libcurl, (optionally) libxml2. + +### Build Options + +In order to enable this SDK, the Automake option *--enable-s3-internal* or the CMake option *-DENABLE_S3_INTERNAL=ON* must be specified. + +### Testing S3 Support {#nccloud_testing_S3_support} + +The pure S3 test(s) are in the _unit_tests_ directory. +Currently, by default, testing of S3 is supported only for Unidata members of the NetCDF Development Group. +This is because it uses a Unidata-specific bucket is inaccessible to the general user. + +# Appendix C. AWS Selection Algorithms. {#nccloud_awsselect} + +If byterange support is enabled, the netcdf-c library will parse the files +```` +${HOME}/.aws/config +and +${HOME}/.aws/credentials +```` +to extract profile names plus a list of key=value pairs. +In case of duplicates, *credentials* takes precedence over *config*. + +This example is typical of the contents of these files. +```` +[default] + aws_access_key_id=XXXXXXXXXXXXXXXXXXXX + aws_secret_access_key=YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY + aws_region=ZZZZZZZZZ +```` +The keys in the profile will be used to set various parameters in the library + +## Profile Selection + +The algorithm for choosing the active profile to use is as follows: + +1. If the "aws.profile" fragment flag is defined in the URL, then it is used. For example, see this URL. +```` +https://...#mode=nczarr,s3&aws.profile=xxx +```` +2. If the "AWS.PROFILE" entry in the .rc file (i.e. .netrc or .dodsrc) is set, then it is used. +3. If defined, then profile "default" is used. +4. Otherwise the profile "no" is used. + +The profile named "no" is a special profile that the netcdf-c library automatically defines. +It should not be defined anywhere else. It signals to the library that no credentialas are to used. +It is equivalent to the "--no-sign-request" option in the AWS CLI. + +## Region Selection + +If the specified URL is of the form +```` +s3:///key +```` +Then this is rebuilt to this form: +```` +s3://s2.<region>.amazonaws.com>/key +```` +However this requires figuring out the region to use. +The algorithm for picking an region is as follows. + +1. If the "aws.region" fragment flag is defined in the URL, then it is used. +2. The active profile is searched for the "aws_region" key. +3. If the "AWS.REGION" entry in the .rc file (i.e. .netrc or .dodsrc) is set, then it is used. +4. Otherwise use "us-east-1" region. + +## Authorization Selection + +Picking an access-key/secret-key pair is always determined +by the current active profile. To choose to not use keys +requires that the active profile must be "no". + +# Change Log {#nccloud_changelog} +[Note: minor text changes are not included.] + +## 3/8/2023 +1. Add an internal, minimal Amazon S3 SDK to support S3 access especially for Windows. + +# Point of Contact {#nccloud_poc} + +__Author__: Dennis Heimbigner
+__Email__: dmh at ucar dot edu
+__Initial Version__: 3/8/2023
+__Last Revised__: 3/8/2023 diff --git a/docs/nczarr.md b/docs/nczarr.md index af6c1cc260..5c92dcf89e 100644 --- a/docs/nczarr.md +++ b/docs/nczarr.md @@ -46,7 +46,7 @@ filters](./md_filters.html "filters") for details. Briefly, the data model supported by NCZarr is netcdf-4 minus the user-defined types. However, a restricted form of String type -is supported (see Appendix H). +is supported (see Appendix E). As with netcdf-4 chunking is supported. Filters and compression are also [supported](./md_filters.html "filters"). @@ -90,7 +90,7 @@ If NCZarr support is enabled, then support for datasets stored as files in a dir However, several addition storage mechanisms are available if additional libraries are installed. 1. Zip format -- if _libzip_ is installed, then it is possible to directly read and write datasets stored in zip files. -2. If the AWS C++ SDK is installed, and _libcurl_ is installed, then it is possible to directly read and write datasets stored in the Amazon S3 cloud storage. +2. If one of the supported AWS SDKs is installed, then it is possible to directly read and write datasets stored in the Amazon S3 cloud storage. # Accessing Data Using the NCZarr Prototocol {#nczarr_accessing_data} @@ -101,29 +101,14 @@ In this case, it is indicated by the URL path. ## URL Format The URL is the usual format. ```` -scheme:://host:port/path?query#fragment format +protocol:://host:port/path?query#fragment ```` -There are some details that are important. -- Scheme: this should be _https_ or _s3_,or _file_. - The _s3_ scheme is equivalent - to "https" plus setting "mode=nczarr,s3" (see below). - Specifying "file" is mostly used for testing, but is used to support - directory tree or zipfile format storage. -- Host: Amazon S3 defines three forms: _Virtual_, _Path_, and _S3_ - + _Virtual_: the host includes the bucket name as in - __bucket.s3.<region>.amazonaws.com__ - + _Path_: the host does not include the bucket name, but - rather the bucket name is the first segment of the path. - For example __s3.<region>.amazonaws.com/bucket__ - + _S3_: the protocol is "s3:" and if the host is a single name, - then it is interpreted as the bucket. The region is determined - using the algorithm in Appendix E. - + _Other_: It is possible to use other non-Amazon cloud storage, but - that is cloud library dependent. -- Query: currently not used. -- Fragment: the fragment is of the form _key=value&key=value&..._. - Depending on the key, the _value_ part may be left out and some - default value will be used. +See the document "quickstart_paths" for details about +using URLs. + +There are, however, some details that are important. +- Protocol: this should be _https_ or _s3_,or _file_. + The _s3_ scheme is equivalent to "https" plus setting "mode=nczarr,s3" (see below). Specifying "file" is mostly used for testing, but is used to support directory tree or zipfile format storage. ## Client Parameters @@ -275,7 +260,7 @@ Another storage format uses a file system tree of directories and files (_mode=n A third storage format uses a zip file (_mode=nczarr,zip_). The latter two are used mostly for debugging and testing. However, the _file_ and _zip_ formats are important because they are intended to match corresponding storage formats used by the Python Zarr implementation. -Hence it should serve to provide interoperability between NCZarr and the Python Zarr, although this interoperability has not been tested. +Hence it should serve to provide interoperability between NCZarr and the Python Zarr, although this interoperability has had only limited testing. Examples of the typical URL form for _file_ and _zip_ are as follows. ```` @@ -284,7 +269,7 @@ file:///xxx/yyy/testdata.zip#mode=nczarr,zip ```` Note that the extension (e.g. ".file" in "testdata.file") -is arbitraty, so this would be equally acceptable. +is arbitrary, so this would be equally acceptable. ```` file:///xxx/yyy/testdata.anyext#mode=nczarr,file ```` @@ -324,24 +309,11 @@ A good value of _n_ is 9. In order to use the _zip_ storage format, the libzip [3] library must be installed. Note that this is different from zlib. -# Amazon S3 Storage {#nczarr_s3} - -The Amazon AWS S3 storage driver currently uses the Amazon AWS S3 Software Development Kit for C++ (aws-s3-sdk-cpp). -In order to use it, the client must provide some configuration information. -Specifically, the ''~/.aws/config'' file should contain something like this. - -``` -[default] -output = json -aws_access_key_id=XXXX... -aws_secret_access_key=YYYY... -``` -See Appendix E for additional information. - ## Addressing Style The notion of "addressing style" may need some expansion. -Amazon S3 accepts two forms for specifying the endpoint for accessing the data. +Amazon S3 accepts two forms for specifying the endpoint for accessing the data +(see the document "quickstart_path). 1. Virtual -- the virtual addressing style places the bucket in the host part of a URL. For example: @@ -351,7 +323,7 @@ https://.s2.<region>.amazonaws.com/ 2. Path -- the path addressing style places the bucket in at the front of the path part of a URL. For example: ``` -https://s2.<region>.amazonaws.com// +https://s3.<region>.amazonaws.com// ``` The NCZarr code will accept either form, although internally, it is standardized on path style. @@ -422,10 +394,10 @@ Specifically it contains the following keys: ## Translation {#nczarr_translation} -With some constraints, it is possible for an nczarr library to read Zarr and for a zarr library to read the nczarr format. -The latter case, zarr reading nczarr is possible if the zarr library is willing to ignore keys whose name it does not recognize; specifically anything beginning with _\_NCZARR\__. +With some constraints, it is possible for an nczarr library to read the pure Zarr format and for a zarr library to read the nczarr format. +The latter case, zarr reading nczarr is possible if the zarr library is willing to ignore keys whose name it does not recognize; specifically anything beginning with _\_nczarr\__. -The former case, nczarr reading zarr is also possible if the nczarr can simulate or infer the contents of the missing _\_NCZARR\_XXX_ objects. +The former case, nczarr reading zarr is also possible if the nczarr can simulate or infer the contents of the missing _\_nczarr\_xxx_ objects. As a rule this can be done as follows. 1. _\_nczarr_group\__ -- The list of contained variables and sub-groups can be computed using the search API to list the keys "contained" in the key for a group. The search looks for occurrences of _.zgroup_, _.zattr_, _.zarray_ to infer the keys for the contained groups, attribute sets, and arrays (variables). @@ -483,11 +455,10 @@ Here are a couple of examples using the _ncgen_ and _ncdump_ utilities. ``` ncgen -4 -lb -o "s3://datasetbucket/rootkey#mode=nczarr,awsprofile=unidata" dataset.cdl ``` - Note that the URLis internally translated to this + Note that the URL is internally translated to this ```` https://s2.<region>.amazonaws.com/datasetbucket/rootkey#mode=nczarr,awsprofile=unidata" dataset.cdl ```` - The region is from the algorithm described in Appendix E1. # References {#nczarr_bib} @@ -508,177 +479,56 @@ collections — High-performance dataset datatypes](https://docs.python.org/2/li # Appendix A. Building NCZarr Support {#nczarr_build} Currently the following build cases are known to work. +Note that this does not include S3 support. +A separate tabulation of S3 support is in the document cloud.md. -
Operating SystemBuild SystemNCZarrS3 Support -
Linux Automake yes yes -
Linux CMake yes yes -
Cygwin Automake yes no -
OSX Automake unknown unknown -
OSX CMake unknown unknown -
Visual Studio CMake yes tests fail +
Operating SystemBuild SystemNCZarr +
Linux Automake yes +
Linux CMake yes +
Cygwin Automake yes +
Cygwin CMake unknown +
OSX Automake unknown +
OSX CMake unknown +
Visual Studio CMake yes
-Note: S3 support includes both compiling the S3 support code as well as running the S3 tests. - ## Automake -There are several options relevant to NCZarr support and to Amazon S3 support. -These are as follows. - -1. _--disable-nczarr_ -- disable the NCZarr support. -If disabled, then all of the following options are disabled or irrelevant. -2. _--enable-nczarr-s3_ -- Enable NCZarr S3 support. -3. _--enable-nczarr-s3-tests_ -- the NCZarr S3 tests are currently only usable by Unidata personnel, so they are disabled by default. - -__A note about using S3 with Automake.__ -If S3 support is desired, and using Automake, then LDFLAGS must be properly set, namely to this. -```` -LDFLAGS="$LDFLAGS -L/usr/local/lib -laws-cpp-sdk-s3" -```` -The above assumes that these libraries were installed in '/usr/local/lib', so the above requires modification if they were installed elsewhere. - -Note also that if S3 support is enabled, then you need to have a C++ compiler installed because part of the S3 support code is written in C++. +The relevant ./configure options are as follows. -## CMake {#nczarr_cmake} +1. *--disable-nczarr* -- disable the NCZarr support. -The necessary CMake flags are as follows (with defaults) +## CMake -1. -DENABLE_NCZARR=off -- equivalent to the Automake _--disable-nczarr_ option. -2. -DENABLE_NCZARR_S3=off -- equivalent to the Automake _--enable-nczarr-s3_ option. -3. -DENABLE_NCZARR_S3_TESTS=off -- equivalent to the Automake _--enable-nczarr-s3-tests_ option. +The relevant CMake flags are as follows. -Note that unlike Automake, CMake can properly locate C++ libraries, so it should not be necessary to specify _-laws-cpp-sdk-s3_ assuming that the aws s3 libraries are installed in the default location. -For CMake with Visual Studio, the default location is here: - -```` -C:/Program Files (x86)/aws-cpp-sdk-all -```` - -It is possible to install the sdk library in another location. -In this case, one must add the following flag to the cmake command. -```` -cmake ... -DAWSSDK_DIR=\ -```` -where "awssdkdir" is the path to the sdk installation. -For example, this might be as follows. -```` -cmake ... -DAWSSDK_DIR="c:\tools\aws-cpp-sdk-all" -```` -This can be useful if blanks in path names cause problems in your build environment. - -## Testing S3 Support {#nczarr_testing_S3_support} +1. *-DENABLE_NCZARR=off* -- equivalent to the Automake *--disable-nczarr* option. +## Testing NCZarr S3 Support {#nczarr_testing_S3_support} The relevant tests for S3 support are in the _nczarr_test_ directory. Currently, by default, testing of S3 with NCZarr is supported only for Unidata members of the NetCDF Development Group. -This is because it uses a Unidata-specific bucket is inaccessible to the general user. - -# Appendix B. Building aws-sdk-cpp {#nczarr_s3sdk} - -In order to use the S3 storage driver, it is necessary to install the Amazon [aws-sdk-cpp library](https://github.com/aws/aws-sdk-cpp.git). - -Building this package from scratch has proven to be a formidable task. -This appears to be due to dependencies on very specific versions of, -for example, openssl. - -## *\*nix\** Build - -For linux, the following context works. Of course your mileage may vary. -* OS: ubuntu 21 -* aws-sdk-cpp version 1.9.96 (or later?) -* Required installed libraries: openssl, libcurl, cmake, ninja (ninja-build in apt) - -### AWS-SDK-CPP Build Recipe - -```` -git clone --recurse-submodules https://www.github.com/aws/aws-sdk-cpp.git -cd aws-sdk-cpp -mkdir build -cd build -PREFIX=/usr/local -FLAGS="-DCMAKE_INSTALL_PREFIX=${PREFIX} \ - -DCMAKE_INSTALL_LIBDIR=lib \ - -DCMAKE_MODULE_PATH=${PREFIX}/lib/cmake \ - -DCMAKE_POLICY_DEFAULT_CMP0075=NEW \ - -DBUILD_ONLY=s3 \ - -DENABLE_UNITY_BUILD=ON \ - -DENABLE_TESTING=OFF \ - -DCMAKE_BUILD_TYPE=$CFG \ - -DSIMPLE_INSTALL=ON \ - _DMY_ASSEMBLER_IS_TOO_OLD_FOR_AVX=ON" -cmake -GNinja $FLAGS .. -ninja all -sudo ninja install -cd .. -cd .. -```` +This is because it uses a Unidata-specific bucket that is inaccessible to the general user. ### NetCDF Build In order to build netcdf-c with S3 sdk support, the following options must be specified for ./configure. ```` ---enable-nczarr-s3 +--enable-s3 ```` If you have access to the Unidata bucket on Amazon, then you can also test S3 support with this option. ```` ---enable-nczarr-s3-tests -```` - -## Windows build -It is possible to build and install aws-sdk-cpp. It is also possible -to build netcdf-c using cmake. Unfortunately, testing currently fails. - -For Windows, the following context work. Of course your mileage may vary. -* OS: Windows 10 64-bit with Visual Studio community edition 2019. -* aws-sdk-cpp version 1.9.96 (or later?) -* Required installed libraries: openssl, libcurl, cmake - -### AWS-SDK-CPP Build Recipe - -This command-line build assumes one is using Cygwin or Mingw to provide -tools such as bash. - -```` -git clone --recurse-submodules https://www.github.com/aws/aws-sdk-cpp -pushd aws-sdk-cpp -mkdir build -cd build -CFG="Release" -PREFIX="c:/tools/aws-sdk-cpp" - -FLAGS="-DCMAKE_INSTALL_PREFIX=${PREFIX} \ - -DCMAKE_INSTALL_LIBDIR=lib" \ - -DCMAKE_MODULE_PATH=${PREFIX}/cmake \ - -DCMAKE_POLICY_DEFAULT_CMP0075=NEW \ - -DBUILD_ONLY=s3 \ - -DENABLE_UNITY_BUILD=ON \ - -DCMAKE_BUILD_TYPE=$CFG \ - -DSIMPLE_INSTALL=ON" - -rm -fr build -mkdir -p build -cd build -cmake -DCMAKE_BUILD_TYPE=${CFG} $FLAGS .. -cmake --build . --config ${CFG} -cmake --install . --config ${CFG} -cd .. -popd +--with-s3-testing=yes ```` -Notice that the sdk is being installed in the directory "c:\tools\aws-sdk-cpp" -rather than the default location "c:\Program Files (x86)/aws-sdk-cpp-all" -This is because when using a command line, an install path that contains -blanks may not work. ### NetCDF CMake Build -Enabling S3 support is controlled by these two cmake options: +Enabling S3 support is controlled by this cmake option: ```` --DENABLE_NCZARR_S3=ON --DENABLE_NCZARR_S3_TESTS=OFF +-DENABLE_S3=ON ```` - However, to find the aws sdk libraries, the following environment variables must be set: ```` @@ -691,8 +541,7 @@ Then the following options must be specified for cmake. -DAWSSDK_ROOT_DIR=${AWSSDK_ROOT_DIR} -DAWSSDK_DIR=${AWSSDK_ROOT_DIR}/lib/cmake/AWSSDK" ```` - -# Appendix C. Amazon S3 Imposed Limits {#nczarr_s3limits} +# Appendix B. Amazon S3 Imposed Limits {#nczarr_s3limits} The Amazon S3 cloud storage imposes some significant limits that are inherited by NCZarr (and Zarr also, for that matter). @@ -702,89 +551,7 @@ Some of the relevant limits are as follows: Note that the limit is defined in terms of bytes and not (Unicode) characters. This affects the depth to which groups can be nested because the key encodes the full path name of a group. -# Appendix D. Alternative Mechanisms for Accessing Remote Datasets {#nczarr_altremote} - -The NetCDF-C library contains an alternate mechanism for accessing traditional netcdf-4 files stored in Amazon S3: The byte-range mechanism. -The idea is to treat the remote data as if it was a big file. -This remote "file" can be randomly accessed using the HTTP Byte-Range header. - -In the Amazon S3 context, a copy of a dataset, a netcdf-3 or netdf-4 file, is uploaded into a single object in some bucket. -Then using the key to this object, it is possible to tell the netcdf-c library to treat the object as a remote file and to use the HTTP Byte-Range protocol to access the contents of the object. -The dataset object is referenced using a URL with the trailing fragment containing the string ````#mode=bytes````. - -An examination of the test program _nc_test/test_byterange.sh_ shows simple examples using the _ncdump_ program. -One such test is specified as follows: -```` -https://s3.us-east-1.amazonaws.com/noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes -```` -Note that for S3 access, it is expected that the URL is in what is called "path" format where the bucket, _noaa-goes16_ in this case, is part of the URL path instead of the host. - -The _#mode=bytes_ mechanism generalizes to work with most servers that support byte-range access. - -Specifically, Thredds servers support such access using the HttpServer access method as can be seen from this URL taken from the above test program. -```` -https://thredds-test.unidata.ucar.edu/thredds/fileServer/irma/metar/files/METAR_20170910_0000.nc#bytes -```` - -# Appendix E. AWS Selection Algorithms. {#nczarr_awsselect} - -If byterange support is enabled, the netcdf-c library will parse the files -```` -${HOME}/.aws/config -and -${HOME}/.aws/credentials -```` -to extract profile names plus a list -of key=value pairs. This example is typical. -```` -[default] - aws_access_key_id=XXXXXXXXXXXXXXXXXXXX - aws_secret_access_key=YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY - aws_region=ZZZZZZZZZ -```` -The keys in the profile will be used to set various parameters in the library - -## Profile Selection - -The algorithm for choosing the active profile to use is as follows: - -1. If the "aws.profile" fragment flag is defined in the URL, then it is used. For example, see this URL. -```` -https://...#mode=nczarr,s3&aws.profile=xxx -```` -2. If the "AWS.PROFILE" entry in the .rc file (i.e. .netrc or .dodsrc) is set, then it is used. -3. Otherwise the profile "default" is used. - -The profile named "none" is a special profile that the netcdf-c library automatically defines. -It should not be defined anywhere else. It signals to the library that no credentialas are to used. -It is equivalent to the "--no-sign-request" option in the AWS CLI. -Also, it must be explicitly specified by name. Otherwise "default" will be used. - -## Region Selection - -If the specified URL is of the form -```` -s3:///key -```` -Then this is rebuilt to this form: -```` -s3://s2.<region>.amazonaws.com>/key -```` -However this requires figuring out the region to use. -The algorithm for picking an region is as follows. - -1. If the "aws.region" fragment flag is defined in the URL, then it is used. -2. The active profile is searched for the "aws_region" key. -3. If the "AWS.REGION" entry in the .rc file (i.e. .netrc or .dodsrc) is set, then it is used. -4. Otherwise use "us-east-1" region. - -## Authorization Selection - -Picking an access-key/secret-key pair is always determined -by the current active profile. To choose to not use keys -requires that the active profile must be "none". - -# Appendix F. NCZarr Version 1 Meta-Data Representation. {#nczarr_version1} +# Appendix C. NCZarr Version 1 Meta-Data Representation. {#nczarr_version1} In NCZarr Version 1, the NCZarr specific metadata was represented using new objects rather than as keys in existing Zarr objects. Due to conflicts with the Zarr specification, that format is deprecated in favor of the one described above. @@ -799,7 +566,7 @@ The content of these objects is the same as the contents of the corresponding ke * ''.nczarray <=> ''_nczarr_array_'' * ''.nczattr <=> ''_nczarr_attr_'' -# Appendix G. JSON Attribute Convention. {#nczarr_json} +# Appendix D. JSON Attribute Convention. {#nczarr_json} The Zarr V2 specification is somewhat vague on what is a legal value for an attribute. The examples all show one of two cases: @@ -887,7 +654,7 @@ actions "read-write-read" is equivalent to a single "read" and "write-read-write The "almost" caveat is necessary because (1) whitespace may be added or lost during the sequence of operations, and (2) numeric precision may change. -# Appendix H. Support for string types +# Appendix E. Support for string types Zarr supports a string type, but it is restricted to fixed size strings. NCZarr also supports such strings, @@ -936,12 +703,16 @@ and the type that signals NC_CHAR (in NCZarr) would be handled by Zarr as a string of length 1. # Change Log {#nczarr_changelog} +[Note: minor text changes are not included.] Note, this log was only started as of 8/11/2022 and is not intended to be a detailed chronology. Rather, it provides highlights that will be of interest to NCZarr users. In order to see exact changes, It is necessary to use the 'git diff' command. +## 3/10/2023 +1. Move most of the S3 text to the cloud.md document. + ## 8/29/2022 1. Zarr fixed-size string types are now supported. @@ -951,11 +722,11 @@ It is necessary to use the 'git diff' command. accepted for back compatibility. 2. The legal values of an attribute has been extended to -include arbitrary JSON expressions; see Appendix G for more details. +include arbitrary JSON expressions; see Appendix D for more details. # Point of Contact {#nczarr_poc} __Author__: Dennis Heimbigner
__Email__: dmh at ucar dot edu
__Initial Version__: 4/10/2020
-__Last Revised__: 8/27/2022 +__Last Revised__: 3/8/2023 diff --git a/docs/quickstart_paths.md b/docs/quickstart_paths.md new file mode 100644 index 0000000000..5f1f4a8aea --- /dev/null +++ b/docs/quickstart_paths.md @@ -0,0 +1,108 @@ +Appendix D.2. Specifying Paths for NetCDF-C {#nc_paths_quickstart} +============================== + +A key concept in netcdf-c is the notion of a "path". +A path specifies some dataset that is of interest to a user. +It is the primary argument to the *nc_open* and *nc_create* +functions in the NetCDF-C API, as defined by the file netcdf.h. +It is also the primary option for the NetCDF-C utilities: +*ncdump*, *nccopy*, and *ncgen*. +Hence understanding what kind of paths are acceptable is important +for using the NetCDF-C library. + +## Classification of Paths {#nc_paths_kinds} + +Basically, there are two kinds of paths: +1. File system paths, and +2. Uniform Resource Locator (URL) paths. + +### File System Paths + +The most common form of path accepted by the NetCDF-C library is a file system path. +Every user of some computer operating system is familiar with the idea of a file system path. + +Each operating system has some special quirks when specifying file system paths. +Here are some example paths for various version of Linux/Unix. +* / +* /a/b/c/d + +As a rule, Linux/Unix has a single root path typically indicated by "/", +and "/" is also used as the file separator. + +For Windows, some example paths would be as follows: +* c:\\ +* d:\\a\\b\\c + +Windows has a notion of a drive ("d:") and each drive serves as the root +of its own file system. Windows uses "\\" as its file separator, although +many programs also accept "/". + +## Uniform Resource Locator (URL) Paths + +The NetCDF-C library can access datasets that reside on remote computers, +Hence NetCDF-C now also accepts URLs to specify those remote datasets. + +The general form of a URL is as follows: +```` +://:@:/?# +```` +* \ - specifies the format of the messages between the local and remote computers. Some examples used by NetCDF-C are *http*, *https*, *s3*, *dap*, *dap4*, or *file*. +* \ - Pass authorization credentials to the remote computer. +* \:\ - The specific remote computer to access, such as *thredds.ucar.edu:8080*. +* \ - A specification of file-path-like string that indicates some specific resource on the remote computer. +* \ - A sequence of (key=value) pairs, separated by "&", and providing special information to the remote computer. +* \ - A sequence of (key=value) pairs, separated by "&", and providing special information to the local computer. + +Most parts of a URL are optional. As a rule, the protocol is always required, +and either the host is required or, if the protocol is "file", then the path is required. +The query and fragment are optional. + +### Examples of URL Paths for NetCDF-C +* https://thredds.ucar.edu/catalog + +## Addendum A. Amazon S3 Specific URLS {#nc_paths_s3_urls} +A URL path is required for accessing datasets on the Amazon S3 storage cloud. +Unfortunately S3 URLs are complicated. +It has the following features: +* Protocol: _https_ or _s3_. The _s3_ scheme is equivalent to "https" plus setting various tags in the query and/or fragment part of the URL. +* Host: Amazon S3 defines three forms: _Virtual_, _Path_, and _S3_ + + _Virtual_: the host includes the bucket name as in __bucket.s3.<region>.amazonaws.com__ or __bucket.s3.amazonaws.com__ + + _Path_: the host does not include the bucket name, but rather the bucket name is the first segment of the path. For example __s3.<region>.amazonaws.com/bucket__ or __s3.amazonaws.com/bucket__ + + _S3_: the protocol is "s3:" and if the host is a single name, then it is interpreted as the bucket. The region is determined using an algorithm defined in the nczarr documentation. + + _Other_: It is possible to use other non-Amazon cloud storage, but that is cloud library dependent. +* Query: currently not used. +* Fragment: the fragment is of the form _key=value&key=value&..._. Depending on the key, the _value_ part may be left out and some default value will be used. The exact set of possible keys is defined in the nczarr documentation. + +## Addendum B. Known Fragment Keys {#nc_paths_frag_keys} + +The fragment part of a URL is used to pass information deep into +the netcdf-c library to control its actions. +This appendix list known keys, although it may be somewhat out-of-date. + +The current set of keys used in the netcdf-c library is as follows. +* _mode_ -- A special key that is used to provide single values for controlling the netcdf-c library. It consists of a comma separated sequence of values +primarily used to control the file format. +The *mode* key supports the following values + - _dap2_ -- Specifies that the URL accesses a resource using the DAP2 protocol + - _dap4_ -- Specifies that the URL accesses a resource using the DAP4 protocol + - _netcdf-3_ -- Specifies that a file is a netcdf-classic file + - _classic_ -- Alias for _netcdf-3_ + - _netcdf-4_ -- Specifies that a file is a netcdf-enhanced file + - _enhanced_ -- Alias for _netcdf-4_ + - _udf0_ -- Specifies that the file format is defined by a User Defined format + - _udf1_ -- Specifies that the file format is defined by a User Defined format + - _nczarr_ -- Specifies that the file is in NCZarr format + - _zarr_ -- Specifies that the file is in Zarr format + - _xarray_ --Specifies that the file is in Zarr format and uses the XARRAY convention + - _noxarray_ --Specifies that the file is in Zarr format and does not use the XARRAY convention + - _s3_ --Specifies that the file is remote and is stored on the Amazon S3 cloud + - _file_ --Specifies that the file is an NCZarr/Zarr file stored as a file tree + - _zip_ --Specifies that the file is an NCZarr/Zarr file stored as a zip file + - _bytes_ -- Specifies that the file is remote and is to be read using byte-range support + in NCZarr format +* _dap2_ -- equivalent to "mode=dap2" +* _dap4_ -- equivalent to "mode=dap4" +* _bytes_ -- equivalent to "mode=bytes" +* _log_ -- turn on logging for the duration of the data request +* _show=fetch_ -- log curl fetch commands + diff --git a/include/hdf5internal.h b/include/hdf5internal.h index 1a15a172ba..58d076567b 100644 --- a/include/hdf5internal.h +++ b/include/hdf5internal.h @@ -60,12 +60,12 @@ struct NCauth; /** Struct to hold HDF5-specific info for the file. */ typedef struct NC_HDF5_FILE_INFO { hid_t hdfid; + NCURI* uri; /* Parse of the incoming path, if url */ #if defined(ENABLE_BYTERANGE) int byterange; - NCURI* uri; /* Parse of the incoming path, if url */ -#if defined(ENABLE_HDF5_ROS3) || defined(ENABLE_S3_SDK) - struct NCauth* auth; #endif +#ifdef ENABLE_S3 + struct NCauth* auth; #endif } NC_HDF5_FILE_INFO_T; diff --git a/include/nc4internal.h b/include/nc4internal.h index 86cb223fe1..d750421964 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -168,10 +168,6 @@ typedef struct NC_ATT_INFO nc_type nc_typeid; /**< NetCDF type of attribute's data. */ void *format_att_info; /**< Pointer to format-specific att info. */ void *data; /**< The attribute data. */ -#ifdef SEPDATA - nc_vlen_t *vldata; /**< VLEN data (only used for vlen types). */ - char **stdata; /**< String data (only for string type). */ -#endif } NC_ATT_INFO_T; /** This is a struct to handle the var metadata. */ diff --git a/include/ncbytes.h b/include/ncbytes.h index 228d8b539e..b00d428c16 100644 --- a/include/ncbytes.h +++ b/include/ncbytes.h @@ -57,6 +57,7 @@ EXTERNL int ncbytessetcontents(NCbytes*, void*, unsigned long); #define ncbytesextend(bb,len) ncbytessetalloc((bb),(len)+(bb->alloc)) #define ncbytesclear(bb) ((bb)!=NULL?(bb)->length=0:0) #define ncbytesavail(bb,n) ((bb)!=NULL?((bb)->alloc - (bb)->length) >= (n):0) +#define ncbytesextendible(bb) ((bb)!=NULL?((bb)->nonextendible?0:1):0) #if defined(_CPLUSPLUS_) || defined(__CPLUSPLUS__) || defined(__CPLUSPLUS) } diff --git a/include/ncconfigure.h b/include/ncconfigure.h index 975ea5f39b..3dccfe9eab 100644 --- a/include/ncconfigure.h +++ b/include/ncconfigure.h @@ -50,10 +50,9 @@ extern "C" { #endif /* WARNING: in some systems, these functions may be defined as macros, so check */ -#ifndef strdup #ifndef HAVE_STRDUP +#ifndef strdup char* strdup(const char*); -#define HAVE_STRDUP #endif #endif @@ -71,7 +70,7 @@ int snprintf(char*, size_t, const char*, ...); #ifndef HAVE_STRCASECMP #ifndef strcasecmp -extern int strcasecmp(const char*, const char*); +int strcasecmp(const char*, const char*); #endif #endif @@ -99,6 +98,9 @@ unsigned long long int strtoull(const char*, char**, int); #define strlcat(d,s,n) strcat_s((d),(n),(s)) #endif +#ifndef HAVE_STRLCPY +#define strlcpy(d,s,n) strcpy_s((d),(n),(s)) +#endif #ifndef __MINGW32__ #ifndef strcasecmp @@ -119,77 +121,9 @@ unsigned long long int strtoull(const char*, char**, int); #endif /*_WIN32*/ -/* handle null arguments */ #ifndef nulldup -#ifndef HAVE_STRDUP -/** Copy s if not NULL. - * - * Implementation in terms of strdup in - * - * @li include/ncconfigure.h - * @li include/netcdf_json.h - * @li libdap4/ncd4.h - * @li libdispatch/dfile.c - * @li libdispatch/dinfermodel.c - * @li libdispatch/drc.c - * @li libdispatch/dutil.c - * @li libdispatch/nc.c - * @li libdispatch/ncjson.c - * @li libdispatch/ncurl.c - * @li libncxml/ncxml_ezxml.c - * @li ncxml_tinyxml2.cpp - * @li libncxml/ncxml_xml2.c - * @li libnczarr/zsync.c - * @li ncdump/ocprint.c - * @li ncgen/cvt.c - * @li ncgen/ncgen.h - * @li ncgen3/ncgen.h - * @li nczarr_test/test_nczarr_utils.h - * @li oc2/ocinternal.h - * - * Declarations as extern: - * - * @li include/ncconfigure.h - * - * I'd like it to be - * static inline const char *nulldup(const char *const s); - * but that's not what's in ncconfigure.h - * - * @param s the string to duplicate - * @pre s is either NULL or a NULL-terminated string - * - * @returns NULL or the duplicated string (caller owns the new - * pointer) - * - * @throws ENOMEM if out of memory - - * - * @post returns NULL if s is NULL, or a new pointer to a - * freshly-allocated copy of s - */ -static char *nulldup(const char* s) { - if (s != NULL) { - ssize_t result_length = strlen(s) + 1; - char *result = malloc(result_length); - if (result == NULL) { -#ifdef ENOMEM - /* C++11, POSIX? */ - errno = ENOMEM; -#else /* ENOMEM */ - errno = 1; -#endif /* ENOMEM */ - return NULL; - } - strncpy(result, s, result_length); - return result; - } else { - return NULL; - } -} -#else /* HAVE_STRDUP */ #define nulldup(s) ((s)==NULL?NULL:strdup(s)) -#endif /* HAVE_STRDUP */ -#endif /* nulldup */ +#endif #ifndef nulllen #define nulllen(s) ((s)==NULL?0:strlen(s)) @@ -226,6 +160,7 @@ typedef unsigned long long uint64_t; #ifndef _WIN32 #ifndef HAVE_UINTPTR_T +#ifndef uintptr_t #if SIZEOF_VOIDP == 8 #define uintptr_t unsigned long #else @@ -233,11 +168,16 @@ typedef unsigned long long uint64_t; #endif #endif #endif +#endif #ifndef HAVE_SIZE64_T typedef unsigned long long size64_t; #endif +#ifndef HAVE_SSIZE64_T +typedef long long ssize64_t; +#endif + #ifndef HAVE_PTRDIFF_T typedef long ptrdiff_t; #endif diff --git a/include/nchttp.h b/include/nchttp.h index 82f7150654..872534dcb5 100644 --- a/include/nchttp.h +++ b/include/nchttp.h @@ -12,38 +12,51 @@ typedef enum HTTPMETHOD { HTTPNONE=0, HTTPGET=1, HTTPPUT=2, HTTPPOST=3, HTTPHEAD=4, HTTPDELETE=5 } HTTPMETHOD; -struct CURL; /* Forward */ +/* Forward */ +struct CURL; +struct NCS3INFO; +struct NCURI; + +/* Common state For S3 vs Simple Curl */ +typedef enum NC_HTTPFORMAT {HTTPS3=1, HTTPCURL=2} NC_HTTPFORMAT; typedef struct NC_HTTP_STATE { - struct CURL* curl; - long httpcode; - struct Response { - NClist* headset; /* which headers to capture */ - NClist* headers; /* Set of captured headers */ - NCbytes* buf; /* response content; call owns; do not free */ - } response; - struct Request { - HTTPMETHOD method; - size_t payloadsize; - void* payload; /* caller owns; do not free */ - size_t payloadpos; - NClist* headers; - } request; - char errbuf[1024]; /* assert(CURL_ERROR_SIZE <= 1024) */ + enum NC_HTTPFORMAT format; /* Discriminator */ + char* path; /* original url */ + struct NCURI* url; /* parsed url */ + long httpcode; + char* errmsg; /* do not free if format is HTTPCURL */ +#ifdef ENABLE_S3 + struct NC_HTTP_S3 { + void* s3client; + struct NCS3INFO* info; + } s3; +#endif + struct NC_HTTP_CURL { + struct CURL* curl; + char errbuf[2048]; /* assert(CURL_ERROR_SIZE <= 2048) */ + struct Response { + NClist* headset; /* which headers to capture */ + NClist* headers; /* Set of captured headers */ + NCbytes* buf; /* response content; call owns; do not free */ + } response; + struct Request { + HTTPMETHOD method; + size_t payloadsize; + void* payload; /* caller owns; do not free */ + size_t payloadpos; + NClist* headers; + } request; + } curl; } NC_HTTP_STATE; -extern int nc_http_init(NC_HTTP_STATE** state); -extern int nc_http_init_verbose(NC_HTTP_STATE** state, int verbose); -extern int nc_http_size(NC_HTTP_STATE* state, const char* url, long long* sizep); -extern int nc_http_read(NC_HTTP_STATE* state, const char* url, size64_t start, size64_t count, NCbytes* buf); -extern int nc_http_write(NC_HTTP_STATE* state, const char* url, NCbytes* payload); +/* External API */ +extern int nc_http_open(const char* url, NC_HTTP_STATE** statep); +extern int nc_http_open_verbose(const char* url, int verbose, NC_HTTP_STATE** statep); +extern int nc_http_size(NC_HTTP_STATE* state, long long* sizep); +extern int nc_http_read(NC_HTTP_STATE* state, size64_t start, size64_t count, NCbytes* buf); +extern int nc_http_write(NC_HTTP_STATE* state, NCbytes* payload); extern int nc_http_close(NC_HTTP_STATE* state); extern int nc_http_reset(NC_HTTP_STATE* state); -extern int nc_http_set_method(NC_HTTP_STATE* state, HTTPMETHOD method); -extern int nc_http_set_response(NC_HTTP_STATE* state, NCbytes* buf); -extern int nc_http_set_payload(NC_HTTP_STATE* state, size_t len, void* payload); -extern int nc_http_response_headset(NC_HTTP_STATE* state, const NClist* headers); /* Set of headers to capture */ -extern int nc_http_response_headers(NC_HTTP_STATE* state, NClist** headersp); /* set of captured headers */ -extern int nc_http_request_setheaders(NC_HTTP_STATE* state, const NClist* headers); /* set of extra request headers */ #endif /*NCHTTP_H*/ diff --git a/include/nclist.h b/include/nclist.h index bdbefe8a91..bad224ac2d 100644 --- a/include/nclist.h +++ b/include/nclist.h @@ -11,7 +11,7 @@ extern "C" { #endif -EXTERNL int nclistnull(void*); +EXTERNL int nclistisnull(void*); typedef struct NClist { size_t alloc; @@ -55,8 +55,12 @@ EXTERNL int nclistunique(NClist*); /* Create a clone of a list; if deep, then assume it is a list of strings */ EXTERNL NClist* nclistclone(const NClist*, int deep); +/* Extract the contents of a list, leaving list empty */ EXTERNL void* nclistextract(NClist*); +/* Append an uncounted NULL to the end of the list */ +EXTERNL int nclistnull(NClist*); + /* Following are always "in-lined"*/ #define nclistclear(l) nclistsetlength((l),0) #define nclistextend(l,len) nclistsetalloc((l),(len)+(l->alloc)) diff --git a/include/nclog.h b/include/nclog.h index 76e0f82de5..e7146c162a 100644 --- a/include/nclog.h +++ b/include/nclog.h @@ -10,9 +10,7 @@ #include #include "ncexternl.h" -#ifndef NCCATCH -#undef NCCATCH -#endif +#define NCCATCH #define NCENVLOGGING "NCLOGGING" #define NCENVTRACING "NCTRACING" @@ -23,6 +21,9 @@ #define NCLOGERR 2 #define NCLOGDBG 3 +/* Support ptr valued arguments that are used to store results */ +#define PTRVAL(t,p,d) ((t)((p) == NULL ? (d) : *(p))) + #if defined(_CPLUSPLUS_) || defined(__CPLUSPLUS__) extern "C" { #endif diff --git a/include/ncrc.h b/include/ncrc.h index 81d62fdffd..b3bb8c512d 100644 --- a/include/ncrc.h +++ b/include/ncrc.h @@ -53,14 +53,6 @@ typedef struct NCRCinfo { NClist* s3profiles; /* NClist */ } NCRCinfo; -typedef struct NCS3INFO { - char* host; /* non-null if other*/ - char* region; /* region */ - char* bucket; /* bucket name */ - char* rootkey; - char* profile; -} NCS3INFO; - #if defined(__cplusplus) extern "C" { #endif @@ -98,14 +90,11 @@ EXTERNL int NC_join(struct NClist* segments, char** pathp); /* From ds3util.c */ /* S3 profiles */ -EXTERNL int NC_s3urlrebuild(NCURI* url, NCURI** newurlp, char** bucketp, char** regionp); EXTERNL int NC_getactives3profile(NCURI* uri, const char** profilep); -EXTERNL int NC_getdefaults3region(NCURI* uri, const char** regionp); -EXTERNL int NC_authgets3profile(const char* profile, struct AWSprofile** profilep); EXTERNL int NC_s3profilelookup(const char* profile, const char* key, const char** valuep); -EXTERNL int NC_s3urlprocess(NCURI* url, NCS3INFO* s3); -EXTERNL int NC_s3clear(NCS3INFO* s3); +EXTERNL int NC_authgets3profile(const char* profile, struct AWSprofile** profilep); EXTERNL int NC_iss3(NCURI* uri); +EXTERNL int NC_s3urlrebuild(NCURI* url, char** inoutbucketp, char** inoutregionp, NCURI** newurlp); #if defined(__cplusplus) } diff --git a/include/ncs3sdk.h b/include/ncs3sdk.h index 2200dcb293..2ca05754e9 100644 --- a/include/ncs3sdk.h +++ b/include/ncs3sdk.h @@ -6,6 +6,14 @@ #ifndef NCS3SDK_H #define NCS3SDK_H 1 +typedef struct NCS3INFO { + char* host; /* non-null if other*/ + char* region; /* region */ + char* bucket; /* bucket name */ + char* rootkey; + char* profile; +} NCS3INFO; + #ifdef __cplusplus extern "C" { #endif @@ -15,7 +23,7 @@ EXTERNL int NC_s3sdkfinalize(void); EXTERNL void* NC_s3sdkcreateclient(NCS3INFO* context); EXTERNL int NC_s3sdkbucketexists(void* s3client, const char* bucket, int* existsp, char** errmsgp); EXTERNL int NC_s3sdkbucketcreate(void* s3client, const char* region, const char* bucket, char** errmsgp); -EXTERNL int NC_s3sdkbucketdelete(void* s3client, const char* region, const char* bucket, char** errmsgp); +EXTERNL int NC_s3sdkbucketdelete(void* s3client, NCS3INFO* info, char** errmsgp); EXTERNL int NC_s3sdkinfo(void* client0, const char* bucket, const char* pathkey, unsigned long long* lenp, char** errmsgp); EXTERNL int NC_s3sdkread(void* client0, const char* bucket, const char* pathkey, unsigned long long start, unsigned long long count, void* content, char** errmsgp); EXTERNL int NC_s3sdkwriteobject(void* client0, const char* bucket, const char* pathkey, unsigned long long count, const void* content, char** errmsgp); @@ -23,6 +31,13 @@ EXTERNL int NC_s3sdkclose(void* s3client0, NCS3INFO* info, int deleteit, char** EXTERNL int NC_s3sdkgetkeys(void* s3client0, const char* bucket, const char* prefix, size_t* nkeysp, char*** keysp, char** errmsgp); EXTERNL int NC_s3sdksearch(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp); EXTERNL int NC_s3sdkdeletekey(void* client0, const char* bucket, const char* pathkey, char** errmsgp); +EXTERNL const char* NC_s3dumps3info(NCS3INFO* info); + +/* From ds3util.c */ +EXTERNL int NC_getdefaults3region(NCURI* uri, const char** regionp); +EXTERNL int NC_s3urlprocess(NCURI* url, NCS3INFO* s3); +EXTERNL int NC_s3clear(NCS3INFO* s3); +EXTERNL int NC_s3clone(NCS3INFO* s3, NCS3INFO** news3p); #ifdef __cplusplus } diff --git a/include/ncuri.h b/include/ncuri.h index c4fabd4049..eafb9be97f 100644 --- a/include/ncuri.h +++ b/include/ncuri.h @@ -8,7 +8,9 @@ #include "ncexternl.h" -/* Define flags to control what is included by ncuribuild*/ +/* Define flags to control what is included by ncuribuild; + protocol+host+port always included +*/ #define NCURIPATH 1 #define NCURIPWD 2 #define NCURIQUERY 4 diff --git a/include/netcdf_json.h b/include/netcdf_json.h index 33eabe4606..e063319d5a 100644 --- a/include/netcdf_json.h +++ b/include/netcdf_json.h @@ -613,13 +613,13 @@ testdouble(const char* word) double d; int count = 0; /* Check for Nan and Infinity */ - if(strcasecmp("nan",word)==0) return NCJTHROW(NCJ_OK); - if(strcasecmp("infinity",word)==0) return NCJTHROW(NCJ_OK); - if(strcasecmp("-infinity",word)==0) return NCJTHROW(NCJ_OK); + if(0==(int)strcasecmp("nan",word)) return NCJTHROW(NCJ_OK); + if(0==(int)strcasecmp("infinity",word)) return NCJTHROW(NCJ_OK); + if(0==(int)strcasecmp("-infinity",word)) return NCJTHROW(NCJ_OK); /* Allow the XXXf versions as well */ - if(strcasecmp("nanf",word)==0) return NCJTHROW(NCJ_OK); - if(strcasecmp("infinityf",word)==0) return NCJTHROW(NCJ_OK); - if(strcasecmp("-infinityf",word)==0) return NCJTHROW(NCJ_OK); + if(0==(int)strcasecmp("nanf",word)) return NCJTHROW(NCJ_OK); + if(0==(int)strcasecmp("infinityf",word)) return NCJTHROW(NCJ_OK); + if(0==(int)strcasecmp("-infinityf",word)) return NCJTHROW(NCJ_OK); /* Try to convert to number */ ncvt = sscanf(word,"%lg%n",&d,&count); return NCJTHROW((ncvt == 1 && strlen(word)==count ? NCJ_OK : NCJ_ERR)); @@ -1226,8 +1226,7 @@ NCJtotext(const NCjson* json) char* text = NULL; if(json == NULL) {strcpy(outtext,""); goto done;} (void)NCJunparse(json,0,&text); - outtext[0] = '\0'; - strlcat(outtext,text,sizeof(outtext)); + strncpy(outtext,text,sizeof(outtext)); nullfree(text); done: return outtext; diff --git a/include/netcdf_meta.h.in b/include/netcdf_meta.h.in index c4e613d218..e1f7fbe1aa 100644 --- a/include/netcdf_meta.h.in +++ b/include/netcdf_meta.h.in @@ -60,14 +60,17 @@ #define NC_RELAX_COORD_BOUND 1 /*!< Always allow 0 counts in parallel I/O. */ #define NC_DISPATCH_VERSION @NC_DISPATCH_VERSION@ /*!< Dispatch table version. */ #define NC_HAS_PAR_FILTERS @NC_HAS_PAR_FILTERS@ /* Parallel I/O with filter support. */ -#define NC_HAS_NCZARR @NC_HAS_NCZARR@ /*!< Parallel I/O with filter support. */ #define NC_HAS_MULTIFILTERS @NC_HAS_MULTIFILTERS@ /*!< Nczarr support. */ #define NC_HAS_LOGGING @NC_HAS_LOGGING@ /*!< Logging support. */ #define NC_HAS_QUANTIZE @NC_HAS_QUANTIZE@ /*!< Quantization support. */ #define NC_HAS_ZSTD @NC_HAS_ZSTD@ /*!< Zstd support. */ #define NC_HAS_BENCHMARKS @NC_HAS_BENCHMARKS@ /*!< Benchmarks. */ -#define NC_HAS_S3 @NC_HAS_S3@ /*!< Amazon S3 Support. */ #define NC_HAS_BLOSC @NC_HAS_BLOSC@ /*!< Blosc Support. */ #define NC_HAS_BZ2 @NC_HAS_BZ2@ /*!< bzip2 support */ +#define NC_HAS_S3 @NC_HAS_S3@ /*!< Amazon S3 Support. */ +#define NC_HAS_S3_AWS @NC_HAS_S3_AWS@ /*!< Amazon S3 SDK. */ +#define NC_HAS_S3_INTERNAL @NC_HAS_S3_INTERNAL@ /*!< Internal S3 SDK. */ +#define NC_HAS_HDF5_ROS3 @NC_HAS_HDF5_ROS3@ /*!< HDF5 ROS3 Support. */ +#define NC_HAS_NCZARR @NC_HAS_NCZARR@ /*!< Parallel I/O with filter support. */ #endif diff --git a/lib_flags.am b/lib_flags.am index 3bc6436711..c7ce5bfbd6 100644 --- a/lib_flags.am +++ b/lib_flags.am @@ -18,8 +18,7 @@ if ENABLE_NCZARR AM_CPPFLAGS += -I${top_srcdir}/libnczarr endif - -if ENABLE_S3_SDK +if ENABLE_S3_AWS AM_LDFLAGS += -lstdc++ endif diff --git a/libdispatch/CMakeLists.txt b/libdispatch/CMakeLists.txt index cc9dcd95c8..f518a1adbc 100644 --- a/libdispatch/CMakeLists.txt +++ b/libdispatch/CMakeLists.txt @@ -6,7 +6,7 @@ # See netcdf-c/COPYRIGHT file for more info. SET(libdispatch_SOURCES dcopy.c dfile.c ddim.c datt.c dattinq.c dattput.c dattget.c derror.c dvar.c dvarget.c dvarput.c dvarinq.c ddispatch.c nclog.c dstring.c dutf8.c dinternal.c doffsets.c ncuri.c nclist.c ncbytes.c nchashmap.c nctime.c nc.c nclistmgr.c utf8proc.h utf8proc.c dpathmgr.c dutil.c drc.c dauth.c dreadonly.c dnotnc4.c dnotnc3.c dinfermodel.c daux.c dinstance.c -dcrc32.c dcrc32.h dcrc64.c ncexhash.c ncxcache.c ncjson.c ds3util.c dparallel.c) +dcrc32.c dcrc32.h dcrc64.c ncexhash.c ncxcache.c ncjson.c ds3util.c dparallel.c dmissing.c) # Netcdf-4 only functions. Must be defined even if not used SET(libdispatch_SOURCES ${libdispatch_SOURCES} dgroup.c dvlen.c dcompound.c dtype.c denum.c dopaque.c dfilter.c) @@ -19,8 +19,12 @@ IF(ENABLE_BYTERANGE) SET(libdispatch_SOURCES ${libdispatch_SOURCES} dhttp.c) ENDIF(ENABLE_BYTERANGE) -IF(ENABLE_S3_SDK) - SET(libdispatch_SOURCES ${libdispatch_SOURCES} ncs3sdk.cpp awsincludes.h) +IF(ENABLE_S3) + IF(ENABLE_S3_INTERNAL) + SET(libdispatch_SOURCES ${libdispatch_SOURCES} ncs3sdk_h5.c nch5s3comms.c nch5s3comms.h nccurl_sha256.c nccurl_sha256.h nccurl_hmac.c nccurl_hmac.h nccurl_setup.h) + ELSE() + SET(libdispatch_SOURCES ${libdispatch_SOURCES} ncs3sdk_aws.cpp awsincludes.h) + ENDIF() ENDIF() IF(REGEDIT) @@ -40,13 +44,19 @@ IF(ENABLE_NCZARR) target_include_directories(dispatch PUBLIC ../libnczarr) ENDIF(ENABLE_NCZARR) -IF(ENABLE_S3_SDK) -target_include_directories(dispatch PUBLIC ${AWSSDK_INCLUDE_DIRS}) -IF(NOT MSVC) -target_compile_features(dispatch PUBLIC cxx_std_11) -ENDIF() +IF(ENABLE_S3) + IF(ENABLE_S3_AWS) + target_include_directories(dispatch PUBLIC ${AWSSDK_INCLUDE_DIRS}) + IF(NOT MSVC) + target_compile_features(dispatch PUBLIC cxx_std_11) + ENDIF() + ELSE() + target_include_directories(dispatch PUBLIC ../libncxml) + ENDIF() ENDIF() +BUILD_BIN_TEST(ncrandom) + FILE(GLOB CUR_EXTRA_DIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*.h ${CMAKE_CURRENT_SOURCE_DIR}/*.c) SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} CMakeLists.txt Makefile.am) ADD_EXTRA_DIST("${CUR_EXTRA_DIST}") diff --git a/libdispatch/Makefile.am b/libdispatch/Makefile.am index 2db1916c15..a5bac12c6b 100644 --- a/libdispatch/Makefile.am +++ b/libdispatch/Makefile.am @@ -21,7 +21,7 @@ dinternal.c ddispatch.c dutf8.c nclog.c dstring.c ncuri.c nclist.c \ ncbytes.c nchashmap.c nctime.c nc.c nclistmgr.c dauth.c doffsets.c \ dpathmgr.c dutil.c dreadonly.c dnotnc4.c dnotnc3.c dinfermodel.c \ daux.c dinstance.c dcrc32.c dcrc32.h dcrc64.c ncexhash.c ncxcache.c \ -ncjson.c ds3util.c dparallel.c +ncjson.c ds3util.c dparallel.c dmissing.c # Add the utf8 codebase libdispatch_la_SOURCES += utf8proc.c utf8proc.h @@ -47,15 +47,27 @@ if ENABLE_BYTERANGE libdispatch_la_SOURCES += dhttp.c endif # ENABLE_BYTERANGE -if ENABLE_S3_SDK -libdispatch_la_SOURCES += ncs3sdk.cpp awsincludes.h +if ENABLE_S3 +if ENABLE_S3_INTERNAL +# Renamed to avoid conflicts with the HDF5 files +libdispatch_la_SOURCES += ncs3sdk_h5.c nch5s3comms.c nch5s3comms.h ncutil.h nccurl_setup.h \ + nccurl_sha256.c nccurl_sha256.h nccurl_hmac.c nccurl_hmac.h +AM_CPPFLAGS += -I$(top_srcdir)/libncxml +libdispatch_la_CPPFLAGS += ${AM_CPPFLAGS} +else +libdispatch_la_SOURCES += ncs3sdk_aws.cpp awsincludes.h AM_CXXFLAGS = -std=c++11 endif +endif if REGEDIT libdispatch_la_SOURCES += dreg.c endif +# Support generation of 32-bit unsigned int random numbers +noinst_PROGRAMS = ncrandom +ncrandom_SOURCES = ncrandom.c + EXTRA_DIST = CMakeLists.txt ncsettings.hdr utf8proc_data.c XGetopt.c # Build ncsettings.c as follows: diff --git a/libdispatch/awsincludes.h b/libdispatch/awsincludes.h index 6359de9829..b175724054 100755 --- a/libdispatch/awsincludes.h +++ b/libdispatch/awsincludes.h @@ -11,11 +11,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include diff --git a/libdispatch/dauth.c b/libdispatch/dauth.c index 86d89b156f..9ccef85937 100644 --- a/libdispatch/dauth.c +++ b/libdispatch/dauth.c @@ -94,6 +94,7 @@ NC_authsetup(NCauth** authp, NCURI* uri) int ret = NC_NOERR; char* uri_hostport = NULL; NCauth* auth = NULL; + struct AWSprofile* ap = NULL; if(uri != NULL) uri_hostport = NC_combinehostport(uri); @@ -175,8 +176,15 @@ NC_authsetup(NCauth** authp, NCURI* uri) nullfree(user); nullfree(pwd); } + /* Get the Default profile */ - auth->s3profile = strdup("default"); + if((ret=NC_authgets3profile("no",&ap))) goto done; + if(ap == NULL) + if((ret=NC_authgets3profile("default",&ap))) goto done; + if(ap != NULL) + auth->s3profile = strdup(ap->name); + else + auth->s3profile = NULL; if(authp) {*authp = auth; auth = NULL;} done: diff --git a/libdispatch/dcopy.c b/libdispatch/dcopy.c index 93e6960360..2491a92b6d 100644 --- a/libdispatch/dcopy.c +++ b/libdispatch/dcopy.c @@ -520,88 +520,6 @@ NC_copy_att(int ncid_in, int varid_in, const char *name, if ((res = nc_inq_att(ncid_in, varid_in, name, &xtype, &len))) return res; -#ifdef SEPDATA - if (xtype < NC_STRING) - { - /* Handle non-string atomic types. */ - if (len) - { - size_t size = NC_atomictypelen(xtype); - - assert(size > 0); - if (!(data = malloc(len * size))) - return NC_ENOMEM; - } - - res = nc_get_att(ncid_in, varid_in, name, data); - if (!res) - res = nc_put_att(ncid_out, varid_out, name, xtype, - len, data); - if (len) - free(data); - } -#ifdef USE_NETCDF4 - else if (xtype == NC_STRING) - { - /* Copy string attributes. */ - char **str_data; - if (!(str_data = malloc(sizeof(char *) * len))) - return NC_ENOMEM; - res = nc_get_att_string(ncid_in, varid_in, name, str_data); - if (!res) - res = nc_put_att_string(ncid_out, varid_out, name, len, - (const char **)str_data); - nc_free_string(len, str_data); - free(str_data); - } - else - { - /* Copy user-defined type attributes. */ - int class; - size_t size; - void *data; - nc_type xtype_out = NC_NAT; - - /* Find out if there is an equal type in the output file. */ - /* Note: original code used a libsrc4 specific internal function - which we had to "duplicate" here */ - if ((res = NC_find_equal_type(ncid_in, xtype, ncid_out, &xtype_out))) - return res; - if (xtype_out) - { - /* We found an equal type! */ - if ((res = nc_inq_user_type(ncid_in, xtype, NULL, &size, - NULL, NULL, &class))) - return res; - if (class == NC_VLEN) /* VLENs are different... */ - { - nc_vlen_t *vldata; - int i; - if (!(vldata = malloc(sizeof(nc_vlen_t) * len))) - return NC_ENOMEM; - if ((res = nc_get_att(ncid_in, varid_in, name, vldata))) - return res; - if ((res = nc_put_att(ncid_out, varid_out, name, xtype_out, - len, vldata))) - return res; - for (i = 0; i < len; i++) - if((res = nc_free_vlen(&vldata[i]))) - return res; - free(vldata); - } - else /* not VLEN */ - { - if (!(data = malloc(size * len))) - return NC_ENOMEM; - res = nc_get_att(ncid_in, varid_in, name, data); - if (!res) - res = nc_put_att(ncid_out, varid_out, name, xtype_out, len, data); - free(data); - } - } - } -#endif /*!USE_NETCDF4*/ -#else /*!SEPDATA*/ { /* Copy arbitrary attributes. */ int class; @@ -629,7 +547,6 @@ NC_copy_att(int ncid_in, int varid_in, const char *name, res = nc_put_att(ncid_out, varid_out, name, xtype_out, len, data); (void)nc_reclaim_data_all(ncid_out,xtype_out,data,len); } -#endif /*SEPDATA*/ return res; } diff --git a/libdispatch/ddispatch.c b/libdispatch/ddispatch.c index d405f343c4..e3d515a0e2 100644 --- a/libdispatch/ddispatch.c +++ b/libdispatch/ddispatch.c @@ -28,7 +28,7 @@ See LICENSE.txt for license information. #include #endif -#ifdef ENABLE_S3_SDK +#ifdef ENABLE_S3 #include "ncs3sdk.h" #endif diff --git a/libdispatch/derror.c b/libdispatch/derror.c index 3b4ebd2b6c..57465c3d33 100644 --- a/libdispatch/derror.c +++ b/libdispatch/derror.c @@ -274,7 +274,7 @@ const char *nc_strerror(int ncerr1) case NC_ENCZARR: return "NetCDF: NCZarr error"; case NC_ES3: - return "NetCDF: AWS S3 error"; + return "NetCDF: S3 error"; case NC_EEMPTY: return "NetCDF: Attempt to read empty NCZarr map key"; case NC_EOBJECT: diff --git a/libdispatch/dhttp.c b/libdispatch/dhttp.c index c97eddde8c..aa4d03031e 100644 --- a/libdispatch/dhttp.c +++ b/libdispatch/dhttp.c @@ -23,9 +23,13 @@ #include "ncbytes.h" #include "nclist.h" #include "ncuri.h" -#include "nchttp.h" #include "ncauth.h" +#ifdef ENABLE_S3 +#include "ncs3sdk.h" +#endif +#include "nchttp.h" + #undef TRACE #define CURLERR(e) reporterror(state,(e)) @@ -36,6 +40,10 @@ static const char* LENGTH_ACCEPT[] = {"content-length","accept-ranges",NULL}; static const char* CONTENTLENGTH[] = {"content-length",NULL}; /* Forward */ +static int nc_http_set_method(NC_HTTP_STATE* state, HTTPMETHOD method); +static int nc_http_set_response(NC_HTTP_STATE* state, NCbytes* buf); +static int nc_http_set_payload(NC_HTTP_STATE* state, size_t len, void* payload); + static int setupconn(NC_HTTP_STATE* state, const char* objecturl); static int execute(NC_HTTP_STATE* state); static int headerson(NC_HTTP_STATE* state, const char** which); @@ -66,44 +74,66 @@ Trace(const char* fcn) /**************************************************/ /** -@param curlp curl handle stored here if non-NULL +@param statep return a pointer to an allocated NC_HTTP_STATE */ int -nc_http_init(NC_HTTP_STATE** statep) +nc_http_open(const char* url, NC_HTTP_STATE** statep) { - return nc_http_init_verbose(statep,0); + return nc_http_open_verbose(url,0,statep); } int -nc_http_init_verbose(NC_HTTP_STATE** statep, int verbose) +nc_http_open_verbose(const char* path, int verbose, NC_HTTP_STATE** statep) { int stat = NC_NOERR; NC_HTTP_STATE* state = NULL; + NCURI* uri = NULL; Trace("open"); + ncuriparse(path,&uri); + if(uri == NULL) {stat = NCTHROW(NC_EURL); goto done;} + if((state = calloc(1,sizeof(NC_HTTP_STATE))) == NULL) - {stat = NC_ENOMEM; goto done;} - /* initialize curl*/ - state->curl = curl_easy_init(); - if (state->curl == NULL) {stat = NC_ECURL; goto done;} - showerrors(state); - if(verbose) { - long onoff = 1; - CURLcode cstat = CURLE_OK; - cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_VERBOSE, onoff)); - if(cstat != CURLE_OK) {stat = NC_ECURL; goto done;} - cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_DEBUGFUNCTION, my_trace)); - if(cstat != CURLE_OK) {stat = NC_ECURL; goto done;} + {stat = NCTHROW(NC_ENOMEM); goto done;} + state->path = strdup(path); + state->url = uri; uri = NULL; + state->format = (NC_iss3(state->url)?HTTPS3:HTTPCURL); + + switch (state->format) { + case HTTPCURL: { + /* initialize curl*/ + state->curl.curl = curl_easy_init(); + if (state->curl.curl == NULL) {stat = NCTHROW(NC_ECURL); goto done;} + showerrors(state); + state->errmsg = state->curl.errbuf; + if(verbose) { + long onoff = 1; + CURLcode cstat = CURLE_OK; + cstat = CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_VERBOSE, onoff)); + if(cstat != CURLE_OK) {stat = NCTHROW(NC_ECURL); goto done;} + cstat = CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_DEBUGFUNCTION, my_trace)); + if(cstat != CURLE_OK) {stat = NCTHROW(NC_ECURL); goto done;} + } + } break; +#ifdef ENABLE_S3 + case HTTPS3: { + if((state->s3.info = (NCS3INFO*)calloc(1,sizeof(NCS3INFO)))==NULL) + {stat = NCTHROW(NC_ENOMEM); goto done;} + if((stat = NC_s3urlprocess(state->url,state->s3.info))) goto done; + if((state->s3.s3client = NC_s3sdkcreateclient(state->s3.info))==NULL) + {stat = NCTHROW(NC_EURL); goto done;} + } break; +#endif + default: return NCTHROW(NC_ENOTBUILT); } stat = nc_http_reset(state); if(statep) {*statep = state; state = NULL;} - done: if(state) nc_http_close(state); dbgflush(); - return stat; + return NCTHROW(stat); } int @@ -113,16 +143,33 @@ nc_http_close(NC_HTTP_STATE* state) Trace("close"); - if(state == NULL) return stat; - if(state->curl != NULL) - (void)curl_easy_cleanup(state->curl); - nclistfreeall(state->response.headset); state->response.headset = NULL; - nclistfreeall(state->response.headers); state->response.headers = NULL; - ncbytesfree(state->response.buf); - nclistfreeall(state->request.headers); state->request.headers = NULL; + if(state == NULL) return NCTHROW(stat); + switch (state->format) { + case HTTPCURL: + if(state->curl.curl != NULL) + (void)curl_easy_cleanup(state->curl.curl); + nclistfreeall(state->curl.response.headset); state->curl.response.headset = NULL; + nclistfreeall(state->curl.response.headers); state->curl.response.headers = NULL; + ncbytesfree(state->curl.response.buf); + nclistfreeall(state->curl.request.headers); state->curl.request.headers = NULL; + break; +#ifdef ENABLE_S3 + case HTTPS3: { + if(state->s3.s3client) + NC_s3sdkclose(state->s3.s3client, state->s3.info, 0, NULL); + NC_s3clear(state->s3.info); + nullfree(state->s3.info); + state->s3.s3client = NULL; + } break; +#endif + default: stat = NCTHROW(NC_ENOTBUILT); goto done; + } + nullfree(state->path); + ncurifree(state->url); nullfree(state); +done: dbgflush(); - return stat; + return NCTHROW(stat); } /* Reset after a request */ @@ -131,75 +178,37 @@ nc_http_reset(NC_HTTP_STATE* state) { int stat = NC_NOERR; CURLcode cstat = CURLE_OK; - cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_HTTPGET, 1L)); - if(cstat != CURLE_OK) {stat = NC_ECURL; goto done;} - cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_NOBODY, 0L)); - if(cstat != CURLE_OK) {stat = NC_ECURL; goto done;} - cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_UPLOAD, 0L)); - if(cstat != CURLE_OK) {stat = NC_ECURL; goto done;} - cstat = curl_easy_setopt(state->curl, CURLOPT_CUSTOMREQUEST, NULL); - if(cstat != CURLE_OK) {stat = NC_ECURL; goto done;} - cstat = curl_easy_setopt(state->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)-1); - if(cstat != CURLE_OK) {stat = NC_ECURL; goto done;} - state->request.method = HTTPGET; - (void)CURLERR(curl_easy_setopt(state->curl, CURLOPT_WRITEFUNCTION, NULL)); - (void)CURLERR(curl_easy_setopt(state->curl, CURLOPT_WRITEDATA, NULL)); - (void)CURLERR(curl_easy_setopt(state->curl, CURLOPT_READFUNCTION, NULL)); - (void)CURLERR(curl_easy_setopt(state->curl, CURLOPT_READDATA, NULL)); - headersoff(state); -done: - return stat; -} - -/**************************************************/ -/* Set misc parameters */ -int -nc_http_set_method(NC_HTTP_STATE* state, HTTPMETHOD method) -{ - int stat = NC_NOERR; - CURLcode cstat = CURLE_OK; - switch (method) { - case HTTPGET: - cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_HTTPGET, 1L)); - break; - case HTTPHEAD: - cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_HTTPGET, 1L)); - cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_NOBODY, 1L)); + switch (state->format) { + case HTTPCURL: + cstat = CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_HTTPGET, 1L)); + if(cstat != CURLE_OK) {stat = NCTHROW(NC_ECURL); goto done;} + cstat = CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_NOBODY, 0L)); + if(cstat != CURLE_OK) {stat = NCTHROW(NC_ECURL); goto done;} + cstat = CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_UPLOAD, 0L)); + if(cstat != CURLE_OK) {stat = NCTHROW(NC_ECURL); goto done;} + cstat = curl_easy_setopt(state->curl.curl, CURLOPT_CUSTOMREQUEST, NULL); + if(cstat != CURLE_OK) {stat = NCTHROW(NC_ECURL); goto done;} + cstat = curl_easy_setopt(state->curl.curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)-1); + if(cstat != CURLE_OK) {stat = NCTHROW(NC_ECURL); goto done;} + state->curl.request.method = HTTPGET; + (void)CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_WRITEFUNCTION, NULL)); + (void)CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_WRITEDATA, NULL)); + (void)CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_READFUNCTION, NULL)); + (void)CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_READDATA, NULL)); + headersoff(state); break; - case HTTPPUT: - cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_UPLOAD, 1L)); - break; - case HTTPDELETE: - cstat = curl_easy_setopt(state->curl, CURLOPT_CUSTOMREQUEST, "DELETE"); - cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_NOBODY, 1L)); - break; - default: stat = NC_EINVAL; break; +#ifdef ENABLE_S3 + case HTTPS3: + break; /* Done automatically */ +#endif + default: stat = NCTHROW(NC_ENOTBUILT); goto done; } - if(cstat != CURLE_OK) {stat = NC_ECURL; goto done;} - state->request.method = method; done: - return stat; -} - -int -nc_http_set_payload(NC_HTTP_STATE* state, size_t size, void* payload) -{ - int stat = NC_NOERR; - state->request.payloadsize = size; - state->request.payload = payload; - state->request.payloadpos = 0; - return stat; -} - -int -nc_http_set_response(NC_HTTP_STATE* state, NCbytes* buf) -{ - int stat = NC_NOERR; - state->response.buf = buf; - return stat; + return NCTHROW(stat); } +/**************************************************/ /**************************************************/ /** @param state state handle @@ -210,7 +219,7 @@ nc_http_set_response(NC_HTTP_STATE* state, NCbytes* buf) */ int -nc_http_read(NC_HTTP_STATE* state, const char* objecturl, size64_t start, size64_t count, NCbytes* buf) +nc_http_read(NC_HTTP_STATE* state, size64_t start, size64_t count, NCbytes* buf) { int stat = NC_NOERR; char range[64]; @@ -219,55 +228,92 @@ nc_http_read(NC_HTTP_STATE* state, const char* objecturl, size64_t start, size64 Trace("read"); if(count == 0) - goto done; /* do not attempt to read */ - - if((stat = nc_http_set_response(state,buf))) goto fail; - if((stat = setupconn(state,objecturl))) - goto fail; + goto done; /* do not attempt to read */ - /* Set to read byte range */ - snprintf(range,sizeof(range),"%ld-%ld",(long)start,(long)((start+count)-1)); - cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_RANGE, range)); - if(cstat != CURLE_OK) - {stat = NC_ECURL; goto done;} - - if((stat = execute(state))) - goto done; + switch (state->format) { + case HTTPCURL: + if((stat = nc_http_set_response(state,buf))) goto fail; + if((stat = setupconn(state,state->path))) + goto fail; + + /* Set to read byte range */ + snprintf(range,sizeof(range),"%ld-%ld",(long)start,(long)((start+count)-1)); + cstat = CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_RANGE, range)); + if(cstat != CURLE_OK) + {stat = NCTHROW(NC_ECURL); goto done;} + + if((stat = execute(state))) + goto done; + break; +#ifdef ENABLE_S3 + case HTTPS3: { + /* Make sure buf has enough space allocated */ + ncbytessetalloc(buf,count); + ncbytessetlength(buf,count); + if((stat = NC_s3sdkread(state->s3.s3client, + state->s3.info->bucket, + state->s3.info->rootkey, + start, + count, + ncbytescontents(buf), + &state->errmsg))) goto done; + } break; +#endif + default: stat = NCTHROW(NC_ENOTBUILT); goto done; + } done: nc_http_reset(state); - state->response.buf = NULL; + if(state->format == HTTPCURL) + state->curl.response.buf = NULL; dbgflush(); - return stat; + return NCTHROW(stat); fail: - stat = NC_ECURL; + stat = NCTHROW(NC_ECURL); goto done; } /** @param state state handle -@param objecturl to write +@param objectpath to write @param payload send as body of a PUT */ int -nc_http_write(NC_HTTP_STATE* state, const char* objecturl, NCbytes* payload) +nc_http_write(NC_HTTP_STATE* state, NCbytes* payload) { int stat = NC_NOERR; Trace("write"); - if((stat = nc_http_set_payload(state,ncbyteslength(payload),ncbytescontents(payload)))) goto fail; - if((stat = nc_http_set_method(state,HTTPPUT))) goto fail; - if((stat = setupconn(state,objecturl))) goto fail; - if((stat = execute(state))) - goto done; + if(payload == NULL || ncbyteslength(payload) == 0) goto done; + + switch (state->format) { + case HTTPCURL: + if((stat = nc_http_set_payload(state,ncbyteslength(payload),ncbytescontents(payload)))) goto fail; + if((stat = nc_http_set_method(state,HTTPPUT))) goto fail; + if((stat = setupconn(state,state->path))) goto fail; + if((stat = execute(state))) + goto done; + break; +#ifdef ENABLE_S3 + case HTTPS3: + if((stat = NC_s3sdkwriteobject(state->s3.s3client, + state->s3.info->bucket, + state->s3.info->rootkey, + ncbyteslength(payload), + ncbytescontents(payload), + &state->errmsg))) goto done; + break; +#endif + default: stat = NCTHROW(NC_ENOTBUILT); goto done; + } done: nc_http_reset(state); - return stat; + return NCTHROW(stat); fail: - stat = NC_ECURL; + stat = NCTHROW(NC_ECURL); goto done; } @@ -278,73 +324,135 @@ Assume URL etc has already been set. */ int -nc_http_size(NC_HTTP_STATE* state, const char* objecturl, long long* sizep) +nc_http_size(NC_HTTP_STATE* state, long long* sizep) { int stat = NC_NOERR; const char* hdr = NULL; Trace("size"); if(sizep == NULL) - goto done; /* do not attempt to read */ - - if((stat = nc_http_set_method(state,HTTPHEAD))) goto done; - if((stat = setupconn(state,objecturl))) - goto done; - /* Make sure we get headers */ - if((stat = headerson(state,CONTENTLENGTH))) goto done; - - state->httpcode = 200; - if((stat = execute(state))) - goto done; - - if(nclistlength(state->response.headers) == 0) - {stat = NC_EURL; goto done;} - - /* Get the content length header */ - if((stat = lookupheader(state,"content-length",&hdr))==NC_NOERR) { - sscanf(hdr,"%llu",sizep); + goto done; /* do not attempt to read */ + + switch (state->format) { + case HTTPCURL: + if((stat = nc_http_set_method(state,HTTPHEAD))) goto done; + if((stat = setupconn(state,state->path))) + goto done; + /* Make sure we get headers */ + if((stat = headerson(state,CONTENTLENGTH))) goto done; + + state->httpcode = 200; + if((stat = execute(state))) + goto done; + + if(nclistlength(state->curl.response.headers) == 0) + {stat = NCTHROW(NC_EURL); goto done;} + + /* Get the content length header */ + if((stat = lookupheader(state,"content-length",&hdr))==NC_NOERR) + sscanf(hdr,"%llu",sizep); + break; +#ifdef ENABLE_S3 + case HTTPS3: { + size64_t len = 0; + if((stat = NC_s3sdkinfo(state->s3.s3client,state->s3.info->bucket,state->s3.info->rootkey,&len,&state->errmsg))) goto done; + if(sizep) *sizep = len; + } break; +#endif + default: stat = NCTHROW(NC_ENOTBUILT); goto done; } - done: nc_http_reset(state); - headersoff(state); + if(state->format == HTTPCURL) + headersoff(state); dbgflush(); - return stat; + return NCTHROW(stat); } -int +/**************************************************/ +/* Set misc parameters */ + +static int +nc_http_set_method(NC_HTTP_STATE* state, HTTPMETHOD method) +{ + int stat = NC_NOERR; + CURLcode cstat = CURLE_OK; + switch (method) { + case HTTPGET: + cstat = CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_HTTPGET, 1L)); + break; + case HTTPHEAD: + cstat = CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_HTTPGET, 1L)); + cstat = CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_NOBODY, 1L)); + break; + case HTTPPUT: + cstat = CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_UPLOAD, 1L)); + break; + case HTTPDELETE: + cstat = curl_easy_setopt(state->curl.curl, CURLOPT_CUSTOMREQUEST, "DELETE"); + cstat = CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_NOBODY, 1L)); + break; + default: stat = NCTHROW(NC_EINVAL); break; + } + if(cstat != CURLE_OK) {stat = NCTHROW(NC_ECURL); goto done;} + state->curl.request.method = method; +done: + return NCTHROW(stat); +} + +static int +nc_http_set_payload(NC_HTTP_STATE* state, size_t size, void* payload) +{ + int stat = NC_NOERR; + state->curl.request.payloadsize = size; + state->curl.request.payload = payload; + state->curl.request.payloadpos = 0; + return NCTHROW(stat); +} + +static int +nc_http_set_response(NC_HTTP_STATE* state, NCbytes* buf) +{ + int stat = NC_NOERR; + state->curl.response.buf = buf; + return NCTHROW(stat); +} + +#if 0 +static int nc_http_response_headset(NC_HTTP_STATE* state, const NClist* keys) { int i; if(keys == NULL) return NC_NOERR; - if(state->response.headset == NULL) - state->response.headset = nclistnew(); + if(state->curl.response.headset == NULL) + state->curl.response.headset = nclistnew(); for(i=0;iresponse.headset,key,0)) /* remove duplicates */ - nclistpush(state->response.headset,strdup(key)); + const char* key = (const char*)nclistget(keys,i); + if(!nclistmatch(state->curl.response.headset,key,0)) /* remove duplicates */ + nclistpush(state->curl.response.headset,strdup(key)); } return NC_NOERR; } -int +static int nc_http_response_headers(NC_HTTP_STATE* state, NClist** headersp) { NClist* headers = NULL; if(headersp != NULL) { - headers = nclistclone(state->response.headers,1); + headers = nclistclone(state->curl.response.headers,1); *headersp = headers; headers = NULL; } return NC_NOERR; } -int +static int nc_http_request_setheaders(NC_HTTP_STATE* state, const NClist* headers) { - nclistfreeall(state->request.headers); - state->request.headers = nclistclone(headers,1); + nclistfreeall(state->curl.request.headers); + state->curl.request.headers = nclistclone(headers,1); return NC_NOERR; } +#endif /**************************************************/ @@ -353,14 +461,14 @@ ReadMemoryCallback(char* buffer, size_t size, size_t nmemb, void *data) { NC_HTTP_STATE* state = data; size_t transfersize = size * nmemb; - size_t avail = (state->request.payloadsize - state->request.payloadpos); + size_t avail = (state->curl.request.payloadsize - state->curl.request.payloadpos); Trace("ReadMemoryCallback"); if(transfersize == 0) nclog(NCLOGWARN,"ReadMemoryCallback: zero sized buffer"); if(transfersize > avail) transfersize = avail; - memcpy(buffer,((char*)state->request.payload)+state->request.payloadpos,transfersize); - state->request.payloadpos += transfersize; + memcpy(buffer,((char*)state->curl.request.payload)+state->curl.request.payloadpos,transfersize); + state->curl.request.payloadpos += transfersize; return transfersize; } @@ -373,7 +481,7 @@ WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *data) Trace("WriteMemoryCallback"); if(realsize == 0) nclog(NCLOGWARN,"WriteMemoryCallback: zero sized chunk"); - ncbytesappendn(state->response.buf, ptr, realsize); + ncbytesappendn(state->curl.response.buf, ptr, realsize); return realsize; } @@ -387,14 +495,14 @@ trim(char* s) q--; /* point to last char of string */ /* Walk backward to first non-whitespace */ for(;q > p;q--) { - if(*q > ' ') break; /* found last non-whitespace */ + if(*q > ' ') break; /* found last non-whitespace */ } /* invariant: p == q || *q > ' ' */ if(p == q) /* string is all whitespace */ - {*p = '\0';} + {*p = '\0';} else {/* *q is last non-whitespace */ - q++; /* point to actual whitespace */ - *q = '\0'; + q++; /* point to actual whitespace */ + *q = '\0'; } /* Ok, skip past leading whitespace */ for(p=s;*p;p++) {if(*p > ' ') break;} @@ -431,29 +539,29 @@ HeaderCallback(char *buffer, size_t size, size_t nitems, void *data) name = malloc(i+1); memcpy(name,buffer,i); name[i] = '\0'; - if(state->response.headset != NULL) { - for(match=0,i=0;iresponse.headset);i++) { - hdr = (const char*)nclistget(state->response.headset,i); - if(strcasecmp(hdr,name)==0) {match = 1; break;} + if(state->curl.response.headset != NULL) { + for(match=0,i=0;icurl.response.headset);i++) { + hdr = (const char*)nclistget(state->curl.response.headset,i); + if(strcasecmp(hdr,name)==0) {match = 1; break;} } if(!match) goto done; } /* Capture this header */ value = NULL; if(havecolon) { - size_t vlen = (realsize - i); + size_t vlen = (realsize - i); value = malloc(vlen+1); - p++; /* skip colon */ + p++; /* skip colon */ memcpy(value,p,vlen); value[vlen] = '\0'; trim(value); } - if(state->response.headers == NULL) - state->response.headers = nclistnew(); - nclistpush(state->response.headers,name); + if(state->curl.response.headers == NULL) + state->curl.response.headers = nclistnew(); + nclistpush(state->curl.response.headers,name); name = NULL; if(value == NULL) value = strdup(""); - nclistpush(state->response.headers,value); + nclistpush(state->curl.response.headers,value); value = NULL; done: nullfree(name); @@ -471,80 +579,80 @@ setupconn(NC_HTTP_STATE* state, const char* objecturl) #ifdef TRACE fprintf(stderr,"curl.setup: url |%s|\n",objecturl); #endif - cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_URL, (void*)objecturl)); + cstat = CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_URL, (void*)objecturl)); if (cstat != CURLE_OK) goto fail; } /* Set options */ - cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_TIMEOUT, 100)); /* 30sec timeout*/ + cstat = CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_TIMEOUT, 100)); /* 30sec timeout*/ if (cstat != CURLE_OK) goto fail; - cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_CONNECTTIMEOUT, 100)); + cstat = CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_CONNECTTIMEOUT, 100)); if (cstat != CURLE_OK) goto fail; - cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_NOPROGRESS, 1)); + cstat = CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_NOPROGRESS, 1)); if (cstat != CURLE_OK) goto fail; - cstat = curl_easy_setopt(state->curl, CURLOPT_FOLLOWLOCATION, 1); + cstat = curl_easy_setopt(state->curl.curl, CURLOPT_FOLLOWLOCATION, 1); if (cstat != CURLE_OK) goto fail; /* Pull some values from .rc tables */ { - NCURI* uri = NULL; - char* hostport = NULL; - char* value = NULL; - ncuriparse(objecturl,&uri); - if(uri == NULL) goto fail; - hostport = NC_combinehostport(uri); - ncurifree(uri); uri = NULL; - value = NC_rclookup("HTTP.SSL.CAINFO",hostport,NULL); - nullfree(hostport); hostport = NULL; - if(value == NULL) - value = NC_rclookup("HTTP.SSL.CAINFO",NULL,NULL); - if(value != NULL) { - cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_CAINFO, value)); - if (cstat != CURLE_OK) goto fail; - } + NCURI* uri = NULL; + char* hostport = NULL; + char* value = NULL; + ncuriparse(objecturl,&uri); + if(uri == NULL) goto fail; + hostport = NC_combinehostport(uri); + ncurifree(uri); uri = NULL; + value = NC_rclookup("HTTP.SSL.CAINFO",hostport,NULL); + nullfree(hostport); hostport = NULL; + if(value == NULL) + value = NC_rclookup("HTTP.SSL.CAINFO",NULL,NULL); + if(value != NULL) { + cstat = CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_CAINFO, value)); + if (cstat != CURLE_OK) goto fail; + } } /* Set the method */ - if((stat = nc_http_set_method(state,state->request.method))) goto done; + if((stat = nc_http_set_method(state,state->curl.request.method))) goto done; - if(state->response.buf) { - /* send all data to this function */ - cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)); + if(state->curl.response.buf) { + /* send all data to this function */ + cstat = CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)); if (cstat != CURLE_OK) goto fail; /* Set argument for WriteMemoryCallback */ - cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_WRITEDATA, (void*)state)); + cstat = CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_WRITEDATA, (void*)state)); if (cstat != CURLE_OK) goto fail; } else {/* turn off data capture */ - (void)CURLERR(curl_easy_setopt(state->curl, CURLOPT_WRITEFUNCTION, NULL)); - (void)CURLERR(curl_easy_setopt(state->curl, CURLOPT_WRITEDATA, NULL)); + (void)CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_WRITEFUNCTION, NULL)); + (void)CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_WRITEDATA, NULL)); } - if(state->request.payloadsize > 0) { - state->request.payloadpos = 0; /* track reading */ - /* send all data to this function */ - cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_READFUNCTION, ReadMemoryCallback)); + if(state->curl.request.payloadsize > 0) { + state->curl.request.payloadpos = 0; /* track reading */ + /* send all data to this function */ + cstat = CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_READFUNCTION, ReadMemoryCallback)); if (cstat != CURLE_OK) goto fail; /* Set argument for ReadMemoryCallback */ - cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_READDATA, (void*)state)); + cstat = CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_READDATA, (void*)state)); if (cstat != CURLE_OK) goto fail; } else {/* turn off data capture */ - (void)CURLERR(curl_easy_setopt(state->curl, CURLOPT_READFUNCTION, NULL)); - (void)CURLERR(curl_easy_setopt(state->curl, CURLOPT_READDATA, NULL)); + (void)CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_READFUNCTION, NULL)); + (void)CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_READDATA, NULL)); } /* Do method specific actions */ - switch(state->request.method) { + switch(state->curl.request.method) { case HTTPPUT: - if(state->request.payloadsize > 0) - cstat = curl_easy_setopt(state->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)state->request.payloadsize); - break; + if(state->curl.request.payloadsize > 0) + cstat = curl_easy_setopt(state->curl.curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)state->curl.request.payloadsize); + break; default: break; } done: - return stat; + return NCTHROW(stat); fail: /* Turn off header capture */ headersoff(state); - stat = NC_ECURL; + stat = NCTHROW(NC_ECURL); goto done; } @@ -554,16 +662,16 @@ execute(NC_HTTP_STATE* state) int stat = NC_NOERR; CURLcode cstat = CURLE_OK; - cstat = CURLERR(curl_easy_perform(state->curl)); + cstat = CURLERR(curl_easy_perform(state->curl.curl)); if(cstat != CURLE_OK) goto fail; - cstat = CURLERR(curl_easy_getinfo(state->curl,CURLINFO_RESPONSE_CODE,&state->httpcode)); + cstat = CURLERR(curl_easy_getinfo(state->curl.curl,CURLINFO_RESPONSE_CODE,&state->httpcode)); if(cstat != CURLE_OK) state->httpcode = 0; done: - return stat; + return NCTHROW(stat); fail: - stat = NC_ECURL; + stat = NCTHROW(NC_ECURL); goto done; } @@ -574,34 +682,34 @@ headerson(NC_HTTP_STATE* state, const char** headset) CURLcode cstat = CURLE_OK; const char** p; - if(state->response.headers != NULL) - nclistfreeall(state->response.headers); - state->response.headers = nclistnew(); - if(state->response.headset != NULL) - nclistfreeall(state->response.headset); - state->response.headset = nclistnew(); + if(state->curl.response.headers != NULL) + nclistfreeall(state->curl.response.headers); + state->curl.response.headers = nclistnew(); + if(state->curl.response.headset != NULL) + nclistfreeall(state->curl.response.headset); + state->curl.response.headset = nclistnew(); for(p=headset;*p;p++) - nclistpush(state->response.headset,strdup(*p)); + nclistpush(state->curl.response.headset,strdup(*p)); - cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_HEADERFUNCTION, HeaderCallback)); + cstat = CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_HEADERFUNCTION, HeaderCallback)); if(cstat != CURLE_OK) goto fail; - cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_HEADERDATA, (void*)state)); + cstat = CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_HEADERDATA, (void*)state)); if (cstat != CURLE_OK) goto fail; done: - return stat; + return NCTHROW(stat); fail: - stat = NC_ECURL; + stat = NCTHROW(NC_ECURL); goto done; } static void headersoff(NC_HTTP_STATE* state) { - nclistfreeall(state->response.headers); - state->response.headers = NULL; - (void)CURLERR(curl_easy_setopt(state->curl, CURLOPT_HEADERFUNCTION, NULL)); - (void)CURLERR(curl_easy_setopt(state->curl, CURLOPT_HEADERDATA, NULL)); + nclistfreeall(state->curl.response.headers); + state->curl.response.headers = NULL; + (void)CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_HEADERFUNCTION, NULL)); + (void)CURLERR(curl_easy_setopt(state->curl.curl, CURLOPT_HEADERDATA, NULL)); } static int @@ -610,23 +718,23 @@ lookupheader(NC_HTTP_STATE* state, const char* key, const char** valuep) int i; const char* value = NULL; /* Get the content length header */ - for(i=0;iresponse.headers);i+=2) { - char* s = nclistget(state->response.headers,i); - if(strcasecmp(s,key)==0) { - value = nclistget(state->response.headers,i+1); - break; - } + for(i=0;icurl.response.headers);i+=2) { + char* s = nclistget(state->curl.response.headers,i); + if(strcasecmp(s,key)==0) { + value = nclistget(state->curl.response.headers,i+1); + break; + } } - if(value == NULL) return NC_ENOOBJECT; + if(value == NULL) return NCTHROW(NC_ENOOBJECT); if(valuep) - *valuep = value; + *valuep = value; return NC_NOERR; } static void showerrors(NC_HTTP_STATE* state) { - (void)curl_easy_setopt(state->curl, CURLOPT_ERRORBUFFER, state->errbuf); + (void)curl_easy_setopt(state->curl.curl, CURLOPT_ERRORBUFFER, state->curl.errbuf); } static int @@ -634,7 +742,8 @@ reporterror(NC_HTTP_STATE* state, CURLcode cstat) { if(cstat != CURLE_OK) fprintf(stderr,"curlcode: (%d)%s : %s\n", - cstat,curl_easy_strerror(cstat),state->errbuf); + cstat,curl_easy_strerror(cstat), + state->errmsg?state->errmsg:"?"); return cstat; } @@ -705,3 +814,52 @@ my_trace(CURL *handle, curl_infotype type, char *data, size_t size,void *userp) dump(text, stderr, (unsigned char *)data, size); return 0; } + +#if 0 +static char* +urlify(NC_HTTP_STATE* state, const char* path) +{ + NCbytes* buf = ncbytesnew(); + char* tmp = NULL; + + tmp = ncuribuild(state->url,NULL,NULL,NCURIPWD); + ncbytescat(buf,tmp); + nullfree(tmp); tmp = NULL; + ncbytescat(buf,"/"); + if(state->url->path != NULL) { + if(state->url->path[0] == '/') + ncbytescat(buf,state->url->path+1); + else + ncbytescat(buf,state->url->path); + if(ncbytesget(buf,ncbyteslength(buf)-1) == '/') + ncbytessetlength(buf,ncbyteslength(buf)-1); + } + if(path != NULL) { + if(path[0] != '/') + ncbytescat(buf,"/"); + ncbytescat(buf,path); + } + tmp = ncbytesextract(buf); + ncbytesfree(buf); + return tmp; +} + +int +nc_http_urisplit(const char* url, char** rootp, char** pathp) +{ + int stat = NC_NOERR; + NCURI* uri = NULL; + + ncuriparse(url,&uri); + if(uri == NULL) {stat = NCTHROW(NC_EURL); goto done;} + if(rootp) { + char* tmp = ncuribuild(uri,NULL,NULL,NCURIPWD); + *rootp = tmp; + nullfree(tmp); + tmp = NULL; + } + if(pathp) {*pathp = strdup(uri->path);} +done: + return NCTHROW(stat); +} +#endif diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index 5bfd27747c..937fd8bd08 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -25,12 +25,10 @@ #include "nclist.h" #include "nclog.h" #include "ncrc.h" -#ifdef ENABLE_BYTERANGE #include "nchttp.h" -#ifdef ENABLE_S3_SDK +#ifdef ENABLE_S3 #include "ncs3sdk.h" #endif -#endif #ifndef nulldup #define nulldup(x) ((x)?strdup(x):(x)) @@ -60,15 +58,13 @@ struct MagicFile { #ifdef USE_PARALLEL MPI_File fh; #endif -#ifdef ENABLE_BYTERANGE char* curlurl; /* url to use with CURLOPT_SET_URL */ NC_HTTP_STATE* state; -#ifdef ENABLE_S3_SDK +#ifdef ENABLE_S3 NCS3INFO s3; void* s3client; char* errmsg; #endif -#endif }; /** @internal Magic number for HDF5 files. To be consistent with @@ -918,7 +914,7 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void /* If s3, then rebuild the url */ if(NC_iss3(uri)) { NCURI* newuri = NULL; - if((stat = NC_s3urlrebuild(uri,&newuri,NULL,NULL))) goto done; + if((stat = NC_s3urlrebuild(uri,NULL,NULL,&newuri))) goto done; ncurifree(uri); uri = newuri; } else if(strcmp(uri->protocol,"file")==0) { @@ -1251,7 +1247,7 @@ check_file_type(const char *path, int omode, int use_parallel, memset((void*)&magicinfo,0,sizeof(magicinfo)); #ifdef _WIN32 /* including MINGW */ - /* Windows does not handle well multiple handles to the same file. + /* Windows does not handle multiple handles to the same file very well. So if file is already open/created, then find it and just get the model from that. */ if((nc = find_in_NCList_by_name(path)) != NULL) { @@ -1325,87 +1321,79 @@ static int openmagic(struct MagicFile* file) { int status = NC_NOERR; - if(fIsSet(file->omode,NC_INMEMORY)) { /* Get its length */ NC_memio* meminfo = (NC_memio*)file->parameters; assert(meminfo != NULL); file->filelen = (long long)meminfo->size; + goto done; + } + if(file->uri != NULL) { #ifdef ENABLE_BYTERANGE - } else if(file->uri != NULL) { -#ifdef ENABLE_S3_SDK - /* If this is an S3 URL, then handle specially */ - if(NC_iss3(file->uri)) { - if((status = NC_s3urlprocess(file->uri,&file->s3))) goto done; - if((file->s3client = NC_s3sdkcreateclient(&file->s3))==NULL) {status = NC_EURL; goto done;} - if((status = NC_s3sdkinfo(file->s3client,file->s3.bucket,file->s3.rootkey,&file->filelen,&file->errmsg))) - goto done; - file->iss3 = 1; - } else -#endif - { - /* Construct a URL minus any fragment */ - file->curlurl = ncuribuild(file->uri,NULL,NULL,NCURISVC); - /* Open the curl handle */ - if((status=nc_http_init(&file->state))) goto done; - if((status=nc_http_size(file->state,file->curlurl,&file->filelen))) goto done; - } + /* Construct a URL minus any fragment */ + file->curlurl = ncuribuild(file->uri,NULL,NULL,NCURISVC); + /* Open the curl handle */ + if((status=nc_http_open(file->curlurl, &file->state))) goto done; + if((status=nc_http_size(file->state,&file->filelen))) goto done; +#else /*!BYTERANGE*/ + {status = NC_ENOTBUILT;} #endif /*BYTERANGE*/ - } else { + goto done; + } #ifdef USE_PARALLEL - if (file->use_parallel) { - int retval; - MPI_Offset size; - assert(file->parameters != NULL); - if((retval = MPI_File_open(((NC_MPI_INFO*)file->parameters)->comm, + if (file->use_parallel) { + int retval; + MPI_Offset size; + assert(file->parameters != NULL); + if((retval = MPI_File_open(((NC_MPI_INFO*)file->parameters)->comm, (char*)file->path,MPI_MODE_RDONLY, ((NC_MPI_INFO*)file->parameters)->info, &file->fh)) != MPI_SUCCESS) { #ifdef MPI_ERR_NO_SUCH_FILE - int errorclass; - MPI_Error_class(retval, &errorclass); - if (errorclass == MPI_ERR_NO_SUCH_FILE) + int errorclass; + MPI_Error_class(retval, &errorclass); + if (errorclass == MPI_ERR_NO_SUCH_FILE) #ifdef NC_ENOENT - status = NC_ENOENT; -#else - status = errno; -#endif - else -#endif - status = NC_EPARINIT; - file->fh = MPI_FILE_NULL; - goto done; - } - /* Get its length */ - if((retval=MPI_File_get_size(file->fh, &size)) != MPI_SUCCESS) - {status = NC_EPARINIT; goto done;} - file->filelen = (long long)size; - } else + status = NC_ENOENT; +#else /*!NC_ENOENT*/ + status = errno; +#endif /*NC_ENOENT*/ + else +#endif /*MPI_ERR_NO_SUCH_FILE*/ + status = NC_EPARINIT; + file->fh = MPI_FILE_NULL; + goto done; + } + /* Get its length */ + if((retval=MPI_File_get_size(file->fh, &size)) != MPI_SUCCESS) + {status = NC_EPARINIT; goto done;} + file->filelen = (long long)size; + goto done; + } #endif /* USE_PARALLEL */ + { + if (file->path == NULL || strlen(file->path) == 0) + {status = NC_EINVAL; goto done;} + file->fp = NCfopen(file->path, "r"); + if(file->fp == NULL) + {status = errno; goto done;} + /* Get its length */ { - if (file->path == NULL || strlen(file->path) == 0) - {status = NC_EINVAL; goto done;} - file->fp = NCfopen(file->path, "r"); - if(file->fp == NULL) - {status = errno; goto done;} - /* Get its length */ - { - int fd = fileno(file->fp); + int fd = fileno(file->fp); #ifdef _WIN32 - __int64 len64 = _filelengthi64(fd); - if(len64 < 0) - {status = errno; goto done;} - file->filelen = (long long)len64; + __int64 len64 = _filelengthi64(fd); + if(len64 < 0) + {status = errno; goto done;} + file->filelen = (long long)len64; #else - off_t size; - size = lseek(fd, 0, SEEK_END); - if(size == -1) - {status = errno; goto done;} + off_t size; + size = lseek(fd, 0, SEEK_END); + if(size == -1) + {status = errno; goto done;} file->filelen = (long long)size; #endif - } - rewind(file->fp); - } + } + rewind(file->fp); } done: return check(status); @@ -1428,26 +1416,17 @@ readmagic(struct MagicFile* file, long pos, char* magic) #ifdef DEBUG printmagic("XXX: readmagic",magic,file); #endif -#ifdef ENABLE_BYTERANGE } else if(file->uri != NULL) { +#ifdef ENABLE_BYTERANGE fileoffset_t start = (size_t)pos; fileoffset_t count = MAGIC_NUMBER_LEN; -#ifdef ENABLE_S3_SDK - if(file->iss3) { - if((status = NC_s3sdkread(file->s3client,file->s3.bucket,file->s3.rootkey,start,count,(void*)magic,&file->errmsg))) - {goto done;} - } - else -#endif - { - status = nc_http_read(file->state, file->curlurl, start, count, buf); + status = nc_http_read(file->state, start, count, buf); if (status == NC_NOERR) { if (ncbyteslength(buf) != count) status = NC_EINVAL; else memcpy(magic, ncbytescontents(buf), count); } - } #endif } else { #ifdef USE_PARALLEL @@ -1492,20 +1471,11 @@ closemagic(struct MagicFile* file) if(fIsSet(file->omode,NC_INMEMORY)) { /* noop */ -#ifdef ENABLE_BYTERANGE } else if(file->uri != NULL) { -#ifdef ENABLE_S3_SDK - if(file->iss3) { - NC_s3sdkclose(file->s3client, &file->s3, 0, &file->errmsg); - NC_s3clear(&file->s3); - nullfree(file->errmsg); - } else -#endif - { +#ifdef ENABLE_BYTERANGE status = nc_http_close(file->state); - nullfree(file->curlurl); - } #endif + nullfree(file->curlurl); } else { #ifdef USE_PARALLEL if (file->use_parallel) { diff --git a/libdispatch/dmissing.c b/libdispatch/dmissing.c new file mode 100644 index 0000000000..b53335cef0 --- /dev/null +++ b/libdispatch/dmissing.c @@ -0,0 +1,208 @@ +/* + * Copyright 2018, University Corporation for Atmospheric Research + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + */ + +/** + * Provide local alternatives for unix functions + * not available on all machines. + * Currently, this defines: + * strdup + * strcpy + * strlcpy + * strlcat +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#ifdef HAVE_STDLIB_H +#include +#endif +#ifdef HAVE_STDIO_H +#include +#endif +#ifdef HAVE_STRING_H +#include +#endif + +/**************************************************/ + +#ifndef HAVE_STRDUP +char* +strdup(const char* s) +{ + char* dup; + size_t len; + if(s == NULL) return NULL; + len = strlen(s); + dup = (char*)malloc(len+1); + memcpy(dup,s,len); + dup[len] = '\0'; + return dup; +} +#endif + + +#ifndef WIN32 + +#ifndef HAVE_STRLCPY +/* + * Copyright (c) 1998, 2015 Todd C. Miller + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ +/* + * Copy string src to buffer dst of size dsize. At most dsize-1 + * chars will be copied. Always NUL terminates (unless dsize == 0). + * Returns strlen(src); if retval >= dsize, truncation occurred. + */ +size_t +strlcpy(char *dst, const char* src, size_t dsize) +{ + const char *osrc = src; + size_t nleft = dsize; + + /* Copy as many bytes as will fit. */ + if (nleft != 0) { + while (--nleft != 0) { + if ((*dst++ = *src++) == '\0') + break; + } + } + /* Not enough room in dst, add NUL and traverse rest of src. */ + if (nleft == 0) { + if (dsize != 0) + *dst = '\0'; /* NUL-terminate dst */ + while (*src++) + ; + } + return(src - osrc - 1); /* count does not include NUL */ +} +#endif + +#ifndef HAVE_STRLCAT +/* strlcat */ +/* + * Copyright (c) 1998, 2015 Todd C. Miller + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +/* + * Appends src to string dst of size dsize (unlike strncat, dsize is the + * full size of dst, not space left). At most dsize-1 characters + * will be copied. Always NUL terminates (unless dsize <= strlen(dst)). + * Returns strlen(src) + MIN(dsize, strlen(initial dst)). + * If retval >= dsize, truncation occurred. + */ +size_t +strlcat(char* dst, const char* src, size_t dsize) +{ + const char *odst = dst; + const char *osrc = src; + size_t n = dsize; + size_t dlen; + + /* Find the end of dst and adjust bytes left but don't go past end. */ + while (n-- != 0 && *dst != '\0') + dst++; + dlen = dst - odst; + n = dsize - dlen; + + if (n-- == 0) + return(dlen + strlen(src)); + while (*src != '\0') { + if (n != 0) { + *dst++ = *src; + n--; + } + src++; + } + *dst = '\0'; + + return(dlen + (src - osrc)); /* count does not include NUL */ +} +#endif /*!HAVE_STRLCAT*/ + +#endif /*WIN32*/ + +#if 0 +Not currently used +/* Define an version of strcasestr renamed to avoid any system definition */ +/* See https://android.googlesource.com/platform/bionic/+/a27d2baa/libc/string/strcasestr.c */ +/* $OpenBSD: strcasestr.c,v 1.3 2006/03/31 05:34:55 deraadt Exp $ */ +/* $NetBSD: strcasestr.c,v 1.2 2005/02/09 21:35:47 kleink Exp $ */ +/*- + * Copyright (c) 1990, 1993 + * The Regents of the University of California. All rights reserved. + * + * This code is derived from software contributed to Berkeley by + * Chris Torek. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * Find the first occurrence of find in s, ignore case. + */ +const char * +NC_strcasestr(const char *s, const char *find) +{ + char c, sc; + size_t len; + if ((c = *find++) != 0) { + c = (char)tolower((unsigned char)c); + len = strlen(find); + do { + do { + if ((sc = *s++) == 0) return (NULL); + } while ((char)tolower((unsigned char)sc) != c); + } while (strncasecmp(s, find, len) != 0); + s--; + } + return ((char *)s); +} +#endif diff --git a/libdispatch/dpathmgr.c b/libdispatch/dpathmgr.c index 9641fa81d7..14a6bb1a6e 100644 --- a/libdispatch/dpathmgr.c +++ b/libdispatch/dpathmgr.c @@ -28,6 +28,7 @@ #include #endif #include + #include "netcdf.h" #include "ncpathmgr.h" #include "nclog.h" diff --git a/libdispatch/drc.c b/libdispatch/drc.c index 07f7ed95ee..337021973b 100644 --- a/libdispatch/drc.c +++ b/libdispatch/drc.c @@ -19,6 +19,7 @@ See COPYRIGHT for license information. #include "ncbytes.h" #include "ncuri.h" #include "ncrc.h" +#include "ncs3sdk.h" #include "nclog.h" #include "ncauth.h" #include "ncpathmgr.h" @@ -67,8 +68,8 @@ static void freeprofilelist(NClist* profiles); /* Define default rc files and aliases, also defines load order*/ static const char* rcfilenames[] = {".ncrc", ".daprc", ".dodsrc",NULL}; -/* Read these files */ -static const char* awsconfigfiles[] = {".aws/credentials",".aws/config",NULL}; +/* Read these files in order and later overriding earlier */ +static const char* awsconfigfiles[] = {".aws/config",".aws/credentials",NULL}; static int NCRCinitialized = 0; @@ -158,7 +159,7 @@ ncrc_initialize(void) if((stat = NC_rcload())) { nclog(NCLOGWARN,".rc loading failed"); } - /* Load .aws/config */ + /* Load .aws/config &/ credentials */ if((stat = aws_load_credentials(ncg))) { nclog(NCLOGWARN,"AWS config file not loaded"); } @@ -488,7 +489,8 @@ rccompile(const char* filepath) NCURI* newuri = NULL; /* Rebuild the url to S3 "path" format */ nullfree(bucket); - if((ret = NC_s3urlrebuild(uri,&newuri,&bucket,NULL))) goto done; + bucket = NULL; + if((ret = NC_s3urlrebuild(uri,&bucket,NULL,&newuri))) goto done; ncurifree(uri); uri = newuri; newuri = NULL; @@ -760,6 +762,7 @@ Get the current active profile. The priority order is as follows: 1. aws.profile key in mode flags 2. aws.profile in .rc entries 4. "default" +5. "no" -- meaning do not use any profile => no secret key @param uri uri with mode flags, may be NULL @param profilep return profile name here or NULL if none found @@ -772,16 +775,27 @@ NC_getactives3profile(NCURI* uri, const char** profilep) { int stat = NC_NOERR; const char* profile = NULL; + struct AWSprofile* ap = NULL; profile = ncurifragmentlookup(uri,"aws.profile"); if(profile == NULL) profile = NC_rclookupx(uri,"AWS.PROFILE"); - if(profile == NULL) - profile = "default"; + + if(profile == NULL) { + if((stat=NC_authgets3profile("default",&ap))) goto done; + if(ap) profile = "default"; + } + + if(profile == NULL) { + if((stat=NC_authgets3profile("no",&ap))) goto done; + if(ap) profile = "no"; + } + #ifdef AWSDEBUG fprintf(stderr,">>> activeprofile = %s\n",(profile?profile:"null")); #endif if(profilep) *profilep = profile; +done: return stat; } @@ -809,7 +823,7 @@ NC_getdefaults3region(NCURI* uri, const char** regionp) if(region == NULL) region = NC_rclookupx(uri,"AWS.REGION"); if(region == NULL) {/* See if we can find a profile */ - if((stat = NC_getactives3profile(uri,&profile))==NC_NOERR) { + if(NC_getactives3profile(uri,&profile)==NC_NOERR) { if(profile) (void)NC_s3profilelookup(profile,"aws_region",®ion); } @@ -1038,13 +1052,14 @@ fprintf(stderr,">>> parse: entry=(%s,%s)\n",entry->key,entry->value); {stat = NCTHROW(NC_EINVAL); goto done;} } - /* If this profile already exists, then ignore new one */ + /* If this profile already exists, then replace old one */ for(i=0;iname,profile->name)==0) { - /* reclaim and ignore */ - freeprofile(profile); - profile = NULL; + nclistset(profiles,i,profile); + profile = NULL; + /* reclaim old one */ + freeprofile(p); break; } } @@ -1069,7 +1084,7 @@ freeentry(struct AWSentry* e) { if(e) { #ifdef AWSDEBUG -fprintf(stderr,">>> freeentry: key=%s value=%s\n",e->key,e->value); +fprintf(stderr,">>> freeentry: key=%p value=%p\n",e->key,e->value); #endif nullfree(e->key); nullfree(e->value); @@ -1108,7 +1123,7 @@ freeprofilelist(NClist* profiles) } } -/* Find, load, and parse the aws credentials file */ +/* Find, load, and parse the aws config &/or credentials file */ static int aws_load_credentials(NCglobalstate* gstate) { @@ -1119,6 +1134,14 @@ aws_load_credentials(NCglobalstate* gstate) NCbytes* buf = ncbytesnew(); char path[8192]; + /* add a "no" credentials */ + { + struct AWSprofile* noprof = (struct AWSprofile*)calloc(1,sizeof(struct AWSprofile)); + noprof->name = strdup("no"); + noprof->entries = nclistnew(); + nclistpush(profiles,noprof); noprof = NULL; + } + for(;*awscfg;awscfg++) { /* Construct the path ${HOME}/ or Windows equivalent. */ const char* cfg = *awscfg; @@ -1137,14 +1160,6 @@ aws_load_credentials(NCglobalstate* gstate) } } - /* add a "none" credentials */ - { - struct AWSprofile* noprof = (struct AWSprofile*)calloc(1,sizeof(struct AWSprofile)); - noprof->name = strdup("none"); - noprof->entries = nclistnew(); - nclistpush(profiles,noprof); noprof = NULL; - } - if(gstate->rcinfo->s3profiles) freeprofilelist(gstate->rcinfo->s3profiles); gstate->rcinfo->s3profiles = profiles; profiles = NULL; @@ -1152,12 +1167,16 @@ aws_load_credentials(NCglobalstate* gstate) #ifdef AWSDEBUG {int i,j; fprintf(stderr,">>> profiles:\n"); - for(i=0;iprofiles);i++) { - struct AWSprofile* p = (struct AWSprofile*)nclistget(creds->profiles,i); + for(i=0;ircinfo->s3profiles);i++) { + struct AWSprofile* p = (struct AWSprofile*)nclistget(gstate->rcinfo->s3profiles,i); fprintf(stderr," [%s]",p->name); for(j=0;jentries);j++) { struct AWSentry* e = (struct AWSentry*)nclistget(p->entries,j); - fprintf(stderr," %s=%s",e->key,e->value); + if(strcmp(e->key,"aws_access_key_id") + fprintf(stderr," %s=%d",e->key,(int)strlen(e->value)); + else if(strcmp(e->key,"aws_secret_access_key") + fprintf(stderr," %s=%d",e->key,(int)strlen(e->value)); + else fprintf(stderr," %s=%s",e->key,e->value); } fprintf(stderr,"\n"); } @@ -1170,6 +1189,13 @@ aws_load_credentials(NCglobalstate* gstate) return stat; } +/* Lookup a profile by name; +@param profilename to lookup +@param profilep return the matching profile; null if profile not found +@return NC_NOERR if no error +@return other error +*/ + int NC_authgets3profile(const char* profilename, struct AWSprofile** profilep) { @@ -1187,6 +1213,13 @@ NC_authgets3profile(const char* profilename, struct AWSprofile** profilep) return stat; } +/** +@param profile name of profile +@param key key to search for in profile +@param value place to store the value if key is found; NULL if not found +@return NC_NOERR if key is found, Some other error otherwise. +*/ + int NC_s3profilelookup(const char* profile, const char* key, const char** valuep) { @@ -1195,8 +1228,7 @@ NC_s3profilelookup(const char* profile, const char* key, const char** valuep) const char* value = NULL; if(profile == NULL) return NC_ES3; - stat = NC_authgets3profile(profile,&awsprof); - if(stat == NC_NOERR && awsprof != NULL) { + if((stat=NC_authgets3profile(profile,&awsprof))==NC_NOERR && awsprof != NULL) { for(i=0;ientries);i++) { struct AWSentry* entry = (struct AWSentry*)nclistget(awsprof->entries,i); if(strcasecmp(entry->key,key)==0) { diff --git a/libdispatch/ds3util.c b/libdispatch/ds3util.c index 21ab90de6c..2f769dd3d1 100644 --- a/libdispatch/ds3util.c +++ b/libdispatch/ds3util.c @@ -23,8 +23,9 @@ #include "netcdf.h" #include "ncuri.h" +#include "nclist.h" #include "ncrc.h" - +#include "ncs3sdk.h" #undef AWSDEBUG @@ -45,12 +46,13 @@ if provided, otherwise us-east-1. @param url (in) the current url @param region (in) region to use if needed; NULL => us-east-1 (out) region from url or the input region +@param bucketp (in) bucket to use if needed + (out) bucket from url @param pathurlp (out) the resulting pathified url string -@param bucketp (out) the bucket from the url */ int -NC_s3urlrebuild(NCURI* url, NCURI** newurlp, char** bucketp, char** outregionp) +NC_s3urlrebuild(NCURI* url, char** inoutbucketp, char** inoutregionp, NCURI** newurlp) { int i,stat = NC_NOERR; NClist* hostsegments = NULL; @@ -76,63 +78,68 @@ NC_s3urlrebuild(NCURI* url, NCURI** newurlp, char** bucketp, char** outregionp) if((stat = NC_split_delim(url->path,'/',pathsegments))) goto done; /* Distinguish path-style from virtual-host style from s3: and from other. - Virtual: https://bucket-name.s3.Region.amazonaws.com/ (1) - or: https://bucket-name.s3.amazonaws.com/ -- region defaults to us-east-1 (2) - Path: https://s3.Region.amazonaws.com/bucket-name/ (3) - or: https://s3.amazonaws.com/bucket-name/ -- region defaults to us-east-1 (4) - S3: s3://bucket-name/ (5) - Other: https:///bucketname/ (6) + Virtual: https://.s3..amazonaws.com/ (1) + or: https://.s3.amazonaws.com/ -- region defaults to us-east-1 (2) + Path: https://s3..amazonaws.com// (3) + or: https://s3.amazonaws.com// -- region defaults to us-east-1 (4) + S3: s3:/// (5) + Other: https://// (6) */ if(url->host == NULL || strlen(url->host) == 0) {stat = NC_EURL; goto done;} - if(strcmp(url->protocol,"s3")==0 && nclistlength(hostsegments)==1) { /* Case (5) */ + if(strcmp(url->protocol,"s3")==0 && nclistlength(hostsegments)==1) { /* Format (5) */ bucket = nclistremove(hostsegments,0); /* region unknown at this point */ } else if(endswith(url->host,AWSHOST)) { /* Virtual or path */ /* If we find a bucket as part of the host, then remove it */ switch (nclistlength(hostsegments)) { default: stat = NC_EURL; goto done; - case 3: /* Case (4) */ + case 3: /* Format (4) */ /* region unknown at this point */ /* bucket unknown at this point */ break; - case 4: /* Case (2) or (3) */ - if(strcasecmp(nclistget(hostsegments,1),"s3")==0) { /* Case (2) */ + case 4: /* Format (2) or (3) */ + if(strcasecmp(nclistget(hostsegments,1),"s3")==0) { /* Format (2) */ /* region unknown at this point */ - bucket = nclistremove(hostsegments,0); /* Note removal */ - } else if(strcasecmp(nclistget(hostsegments,0),"s3")==0) { /* Case (3) */ + bucket = nclistremove(hostsegments,0); /* Note removeal */ + } else if(strcasecmp(nclistget(hostsegments,0),"s3")==0) { /* Format (3) */ region = strdup(nclistget(hostsegments,1)); /* bucket unknown at this point */ - } else /* ! (2) and !(3) => error */ + } else /* ! Format (2) and ! Format (3) => error */ {stat = NC_EURL; goto done;} break; - case 5: /* Case (1) */ + case 5: /* Format (1) */ if(strcasecmp(nclistget(hostsegments,1),"s3")!=0) {stat = NC_EURL; goto done;} region = strdup(nclistget(hostsegments,2)); bucket = strdup(nclistremove(hostsegments,0)); break; } - } else { /* Presume Case (6) */ + } else { /* Presume Format (6) */ if((host = strdup(url->host))==NULL) {stat = NC_ENOMEM; goto done;} /* region is unknown */ /* bucket is unknown */ } - /* If region is null, use default */ + + /* region = (1) from url, (2) inoutregion, (3) default */ + if(region == NULL) + region = (inoutregionp?nulldup(*inoutregionp):NULL); if(region == NULL) { const char* region0 = NULL; /* Get default region */ if((stat = NC_getdefaults3region(url,®ion0))) goto done; region = strdup(region0); } - /* if bucket is null, use first segment of the path, if any */ - if(bucket == NULL) { - if(nclistlength(pathsegments) > 0) - bucket = nclistremove(pathsegments,0); + if(region == NULL) {stat = NC_ES3; goto done;} + + /* bucket = (1) from url, (2) inoutbucket */ + if(bucket == NULL && nclistlength(pathsegments) > 0) { + bucket = nclistremove(pathsegments,0); /* Get from the URL path; will reinsert below */ } - assert(bucket != NULL); - /* bucket may still be null */ + if(bucket == NULL) + bucket = (inoutbucketp?nulldup(*inoutbucketp):NULL); + if(bucket == NULL) {stat = NC_ES3; goto done;} if(host == NULL) { /* Construct the revised host */ ncbytescat(buf,"s3."); @@ -164,8 +171,8 @@ NC_s3urlrebuild(NCURI* url, NCURI** newurlp, char** bucketp, char** outregionp) fprintf(stderr,">>> NC_s3urlrebuild: final=%s bucket=%s region=%s\n",uri->uri,bucket,region); #endif if(newurlp) {*newurlp = newurl; newurl = NULL;} - if(bucketp) {*bucketp = bucket; bucket = NULL;} - if(outregionp) {*outregionp = region; region = NULL;} + if(inoutbucketp) {*inoutbucketp = bucket; bucket = NULL;} + if(inoutregionp) {*inoutregionp = region; region = NULL;} done: nullfree(region); @@ -207,11 +214,11 @@ NC_s3urlprocess(NCURI* url, NCS3INFO* s3) {stat = NC_EURL; goto done;} /* Get current profile */ if((stat = NC_getactives3profile(url,&profile0))) goto done; - if(profile0 == NULL) profile0 = "none"; + if(profile0 == NULL) profile0 = "no"; s3->profile = strdup(profile0); - /* Rebuild the URL to path format and get a usable region*/ - if((stat = NC_s3urlrebuild(url,&url2,&s3->bucket,&s3->region))) goto done; + /* Rebuild the URL to path format and get a usable region and optional bucket*/ + if((stat = NC_s3urlrebuild(url,&s3->bucket,&s3->region,&url2))) goto done; s3->host = strdup(url2->host); /* construct the rootkey minus the leading bucket */ pathsegments = nclistnew(); @@ -228,6 +235,24 @@ NC_s3urlprocess(NCURI* url, NCS3INFO* s3) return stat; } +int +NC_s3clone(NCS3INFO* s3, NCS3INFO** news3p) +{ + NCS3INFO* news3 = NULL; + if(s3 && news3p) { + if((news3 = (NCS3INFO*)calloc(1,sizeof(NCS3INFO)))==NULL) + return NC_ENOMEM; + if((news3->host = nulldup(s3->host))==NULL) return NC_ENOMEM; + if((news3->region = nulldup(s3->region))==NULL) return NC_ENOMEM; + if((news3->bucket = nulldup(s3->bucket))==NULL) return NC_ENOMEM; + if((news3->rootkey = nulldup(s3->rootkey))==NULL) return NC_ENOMEM; + if((news3->profile = nulldup(s3->profile))==NULL) return NC_ENOMEM; + } + if(news3p) {*news3p = news3; news3 = NULL;} + else {NC_s3clear(news3); nullfree(news3);} + return NC_NOERR; +} + int NC_s3clear(NCS3INFO* s3) { @@ -262,3 +287,16 @@ NC_iss3(NCURI* uri) return iss3; } +const char* +NC_s3dumps3info(NCS3INFO* info) +{ + static char text[8192]; + snprintf(text,sizeof(text),"host=%s region=%s bucket=%s rootkey=%s profile=%s", + (info->host?info->host:"null"), + (info->region?info->region:"null"), + (info->bucket?info->bucket:"null"), + (info->rootkey?info->rootkey:"null"), + (info->profile?info->profile:"null")); + return text; +} + diff --git a/libdispatch/dstring.c b/libdispatch/dstring.c index 468d36e8bb..1349717c8c 100644 --- a/libdispatch/dstring.c +++ b/libdispatch/dstring.c @@ -281,78 +281,3 @@ int return NC_NOERR; } - -/**************************************************/ -/* Provide local alternatives for unix functions - not available on all machines. Place here so that - all subsequence code modules can use it. -*/ - -#ifndef HAVE_STRDUP -char* -strdup(const char* s) -{ - char* dup; - if(s == NULL) return NULL; - dup = malloc(strlen(s)+1); - strcpy(dup,s); - return dup; -} -#endif - -/**************************************************/ -/* strlcat */ -/* - * Copyright (c) 1998, 2015 Todd C. Miller - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#ifndef HAVE_STRLCAT -#ifndef _WIN32 /* We will use strcat_s */ -/* - * Appends src to string dst of size dsize (unlike strncat, dsize is the - * full size of dst, not space left). At most dsize-1 characters - * will be copied. Always NUL terminates (unless dsize <= strlen(dst)). - * Returns strlen(src) + MIN(dsize, strlen(initial dst)). - * If retval >= dsize, truncation occurred. - */ -EXTERNL size_t -strlcat(char* dst, const char* src, size_t dsize) -{ - const char *odst = dst; - const char *osrc = src; - size_t n = dsize; - size_t dlen; - - /* Find the end of dst and adjust bytes left but don't go past end. */ - while (n-- != 0 && *dst != '\0') - dst++; - dlen = dst - odst; - n = dsize - dlen; - - if (n-- == 0) - return(dlen + strlen(src)); - while (*src != '\0') { - if (n != 0) { - *dst++ = *src; - n--; - } - src++; - } - *dst = '\0'; - - return(dlen + (src - osrc)); /* count does not include NUL */ -} -#endif /*!_WIN32*/ -#endif /*!HAVE_STRLCAT*/ diff --git a/libdispatch/ncbytes.c b/libdispatch/ncbytes.c index dc9144e8d8..84eb93e090 100644 --- a/libdispatch/ncbytes.c +++ b/libdispatch/ncbytes.c @@ -122,6 +122,7 @@ ncbytesappend(NCbytes* bb, char elem) int ncbytescat(NCbytes* bb, const char* s) { + if(bb == NULL) return ncbytesfail(); if(s == NULL) return 1; ncbytesappendn(bb,(void*)s,strlen(s)+1); /* include trailing null*/ /* back up over the trailing null*/ @@ -135,10 +136,9 @@ ncbytesappendn(NCbytes* bb, const void* elem, unsigned long n) { if(bb == NULL || elem == NULL) return ncbytesfail(); if(n == 0) {n = strlen((char*)elem);} - ncbytessetalloc(bb,bb->length+n+1); + ncbytessetalloc(bb,bb->length+n); memcpy((void*)&bb->content[bb->length],(void*)elem,n); bb->length += n; - bb->content[bb->length] = '\0'; return TRUE; } diff --git a/libdispatch/nccurl_hmac.c b/libdispatch/nccurl_hmac.c new file mode 100644 index 0000000000..7672f4882a --- /dev/null +++ b/libdispatch/nccurl_hmac.c @@ -0,0 +1,165 @@ +/********************************************************************* +* Copyright 2018, UCAR/Unidata +* See netcdf/COPYRIGHT file for copying and redistribution conditions. +* ********************************************************************/ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + * SPDX-License-Identifier: curl + * + * RFC2104 Keyed-Hashing for Message Authentication + * + ***************************************************************************/ + +#include "nccurl_setup.h" +#include "nccurl_hmac.h" + +/* + * Generic HMAC algorithm. + * + * This module computes HMAC digests based on any hash function. Parameters + * and computing procedures are set-up dynamically at HMAC computation context + * initialization. + */ + +static const unsigned char hmac_ipad = 0x36; +static const unsigned char hmac_opad = 0x5C; + + + +struct HMAC_context * +Curl_HMAC_init(const struct HMAC_params *hashparams, + const unsigned char *key, + unsigned int keylen) +{ + size_t i; + struct HMAC_context *ctxt; + unsigned char *hkey; + unsigned char b; + + /* Create HMAC context. */ + i = sizeof(*ctxt) + 2 * hashparams->hmac_ctxtsize + + hashparams->hmac_resultlen; + ctxt = malloc(i); + + if(!ctxt) + return ctxt; + + ctxt->hmac_hash = hashparams; + ctxt->hmac_hashctxt1 = (void *) (ctxt + 1); + ctxt->hmac_hashctxt2 = (void *) ((char *) ctxt->hmac_hashctxt1 + + hashparams->hmac_ctxtsize); + + /* If the key is too long, replace it by its hash digest. */ + if(keylen > hashparams->hmac_maxkeylen) { + (*hashparams->hmac_hinit)(ctxt->hmac_hashctxt1); + (*hashparams->hmac_hupdate)(ctxt->hmac_hashctxt1, key, keylen); + hkey = (unsigned char *) ctxt->hmac_hashctxt2 + hashparams->hmac_ctxtsize; + (*hashparams->hmac_hfinal)(hkey, ctxt->hmac_hashctxt1); + key = hkey; + keylen = hashparams->hmac_resultlen; + } + + /* Prime the two hash contexts with the modified key. */ + (*hashparams->hmac_hinit)(ctxt->hmac_hashctxt1); + (*hashparams->hmac_hinit)(ctxt->hmac_hashctxt2); + + for(i = 0; i < keylen; i++) { + b = (unsigned char)(*key ^ hmac_ipad); + (*hashparams->hmac_hupdate)(ctxt->hmac_hashctxt1, &b, 1); + b = (unsigned char)(*key++ ^ hmac_opad); + (*hashparams->hmac_hupdate)(ctxt->hmac_hashctxt2, &b, 1); + } + + for(; i < hashparams->hmac_maxkeylen; i++) { + (*hashparams->hmac_hupdate)(ctxt->hmac_hashctxt1, &hmac_ipad, 1); + (*hashparams->hmac_hupdate)(ctxt->hmac_hashctxt2, &hmac_opad, 1); + } + + /* Done, return pointer to HMAC context. */ + return ctxt; +} + +int Curl_HMAC_update(struct HMAC_context *ctxt, + const unsigned char *data, + unsigned int len) +{ + /* Update first hash calculation. */ + (*ctxt->hmac_hash->hmac_hupdate)(ctxt->hmac_hashctxt1, data, len); + return 0; +} + + +int Curl_HMAC_final(struct HMAC_context *ctxt, unsigned char *result) +{ + const struct HMAC_params *hashparams = ctxt->hmac_hash; + + /* Do not get result if called with a null parameter: only release + storage. */ + + if(!result) + result = (unsigned char *) ctxt->hmac_hashctxt2 + + ctxt->hmac_hash->hmac_ctxtsize; + + (*hashparams->hmac_hfinal)(result, ctxt->hmac_hashctxt1); + (*hashparams->hmac_hupdate)(ctxt->hmac_hashctxt2, + result, hashparams->hmac_resultlen); + (*hashparams->hmac_hfinal)(result, ctxt->hmac_hashctxt2); + free((char *) ctxt); + return 0; +} + +/* + * Curl_hmacit() + * + * This is used to generate a HMAC hash, for the specified input data, given + * the specified hash function and key. + * + * Parameters: + * + * hashparams [in] - The hash function (Curl_HMAC_MD5). + * key [in] - The key to use. + * keylen [in] - The length of the key. + * data [in] - The data to encrypt. + * datalen [in] - The length of the data. + * output [in/out] - The output buffer. + * + * Returns CURLE_OK on success. + */ +CURLcode +Curl_hmacit(const struct HMAC_params *hashparams, + const unsigned char *key, const size_t keylen, + const unsigned char *data, const size_t datalen, + unsigned char *output) +{ + struct HMAC_context *ctxt = + Curl_HMAC_init(hashparams, key, nccurlx_uztoui(keylen)); + + if(!ctxt) + return CURLE_OUT_OF_MEMORY; + + /* Update the digest with the given challenge */ + Curl_HMAC_update(ctxt, data, nccurlx_uztoui(datalen)); + + /* Finalise the digest */ + Curl_HMAC_final(ctxt, output); + + return CURLE_OK; +} diff --git a/libdispatch/nccurl_hmac.h b/libdispatch/nccurl_hmac.h new file mode 100644 index 0000000000..df8ace72f7 --- /dev/null +++ b/libdispatch/nccurl_hmac.h @@ -0,0 +1,80 @@ +#ifndef HEADER_CURL_HMAC_H +#define HEADER_CURL_HMAC_H +/********************************************************************* +* Copyright 2018, UCAR/Unidata +* See netcdf/COPYRIGHT file for copying and redistribution conditions. +* ********************************************************************/ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + * SPDX-License-Identifier: curl + * + ***************************************************************************/ + +#ifndef CURL_DISABLE_CRYPTO_AUTH + +#include + +#define HMAC_MD5_LENGTH 16 + +typedef CURLcode (* HMAC_hinit_func)(void *context); +typedef void (* HMAC_hupdate_func)(void *context, + const unsigned char *data, + unsigned int len); +typedef void (* HMAC_hfinal_func)(unsigned char *result, void *context); + + +/* Per-hash function HMAC parameters. */ +struct HMAC_params { + HMAC_hinit_func + hmac_hinit; /* Initialize context procedure. */ + HMAC_hupdate_func hmac_hupdate; /* Update context with data. */ + HMAC_hfinal_func hmac_hfinal; /* Get final result procedure. */ + unsigned int hmac_ctxtsize; /* Context structure size. */ + unsigned int hmac_maxkeylen; /* Maximum key length (bytes). */ + unsigned int hmac_resultlen; /* Result length (bytes). */ +}; + + +/* HMAC computation context. */ +struct HMAC_context { + const struct HMAC_params *hmac_hash; /* Hash function definition. */ + void *hmac_hashctxt1; /* Hash function context 1. */ + void *hmac_hashctxt2; /* Hash function context 2. */ +}; + + +/* Prototypes. */ +struct HMAC_context *Curl_HMAC_init(const struct HMAC_params *hashparams, + const unsigned char *key, + unsigned int keylen); +int Curl_HMAC_update(struct HMAC_context *context, + const unsigned char *data, + unsigned int len); +int Curl_HMAC_final(struct HMAC_context *context, unsigned char *result); + +extern CURLcode Curl_hmacit(const struct HMAC_params *hashparams, + const unsigned char *key, const size_t keylen, + const unsigned char *data, const size_t datalen, + unsigned char *output); + +#endif + +#endif /* HEADER_CURL_HMAC_H */ diff --git a/libdispatch/nccurl_setup.h b/libdispatch/nccurl_setup.h new file mode 100644 index 0000000000..787f3f0bcf --- /dev/null +++ b/libdispatch/nccurl_setup.h @@ -0,0 +1,51 @@ +/********************************************************************* +* Copyright 2018, UCAR/Unidata +* See netcdf/COPYRIGHT file for copying and redistribution conditions. +* ********************************************************************/ + +/* The Curl code used here (nccurl_sha256.[ch] and nccurl_hmac.[ch] + were taken from libcurl version 7.88.1. To upgrade this code, + do a diff between that version of curl and the new one and transfer + any relevant changes to this code. +*/ + +#ifndef NCCURL_SETUP_H +#define NCCURL_SETUP_H + +#include "config.h" +#ifdef HAVE_STDLIB_H +#include +#endif +#ifdef HAVE_INTTYPES_H +#include +#endif +#include "ncexternl.h" + +/* Please keep the SSL backend-specific #if branches in this order: + * + * 1. USE_OPENSSL + * 2. USE_GNUTLS + * 3. USE_MBEDTLS + * 4. USE_COMMON_CRYPTO + * 5. USE_WIN32_CRYPTO + * + * This ensures that the same SSL branch gets activated throughout this source + * file even if multiple backends are enabled at the same time. + */ + +#if defined(_WIN32) +#define USE_WIN32_CRYPTO +#elif ! defined(__APPLE__) +#define USE_OPENSSL +#endif + +#define CURLX_FUNCTION_CAST(target_type, func) (target_type)(void (*) (void))(func) + +#define DEBUGASSERT(expr) + +#define CURL_MASK_UINT ((unsigned int)~0) + +extern uintmax_t strtoumax(const char *nptr, char **endptr, int base); +extern unsigned int nccurlx_uztoui(size_t uznum); + +#endif /*NCCURL_SETUP_H*/ diff --git a/libdispatch/nccurl_sha256.c b/libdispatch/nccurl_sha256.c new file mode 100644 index 0000000000..4db391e492 --- /dev/null +++ b/libdispatch/nccurl_sha256.c @@ -0,0 +1,556 @@ +/********************************************************************* +* Copyright 2018, UCAR/Unidata +* See netcdf/COPYRIGHT file for copying and redistribution conditions. +* ********************************************************************/ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) Florin Petriuc, + * Copyright (C) Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + * SPDX-License-Identifier: curl + * + ***************************************************************************/ + +#include "nccurl_setup.h" + +#include "nccurl_sha256.h" +#include "nccurl_hmac.h" + +#ifdef USE_WOLFSSL +#include +#ifndef NO_SHA256 +#define USE_OPENSSL_SHA256 +#endif +#endif + +#if defined(USE_OPENSSL) + +#include + +#if (OPENSSL_VERSION_NUMBER >= 0x0090800fL) +#define USE_OPENSSL_SHA256 +#endif + +#endif /* USE_OPENSSL */ + +#ifdef USE_MBEDTLS +#include + +#if(MBEDTLS_VERSION_NUMBER >= 0x02070000) && \ + (MBEDTLS_VERSION_NUMBER < 0x03000000) + #define HAS_MBEDTLS_RESULT_CODE_BASED_FUNCTIONS +#endif +#endif /* USE_MBEDTLS */ + +#if defined(USE_OPENSSL_SHA256) + +/* When OpenSSL or wolfSSL is available is available we use their + * SHA256-functions. + */ +#if defined(USE_OPENSSL) +#include +#elif defined(USE_WOLFSSL) +#include +#endif + +#elif defined(USE_GNUTLS) +#include +#elif defined(USE_MBEDTLS) +#include +#elif (defined(__MAC_OS_X_VERSION_MAX_ALLOWED) && \ + (__MAC_OS_X_VERSION_MAX_ALLOWED >= 1040)) || \ + (defined(__IPHONE_OS_VERSION_MAX_ALLOWED) && \ + (__IPHONE_OS_VERSION_MAX_ALLOWED >= 20000)) +#include +#define AN_APPLE_OS +#elif defined(__APPLE__) +#include +#define AN_APPLE_OS +#elif defined(USE_WIN32_CRYPTO) +#include +#endif + +/* Please keep the SSL backend-specific #if branches in this order: + * + * 1. USE_OPENSSL + * 2. USE_GNUTLS + * 3. USE_MBEDTLS + * 4. USE_COMMON_CRYPTO + * 5. USE_WIN32_CRYPTO + * + * This ensures that the same SSL branch gets activated throughout this source + * file even if multiple backends are enabled at the same time. + */ + +#if defined(USE_OPENSSL_SHA256) + +struct sha256_ctx { + EVP_MD_CTX *openssl_ctx; +}; +typedef struct sha256_ctx my_sha256_ctx; + +static CURLcode my_sha256_init(my_sha256_ctx *ctx) +{ + ctx->openssl_ctx = EVP_MD_CTX_create(); + if(!ctx->openssl_ctx) + return CURLE_OUT_OF_MEMORY; + + EVP_DigestInit_ex(ctx->openssl_ctx, EVP_sha256(), NULL); + return CURLE_OK; +} + +static void my_sha256_update(my_sha256_ctx *ctx, + const unsigned char *data, + unsigned int length) +{ + EVP_DigestUpdate(ctx->openssl_ctx, data, length); +} + +static void my_sha256_final(unsigned char *digest, my_sha256_ctx *ctx) +{ + EVP_DigestFinal_ex(ctx->openssl_ctx, digest, NULL); + EVP_MD_CTX_destroy(ctx->openssl_ctx); +} + +#elif defined(USE_GNUTLS) + +typedef struct sha256_ctx my_sha256_ctx; + +static CURLcode my_sha256_init(my_sha256_ctx *ctx) +{ + sha256_init(ctx); + return CURLE_OK; +} + +static void my_sha256_update(my_sha256_ctx *ctx, + const unsigned char *data, + unsigned int length) +{ + sha256_update(ctx, length, data); +} + +static void my_sha256_final(unsigned char *digest, my_sha256_ctx *ctx) +{ + sha256_digest(ctx, SHA256_DIGEST_SIZE, digest); +} + +#elif defined(USE_MBEDTLS) + +typedef mbedtls_sha256_context my_sha256_ctx; + +static CURLcode my_sha256_init(my_sha256_ctx *ctx) +{ +#if !defined(HAS_MBEDTLS_RESULT_CODE_BASED_FUNCTIONS) + (void) mbedtls_sha256_starts(ctx, 0); +#else + (void) mbedtls_sha256_starts_ret(ctx, 0); +#endif + return CURLE_OK; +} + +static void my_sha256_update(my_sha256_ctx *ctx, + const unsigned char *data, + unsigned int length) +{ +#if !defined(HAS_MBEDTLS_RESULT_CODE_BASED_FUNCTIONS) + (void) mbedtls_sha256_update(ctx, data, length); +#else + (void) mbedtls_sha256_update_ret(ctx, data, length); +#endif +} + +static void my_sha256_final(unsigned char *digest, my_sha256_ctx *ctx) +{ +#if !defined(HAS_MBEDTLS_RESULT_CODE_BASED_FUNCTIONS) + (void) mbedtls_sha256_finish(ctx, digest); +#else + (void) mbedtls_sha256_finish_ret(ctx, digest); +#endif +} + +#elif defined(AN_APPLE_OS) +typedef CC_SHA256_CTX my_sha256_ctx; + +static CURLcode my_sha256_init(my_sha256_ctx *ctx) +{ + (void) CC_SHA256_Init(ctx); + return CURLE_OK; +} + +static void my_sha256_update(my_sha256_ctx *ctx, + const unsigned char *data, + unsigned int length) +{ + (void) CC_SHA256_Update(ctx, data, length); +} + +static void my_sha256_final(unsigned char *digest, my_sha256_ctx *ctx) +{ + (void) CC_SHA256_Final(digest, ctx); +} + +#elif defined(USE_WIN32_CRYPTO) + +struct sha256_ctx { + HCRYPTPROV hCryptProv; + HCRYPTHASH hHash; +}; +typedef struct sha256_ctx my_sha256_ctx; + +#if !defined(CALG_SHA_256) +#define CALG_SHA_256 0x0000800c +#endif + +static CURLcode my_sha256_init(my_sha256_ctx *ctx) +{ + if(CryptAcquireContext(&ctx->hCryptProv, NULL, NULL, PROV_RSA_AES, + CRYPT_VERIFYCONTEXT | CRYPT_SILENT)) { + CryptCreateHash(ctx->hCryptProv, CALG_SHA_256, 0, 0, &ctx->hHash); + } + + return CURLE_OK; +} + +static void my_sha256_update(my_sha256_ctx *ctx, + const unsigned char *data, + unsigned int length) +{ + CryptHashData(ctx->hHash, (unsigned char *) data, length, 0); +} + +static void my_sha256_final(unsigned char *digest, my_sha256_ctx *ctx) +{ + unsigned long length = 0; + + CryptGetHashParam(ctx->hHash, HP_HASHVAL, NULL, &length, 0); + if(length == SHA256_DIGEST_LENGTH) + CryptGetHashParam(ctx->hHash, HP_HASHVAL, digest, &length, 0); + + if(ctx->hHash) + CryptDestroyHash(ctx->hHash); + + if(ctx->hCryptProv) + CryptReleaseContext(ctx->hCryptProv, 0); +} + +#else + +/* When no other crypto library is available we use this code segment */ + +/* This is based on SHA256 implementation in LibTomCrypt that was released into + * public domain by Tom St Denis. */ + +#define WPA_GET_BE32(a) ((((unsigned long)(a)[0]) << 24) | \ + (((unsigned long)(a)[1]) << 16) | \ + (((unsigned long)(a)[2]) << 8) | \ + ((unsigned long)(a)[3])) +#define WPA_PUT_BE32(a, val) \ +do { \ + (a)[0] = (unsigned char)((((unsigned long) (val)) >> 24) & 0xff); \ + (a)[1] = (unsigned char)((((unsigned long) (val)) >> 16) & 0xff); \ + (a)[2] = (unsigned char)((((unsigned long) (val)) >> 8) & 0xff); \ + (a)[3] = (unsigned char)(((unsigned long) (val)) & 0xff); \ +} while(0) + +#ifdef HAVE_LONGLONG +#define WPA_PUT_BE64(a, val) \ +do { \ + (a)[0] = (unsigned char)(((unsigned long long)(val)) >> 56); \ + (a)[1] = (unsigned char)(((unsigned long long)(val)) >> 48); \ + (a)[2] = (unsigned char)(((unsigned long long)(val)) >> 40); \ + (a)[3] = (unsigned char)(((unsigned long long)(val)) >> 32); \ + (a)[4] = (unsigned char)(((unsigned long long)(val)) >> 24); \ + (a)[5] = (unsigned char)(((unsigned long long)(val)) >> 16); \ + (a)[6] = (unsigned char)(((unsigned long long)(val)) >> 8); \ + (a)[7] = (unsigned char)(((unsigned long long)(val)) & 0xff); \ +} while(0) +#else +#define WPA_PUT_BE64(a, val) \ +do { \ + (a)[0] = (unsigned char)(((unsigned __int64)(val)) >> 56); \ + (a)[1] = (unsigned char)(((unsigned __int64)(val)) >> 48); \ + (a)[2] = (unsigned char)(((unsigned __int64)(val)) >> 40); \ + (a)[3] = (unsigned char)(((unsigned __int64)(val)) >> 32); \ + (a)[4] = (unsigned char)(((unsigned __int64)(val)) >> 24); \ + (a)[5] = (unsigned char)(((unsigned __int64)(val)) >> 16); \ + (a)[6] = (unsigned char)(((unsigned __int64)(val)) >> 8); \ + (a)[7] = (unsigned char)(((unsigned __int64)(val)) & 0xff); \ +} while(0) +#endif + +struct sha256_state { +#ifdef HAVE_LONGLONG + unsigned long long length; +#else + unsigned __int64 length; +#endif + unsigned long state[8], curlen; + unsigned char buf[64]; +}; +typedef struct sha256_state my_sha256_ctx; + +/* The K array */ +static const unsigned long K[64] = { + 0x428a2f98UL, 0x71374491UL, 0xb5c0fbcfUL, 0xe9b5dba5UL, 0x3956c25bUL, + 0x59f111f1UL, 0x923f82a4UL, 0xab1c5ed5UL, 0xd807aa98UL, 0x12835b01UL, + 0x243185beUL, 0x550c7dc3UL, 0x72be5d74UL, 0x80deb1feUL, 0x9bdc06a7UL, + 0xc19bf174UL, 0xe49b69c1UL, 0xefbe4786UL, 0x0fc19dc6UL, 0x240ca1ccUL, + 0x2de92c6fUL, 0x4a7484aaUL, 0x5cb0a9dcUL, 0x76f988daUL, 0x983e5152UL, + 0xa831c66dUL, 0xb00327c8UL, 0xbf597fc7UL, 0xc6e00bf3UL, 0xd5a79147UL, + 0x06ca6351UL, 0x14292967UL, 0x27b70a85UL, 0x2e1b2138UL, 0x4d2c6dfcUL, + 0x53380d13UL, 0x650a7354UL, 0x766a0abbUL, 0x81c2c92eUL, 0x92722c85UL, + 0xa2bfe8a1UL, 0xa81a664bUL, 0xc24b8b70UL, 0xc76c51a3UL, 0xd192e819UL, + 0xd6990624UL, 0xf40e3585UL, 0x106aa070UL, 0x19a4c116UL, 0x1e376c08UL, + 0x2748774cUL, 0x34b0bcb5UL, 0x391c0cb3UL, 0x4ed8aa4aUL, 0x5b9cca4fUL, + 0x682e6ff3UL, 0x748f82eeUL, 0x78a5636fUL, 0x84c87814UL, 0x8cc70208UL, + 0x90befffaUL, 0xa4506cebUL, 0xbef9a3f7UL, 0xc67178f2UL +}; + +/* Various logical functions */ +#define RORc(x, y) \ +(((((unsigned long)(x) & 0xFFFFFFFFUL) >> (unsigned long)((y) & 31)) | \ + ((unsigned long)(x) << (unsigned long)(32 - ((y) & 31)))) & 0xFFFFFFFFUL) +#define Ch(x,y,z) (z ^ (x & (y ^ z))) +#define Maj(x,y,z) (((x | y) & z) | (x & y)) +#define S(x, n) RORc((x), (n)) +#define R(x, n) (((x)&0xFFFFFFFFUL)>>(n)) +#define Sigma0(x) (S(x, 2) ^ S(x, 13) ^ S(x, 22)) +#define Sigma1(x) (S(x, 6) ^ S(x, 11) ^ S(x, 25)) +#define Gamma0(x) (S(x, 7) ^ S(x, 18) ^ R(x, 3)) +#define Gamma1(x) (S(x, 17) ^ S(x, 19) ^ R(x, 10)) + +/* Compress 512-bits */ +static int sha256_compress(struct sha256_state *md, + unsigned char *buf) +{ + unsigned long S[8], W[64]; + int i; + + /* Copy state into S */ + for(i = 0; i < 8; i++) { + S[i] = md->state[i]; + } + /* copy the state into 512-bits into W[0..15] */ + for(i = 0; i < 16; i++) + W[i] = WPA_GET_BE32(buf + (4 * i)); + /* fill W[16..63] */ + for(i = 16; i < 64; i++) { + W[i] = Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + + W[i - 16]; + } + + /* Compress */ +#define RND(a,b,c,d,e,f,g,h,i) \ + do { \ + unsigned long t0 = h + Sigma1(e) + Ch(e, f, g) + K[i] + W[i]; \ + unsigned long t1 = Sigma0(a) + Maj(a, b, c); \ + d += t0; \ + h = t0 + t1; \ + } while(0) + + for(i = 0; i < 64; ++i) { + unsigned long t; + RND(S[0], S[1], S[2], S[3], S[4], S[5], S[6], S[7], i); + t = S[7]; S[7] = S[6]; S[6] = S[5]; S[5] = S[4]; + S[4] = S[3]; S[3] = S[2]; S[2] = S[1]; S[1] = S[0]; S[0] = t; + } + + /* Feedback */ + for(i = 0; i < 8; i++) { + md->state[i] = md->state[i] + S[i]; + } + + return 0; +} + +/* Initialize the hash state */ +static CURLcode my_sha256_init(struct sha256_state *md) +{ + md->curlen = 0; + md->length = 0; + md->state[0] = 0x6A09E667UL; + md->state[1] = 0xBB67AE85UL; + md->state[2] = 0x3C6EF372UL; + md->state[3] = 0xA54FF53AUL; + md->state[4] = 0x510E527FUL; + md->state[5] = 0x9B05688CUL; + md->state[6] = 0x1F83D9ABUL; + md->state[7] = 0x5BE0CD19UL; + + return CURLE_OK; +} + +/* + Process a block of memory though the hash + @param md The hash state + @param in The data to hash + @param inlen The length of the data (octets) + @return 0 if successful +*/ +static int my_sha256_update(struct sha256_state *md, + const unsigned char *in, + unsigned long inlen) +{ + unsigned long n; + +#define block_size 64 + if(md->curlen > sizeof(md->buf)) + return -1; + while(inlen > 0) { + if(md->curlen == 0 && inlen >= block_size) { + if(sha256_compress(md, (unsigned char *)in) < 0) + return -1; + md->length += block_size * 8; + in += block_size; + inlen -= block_size; + } + else { + n = CURLMIN(inlen, (block_size - md->curlen)); + memcpy(md->buf + md->curlen, in, n); + md->curlen += n; + in += n; + inlen -= n; + if(md->curlen == block_size) { + if(sha256_compress(md, md->buf) < 0) + return -1; + md->length += 8 * block_size; + md->curlen = 0; + } + } + } + + return 0; +} + +/* + Terminate the hash to get the digest + @param md The hash state + @param out [out] The destination of the hash (32 bytes) + @return 0 if successful +*/ +static int my_sha256_final(unsigned char *out, + struct sha256_state *md) +{ + int i; + + if(md->curlen >= sizeof(md->buf)) + return -1; + + /* Increase the length of the message */ + md->length += md->curlen * 8; + + /* Append the '1' bit */ + md->buf[md->curlen++] = (unsigned char)0x80; + + /* If the length is currently above 56 bytes we append zeros + * then compress. Then we can fall back to padding zeros and length + * encoding like normal. + */ + if(md->curlen > 56) { + while(md->curlen < 64) { + md->buf[md->curlen++] = (unsigned char)0; + } + sha256_compress(md, md->buf); + md->curlen = 0; + } + + /* Pad up to 56 bytes of zeroes */ + while(md->curlen < 56) { + md->buf[md->curlen++] = (unsigned char)0; + } + + /* Store length */ + WPA_PUT_BE64(md->buf + 56, md->length); + sha256_compress(md, md->buf); + + /* Copy output */ + for(i = 0; i < 8; i++) + WPA_PUT_BE32(out + (4 * i), md->state[i]); + + return 0; +} + +#endif /* CRYPTO LIBS */ + +/* + * Curl_sha256it() + * + * Generates a SHA256 hash for the given input data. + * + * Parameters: + * + * output [in/out] - The output buffer. + * input [in] - The input data. + * length [in] - The input length. + * + * Returns CURLE_OK on success. + */ +CURLcode +Curl_sha256it(unsigned char *output, const unsigned char *input, + const size_t length) +{ + CURLcode result; + my_sha256_ctx ctx; + + result = my_sha256_init(&ctx); + if(!result) { + my_sha256_update(&ctx, input, nccurlx_uztoui(length)); + my_sha256_final(output, &ctx); + } + return result; +} + + +const struct HMAC_params Curl_HMAC_SHA256[] = { + { + /* Hash initialization function. */ + CURLX_FUNCTION_CAST(HMAC_hinit_func, my_sha256_init), + /* Hash update function. */ + CURLX_FUNCTION_CAST(HMAC_hupdate_func, my_sha256_update), + /* Hash computation end function. */ + CURLX_FUNCTION_CAST(HMAC_hfinal_func, my_sha256_final), + /* Size of hash context structure. */ + sizeof(my_sha256_ctx), + /* Maximum key length. */ + 64, + /* Result size. */ + 32 + } +}; + +/* +** unsigned size_t to unsigned int +*/ +unsigned int +nccurlx_uztoui(size_t uznum) +{ +#ifdef __INTEL_COMPILER +# pragma warning(push) +# pragma warning(disable:810) /* conversion may lose significant bits */ +#endif + +#if UINT_MAX < SIZE_T_MAX + DEBUGASSERT(uznum <= (size_t) CURL_MASK_UINT); +#endif + return (unsigned int)(uznum & (size_t) CURL_MASK_UINT); + +#ifdef __INTEL_COMPILER +# pragma warning(pop) +#endif +} diff --git a/libdispatch/nccurl_sha256.h b/libdispatch/nccurl_sha256.h new file mode 100644 index 0000000000..27395928d8 --- /dev/null +++ b/libdispatch/nccurl_sha256.h @@ -0,0 +1,53 @@ +#ifndef HEADER_CURL_SHA256_H +#define HEADER_CURL_SHA256_H +/********************************************************************* +* Copyright 2018, UCAR/Unidata +* See netcdf/COPYRIGHT file for copying and redistribution conditions. +* ********************************************************************/ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) Florin Petriuc, + * Copyright (C) Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + * SPDX-License-Identifier: curl + * + ***************************************************************************/ + + +#include "nccurl_setup.h" +#include "nccurl_hmac.h" + +#ifndef CURL_DISABLE_CRYPTO_AUTH + +extern const struct HMAC_params Curl_HMAC_SHA256[1]; + +#ifdef USE_WOLFSSL +/* SHA256_DIGEST_LENGTH is an enum value in wolfSSL. Need to import it from + * sha.h */ +#include +#include +#else +#define SHA256_DIGEST_LENGTH 32 +#endif + +EXTERNL CURLcode Curl_sha256it(unsigned char *outbuffer, const unsigned char *input, + const size_t len); +#endif + +#endif /* HEADER_CURL_SHA256_H */ diff --git a/libdispatch/nch5s3comms.c b/libdispatch/nch5s3comms.c new file mode 100644 index 0000000000..fd473f64e5 --- /dev/null +++ b/libdispatch/nch5s3comms.c @@ -0,0 +1,2905 @@ +/********************************************************************* +* Copyright 2018, UCAR/Unidata +* See netcdf/COPYRIGHT file for copying and redistribution conditions. +* ********************************************************************/ + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/***************************************************************************** + * Read-Only S3 Virtual File Driver (VFD) + * Source for S3 Communications module + * ***NOT A FILE DRIVER*** + * Provide functions and structures required for interfacing with Amazon + * Simple Storage Service (S3). + * Provide S3 object access as if it were a local file. + * Connect to remote host, send and receive HTTP requests and responses + * as part of the AWS REST API, authenticating requests as appropriate. + * Programmer: Jacob Smith + * 2017-11-30 + *****************************************************************************/ + +/** + * Unidata Changes: + * Derived from HDF5-1.14.0 H5FDs3comms.[ch] + * Modified to support Write operations and support NCZarr. + * Primary Changes: + * - rename H5FD_s3comms to NCH5_s3comms to avoid name conflicts + * - Remove HDF5 dependencies + * - Support zmap API + * + * Note that this code is very ugly because it is the bastard + * child of the HDF5 coding style and the NetCDF-C coding style + * and some libcurl as well. + * + * A note about the nccurl_hmac.c and nccurl_sha256.c files. + * The code in this file depends on having access to two + * cryptographic functions: + * 1. HMAC signing function + * 2. SHA256 digest function + * + * There are a number of libraries providing these functions. + * For example, OPENSSL, WOLFSSL, GNUTLS, Windows crypto package + * etc. It turns out that libcurl has identified all of these + * possible sources and set up a wrapper to handle the + * possibilities. So, this code copies the libcurl wrapper to + * inherit its multi-source capabilities. + * + * Author: Dennis Heimbigner + * Creation Date: 2/12/2023 + * Last Modified: 5/1/2023 + */ + +/****************/ +/* Module Setup */ +/****************/ + +/***********/ +/* Headers */ +/***********/ + +/*****************/ +#include "config.h" + +#ifdef HAVE_STDLIB_H +#include +#endif +#ifdef HAVE_STRING_H +#include +#endif +#ifdef HAVE_CTYPE_H +#include +#endif +#include + +#include "nccurl_sha256.h" +#include "nccurl_hmac.h" + +/* Necessary S3 headers */ +#include +//#include +//#include +//#include + +#include "netcdf.h" +#include "ncuri.h" +#include "ncutil.h" + +/*****************/ + +#include "nch5s3comms.h" /* S3 Communications */ + +/****************/ +/* Local Macros */ +/****************/ + +#undef TRACING +#undef DEBUG + +#define SUCCEED NC_NOERR +#ifndef TRUE +#define TRUE 1 +#define FALSE 0 +#endif + +/* enable debugging */ +#define S3COMMS_DEBUG 0 +#define S3COMMS_DEBUG_TRACE 0 + +/* manipulate verbosity of CURL output + * operates separately from S3COMMS_DEBUG + * 0 -> no explicit curl output + * 1 -> on error, print failure info to stderr + * 2 -> in addition to above, print information for all performs; sets all + * curl handles with CURLOPT_VERBOSE + */ +#define S3COMMS_CURL_VERBOSITY 0 + +/* Apparently Apple/OSX C Compiler does not (yet) accept __VA_OPT__(,), + so we have to case it out (ugh!) +*/ +#if S3COMMS_CURL_VERBOSITY > 1 +#define HDONE_ERROR(ignore1,ncerr,ignore2,msg) do {ret_value=report(ncerr,__func__,__LINE__,msg);} while(0) +#define HDONE_ERRORVA(ignore1,ncerr,ignore2,msg,...) do {ret_value=report(ncerr,__func__,__LINE__,msg, __VA_ARGS__);} while(0) +#define HGOTO_ERROR(ignore1,ncerr,ignore2,msg,...) do {ret_value=report(ncerr,__func__,__LINE__,msg); goto done;} while(0) +#define HGOTO_ERRORVA(ignore1,ncerr,ignore2,msg,...) do {ret_value=report(ncerr,__func__,__LINE__,msg, __VA_ARGS__); goto done;} while(0) +#else /*S3COMMS_CURL_VERBOSITY*/ +#define HDONE_ERROR(ignore1,ncerr,ignore2,msg,...) do {ret_value=(ncerr);} while(0) +#define HDONE_ERRORVA(ignore1,ncerr,ignore2,msg,...) HDONE_ERROR(ignore1,ncerr,ignore2,msg) +#define HGOTO_ERROR(ignore1,ncerr,ignore2,msg,...) do {ret_value=(ncerr);; goto done;} while(0) +#define HGOTO_ERRORVA(ignore1,ncerr,ignore2,msg,...) HGOTO_ERROR(ignore1,ncerr,ignore2,msg) +#endif /*S3COMMS_CURL_VERBOSITY*/ + +/* size to allocate for "bytes=[-]" HTTP Range value + */ +#define S3COMMS_MAX_RANGE_STRING_SIZE 128 + +#define SNULL(x) ((x)==NULL?"NULL":(x)) +#define INULL(x) ((x)==NULL?-1:(int)(*x)) + +#ifdef TRACING +#define TRACE(level,fmt,...) s3trace((level),__func__,fmt,##__VA_ARGS__) +#define TRACEMORE(level,fmt,...) s3tracemore((level),fmt,##__VA_ARGS__) +#define UNTRACE(e) s3untrace(__func__,NCTHROW(e),NULL) +#define UNTRACEX(e,fmt,...) s3untrace(__func__,NCTHROW(e),fmt,##__VA_ARGS__) +#else +#define TRACE(level,fmt,...) +#define TRACEMORE(level,fmt,...) +#define UNTRACE(e) (e) +#define UNTRACEX(e,fmt,...) (e) +#endif + +#ifdef TRACING +static struct S3LOGGLOBAL { + FILE* stream; + int depth; + struct Frame { + const char* fcn; + int level; + int depth; + } frames[1024]; +} s3log_global = {NULL,0}; + +static int +s3breakpoint(int err) +{ + return err; +} + +static void +s3vtrace(int level, const char* fcn, const char* fmt, va_list ap) +{ + struct Frame* frame; + if(s3log_global.stream == NULL) s3log_global.stream = stderr; + if(fcn != NULL) { + frame = &s3log_global.frames[s3log_global.depth]; + frame->fcn = fcn; + frame->level = level; + frame->depth = s3log_global.depth; + } + { + if(fcn != NULL) + fprintf(s3log_global.stream,"%s: (%d): %s:","Enter",level,fcn); + if(fmt != NULL) + vfprintf(s3log_global.stream, fmt, ap); + fprintf(s3log_global.stream, "\n" ); + fflush(s3log_global.stream); + } + if(fcn != NULL) s3log_global.depth++; +} + +static void +s3trace(int level, const char* fcn, const char* fmt, ...) +{ + va_list args; + va_start(args, fmt); + s3vtrace(level,fcn,fmt,args); + va_end(args); +} + +static void +s3tracemore(int level, const char* fmt, ...) +{ + va_list args; + va_start(args, fmt); + s3vtrace(level,NULL,fmt,args); + va_end(args); +} + +static int +s3untrace(const char* fcn, int err, const char* fmt, ...) +{ + va_list args; + struct Frame* frame; + va_start(args, fmt); + if(s3log_global.depth == 0) { + fprintf(s3log_global.stream,"*** Unmatched untrace: %s: depth==0\n",fcn); + goto done; + } + s3log_global.depth--; + frame = &s3log_global.frames[s3log_global.depth]; + if(frame->depth != s3log_global.depth || strcmp(frame->fcn,fcn) != 0) { + fprintf(s3log_global.stream,"*** Unmatched untrace: fcn=%s expected=%s\n",frame->fcn,fcn); + goto done; + } + { + fprintf(s3log_global.stream,"%s: (%d): %s: ","Exit",frame->level,frame->fcn); + if(err) + fprintf(s3log_global.stream,"err=(%d) '%s':",err,nc_strerror(err)); + if(fmt != NULL) + vfprintf(s3log_global.stream, fmt, args); + fprintf(s3log_global.stream, "\n" ); + fflush(s3log_global.stream); + } +done: + va_end(args); + if(err != 0) + return s3breakpoint(err); + else + return err; +} + +#endif + + +/******************/ +/* Local Decls*/ +/******************/ + +#define S3COMMS_VERB_MAX 16 + +/********************/ +/* Local Structures */ +/********************/ + +/* Provide a single, unified argument for curl callbacks */ +/* struct s3r_cbstruct + * Structure passed to curl callback + */ +struct s3r_cbstruct { + unsigned long magic; + VString* data; + const char* key; /* headcallback: header search key */ + size_t pos; /* readcallback: write from this point in data */ +}; +#define S3COMMS_CALLBACK_STRUCT_MAGIC 0x28c2b2ul + +/********************/ +/* Local Prototypes */ +/********************/ + +/* Forward */ +static int NCH5_s3comms_s3r_execute(s3r_t *handle, const char* url, HTTPVerb verb, const char* byterange, const char* header, const char** otherheaders, long* httpcodep, VString* data); +static size_t curlwritecallback(char *ptr, size_t size, size_t nmemb, void *userdata); +static size_t curlheadercallback(char *ptr, size_t size, size_t nmemb, void *userdata); +static int curl_reset(s3r_t* handle); +static int perform_request(s3r_t* handle, long* httpcode); +static int build_request(s3r_t* handle, NCURI* purl, const char* byterange, const char** otherheaders, VString* payload, HTTPVerb verb); +static int request_setup(s3r_t* handle, const char* url, HTTPVerb verb, struct s3r_cbstruct*); +static int validate_handle(s3r_t* handle, const char* url); +static int validate_url(NCURI* purl); +static int build_range(size_t offset, size_t len, char** rangep); +static const char* verbtext(HTTPVerb verb); +static int trace(CURL* curl, int onoff); +static int sortheaders(VList* headers); +static int httptonc(long httpcode); +static void hrb_node_free(hrb_node_t *node); + +#if S3COMMS_DEBUG_HRB +static void dumphrbnodes(VList* nodes); +static void dumphrb(hrb_t* hrb); +#endif + +/*********************/ +/* Package Variables */ +/*********************/ + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + +/*******************/ +/* Local Variables */ +/*******************/ + +/*************/ +/* Functions */ +/*************/ + +#if S3COMMS_CURL_VERBOSITY > 0 +static void +nch5breakpoint(int stat) +{ + if(stat == -78) abort(); + ncbreakpoint(stat); +} + +static int +report(int stat, const char* fcn, int lineno, const char* fmt, ...) +{ + va_list args; + char bigfmt[1024]; + + if(stat == NC_NOERR) goto done; + snprintf(bigfmt,sizeof(bigfmt),"(%d)%s ; fcn=%s line=%d ; %s",stat,nc_strerror(stat),fcn,lineno,fmt); + va_start(args,fmt); + ncvlog(NCLOGERR,bigfmt,args); + nch5breakpoint(stat); + +done: + va_end(args); + return stat; +} +#endif + +/*---------------------------------------------------------------------------- + * Function: curlwritecallback() + * Purpose: + * Function called by CURL to write received data. + * Writes bytes to `userdata`. + * Internally manages number of bytes processed. + * Return: + * - Number of bytes processed. + * - Should equal number of bytes passed to callback. + * - Failure will result in curl error: CURLE_WRITE_ERROR. + * Programmer: Jacob Smith + * 2017-08-17 + *---------------------------------------------------------------------------- + */ +static size_t +curlwritecallback(char *ptr, size_t size, size_t nmemb, void *userdata) +{ + struct s3r_cbstruct *sds = (struct s3r_cbstruct *)userdata; + size_t product = (size * nmemb); + size_t written = 0; + + if (sds->magic != S3COMMS_CALLBACK_STRUCT_MAGIC) + return written; + + if (product > 0) { + vsappendn(sds->data,ptr,product); + written = product; + } + + return written; +} /* end curlwritecallback() */ + +/*---------------------------------------------------------------------------- + * Function: curlreadcallback() + * Purpose: + * Function called by CURL to write PUT data. + * Reads bytes from `userdata`. + * Internally manages number of bytes processed. + * Return: + * - Number of bytes processed. + * - Should equal number of bytes passed to callback. + * - Failure will result in curl error: CURLE_WRITE_ERROR. + * Programmer: Dennis Heimbigner + *---------------------------------------------------------------------------- + */ +static size_t +curlreadcallback(char *ptr, size_t size, size_t nmemb, void *userdata) +{ + struct s3r_cbstruct *sds = (struct s3r_cbstruct *)userdata; + size_t product = (size * nmemb); + size_t written = 0; + size_t avail = 0; + size_t towrite = 0; + + if (sds->magic != S3COMMS_CALLBACK_STRUCT_MAGIC) + return CURL_READFUNC_ABORT; + + avail = (vslength(sds->data) - sds->pos); + towrite = (product > avail ? avail : product); + if (towrite > 0) { + const char* data = vscontents(sds->data); + memcpy(ptr,&data[sds->pos],towrite); + } + sds->pos += towrite; + written = towrite; + + return written; +} /* end curlreadcallback() */ + +/*---------------------------------------------------------------------------- + * Function: curlheadercallback() + * Purpose: + * Function called by CURL to write headers. + * Writes target header line to value field; + * Internally manages number of bytes processed. + * Return: + * - Number of bytes processed. + * - Should equal number of bytes passed to callback. + * - Failure will result in curl error: CURLE_WRITE_ERROR. + * Programmer: Dennis Heimbigner + * 2017-08-17 + *---------------------------------------------------------------------------- + */ +static size_t +curlheadercallback(char *ptr, size_t size, size_t nmemb, void *userdata) +{ + struct s3r_cbstruct *sds = (struct s3r_cbstruct *)userdata; + size_t len = (size * nmemb); + char* line = ptr; + size_t i,j; + + if (sds->magic != S3COMMS_CALLBACK_STRUCT_MAGIC) + return 0; + if(vslength(sds->data) > 0) + goto done; /* already found */ + + /* skip leading white space */ + for(j=0,i=0;ikey && strncasecmp(line,sds->key,strlen(sds->key)) == 0) { + vsappendn(sds->data,line,len); + } + +done: + return size * nmemb; + +} /* end curlwritecallback() */ + +/*---------------------------------------------------------------------------- + * Function: NCH5_s3comms_hrb_node_insert() + * Purpose: + * Insert elements in a field node list. + * `name` cannot be null; will return FAIL and list will be unaltered. + * Entries are accessed via the lowercase representation of their name: + * "Host", "host", and "hOSt" would all access the same node, + * but name's case is relevant in HTTP request output. + *---------------------------------------------------------------------------- + */ + +int +NCH5_s3comms_hrb_node_insert(VList* list, const char *name, const char *value) +{ + size_t i = 0; + int ret,ret_value = SUCCEED; + size_t catlen, namelen; + size_t catwrite; + char* lowername = NULL; + char* nvcat = NULL; + hrb_node_t* new_node = NULL; + +#if S3COMMS_DEBUG_HRB + fprintf(stdout, "called NCH5_s3comms_hrb_node_insert."); + printf("NAME: %s\n", name); + printf("VALUE: %s\n", value); + printf("LIST:\n->"); + dumphrbnodes(list); + fflush(stdout); +#endif + + if (name == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "unable to operate on null name"); + namelen = nulllen(name); + + /* get lowercase name */ + lowername = (char *)malloc(sizeof(char) * (namelen + 1)); + if (lowername == NULL) + HGOTO_ERROR(H5E_RESOURCE, NC_ENOMEM, FAIL, "cannot make space for lowercase name copy."); + for (i = 0; i < namelen; i++) + lowername[i] = (char)tolower((int)name[i]); + lowername[namelen] = 0; + + if(value == NULL) value = ""; + + /* create new_node */ + new_node = (hrb_node_t *)calloc(1,sizeof(hrb_node_t)); + if (new_node == NULL) + HGOTO_ERROR(H5E_RESOURCE, NC_ENOMEM, FAIL, "cannot make space for new set."); + new_node->magic = S3COMMS_HRB_NODE_MAGIC; + new_node->name = strdup(name); + new_node->value = strdup(value); + + catlen = namelen + strlen(value) + 2; /* +2 from ": " */ + catwrite = catlen + 3; /* 3 not 1 to quiet compiler warning */ + nvcat = (char *)malloc(catwrite); + if (nvcat == NULL) + HGOTO_ERROR(H5E_RESOURCE, NC_ENOMEM, FAIL, "cannot make space for concatenated string."); + ret = snprintf(nvcat, catwrite, "%s: %s", lowername, value); + if (ret < 0 || (size_t)ret > catlen) + HGOTO_ERRORVA(H5E_ARGS, NC_EINVAL, FAIL, "cannot concatenate `%s: %s", name, value); + assert(catlen == nulllen(nvcat)); + new_node->cat = nvcat; nvcat = NULL; + + new_node->lowername = lowername; lowername = NULL; + + vlistpush(list,new_node); new_node = NULL; + +done: + /* clean up */ + if (nvcat != NULL) free(nvcat); + if (lowername != NULL) free(lowername); + hrb_node_free(new_node); + return (ret_value); +} + + +/*---------------------------------------------------------------------------- + * Function: NCH5_s3comms_hrb_destroy() + * Purpose: + * Destroy and free resources _directly_ associated with an HTTP Buffer. + * Takes a pointer to pointer to the buffer structure. + * This allows for the pointer itself to be NULLed from within the call. + * If buffer or buffer pointer is NULL, there is no effect. + * Headers list at `first_header` is not touched. + * - Programmer should re-use or destroy `first_header` pointer + * (hrb_node_t *) as suits their purposes. + * - Recommend fetching prior to destroy() + * e.g., `reuse_node = hrb_to_die->first_header; destroy(hrb_to_die);` + * or maintaining an external reference. + * - Destroy node/list separately as appropriate + * - Failure to account for this will result in a memory leak. + * Return: + * - SUCCESS: `SUCCEED` + * - successfully released buffer resources + * - if `buf` is NULL or `*buf` is NULL, no effect + * - FAILURE: `FAIL` + * - `buf->magic != S3COMMS_HRB_MAGIC` + * Programmer: Jacob Smith + * 2017-07-21 + *---------------------------------------------------------------------------- + */ +int +NCH5_s3comms_hrb_destroy(hrb_t *buf) +{ + int ret_value = SUCCEED; + size_t i; + +#if S3COMMS_DEBUG_TRACE + fprintf(stdout, "called NCH5_s3comms_hrb_destroy.\n"); +#endif + + if(buf == NULL) return ret_value; + + if (buf->magic != S3COMMS_HRB_MAGIC) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "pointer's magic does not match."); + free(buf->version); + free(buf->resource); + buf->magic += 1ul; + vsfree(buf->body); + for(i=0;iheaders);i++) { + hrb_node_t* node = (hrb_node_t*)vlistget(buf->headers,i); + hrb_node_free(node); + } + vlistfree(buf->headers); + free(buf); +done: + return (ret_value); +} /* end NCH5_s3comms_hrb_destroy() */ + +/*---------------------------------------------------------------------------- + * Function: NCH5_s3comms_hrb_init_request() + * Purpose: + * Create a new HTTP Request Buffer + * All non-null arguments should be null-terminated strings. + * If `verb` is NULL, defaults to "GET". + * If `http_version` is NULL, defaults to "HTTP/1.1". + * `resource` cannot be NULL; should be string beginning with slash + * character ('/'). + * All strings are copied into the structure, making them safe from + * modification in source strings. + * Return: + * - SUCCESS: pointer to new `hrb_t` + * - FAILURE: `NULL` + * Programmer: Jacob Smith + * 2017-07-21 + *---------------------------------------------------------------------------- + */ +hrb_t * +NCH5_s3comms_hrb_init_request(const char *_resource, const char *_http_version) +{ + hrb_t *request = NULL; + char *res = NULL; + size_t reslen = 0; + int ret_value = SUCCEED; + char *vrsn = NULL; + size_t vrsnlen = 0; + +#if S3COMMS_DEBUG_TRACE + fprintf(stdout, "called NCH5_s3comms_hrb_init_request.\n"); +#endif + + if (_resource == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "resource string cannot be null."); + + /* populate valid NULLs with defaults */ + if (_http_version == NULL) + _http_version = "HTTP/1.1"; + + /* malloc space for and prepare structure */ + request = (hrb_t *)malloc(sizeof(hrb_t)); + if (request == NULL) + HGOTO_ERROR(H5E_ARGS, NC_ENOMEM, NULL, "no space for request structure"); + request->magic = S3COMMS_HRB_MAGIC; + request->body = vsnew(); + request->headers = vlistnew(); + + /* malloc and copy strings for the structure */ + reslen = nulllen(_resource); + + if (_resource[0] == '/') { + res = (char *)malloc(sizeof(char) * (reslen + 1)); + if (res == NULL) + HGOTO_ERROR(H5E_ARGS, NC_ENOMEM, NULL, "no space for resource string"); + memcpy(res, _resource, (reslen + 1)); + } + else { + res = (char *)malloc(sizeof(char) * (reslen + 2)); + if (res == NULL) + HGOTO_ERROR(H5E_ARGS, NC_ENOMEM, NULL, "no space for resource string"); + *res = '/'; + memcpy((&res[1]), _resource, (reslen + 1)); + assert((reslen + 1) == nulllen(res)); + } /* end if (else resource string not starting with '/') */ + + vrsnlen = nulllen(_http_version) + 1; + vrsn = (char *)malloc(sizeof(char) * vrsnlen); + if (vrsn == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "no space for http-version string"); + strncpy(vrsn, _http_version, vrsnlen); + + /* place new copies into structure */ + request->resource = res; + request->version = vrsn; + +done: + /* if there is an error, clean up after ourselves */ + if (ret_value != SUCCEED) { + if (request != NULL) + free(request); + if (vrsn != NULL) + free(vrsn); + if (res != NULL) + free(res); + request = NULL; + } + + (void)(ret_value); + return request; +} /* end NCH5_s3comms_hrb_init_request() */ + +#if S3COMMS_DEBUG_HRB +static void +dumphrbnodes(VList* nodes) +{ + int i; + if(nodes != NULL) { + fprintf(stderr,"\tnodes={\n"); + for(i=0;iname,node->value); + } + fprintf(stderr,"\t}\n"); + } +} + +static void +dumphrb(hrb_t* hrb) +{ + fprintf(stderr,"hrb={\n"); + if(hrb != NULL) { + fprintf(stderr,"\tresource=%s\n",hrb->resource); + fprintf(stderr,"\tversion=%s\n",hrb->version); + fprintf(stderr,"\tbody=|%.*s|\n",(int)ncbyteslength(hrb->body),ncbytescontents(hrb->body)); + dumphrbnodes(hrb->headers); + } + fprintf(stderr,"}\n"); + +} +#endif + +static void +hrb_node_free(hrb_node_t *node) +{ + if(node != NULL) { + nullfree(node->name); + nullfree(node->value); + nullfree(node->cat); + nullfree(node->lowername); + free(node); + } +} + +/**************************************************************************** + * S3R FUNCTIONS + ****************************************************************************/ + +/*---------------------------------------------------------------------------- + * Function: NCH5_s3comms_s3r_close() + * Purpose: + * Close communications through given S3 Request Handle (`s3r_t`) + * and clean up associated resources. + * Return: + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * - fails if handle is null or has invalid magic number + * Programmer: Jacob Smith + * 2017-08-31 + *---------------------------------------------------------------------------- + */ +int +NCH5_s3comms_s3r_close(s3r_t *handle) +{ + int ret_value = SUCCEED; + + TRACE(0,"handle=%p",handle); + +#if S3COMMS_DEBUG_TRACE + fprintf(stdout, "called NCH5_s3comms_s3r_close.\n"); +#endif + + if (handle == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "handle cannot be null."); + if (handle->magic != S3COMMS_S3R_MAGIC) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "handle has invalid magic."); + + if(handle->curlheaders != NULL) { + curl_slist_free_all(handle->curlheaders); + handle->curlheaders = NULL; + } + curl_easy_cleanup(handle->curlhandle); + + nullfree(handle->rootpath); + nullfree(handle->region); + nullfree(handle->accessid); + nullfree(handle->accesskey); + nullfree(handle->reply); + nullfree(handle->signing_key); + free(handle); + +done: + return UNTRACE(ret_value); +} /* NCH5_s3comms_s3r_close */ + + +/*---------------------------------------------------------------------------- + * Function: NCH5_s3comms_s3r_getsize() + * Purpose: + * Get the number of bytes of handle's target resource. + * Sets handle and curlhandle with to enact an HTTP HEAD request on file, + * and parses received headers to extract "Content-Length" from response + * headers, storing file size at `handle->filesize`. + * Critical step in opening (initiating) an `s3r_t` handle. + * Wraps `s3r_read()`. + * Sets curlhandle to write headers to a temporary buffer (using extant + * write callback) and provides no buffer for body. + * Upon exit, unsets HTTP HEAD settings from curl handle, returning to + * initial state. In event of error, curl handle state is undefined and is + * not to be trusted. + * Return: + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * Programmer: Jacob Smith + * 2017-08-23 + *---------------------------------------------------------------------------- + */ +int +NCH5_s3comms_s3r_getsize(s3r_t *handle, const char* url, long long* sizep) +{ + int ret_value = SUCCEED; + char* contentlength = NULL; + char* value = NULL; + long long content_length = -1; + long httpcode = 0; + + TRACE(0,"handle=%p url=%s sizep=%p",handle,url,sizep); + +#if S3COMMS_DEBUG_TRACE + fprintf(stdout, "called NCH5_s3comms_s3r_getsize.\n"); +#endif + + if((ret_value = NCH5_s3comms_s3r_head(handle, url, "Content-Length", NULL, &httpcode, &contentlength))) + HGOTO_ERROR(H5E_ARGS, ret_value, FAIL, "NCH5_s3comms_s3r_head failed."); + + if((ret_value = httptonc(httpcode))) goto done; + + /****************** + * PARSE RESPONSE * + ******************/ + + value = strchr(contentlength,':'); + if(value == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "could not find content length value"); + value++; + content_length = strtoumax(value, NULL, 0); + if (UINTMAX_MAX > SIZE_MAX && content_length > SIZE_MAX) + HGOTO_ERROR(H5E_ARGS, NC_ERANGE, FAIL, "content_length overflows size_t"); + + if (errno == ERANGE) /* errno set by strtoumax*/ + HGOTO_ERRORVA(H5E_ARGS, NC_EINVAL, FAIL, + "could not convert found \"Content-Length\" response (\"%s\")", + contentlength); /* range is null-terminated, remember */ + + if(sizep) {*sizep = (long long)content_length;} + +done: + nullfree(contentlength); + return UNTRACEX(ret_value,"size=%lld",(sizep?-1:*sizep)); +} /* NCH5_s3comms_s3r_getsize */ + +/*---------------------------------------------------------------------------- + * Function: NCH5_s3comms_s3r_deletekey() + * Return: + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * Programmer: Dennis Heimbigner + *---------------------------------------------------------------------------- + */ +int +NCH5_s3comms_s3r_deletekey(s3r_t *handle, const char* url, long* httpcodep) +{ + int ret_value = SUCCEED; + VString* data = vsnew(); + long httpcode = 0; + + TRACE(0,"handle=%p url=%s httpcodep=%p",handle,url,httpcodep); + +#if S3COMMS_DEBUG_TRACE + fprintf(stdout, "called NCH5_s3comms_s3r_deletekey.\n"); +#endif + + /********************* + * Execute * + *********************/ + + if((ret_value = NCH5_s3comms_s3r_execute(handle, url, HTTPDELETE, NULL, NULL, NULL, &httpcode, data))) + HGOTO_ERROR(H5E_ARGS, ret_value, FAIL, "execute failed."); + + /****************** + * RESPONSE * + ******************/ + if((ret_value = httptonc(httpcode))) goto done; + if(httpcode != 204) + HGOTO_ERROR(H5E_ARGS, NC_ECANTREMOVE, FAIL, "deletekey failed."); + +done: + vsfree(data); + if(httpcodep) *httpcodep = httpcode; + return UNTRACEX(ret_value,"httpcode=%d",INULL(httpcodep)); +} /* NCH5_s3comms_s3r_getsize */ + +/*---------------------------------------------------------------------------- + * Function: NCH5_s3comms_s3r_head() + * Purpose: + * Generic HEAD request + * @param + * @return NC_NOERR if exists + * @return NC_EINVAL if not exits + * @return error otherwise + *---------------------------------------------------------------------------- + */ +int +NCH5_s3comms_s3r_head(s3r_t *handle, const char* url, const char* header, const char* query, long* httpcodep, char** valuep) +{ + int ret_value = SUCCEED; + VString* data = vsnew(); + long httpcode = 0; + + TRACE(0,"handle=%p url=%s header=%s query=%s httpcodep=%p valuep=%p",handle,url,SNULL(header),SNULL(query),httpcodep,valuep); + +#if S3COMMS_DEBUG_TRACE + fprintf(stdout, "called NCH5_s3comms_s3r_head.\n"); +#endif + + if (url == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "handle has bad (null) url."); + + if((ret_value = validate_handle(handle,url))) + HGOTO_ERROR(H5E_ARGS, ret_value, FAIL, "invalid handle."); + + /******************* + * PERFORM REQUEST * + *******************/ + + /* only http metadata will be sent by server and recorded by s3comms + */ + if (SUCCEED != NCH5_s3comms_s3r_execute(handle, url, HTTPHEAD, NULL, header, NULL, &httpcode, data)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "problem in reading during getsize."); + + if((ret_value = httptonc(httpcode))) goto done; + + if(header != NULL) { + if(vslength(data) == 0) + HGOTO_ERRORVA(H5E_ARGS, NC_EINVAL, FAIL, "HTTP metadata: header=%s; not found",header); + else if (vslength(data) > CURL_MAX_HTTP_HEADER) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "HTTP metadata buffer overrun"); +#if S3COMMS_DEBUG + else + fprintf(stderr, "HEAD: OK\n"); +#endif + } + + /****************** + * PARSE RESPONSE * + ******************/ + + if(header != NULL) { + char* content; + content = vsextract(data); + if(valuep) {*valuep = content;} + } + + /********************** + * UNDO HEAD SETTINGS * + **********************/ + + if((ret_value = curl_reset(handle))) + HGOTO_ERROR(H5E_ARGS, ret_value, FAIL, "error while re-setting CURL options."); + +done: + if(httpcodep) *httpcodep = httpcode; + vsfree(data); + return UNTRACEX(ret_value,"httpcodep=%d",INULL(httpcodep)); +} /* NCH5_s3comms_s3r_getsize */ + +/*---------------------------------------------------------------------------- + * Function: NCH5_s3comms_s3r_execute() + * Purpose: + * Execute an HTTP verb and optionally return the response. + * Uses configured "curl easy handle" to perform request. + * In event of error, buffer should remain unaltered. + * If handle is set to authorize a request, creates a new (temporary) + * HTTP Request object (hrb_t) for generating requisite headers, + * which is then translated to a `curl slist` and set in the curl handle + * for the request. + * Return: + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * Programmer: Dennis Heimbigner + *---------------------------------------------------------------------------- + */ +/* TODO: Need to simplify this signature; it is too long */ +static int +NCH5_s3comms_s3r_execute(s3r_t *handle, const char* url, + HTTPVerb verb, + const char* range, + const char* searchheader, + const char** otherheaders, + long* httpcodep, + VString* data) +{ + int ret_value = SUCCEED; + NCURI* purl= NULL; + struct s3r_cbstruct sds = {S3COMMS_CALLBACK_STRUCT_MAGIC, NULL, NULL, 0}; + long httpcode = 0; + +#ifdef DEBUG + printf(">>> NCH5_s3comms_s3r_execute(url=%s verb=%s range=%s searchheader=%s)\n",url,verbtext(verb),SNULL(range),SNULL(searchheader)); + fflush(stdout); +#endif + +#if S3COMMS_DEBUG_TRACE + fprintf(stdout, "called NCH5_s3comms_s3r_execute.\n"); +#endif + + /************************************** + * ABSOLUTELY NECESSARY SANITY-CHECKS * + **************************************/ + + if((ret_value = validate_handle(handle, url))) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "invalid handle."); + + ncuriparse(url,&purl); + if((ret_value = validate_url(purl))) + HGOTO_ERRORVA(H5E_ARGS, NC_EINVAL, FAIL, "unparseable url: %s", url); + + /********************* + * Setup * + *********************/ + + sds.data = data; + if (verb == HTTPHEAD) + sds.key = searchheader; + + /******************* + * COMPILE REQUEST * + *******************/ + + if((ret_value = build_request(handle,purl,range,otherheaders,data,verb))) + HGOTO_ERROR(H5E_ARGS, ret_value, FAIL, "unable to build request."); + + /********************* + * PREPARE CURL + *********************/ + + if((ret_value = request_setup(handle, url, verb, &sds))) + HGOTO_ERROR(H5E_ARGS, ret_value, FAIL, "read_request_setup failed."); + + /******************* + * PERFORM REQUEST * + *******************/ + + if((ret_value = perform_request(handle,&httpcode))) + HGOTO_ERROR(H5E_ARGS, ret_value, FAIL, "unable perform request."); + +done: + if(httpcodep) *httpcodep = httpcode; + ncurifree(purl); + /* clean any malloc'd resources */ + curl_reset(handle); + return (ret_value);; +} /* NCH5_s3comms_s3r_read */ + +/*---------------------------------------------------------------------------- + * Function: NCH5_s3comms_s3r_open() + * Purpose: + * Logically 'open' a file hosted on S3. + * - create new Request Handle + * - copy supplied url + * - copy authentication info if supplied + * - create CURL handle + * - fetch size of file + * - connect with server and execute HEAD request + * - return request handle ready for reads + * To use 'default' port to connect, `port` should be 0. + * To prevent AWS4 authentication, pass null pointer to `region`, `id`, + * and `signing_key`. + * Uses `NCH5_s3comms_parse_url()` to validate and parse url input. + * Return: + * - SUCCESS: Pointer to new request handle. + * - FAILURE: NULL + * - occurs if: + * - authentication strings are inconsistent + * - must _all_ be null, or have at least `region` and `id` + * - url is NULL (no filename) + * - unable to parse url (malformed?) + * - error while performing `getsize()` + * Programmer: Jacob Smith + * 2017-09-01 + *---------------------------------------------------------------------------- + */ +s3r_t * +NCH5_s3comms_s3r_open(const char* root, const char *region, const char *access_id, const char* access_key) +{ + int ret_value = SUCCEED; + size_t tmplen = 0; + CURL *curlh = NULL; + s3r_t *handle = NULL; + unsigned char *signing_key = NULL; + char iso8601now[ISO8601_SIZE]; + struct tm *now = NULL; + + TRACE(0,"root=%s region=%s access_id=%s access_key=%s",root,region,access_id,access_key); + +#if S3COMMS_DEBUG_TRACE + fprintf(stdout, "called NCH5_s3comms_s3r_open.\n"); +#endif + + /* setup */ + iso8601now[0] = '\0'; + + handle = (s3r_t *)calloc(1,sizeof(s3r_t)); + if (handle == NULL) + HGOTO_ERROR(H5E_ARGS, NC_ENOMEM, NULL, "could not malloc space for handle."); + + handle->magic = S3COMMS_S3R_MAGIC; + + /************************************* + * RECORD THE ROOT PATH + *************************************/ + + /* Verify that the region is a substring of root */ + if(region != NULL && region[0] != '\0') { + if(strstr(root,region) == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "region not present in root path."); + } + handle->rootpath = nulldup(root); + + /************************************* + * RECORD AUTHENTICATION INFORMATION * + *************************************/ + + /* copy strings */ + if(nulllen(region) != 0) { + tmplen = nulllen(region) + 1; + handle->region = (char *)malloc(sizeof(char) * tmplen); + if (handle->region == NULL) + HGOTO_ERROR(H5E_ARGS, NC_ENOMEM, NULL, "could not malloc space for handle region copy."); + memcpy(handle->region, region, tmplen); + } + + if(nulllen(access_id) != 0) { + tmplen = nulllen(access_id) + 1; + handle->accessid = (char *)malloc(sizeof(char) * tmplen); + if (handle->accessid == NULL) + HGOTO_ERROR(H5E_ARGS, NC_ENOMEM, NULL, "could not malloc space for handle ID copy."); + memcpy(handle->accessid, access_id, tmplen); + } + + if(nulllen(access_key) != 0) { + tmplen = nulllen(access_key) + 1; + handle->accesskey = (char *)malloc(sizeof(char) * tmplen); + if (handle->accesskey == NULL) + HGOTO_ERROR(H5E_ARGS, NC_ENOMEM, NULL, "could not malloc space for handle access key copy."); + memcpy(handle->accesskey, access_key, tmplen); + } + + now = gmnow(); + if (ISO8601NOW(iso8601now, now) != (ISO8601_SIZE - 1)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "unable to get current time"); + memcpy(handle->iso8601now,iso8601now,ISO8601_SIZE); + + /* Do optional authentication */ + if(access_id != NULL && access_key != NULL) { /* We are authenticating */ + /* Need several pieces of info for authentication */ + if (nulllen(handle->region) == 0) + HGOTO_ERROR(H5E_ARGS, NC_EAUTH, NULL, "region cannot be null."); + if (nulllen(handle->accessid)==0) + HGOTO_ERROR(H5E_ARGS, NC_EAUTH, NULL, "access id cannot be null."); + if (nulllen(handle->accesskey)==0) + HGOTO_ERROR(H5E_ARGS, NC_EAUTH, NULL, "signing key cannot be null."); + + /* Compute the signing key */ + if (SUCCEED != NCH5_s3comms_signing_key(&signing_key, access_key, region, iso8601now)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "problem in NCH5_s3comms_s3comms_signing_key."); + if (nulllen(signing_key)==0) + HGOTO_ERROR(H5E_ARGS, NC_EAUTH, NULL, "signing key cannot be null."); + handle->signing_key = signing_key; + signing_key = NULL; + + } /* if authentication information provided */ + + /************************ + * INITIATE CURL HANDLE * + ************************/ + + curlh = curl_easy_init(); + if (curlh == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "problem creating curl easy handle!"); + + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "error while setting CURL option (CURLOPT_HTTP_VERSION)."); + + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_FAILONERROR, 1L)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "error while setting CURL option (CURLOPT_FAILONERROR)."); + + handle->curlhandle = curlh; + + /********************* + * FINAL PREPARATION * + *********************/ + + assert(handle->httpverb != NULL); + strcpy(handle->httpverb, "GET"); + +done: + nullfree(signing_key); + if (ret_value != SUCCEED) { + if (curlh != NULL) + curl_easy_cleanup(curlh); + if (handle != NULL) { + if(handle->region != NULL) free(handle->region); + if(handle->accessid != NULL) free(handle->accessid); + if(handle->accesskey != NULL) free(handle->accesskey); + if(handle->rootpath != NULL) free(handle->rootpath); + free(handle); + handle = NULL; + } + } + + (void)UNTRACEX(ret_value,"handle=%p",handle); + return handle; +} /* NCH5_s3comms_s3r_open */ + +/*---------------------------------------------------------------------------- + * Function: NCH5_s3comms_s3r_read() + * Purpose: + * Read file pointed to by request handle. + * Optionally specify byterange of `offset` .. `offset + len` bytes to buffer `dest`. + * In event of error, buffer should remain unaltered. + * If handle is set to authorize a request, creates a new (temporary) + * HTTP Request object (hrb_t) for generating requisite headers, + * which is then translated to a `curl slist` and set in the curl handle + * for the request. + * `dest` _may_ be NULL, but no body data will be recorded. + * Return: + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * Programmer: Jacob Smith + * 2017-08-22 + *---------------------------------------------------------------------------- + */ +int +NCH5_s3comms_s3r_read(s3r_t *handle, const char* url, size_t offset, size_t len, s3r_buf_t* dest) +{ + char *rangebytesstr = NULL; + int ret_value = SUCCEED; + long httpcode; + VString *wrap = vsnew(); + + TRACE(0,"handle=%p url=%s offset=%ld len=%ld, dest=%p",handle,url,(long)offset,(long)len,dest); + +#if S3COMMS_DEBUG_TRACE + fprintf(stdout, "called NCH5_s3comms_s3r_read.\n"); +#endif + + /********************* + * FORMAT HTTP RANGE * + *********************/ + + if((ret_value = build_range(offset,len,&rangebytesstr))) + HGOTO_ERROR(H5E_ARGS, ret_value, FAIL, "build_range failed."); + + /********************* + * Execute * + *********************/ + + vssetcontents(wrap,dest->content,dest->count); + vssetlength(wrap,0); + + if((ret_value = NCH5_s3comms_s3r_execute(handle, url, HTTPGET, rangebytesstr, NULL, NULL, &httpcode, wrap))) + HGOTO_ERROR(H5E_ARGS, ret_value, FAIL, "execute failed."); + if((ret_value = httptonc(httpcode))) goto done; + +done: + (void)vsextract(wrap); + vsfree(wrap); + /* clean any malloc'd resources */ + nullfree(rangebytesstr); + curl_reset(handle); + return UNTRACE(ret_value);; +} /* NCH5_s3comms_s3r_read */ + +/*---------------------------------------------------------------------------- + * Function: NCH5_s3comms_s3r_write() + * Return: + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * Programmer: Dennis Heimbigner + *---------------------------------------------------------------------------- + */ +int +NCH5_s3comms_s3r_write(s3r_t *handle, const char* url, const s3r_buf_t* data) +{ + int ret_value = SUCCEED; + VList* otherheaders = vlistnew(); + char digits[64]; + long httpcode = 0; + VString* wrap = vsnew(); + + TRACE(0,"handle=%p url=%s |data|=%d",handle,url,data->count); + + snprintf(digits,sizeof(digits),"%llu",(unsigned long long)data->count); + + vlistpush(otherheaders,strdup("Content-Length")); + vlistpush(otherheaders,strdup(digits)); + vlistpush(otherheaders,strdup("Content-Type")); + vlistpush(otherheaders,strdup("binary/octet-stream")); + vlistpush(otherheaders,NULL); + +#if S3COMMS_DEBUG_TRACE + fprintf(stdout, "called NCH5_s3comms_s3r_write.\n"); +#endif + + /********************* + * Execute * + *********************/ + + vssetcontents(wrap,data->content,data->count); + vssetlength(wrap,data->count); + if((ret_value = NCH5_s3comms_s3r_execute(handle, url, HTTPPUT, NULL, NULL, (const char**)vlistcontents(otherheaders), &httpcode, wrap))) + HGOTO_ERROR(H5E_ARGS, ret_value, FAIL, "execute failed."); + if((ret_value = httptonc(httpcode))) goto done; + +done: + (void)vsextract(wrap); + vsfree(wrap); + /* clean any malloc'd resources */ + vlistfreeall(otherheaders); + curl_reset(handle); + return UNTRACE(ret_value); +} /* NCH5_s3comms_s3r_write */ + +/*---------------------------------------------------------------------------- + * Function: NCH5_s3comms_s3r_getkeys() + * Return: + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * Programmer: Dennis Heimbigner + *---------------------------------------------------------------------------- + */ +int +NCH5_s3comms_s3r_getkeys(s3r_t *handle, const char* url, s3r_buf_t* response) +{ + int ret_value = SUCCEED; + const char* otherheaders[3] = {"Content-Type", "application/xml", NULL}; + long httpcode = 0; + VString* content = vsnew(); + + TRACE(0,"handle=%p url=%s response=%p",handle,url,response->content); + +#if S3COMMS_DEBUG_TRACE + fprintf(stdout, "called NCH5_s3comms_s3r_getkeys.\n"); +#endif + + /********************* + * Execute * + *********************/ + + if((SUCCEED != NCH5_s3comms_s3r_execute(handle, url, HTTPGET, NULL, NULL, otherheaders, &httpcode, content))) + HGOTO_ERROR(H5E_ARGS, ret_value, FAIL, "execute failed."); + if((ret_value = httptonc(httpcode))) goto done; + if(response) { + response->count = vslength(content); + response->content = vsextract(content); + } + +done: + vsfree(content); + /* clean any malloc'd resources */ + curl_reset(handle); + return UNTRACEX(ret_value,"response=[%d]",ncbyteslength(response)); +} /* NCH5_s3comms_s3r_getkeys */ + +/**************************************************************************** + * MISCELLANEOUS FUNCTIONS + ****************************************************************************/ + +/*---------------------------------------------------------------------------- + * Function: gmnow() + * Purpose: + * Get the output of `time.h`'s `gmtime()` call while minimizing setup + * clutter where important. + * Return: + * Pointer to resulting `struct tm`,as created by gmtime(time_t * T). + * Programmer: Jacob Smith + * 2017-07-12 + *---------------------------------------------------------------------------- + */ +struct tm * +gmnow(void) +{ + time_t now; + time_t *now_ptr = &now; + struct tm *ret_value = NULL; + + /* Doctor assert, checks against error in time() */ + if ((time_t)(-1) != time(now_ptr)) + ret_value = gmtime(now_ptr); + + assert(ret_value != NULL); + + return ret_value; +} /* end gmnow() */ + +/*---------------------------------------------------------------------------- + * Function: NCH5_s3comms_aws_canonical_request() + * Purpose: + * Compose AWS "Canonical Request" (and signed headers string) + * as defined in the REST API documentation. + * Both destination strings are null-terminated. + * Destination string arguments must be provided with adequate space. + * Canonical Request format: + * "\n" + * "\n" + * "\n" + * "\n" (`lowercase(name)`":"`trim(value)`) + * "\n" + * ... (headers sorted by name) + * "\n" + * "\n" + * "\n" (`lowercase(header 1 name)`";"`header 2 name`;...) + * ("e3b0c4429...", e.g.) + * Return: + * - SUCCESS: `SUCCEED` + * - writes canonical request to respective `...dest` strings + * - FAILURE: `FAIL` + * - one or more input argument was NULL + * - internal error + * Programmer: Jacob Smith + * 2017-10-04 + *---------------------------------------------------------------------------- + */ +int +NCH5_s3comms_aws_canonical_request(VString* canonical_request_dest, VString* signed_headers_dest, + HTTPVerb verb, + const char* query, + const char* payloadsha256, + hrb_t *http_request) +{ + hrb_node_t *node = NULL; + int ret_value = SUCCEED; + int i; + + const char* sverb = verbtext(verb); + const char* query_params = (query?query:""); + + /* "query params" refers to the optional element in the URL, e.g. + * http://bucket.aws.com/myfile.txt?max-keys=2&prefix=J + * ^-----------------^ + * Not handled/implemented as of 2017-10-xx. + * Element introduced as empty placeholder and reminder. + * Further research to be done if this is ever relevant for the + * VFD use-cases. + */ + +#if S3COMMS_DEBUG_TRACE + fprintf(stdout, "called NCH5_s3comms_aws_canonical_request.\n"); +#endif + + if (http_request == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "hrb object cannot be null."); + assert(http_request->magic == S3COMMS_HRB_MAGIC); + + if (canonical_request_dest == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "canonical request destination cannot be null."); + + if (signed_headers_dest == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "signed headers destination cannot be null."); + + /* HTTP verb, resource path, and query string lines */ + vscat(canonical_request_dest,sverb); + vscat(canonical_request_dest,"\n"); + vscat(canonical_request_dest,http_request->resource); + vscat(canonical_request_dest,"\n"); + vscat(canonical_request_dest,query_params); + vscat(canonical_request_dest,"\n"); + + /* write in canonical headers, building signed headers concurrently */ + for(i=0;iheaders);i++) { + node = (hrb_node_t*)vlistget(http_request->headers,i); /* assumed sorted */ + if(i>0) vscat(signed_headers_dest,";"); + assert(node->magic == S3COMMS_HRB_NODE_MAGIC); + vscat(canonical_request_dest,node->lowername); + vscat(canonical_request_dest,":"); + vscat(canonical_request_dest,node->value); + vscat(canonical_request_dest,"\n"); + vscat(signed_headers_dest,node->lowername); + } /* end while node is not NULL */ + + /* append signed headers and payload hash + * NOTE: at present, no HTTP body is handled, per the nature of + * requests/range-gets + */ + vscat(canonical_request_dest, "\n"); + vscat(canonical_request_dest, vscontents(signed_headers_dest)); + vscat(canonical_request_dest, "\n"); + vscat(canonical_request_dest, payloadsha256); + +done: + return (ret_value); +} /* end NCH5_s3comms_aws_canonical_request() */ + +/*---------------------------------------------------------------------------- + * Function: NCH5_s3comms_bytes_to_hex() + * Purpose: + * Produce human-readable hex string [0-9A-F] from sequence of bytes. + * For each byte (char), writes two-character hexadecimal representation. + * No null-terminator appended. + * Assumes `dest` is allocated to enough size (msg_len * 2). + * Fails if either `dest` or `msg` are null. + * `msg_len` message length of 0 has no effect. + * Return: + * - SUCCESS: `SUCCEED` + * - hex string written to `dest` (not null-terminated) + * - FAILURE: `FAIL` + * - `dest == NULL` + * - `msg == NULL` + * Programmer: Jacob Smith + * 2017-07-12 + *---------------------------------------------------------------------------- + */ +int +NCH5_s3comms_bytes_to_hex(char *dest, const unsigned char *msg, size_t msg_len, int lowercase) +{ + size_t i = 0; + int ret_value = SUCCEED; + +#if S3COMMS_DEBUG_TRACE + fprintf(stdout, "called NCH5_s3comms_bytes_to_hex.\n"); +#endif + + if (dest == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "hex destination cannot be null."); + if (msg == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "bytes sequence cannot be null."); + + for (i = 0; i < msg_len; i++) { + int chars_written = snprintf(&(dest[i * 2]), 3, /* 'X', 'X', '\n' */ + (lowercase == TRUE) ? "%02x" : "%02X", msg[i]); + if (chars_written != 2) + HGOTO_ERRORVA(H5E_ARGS, NC_EINVAL, FAIL, "problem while writing hex chars for %c", msg[i]); + } + +done: + return (ret_value); +} /* end NCH5_s3comms_bytes_to_hex() */ + +/*---------------------------------------------------------------------------- + * Function: NCH5_s3comms_HMAC_SHA256() + * Purpose: + * Generate Hash-based Message Authentication Checksum using the SHA-256 + * hashing algorithm. + * Given a key, message, and respective lengths (to accommodate null + * characters in either), generate _hex string_ of authentication checksum + * and write to `dest`. + * `dest` must be at least `SHA256_DIGEST_LENGTH * 2` characters in size. + * Not enforceable by this function. + * `dest` will _not_ be null-terminated by this function. + * Return: + * - SUCCESS: `SUCCEED` + * - hex string written to `dest` (not null-terminated) + * - FAILURE: `FAIL` + * - `dest == NULL` + * - error while generating hex string output + * Programmer: Jacob Smith + * 2017-07-?? + *---------------------------------------------------------------------------- + */ +int +NCH5_s3comms_HMAC_SHA256(const unsigned char *key, size_t key_len, const char *msg, size_t msg_len, + char *dest) +{ + unsigned char md[SHA256_DIGEST_LENGTH]; + unsigned int md_len = SHA256_DIGEST_LENGTH; + int ret_value = SUCCEED; + +#if S3COMMS_DEBUG_TRACE + fprintf(stdout, "called NCH5_s3comms_HMAC_SHA256.\n"); +#endif + + if (dest == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "destination cannot be null."); + +#if 0 + HMAC(EVP_sha256(), key, (int)key_len, (const unsigned char *)msg, msg_len, md, &md_len); +#else + if(CURLE_OK != Curl_hmacit(Curl_HMAC_SHA256, + key, key_len, + msg, msg_len, + md)) + HGOTO_ERROR(H5E_ARGS, NC_EINTERNAL, FAIL, "Curl_hmacit failure."); +#endif + + if (NCH5_s3comms_bytes_to_hex(dest, (const unsigned char *)md, (size_t)md_len, TRUE) != SUCCEED) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "could not convert to hex string."); + +done: + return (ret_value); +} /* NCH5_s3comms_HMAC_SHA256 */ + +/*----------------------------------------------------------------------------- + * Function: H5FD__s3comms_load_aws_creds_from_file() + * Purpose: + * Extract AWS configuration information from a target file. + * Given a file and a profile name, e.g. "ros3_vfd_test", attempt to locate + * that region in the file. If not found, returns in error and output + * pointers are not modified. + * If the profile label is found, attempts to locate and parse configuration + * data, stopping at the first line where: + * + reached end of file + * + line does not start with a recognized setting name + * Following AWS documentation, looks for any of: + * + aws_access_key_id + * + aws_secret_access_key + * + region + * To be valid, the setting must begin the line with one of the keywords, + * followed immediately by an equals sign '=', and have some data before + * newline at end of line. + * + `spam=eggs` would be INVALID because name is unrecognized + * + `region = us-east-2` would be INVALID because of spaces + * + `region=` would be INVALID because no data. + * Upon successful parsing of a setting line, will store the result in the + * corresponding output pointer. If the output pointer is NULL, will skip + * any matching setting line while parsing -- useful to prevent overwrite + * when reading from multiple files. + * Return: + * + SUCCESS: `SUCCEED` + * + no error. settings may or may not have been loaded. + * + FAILURE: `FAIL` + * + internal error occurred. + * + -1 :: unable to format profile label + * + -2 :: profile name/label not found in file + * + -3 :: some other error + * Programmer: Jacob Smith + * 2018-02-27 + *----------------------------------------------------------------------------- + */ +static int +H5FD__s3comms_load_aws_creds_from_file(FILE *file, const char *profile_name, char *key_id, char *access_key, + char *aws_region) +{ + char profile_line[32]; + char buffer[128]; + const char *setting_names[] = { + "region", + "aws_access_key_id", + "aws_secret_access_key", + }; + char *const setting_pointers[] = { + aws_region, + key_id, + access_key, + }; + unsigned setting_count = 3; + int ret_value = SUCCEED; + unsigned buffer_i = 0; + unsigned setting_i = 0; + int found_setting = 0; + char *line_buffer = &(buffer[0]); + +#if S3COMMS_DEBUG_TRACE + fprintf(stdout, "called load_aws_creds_from_file.\n"); +#endif + + /* format target line for start of profile */ + if (32 < snprintf(profile_line, 32, "[%s]", profile_name)) + HGOTO_ERROR(H5E_ARGS, NC_EINTERNAL, FAIL, "unable to format profile label"); + + /* look for start of profile */ + do { + /* clear buffer */ + for (buffer_i = 0; buffer_i < 128; buffer_i++) + buffer[buffer_i] = 0; + + line_buffer = fgets(line_buffer, 128, file); + if (line_buffer == NULL) /* reached end of file */ + goto done; + } while (strncmp(line_buffer, profile_line, nulllen(profile_line))); + + /* extract credentials from lines */ + do { + /* clear buffer */ + for (buffer_i = 0; buffer_i < 128; buffer_i++) + buffer[buffer_i] = 0; + + /* collect a line from file */ + line_buffer = fgets(line_buffer, 128, file); + if (line_buffer == NULL) + goto done; /* end of file */ + + /* loop over names to see if line looks like assignment */ + for (setting_i = 0; setting_i < setting_count; setting_i++) { + size_t setting_name_len = 0; + const char *setting_name = NULL; + char line_prefix[128]; + + setting_name = setting_names[setting_i]; + setting_name_len = nulllen(setting_name); + if (snprintf(line_prefix, 128, "%s=", setting_name) < 0) + HGOTO_ERROR(H5E_ARGS, NC_EINTERNAL, FAIL, "unable to format line prefix"); + + /* found a matching name? */ + if (!strncmp(line_buffer, line_prefix, setting_name_len + 1)) { + found_setting = 1; + + /* skip NULL destination buffer */ + if (setting_pointers[setting_i] == NULL) + break; + + /* advance to end of name in string */ + do { + line_buffer++; + } while (*line_buffer != 0 && *line_buffer != '='); + + if (*line_buffer == 0 || *(line_buffer + 1) == 0) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "incomplete assignment in file"); + line_buffer++; /* was pointing at '='; advance */ + + /* copy line buffer into out pointer */ + strncpy(setting_pointers[setting_i], (const char *)line_buffer, nulllen(line_buffer)); + + /* "trim" tailing whitespace by replacing with null terminator*/ + buffer_i = 0; + while (!isspace(setting_pointers[setting_i][buffer_i])) + buffer_i++; + setting_pointers[setting_i][buffer_i] = '\0'; + + break; /* have read setting; don't compare with others */ + } /* end if possible name match */ + } /* end for each setting name */ + } while (found_setting); + +done: + return (ret_value); +} /* end H5FD__s3comms_load_aws_creds_from_file() */ + +/*---------------------------------------------------------------------------- + * Function: NCH5_s3comms_load_aws_profile() + * Purpose : + * Read aws profile elements from standard location on system and store + * settings in memory. + * Looks for both `~/.aws/config` and `~/.aws/credentials`, the standard + * files for AWS tools. If a file exists (can be opened), looks for the + * given profile name and reads the settings into the relevant buffer. + * Any setting duplicated in both files will be set to that from + * `credentials`. + * Settings are stored in the supplied buffers as null-terminated strings. + * Return: + * + SUCCESS: `SUCCEED` (0) + * + no error occurred and all settings were populated + * + FAILURE: `FAIL` (-1) + * + internal error occurred + * + unable to locate profile + * + region, key id, and secret key were not all found and set + * Programmer: Jacob Smith + * 2018-02-27 + *---------------------------------------------------------------------------- + */ +int +NCH5_s3comms_load_aws_profile(const char *profile_name, char *key_id_out, char *secret_access_key_out, + char *aws_region_out) +{ + int ret_value = SUCCEED; + FILE *credfile = NULL; + char awspath[117]; + char filepath[128]; + int ret = 0; + +#if S3COMMS_DEBUG_TRACE + fprintf(stdout, "called NCH5_s3comms_load_aws_profile.\n"); +#endif + +#ifdef H5_HAVE_WIN32_API + ret = snprintf(awspath, 117, "%s/.aws/", getenv("USERPROFILE")); +#else + ret = snprintf(awspath, 117, "%s/.aws/", getenv("HOME")); +#endif + if (ret < 0 || (size_t)ret >= 117) + HGOTO_ERROR(H5E_ARGS, NC_EINTERNAL, FAIL, "unable to format home-aws path"); + ret = snprintf(filepath, 128, "%s%s", awspath, "credentials"); + if (ret < 0 || (size_t)ret >= 128) + HGOTO_ERROR(H5E_ARGS, NC_EINTERNAL, FAIL, "unable to format credentials path"); + + credfile = fopen(filepath, "r"); + if (credfile != NULL) { + if (H5FD__s3comms_load_aws_creds_from_file(credfile, profile_name, key_id_out, secret_access_key_out, + aws_region_out) != SUCCEED) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "unable to load from aws credentials"); + if (fclose(credfile) == EOF) + HGOTO_ERROR(H5E_FILE, NC_EACCESS, FAIL, "unable to close credentials file"); + credfile = NULL; + } /* end if credential file opened */ + + ret = snprintf(filepath, 128, "%s%s", awspath, "config"); + if (ret < 0 || (size_t)ret >= 128) + HGOTO_ERROR(H5E_ARGS, NC_EINTERNAL, FAIL, "unable to format config path"); + credfile = fopen(filepath, "r"); + if (credfile != NULL) { + if (H5FD__s3comms_load_aws_creds_from_file( + credfile, profile_name, (*key_id_out == 0) ? key_id_out : NULL, + (*secret_access_key_out == 0) ? secret_access_key_out : NULL, + (*aws_region_out == 0) ? aws_region_out : NULL) != SUCCEED) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "unable to load from aws config"); + if (fclose(credfile) == EOF) + HGOTO_ERROR(H5E_FILE, NC_EACCESS, FAIL, "unable to close config file"); + credfile = NULL; + } /* end if credential file opened */ + + /* fail if not all three settings were loaded */ + if (*key_id_out == 0 || *secret_access_key_out == 0 || *aws_region_out == 0) + ret_value = NC_EINVAL; + +done: + if (credfile != NULL) + if (fclose(credfile) == EOF) + HDONE_ERROR(H5E_ARGS, NC_EACCESS, FAIL, "problem error-closing aws configuration file"); + + return (ret_value); +} /* end NCH5_s3comms_load_aws_profile() */ + +/*---------------------------------------------------------------------------- + * Function: NCH5_s3comms_nlowercase() + * Purpose: + * From string starting at `s`, write `len` characters to `dest`, + * converting all to lowercase. + * Behavior is undefined if `s` is NULL or `len` overruns the allocated + * space of either `s` or `dest`. + * Provided as convenience. + * Return: + * - SUCCESS: `SUCCEED` + * - upon completion, `dest` is populated + * - FAILURE: `FAIL` + * - `dest == NULL` + * Programmer: Jacob Smith + * 2017-09-18 + *---------------------------------------------------------------------------- + */ +int +NCH5_s3comms_nlowercase(char *dest, const char *s, size_t len) +{ + int ret_value = SUCCEED; + +#if S3COMMS_DEBUG_TRACE + fprintf(stdout, "called NCH5_s3comms_nlowercase.\n"); +#endif + + if (dest == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "destination cannot be null."); + + if (len > 0) { + memcpy(dest, s, len); + do { + len--; + dest[len] = (char)tolower((int)dest[len]); + } while (len > 0); + } + +done: + return (ret_value); +} /* end NCH5_s3comms_nlowercase() */ + +/*---------------------------------------------------------------------------- + * Function: NCH5_s3comms_percent_encode_char() + * Purpose: + * "Percent-encode" utf-8 character `c`, e.g., + * '$' -> "%24" + * '¢' -> "%C2%A2" + * `c` cannot be null. + * Does not (currently) accept multi-byte characters... + * limit to (?) u+00ff, well below upper bound for two-byte utf-8 encoding + * (u+0080..u+07ff). + * Writes output to `repr`. + * `repr` cannot be null. + * Assumes adequate space i `repr`... + * >>> char[4] or [7] for most characters, + * >>> [13] as theoretical maximum. + * Representation `repr` is null-terminated. + * Stores length of representation (without null terminator) at pointer + * `repr_len`. + * Return : SUCCEED/FAIL + * - SUCCESS: `SUCCEED` + * - percent-encoded representation written to `repr` + * - 'repr' is null-terminated + * - FAILURE: `FAIL` + * - `c` or `repr` was NULL + * Programmer: Jacob Smith + *---------------------------------------------------------------------------- + */ +int +NCH5_s3comms_percent_encode_char(char *repr, const unsigned char c, size_t *repr_len) +{ + unsigned int i = 0; + int chars_written = 0; + int ret_value = SUCCEED; +#if S3COMMS_DEBUG + unsigned char s[2] = {c, 0}; + unsigned char hex[3] = {0, 0, 0}; +#endif + +#if S3COMMS_DEBUG_TRACE + fprintf(stdout, "called NCH5_s3comms_percent_encode_char.\n"); +#endif + + if (repr == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "no destination `repr`."); + +#if S3COMMS_DEBUG + NCH5_s3comms_bytes_to_hex((char *)hex, s, 1, FALSE); + fprintf(stdout, " CHAR: \'%s\'\n", s); + fprintf(stdout, " CHAR-HEX: \"%s\"\n", hex); +#endif + + if (c <= (unsigned char)0x7f) { + /* character represented in a single "byte" + * and single percent-code + */ +#if S3COMMS_DEBUG + fprintf(stdout, " SINGLE-BYTE\n"); +#endif + *repr_len = 3; + chars_written = snprintf(repr, 4, "%%%02X", c); + if (chars_written < 0) + HGOTO_ERRORVA(H5E_ARGS, NC_EINVAL, FAIL, "cannot write char %c", c); + } /* end if single-byte unicode char */ + else { + /* multi-byte, multi-percent representation + */ + unsigned int acc = 0; /* byte accumulator */ + unsigned int k = 0; /* uint character representation */ + unsigned int stack_size = 0; + unsigned char stack[4] = {0, 0, 0, 0}; +#if S3COMMS_DEBUG + fprintf(stdout, " MULTI-BYTE\n"); +#endif + stack_size = 0; + k = (unsigned int)c; + *repr_len = 0; + do { + /* push number onto stack in six-bit slices + */ + acc = k; + acc >>= 6; /* cull least */ + acc <<= 6; /* six bits */ + stack[stack_size++] = (unsigned char)(k - acc); + k = acc >> 6; + } while (k > 0); + + /* `stack` now has two to four six-bit 'numbers' to be put into + * UTF-8 byte fields. + */ + +#if S3COMMS_DEBUG + fprintf(stdout, " STACK:\n {\n"); + for (i = 0; i < stack_size; i++) { + NCH5_s3comms_bytes_to_hex((char *)hex, (&stack[i]), 1, FALSE); + hex[2] = 0; + fprintf(stdout, " %s,\n", hex); + } + fprintf(stdout, " }\n"); +#endif + + /**************** + * leading byte * + ****************/ + + /* prepend 11[1[1]]0 to first byte */ + /* 110xxxxx, 1110xxxx, or 11110xxx */ + acc = 0xC0; /* 0x11000000 */ + acc += (stack_size > 2) ? 0x20 : 0; /* 0x00100000 */ + acc += (stack_size > 3) ? 0x10 : 0; /* 0x00010000 */ + stack_size--; + chars_written = snprintf(repr, 4, "%%%02X", (unsigned char)(acc + stack[stack_size])); + if (chars_written < 0) + HGOTO_ERRORVA(H5E_ARGS, NC_EINVAL, FAIL, "cannot write char %c", c); + *repr_len += 3; + + /************************ + * continuation byte(s) * + ************************/ + + /* 10xxxxxx */ + for (i = 0; i < stack_size; i++) { + chars_written = + snprintf(&repr[i * 3 + 3], 4, "%%%02X", (unsigned char)(0x80 + stack[stack_size - 1 - i])); + if (chars_written < 0) + HGOTO_ERRORVA(H5E_ARGS, NC_EINVAL, FAIL, "cannot write char %c", c); + *repr_len += 3; + } /* end for each continuation byte */ + } /* end else (multi-byte) */ + + *(repr + *repr_len) = '\0'; + +done: + return (ret_value); +} /* NCH5_s3comms_percent_encode_char */ + +/*---------------------------------------------------------------------------- + * Function: NCH5_s3comms_signing_key() + * Purpose: + * Create AWS4 "Signing Key" from secret key, AWS region, and timestamp. + * Sequentially runs HMAC_SHA256 on strings in specified order, + * generating re-usable checksum (according to documentation, valid for + * 7 days from time given). + * `secret` is `access key id` for targeted service/bucket/resource. + * `iso8601now` must conform to format, yyyyMMDD'T'hhmmss'Z' + * e.g. "19690720T201740Z". + * `region` should be one of AWS service region names, e.g. "us-east-1". + * Hard-coded "service" algorithm requirement to "s3". + * Inputs must be null-terminated strings. + * Writes to `md` the raw byte data, length of `SHA256_DIGEST_LENGTH`. + * Programmer must ensure that `md` is appropriately allocated. + * Return: + * - SUCCESS: `SUCCEED` + * - raw byte data of signing key written to `md` + * - FAILURE: `FAIL` + * - if any input arguments was NULL + * Programmer: Jacob Smith + * 2017-07-13 + *---------------------------------------------------------------------------- + */ +int +NCH5_s3comms_signing_key(unsigned char **mdp, const char *secret, const char *region, const char *iso8601now) +{ + char *AWS4_secret = NULL; + size_t AWS4_secret_len = 0; + unsigned char datekey[SHA256_DIGEST_LENGTH]; + unsigned char dateregionkey[SHA256_DIGEST_LENGTH]; + unsigned char dateregionservicekey[SHA256_DIGEST_LENGTH]; + int ret = 0; /* return value of snprintf */ + int ret_value = SUCCEED; + unsigned char* md = NULL; + +#if S3COMMS_DEBUG + fprintf(stdout, "called NCH5_s3comms_signing_key.\n"); +#endif + + if (mdp == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "Destination `mdp` cannot be NULL."); + if (secret == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EAUTH, FAIL, "`secret` cannot be NULL."); + if (region == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EAUTH, FAIL, "`region` cannot be NULL."); + if (iso8601now == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "`iso8601now` cannot be NULL."); + + AWS4_secret_len = 4 + nulllen(secret) + 1; + AWS4_secret = (char *)malloc(sizeof(char *) * AWS4_secret_len); + if (AWS4_secret == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "Could not allocate space."); + + /* prepend "AWS4" to start of the secret key */ + ret = snprintf(AWS4_secret, AWS4_secret_len, "%s%s", "AWS4", secret); + if ((size_t)ret != (AWS4_secret_len - 1)) + HGOTO_ERRORVA(H5E_ARGS, NC_EINVAL, FAIL, "problem writing AWS4+secret `%s`", secret); + + if((md = (unsigned char*)malloc(SHA256_DIGEST_LENGTH))==NULL) + HGOTO_ERROR(H5E_ARGS, NC_ENOMEM, NULL, "could not malloc space for signing key ."); + + /* hash_func, key, len(key), msg, len(msg), digest_dest, digest_len_dest + * we know digest length, so ignore via NULL + */ +#if 0 + HMAC(EVP_sha256(), (const unsigned char *)AWS4_secret, (int)nulllen(AWS4_secret), + (const unsigned char *)iso8601now, 8, /* 8 --> length of 8 --> "yyyyMMDD" */ + datekey, NULL); + HMAC(EVP_sha256(), (const unsigned char *)datekey, SHA256_DIGEST_LENGTH, (const unsigned char *)region, + nulllen(region), dateregionkey, NULL); + HMAC(EVP_sha256(), (const unsigned char *)dateregionkey, SHA256_DIGEST_LENGTH, + (const unsigned char *)"s3", 2, dateregionservicekey, NULL); + HMAC(EVP_sha256(), (const unsigned char *)dateregionservicekey, SHA256_DIGEST_LENGTH, + (const unsigned char *)"aws4_request", 12, md, NULL); +#else + Curl_hmacit(Curl_HMAC_SHA256, (const unsigned char *)AWS4_secret, (int)nulllen(AWS4_secret), + (const unsigned char *)iso8601now, 8, /* 8 --> length of 8 --> "yyyyMMDD" */ + datekey); + Curl_hmacit(Curl_HMAC_SHA256, (const unsigned char *)datekey, SHA256_DIGEST_LENGTH, (const unsigned char *)region, + nulllen(region), dateregionkey); + Curl_hmacit(Curl_HMAC_SHA256, (const unsigned char *)dateregionkey, SHA256_DIGEST_LENGTH, + (const unsigned char *)"s3", 2, dateregionservicekey); + Curl_hmacit(Curl_HMAC_SHA256, (const unsigned char *)dateregionservicekey, SHA256_DIGEST_LENGTH, + (const unsigned char *)"aws4_request", 12, md); +#endif + if(mdp) {*mdp = md; md = NULL;} + +done: + nullfree(md); + free(AWS4_secret); + return (ret_value); +} /* end NCH5_s3comms_signing_key() */ + +/*---------------------------------------------------------------------------- + * Function: NCH5_s3comms_tostringtosign() + * Purpose: + * Get AWS "String to Sign" from Canonical Request, timestamp, + * and AWS "region". + * Common between single request and "chunked upload", + * conforms to: + * "AWS4-HMAC-SHA256\n" + + * + "\n" + // yyyyMMDD'T'hhmmss'Z' + * + "/" + + "/s3/aws4-request\n" + + * hex(SHA256()) + * Inputs `creq` (canonical request string), `now` (ISO8601 format), + * and `region` (s3 region designator string) must all be + * null-terminated strings. + * Result is written to `dest` with null-terminator. + * It is left to programmer to ensure `dest` has adequate space. + * Return: + * - SUCCESS: `SUCCEED` + * - "string to sign" written to `dest` and null-terminated + * - FAILURE: `FAIL` + * - if any of the inputs are NULL + * - if an error is encountered while computing checksum + * Programmer: Jacob Smith + * 2017-07-?? + *---------------------------------------------------------------------------- + */ +int +NCH5_s3comms_tostringtosign(VString* dest, const char *req, const char *now, const char *region) +{ + unsigned char checksum[SHA256_DIGEST_LENGTH * 2 + 1]; + char day[9]; + char hexsum[SHA256_DIGEST_LENGTH * 2 + 1]; + size_t i = 0; + int ret = 0; /* snprintf return value */ + int ret_value = SUCCEED; + char tmp[128]; + +#if S3COMMS_DEBUG_TRACE + fprintf(stdout, "called NCH5_s3comms_tostringtosign.\n"); +#endif + + if (dest == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "destination buffer cannot be null."); + if (req == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "canonical request cannot be null."); + if (now == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "Timestring cannot be NULL."); + if (region == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EAUTH, FAIL, "Region cannot be NULL."); + + for (i = 0; i < 128; i++) + tmp[i] = '\0'; + for (i = 0; i < SHA256_DIGEST_LENGTH * 2 + 1; i++) { + checksum[i] = '\0'; + hexsum[i] = '\0'; + } + strncpy(day, now, 8); + day[8] = '\0'; + ret = snprintf(tmp, 127, "%s/%s/s3/aws4_request", day, region); + if (ret <= 0 || ret >= 127) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "problem adding day and region to string"); + + vscat(dest,"AWS4-HMAC-SHA256\n"); + vsappendn(dest, now, nulllen(now)); + vscat(dest,"\n"); + + vsappendn(dest, tmp, nulllen(tmp)); + vscat(dest,"\n"); + +#if 0 + SHA256((const unsigned char *)req, nulllen(req), checksum); +#else + Curl_sha256it(checksum, (const unsigned char *)req, nulllen(req)); +#endif + + if (NCH5_s3comms_bytes_to_hex(hexsum, (const unsigned char *)checksum, SHA256_DIGEST_LENGTH, TRUE) != SUCCEED) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "could not create hex string"); + + vsappendn(dest,hexsum,SHA256_DIGEST_LENGTH * 2); + +done: + return (ret_value); +} /* end H5ros3_tostringtosign() */ + +/*---------------------------------------------------------------------------- + * Function: NCH5_s3comms_trim() + * Purpose: + * Remove all whitespace characters from start and end of a string `s` + * of length `s_len`, writing trimmed string copy to `dest`. + * Stores number of characters remaining at `n_written`. + * Destination for trimmed copy `dest` cannot be null. + * `dest` must have adequate space allocated for trimmed copy. + * If inadequate space, behavior is undefined, possibly resulting + * in segfault or overwrite of other data. + * If `s` is NULL or all whitespace, `dest` is untouched and `n_written` + * is set to 0. + * Return: + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * - `dest == NULL` + * Programmer: Jacob Smith + * 2017-09-18 + *---------------------------------------------------------------------------- + */ +int +NCH5_s3comms_trim(char *dest, char *s, size_t s_len, size_t *n_written) +{ + int ret_value = SUCCEED; + +#if S3COMMS_DEBUG_TRACE + fprintf(stdout, "called NCH5_s3comms_trim.\n"); +#endif + + if (dest == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "destination cannot be null."); + if (s == NULL) + s_len = 0; + + if (s_len > 0) { + /* Find first non-whitespace character from start; + * reduce total length per character. + */ + while ((s_len > 0) && isspace((unsigned char)s[0]) && s_len > 0) { + s++; + s_len--; + } + + /* Find first non-whitespace character from tail; + * reduce length per-character. + * If length is 0 already, there is no non-whitespace character. + */ + if (s_len > 0) { + do { + s_len--; + } while (isspace((unsigned char)s[s_len])); + s_len++; + + /* write output into dest */ + memcpy(dest, s, s_len); + } + } + + *n_written = s_len; + +done: + return (ret_value); +} /* end NCH5_s3comms_trim() */ + +/*---------------------------------------------------------------------------- + * Function: NCH5_s3comms_uriencode() + * Purpose: + * URIencode (percent-encode) every byte except "[a-zA-Z0-9]-._~". + * For each character in source string `_s` from `s[0]` to `s[s_len-1]`, + * writes to `dest` either the raw character or its percent-encoded + * equivalent. + * See `NCH5_s3comms_bytes_to_hex` for information on percent-encoding. + * Space (' ') character encoded as "%20" (not "+") + * Forward-slash ('/') encoded as "%2F" only when `encode_slash == true`. + * Records number of characters written at `n_written`. + * Assumes that `dest` has been allocated with enough space. + * Neither `dest` nor `s` can be NULL. + * `s_len == 0` will have no effect. + * Return: + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * - source strings `s` or destination `dest` are NULL + * - error while attempting to percent-encode a character + * Programmer: Jacob Smith + * 2017-07-?? + *---------------------------------------------------------------------------- + */ +int +NCH5_s3comms_uriencode(char** destp, const char *s, size_t s_len, int encode_slash, size_t *n_written) +{ + char c = 0; + char hex_buffer[13]; + size_t hex_len = 0; + int ret_value = SUCCEED; + size_t s_off = 0; + VString* dest = vsnew(); + +#if S3COMMS_DEBUG_TRACE + fprintf(stdout, "NCH5_s3comms_uriencode called.\n"); +#endif + + if (s == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "source string cannot be NULL"); + if (dest == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "destination cannot be NULL"); + + /* Write characters to destination, converting to percent-encoded + * "hex-utf-8" strings if necessary. + * e.g., '$' -> "%24" + */ + for (s_off = 0; s_off < s_len; s_off++) { + c = s[s_off]; + if (isalnum(c) || c == '.' || c == '-' || c == '_' || c == '~' || + (c == '/' && encode_slash == FALSE)) + vsappend(dest, c); + else { + if (NCH5_s3comms_percent_encode_char(hex_buffer, (const unsigned char)c, &hex_len) != SUCCEED) { + hex_buffer[0] = c; + hex_buffer[1] = 0; + HGOTO_ERRORVA(H5E_ARGS, NC_EINVAL, FAIL, + "unable to percent-encode character \'%s\' " + "at %d in \"%s\"", + hex_buffer, (int)s_off, s); + } + vsappendn(dest, hex_buffer, hex_len); + } /* end else (not a regular character) */ + } /* end for each character */ + + if(n_written) {*n_written = vslength(dest);} + if(destp) {*destp = vsextract(dest);} + +done: + vsfree(dest); + return (ret_value); +} /* NCH5_s3comms_uriencode */ + + +/**************************************************/ +/* Extensions */ + +static int +validate_url(NCURI* purl) +{ + int ret_value = SUCCEED; + if (purl == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "parsed url cannot be null."); + if (purl->host == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "parsed url must have non-null host."); + if (purl->path == NULL) + + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "parsed url must have non-null resource."); +done: + return (ret_value); +} + +static int +validate_handle(s3r_t* handle, const char* url) +{ + int ret_value = SUCCEED; + if (handle == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "handle cannot be null."); + if (handle->magic != S3COMMS_S3R_MAGIC) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "handle has invalid magic."); + if (handle->curlhandle == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "handle has bad (null) curlhandle."); + +done: + return (ret_value); +} + +static int +request_setup(s3r_t* handle, const char* url, HTTPVerb verb, struct s3r_cbstruct* sds) +{ + int ret_value = SUCCEED; + CURL* curlh = handle->curlhandle; + + (void)trace(curlh,1); + + /* Common setup (possibly overridden below) */ + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_URL, url)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "error while setting CURL option (CURLOPT_URL)."); + + switch (verb) { + case HTTPGET: + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_HTTPGET, 1L)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "error while setting CURL option (CURLOPT_HTTPGET)."); + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_WRITEDATA, sds)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "error while setting CURL option (CURLOPT_WRITEDATA)."); + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_WRITEFUNCTION, curlwritecallback)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "error while setting CURL option (CURLOPT_WRITEFUNCTION)."); + break; + case HTTPPUT: + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_UPLOAD, 1L)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "error while setting CURL option (CURLOPT_UPLOAD)."); + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_READDATA, sds)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "error while setting CURL option (CURLOPT_READDATA)."); + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_READFUNCTION, curlreadcallback)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "error while setting CURL option (CURLOPT_READFUNCTION)."); + break; + case HTTPHEAD: + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_NOBODY, 1L)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "error while setting CURL option (CURLOPT_NOBODY)."); + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_HEADERDATA, sds)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "error while setting CURL option (CURLOPT_HEADERDATA)."); + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_HEADERFUNCTION, curlheadercallback)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "error while setting CURL option (CURLOPT_HEADERFUNCTION)."); + break; + case HTTPDELETE: + if( CURLE_OK != curl_easy_setopt(curlh, CURLOPT_CUSTOMREQUEST, "DELETE")) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "error while setting CURL option (CURLOPT_CUSTOMREQUEST)."); + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_HEADERDATA, sds)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "error while setting CURL option (CURLOPT_HEADERDATA)."); + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_HEADERFUNCTION, curlheadercallback)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "error while setting CURL option (CURLOPT_HEADERFUNCTION)."); + break; + case HTTPPOST: + default: + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "Illegal verb: %d.",(int)verb); + break; + } + +done: + return (ret_value); +} + + +/** + otherheaders is a vector of (header,value) pairs + */ +static int +build_request(s3r_t* handle, NCURI* purl, + const char* byterange, + const char** otherheaders, + VString* payload, + HTTPVerb verb) +{ + int i,ret_value = SUCCEED; + struct curl_slist *curlheaders = NULL; + hrb_node_t *node = NULL; + hrb_t *request = NULL; + struct tm *now = NULL; + CURL *curlh = handle->curlhandle; + VString *authorization = vsnew(); + VString *signed_headers = vsnew(); + VString* creds = vsnew(); + VString *canonical_request = vsnew(); + VString *buffer1 = vsnew(); + VString *buffer2 = vsnew(); + char hexsum[SHA256_DIGEST_LENGTH * 2 + 1]; + char* payloadsha256 = NULL; /* [SHA256_DIGEST_LENGTH * 2 + 1]; */ +#if 0 + char buffer1[512 + 1]; /* -> Canonical Request -> Signature */ + char buffer2[256 + 1]; /* -> String To Sign -> Credential */ + char signed_headers[48 + 1]; /* + * should be large enough for nominal listing: + * "host;range;x-amz-content-sha256;x-amz-date" + * + '\0', with "range;" possibly absent + */ +#endif + char iso8601now[ISO8601_SIZE]; + + /* zero start of strings */ +#if 0 + authorization[0] = 0; + buffer1[0] = 0; + buffer2[0] = 0; + signed_headers[0] = 0; +#endif + iso8601now[0] = 0; + + + /**** CREATE HTTP REQUEST STRUCTURE (hrb_t) ****/ + + request = NCH5_s3comms_hrb_init_request((const char *)purl->path, "HTTP/1.1"); + if (request == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "could not allocate hrb_t request."); + assert(request->magic == S3COMMS_HRB_MAGIC); + + /* These headers are independent of auth */ + if (SUCCEED != NCH5_s3comms_hrb_node_insert(request->headers, "Host", purl->host)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "unable to set host header"); + + if (byterange != NULL) { + if (SUCCEED != NCH5_s3comms_hrb_node_insert(request->headers, "Range", byterange)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "unable to set range header"); + } + + /* Add other headers */ + if(otherheaders != NULL) { + const char** hdrs = otherheaders; + for(;*hdrs;hdrs+=2) { + const char* key = (const char*)hdrs[0]; + const char* value = (const char*)hdrs[1]; + if (SUCCEED != NCH5_s3comms_hrb_node_insert(request->headers, key, value)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "unable to set host header"); + } + } + + now = gmnow(); + if (ISO8601NOW(iso8601now, now) != (ISO8601_SIZE - 1)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "could not format ISO8601 time."); + + if (SUCCEED != NCH5_s3comms_hrb_node_insert(request->headers, "x-amz-date", (const char *)iso8601now)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "unable to set x-amz-date header"); + + /* Compute SHA256 of upload data, if any */ + if(verb == HTTPPUT && payload != NULL) { + unsigned char sha256csum[SHA256_DIGEST_LENGTH]; +#if 0 + SHA256((const unsigned char*)vscontents(payload),vslength(payload),sha256csum); +#else + Curl_sha256it(sha256csum, (const unsigned char *)vscontents(payload), vslength(payload)); +#endif + if((SUCCEED != NCH5_s3comms_bytes_to_hex(hexsum,sha256csum,sizeof(sha256csum),1/*lowercase*/))) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "unable to compute hex form of payload."); + payloadsha256 = hexsum; + } else {/* Everything else has no body */ + payloadsha256 = EMPTY_SHA256; + } + if (SUCCEED != NCH5_s3comms_hrb_node_insert(request->headers, "x-amz-content-sha256", (const char *)payloadsha256)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "unable to set x-amz-content-sha256 header"); + + if (handle->signing_key != NULL) { + /* authenticate request + */ + /* char authorization[512 + 1]; + * 512 := approximate max length... + * 67 + * + 8 + * + 64 + * + 128 + * + 20 + * + 128 + */ + + /**** VERIFY INFORMATION EXISTS ****/ + + if (handle->region == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "handle must have non-null region."); + if (handle->accessid == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "handle must have non-null accessid."); + if (handle->accesskey == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "handle must have non-null accesskey."); + if (handle->signing_key == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "handle must have non-null signing_key."); + + sortheaders(request->headers); /* ensure sorted order */ + + /**** COMPUTE AUTHORIZATION ****/ + + /* buffer1 -> canonical request */ + if (SUCCEED != NCH5_s3comms_aws_canonical_request(buffer1, signed_headers, verb, purl->query, payloadsha256, request)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "bad canonical request"); +#if S3COMMS_DEBUG >= 2 + fprintf(stderr,"canonical_request=\n%s\n",vscontents(buffer1)); +#endif + /* buffer2->string-to-sign */ + if (SUCCEED != NCH5_s3comms_tostringtosign(buffer2, vscontents(buffer1), iso8601now, handle->region)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "bad string-to-sign"); +#if S3COMMS_DEBUG >= 2 + fprintf(stderr,"stringtosign=\n%s\n",vscontents(buffer2)); +#endif + /* hexsum -> signature */ + if (SUCCEED != NCH5_s3comms_HMAC_SHA256(handle->signing_key, SHA256_DIGEST_LENGTH, vscontents(buffer2), vslength(buffer2), hexsum)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "bad signature"); +#if S3COMMS_DEBUG >= 2 + fprintf(stderr,"HMAX_SHA256=|%s|\n",hexsum); +#endif + iso8601now[8] = 0; /* trim to yyyyMMDD */ + S3COMMS_FORMAT_CREDENTIAL(creds, handle->accessid, iso8601now, handle->region, "s3"); + if (vslength(creds) >= S3COMMS_MAX_CREDENTIAL_SIZE) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "unable to format aws4 credential string"); +#if S3COMMS_DEBUG >= 2 + fprintf(stderr,"Credentials=|%s|\n",vscontents(creds)); +#endif + + vscat(authorization,"AWS4-HMAC-SHA256 Credential="); + vsappendn(authorization,vscontents(creds),vslength(creds)); + vscat(authorization,",SignedHeaders="); + vscat(authorization,vscontents(signed_headers)); + vscat(authorization,",Signature="); + vsappendn(authorization,hexsum,2*SHA256_DIGEST_LENGTH); +#if S3COMMS_DEBUG >= 2 + fprintf(stderr,"Authorization=|%s|\n",vscontents(authorization)); +#endif + + /* append authorization header to http request buffer */ + if (NCH5_s3comms_hrb_node_insert(request->headers, "authorization", (const char *)vscontents(authorization)) != SUCCEED) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "unable to set Authorization header"); + + } /* end if should authenticate (info provided) */ + + sortheaders(request->headers); /* re-sort */ + + /**** SET CURLHANDLE HTTP HEADERS FROM GENERATED DATA ****/ + + for(i=0;iheaders);i++) { + node = vlistget(request->headers,i); + if(node != NULL) { + assert(node->magic == S3COMMS_HRB_NODE_MAGIC); + curlheaders = curl_slist_append(curlheaders, (const char *)node->cat); + if (curlheaders == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "could not append header to curl slist."); + node = node->next; + } + } + + /* sanity-check */ + if (curlheaders == NULL) + /* above loop was probably never run */ + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "curlheaders was never populated."); + + /* Apparently, Chunked Transfer Encoding cannot be used, so disable it explicitly */ + if((curlheaders = curl_slist_append(curlheaders, "Transfer-Encoding:"))==NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "could not disable Transfer-Encoding."); + + /* finally, set http headers in curl handle */ + if (curl_easy_setopt(curlh, CURLOPT_HTTPHEADER, curlheaders) != CURLE_OK) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, + "error while setting CURL option (CURLOPT_HTTPHEADER)."); + + /* We need to save the curlheaders so we can release them after the transfer + (see https://curl.se/libcurl/c/CURLOPT_HTTPHEADER.html). */ + if(handle->curlheaders != NULL) { + curl_slist_free_all(handle->curlheaders); + handle->curlheaders = NULL; + } + assert(handle->curlheaders == NULL); + handle->curlheaders = curlheaders; + curlheaders = NULL; + +done: + vsfree(buffer1); + vsfree(buffer2); + vsfree(authorization); + vsfree(signed_headers); + vsfree(canonical_request); + vsfree(creds); + if (curlheaders != NULL) { + curl_slist_free_all(curlheaders); + curlheaders = NULL; + } + if (request != NULL) { + if (SUCCEED != NCH5_s3comms_hrb_destroy(request)) + HDONE_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "cannot release header request structure"); + request = NULL; + } + return (ret_value); +} + +static int +perform_request(s3r_t* handle, long* httpcodep) +{ + int ret_value = SUCCEED; + CURL *curlh = NULL; + CURLcode p_status = CURLE_OK; + long httpcode = 0; + char curlerrbuf[CURL_ERROR_SIZE]; + + curlerrbuf[0] = '\0'; + curlh = handle->curlhandle; + +#if S3COMMS_CURL_VERBOSITY > 1 + /* CURL will print (to stdout) information for each operation */ + (void)trace(curlh,1); +#endif + + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_ERRORBUFFER, curlerrbuf)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "problem setting error buffer"); + + p_status = curl_easy_perform(curlh); + + /* Get response code */ + if (CURLE_OK != curl_easy_getinfo(curlh, CURLINFO_RESPONSE_CODE, &httpcode)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "problem getting response code"); + +#ifdef DEBUG + { +char* eurl = NULL; +curl_easy_getinfo(curlh, CURLINFO_EFFECTIVE_URL, &eurl); +if(eurl == NULL) eurl = ""; +printf(">>>> url=%s verb=%s httpcode=%d",eurl,handle->httpverb,(int)httpcode); +if(p_status != CURLE_OK) +printf(" errmsg=(%d) |%s|",(int)p_status,curl_easy_strerror(p_status)); +printf("\n"); +fflush(stdout); + } +#endif + + if(p_status == CURLE_HTTP_RETURNED_ERROR) { + /* signal ok , but return the bad error code */ + p_status = CURLE_OK; + } + +#if S3COMMS_CURL_VERBOSITY > 0 + /* In event of error, print detailed information to stderr + * This is not the default behavior. + */ + { + if (p_status != CURLE_OK) { + fprintf(stderr, "CURL ERROR CODE: %d\nHTTP CODE: %ld\n", p_status, httpcode); + fprintf(stderr, "%s\n", curl_easy_strerror(p_status)); + } + } /* verbose error reporting */ +#endif + if (p_status != CURLE_OK) + HGOTO_ERROR(H5E_VFL, NC_EACCESS, FAIL, "curl cannot perform request"); + + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_ERRORBUFFER, NULL)) /* reset */ + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "problem unsetting error buffer"); + + /* Should be safe to reclaim the curl headers */ + if(handle->curlheaders != NULL) { + curl_slist_free_all(handle->curlheaders); + handle->curlheaders = NULL; + } + assert(handle->curlheaders == NULL); + +done: + if(httpcodep) *httpcodep = httpcode; + return (ret_value); +} + +static int +curl_reset(s3r_t* handle) +{ + int ret_value = SUCCEED; + CURL* curlh = handle->curlhandle; + + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_NOBODY, NULL)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "error while setting CURL option (CURLOPT_NOBODY)."); + + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_HEADERDATA, NULL)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "error while setting CURL option (CURLOPT_HEADERDATA)."); + + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_HEADERFUNCTION, NULL)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "error while setting CURL option (CURLOPT_HEADERFUNCTION)."); + + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_URL, NULL)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "error while setting CURL option (CURLOPT_URL)."); + + if(CURLE_OK != curl_easy_setopt(curlh, CURLOPT_CUSTOMREQUEST, NULL)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "error while setting CURL option (CURLOPT_CUSTOMREQUEST)."); + + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_HTTPGET, 1L)) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "error while setting CURL option (CURLOPT_HTTPGET)."); + + /* clear any Range */ + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_RANGE, NULL)) + HDONE_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "cannot unset CURLOPT_RANGE"); + + /* clear headers */ + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_HTTPHEADER, NULL)) + HDONE_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "cannot unset CURLOPT_HTTPHEADER"); + +done: + return (ret_value); +} + +static int +build_range(size_t offset, size_t len, char** rangep) +{ + int ret_value = SUCCEED; + char *rangebytesstr = NULL; + int ret = 0; /* working variable to check */ + /* return value of snprintf */ + if (len > 0) { + rangebytesstr = (char *)malloc(sizeof(char) * (S3COMMS_MAX_RANGE_STRING_SIZE + 1)); + if (rangebytesstr == NULL) + HGOTO_ERROR(H5E_ARGS, NC_ENOMEM, FAIL, "could not malloc range format string."); + ret = snprintf(rangebytesstr, (S3COMMS_MAX_RANGE_STRING_SIZE), "bytes=%lld-%lld", + (long long)offset, (long long)(offset + len - 1)); + if (ret <= 0 || ret >= S3COMMS_MAX_RANGE_STRING_SIZE) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "unable to format HTTP Range value"); + } + else if (offset > 0) { + rangebytesstr = (char *)malloc(sizeof(char) * (S3COMMS_MAX_RANGE_STRING_SIZE + 1)); + if (rangebytesstr == NULL) + HGOTO_ERROR(H5E_ARGS, NC_ENOMEM, FAIL, "could not malloc range format string."); + ret = snprintf(rangebytesstr, (S3COMMS_MAX_RANGE_STRING_SIZE), "bytes=%lld-", (long long)offset); + if (ret <= 0 || ret >= S3COMMS_MAX_RANGE_STRING_SIZE) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, FAIL, "unable to format HTTP Range value"); + } + + if(rangep) {*rangep = rangebytesstr; rangebytesstr = NULL;} + +done: + nullfree(rangebytesstr); + return (ret_value); +} + +static const char* +verbtext(HTTPVerb verb) +{ + switch(verb) { + case HTTPGET: return "GET"; + case HTTPPUT: return "PUT"; + case HTTPPOST: return "POST"; + case HTTPHEAD: return "HEAD"; + case HTTPDELETE: return "DELETE"; + default: break; + } + return NULL; +} + +/* Qsort comparison function */ +static int +hdrcompare(const void* arg1, const void* arg2) +{ + hrb_node_t* n1 = *((hrb_node_t**)arg1); + hrb_node_t* n2 = *((hrb_node_t**)arg2); + return strcasecmp(n1->name,n2->name); +} + +static int +sortheaders(VList* headers) +{ + int ret_value = SUCCEED; + size_t nnodes; + void** listcontents = NULL; + +#if S3COMMS_DEBUG_TRACE + fprintf(stdout, "called sortheaders.\n"); +#endif + + if (headers == NULL || vlistlength(headers) < 2) return ret_value; + nnodes = vlistlength(headers); + listcontents = vlistcontents(headers); + /* sort */ + qsort(listcontents, nnodes, sizeof(hrb_node_t*), hdrcompare); + return (ret_value); +} + +static int +httptonc(long httpcode) +{ + int stat = NC_NOERR; + if(httpcode <= 99) stat = NC_EINTERNAL; /* should never happen */ + else if(httpcode <= 199) + stat = NC_NOERR; /* I guess */ + else if(httpcode <= 299) { + switch (httpcode) { + default: stat = NC_NOERR; break; + } + } else if(httpcode <= 399) + stat = NC_NOERR; /* ? */ + else if(httpcode <= 499) { + switch (httpcode) { + case 400: stat = NC_EINVAL; break; + case 401: case 402: case 403: + stat = NC_EAUTH; break; + case 404: stat = NC_EEMPTY; break; + default: stat = NC_EINVAL; break; + } + } else + stat = NC_ES3; + return stat; +} + +/**************************************************/ +/* Request Tracing */ + +static void +dump(const char *text, FILE *stream, unsigned char *ptr, size_t size) +{ + fprintf(stream, "%s, %10.10ld bytes (0x%8.8lx)\n", + text, (long)size, (long)size); +#if 1 + fprintf(stream,"|%.*s|\n",(int)size,ptr); +#else + { + size_t i; + size_t c; + unsigned int width=0x10; + + for(i=0; i= 0x20 && ptr[i+c] < 0x80) ? ptr[i+c] : '.'; + fputc(x, stream); + } + fputc('\n', stream); /* newline */ + } + } +#endif +} + +static int +my_trace(CURL *handle, curl_infotype type, char *data, size_t size,void *userp) +{ + int dumpdata = 0; + int ssl = 0; + const char *text; + (void)handle; /* prevent compiler warning */ + (void)userp; + + switch (type) { + case CURLINFO_TEXT: + dumpdata=1; + fprintf(stderr, "== Info: %s", data); + default: /* in case a new one is introduced to shock us */ + return 0; + case CURLINFO_HEADER_OUT: + dumpdata=1; + text = "=> Send header"; + break; + case CURLINFO_DATA_OUT: + dumpdata=1; + text = "=> Send data"; + break; + case CURLINFO_HEADER_IN: + dumpdata=1; + text = "<= Recv header"; + break; + case CURLINFO_DATA_IN: + dumpdata=1; + text = "<= Recv data"; + break; + case CURLINFO_SSL_DATA_OUT: + if(!ssl) return 0; + text = "=> Send SSL data"; + break; + case CURLINFO_SSL_DATA_IN: + if(!ssl) return 0; + text = "<= Recv SSL data"; + break; + } + if(dumpdata) + dump(text, stderr, (unsigned char *)data, size); + return 0; +} + +static int +trace(CURL* curl, int onoff) +{ + int stat = NC_NOERR; + CURLcode cstat = CURLE_OK; + if(getenv("S3TRACE") == NULL) goto done; + cstat = curl_easy_setopt(curl, CURLOPT_VERBOSE, onoff?1L:0L); + if(cstat != CURLE_OK) {stat = NC_ECURL; goto done;} + cstat = curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, my_trace); + if(cstat != CURLE_OK) {stat = NC_ECURL; goto done;} +done: + return (stat); +} + diff --git a/libdispatch/nch5s3comms.h b/libdispatch/nch5s3comms.h new file mode 100644 index 0000000000..acfffb2858 --- /dev/null +++ b/libdispatch/nch5s3comms.h @@ -0,0 +1,561 @@ +/********************************************************************* +* Copyright 2018, UCAR/Unidata +* See netcdf/COPYRIGHT file for copying and redistribution conditions. +* ********************************************************************/ + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/***************************************************************************** + * Read-Only S3 Virtual File Driver (VFD) + * + * This is the header for the S3 Communications module + * + * ***NOT A FILE DRIVER*** + * + * Purpose: + * + * - Provide structures and functions related to communicating with + * Amazon S3 (Simple Storage Service). + * - Abstract away the REST API (HTTP, + * networked communications) behind a series of uniform function calls. + * - Handle AWS4 authentication, if appropriate. + * - Fail predictably in event of errors. + * - Eventually, support more S3 operations, such as creating, writing to, + * and removing Objects remotely. + * + * translates: + * `read(some_file, bytes_offset, bytes_length, &dest_buffer);` + * to: + * ``` + * GET myfile HTTP/1.1 + * Host: somewhere.me + * Range: bytes=4096-5115 + * ``` + * and places received bytes from HTTP response... + * ``` + * HTTP/1.1 206 Partial-Content + * Content-Range: 4096-5115/63239 + * + * + * ``` + * ...in destination buffer. + * + * TODO: put documentation in a consistent place and point to it from here. + * + * Programmer: Jacob Smith + * 2017-11-30 + * + *****************************************************************************/ + +/** + * Unidata Changes: + * Derived from HDF5-1.14.0 H5FDs3comms.[ch] + * Modified to be in netcdf-c style + * Support Write operations and support NCZarr. + * See ncs3comms.c for detailed list of changes. + * Author: Dennis Heimbigner + */ + +#ifndef NCS3COMMS_H +#define NCS3COMMS_H + +/*****************/ + +/* Opaque Handles */ +struct CURL; +struct NCURI; +struct VString; + +/***************** + * PUBLIC MACROS * + *****************/ + +/* hexadecimal string of pre-computed sha256 checksum of the empty string + * hex(sha256sum("")) + */ +#define EMPTY_SHA256 "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + +/* string length (plus null terminator) + * example ISO8601-format string: "20170713T145903Z" (YYYYmmdd'T'HHMMSS'_') + */ +#define ISO8601_SIZE 17 + +/* string length (plus null terminator) + * example RFC7231-format string: "Fri, 30 Jun 2017 20:41:55 GMT" + */ +#define RFC7231_SIZE 30 + +/* + *String length (including nul term) for HTTP Verb + */ +#define S3COMMS_VERB_MAX 16 + +/* + * Size of a SHA256 digest in bytes + */ +#ifndef SHA256_DIGEST_LENGTH +#define SHA256_DIGEST_LENGTH 32 +#endif + + +/*--------------------------------------------------------------------------- + * + * Macro: ISO8601NOW() + * + * Purpose: + * + * write "YYYYmmdd'T'HHMMSS'Z'" (less single-quotes) to dest + * e.g., "20170630T204155Z" + * + * wrapper for strftime() + * + * It is left to the programmer to check return value of + * ISO8601NOW (should equal ISO8601_SIZE - 1). + * + *--------------------------------------------------------------------------- + */ +#define ISO8601NOW(dest, now_gm) strftime((dest), ISO8601_SIZE, "%Y%m%dT%H%M%SZ", (now_gm)) + +/*--------------------------------------------------------------------------- + * + * Macro: RFC7231NOW() + * + * Purpose: + * + * write "Day, dd Mmm YYYY HH:MM:SS GMT" to dest + * e.g., "Fri, 30 Jun 2017 20:41:55 GMT" + * + * wrapper for strftime() + * + * It is left to the programmer to check return value of + * RFC7231NOW (should equal RFC7231_SIZE - 1). + * + *--------------------------------------------------------------------------- + */ +#define RFC7231NOW(dest, now_gm) strftime((dest), RFC7231_SIZE, "%a, %d %b %Y %H:%M:%S GMT", (now_gm)) + +/* Reasonable maximum length of a credential string. + * Provided for error-checking S3COMMS_FORMAT_CREDENTIAL (below). + * 17 <- "////aws4_request\0" + * 2 < "s3" (service) + * 8 <- "YYYYmmdd" (date) + * 128 <- (access_id) + * 155 :: sum + */ +#define S3COMMS_MAX_CREDENTIAL_SIZE 155 + +/*--------------------------------------------------------------------------- + * + * Macro: H5FD_S3COMMS_FORMAT_CREDENTIAL() + * + * Purpose: + * + * Format "S3 Credential" string from inputs, for AWS4. + * + * Wrapper for HDsnprintf(). + * + * _HAS NO ERROR-CHECKING FACILITIES_ + * It is left to programmer to ensure that return value confers success. + * e.g., + * ``` + * assert( S3COMMS_MAX_CREDENTIAL_SIZE >= + * S3COMMS_FORMAT_CREDENTIAL(...) ); + * ``` + * + * "////aws4_request" + * assuming that `dest` has adequate space. + * + * ALL inputs must be null-terminated strings. + * + * `access` should be the user's access key ID. + * `date` must be of format "YYYYmmdd". + * `region` should be relevant AWS region, i.e. "us-east-1". + * `service` should be "s3". + * + *--------------------------------------------------------------------------- + */ +#define S3COMMS_FORMAT_CREDENTIAL(dest, access, iso8601_date, region, service) \ + vscat((dest),(access)); vscat((dest),"/"); \ + vscat((dest),(iso8601_date)); vscat((dest),"/"); \ + vscat((dest),(region)); vscat((dest),"/"); \ + vscat((dest),(service)); vscat((dest),"/"); \ + vscat((dest),"aws4_request"); + +#if 0 + snprintf((dest), S3COMMS_MAX_CREDENTIAL_SIZE, "%s/%s/%s/%s/aws4_request", (access), (iso8601_date), \ + (region), (service)) +#endif + +/********************* + * PUBLIC STRUCTURES * + *********************/ + +/*---------------------------------------------------------------------------- + * + * Structure: hrb_node_t + * + * HTTP Header Field Node + * + * + * + * Maintain a ordered (linked) list of HTTP Header fields. + * + * Provides efficient access and manipulation of a logical sequence of + * HTTP header fields, of particular use when composing an + * "S3 Canonical Request" for authentication. + * + * - The creation of a Canonical Request involves: + * - convert field names to lower case + * - sort by this lower-case name + * - convert ": " name-value separator in HTTP string to ":" + * - get sorted lowercase names without field or separator + * + * As HTTP headers allow headers in any order (excepting the case of multiple + * headers with the same name), the list ordering can be optimized for Canonical + * Request creation, suggesting alphabtical order. For more expedient insertion + * and removal of elements in the list, linked list seems preferable to a + * dynamically-expanding array. The usually-smaller number of entries (5 or + * fewer) makes performance overhead of traversing the list trivial. + * + * The above requirements of creating at Canonical Request suggests a reasonable + * trade-off of speed for space with the option to compute elements as needed + * or to have the various elements prepared and stored in the structure + * (e.g. name, value, lowername, concatenated name:value) + * The structure currently is implemented to pre-compute. + * + * At all times, the "first" node of the list should be the least, + * alphabetically. For all nodes, the `next` node should be either NULL or + * of greater alphabetical value. + * + * Each node contains its own header field information, plus a pointer to the + * next node. + * + * It is not allowed to have multiple nodes with the same _lowercase_ `name`s + * in the same list + * (i.e., name is case-insensitive for access and modification.) + * + * All data (`name`, `value`, `lowername`, and `cat`) are null-terminated + * strings allocated specifically for their node. + * + * + * + * `magic` (unsigned long) + * + * "unique" idenfier number for the structure type + * + * `name` (char *) + * + * Case-meaningful name of the HTTP field. + * Given case is how it is supplied to networking code. + * e.g., "Range" + * + * `lowername` (char *) + * + * Lowercase copy of name. + * e.g., "range" + * + * `value` (char *) + * + * Case-meaningful value of HTTP field. + * e.g., "bytes=0-9" + * + * `cat` (char *) + * + * Concatenated, null-terminated string of HTTP header line, + * as the field would appear in an HTTP request. + * e.g., "Range: bytes=0-9" + * + *---------------------------------------------------------------------------- + */ +typedef struct hrb_node_t { + unsigned long magic; + char *name; + char *value; + char *cat; + char *lowername; + struct hrb_node_t *next; +} hrb_node_t; +#define S3COMMS_HRB_NODE_MAGIC 0x7F5757UL + +/*---------------------------------------------------------------------------- + * + * Structure: hrb_t + * + * HTTP Request Buffer structure + * + * + * + * Logically represent an HTTP request + * + * GET /myplace/myfile.h5 HTTP/1.1 + * Host: over.rainbow.oz + * Date: Fri, 01 Dec 2017 12:35:04 CST + * + * + * + * ...with fast, efficient access to and modification of primary and field + * elements. + * + * Structure for building HTTP requests while hiding much of the string + * processing required "under the hood." + * + * Information about the request target -- the first line -- and the body text, + * if any, are managed directly with this structure. All header fields, e.g., + * "Host" and "Date" above, are created with a linked list of `hrb_node_t` and + * included in the request by a pointer to the head of the list. + * + * + * + * `magic` (unsigned long) + * + * "Magic" number confirming that this is an hrb_t structure and + * what operations are valid for it. + * + * Must be S3COMMS_HRB_MAGIC to be valid. + * + * `body` (char *) : + * + * Pointer to start of HTTP body. + * + * Can be NULL, in which case it is treated as the empty string, "". + * + * `body_len` (size_t) : + * + * Number of bytes (characters) in `body`. 0 if empty or NULL `body`. + * + * `first_header` (hrb_node_t *) : + * + * Pointer to first SORTED header node, if any. + * It is left to the programmer to ensure that this node and associated + * list is destroyed when done. + * + * `resource` (char *) : + * + * Pointer to resource URL string, e.g., "/folder/page.xhtml". + * + * `verb` (char *) : + * + * Pointer to HTTP verb string, e.g., "GET". + * + * `version` (char *) : + * + * Pointer to HTTP version string, e.g., "HTTP/1.1". + * + *---------------------------------------------------------------------------- + */ +typedef struct { + unsigned long magic; + struct VString *body; + struct VList *headers; + char *resource; + char *version; +} hrb_t; +#define S3COMMS_HRB_MAGIC 0x6DCC84UL + +/*---------------------------------------------------------------------------- + * Structure: s3r_byterange + * HTTP Request byterange info + * + * `magic` (unsigned long) + * + * "Magic" number confirming that this is an s3r_byterange structure and + * what operations are valid for it. + * + * Must be S3COMMS_BYTERANGE_MAGIC to be valid. + * + * `offset` (size_t) : + * Read bytes starting at position `offset` + * + * `len` (size_t) : + * Read `len` bytes + *---------------------------------------------------------------------------- + */ +typedef struct { + unsigned long magic; + size_t offset; + size_t len; +} s3r_byterange; +#define S3COMMS_BYTERANGE_MAGIC 0x41fab3UL + +/*---------------------------------------------------------------------------- + * + * Structure: s3r_t + * + * + * + * S3 request structure "handle". + * + * Holds persistent information for Amazon S3 requests. + * + * Instantiated through `NCH5_s3comms_s3r_open()`, copies data into self. + * + * Intended to be re-used for operations on a remote object. + * + * Cleaned up through `NCH5_s3comms_s3r_close()`. + * + * _DO NOT_ share handle between threads: curl easy handle `curlhandle` has + * undefined behavior if called to perform in multiple threads. + * + * + * + * `magic` (unsigned long) + * + * "magic" number identifying this structure as unique type. + * MUST equal `S3R_MAGIC` to be valid. + * + * `curlhandle` (CURL) + * + * Pointer to the curl_easy handle generated for the request. + * + * `httpverb` (char *) + * + * Pointer to NULL-terminated string. HTTP verb, + * e.g. "GET", "HEAD", "PUT", etc. + * + * Default is NULL, resulting in a "GET" request. + * + * `purl` (NCuri*) see ncuri.h + * Cannot be NULL. + * + * `region` (char *) + * + * Pointer to NULL-terminated string, specifying S3 "region", + * e.g., "us-east-1". + * + * Required to authenticate. + * + * `secret_id` (char *) + * + * Pointer to NULL-terminated string for "secret" access id to S3 resource. + * + * Required to authenticate. + * + * `signing_key` (unsigned char *) + * + * Pointer to `SHA256_DIGEST_LENGTH`-long string for "re-usable" signing + * key, generated via + * `HMAC-SHA256(HMAC-SHA256(HMAC-SHA256(HMAC-SHA256("AWS4", + * ""), ""), "aws4_request")` + * which may be re-used for several (up to seven (7)) days from creation? + * Computed once upon file open. + * + * Required to authenticate. + * + *---------------------------------------------------------------------------- + */ +typedef struct { + unsigned long magic; + struct CURL *curlhandle; + char *rootpath; /* All keys are WRT this path */ + char *region; + char *accessid; + char *accesskey; + char httpverb[S3COMMS_VERB_MAX]; + unsigned char *signing_key; /*|signing_key| = SHA256_DIGEST_LENGTH*/ + char iso8601now[ISO8601_SIZE]; + char *reply; + struct curl_slist *curlheaders; +} s3r_t; + +/* Combined storage for space + size */ +typedef struct s3r_buf_t { + unsigned long long count; /* |content| */ + void* content; +} s3r_buf_t; + + +#define S3COMMS_S3R_MAGIC 0x44d8d79 + +typedef enum HTTPVerb { +HTTPNONE=0, HTTPGET=1, HTTPPUT=2, HTTPPOST=3, HTTPHEAD=4, HTTPDELETE=5 +} HTTPVerb; + +#ifdef __cplusplus +extern "C" { +#endif + +/******************************************* + * DECLARATION OF HTTP FIELD LIST ROUTINES * + *******************************************/ + +EXTERNL int NCH5_s3comms_hrb_node_set(hrb_node_t **L, const char *name, const char *value); + +/*********************************************** + * DECLARATION OF HTTP REQUEST BUFFER ROUTINES * + ***********************************************/ + +EXTERNL int NCH5_s3comms_hrb_destroy(hrb_t *buf); + +EXTERNL hrb_t *NCH5_s3comms_hrb_init_request(const char *resource, const char *host); + +/************************************* + * DECLARATION OF S3REQUEST ROUTINES * + *************************************/ + +EXTERNL s3r_t *NCH5_s3comms_s3r_open(const char* root, const char* region, const char* id, const char* access_key); + +EXTERNL int NCH5_s3comms_s3r_close(s3r_t *handle); + +EXTERNL int NCH5_s3comms_s3r_read(s3r_t *handle, const char* url, size_t offset, size_t len, s3r_buf_t* data); + +EXTERNL int NCH5_s3comms_s3r_write(s3r_t *handle, const char* url, const s3r_buf_t* data); + +EXTERNL int NCH5_s3comms_s3r_getkeys(s3r_t *handle, const char* url, s3r_buf_t* response); + +EXTERNL int NCH5_s3comms_s3r_getsize(s3r_t *handle, const char* url, long long * sizep); + +EXTERNL int NCH5_s3comms_s3r_deletekey(s3r_t *handle, const char* url, long* httpcodep); + +EXTERNL int NCH5_s3comms_s3r_head(s3r_t *handle, const char* url, const char* header, const char* query, long* httpcodep, char** valuep); + +/********************************* + * DECLARATION OF OTHER ROUTINES * + *********************************/ + +EXTERNL struct tm *gmnow(void); + +EXTERNL int NCH5_s3comms_aws_canonical_request(struct VString* canonical_request_dest, + struct VString* signed_headers_dest, + HTTPVerb verb, + const char* query, + const char* payloadsha256, + hrb_t *http_request); + +EXTERNL int NCH5_s3comms_bytes_to_hex(char *dest, const unsigned char *msg, size_t msg_len, + int lowercase); + +EXTERNL int NCH5_s3comms_HMAC_SHA256(const unsigned char *key, size_t key_len, const char *msg, + size_t msg_len, char *dest); + +EXTERNL int NCH5_s3comms_load_aws_profile(const char *name, char *key_id_out, char *secret_access_key_out, + char *aws_region_out); + +EXTERNL int NCH5_s3comms_nlowercase(char *dest, const char *s, size_t len); + +EXTERNL int NCH5_s3comms_percent_encode_char(char *repr, const unsigned char c, size_t *repr_len); + +EXTERNL int NCH5_s3comms_signing_key(unsigned char **mdp, const char *secret, const char *region, + const char *iso8601now); + +EXTERNL int NCH5_s3comms_tostringtosign(struct VString* dest, const char *req_str, const char *now, + const char *region); + +EXTERNL int NCH5_s3comms_trim(char *dest, char *s, size_t s_len, size_t *n_written); + +EXTERNL int NCH5_s3comms_uriencode(char** destp, const char *s, size_t s_len, int encode_slash, size_t *n_written); + +#ifdef __cplusplus +} +#endif + +#endif /*NCS3COMMS_H*/ diff --git a/libdispatch/ncjson.c b/libdispatch/ncjson.c index a50b1a93aa..7730f42f72 100644 --- a/libdispatch/ncjson.c +++ b/libdispatch/ncjson.c @@ -1080,8 +1080,7 @@ NCJtotext(const NCjson* json) char* text = NULL; if(json == NULL) {strcpy(outtext,""); goto done;} (void)NCJunparse(json,0,&text); - outtext[0] = '\0'; - strlcat(outtext,text,sizeof(outtext)); + strncpy(outtext,text,sizeof(outtext)); nullfree(text); done: return outtext; diff --git a/libdispatch/nclist.c b/libdispatch/nclist.c index b5f815864c..f2c3f4d47b 100644 --- a/libdispatch/nclist.c +++ b/libdispatch/nclist.c @@ -10,7 +10,7 @@ #define strcasecmp _stricmp #endif -int nclistnull(void* e) {return e == NULL;} +int nclistisnull(void* e) {return e == NULL;} #ifndef TRUE #define TRUE 1 @@ -292,3 +292,17 @@ nclistextract(NClist* l) l->content = NULL; return result; } + +/* Extends nclist to include a NULL that is not included + in list length. + return value is always 1. +*/ +int +nclistnull(NClist* l) +{ + if(l == NULL || l->length == 0) return 1; + nclistpush(l,NULL); + nclistsetlength(l,l->length-1); + return 1; +} + diff --git a/libdispatch/nclog.c b/libdispatch/nclog.c index b474b8c4b9..288d93e308 100644 --- a/libdispatch/nclog.c +++ b/libdispatch/nclog.c @@ -141,7 +141,7 @@ ncvlog(int tag, const char* fmt, va_list ap) if(tag == NCLOGERR) was = ncsetlogging(1); if(!nclog_global.nclogging || nclog_global.nclogstream == NULL) return was; prefix = nctagname(tag); - fprintf(nclog_global.nclogstream,"%s:",prefix); + fprintf(nclog_global.nclogstream,"%s: ",prefix); if(fmt != NULL) { vfprintf(nclog_global.nclogstream, fmt, ap); } diff --git a/libdispatch/ncrandom.c b/libdispatch/ncrandom.c new file mode 100644 index 0000000000..bfb1fbe3ca --- /dev/null +++ b/libdispatch/ncrandom.c @@ -0,0 +1,31 @@ +/* + * Copyright 2018, University Corporation for Atmospheric Research + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#ifdef HAVE_STDLIB_H +#include +#endif +#ifdef HAVE_STDIO_H +#include +#endif + + +/* Support platform independent generation of 32-bit unsigned int random numbers */ + +int +main() { + unsigned int urnd = 0; /* range 0..2147483647 */ +#ifdef WIN32 + (void)rand_s(&urnd); +#else + long rnd; + rnd = random(); + urnd = (unsigned)(rnd & 0xffffffff); +#endif + printf("%u",urnd); + exit(0); +} diff --git a/libdispatch/ncs3sdk.cpp b/libdispatch/ncs3sdk_aws.cpp similarity index 69% rename from libdispatch/ncs3sdk.cpp rename to libdispatch/ncs3sdk_aws.cpp index d415052d2f..18ca603905 100644 --- a/libdispatch/ncs3sdk.cpp +++ b/libdispatch/ncs3sdk_aws.cpp @@ -37,6 +37,46 @@ extern char* strdup(const char*); #define size64_t unsigned long long +struct KeySet { + size_t nkeys; + size_t alloc; + char** keys; + KeySet() {nkeys = 0; alloc = 0; keys = NULL;} + ~KeySet() {clear();} + void push(char* key) { + if(alloc == 0) { + keys = (char**)calloc(10,sizeof(char*)); + alloc = 10; + } + if(nkeys >= alloc) { + char** newkeys = (char**)calloc(alloc*2,sizeof(char*)); + memcpy(newkeys,keys,sizeof(char*)*nkeys); + alloc *= 2; + free(keys); + keys = newkeys; + } + keys[nkeys++] = key; + } + char** extractkeys() {char** outkeys = keys; keys = NULL; clear(); return outkeys;} + char* extractithkey(size_t i) { + if(i >= nkeys) return NULL; + char* k = keys[i]; + keys[i] = NULL; + return k; + } + size_t getnkeys() {return nkeys;} + void clear() { + if(keys) { + for(size_t i=0;i list, size_t*, char*** keysp, size64_t** lenp); -static int s3commonprefixes(Aws::Vector list, char*** keysp); -static void freestringenvv(char** ss); +static int s3objectinfo1(const Aws::S3::Model::Object& s3_object, char** fullkeyp); +static int s3objectsinfo(Aws::Vector list, KeySet* keys); +static int s3commonprefixes(Aws::Vector list, KeySet* keys); static int makes3key(const char* pathkey, const char** keyp); static int makes3keydir(const char* prefix, char** prefixdirp); -static char** mergekeysets(size_t nkeys1, char** keys1, size_t nkeys2, char** keys2); +static int mergekeysets(KeySet* keys1, KeySet* keys2, KeySet* merge); -static const char* -dumps3info(NCS3INFO* info) +const char* +NCS3_dumps3info(NCS3INFO* info) { static char text[8192]; snprintf(text,sizeof(text),"host=%s region=%s bucket=%s rootkey=%s profile=%s", @@ -73,6 +112,9 @@ NC_s3sdkinitialize(void) ncs3_finalized = 0; NCTRACE(11,NULL); Aws::InitAPI(ncs3options); +#ifdef DEBUG + ncs3options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug; +#endif } return NCUNTRACE(NC_NOERR); } @@ -114,13 +156,7 @@ s3sdkcreateconfig(NCS3INFO* info) Aws::Client::ClientConfiguration config; if(info->profile) -#if 0 - config = new Aws::Client::ClientConfiguration(info->profile); - else - config = new Aws::Client::ClientConfiguration(); -#else config.profileName = info->profile; -#endif config.scheme = Aws::Http::Scheme::HTTPS; config.connectTimeoutMs = 300000; config.requestTimeoutMs = 600000; @@ -128,13 +164,14 @@ s3sdkcreateconfig(NCS3INFO* info) if(info->host) config.endpointOverride = info->host; config.enableEndpointDiscovery = true; config.followRedirects = Aws::Client::FollowRedirectsPolicy::ALWAYS; - NCUNTRACE(NC_NOERR); + stat = NCUNTRACE(stat); return config; } EXTERNL void* NC_s3sdkcreateclient(NCS3INFO* info) { + int stat = NC_NOERR; NCTRACE(11,NULL); Aws::Client::ClientConfiguration config = s3sdkcreateconfig(info); @@ -153,7 +190,7 @@ NC_s3sdkcreateclient(NCS3INFO* info) false); } // delete config; - NCUNTRACE(NC_NOERR); + stat = NCUNTRACE(stat); return (void*)s3client; } @@ -166,9 +203,14 @@ NC_s3sdkbucketexists(void* s3client0, const char* bucket, int* existsp, char** e NCTRACE(11,"bucket=%s",bucket); if(errmsgp) *errmsgp = NULL; +#if 0 + char* errmsg = NULL; auto result = s3client->ListBuckets(); if(!result.IsSuccess()) { - if(errmsgp) *errmsgp = makeerrmsg(result.GetError()); + auto re = result.GetError(); + errmsg = makeerrmsg(re); + fprintf(stderr,">>> errmsg=%s\n",errmsg); + if(errmsgp) {*errmsgp = errmsg; errmsg = NULL;} stat = NC_ES3; } else { Aws::Vector bucket_list = result.GetResult().GetBuckets(); @@ -177,6 +219,12 @@ NC_s3sdkbucketexists(void* s3client0, const char* bucket, int* existsp, char** e if(name == bucket) {exists = 1; break;} } } +#else + Aws::S3::Model::HeadBucketRequest request; + request.SetBucket(bucket); + auto result = s3client->HeadBucket(request); + exists = result.IsSuccess() ? 1 : 0; +#endif if(existsp) *existsp = exists; return NCUNTRACEX(stat,"exists=%d",*existsp); } @@ -398,61 +446,87 @@ Return a list of names of legal objects immediately below a specified key. In theory, the returned list should be sorted in lexical order, but it possible that it is not. */ -EXTERNL int -NC_s3sdkgetkeys(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp) +static int +getkeys(void* s3client0, const char* bucket, const char* prefixkey0, const char* delim, size_t* nkeysp, char*** keysp, char** errmsgp) { int stat = NC_NOERR; - size_t nkeys = 0; const char* prefix = NULL; char* prefixdir = NULL; - char** realkeys = NULL; - char** commonkeys = NULL; - char** allkeys = NULL; - + bool istruncated = false; + char* continuetoken = NULL; + Aws::S3::S3Client* s3client = NULL; + KeySet commonkeys; + KeySet realkeys; + KeySet allkeys; + NCTRACE(11,"bucket=%s prefixkey0=%s",bucket,prefixkey0); - Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0; - Aws::S3::Model::ListObjectsV2Request objects_request; + if(*prefixkey0 != '/') {stat = NC_EINTERNAL; goto done;} + if(errmsgp) *errmsgp = NULL; - if(*prefixkey0 != '/') return NCUNTRACE(NC_EINTERNAL); - /* Make sure that the prefix ends with '/' */ - if((stat = makes3keydir(prefixkey0,&prefixdir))) return NCUNTRACE(stat); - /* remove leading '/' */ - if((stat = makes3key(prefixdir,&prefix))) return NCUNTRACE(stat); + s3client = (Aws::S3::S3Client*)s3client0; + + do { + Aws::S3::Model::ListObjectsV2Request objects_request; + if(prefixdir != NULL) free(prefixdir); + realkeys.clear(); + commonkeys.clear(); + /* Make sure that the prefix ends with '/' */ + if((stat = makes3keydir(prefixkey0,&prefixdir))) goto done; + /* remove leading '/' */ + if((stat = makes3key(prefixdir,&prefix))) goto done; + objects_request.SetBucket(bucket); + objects_request.SetPrefix(prefix); + if(delim != NULL) { + objects_request.SetDelimiter(delim); /* Force return of common prefixes */ + } + if(istruncated && continuetoken != NULL) { + objects_request.SetContinuationToken(continuetoken); + free(continuetoken); continuetoken = NULL; + } + auto objects_outcome = s3client->ListObjectsV2(objects_request); + if(objects_outcome.IsSuccess()) { + const Aws::S3::Model::ListObjectsV2Result& result = objects_outcome.GetResult(); + istruncated = result.GetIsTruncated(); + if(istruncated) + continuetoken = strdup(result.GetNextContinuationToken().c_str()); + Aws::Vector object_list = + objects_outcome.GetResult().GetContents(); + if((stat = s3objectsinfo(object_list,&realkeys))) goto done; + /* Add common prefixes */ + Aws::Vector common_list = + objects_outcome.GetResult().GetCommonPrefixes(); + if((stat = s3commonprefixes(common_list,&commonkeys))) goto done; + /* merge the two lists; target list might not be empty */ + if((stat=mergekeysets(&realkeys, &commonkeys, &allkeys))) goto done; + } else { + if(errmsgp) *errmsgp = makeerrmsg(objects_outcome.GetError()); + stat = NC_ES3; + goto done; + } + } while(istruncated); + if(nkeysp) {*nkeysp = allkeys.getnkeys();} + if(keysp) {*keysp = allkeys.extractkeys();} +done: + realkeys.clear(); + commonkeys.clear(); + allkeys.clear(); + if(prefixdir != NULL) free(prefixdir); + if(continuetoken != NULL) free(continuetoken); + if(stat) + return NCUNTRACE(stat); + else + return NCUNTRACEX(stat,"nkeys=%u",(unsigned)*nkeysp); +} - if(errmsgp) *errmsgp = NULL; - objects_request.SetBucket(bucket); - objects_request.SetPrefix(prefix); - objects_request.SetDelimiter("/"); /* Force return of common prefixes */ - auto objects_outcome = s3client->ListObjectsV2(objects_request); - if(objects_outcome.IsSuccess()) { - size_t nrealkeys = 0; - size_t ncommonkeys = 0; - Aws::Vector object_list = - objects_outcome.GetResult().GetContents(); - nrealkeys = (size_t)object_list.size(); - stat = s3objectsinfo(object_list,NULL,&realkeys,NULL); - /* Add common prefixes */ - Aws::Vector common_list = - objects_outcome.GetResult().GetCommonPrefixes(); - ncommonkeys = (size_t)common_list.size(); - stat = s3commonprefixes(common_list,&commonkeys); - /* merge the two lists */ - if((allkeys=mergekeysets(nrealkeys, realkeys, ncommonkeys, commonkeys))==NULL) - stat = NC_ENOMEM; - if(stat == NC_NOERR) { - if(nkeysp) {*nkeysp = nrealkeys + ncommonkeys;} - if(keysp) {*keysp = allkeys; allkeys = NULL;} - } - freestringenvv(allkeys); - freestringenvv(realkeys); - freestringenvv(commonkeys); - } else { - if(errmsgp) *errmsgp = makeerrmsg(objects_outcome.GetError()); - stat = NC_ES3; - } - if(prefixdir) free(prefixdir); - return NCUNTRACEX(stat,"nkeys=%u",(unsigned)*nkeysp); +/* +Return a list of full keys of legal objects immediately below a specified key. +Not necessarily sorted. +*/ +EXTERNL int +NC_s3sdkgetkeys(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp) +{ + return getkeys(s3client0, bucket, prefixkey0, "/", nkeysp, keysp, errmsgp); } /* @@ -462,63 +536,7 @@ Not necessarily sorted. EXTERNL int NC_s3sdksearch(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp) { - int stat = NC_NOERR; - size_t nkeys = 0; - const char* prefix = NULL; - char* prefixdir = NULL; - char** keys = NULL; - - NCTRACE(11,"bucket=%s prefixkey0=%s",bucket,prefixkey0); - - Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0; - Aws::S3::Model::ListObjectsV2Request objects_request; - - if(*prefixkey0 != '/') return NCUNTRACE(NC_EINTERNAL); - /* Make sure that the prefix ends with '/' */ - if((stat = makes3keydir(prefixkey0,&prefixdir))) return NCUNTRACE(stat); - /* remove leading '/' */ - if((stat = makes3key(prefixdir,&prefix))) return NCUNTRACE(stat); - - if(errmsgp) *errmsgp = NULL; - objects_request.SetBucket(bucket); - objects_request.SetPrefix(prefix); - /* Do not use delimiter so we get all full key paths */ - auto objects_outcome = s3client->ListObjectsV2(objects_request); - if(objects_outcome.IsSuccess()) { - size_t nkeys = 0; - Aws::Vector object_list = - objects_outcome.GetResult().GetContents(); - nkeys = (size_t)object_list.size(); - if((keys=(char**)calloc(sizeof(char*),(nkeys+1)))==NULL) /* NULL terminate list */ - stat = NC_ENOMEM; - if(!stat) { - int i; - i = 0; - for (auto const &s3obj : object_list) { - const char* s; - const Aws::String& name = s3obj.GetKey(); - s = name.c_str(); - if(s != NULL) { - char* p; - size_t slen = name.length(); - p = (char*)malloc(slen+1+1); /* nul plus leading '/' */ - if(s[0] != '/') {p[0] = '/'; memcpy(p+1,s,slen); slen++;} else {memcpy(p,s,slen);} - p[slen] = '\0'; - keys[i++] = p; - } - } - } - if(stat == NC_NOERR) { - if(nkeysp) {*nkeysp = nkeys;} - if(keysp) {*keysp = keys; keys = NULL;} - } - freestringenvv(keys); - } else { - if(errmsgp) *errmsgp = makeerrmsg(objects_outcome.GetError()); - stat = NC_ES3; - } - if(prefixdir) free(prefixdir); - return NCUNTRACEX(stat,"nkeys=%u",(unsigned)*nkeysp); + return getkeys(s3client0, bucket, prefixkey0, NULL, nkeysp, keysp, errmsgp); } EXTERNL int @@ -532,7 +550,7 @@ NC_s3sdkdeletekey(void* s3client0, const char* bucket, const char* pathkey, char Aws::S3::S3Client* s3client = (Aws::S3::S3Client*)s3client0; Aws::S3::Model::DeleteObjectRequest delete_request; - if(*pathkey != '/') return NC_EINTERNAL; + assert(pathkey != NULL && *pathkey == '/'); if((stat = makes3key(pathkey,&key))) return NCUNTRACE(stat); /* Delete this key object */ delete_request.SetBucket(bucket); @@ -549,7 +567,7 @@ NC_s3sdkdeletekey(void* s3client0, const char* bucket, const char* pathkey, char Get Info about a single object from a vector */ static int -s3objectinfo1(const Aws::S3::Model::Object& s3_object, char** fullkeyp, size64_t* lenp) +s3objectinfo1(const Aws::S3::Model::Object& s3_object, char** fullkeyp) { int stat = NC_NOERR; char* cstr = NULL; @@ -577,9 +595,6 @@ s3objectinfo1(const Aws::S3::Model::Object& s3_object, char** fullkeyp, size64_t cstr = NULL; } } - if(!stat) { - if(lenp) *lenp = (size64_t)s3_object.GetSize(); - } if(cstr) free(cstr); return stat; } @@ -588,73 +603,45 @@ s3objectinfo1(const Aws::S3::Model::Object& s3_object, char** fullkeyp, size64_t Get Info about a vector of objects; Keys are fixed up to start with a '/' */ static int -s3objectsinfo(Aws::Vector list, size_t* nkeysp, char*** keysp, size64_t** lenp) +s3objectsinfo(Aws::Vector list, KeySet* keys) { int stat = NC_NOERR; - char** keys = NULL; - size_t nkeys; - size64_t *lengths = NULL; int i; - nkeys = list.size(); - if(nkeysp) *nkeysp = nkeys; - if((keys=(char**)calloc(sizeof(char*),(nkeys+1)))==NULL) - stat = NC_ENOMEM; - if(!stat) { - if((lengths=(size64_t*)calloc(sizeof(size64_t),(nkeys)))==NULL) - stat = NC_ENOMEM; - } - if(!stat) { - i = 0; - for (auto const &s3_object : list) { - stat = s3objectinfo1(s3_object,&keys[i],&lengths[i]); - i++; - if(stat) break; - } + i = 0; + for (auto const &s3_object : list) { + char* akey = NULL; + stat = s3objectinfo1(s3_object,&akey); + i++; + if(stat) break; + keys->push(akey); } - if(!stat) { - if(keysp) {keys[nkeys] = NULL; *keysp = keys; keys = NULL;} - if(lenp) {*lenp = lengths; lengths = NULL;} - } - if(keys != NULL) freestringenvv(keys); - if(lengths != NULL) free(lengths); return stat; } static int -s3commonprefixes(Aws::Vector list, char*** keysp) +s3commonprefixes(Aws::Vector list, KeySet* keys) { int stat = NC_NOERR; - char** keys = NULL; - size_t nkeys; - int i; - nkeys = list.size(); - if((keys=(char**)calloc(sizeof(char*),(nkeys+1)))==NULL) - stat = NC_ENOMEM; - if(!stat) { - i = 0; - for (auto const &s3prefix : list) { - char* p; const char* px; char* p1; - size_t len, alloc; - const Aws::String& prefix = s3prefix.GetPrefix(); - len = prefix.length(); - alloc = len + 1 + 1; /* for nul + leading '/' */ - if((p = (char*) malloc(alloc))==NULL) - stat = NC_ENOMEM; - if(stat == NC_NOERR) { - px = prefix.c_str(); - p1 = p; - if(*px != '/') *p1++ = '/'; - memcpy(p1,px,len); - p1 += len; - *p1 = '\0'; - keys[i++] = p; - } + for (auto const &s3prefix : list) { + char* p; const char* px; char* p1; + size_t len, alloc; + const Aws::String& prefix = s3prefix.GetPrefix(); + len = prefix.length(); + alloc = len + 1 + 1; /* for nul + leading '/' */ + if((p = (char*) malloc(alloc))==NULL) + stat = NC_ENOMEM; + if(stat == NC_NOERR) { + px = prefix.c_str(); + p1 = p; + if(*px != '/') *p1++ = '/'; + memcpy(p1,px,len); + p1 += len; + *p1 = '\0'; + keys->push(p); } } - if(keysp) {keys[nkeys] = NULL; *keysp = keys; keys = NULL;} - if(keys != NULL) freestringenvv(keys); return stat; } @@ -671,6 +658,7 @@ struct MemBuf: std::streambuf { } }; +#if 0 static void freestringenvv(char** ss) { @@ -681,6 +669,7 @@ freestringenvv(char** ss) free(ss); } } +#endif static int makes3key(const char* pathkey, const char** keyp) @@ -705,25 +694,13 @@ makes3keydir(const char* prefix, char** prefixdirp) return NC_NOERR; } -static char** -mergekeysets(size_t nkeys1, char** keys1, size_t nkeys2, char** keys2) +static int +mergekeysets(KeySet* keys1, KeySet* keys2, KeySet* merge) { - char** merge = NULL; - char** p1; - char** p2; - - merge = (char**)malloc(sizeof(char*)*(nkeys1+nkeys2+1)); - if(merge == NULL) return NULL; - if(nkeys1 > 0) { - memcpy(merge,keys1,sizeof(char*)*nkeys1); - /* avoid double free */ - memset(keys1,0,sizeof(char*)*nkeys1); - } - if(nkeys2 > 0) { - memcpy(merge+nkeys1,keys2,sizeof(char*)*nkeys2); - /* avoid double free */ - memset(keys2,0,sizeof(char*)*nkeys2); - } - merge[nkeys1+nkeys2] = NULL; - return merge; + size_t i; + for(i=0;igetnkeys();i++) + merge->push(keys1->extractithkey(i)); + for(i=0;igetnkeys();i++) + merge->push(keys2->extractithkey(i)); + return NC_NOERR; } diff --git a/libdispatch/ncs3sdk_h5.c b/libdispatch/ncs3sdk_h5.c new file mode 100644 index 0000000000..15e96cefdc --- /dev/null +++ b/libdispatch/ncs3sdk_h5.c @@ -0,0 +1,1045 @@ +/* + * Copyright 2018, University Corporation for Atmospheric Research + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + */ + +#define NOOP +#undef DEBUG + +#include "config.h" +#include +#include +#include +#include "netcdf.h" +#include "nclog.h" +#include "ncrc.h" +#include "ncxml.h" + +#include "nch5s3comms.h" +#include "ncs3sdk.h" + +#define NCTRACING +#ifdef NCTRACING +#define NCTRACE(level,fmt,...) nctrace((level),__func__,fmt,##__VA_ARGS__) +#define NCTRACEMORE(level,fmt,...) nctracemore((level),fmt,##__VA_ARGS__) +#define NCUNTRACE(e) ncuntrace(__func__,NCTHROW(e),NULL) +#define NCUNTRACEX(e,fmt,...) ncuntrace(__func__,NCTHROW(e),fmt,##__VA_ARGS__) +#define NCNILTRACE(e) ((void)NCUNTRACE(e)) +#else +#define NCTRACE(level,fmt,...) +#define NCTRACEMORE(level,fmt,...) +#define NCUNTRACE(e) (e) +#define NCNILTRACE(e) +#define NCUNTRACEX(e,fmt,...) (e) +#endif + +#ifdef __CYGWIN__ +extern char* strdup(const char*); +#endif + +#ifndef SHA256_DIGEST_LENGTH +#define SHA256_DIGEST_LENGTH 32 +#endif + +/* Mnemonic */ +#define RECLAIM 1 + +#define size64 unsigned long long + +typedef struct NCS3CLIENT { + char* rooturl; /* The URL (minus any fragment) for the dataset root path (excludes bucket on down) */ + s3r_t* h5s3client; /* From h5s3comms */ +} NCS3CLIENT; + +struct Object { + NClist* checksumalgorithms; /* NClist */ + char* etag; + char* key; + char* lastmodified; + struct Owner { + char* displayname; + char* id; + } owner; + char* size; + char* storageclass; +}; + +typedef char* CommonPrefix; + +/* Hold essential info from a ListObjectsV2 response */ +struct LISTOBJECTSV2 { + char* istruncated; + NClist* contents; /* NClist */ + char* name; + char* prefix; + char* delimiter; + char* maxkeys; + NClist* commonprefixes; /* NClist */ + char* encodingtype; + char* keycount; + char* continuationtoken; + char* nextcontinuationtoken; + char* startafter; +}; + +static int ncs3_initialized = 0; +static int ncs3_finalized = 0; + +/* Forward */ +static void s3client_destroy(NCS3CLIENT* s3client); +static char* makes3rooturl(NCS3INFO* info); +static int makes3fullpath(const char* pathkey, const char* bucket, const char* prefix, const char* key, NCbytes* url); +static int parse_listbucketresult(char* xml, unsigned long long xmllen, struct LISTOBJECTSV2**); +static int parse_object(ncxml_t root, NClist* objects); +static int parse_owner(ncxml_t root, struct Owner* ownerp); +static int parse_prefix(ncxml_t root, NClist* prefixes); +static int parse_checksumalgorithm(ncxml_t root, NClist* algorithms); +static struct LISTOBJECTSV2* alloclistobjectsv2(void); +static struct Object* allocobject(void); +static void reclaim_listobjectsv2(struct LISTOBJECTSV2* lo); +static void reclaim_object(struct Object* o); +static char* trim(char* s, int reclaim); +static int makes3prefix(const char* prefix, char** prefixdirp); +static int s3objectsinfo(NClist* contents, NClist* keys, NClist* lens); +static int s3commonprefixes(NClist* list, NClist* keys); +static int mergekeysets(NClist*,NClist*,NClist*); +static int rawtokeys(s3r_buf_t* response, NClist* keys, NClist* lengths, struct LISTOBJECTSV2** listv2p); + +static int queryadd(NClist* query, const char* key, const char* value); +static int queryend(NClist* query, char** querystring); +static int queryinsert(NClist* list, char* ekey, char* evalue); + +#define NT(x) ((x)==NULL?"null":x) + +#if 0 +static void +dumps3info(NCS3INFO* s3info, const char* tag) +{ + if(tag == NULL) tag = "dumps3info"; + fprintf(stderr,">>> %s: s3info=%p\n",tag,s3info); + if(s3info != NULL) { + fprintf(stderr,">>> %s: s3info->host=%s\n",tag,NT(s3info->host)); + fprintf(stderr,">>> %s: s3info->region=%s\n",tag,NT(s3info->region)); + fprintf(stderr,">>> %s: s3info->bucket=%s\n",tag,NT(s3info->bucket)); + fprintf(stderr,">>> %s: s3info->rootkey=%s\n",tag,NT(s3info->rootkey)); + fprintf(stderr,">>> %s: s3info->profile=%s\n",tag,NT(s3info->profile)); + } +} + +static void +dumps3client(void* s3client0, const char* tag) +{ + NCS3CLIENT* s3client = (NCS3CLIENT*)s3client0; + if(tag == NULL) tag = "dumps3client"; + fprintf(stderr,">>> %s: s3client=%p\n",tag,s3client); + if(s3client != NULL) { + fprintf(stderr,">>> %s: s3client->rooturl=%s\n",tag,NT(s3client->rooturl)); + fprintf(stderr,">>> %s: s3client->h5s3client=%p\n",tag,s3client->rooturl); + } +} +#endif + +/**************************************************/ + +EXTERNL int +NC_s3sdkinitialize(void) +{ + NCTRACE(11,NULL); + if(!ncs3_initialized) { + ncs3_initialized = 1; + ncs3_finalized = 0; + } + return NCUNTRACE(NC_NOERR); +} + +EXTERNL int +NC_s3sdkfinalize(void) +{ + NCTRACE(11,NULL); + if(!ncs3_finalized) { + ncs3_initialized = 0; + ncs3_finalized = 1; + } + return NCUNTRACE(NC_NOERR); +} + +EXTERNL void* +NC_s3sdkcreateclient(NCS3INFO* info) +{ + int stat = NC_NOERR; + const char* accessid = NULL; + const char* accesskey = NULL; + char* urlroot = NULL; + NCS3CLIENT* s3client = NULL; + + NCTRACE(11,"info=%s",NC_s3dumps3info(info)); + + s3client = (NCS3CLIENT*)calloc(1,sizeof(NCS3CLIENT)); + if(s3client == NULL) goto done; + if(info->profile != NULL) { + if((stat = NC_s3profilelookup(info->profile, "aws_access_key_id", &accessid))) goto done; + if((stat = NC_s3profilelookup(info->profile, "aws_secret_access_key", &accesskey))) goto done; + } + if((s3client->rooturl = makes3rooturl(info))==NULL) {stat = NC_ENOMEM; goto done;} + s3client->h5s3client = NCH5_s3comms_s3r_open(s3client->rooturl,info->region,accessid,accesskey); + if(s3client->h5s3client == NULL) {stat = NC_ES3; goto done;} + +done: + nullfree(urlroot); + if(stat && s3client) { + NC_s3sdkclose(s3client,info,0,NULL); + s3client = NULL; + } + NCNILTRACE(NC_NOERR); + return (void*)s3client; +} + +EXTERNL int +NC_s3sdkbucketexists(void* s3client0, const char* bucket, int* existsp, char** errmsgp) +{ + int stat = NC_NOERR; + NCS3CLIENT* s3client = (NCS3CLIENT*)s3client0; + NCbytes* url = ncbytesnew(); + long httpcode = 0; + + NCTRACE(11,"bucket=%s",bucket); + if(errmsgp) *errmsgp = NULL; + + if((stat = makes3fullpath(s3client->rooturl,bucket,NULL,NULL,url))) goto done; + if((stat = NCH5_s3comms_s3r_head(s3client->h5s3client, ncbytescontents(url), NULL, NULL, &httpcode, NULL))) goto done; + + if(existsp) {*existsp = (stat == 0 && httpcode == 200);} +done: + ncbytesfree(url); + return NCUNTRACEX(stat,"exists=%d",PTRVAL(int,existsp,-1)); +} + +EXTERNL int +NC_s3sdkbucketcreate(void* s3client0, const char* region, const char* bucket, char** errmsgp) +{ + int stat = NC_NOERR; + NCS3CLIENT* s3client = (NCS3CLIENT*)s3client0; + + NC_UNUSED(s3client); + + NCTRACE(11,"region=%s bucket=%s",region,bucket); + if(errmsgp) *errmsgp = NULL; + fprintf(stderr,"create bucket: %s\n",bucket); fflush(stderr); + return NCUNTRACE(stat); +} + +EXTERNL int +NC_s3sdkbucketdelete(void* s3client0, NCS3INFO* info, char** errmsgp) +{ + int stat = NC_NOERR; + NCS3CLIENT* s3client = (NCS3CLIENT*)s3client0; + + NC_UNUSED(s3client); + + NCTRACE(11,"info=%s%s",NC_s3dumps3info(info)); + + if(errmsgp) *errmsgp = NULL; + fprintf(stderr,"delete bucket: %s\n",info->bucket); fflush(stderr); + + return NCUNTRACE(stat); +} + +/**************************************************/ +/* Object API */ + +/* +@return NC_NOERR if key points to a content-bearing object. +@return NC_EEMPTY if object at key has no content. +@return NC_EXXX return true error +*/ +EXTERNL int +NC_s3sdkinfo(void* s3client0, const char* bucket, const char* pathkey, size64_t* lenp, char** errmsgp) +{ + int stat = NC_NOERR; + NCS3CLIENT* s3client = (NCS3CLIENT*)s3client0; + NCbytes* url = ncbytesnew(); + long long len = -1; + + NCTRACE(11,"bucket=%s pathkey=%s",bucket,pathkey); + + if((stat = makes3fullpath(s3client->rooturl,bucket,pathkey,NULL,url))) goto done; + if((stat = NCH5_s3comms_s3r_getsize(s3client->h5s3client, ncbytescontents(url), &len))) goto done; + + if(lenp) {*lenp = len;} + +done: + ncbytesfree(url); + return NCUNTRACEX(stat,"len=%d",PTRVAL(int,lenp,-1)); +} + +/* +@return NC_NOERR if success +@return NC_EXXX if fail +*/ +EXTERNL int +NC_s3sdkread(void* s3client0, const char* bucket, const char* pathkey, size64_t start, size64_t count, void* content, char** errmsgp) +{ + int stat = NC_NOERR; + NCS3CLIENT* s3client = (NCS3CLIENT*)s3client0; + NCbytes* url = ncbytesnew(); + struct s3r_buf_t data = {0,NULL}; + + NCTRACE(11,"bucket=%s pathkey=%s start=%llu count=%llu content=%p",bucket,pathkey,start,count,content); + + if((stat = makes3fullpath(s3client->rooturl,bucket,pathkey,NULL,url))) goto done; + + /* Read the data */ + data.count = count; + data.content = content; + if((stat = NCH5_s3comms_s3r_read(s3client->h5s3client,ncbytescontents(url),(size_t)start,(size_t)count,&data))) goto done; + +done: + ncbytesfree(url); + return NCUNTRACE(stat); +} + +/* +For S3, I can see no way to do a byterange write; +so we are effectively writing the whole object +*/ +EXTERNL int +NC_s3sdkwriteobject(void* s3client0, const char* bucket, const char* pathkey, size64_t count, const void* content, char** errmsgp) +{ + int stat = NC_NOERR; + NCS3CLIENT* s3client = (NCS3CLIENT*)s3client0; + NCbytes* url = ncbytesnew(); + s3r_buf_t data; + + NCTRACE(11,"bucket=%s pathkey=%s count=%llu content=%p",bucket,pathkey,count,content); + + if((stat = makes3fullpath(s3client->rooturl,bucket,pathkey,NULL,url))) goto done; + + /* Write the data */ + data.count = count; + data.content = (void*)content; + if((stat = NCH5_s3comms_s3r_write(s3client->h5s3client,ncbytescontents(url),&data))) goto done; + +done: + ncbytesfree(url); + return NCUNTRACE(stat); +} + +EXTERNL int +NC_s3sdkclose(void* s3client0, NCS3INFO* info, int deleteit, char** errmsgp) +{ + int stat = NC_NOERR; + NCS3CLIENT* s3client = (NCS3CLIENT*)s3client0; + + NCTRACE(11,"info=%s deleteit=%d",NC_s3dumps3info(info),deleteit); + + if(deleteit) { + /* Delete the root key; ok it if does not exist */ + switch (stat = NC_s3sdkdeletekey(s3client0,info->bucket,info->rootkey,errmsgp)) { + case NC_NOERR: break; + case NC_EEMPTY: case NC_ENOTFOUND: stat = NC_NOERR; break; + default: break; + } + } + s3client_destroy(s3client); + return NCUNTRACE(stat); +} + +/* +Common code for getkeys and searchkeys. +Return a list of names of legal objects immediately below a specified key. +In theory, the returned list should be sorted in lexical order, +but it possible that it is not. +*/ +static int +getkeys(void* s3client0, const char* bucket, const char* prefixkey0, const char* delim, size_t* nkeysp, char*** keysp, char** errmsgp) +{ + int stat = NC_NOERR; + NCS3CLIENT* s3client = (NCS3CLIENT*)s3client0; + char* prefixdir = NULL; + NClist* query = NULL; + char* querystring = NULL; + NCURI* purl = NULL; + NCbytes* listurl = ncbytesnew(); + NClist* allkeys = nclistnew(); + struct LISTOBJECTSV2* listv2 = NULL; + int istruncated = 0; + char* continuetoken = NULL; + s3r_buf_t response = {0,NULL}; + + NCTRACE(11,"bucket=%s prefixkey0=%s",bucket,prefixkey0); + + /* cleanup the prefix */ + if((stat = makes3prefix(prefixkey0,&prefixdir))) return NCUNTRACE(stat); + + do { + nclistfreeall(query); + query = nclistnew(); + nullfree(querystring); + querystring = NULL; + ncbytesclear(listurl); + nullfree(response.content); response.content = NULL; response.count = 0; + /* Make sure order is sorted (after encoding) */ + if((stat = queryadd(query,"list-type","2"))) goto done; + if((stat = queryadd(query,"prefix",prefixdir))) goto done; + if(delim != NULL) { + if((stat = queryadd(query,"delimiter",delim))) goto done; + } + if(istruncated && continuetoken != NULL) { + if((stat = queryadd(query,"continuation-token",continuetoken))) goto done; + } + if((stat = queryend(query,&querystring))) goto done; + + /* Build the proper url; leave off prefix because it will appear in the query */ + if((stat = makes3fullpath(s3client->rooturl, bucket, NULL, NULL, listurl))) goto done; + /* Append the query string */ + ncbytescat(listurl,"?"); + ncbytescat(listurl,querystring); + + if((stat = NCH5_s3comms_s3r_getkeys(s3client->h5s3client, ncbytescontents(listurl), &response))) goto done; + if((stat = rawtokeys(&response,allkeys,NULL,&listv2))) goto done; + istruncated = (strcasecmp(listv2->istruncated,"true")==0?1:0); + nullfree(continuetoken); + continuetoken = nulldup(listv2->nextcontinuationtoken); + reclaim_listobjectsv2(listv2); listv2 = NULL; + } while(istruncated); + if(nkeysp) {*nkeysp = nclistlength(allkeys);} + if(keysp) {*keysp = nclistextract(allkeys);} + +done: + nullfree(continuetoken); + reclaim_listobjectsv2(listv2); + nclistfreeall(allkeys); + nclistfreeall(query); + nullfree(querystring); + ncurifree(purl); + ncbytesfree(listurl); + nullfree(response.content); + if(prefixdir) free(prefixdir); + return NCUNTRACEX(stat,"nkeys=%u",PTRVAL(unsigned,nkeysp,0)); +} + +/* +Return a list of names of legal objects immediately below a specified key. +In theory, the returned list should be sorted in lexical order, +but it possible that it is not. +*/ +EXTERNL int +NC_s3sdkgetkeys(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp) +{ + NCTRACE(11,"bucket=%s prefixkey0=%s",bucket,prefixkey0); + return getkeys(s3client0, bucket, prefixkey0, "/", nkeysp, keysp, errmsgp); +} + +/* +Return a list of full keys of legal objects immediately below a specified key. +Not necessarily sorted. +Essentially same as getkeys, but with no delimiter. +*/ +EXTERNL int +NC_s3sdksearch(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp) +{ + NCTRACE(11,"bucket=%s prefixkey0=%s",bucket,prefixkey0); + return getkeys(s3client0, bucket, prefixkey0, NULL, nkeysp, keysp, errmsgp); +} + +EXTERNL int +NC_s3sdkdeletekey(void* s3client0, const char* bucket, const char* pathkey, char** errmsgp) +{ + int stat = NC_NOERR; + NCS3CLIENT* s3client = (NCS3CLIENT*)s3client0; + NCbytes* url = ncbytesnew(); + long httpcode = 0; + + NCTRACE(11,"s3client0=%p bucket=%s pathkey=%s",s3client0,bucket,pathkey); + + if((stat = makes3fullpath(s3client->rooturl,bucket,pathkey,NULL,url))) goto done; + + if((stat = NCH5_s3comms_s3r_deletekey(s3client->h5s3client, ncbytescontents(url), &httpcode))) goto done; + +done: + ncbytesfree(url); + return NCUNTRACE(stat); +} + +/**************************************************/ +/* Utilities */ + +/* +Convert raw getkeys response to vector of keys +*/ +static int +rawtokeys(s3r_buf_t* response, NClist* allkeys, NClist* lengths, struct LISTOBJECTSV2** listv2p) +{ + int stat = NC_NOERR; + struct LISTOBJECTSV2* listv2 = NULL; + NClist* realkeys = nclistnew(); + NClist* commonkeys = nclistnew(); + + if((stat = parse_listbucketresult(response->content,response->count,&listv2))) goto done; + + if(nclistlength(listv2->contents) > 0) { + if((stat = s3objectsinfo(listv2->contents,realkeys,lengths))) goto done; + } + /* Add common prefixes */ + if(nclistlength(listv2->commonprefixes) > 0) { + if((stat = s3commonprefixes(listv2->commonprefixes,commonkeys))) goto done; + } + if((stat=mergekeysets(realkeys, commonkeys, allkeys))) goto done; + + if(listv2p) {*listv2p = listv2; listv2 = NULL;} +done: + nclistfreeall(realkeys); + nclistfreeall(commonkeys); + reclaim_listobjectsv2(listv2); + return stat; +} + +/* Create a path-format root url */ +static char* +makes3rooturl(NCS3INFO* info) +{ + NCbytes* buf = ncbytesnew(); + char* result = NULL; + + ncbytescat(buf,"https://"); + ncbytescat(buf,info->host); + result = ncbytesextract(buf); + ncbytesfree(buf); + return result; +} + +static int +makes3fullpath(const char* rooturl, const char* bucket, const char* prefix, const char* key, NCbytes* url) +{ + int stat = NC_NOERR; + + assert(url != NULL); + assert(rooturl != NULL); + + ncbytescat(url,rooturl); + + if(bucket) { + if(ncbyteslength(url) > 0 && ncbytesget(url,ncbyteslength(url)-1) != '/') ncbytescat(url,"/"); + if(*bucket == '/') bucket++; + ncbytescat(url,bucket); + } + if(prefix) { + if(ncbyteslength(url) > 0 && ncbytesget(url,ncbyteslength(url)-1) != '/') ncbytescat(url,"/"); + if(*prefix == '/') prefix++; + ncbytescat(url,prefix); + } + if(key) { + if(ncbyteslength(url) > 0 && ncbytesget(url,ncbyteslength(url)-1) != '/') ncbytescat(url,"/"); + if(*key == '/') key++; + ncbytescat(url,key); + } + /* Remove any trailing '/' */ + if(ncbyteslength(url) > 0 && ncbytesget(url,ncbyteslength(url)-1) == '/') ncbytessetlength(url,ncbyteslength(url)-1); + return stat; +} + +/* (1) Ensure trailing '/' and (2) Strip off leading '/' */ +/* Note that order of '/' handling is important; we want + "/" to be converted to "", but "/a/b/c" to be converted to "a/b/c/" +*/ +static int +makes3prefix(const char* prefix, char** prefixdirp) +{ + size_t plen; + char* prefixdir; + + if(prefix == NULL) prefix = ""; + if(*prefix == '/') prefix++; /* Remove any leading '/' */ + plen = strlen(prefix); + prefixdir = (char*)malloc(plen+1+1); /* possible '/' and nul */ + if(prefixdir == NULL) return NC_ENOMEM; + memcpy(prefixdir,prefix,plen+1); /* include trailing nul */ + if(plen > 0) + if(prefixdir[plen-1] != '/') strlcat(prefixdir,"/",plen+1+1); /* Ensure trailing '/' */ + /* else leave prefix as "" */ + *prefixdirp = prefixdir; prefixdir = NULL; + return NC_NOERR; +} + +/* Copy keys1 concat keys2 into merge; note that merge list may not be empty. */ +static int +mergekeysets(NClist* keys1, NClist* keys2, NClist* merge) +{ + int stat = NC_NOERR; + int i; + size_t nkeys1 = nclistlength(keys1); + size_t nkeys2 = nclistlength(keys2); + for(i=0;irooturl); + (void)NCH5_s3comms_s3r_close(s3client->h5s3client); + free(s3client); + } +} + +/**************************************************/ +/* XML Response Parser(s) */ + +/** +Action: List objects (V2) +Response XML: +========================= +HTTP/1.1 200 + + + string + string + integer + integer + string + boolean + + string + timestamp + string + integer + string + string + + string + string + + ... + + ... + + string + + ... + string + string + string + string + +========================= +*/ + +static int +parse_listbucketresult(char* xml, unsigned long long xmllen, struct LISTOBJECTSV2** resultp) +{ + int stat = NC_NOERR; + ncxml_doc_t doc = NULL; + ncxml_t x; + struct LISTOBJECTSV2* result = NULL; + +#ifdef DEBUG + printf("-------------------------%s\n-------------------------\n",xml); +#endif + + doc = ncxml_parse(xml,xmllen); + if(doc == NULL) {stat = NC_ES3; goto done;} + ncxml_t dom = ncxml_root(doc); + + /* Verify top level element */ + if(strcmp(ncxml_name(dom),"ListBucketResult")!=0) { + nclog(NCLOGERR,"Expected: actual: <%s>",ncxml_name(dom)); + stat = NC_ES3; + goto done; + } + if((result = alloclistobjectsv2())==NULL) {stat = NC_ENOMEM; goto done;} + + /* Iterate next-level elements */ + for(x=ncxml_child_first(dom);x != NULL;x=ncxml_child_next(x)) { + const char* elem = ncxml_name(x); + if(strcmp(elem,"IsTruncated")==0) { + result->istruncated = trim(ncxml_text(x),RECLAIM); + } else if(strcmp(elem,"Contents")==0) { + if((stat = parse_object(x,result->contents))) goto done; + } else if(strcmp(elem,"Name")==0) { + result->name = trim(ncxml_text(x),RECLAIM); + } else if(strcmp(elem,"Prefix")==0) { + result->prefix = trim(ncxml_text(x),RECLAIM); + } else if(strcmp(elem,"Delimiter")==0) { + result->delimiter = trim(ncxml_text(x),RECLAIM); + } else if(strcmp(elem,"MaxKeys")==0) { + result->maxkeys = trim(ncxml_text(x),RECLAIM); + } else if(strcmp(elem,"CommonPrefixes")==0) { + if((stat = parse_prefix(x,result->commonprefixes))) goto done; + } else if(strcmp(elem,"EncodingType")==0) { + result->encodingtype = trim(ncxml_text(x),RECLAIM); + } else if(strcmp(elem,"KeyCount")==0) { + result->keycount = trim(ncxml_text(x),RECLAIM); + } else if(strcmp(elem,"ContinuationToken")==0) { + result->continuationtoken = trim(ncxml_text(x),RECLAIM); + } else if(strcmp(elem,"NextContinuationToken")==0) { + result->nextcontinuationtoken = trim(ncxml_text(x),RECLAIM); + } else if(strcmp(elem,"StartAfter")==0) { + result->startafter = trim(ncxml_text(x),RECLAIM); + } else { + nclog(NCLOGERR,"Unexpected Element: <%s>",elem); + stat = NC_ES3; + goto done; + } + } + if(resultp) {*resultp = result; result = NULL;} + +done: + if(result) reclaim_listobjectsv2(result); + if(doc) ncxml_free(doc); + return NCTHROW(stat); +} + +static int +parse_object(ncxml_t root, NClist* objects) +{ + int stat = NC_NOERR; + ncxml_t x; + struct Object* object = NULL; + + /* Verify top level element */ + if(strcmp(ncxml_name(root),"Contents")!=0) { + nclog(NCLOGERR,"Expected: actual: <%s>",ncxml_name(root)); + stat = NC_ES3; + goto done; + } + + if((object = allocobject())==NULL) {stat = NC_ENOMEM; goto done;} + + for(x=ncxml_child_first(root);x != NULL;x=ncxml_child_next(x)) { + const char* elem = ncxml_name(x); + if(strcmp(elem,"ChecksumAlorithm")==0) { + if((stat = parse_checksumalgorithm(x,object->checksumalgorithms))) goto done; + } else if(strcmp(elem,"ETag")==0) { + object->etag = trim(ncxml_text(x),RECLAIM); + } else if(strcmp(elem,"Key")==0) { + object->key = trim(ncxml_text(x),RECLAIM); + } else if(strcmp(elem,"LastModified")==0) { + object->lastmodified = trim(ncxml_text(x),RECLAIM); + } else if(strcmp(elem,"Owner")==0) { + if((stat = parse_owner(x,&object->owner))) goto done; + } else if(strcmp(elem,"Size")==0) { + object->size = trim(ncxml_text(x),RECLAIM); + } else if(strcmp(elem,"StorageClass")==0) { + object->storageclass = trim(ncxml_text(x),RECLAIM); + } else { + nclog(NCLOGERR,"Unexpected Element: <%s>",elem); + stat = NC_ES3; + goto done; + } + } + nclistpush(objects,object); + object = NULL; + +done: + if(object) reclaim_object(object); + return NCTHROW(stat); +} + +static int +parse_owner(ncxml_t root, struct Owner* owner) +{ + int stat = NC_NOERR; + ncxml_t x; + + /* Verify top level element */ + if(strcmp(ncxml_name(root),"Owner")!=0) { + nclog(NCLOGERR,"Expected: actual: <%s>",ncxml_name(root)); + stat = NC_ES3; + goto done; + } + + for(x=ncxml_child_first(root);x != NULL;x=ncxml_child_next(x)) { + const char* elem = ncxml_name(x); + if(strcmp(elem,"DisplayName")==0) { + owner->displayname = trim(ncxml_text(x),RECLAIM); + } else if(strcmp(elem,"ID")==0) { + owner->id = trim(ncxml_text(x),RECLAIM); + } else { + nclog(NCLOGERR,"Unexpected Element: <%s>",elem); + stat = NC_ES3; + goto done; + } + } + +done: + return NCTHROW(stat); +} + +static int +parse_prefix(ncxml_t root, NClist* prefixes) +{ + int stat = NC_NOERR; + ncxml_t x; + char* prefix = NULL; + + /* Verify top level element */ + if(strcmp(ncxml_name(root),"CommonPrefixes")!=0) { + nclog(NCLOGERR,"Expected: actual: <%s>",ncxml_name(root)); + stat = NC_ES3; + goto done; + } + + for(x=ncxml_child_first(root);x != NULL;x=ncxml_child_next(x)) { + const char* elem = ncxml_name(x); + if(strcmp(elem,"Prefix")==0) { + prefix = trim(ncxml_text(x),RECLAIM); + nclistpush(prefixes,prefix); + prefix = NULL; + } else { + nclog(NCLOGERR,"Unexpected Element: <%s>",elem); + stat = NC_ES3; + goto done; + } + } + +done: + nullfree(prefix); + return NCTHROW(stat); +} + +static int +parse_checksumalgorithm(ncxml_t root, NClist* algorithms) +{ + int stat = NC_NOERR; + char* alg = NULL; + + /* Verify top level element */ + if(strcmp(ncxml_name(root),"ChecksumAlgorithm")!=0) { + nclog(NCLOGERR,"Expected: actual: <%s>",ncxml_name(root)); + stat = NC_ES3; + goto done; + } + alg = trim(ncxml_text(root),RECLAIM); + nclistpush(algorithms,alg); + alg = NULL; + +done: + nullfree(alg); + return NCTHROW(stat); +} + +static struct LISTOBJECTSV2* +alloclistobjectsv2(void) +{ + struct LISTOBJECTSV2* lov2 = NULL; + if((lov2 = calloc(1,sizeof(struct LISTOBJECTSV2))) == NULL) + return lov2; + lov2->contents = nclistnew(); + lov2->commonprefixes = nclistnew(); + return lov2; +} + +static struct Object* +allocobject(void) +{ + struct Object* obj = NULL; + if((obj = calloc(1,sizeof(struct Object))) == NULL) + return obj; + obj->checksumalgorithms = nclistnew(); + return obj; +} + +static void +reclaim_listobjectsv2(struct LISTOBJECTSV2* lo) +{ + int i; + if(lo == NULL) return; + nullfree(lo->istruncated); + for(i=0;icontents);i++) + reclaim_object((struct Object*)nclistget(lo->contents,i)); + nclistfree(lo->contents); + nullfree(lo->name); + nullfree(lo->prefix); + nullfree(lo->delimiter); + nullfree(lo->maxkeys); + nclistfreeall(lo->commonprefixes); + nullfree(lo->encodingtype); + nullfree(lo->keycount); + nullfree(lo->continuationtoken); + nullfree(lo->nextcontinuationtoken); + nullfree(lo->startafter); + free(lo); +} + +static void +reclaim_object(struct Object* o) +{ + if(o == NULL) return; + nclistfreeall(o->checksumalgorithms); + nullfree(o->etag); + nullfree(o->key); + nullfree(o->lastmodified); + nullfree(o->owner.displayname); + nullfree(o->owner.id); + nullfree(o->size); + nullfree(o->storageclass); + free(o); +} + +static char* +trim(char* s, int reclaim) +{ + ptrdiff_t first=0,last=0; + const char* p; + char* t = NULL; + size_t len; + + for(p=s;*p;p++) { + if(*p > ' ') {first = (p - s); break;} + } + for(p=s+(strlen(s)-1);p >= s;p--) { + if(*p > ' ') {last = (p - s); break;} + } + len = (last - first) + 1; + if((t = (char*)malloc(len+1))==NULL) return t; + memcpy(t,s+first,len); + t[len] = '\0'; + if(reclaim) nullfree(s); + return t; +} + +/* +Get Info about a single object from a vector +*/ +static int +s3objectinfo1(const struct Object* s3_object, char** fullkeyp, uintptr_t* lenp) +{ + int stat = NC_NOERR; + const char* key = NULL; + char* tmp = NULL; + unsigned long long len; + + assert(fullkeyp); + + key = s3_object->key; + len = strlen(key); + if((tmp = (char*)malloc(len+1+1))==NULL) {stat = NC_ENOMEM; goto done;} + tmp[0] = '\0'; + if(key[0] != '/') strlcat(tmp,"/",len+1+1); + strlcat(tmp,key,len+1+1); + sscanf(s3_object->size,"%llu",&len); + if(fullkeyp) {*fullkeyp = tmp; tmp = NULL;} + if(lenp) *lenp = (size64_t)len; +done: + if(tmp) free(tmp); + return NCTHROW(stat); +} + +/* +Get Info about a vector of objects; Keys are fixed up to start with a '/' +*/ +static int +s3objectsinfo(NClist* contents, NClist* keys, NClist* lengths) +{ + int stat = NC_NOERR; + size_t i; + char* key = NULL; + uintptr_t length = 0; + + for(i=0;i 0) ncbytescat(buf,"&"); + ncbytescat(buf,key); + ncbytescat(buf,"="); + if(value != NULL) + ncbytescat(buf,value); + } + if(querystring) {*querystring = ncbytesextract(buf);} + + ncbytesfree(buf); + return NCTHROW(stat); + +} + +/* Insert encoded key+value keeping sorted order */ +static int +queryinsert(NClist* list, char* ekey, char* evalue) +{ + int pos,i,stat = NC_NOERR; + for(pos=-1,i=0;i 0) {pos = i; break;} /* key > ekey => insert ekey before key */ + } + if(pos < 0) pos = nclistlength(list); /* insert at end; also works if |list|==0 */ + nclistinsert(list,pos,evalue); + nclistinsert(list,pos,ekey); +done: + return NCTHROW(stat); +} + diff --git a/libdispatch/ncuri.c b/libdispatch/ncuri.c index 3fbdbc14cc..7bca11a228 100644 --- a/libdispatch/ncuri.c +++ b/libdispatch/ncuri.c @@ -1187,7 +1187,7 @@ ensurefraglist(NCURI* uri) } else if(!nolist && nofrag) { /* Create the fragment string from fraglist */ frag = ncbytesnew(); - buildlist((const char**)uri->fraglist,1,frag); + buildlist((const char**)uri->fraglist,0,frag); /* do not encode */ uri->fragment = ncbytesextract(frag); } diff --git a/libdispatch/ncutil.h b/libdispatch/ncutil.h new file mode 100644 index 0000000000..44ab508b36 --- /dev/null +++ b/libdispatch/ncutil.h @@ -0,0 +1,242 @@ +/* Copyright 2018, UCAR/Unidata and OPeNDAP, Inc. + See the COPYRIGHT file for more information. */ + +#ifndef UTILS_H +#define UTILS_H 1 + +/* Define a header-only simple version of a dynamically expandable list and byte buffer */ +/* To be used in code that should be independent of libnetcdf */ + +typedef struct VList { + unsigned alloc; + unsigned length; + void** content; +} VList; + +typedef struct VString { + int nonextendible; /* 1 => fail if an attempt is made to extend this string*/ + unsigned int alloc; + unsigned int length; + char* content; +} VString; + +/* VString has a fixed expansion size */ +#define VSTRALLOC 64 + +#if defined(_CPLUSPLUS_) || defined(__CPLUSPLUS__) +#define EXTERNC extern "C" +#else +#define EXTERNC extern +#endif + +static int util_initialized = 0; + +static void util_initialize(void); + +static VList* +vlistnew(void) +{ + VList* l; + if(!util_initialized) util_initialize(); + l = (VList*)calloc(1,sizeof(VList)); + assert(l != NULL); + return l; +} + +static void +vlistfree(VList* l) +{ + if(l == NULL) return; + if(l->content != NULL) {free(l->content); l->content = NULL;} + free(l); +} + +static void +vlistexpand(VList* l) +{ + void** newcontent = NULL; + size_t newsz; + + if(l == NULL) return; + newsz = (l->length * 2) + 1; /* basically double allocated space */ + if(l->alloc >= newsz) return; /* space already allocated */ + newcontent=(void**)calloc(newsz,sizeof(void*)); + assert(newcontent != NULL); + if(l->alloc > 0 && l->length > 0 && l->content != NULL) { /* something to copy */ + memcpy((void*)newcontent,(void*)l->content,sizeof(void*)*l->length); + } + if(l->content != NULL) free(l->content); + l->content=newcontent; + l->alloc=newsz; + /* size is the same */ +} + +static void* +vlistget(VList* l, unsigned index) /* Return the ith element of l */ +{ + if(l == NULL || l->length == 0) return NULL; + assert(index < l->length); + return l->content[index]; +} + +static void +vlistpush(VList* l, void* elem) +{ + if(l == NULL) return; + while(l->length >= l->alloc) vlistexpand(l); + l->content[l->length] = elem; + l->length++; +} + +static void* +vlistfirst(VList* l) /* remove first element */ +{ + unsigned i,len; + void* elem; + if(l == NULL || l->length == 0) return NULL; + elem = l->content[0]; + len = l->length; + for(i=1;icontent[i-1] = l->content[i]; + l->length--; + return elem; +} + +static void +vlistfreeall(VList* l) /* call free() on each list element*/ +{ + unsigned i; + if(l == NULL || l->length == 0) return; + for(i=0;ilength;i++) if(l->content[i] != NULL) {free(l->content[i]);} + vlistfree(l); +} + +static VString* +vsnew(void) +{ + VString* vs = NULL; + if(!util_initialized) util_initialize(); + vs = (VString*)calloc(1,sizeof(VString)); + assert(vs != NULL); + return vs; +} + +static void +vsfree(VString* vs) +{ + if(vs == NULL) return; + if(vs->content != NULL) free(vs->content); + free(vs); +} + +static void +vsexpand(VString* vs) +{ + char* newcontent = NULL; + size_t newsz; + + if(vs == NULL) return; + assert(vs->nonextendible == 0); + newsz = (vs->alloc + VSTRALLOC); /* basically double allocated space */ + if(vs->alloc >= newsz) return; /* space already allocated */ + newcontent=(char*)calloc(1,newsz+1);/* always room for nul term */ + assert(newcontent != NULL); + if(vs->alloc > 0 && vs->length > 0 && vs->content != NULL) /* something to copy */ + memcpy((void*)newcontent,(void*)vs->content,vs->length); + newcontent[vs->length] = '\0'; /* ensure null terminated */ + if(vs->content != NULL) free(vs->content); + vs->content=newcontent; + vs->alloc=newsz; + /* length is the same */ +} + +static void +vsappendn(VString* vs, const char* elem, unsigned n) +{ + size_t need; + assert(vs != NULL && elem != NULL); + if(n == 0) {n = strlen(elem);} + need = vs->length + n; + if(vs->nonextendible) { + /* Space must already be available */ + assert(vs->alloc >= need); + } else { + while(vs->alloc < need) + vsexpand(vs); + } + memcpy(&vs->content[vs->length],elem,n); + vs->length += n; + if(!vs->nonextendible) + vs->content[vs->length] = '\0'; +} + +static void +vsappend(VString* vs, char elem) +{ + char s[2]; + s[0] = elem; + s[1] = '\0'; + vsappendn(vs,s,1); +} + +/* Set unexpandible contents */ +static void +vssetcontents(VString* vs, char* contents, unsigned alloc) +{ + assert(vs != NULL && contents != NULL); + vs->length = 0; + if(!vs->nonextendible && vs->content != NULL) free(vs->content); + vs->content = contents; + vs->length = alloc; + vs->alloc = alloc; + vs->nonextendible = 1; +} + +/* Extract the content and leave content null */ +static char* +vsextract(VString* vs) +{ + char* x = NULL; + if(vs == NULL || vs->content == NULL) return NULL; + x = vs->content; + vs->content = NULL; + vs->length = 0; + vs->alloc = 0; + return x; +} + +static void +util_initialize(void) +{ + /* quiet compiler */ + void* f = NULL; + f = f; + f = (void*)vlistnew; + f = (void*)vlistfree; + f = (void*)vlistexpand; + f = (void*)vlistget; + f = (void*)vlistpush; + f = (void*)vlistfirst; + f = (void*)vlistfreeall; + f = (void*)vsnew; + f = (void*)vsfree; + f = (void*)vsexpand; + f = (void*)vssetcontents; + f = (void*)vsappendn; + f = (void*)vsappend; + f = (void*)vsextract; + util_initialized = 1; +} + +/* Following are always "in-lined"*/ +#define vlistcontents(l) ((l)==NULL?NULL:(l)->content) +#define vlistlength(l) ((l)==NULL?0:(int)(l)->length) +#define vlistclear(l) vlistsetlength(l,0) +#define vlistsetlength(l,len) do{if((l)!=NULL) (l)->length=len;} while(0) + +#define vscontents(vs) ((vs)==NULL?NULL:(vs)->content) +#define vslength(vs) ((vs)==NULL?0:(int)(vs)->length) +#define vscat(vs,s) vsappendn(vs,s,0) +#define vsclear(vs) vssetlength(vs,0) +#define vssetlength(vs,len) do{if((vs)!=NULL) (vs)->length=len;} while(0) + +#endif /*UTILS_H*/ diff --git a/libhdf5/H5FDhttp.c b/libhdf5/H5FDhttp.c index e004602523..c6658f46f2 100644 --- a/libhdf5/H5FDhttp.c +++ b/libhdf5/H5FDhttp.c @@ -347,10 +347,10 @@ H5FD_http_open( const char *name, unsigned flags, hid_t /*UNUSED*/ fapl_id, write_access = 0; /* Open file in read-only mode, to check for existence and get length */ - if((ncstat = nc_http_init(&state))) { + if((ncstat = nc_http_open(name,&state))) { H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_CANTOPENFILE, "cannot access object", NULL); } - if((ncstat = nc_http_size(state,name,&len))) { + if((ncstat = nc_http_size(state,&len))) { H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_CANTOPENFILE, "cannot access object", NULL); } @@ -729,7 +729,7 @@ H5FD_http_read(H5FD_t *_file, H5FD_mem_t /*UNUSED*/ type, hid_t /*UNUSED*/ dxpl_ { NCbytes* bbuf = ncbytesnew(); - if((ncstat = nc_http_read(file->state,file->url,addr,size,bbuf))) { + if((ncstat = nc_http_read(file->state,addr,size,bbuf))) { file->op = H5FD_HTTP_OP_UNKNOWN; file->pos = HADDR_UNDEF; ncbytesfree(bbuf); bbuf = NULL; diff --git a/libhdf5/hdf5attr.c b/libhdf5/hdf5attr.c index 96b1d4682d..8b15ef3101 100644 --- a/libhdf5/hdf5attr.c +++ b/libhdf5/hdf5attr.c @@ -552,26 +552,6 @@ nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type, if ((retval = nc4_get_typelen_mem(h5, file_type, &type_size))) return retval; -#ifdef SEPDATA - /* If this att has vlen or string data, release it before we lose the length value. */ - if (att->stdata) - { - int i; - for (i = 0; i < att->len; i++) - if(att->stdata[i]) - free(att->stdata[i]); - free(att->stdata); - att->stdata = NULL; - } - if (att->vldata) - { - int i; - for (i = 0; i < att->len; i++) { - nc_free_vlen(&att->vldata[i]); /* FIX: see warning of nc_free_vlen */ - free(att->vldata); - att->vldata = NULL; - } -#else if (att->data) { assert(attsave.data == NULL); @@ -579,7 +559,6 @@ nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type, attsave.len = att->len; att->data = NULL; } -#endif /* If this is the _FillValue attribute, then we will also have to * copy the value to the fill_vlue pointer of the NC_VAR_INFO_T @@ -611,74 +590,15 @@ nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type, * one. Make up your damn mind, would you? */ if (var->fill_value) { -#ifdef SEPDATA - if (var->type_info->nc_type_class == NC_VLEN) - { - if ((retval = nc_free_vlen(var->fill_value))) - BAIL(retval); - } - else if (var->type_info->nc_type_class == NC_STRING) - { - if (*(char **)var->fill_value) - free(*(char **)var->fill_value); - } - free(var->fill_value); -#else /* reclaim later */ fillsave.data = var->fill_value; fillsave.type = var->type_info->hdr.id; fillsave.len = 1; -#endif var->fill_value = NULL; } /* Determine the size of the fill value in bytes. */ -#ifdef SEPDATA - if (var->type_info->nc_type_class == NC_VLEN) - size = sizeof(hvl_t); - else if (var->type_info->nc_type_class == NC_STRING) - size = sizeof(char *); - else - size = type_size; -#endif - -#ifdef SEPDATA - /* Allocate space for the fill value. */ - if (!(var->fill_value = calloc(1, size))) - BAIL(NC_ENOMEM); - - /* Copy the fill_value. */ - LOG((4, "Copying fill value into metadata for variable %s", var->hdr.name)); - if (var->type_info->nc_type_class == NC_VLEN) - { - nc_vlen_t *in_vlen = (nc_vlen_t *)data, *fv_vlen = (nc_vlen_t *)(var->fill_value); - NC_TYPE_INFO_T* basetype; - size_t basetypesize = 0; - /* get the basetype and its size */ - basetype = var->type_info; - if ((retval = nc4_get_typelen_mem(grp->nc4_info, basetype->hdr.id, &basetypesize))) - BAIL(retval); - /* shallow clone the content of the vlen; shallow because it has only a temporary existence */ - fv_vlen->len = in_vlen->len; - if (!(fv_vlen->p = malloc(basetypesize * in_vlen->len))) - BAIL(NC_ENOMEM); - memcpy(fv_vlen->p, in_vlen->p, in_vlen->len * basetypesize); - } - else if (var->type_info->nc_type_class == NC_STRING) - { - if (*(char **)data) - { - if (!(*(char **)(var->fill_value) = malloc(strlen(*(char **)data) + 1))) - BAIL(NC_ENOMEM); - strcpy(*(char **)var->fill_value, *(char **)data); - } - else - *(char **)var->fill_value = NULL; - } - else - memcpy(var->fill_value, data, type_size); -#else { nc_type var_type = var->type_info->hdr.id; size_t var_type_size = var->type_info->size; @@ -704,7 +624,6 @@ nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type, var->fill_value = copy; copy = NULL; } -#endif /* Indicate that the fill value was changed, if the variable has already * been created in the file, so the dataset gets deleted and re-created. */ if (var->created) @@ -721,82 +640,6 @@ nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type, BAIL(retval); assert(data); -#ifdef SEPDATA - if (type_class == NC_VLEN) - { - const hvl_t *vldata1; - NC_TYPE_INFO_T *vltype; - size_t base_typelen; - - /* Get the type object for the attribute's type */ - if ((retval = nc4_find_type(h5, file_type, &vltype))) - BAIL(retval); - - /* Retrieve the size of the base type */ - if ((retval = nc4_get_typelen_mem(h5, vltype->u.v.base_nc_typeid, &base_typelen))) - BAIL(retval); - - vldata1 = data; - if (!(att->vldata = (nc_vlen_t*)malloc(att->len * sizeof(hvl_t)))) - BAIL(NC_ENOMEM); - for (i = 0; i < att->len; i++) - { - att->vldata[i].len = vldata1[i].len; - /* Warning, this only works for cases described for nc_free_vlen() */ - if (!(att->vldata[i].p = malloc(base_typelen * att->vldata[i].len))) - BAIL(NC_ENOMEM); - memcpy(att->vldata[i].p, vldata1[i].p, base_typelen * att->vldata[i].len); - } - } - else if (type_class == NC_STRING) - { - LOG((4, "copying array of NC_STRING")); - if (!(att->stdata = malloc(sizeof(char *) * att->len))) { - BAIL(NC_ENOMEM); - } - - /* If we are overwriting an existing attribute, - specifically an NC_CHAR, we need to clean up - the pre-existing att->data. */ - if (!new_att && att->data) { - - free(att->data); - att->data = NULL; - } - - for (i = 0; i < att->len; i++) - { - if(NULL != ((char **)data)[i]) { - LOG((5, "copying string %d of size %d", i, strlen(((char **)data)[i]) + 1)); - if (!(att->stdata[i] = strdup(((char **)data)[i]))) - BAIL(NC_ENOMEM); - } - else - att->stdata[i] = ((char **)data)[i]; - } - } - else - { - /* [Re]allocate memory for the attribute data */ - if (!new_att) - free (att->data); - if (!(att->data = malloc(att->len * type_size))) - BAIL(NC_ENOMEM); - - /* Just copy the data, for non-atomic types */ - if (type_class == NC_OPAQUE || type_class == NC_COMPOUND || type_class == NC_ENUM) - memcpy(att->data, data, len * type_size); - else - { - /* Data types are like religions, in that one can convert. */ - if ((retval = nc4_convert_type(data, att->data, mem_type, file_type, - len, &range_error, NULL, - (h5->cmode & NC_CLASSIC_MODEL), - NC_NOQUANTIZE, 0))) - BAIL(retval); - } - } -#else { /* Allocate top level of the copy */ if (!(copy = malloc(len * type_size))) @@ -816,7 +659,6 @@ nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type, /* Store it */ att->data = copy; copy = NULL; } -#endif } att->dirty = NC_TRUE; att->created = NC_FALSE; diff --git a/libhdf5/hdf5debug.c b/libhdf5/hdf5debug.c index 5462eb5475..40871c148a 100644 --- a/libhdf5/hdf5debug.c +++ b/libhdf5/hdf5debug.c @@ -9,6 +9,7 @@ #include #endif +#include "nclog.h" #include "hdf5debug.h" #ifdef H5CATCH diff --git a/libhdf5/hdf5file.c b/libhdf5/hdf5file.c index 3542e3f386..8fab07a6c1 100644 --- a/libhdf5/hdf5file.c +++ b/libhdf5/hdf5file.c @@ -220,12 +220,10 @@ nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, NC_memio *memio) * hidden attribute. */ NC4_clear_provenance(&h5->provenance); -#if defined(ENABLE_BYTERANGE) ncurifree(hdf5_info->uri); -#if defined(ENABLE_HDF5_ROS3) || defined(ENABLE_S3_SDK) +#ifdef ENABLE_S3 /* Free the http info */ NC_authfree(hdf5_info->auth); -#endif #endif /* Close hdf file. It may not be open, since this function is also diff --git a/libhdf5/hdf5internal.c b/libhdf5/hdf5internal.c index 83add362dc..40f80a9681 100644 --- a/libhdf5/hdf5internal.c +++ b/libhdf5/hdf5internal.c @@ -613,18 +613,11 @@ close_vars(NC_GRP_INFO_T *grp) { if (var->type_info) { -#ifdef SEPDATA - if (var->type_info->nc_type_class == NC_VLEN) - nc_free_vlen((nc_vlen_t *)var->fill_value); - else if (var->type_info->nc_type_class == NC_STRING && *(char **)var->fill_value) - free(*(char **)var->fill_value); -#else int stat = NC_NOERR; if((stat = nc_reclaim_data(grp->nc4_info->controller->ext_ncid,var->type_info->hdr.id,var->fill_value,1))) return stat; nullfree(var->fill_value); } -#endif var->fill_value = NULL; } } diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index cd24a9ea99..4bdddbca74 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -885,7 +885,7 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) if(iss3) { /* Rebuild the URL */ NCURI* newuri = NULL; - if((retval = NC_s3urlrebuild(h5->uri,&newuri,NULL,&awsregion0))) goto exit; + if((retval = NC_s3urlrebuild(h5->uri,NULL,&awsregion0,&newuri))) goto exit; if((newpath = ncuribuild(newuri,NULL,NULL,NCURISVC))==NULL) {retval = NC_EURL; goto exit;} ncurifree(h5->uri); @@ -1158,20 +1158,6 @@ static int get_fill_info(hid_t propid, NC_VAR_INFO_T *var) /* Allocate space to hold the fill value. */ if (!var->fill_value) { -#ifdef SEPDATA - - if (var->type_info->nc_type_class == NC_VLEN) - { - if (!(var->fill_value = malloc(sizeof(nc_vlen_t)))) - return NC_ENOMEM; - } - else if (var->type_info->nc_type_class == NC_STRING) - { - if (!(var->fill_value = malloc(sizeof(char *)))) - return NC_ENOMEM; - } - else -#endif { assert(var->type_info->size); if (!(var->fill_value = malloc(var->type_info->size))) @@ -1870,70 +1856,6 @@ read_hdf5_att(NC_GRP_INFO_T *grp, hid_t attid, NC_ATT_INFO_T *att) if ((retval = nc4_get_typelen_mem(grp->nc4_info, att->nc_typeid, &type_size))) return retval; -#ifdef SEPDATA - if (att_class == H5T_VLEN) - { - if (!(att->vldata = malloc((unsigned int)(att->len * sizeof(hvl_t))))) - BAIL(NC_ENOMEM); - if (H5Aread(attid, hdf5_att->native_hdf_typeid, att->vldata) < 0) - BAIL(NC_EATTMETA); - } - else if (att->nc_typeid == NC_STRING) - { - if (!(att->stdata = calloc(att->len, sizeof(char *)))) - BAIL(NC_ENOMEM); - /* For a fixed length HDF5 string, the read requires - * contiguous memory. Meanwhile, the netCDF API requires that - * nc_free_string be called on string arrays, which would not - * work if one contiguous memory block were used. So here I - * convert the contiguous block of strings into an array of - * malloced strings -- each string with its own malloc. Then I - * copy the data and free the contiguous memory. This - * involves copying the data, which is bad, but this only - * occurs for fixed length string attributes, and presumably - * these are small. Note also that netCDF-4 does not create them - it - * always uses variable length strings. */ - if (fixed_len_string) - { - int i; - char *contig_buf, *cur; - - /* Alloc space for the contiguous memory read. */ - if (!(contig_buf = malloc(att->len * fixed_size * sizeof(char)))) - BAIL(NC_ENOMEM); - - /* Read the fixed-len strings as one big block. */ - if (H5Aread(attid, hdf5_att->native_hdf_typeid, contig_buf) < 0) { - free(contig_buf); - BAIL(NC_EATTMETA); - } - - /* Copy strings, one at a time, into their new home. Alloc - space for each string. The user will later free this - space with nc_free_string. */ - cur = contig_buf; - for (i = 0; i < att->len; i++) - { - if (!(att->stdata[i] = malloc(fixed_size))) { - free(contig_buf); - BAIL(NC_ENOMEM); - } - strncpy(att->stdata[i], cur, fixed_size); - cur += fixed_size; - } - - /* Free contiguous memory buffer. */ - free(contig_buf); - } - else - { - /* Read variable-length string atts. */ - if (H5Aread(attid, hdf5_att->native_hdf_typeid, att->stdata) < 0) - BAIL(NC_EATTMETA); - } - } - else -#else { if (!(att->data = malloc((unsigned int)(att->len * type_size)))) BAIL(NC_ENOMEM); @@ -1990,7 +1912,6 @@ read_hdf5_att(NC_GRP_INFO_T *grp, hid_t attid, NC_ATT_INFO_T *att) BAIL(NC_EATTMETA); } } -#endif } if (H5Tclose(file_typeid) < 0) diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index 3a8c07bd2b..f8ba81db20 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -2184,35 +2184,11 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, for (i = 0; i < fill_len; i++) { -#ifdef SEPDATA - if (var->type_info->nc_type_class == NC_STRING) - { - if (*(char **)fillvalue) - { - if (!(*(char **)filldata = strdup(*(char **)fillvalue))) - BAIL(NC_ENOMEM); - } - else - *(char **)filldata = NULL; - } - else if (var->type_info->nc_type_class == NC_VLEN) - { - if (fillvalue) - { - memcpy(filldata,fillvalue,file_type_size); - } else { - *(char **)filldata = NULL; - } - } - else - memcpy(filldata, fillvalue, file_type_size); -#else { /* Copy one instance of the fill_value */ if((retval = nc_copy_data(ncid,var->type_info->hdr.id,fillvalue,1,filldata))) BAIL(retval); } -#endif filldata = (char *)filldata + file_type_size; } } diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index b2c22ddcbd..fb3770afcc 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -470,12 +470,6 @@ put_att_grpa(NC_GRP_INFO_T *grp, int varid, NC_ATT_INFO_T *att) * some phoney data (which won't be written anyway.)*/ if (!dims[0]) data = &phoney_data; -#ifdef SEPDATA - else if (att->vldata) - data = att->vldata; - else if (att->stdata) - data = att->stdata; -#endif else data = att->data; diff --git a/liblib/CMakeLists.txt b/liblib/CMakeLists.txt index e3eddc0fb1..ce2bd85af4 100644 --- a/liblib/CMakeLists.txt +++ b/liblib/CMakeLists.txt @@ -38,6 +38,10 @@ IF(ENABLE_NCZARR) SET(liblib_LIBS ${liblib_LIBS} nczarr) ENDIF() +IF(ENABLE_S3_INTERNAL) + SET(liblib_LIBS ${liblib_LIBS} ncxml) +ENDIF() + IF(ENABLE_PLUGINS) SET(liblib_LIBS ${liblib_LIBS} ncpoco) ENDIF() @@ -134,7 +138,7 @@ IF(ENABLE_PNETCDF AND PNETCDF) SET(TLL_LIBS ${TLL_LIBS} ${PNETCDF}) ENDIF() -IF(ENABLE_S3_SDK) +IF(ENABLE_S3_AWS) TARGET_LINK_DIRECTORIES(netcdf PUBLIC ${AWSSDK_LIB_DIR}) TARGET_LINK_LIBRARIES(netcdf ${AWS_LINK_LIBRARIES}) ENDIF() diff --git a/liblib/Makefile.am b/liblib/Makefile.am index b456aa439a..f719ff4733 100644 --- a/liblib/Makefile.am +++ b/liblib/Makefile.am @@ -62,9 +62,16 @@ endif # ENABLE_DAP if ENABLE_DAP4 AM_CPPFLAGS += -I${top_srcdir}/libdap4 libnetcdf_la_LIBADD += ${top_builddir}/libdap4/libdap4.la +endif # ENABLE_DAP4 + AM_CPPFLAGS += -I${top_srcdir}/libncxml +if ENABLE_DAP4 libnetcdf_la_LIBADD += ${top_builddir}/libncxml/libncxml.la -endif # ENABLE_DAP4 +else +if ENABLE_S3_INTERNAL +libnetcdf_la_LIBADD += ${top_builddir}/libncxml/libncxml.la +endif +endif # ENABLE_S3_INTERNAL || ENABLE_DAP4 # NetCDF-4 ... if USE_NETCDF4 @@ -75,7 +82,7 @@ endif #USE_NETCDF4 if ENABLE_NCZARR AM_CPPFLAGS += -I${top_srcdir}/libnczarr libnetcdf_la_LIBADD += ${top_builddir}/libnczarr/libnczarr.la -if ENABLE_S3_SDK +if ENABLE_S3_AWS libnetcdf_la_LIBADD += ${aws_cpp_sdk_core_LIBS} ${aws_cpp_sdk_s3_LIBS} endif endif #ENABLE_NCZARR diff --git a/liblib/nc_initialize.c b/liblib/nc_initialize.c index 982f6be7cb..e98a5bc1e5 100644 --- a/liblib/nc_initialize.c +++ b/liblib/nc_initialize.c @@ -46,7 +46,7 @@ extern int NC_HDF4_initialize(void); extern int NC_HDF4_finalize(void); #endif -#ifdef ENABLE_S3_SDK +#ifdef ENABLE_S3 EXTERNL int NC_s3sdkinitialize(void); EXTERNL int NC_s3sdkfinalize(void); #endif @@ -109,7 +109,7 @@ nc_initialize() #ifdef USE_HDF4 if((stat = NC_HDF4_initialize())) goto done; #endif -#ifdef ENABLE_S3_SDK +#ifdef ENABLE_S3 if((stat = NC_s3sdkinitialize())) goto done; #endif #ifdef ENABLE_NCZARR @@ -174,7 +174,7 @@ nc_finalize(void) if((stat = NCZ_finalize())) failed = stat; #endif -#ifdef ENABLE_S3_SDK +#ifdef ENABLE_S3 if((stat = NC_s3sdkfinalize())) failed = stat; #endif diff --git a/libnczarr/CMakeLists.txt b/libnczarr/CMakeLists.txt index 59a3c0dfe9..a224c4f24d 100644 --- a/libnczarr/CMakeLists.txt +++ b/libnczarr/CMakeLists.txt @@ -50,7 +50,7 @@ IF(ENABLE_NCZARR_ZIP) SET(libnczarr_SOURCES ${libnczarr_SOURCES} zmap_zip.c) ENDIF() -IF(ENABLE_S3_SDK) +IF(ENABLE_S3) SET(libnczarr_SOURCES ${libnczarr_SOURCES} zmap_s3sdk.c) ENDIF() diff --git a/libnczarr/Makefile.am b/libnczarr/Makefile.am index 397514c124..d7c2b3cf5f 100644 --- a/libnczarr/Makefile.am +++ b/libnczarr/Makefile.am @@ -72,10 +72,12 @@ if ENABLE_NCZARR_FILTERS libnczarr_la_SOURCES += zfilter.c endif -if ENABLE_S3_SDK +if ENABLE_S3 libnczarr_la_SOURCES += zmap_s3sdk.c +if ENABLE_S3_AWS AM_CXXFLAGS += -std=c++11 endif +endif # For now, ignore these IGNORED=ztype.c diff --git a/libnczarr/zdebug.h b/libnczarr/zdebug.h index e57e2d183a..71a9e43537 100644 --- a/libnczarr/zdebug.h +++ b/libnczarr/zdebug.h @@ -81,4 +81,3 @@ struct ZUTEST { EXTERNL struct ZUTEST* zutest; #endif /*ZDEBUG_H*/ - diff --git a/libnczarr/zfilter.c b/libnczarr/zfilter.c index 72d0e139af..03047d95f7 100644 --- a/libnczarr/zfilter.c +++ b/libnczarr/zfilter.c @@ -368,7 +368,7 @@ NCZ_addfilter(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, unsigned int id, size_t if((stat = NCZ_plugin_loaded(id,&plugin))) goto done; if(plugin == NULL) { ZLOG(NCLOGWARN,"no such plugin: %u",(unsigned)id); - stat = NC_ENOFILTER; + stat = THROW(NC_ENOFILTER); goto done; } @@ -432,7 +432,7 @@ NCZ_filter_remove(NC_VAR_INFO_T* var, unsigned int id) } } ZLOG(NCLOGERR,"no such filter: %u",(unsigned)id); - stat = NC_ENOFILTER; + stat = THROW(NC_ENOFILTER); done: return ZUNTRACE(stat); } @@ -705,7 +705,7 @@ NCZ_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparamsp, memcpy(params,spec->hdf5.visible.params,sizeof(unsigned int)*spec->hdf5.visible.nparams); } else { ZLOG(NCLOGWARN,"no such filter: %u",(unsigned)id); - stat = NC_ENOFILTER; + stat = THROW(NC_ENOFILTER); } done: return ZUNTRACEX(stat,"nparams=%u",(unsigned)(nparamsp?*nparamsp:0)); @@ -731,7 +731,7 @@ NCZ_inq_filter_avail(int ncid, unsigned id) /* Check the available filters list */ if((stat = NCZ_plugin_loaded((int)id, &plug))) goto done; if(plug == NULL || plug->incomplete) - stat = NC_ENOFILTER; + stat = THROW(NC_ENOFILTER); done: return ZUNTRACE(stat); } @@ -846,7 +846,7 @@ NCZ_applyfilterchain(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, NClist* cha for(i=0;ihdf5.id > 0 && f->plugin != NULL); if(!(f->flags & FLAG_WORKING)) {/* working not yet available */ if((stat = ensure_working(var,f))) goto done; @@ -984,7 +984,7 @@ NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* j if(NCJdictget(jfilter,"id",&jvalue)<0) {stat = NC_EFILTER; goto done;} if(NCJsort(jvalue) != NCJ_STRING) { ZLOG(NCLOGERR,"no such filter: %s",NCJstring(jvalue)); - stat = NC_ENOFILTER; goto done; + stat = THROW(NC_ENOFILTER); goto done; } /* Build the codec */ @@ -1451,7 +1451,7 @@ NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp) const NCZ_codec_info_defaults_proto cpd = (NCZ_codec_info_defaults_proto)ncpgetsymbol(lib,"NCZ_codec_info_defaults"); if(gpt == NULL && gpi == NULL && npi == NULL && cpd == NULL) - {stat = NC_ENOFILTER; goto done;} + {stat = THROW(NC_ENOFILTER); goto done;} /* We can have cpd or we can have (gpt && gpi && npi) but not both sets */ if(cpd != NULL) { @@ -1656,7 +1656,7 @@ static int ensure_working(const NC_VAR_INFO_T* var, NCZ_Filter* filter) { int stat = NC_NOERR; - if(FILTERINCOMPLETE(filter)) {stat = NC_ENOFILTER; goto done;} + if(FILTERINCOMPLETE(filter)) {stat = THROW(NC_ENOFILTER); goto done;} if(!(filter->flags & FLAG_WORKING)) { const size_t oldnparams = filter->hdf5.visible.nparams; const unsigned* oldparams = filter->hdf5.visible.params; diff --git a/libnczarr/zincludes.h b/libnczarr/zincludes.h index c6df092d7e..3fdae6c6fd 100644 --- a/libnczarr/zincludes.h +++ b/libnczarr/zincludes.h @@ -43,7 +43,7 @@ extern "C" { #include "ncbytes.h" #include "ncauth.h" #include "nclog.h" -#include "ncrc.h" +#include "ncs3sdk.h" #include "ncindex.h" #include "ncjson.h" diff --git a/libnczarr/zinternal.c b/libnczarr/zinternal.c index ba1b907c4c..802cc07cfd 100644 --- a/libnczarr/zinternal.c +++ b/libnczarr/zinternal.c @@ -85,7 +85,7 @@ NCZ_finalize_internal(void) #ifdef ENABLE_NCZARR_FILTERS NCZ_filter_finalize(); #endif -#ifdef ENABLE_S3_SDK +#ifdef ENABLE_S3 NCZ_s3finalize(); #endif return NC_NOERR; diff --git a/libnczarr/zmap.c b/libnczarr/zmap.c index 46e1ac8345..63844ae4ab 100644 --- a/libnczarr/zmap.c +++ b/libnczarr/zmap.c @@ -21,7 +21,7 @@ nczmap_features(NCZM_IMPL impl) #ifdef ENABLE_NCZARR_ZIP case NCZM_ZIP: return zmap_zip.features; #endif -#ifdef ENABLE_S3_SDK +#ifdef ENABLE_S3 case NCZM_S3: return zmap_s3sdk.features; #endif default: break; @@ -52,7 +52,7 @@ nczmap_create(NCZM_IMPL impl, const char *path, int mode, size64_t flags, void* if(stat) goto done; break; #endif -#ifdef ENABLE_S3_SDK +#ifdef ENABLE_S3 case NCZM_S3: stat = zmap_s3sdk.create(path, mode, flags, parameters, &map); if(stat) goto done; @@ -90,7 +90,7 @@ nczmap_open(NCZM_IMPL impl, const char *path, int mode, size64_t flags, void* pa if(stat) goto done; break; #endif -#ifdef ENABLE_S3_SDK +#ifdef ENABLE_S3 case NCZM_S3: stat = zmap_s3sdk.open(path, mode, flags, parameters, &map); if(stat) goto done; @@ -477,40 +477,40 @@ nczm_sortlist(NClist* l) nczm_sortenvv(nclistlength(l),(char**)nclistcontents(l)); } -/* bubble sort a list of strings */ +static int +nczm_compare(const void* arg1, const void* arg2) +{ + char* n1 = *((char**)arg1); + char* n2 = *((char**)arg2); + return strcmp(n1,n2); +} + +/* quick sort a list of strings */ void nczm_sortenvv(int n, char** envv) { - size_t i, switched; - if(n <= 1) return; - do { - switched = 0; - for(i=0;i 0) { - envv[i] = ith1; - envv[i+1] = ith; - switched = 1; - } - } - } while(switched); + qsort(envv, n, sizeof(char*), nczm_compare); #if 0 +{int i; for(i=0;i>> sorted: [%d] %s\n",i,(const char*)envv[i]); +} #endif } void NCZ_freeenvv(int n, char** envv) { - int i; - char** p; - if(envv == NULL) return; - if(n < 0) + int i; + char** p; + if(envv == NULL) return; + if(n < 0) {for(n=0, p = envv; *p; n++); /* count number of strings */} - for(i=0;iformat_file_info; int validate = 0; NCbytes* prefix = ncbytesnew(); @@ -2447,15 +2448,14 @@ ncz_validate(NC_FILE_INFO_T* file) nclistpush(queue,path); path = NULL; do { - /* This should be full path key */ nullfree(path); path = NULL; + /* This should be full path key */ path = nclistremove(queue,0); /* remove from front of queue */ /* get list of next level segments (partial keys) */ - nclistclear(nextlevel); + assert(nclistlength(nextlevel)==0); if((stat=nczmap_search(map,path,nextlevel))) {validate = 0; goto done;} /* For each s in next level, test, convert to full path, and push onto queue */ - for(i=0;i 0) { segment = nclistremove(nextlevel,0); seglen = nulllen(segment); if((seglen >= 2 && memcmp(segment,".z",2)==0) || (seglen >= 4 && memcmp(segment,".ncz",4)==0)) { @@ -2469,6 +2469,7 @@ ncz_validate(NC_FILE_INFO_T* file) ncbytescat(prefix,segment); /* push onto queue */ nclistpush(queue,ncbytesextract(prefix)); + nullfree(segment); segment = NULL; } } while(nclistlength(queue) > 0); done: diff --git a/libnetcdf.settings.in b/libnetcdf.settings.in index d467c7b0fb..5d24d1baed 100644 --- a/libnetcdf.settings.in +++ b/libnetcdf.settings.in @@ -41,9 +41,9 @@ DAP4 Support: @HAS_DAP4@ Byte-Range Support: @HAS_BYTERANGE@ S3 Support: @HAS_S3@ +S3 SDK: @WHICH_S3_SDK@ NCZarr Support: @HAS_NCZARR@ -NCZarr Zip Support: @HAS_NCZARR_ZIP@ Diskless Support: @HAS_DISKLESS@ MMap Support: @HAS_MMAP@ diff --git a/libsrc/CMakeLists.txt b/libsrc/CMakeLists.txt index 7b9b2aad94..c7ae7762ff 100644 --- a/libsrc/CMakeLists.txt +++ b/libsrc/CMakeLists.txt @@ -29,9 +29,9 @@ ENDIF (USE_FFIO) IF (ENABLE_BYTERANGE) LIST(APPEND libsrc_SOURCES httpio.c) -IF (ENABLE_S3_SDK) +IF (ENABLE_S3) LIST(APPEND libsrc_SOURCES s3io.c) -ENDIF(ENABLE_S3_SDK) +ENDIF(ENABLE_S3) ENDIF(ENABLE_BYTERANGE) add_library(netcdf3 OBJECT ${libsrc_SOURCES}) diff --git a/libsrc/Makefile.am b/libsrc/Makefile.am index 08f074131d..ca5333c6b7 100644 --- a/libsrc/Makefile.am +++ b/libsrc/Makefile.am @@ -32,10 +32,9 @@ endif !USE_FFIO if ENABLE_BYTERANGE libnetcdf3_la_SOURCES += httpio.c -if ENABLE_S3_SDK - libnetcdf3_la_SOURCES += s3io.c -AM_CXXFLAGS = -std=c++11 -endif ENABLE_S3_SDK +if ENABLE_S3 + libnetcdf3_la_SOURCES += s3io.c +endif endif ENABLE_BYTERANGE diff --git a/libsrc/httpio.c b/libsrc/httpio.c index 3e8243280c..d9b376d2e7 100644 --- a/libsrc/httpio.c +++ b/libsrc/httpio.c @@ -23,6 +23,7 @@ #include "nc3internal.h" #include "nclist.h" #include "ncbytes.h" +#include "ncuri.h" #undef DEBUG @@ -160,15 +161,19 @@ httpio_open(const char* path, int status; NCHTTP* http = NULL; size_t sizehint; + NCURI* uri = NULL; if(path == NULL ||* path == 0) return EINVAL; + ncuriparse(path,&uri); + if(uri == NULL) {status = NC_EURL; goto done;} + /* Create private data */ if((status = httpio_new(path, ioflags, &nciop, &http))) goto done; /* Open the path and get curl handle and object size */ - if((status = nc_http_init(&http->state))) goto done; - if((status = nc_http_size(http->state,path,&http->size))) goto done; + if((status = nc_http_open(path,&http->state))) goto done; + if((status = nc_http_size(http->state,&http->size))) goto done; sizehint = pagesize; @@ -256,7 +261,7 @@ httpio_get(ncio* const nciop, off_t offset, size_t extent, int rflags, void** co assert(http->region == NULL); http->region = ncbytesnew(); ncbytessetalloc(http->region,(unsigned long)extent); - if((status = nc_http_read(http->state,nciop->path,offset,extent,http->region))) + if((status = nc_http_read(http->state,offset,extent,http->region))) goto done; assert(ncbyteslength(http->region) == extent); if(vpp) *vpp = ncbytescontents(http->region); diff --git a/libsrc/ncio.c b/libsrc/ncio.c index 7574903a2b..b9095b2f0c 100644 --- a/libsrc/ncio.c +++ b/libsrc/ncio.c @@ -41,7 +41,7 @@ extern int ffio_open(const char*,int,off_t,size_t,size_t*,void*,ncio**,void** co extern int httpio_open(const char*,int,off_t,size_t,size_t*,void*,ncio**,void** const); #endif -#ifdef ENABLE_S3_SDK +#ifdef ENABLE_S3 extern int s3io_open(const char*,int,off_t,size_t,size_t*,void*,ncio**,void** const); #endif @@ -107,7 +107,7 @@ ncio_open(const char *path, int ioflags, if(modetest == NC_HTTP) { return httpio_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp); } -# ifdef ENABLE_S3_SDK +# ifdef ENABLE_S3 if(modetest == NC_S3SDK) { return s3io_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp); } diff --git a/libsrc4/nc4attr.c b/libsrc4/nc4attr.c index 2dcd8a9f05..5b52c3c4a9 100644 --- a/libsrc4/nc4attr.c +++ b/libsrc4/nc4attr.c @@ -139,55 +139,10 @@ nc4_get_att_ptrs(NC_FILE_INFO_T *h5, NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, bugs! */ if (data) { -#ifdef SEPDATA - if (att->vldata) - { - size_t base_typelen; - nc_hvl_t *vldest = data; - NC_TYPE_INFO_T *type; - int i; - - /* Get the type object for the attribute's type */ - if ((retval = nc4_find_type(h5, att->nc_typeid, &type))) - BAIL(retval); - - /* Retrieve the size of the base type */ - if ((retval = nc4_get_typelen_mem(h5, type->u.v.base_nc_typeid, &base_typelen))) - BAIL(retval); - - for (i = 0; i < att->len; i++) - { - vldest[i].len = att->vldata[i].len; - if (!(vldest[i].p = malloc(vldest[i].len * base_typelen))) - BAIL(NC_ENOMEM); - memcpy(vldest[i].p, att->vldata[i].p, vldest[i].len * base_typelen); - } - } - else if (att->stdata) - { - int i; - for (i = 0; i < att->len; i++) - { - /* Check for NULL pointer for string (valid in HDF5) */ - if(att->stdata[i]) - { - if (!(((char **)data)[i] = strdup(att->stdata[i]))) - BAIL(NC_ENOMEM); - } - else - ((char **)data)[i] = att->stdata[i]; - } - } - else - { - memcpy(data, bufr, (size_t)(att->len * type_size)); - } -#else { if((retval = nc_copy_data(h5->controller->ext_ncid,mem_type,bufr,att->len,data))) BAIL(retval); } -#endif } exit: diff --git a/libsrc4/nc4internal.c b/libsrc4/nc4internal.c index 25270e6598..d64f0c12a0 100644 --- a/libsrc4/nc4internal.c +++ b/libsrc4/nc4internal.c @@ -1327,37 +1327,6 @@ nc4_att_free(NC_ATT_INFO_T *att) if (att->hdr.name) free(att->hdr.name); -#ifdef SEPDATA - /* Free memory that was malloced to hold data for this - * attribute. */ - if (att->data) { - free(att->data); - } - - /* If this is a string array attribute, delete all members of the - * string array, then delete the array of pointers to strings. (The - * array was filled with pointers by HDF5 when the att was read, - * and memory for each string was allocated by HDF5. That's why I - * use free and not nc_free, because the netCDF library didn't - * allocate the memory that is being freed.) */ - if (att->stdata) - { - int i; - for (i = 0; i < att->len; i++) - if(att->stdata[i]) - free(att->stdata[i]); - free(att->stdata); - } - - /* If this att has vlen data, release it. */ - if (att->vldata) - { - int i; - for (i = 0; i < att->len; i++) - nc_free_vlen(&att->vldata[i]); - free(att->vldata); - } -#else if (att->data) { NC_OBJ* parent; NC_FILE_INFO_T* h5 = NULL; @@ -1372,7 +1341,6 @@ nc4_att_free(NC_ATT_INFO_T *att) free(att->data); /* reclaim top level */ att->data = NULL; } -#endif done: free(att); diff --git a/libsrc4/nc4var.c b/libsrc4/nc4var.c index bb01a00c9f..f93a3f4660 100644 --- a/libsrc4/nc4var.c +++ b/libsrc4/nc4var.c @@ -263,53 +263,14 @@ NC4_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep, { /* Do we have a fill value for this var? */ if (var->fill_value) -#ifdef SEPDATA - { - if (var->type_info->nc_type_class == NC_STRING) - { - assert(*(char **)var->fill_value); - /* This will allocate memory and copy the string. */ - if (!(*(char **)fill_valuep = strdup(*(char **)var->fill_value))) - { - free(*(char **)fill_valuep); - return NC_ENOMEM; - } - } - else - { - assert(var->type_info->size); - memcpy(fill_valuep, var->fill_value, var->type_info->size); - } - } -#else { int xtype = var->type_info->hdr.id; if((retval = nc_copy_data(ncid,xtype,var->fill_value,1,fill_valuep))) return retval; } -#endif else { -#ifdef SEPDATA - if (var->type_info->nc_type_class == NC_STRING) - { - if (!(*(char **)fill_valuep = calloc(1, sizeof(char *)))) - return NC_ENOMEM; - - if ((retval = nc4_get_default_fill_value(var->type_info->hdr.ud, (char **)fill_valuep))) - { - free(*(char **)fill_valuep); - return retval; - } - } - else - { - if ((retval = nc4_get_default_fill_value(var->type_info->hdr.id, fill_valuep))) - return retval; - } -#else if ((retval = nc4_get_default_fill_value(var->type_info, fill_valuep))) return retval; -#endif } } diff --git a/nc_test/test_byterange.sh b/nc_test/test_byterange.sh index ee5c567bb2..38923bfa03 100755 --- a/nc_test/test_byterange.sh +++ b/nc_test/test_byterange.sh @@ -12,15 +12,17 @@ if test "x$FEATURE_THREDDSTEST" = x1 ; then URL3="https://thredds-test.unidata.ucar.edu/thredds/fileServer/pointData/cf_dsg/example/point.nc#mode=bytes" URL4b="https://thredds-test.unidata.ucar.edu/thredds/fileServer/irma/metar/files/METAR_20170910_0000.nc#bytes" fi -if test "x$FEATURE_S3TESTS" = xyes ; then +if test "x$FEATURE_S3TESTS" != xno ; then URL4a="https://s3.us-east-1.amazonaws.com/noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes" URL4c="s3://noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes" +# Test alternate URL with no specified region +URL4e="http://noaa-goes16.s3.amazonaws.com/ABI-L1b-RadF/2022/001/18/OR_ABI-L1b-RadF-M6C01_G16_s20220011800205_e20220011809513_c20220011809562.nc#mode=bytes,s3" +fi +if test "x$FEATURE_S3TESTS" = xyes ; then # Requires auth URL3b="s3://unidata-zarr-test-data/byterangefiles/upload3.nc#bytes" # Requires auth URL4d="s3://unidata-zarr-test-data/byterangefiles/upload4.nc#bytes&aws.profile=unidata" -# Test alternate URL with no specified region -URL4e="http://noaa-goes16.s3.amazonaws.com/ABI-L1b-RadF/2022/001/18/OR_ABI-L1b-RadF-M6C01_G16_s20220011800205_e20220011809513_c20220011809562.nc#mode=bytes,s3" fi URL4f="https://crudata.uea.ac.uk/cru/data/temperature/HadCRUT.4.6.0.0.median.nc#mode=bytes" @@ -33,17 +35,15 @@ echo "" testsetup() { U=$1 # Create and upload test files -if test "x$FEATURE_S3TESTS" = xyes ; then rm -f upload4.nc upload3.nc -${execdir}/../nczarr_test/s3util clear -u ${U} -k /byterangefiles +${execdir}/../nczarr_test/s3util -u ${U} -k /byterangefiles clear ${NCGEN} -lb -3 ${srcdir}/nc_enddef.cdl mv nc_enddef.nc upload3.nc -${execdir}/../nczarr_test/s3util upload -u ${U} -k /byterangefiles/upload3.nc -f upload3.nc +${execdir}/../nczarr_test/s3util -u ${U} -k /byterangefiles/upload3.nc -f upload3.nc upload if test "x$FEATURE_HDF5" = xyes ; then ${NCGEN} -lb -4 ${srcdir}/nc_enddef.cdl mv nc_enddef.nc upload4.nc -${execdir}/../nczarr_test/s3util upload -u ${U} -k /byterangefiles/upload4.nc -f upload4.nc -fi +${execdir}/../nczarr_test/s3util -u ${U} -k /byterangefiles/upload4.nc -f upload4.nc upload fi rm -f tst_http_nc3.cdl tst_http_nc4?.cdl } @@ -51,16 +51,14 @@ rm -f tst_http_nc3.cdl tst_http_nc4?.cdl testcleanup() { U=$1 rm -f upload4.nc upload3.nc -if test "x$FEATURE_S3TESTS" = xyes ; then -${execdir}/../nczarr_test/s3util clear -u ${U} -k /byterangefiles -fi +${execdir}/../nczarr_test/s3util -u ${U} -k /byterangefiles clear } testbytes() { TAG="$1" EXPECTED="$2" U="$3" -K=`${NCDUMP} -k "$U" | tr -d '\r'` +K=`${NCDUMP} -k "$U" | tr -d '\r\n'` if test "x$K" != "x$EXPECTED" ; then echo "test_http: -k flag mismatch: expected=$EXPECTED have=$K" exit 1 @@ -76,7 +74,7 @@ tests3auth() { TAG="$1" EXPECTED="$2" U="$3" -K=`${NCDUMP} -k "$U" | tr -d '\r'` +K=`${NCDUMP} -k "$U" | tr -d '\r\n'` if test "x$K" != "x$EXPECTED" ; then echo "test_http: -k flag mismatch: expected=$EXPECTED have=$K" exit 1 @@ -105,23 +103,31 @@ if test "x$FEATURE_HDF5" = xyes ; then testbytes nc4f netCDF-4 "$URL4f" fi -if test "x$FEATURE_S3" = xyes ; then +if test "x$URL3B" != x ; then echo "***Test remote netcdf-3 file: s3 auth" tests3auth nc3b classic "$URL3b" fi -if test "x$FEATURE_S3" = xyes && test "x$FEATURE_HDF5" = xyes ; then +if test "x$URL4a" != x ; then echo "***Test remote netdf-4 file: s3" testbytes nc4a netCDF-4 "$URL4a" +fi +if test "x$URL4c" != x ; then echo "***Test remote netcdf-4 file: s3" testbytes nc4c netCDF-4 "$URL4c" +fi +if test "x$URL4d" != x ; then echo "***Test remote netcdf-4 file: s3 auth" tests3auth nc4d netCDF-4 "$URL4d" +fi +if test "x$URL4e" != x ; then echo "***Test remote netcdf-4 file: s3 noauth" testbytes nc4e netCDF-4 "$URL4e" fi # Cleanup +if test "x$FEATURE_S3TESTS" = xyes ; then testcleanup https://s3.us-east-1.amazonaws.com/unidata-zarr-test-data +fi exit diff --git a/nc_test4/tst_filterinstall.sh b/nc_test4/tst_filterinstall.sh index 9b119d88e9..942dc678c3 100755 --- a/nc_test4/tst_filterinstall.sh +++ b/nc_test4/tst_filterinstall.sh @@ -7,12 +7,20 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -if test "x$TESTNCZARR" = x1 ; then -. ./test_nczarr.sh +if test "x$TESTNCZARR" = x1; then +. $srcdir/test_nczarr.sh fi set -e +isolate "testdir_filterinstall" +THISDIR=`pwd` +cd $ISOPATH + +if test "x$TESTNCZARR" = x1; then +s3isolate +fi + # Use this plugin path export HDF5_PLUGIN_PATH="${FEATURE_PLUGIN_INSTALL_DIR}" diff --git a/nc_test4/tst_specific_filters.sh b/nc_test4/tst_specific_filters.sh index e094852049..64333edaa0 100755 --- a/nc_test4/tst_specific_filters.sh +++ b/nc_test4/tst_specific_filters.sh @@ -12,6 +12,12 @@ set -e if test "x$TESTNCZARR" = x1 ; then . "$srcdir/test_nczarr.sh" +s3isolate "testdir_specific_filters" +THISDIR=`pwd` +cd $ISOPATH +fi + +if test "x$TESTNCZARR" = x1 ; then BLOSCARGS="32001,0,0,0,256,5,1,1" BLOSCCODEC='[{\"id\": \"blosc\",\"clevel\": 5,\"blocksize\": 256,\"cname\": \"lz4\",\"shuffle\": 1}]' else @@ -178,6 +184,7 @@ if test "x$TESTNCZARR" = x1 ; then testset file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testset zip ; fi if test "x$FEATURE_S3TESTS" = xyes ; then testset s3 ; fi + if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup else testset nc fi diff --git a/nc_test4/tst_unknown.sh b/nc_test4/tst_unknown.sh index 76a7b1b6ab..8bf30c04d8 100755 --- a/nc_test4/tst_unknown.sh +++ b/nc_test4/tst_unknown.sh @@ -3,8 +3,13 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh +isolate "testdir_unknown" +THISDIR=`pwd` +cd $ISOPATH + if test "x$TESTNCZARR" = x1 ; then . "$srcdir/test_nczarr.sh" +s3isolate fi set -e @@ -101,6 +106,7 @@ if test "x$TESTNCZARR" = x1 ; then testunk file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testunk zip ; fi if test "x$FEATURE_S3TESTS" = xyes ; then testunk s3 ; fi + if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup else testunk nc fi diff --git a/ncdap_test/tst_fillmismatch.sh b/ncdap_test/tst_fillmismatch.sh index 1b7d0ffaa2..933144abf2 100755 --- a/ncdap_test/tst_fillmismatch.sh +++ b/ncdap_test/tst_fillmismatch.sh @@ -13,19 +13,18 @@ EXPECTED="${srcdir}/expected3" URL='file://' URL="${URL}${srcdir}/testdata3/$F" -# First check that with [nofillmismatch], we get a failure -rm -f ./tmp_tst_mismatch -NOURL="[nofillmismatch]$URL" -if ${NCDUMP} "${NOURL}" > ./tmp_tst_mismatch 2>&1 ; then +echo "Check that with [nofillmismatch], we get a failure" +NOURL="$URL#nofillmismatch" +if ${NCDUMP} "${NOURL}" > /dev/null ; then echo "*** Fail: ${NCDUMP} ${NOURL} passed" exit 1 else echo "*** XFail: ${NCDUMP} ${NOURL} failed" fi -# Now check that with [fillmismatch] (default), we get success -rm -f ./tmp_tst_mismatch -if ${NCDUMP} "${URL}" > ./tmp_tst_mismatch ; then +echo "Check that with [fillmismatch] (default), we get success" +rm -f tmp_tst_mismatch +if ${NCDUMP} "${URL}" > tmp_tst_mismatch ; then echo "*** Pass: ${NCDUMP} ${URL} passed" else echo "*** Fail: ${NCDUMP} ${URL} failed" diff --git a/ncdump/Makefile.am b/ncdump/Makefile.am index 6407a755ba..49e34c0ffa 100644 --- a/ncdump/Makefile.am +++ b/ncdump/Makefile.am @@ -205,14 +205,14 @@ test_keywords.sh ref_keyword1.cdl ref_keyword2.cdl ref_keyword3.cdl ref_keyword4 ref_tst_nofilters.cdl test_scope.sh \ test_rcmerge.sh ref_rcmerge1.txt ref_rcmerge2.txt ref_rcmerge3.txt \ scope_ancestor_only.cdl scope_ancestor_subgroup.cdl scope_group_only.cdl scope_preorder.cdl \ -ref_rcapi.txt ref_tst_enum_undef.cdl tst_calendars_nc4.cdl ref_times_nc4.cdl +ref_rcapi.txt ref_tst_enum_undef.cdl tst_calendars_nc4.cdl ref_times_nc4.cdl ref_tst_comp2.cdl # The L512.bin file is file containing exactly 512 bytes each of value 0. # It is used for creating hdf5 files with varying offsets for testing. EXTRA_DIST += L512.bin EXTRA_DIST += tst_ctests.sh ref_ctest_small_3.c ref_ctest_small_4.c \ - ref_ctest_special_atts_4.c + ref_ctest_special_atts_4.c test_ncdump.sh EXTRA_DIST += testpathcvt.sh ref_pathcvt.txt @@ -251,5 +251,4 @@ scope_*.nc copy_scope_*.cdl keyword5.nc tst_enum_undef.cdl tst_times_nc4.cdl # Remove directories clean-local: - rm -fr rcmergedir rchome - + rm -fr rcmergedir rchome testdir_ncdump_* diff --git a/ncdump/test_ncdump.sh b/ncdump/test_ncdump.sh new file mode 100644 index 0000000000..170daf5092 --- /dev/null +++ b/ncdump/test_ncdump.sh @@ -0,0 +1,85 @@ +#!/bin/sh + +if test "x$SETX" != x; then set -x; fi + +ERR() { + RES=$? + if [ $RES -ne 0 ]; then + echo "Error found: $RES" + exit $RES + fi +} + +# Remove the version information from _NCProperties +cleanncprops() { + src="$1" + dst="$2" + rm -f $dst + cat $src \ + | sed -e '/:_Endianness/d' \ + | sed -e 's/_SuperblockVersion = [12]/_SuperblockVersion = 0/' \ + | sed -e 's/\(netcdflibversion\|netcdf\)=.*|/\1=NNNN|/' \ + | sed -e 's/\(hdf5libversion\|hdf5\)=.*"/\1=HHHH"/' \ + | grep -v '_NCProperties' \ + | cat >$dst +} + +createtestinputs() { +echo "*** Running tst_create_files.c to create solar test files." +${execdir}/tst_create_files ; ERR +echo "*** Testing tst_create_files output for netCDF-4 features." +${NCDUMP} tst_solar_1.nc | sed 's/e+0/e+/g' > tst_solar_1.cdl ; ERR +diff -b tst_solar_1.cdl $srcdir/ref_tst_solar_1.cdl ; ERR +${NCDUMP} tst_solar_2.nc | sed 's/e+0/e+/g' > tst_solar_2.cdl ; ERR +diff -b tst_solar_2.cdl $srcdir/ref_tst_solar_2.cdl ; ERR + +echo "*** Running tst_group_data.c to create test files." +${execdir}/tst_group_data ; ERR +${NCDUMP} tst_group_data.nc | sed 's/e+0/e+/g' > tst_group_data.cdl ; ERR +diff -b tst_group_data.cdl $srcdir/ref_tst_group_data.cdl ; ERR + +echo "*** Testing -v option with relative name and groups..." +${NCDUMP} -v var,var2 tst_group_data.nc | sed 's/e+0/e+/g' > tst_group_data.cdl ; ERR +diff -b tst_group_data.cdl $srcdir/ref_tst_group_data.cdl ; ERR + +echo "*** Running tst_enum_data.c to create test files." +${execdir}/tst_enum_data ; ERR +${NCDUMP} tst_enum_data.nc | sed 's/e+0/e+/g' > tst_enum_data.cdl ; ERR +diff -b tst_enum_data.cdl $srcdir/ref_tst_enum_data.cdl ; ERR + +echo "*** Running tst_opaque_data.c to create test files." +${execdir}/tst_opaque_data ; ERR +${NCDUMP} tst_opaque_data.nc | sed 's/e+0/e+/g' > tst_opaque_data.cdl ; ERR +diff -b tst_opaque_data.cdl $srcdir/ref_tst_opaque_data.cdl ; ERR + +echo "*** Running tst_comp.c to create test files." +${execdir}/tst_comp ; ERR +${NCDUMP} tst_comp.nc | sed 's/e+0/e+/g' > tst_comp.cdl ; ERR +diff -b tst_comp.cdl $srcdir/ref_tst_comp.cdl ; ERR + +echo "*** Running tst_comp2.c to create test files." +${execdir}/tst_comp2 ; ERR +${NCDUMP} tst_comp2.nc | sed 's/e+0/e+/g' > tst_comp2.cdl ; ERR +diff -b tst_comp2.cdl $srcdir/ref_tst_comp2.cdl ; ERR + +echo "*** Running tst_nans.c to create test files." +${execdir}/tst_nans ; ERR +${NCDUMP} tst_nans.nc | sed 's/e+0/e+/g' > tst_nans.cdl ; ERR +diff -b tst_nans.cdl $srcdir/ref_tst_nans.cdl ; ERR + +echo "*** Running tst_special_atts.c to create test files." +${execdir}/tst_special_atts ; ERR +${NCDUMP} -c -s tst_special_atts.nc > tst_special_atts.cdl ; ERR +cleanncprops tst_special_atts.cdl tst_special_atts.tmp +cleanncprops $srcdir/ref_tst_special_atts.cdl ref_tst_special_atts.tmp +echo "*** comparing tst_special_atts.cdl with ref_tst_special_atts.cdl..." +diff -b tst_special_atts.tmp ref_tst_special_atts.tmp ; ERR + +${execdir}/tst_string_data ; ERR + +${execdir}/tst_fillbug +# echo "*** dumping tst_fillbug.nc to tst_fillbug.cdl..." +${NCDUMP} tst_fillbug.nc > tst_fillbug.cdl +# echo "*** comparing tst_fillbug.cdl with ref_tst_fillbug.cdl..." +diff -b tst_fillbug.cdl $srcdir/ref_tst_fillbug.cdl +} diff --git a/ncdump/tst_nccopy4.sh b/ncdump/tst_nccopy4.sh index 1700aeee5d..baaa0e6140 100755 --- a/ncdump/tst_nccopy4.sh +++ b/ncdump/tst_nccopy4.sh @@ -3,22 +3,23 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh + +. $srcdir/test_ncdump.sh + +isolate "testdir_ncdump_nccopy4" + +# Move into test directory +cd $ISOPATH + set -e # For a netCDF-4 build, test nccopy on netCDF files in this directory -#if 0 -if test -f tst_group_data${ext} ; then ${execdir}/tst_group_data ; fi -if test -f tst_enum_data${ext} ; then ${execdir}/tst_enum_data ; fi -if test -f tst_comp${ext} ; then ${execdir}/tst_comp ; fi -if test -f tst_comp2${ext} ; then ${execdir}/tst_comp2 ; fi -#endif - echo "" -# These files are actually in $srcdir in distcheck builds, so they -# need to be handled differently. -# ref_tst_compounds2 ref_tst_compounds3 ref_tst_compounds4 +# Create common test inputs +createtestinputs + TESTFILES0='tst_comp tst_comp2 tst_enum_data tst_fillbug tst_group_data tst_nans tst_opaque_data tst_solar_1 tst_solar_2 tst_solar_cmp tst_special_atts' @@ -44,7 +45,7 @@ for i in $TESTFILES ; do done # echo "*** Testing compression of deflatable files ..." -./tst_compress +${execdir}/tst_compress echo "*** Test nccopy -d1 can compress a classic format file ..." ${NCCOPY} -d1 tst_inflated.nc tst_deflated.nc if test `wc -c < tst_deflated.nc` -ge `wc -c < tst_inflated.nc`; then @@ -86,7 +87,7 @@ for i in $TESTFILES0 ; do diff copy_of_$i.cdl tmp_ncc4.cdl rm copy_of_$i.nc copy_of_$i.cdl tmp_ncc4.cdl done -./tst_chunking +${execdir}/tst_chunking echo "*** Test that nccopy -c can chunk and unchunk files" ${NCCOPY} -M0 tst_chunking.nc tmp_ncc4.nc ${NCDUMP} tmp_ncc4.nc > tmp_ncc4.cdl diff --git a/ncdump/tst_ncgen4_diff.sh b/ncdump/tst_ncgen4_diff.sh index 044ee8e8b8..75666fe985 100755 --- a/ncdump/tst_ncgen4_diff.sh +++ b/ncdump/tst_ncgen4_diff.sh @@ -42,7 +42,7 @@ for x in ${TESTSET} ; do ${NCDUMP} ${headflag} ${specflag} -n ${x} ${x}_$$.nc | sed 's/e+0/e+/g' > ${x}.dmp # compare the expected (silently if XFAIL) if test "x$isxfail" = "x1" -a "x$SHOWXFAILS" = "x" ; then - if diff -b -bw ${expected}/${x}.dmp ${x}.dmp >/dev/null 2>&1; then ok=1; else ok=0; fi + if diff -b -bw ${expected}/${x}.dmp ${x}.dmp &>/dev/null ; then ok=1; else ok=0; fi else if diff -b -w ${expected}/${x}.dmp ${x}.dmp ; then ok=1; else ok=0; fi fi diff --git a/ncdump/tst_netcdf4.sh b/ncdump/tst_netcdf4.sh index 8a9dbb7823..d878656543 100755 --- a/ncdump/tst_netcdf4.sh +++ b/ncdump/tst_netcdf4.sh @@ -5,32 +5,21 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh +. $srcdir/test_ncdump.sh + +isolate "testdir_ncdump_netcdf4" + +# Move into isolation directory +cd $ISOPATH + set -e -# Remove the version information from _NCProperties -cleanncprops() { - src="$1" - dst="$2" - rm -f $dst - cat $src \ - | sed -e '/:_Endianness/d' \ - | sed -e 's/_SuperblockVersion = [12]/_SuperblockVersion = 0/' \ - | sed -e 's/\(netcdflibversion\|netcdf\)=.*|/\1=NNNN|/' \ - | sed -e 's/\(hdf5libversion\|hdf5\)=.*"/\1=HHHH"/' \ - | grep -v '_NCProperties' \ - | cat >$dst -} - -ERR() { - RES=$? - if [ $RES -ne 0 ]; then - echo "Error found: $RES" - exit $RES - fi -} +# Create common test inputs +createtestinputs echo "" echo "*** Testing ncgen and ncdump for netCDF-4 format." + ${NCGEN} -k nc4 -b -o tst_netcdf4_c0_4.nc ${ncgenc04} ;ERR ${NCDUMP} -n c1 tst_netcdf4_c0_4.nc | sed 's/e+0/e+/g' > tst_netcdf4_c1_4.cdl ; ERR diff -b tst_netcdf4_c1_4.cdl $srcdir/ref_ctest1_nc4.cdl ; ERR @@ -41,14 +30,6 @@ ${NCGEN} -k nc7 -b -o tst_netcdf4_c0.nc ${ncgenc0} ; ERR echo "*** Testing that program tst_h_rdc0 can read tst_netcdf4_c0.nc." ${execdir}/tst_h_rdc0 ; ERR -echo "*** Running tst_create_files.c to create test files." -${execdir}/tst_create_files ; ERR -echo "*** Testing tst_create_files output for netCDF-4 features." -${NCDUMP} tst_solar_1.nc | sed 's/e+0/e+/g' > tst_solar_1.cdl ; ERR -diff -b tst_solar_1.cdl $srcdir/ref_tst_solar_1.cdl ; ERR -${NCDUMP} tst_solar_2.nc | sed 's/e+0/e+/g' > tst_solar_2.cdl ; ERR -diff -b tst_solar_2.cdl $srcdir/ref_tst_solar_2.cdl ; ERR - if test -f tst_roman_szip_simple.nc; then echo "*** Testing szip compression." ${NCDUMP} tst_roman_szip_simple.nc | sed 's/e+0/e+/g' > tst_roman_szip_simple.cdl ; ERR @@ -57,52 +38,15 @@ if test -f tst_roman_szip_simple.nc; then diff -b tst_roman_szip_unlim.cdl $srcdir/ref_roman_szip_unlim.cdl ; ERR fi -echo "*** Running tst_group_data.c to create test files." -${execdir}/tst_group_data ; ERR -${NCDUMP} tst_group_data.nc | sed 's/e+0/e+/g' > tst_group_data.cdl ; ERR -diff -b tst_group_data.cdl $srcdir/ref_tst_group_data.cdl ; ERR - echo "*** Testing -v option with absolute name and groups..." ${NCDUMP} -v g2/g3/var tst_group_data.nc | sed 's/e+0/e+/g' > tst_group_data.cdl ; ERR diff -b tst_group_data.cdl $srcdir/ref_tst_group_data_v23.cdl ; ERR -echo "*** Testing -v option with relative name and groups..." -${NCDUMP} -v var,var2 tst_group_data.nc | sed 's/e+0/e+/g' > tst_group_data.cdl ; ERR -diff -b tst_group_data.cdl $srcdir/ref_tst_group_data.cdl ; ERR - -echo "*** Running tst_enum_data.c to create test files." -${execdir}/tst_enum_data ; ERR -${NCDUMP} tst_enum_data.nc | sed 's/e+0/e+/g' > tst_enum_data.cdl ; ERR -diff -b tst_enum_data.cdl $srcdir/ref_tst_enum_data.cdl ; ERR - echo "*** Running tst_enum_undef.c to create test files." ${execdir}/tst_enum_undef ; ERR ${NCDUMP} tst_enum_undef.nc | sed 's/e+0/e+/g' > tst_enum_undef.cdl ; ERR diff -b tst_enum_undef.cdl $srcdir/ref_tst_enum_undef.cdl ; ERR -echo "*** Running tst_opaque_data.c to create test files." -${execdir}/tst_opaque_data ; ERR -${NCDUMP} tst_opaque_data.nc | sed 's/e+0/e+/g' > tst_opaque_data.cdl ; ERR -diff -b tst_opaque_data.cdl $srcdir/ref_tst_opaque_data.cdl ; ERR - -echo "*** Running tst_comp.c to create test files." -${execdir}/tst_comp ; ERR -${NCDUMP} tst_comp.nc | sed 's/e+0/e+/g' > tst_comp.cdl ; ERR -diff -b tst_comp.cdl $srcdir/ref_tst_comp.cdl ; ERR - -echo "*** Running tst_nans.c to create test files." -${execdir}/tst_nans ; ERR -${NCDUMP} tst_nans.nc | sed 's/e+0/e+/g' > tst_nans.cdl ; ERR -diff -b tst_nans.cdl $srcdir/ref_tst_nans.cdl ; ERR - -echo "*** Running tst_special_atts.c to create test files." -${execdir}/tst_special_atts ; ERR -${NCDUMP} -c -s tst_special_atts.nc > tst_special_atts.cdl ; ERR -cleanncprops tst_special_atts.cdl tst_special_atts.tmp -cleanncprops $srcdir/ref_tst_special_atts.cdl ref_tst_special_atts.tmp -echo "*** comparing tst_special_atts.cdl with ref_tst_special_atts.cdl..." -diff -b tst_special_atts.tmp ref_tst_special_atts.tmp ; ERR - # This creates a memory leak if test 0 = 1 ; then echo "*** Running tst_vlen_data.c to create test files." @@ -114,7 +58,7 @@ fi #echo "" #echo "*** Testing ncdump on file with corrupted header " #rm -f ./ignore_tst_netcdf4 -#if ${NCDUMP} ${srcdir}/ref_test_corrupt_magic.nc > ./ignore_tst_netcdf4 2>&1 ; then +#if ${NCDUMP} ${srcdir}/ref_test_corrupt_magic.nc &> ./ignore_tst_netcdf4 ; then #echo "***Fail: ncdump should have failed on ref_test_corrupt_magic.nc" #else #echo "***XFail: ncdump properly failed on ref_test_corrupt_magic.nc" diff --git a/nczarr_test/CMakeLists.txt b/nczarr_test/CMakeLists.txt index d6e37aed8c..a0c6b80c87 100644 --- a/nczarr_test/CMakeLists.txt +++ b/nczarr_test/CMakeLists.txt @@ -76,16 +76,18 @@ IF(ENABLE_TESTS) TARGET_INCLUDE_DIRECTORIES(tst_fillonlyz PUBLIC ../libnczarr) # Helper programs for testing - BUILD_BIN_TEST(zmapio ${COMMONSRC}) - TARGET_INCLUDE_DIRECTORIES(zmapio PUBLIC ../libnczarr) BUILD_BIN_TEST(zhex) BUILD_BIN_TEST(zisjson ${COMMONSRC}) TARGET_INCLUDE_DIRECTORIES(zisjson PUBLIC ../libnczarr) BUILD_BIN_TEST(zs3parse ${COMMONSRC}) TARGET_INCLUDE_DIRECTORIES(zs3parse PUBLIC ../libnczarr) - if(ENABLE_S3) - BUILD_BIN_TEST(s3util ${COMMONSRC}) - TARGET_INCLUDE_DIRECTORIES(s3util PUBLIC ../libnczarr) + BUILD_BIN_TEST(zmapio ${COMMONSRC}) + TARGET_INCLUDE_DIRECTORIES(zmapio PUBLIC ../libnczarr) + + IF(ENABLE_S3 AND NOT WITH_S3_TESTING STREQUAL "NO") + # Helper programs for testing + BUILD_BIN_TEST(s3util ${COMMONSRC}) + TARGET_INCLUDE_DIRECTORIES(s3util PUBLIC ../libnczarr) endif() SET(ncdumpchunks_SOURCE ncdumpchunks.c) @@ -116,6 +118,7 @@ IF(ENABLE_TESTS) add_sh_test(nczarr_test run_strings) add_sh_test(nczarr_test run_scalar) add_sh_test(nczarr_test run_nulls) + add_sh_test(nczarr_test run_external) BUILD_BIN_TEST(test_quantize ${TSTCOMMONSRC}) add_sh_test(nczarr_test run_quantize) @@ -137,8 +140,12 @@ IF(ENABLE_TESTS) build_bin_test(test_filter_avail) ADD_SH_TEST(nczarr_test run_nczfilter) ADD_SH_TEST(nczarr_test run_filter) - ADD_SH_TEST(nczarr_test run_unknown) ADD_SH_TEST(nczarr_test run_specific_filters) + IF(FALSE) + # This test is too dangerous to run in a parallel make environment. + # It causes race conditions. So suppress and only test by hand. + ADD_SH_TEST(nczarr_test run_unknown) + ENDIF(FALSE) ENDIF(ENABLE_FILTER_TESTING) if(ENABLE_NCZARR_ZIP) add_sh_test(nczarr_test run_newformat) diff --git a/nczarr_test/Makefile.am b/nczarr_test/Makefile.am index f088765d79..de0a66a637 100644 --- a/nczarr_test/Makefile.am +++ b/nczarr_test/Makefile.am @@ -7,13 +7,12 @@ include $(top_srcdir)/lib_flags.am TESTS_ENVIRONMENT = - #TEST_EXTENSIONS = .sh - #SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose #sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose #LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose #TESTS_ENVIRONMENT += export SETX=1; +#TESTS_ENVIRONMENT += export NCTRACING=1; AM_CPPFLAGS += -I${top_srcdir} -I${top_srcdir}/libnczarr AM_LDFLAGS += ${top_builddir}/liblib/libnetcdf.la @@ -23,7 +22,7 @@ LDADD = ${top_builddir}/liblib/libnetcdf.la check_PROGRAMS = TESTS = -check_PROGRAMS += ut_map ut_mapapi ut_json ut_projections ut_chunking +check_PROGRAMS += ut_map ut_mapapi ut_json ut_projections ut_chunking commonsrc = ut_util.c ut_test.c ut_includes.h ut_test.h ut_util.h test_nczarr_utils.h tstcommonsrc = tst_utils.c tst_utils.h @@ -42,8 +41,8 @@ TESTS += run_ut_chunk.sh if BUILD_UTILITIES TESTS += run_ut_map.sh -TESTS += run_ut_mapapi.sh TESTS += run_ut_misc.sh +TESTS += run_ut_mapapi.sh TESTS += run_ncgen4.sh if USE_HDF5 @@ -65,6 +64,7 @@ TESTS += run_strings.sh TESTS += run_scalar.sh TESTS += run_nulls.sh TESTS += run_notzarr.sh +TESTS += run_external.sh endif #BUILD_UTILITIES @@ -98,11 +98,14 @@ TESTS += run_nczfilter.sh # Echo filter tests from nc_test4 check_PROGRAMS += testfilter testfilter_misc testfilter_order testfilter_repeat testfilter_multi test_filter_avail TESTS += run_filter.sh -TESTS += run_unknown.sh TESTS += run_specific_filters.sh +# This test is too dangerous to run in a parallel make environment. +# It causes race conditions. So suppress and only test by hand. +#TESTS += run_unknown.sh + if ISMINGW -XFAIL_TESTS = run_filter.sh run_unknown.sh run_specific_filters.sh +XFAIL_TESTS = run_filter.sh run_specific_filters.sh endif # ISMINGW endif #ENABLE_FILTER_TESTING @@ -110,22 +113,18 @@ endif #ENABLE_FILTER_TESTING endif #BUILD_UTILITIES # These programs are used by the test cases -noinst_PROGRAMS = zmapio -zmapio_SOURCES = zmapio.c -noinst_PROGRAMS += zhex +noinst_PROGRAMS = zhex zhex_SOURCES = zhex.c noinst_PROGRAMS += zisjson zisjson_SOURCES = zisjson.c +noinst_PROGRAMS += zmapio +zmapio_SOURCES = zmapio.c noinst_PROGRAMS += zs3parse zs3parse_SOURCES = zs3parse.c if ENABLE_S3 -noinst_PROGRAMS += s3util +noinst_PROGRAMS += s3util s3util_SOURCES = s3util.c -TESTS += run_s3_cleanup.sh -if ISCYGWIN -XFAIL_TESTS = run_s3_cleanup.sh -endif # ISCYGWIN endif # Given a netcdf4|NCZarr file, dump the actual chunk contents. @@ -141,7 +140,7 @@ run_purezarr.sh run_interop.sh run_misc.sh \ run_filter.sh \ run_newformat.sh run_nczarr_fill.sh run_quantize.sh \ run_jsonconvention.sh run_nczfilter.sh run_unknown.sh \ -run_scalar.sh run_strings.sh run_nulls.sh run_notzarr.sh +run_scalar.sh run_strings.sh run_nulls.sh run_notzarr.sh run_external.sh EXTRA_DIST += \ ref_ut_map_create.cdl ref_ut_map_writedata.cdl ref_ut_map_writemeta2.cdl ref_ut_map_writemeta.cdl \ @@ -163,13 +162,13 @@ ref_groups.h5 ref_byte.zarr.zip ref_byte_fill_value_null.zarr.zip \ ref_groups_regular.cdl ref_byte.cdl ref_byte_fill_value_null.cdl \ ref_jsonconvention.cdl ref_jsonconvention.zmap \ ref_string.cdl ref_string_nczarr.baseline ref_string_zarr.baseline ref_scalar.cdl \ -ref_nulls_nczarr.baseline ref_nulls_zarr.baseline ref_nulls.cdl +ref_nulls_nczarr.baseline ref_nulls_zarr.baseline ref_nulls.cdl ref_notzarr.tar.gz # Interoperability files EXTRA_DIST += ref_power_901_constants_orig.zip ref_power_901_constants.cdl ref_quotes_orig.zip ref_quotes.cdl \ ref_zarr_test_data.cdl.gz -CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.nc tmp*.dump tmp*.tmp tmp_ngc.c ref_zarr_test_data.cdl tst_*.nc.zip ref_quotes.zip ref_power_901_constants.zip +CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.nc tmp*.dump tmp*.tmp tmp*.zmap tmp_ngc.c ref_zarr_test_data.cdl tst_*.nc.zip ref_quotes.zip ref_power_901_constants.zip BUILT_SOURCES = test_quantize.c test_filter_avail.c run_specific_filters.sh run_filterinstall.sh run_unknown.sh test_quantize.c: $(top_srcdir)/nc_test4/tst_quantize.c @@ -211,11 +210,16 @@ run_filterinstall.sh: $(top_srcdir)/nc_test4/tst_filterinstall.sh # Remove directories clean-local: + rm -fr testdir_* rm -fr tmp_*.nc tmp_*.zarr tst_quantize*.zarr tmp*.file results.file results.s3 results.zip rm -fr rcmiscdir ref_power_901_constants.file DISTCLEANFILES = findplugin.sh test_quantize.c run_specific_filters.sh run_filterinstall.sh run_unknown.sh test_filter_avail.c +# Provide a specific s3 cleanup action +s3cleanup:: + ${srcdir}/run_s3_cleanup.sh + # If valgrind is present, add valgrind targets. @VALGRIND_CHECK_RULES@ diff --git a/nczarr_test/ref_jsonconvention.zmap b/nczarr_test/ref_jsonconvention.zmap index eb12c150f2..4687f6c674 100644 --- a/nczarr_test/ref_jsonconvention.zmap +++ b/nczarr_test/ref_jsonconvention.zmap @@ -1,5 +1,5 @@ -[0] /.zattrs : (354) |{"globalfloat": 1, "globalfloatvec": [1,2], "globalchar": "abc", "globalillegal": "[ [ 1.0, 0.0, 0.0 ], [ 0.0, 1.0, 0.0 ], [ 0.0, 0.0, 1.0 ", "_nczarr_attr": {"types": {"globalfloat": "S1", "globalillegal": ">S1", "_NCProperties": ">S1"}}}| -[1] /.zgroup : (129) |{"zarr_format": 2, "_nczarr_superblock": {"version": "2.0.0"}, "_nczarr_group": {"dims": {"d1": 1}, "vars": ["v"], "groups": []}}| -[3] /v/.zarray : (202) |{"zarr_format": 2, "shape": [1], "dtype": "S1", "varjson2": ">S1", "varvec1": ">S1", "varvec2": ">S1"}}}| +[0] /.zattrs : () |{"globalfloat": 1, "globalfloatvec": [1,2], "globalchar": "abc", "globalillegal": "[ [ 1.0, 0.0, 0.0 ], [ 0.0, 1.0, 0.0 ], [ 0.0, 0.0, 1.0 ", "_nczarr_attr": {"types": {"globalfloat": "S1", "globalillegal": ">S1", "_NCProperties": ">S1"}}}| +[1] /.zgroup : () |{"zarr_format": 2, "_nczarr_superblock": {"version": "2.0.0"}, "_nczarr_group": {"dims": {"d1": 1}, "vars": ["v"], "groups": []}}| +[3] /v/.zarray : () |{"zarr_format": 2, "shape": [1], "dtype": "S1", "varjson2": ">S1", "varvec1": ">S1", "varvec2": ">S1"}}}| [5] /v/0 : (4) (ubyte) |...| diff --git a/nczarr_test/ref_notzarr.tar.gz b/nczarr_test/ref_notzarr.tar.gz new file mode 100644 index 0000000000..efc55444e5 Binary files /dev/null and b/nczarr_test/ref_notzarr.tar.gz differ diff --git a/nczarr_test/run_chunkcases.sh b/nczarr_test/run_chunkcases.sh index 6323e0219d..02adbef9bd 100755 --- a/nczarr_test/run_chunkcases.sh +++ b/nczarr_test/run_chunkcases.sh @@ -17,6 +17,10 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi set -e +s3isolate "testdir_chunkcases" +THISDIR=`pwd` +cd $ISOPATH + TC="${execdir}/tst_chunkcases -4" ZM="${execdir}/zmapio -t int" @@ -52,25 +56,35 @@ makefile() { esac } -testcases() { - +testcasesxfail() { zext=$1 -echo ""; echo "*** Test format $1" - +echo ""; echo "*** XFAIL Test format $1" # Test whole chunk write and read echo "Test whole chunk write then read" -makefile tmp_whole -rm -f tmp_whole_${zext}.txt tmp_whole_${zext}.cdl tmp_err_${zext}.txt +makefile tmp_xwhole +rm -f tmp_xwhole_${zext}.txt tmp_xwhole_${zext}.cdl tmp_xerr_${zext}.txt # This should fail -if ! $TC -d 8,8 -c 4,4 -f 4,3 -e 4,4 -X w -OWw $F >> tmp_err_${zext}.txt ; then +if ! $TC -d 8,8 -c 4,4 -f 4,3 -e 4,4 -X w -OWw $F >> tmp_xerr_${zext}.txt ; then echo "XFAIL: wholechunk with bad -f" +else +echo "Unexpected PASS: wholechunk with bad -f" +exit 1 fi remfile $file -if ! $TC -d 8,8 -c 4,4 -f 4,4 -e 1,4 -X w -OWw $F >> tmp_err_${zext}.txt ; then - +if ! $TC -d 8,8 -c 4,4 -f 4,4 -e 1,4 -X w -OWw $F >> tmp_xerr_${zext}.txt ; then echo "XFAIL: wholechunk with bad -e" +else +echo "Unexpected PASS: wholechunk with bad -e" +exit 1 fi -remfile $file +} # testcasesxfail() + +testcasespass() { +zext=$1 +echo ""; echo "*** Test format $1" +makefile tmp_whole +rm -f tmp_whole_${zext}.txt tmp_whole_${zext}.cdl tmp_err_${zext}.txt +makefile tmp_whole # This should succeed $TC -d 8,8 -c 4,4 -f 4,4 -e 4,4 -X w -OWw $F $TC -d 8,8 -c 4,4 -f 4,4 -e 4,4 -X w -OWr $F > tmp_whole_${zext}.txt @@ -91,14 +105,12 @@ diff -b ${srcdir}/ref_skip.cdl tmp_skip_${zext}.cdl echo "Test chunk skipping during write" makefile tmp_skipw rm -f tmp_skipw_${zext}.cdl - $TC -d 6,6 -s 5,5 -p 6,6 -Ow $F ${NCDUMP} $F > tmp_skipw_${zext}.cdl diff -b ${srcdir}/ref_skipw.cdl tmp_skipw_${zext}.cdl echo "Test dimlen % chunklen != 0" makefile tmp_rem - rm -f tmp_rem_${zext}.txt tmp_rem_${zext}.cdl $TC -d 8,8 -c 3,3 -Ow $F ${NCDUMP} $F > tmp_rem_${zext}.cdl @@ -119,14 +131,22 @@ echo "Test miscellaneous 1" makefile tmp_misc1 rm -f tmp_misc1_${zext}.txt tmp_misc1_${zext}.cdl $TC -d 6,12,4 -c 2,3,1 -f 0,0,0 -e 6,1,4 -Ow $F +if test "x$FEATURE_S3TESTS" = xyes ; then +${S3UTIL} -u 'https://s3.us-east-1.amazonaws.com/unidata-zarr-test-data' -k '/netcdf-c' list +fi ${NCDUMP} $F > tmp_misc1_${zext}.cdl diff -b ${srcdir}/ref_misc1.cdl tmp_misc1_${zext}.cdl ${execdir}/ncdumpchunks -v v $F > tmp_misc1_${zext}.dmp diff -b ${srcdir}/ref_misc1.dmp tmp_misc1_${zext}.dmp +} # testcasespass() -} # testcases() +testcases() { + testcasesxfail $1 + testcasespass $1 +} testcases file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcases zip; fi if test "x$FEATURE_S3TESTS" = xyes ; then testcases s3; fi +if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_external.sh b/nczarr_test/run_external.sh new file mode 100755 index 0000000000..c243bbbde2 --- /dev/null +++ b/nczarr_test/run_external.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +set -e + +. "$srcdir/test_nczarr.sh" + +s3isolate "testdir_external" +THISDIR=`pwd` +cd $ISOPATH + +# This shell script tests reading of +# publically accessible S3 zarr datasets. + +TESTCASES= +if test "x$FEATURE_BYTERANGE" = xyes && test "x$FEATURE_S3" = xyes && test "x$FP_ISCYGWIN" = x ; then +TESTCASES="${TESTCASES} OR_ABI;http://s3.amazonaws.com/noaa-goes16/ABI-L1b-RadF/2022/001/18/OR_ABI-L1b-RadF-M6C01_G16_s20220011800205_e20220011809513_c20220011809562.nc#mode=bytes,s3" +fi + +testcase() { +NM=`echo "$1" | cut -d';' -f1` +URL=`echo "$1" | cut -d';' -f2` +echo "*** Test: $NM = $URL" +rm -f "tmp_external_$NM.cdl" +${NCDUMP} -h -n $NM $URL > "tmp_external_${NM}.cdl" +} + +if test "x$FEATURE_S3" = xyes ; then +for t in $TESTCASES ; do +testcase "$t" +done +fi + +if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_fillonlyz.sh b/nczarr_test/run_fillonlyz.sh index 1c00e003cf..27977c1418 100755 --- a/nczarr_test/run_fillonlyz.sh +++ b/nczarr_test/run_fillonlyz.sh @@ -10,6 +10,10 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi set -e +s3isolate "testdir_fillonlyz" +THISDIR=`pwd` +cd $ISOPATH + echo "" echo "*** Testing data conversions when a variable has fill value but never written" @@ -25,4 +29,4 @@ testcase file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi -exit 0 +if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_filter.sh b/nczarr_test/run_filter.sh index 0bf6437d15..9ed36c1f20 100755 --- a/nczarr_test/run_filter.sh +++ b/nczarr_test/run_filter.sh @@ -7,6 +7,10 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi set -e +s3isolate "testdir_filter" +THISDIR=`pwd` +cd $ISOPATH + testset() { # Which test cases to exercise testapi $1 @@ -177,4 +181,4 @@ testset file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testset zip ; fi if test "x$FEATURE_S3TESTS" = xyes ; then testset s3 ; fi -exit 0 +if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_interop.sh b/nczarr_test/run_interop.sh index 22baf1ebf4..534bcd5874 100755 --- a/nczarr_test/run_interop.sh +++ b/nczarr_test/run_interop.sh @@ -5,24 +5,26 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . "$srcdir/test_nczarr.sh" +set -e + +s3isolate "testdir_interop" +THISDIR=`pwd` +cd $ISOPATH + # This shell script tests compatibility between # this implementation and other implementations # by means of files constructed by that other implementation -set -e - UH="${NCZARR_S3_TEST_HOST}" UB="${NCZARR_S3_TEST_BUCKET}" -RESULTSDIR="${builddir}/tmp_interop" - testcasefile() { zext=file base=$1 mode=$2 metaonly=$3 if test "x$metaonly" = xmetaonly ; then flags="-h"; fi - fileargs ${RESULTSDIR}/ref_$base "mode=$mode,$zext" + fileargs ${ISOPATH}/ref_$base "mode=$mode,$zext" rm -f tmp_${base}_${zext}.cdl ${NCDUMP} $flags $fileurl > tmp_${base}_${zext}.cdl diff -b ${srcdir}/ref_${base}.cdl tmp_${base}_${zext}.cdl @@ -32,7 +34,7 @@ testcasezip() { zext=zip base=$1 mode=$2 - fileargs ${RESULTSDIR}/ref_$base "mode=$mode,$zext" + fileargs ${ISOPATH}/ref_$base "mode=$mode,$zext" rm -f tmp_${base}_${zext}.cdl ${NCDUMP} -h $flags $fileurl > tmp_${base}_${zext}.cdl diff -b ${srcdir}/ref_${base}.cdl tmp_${base}_${zext}.cdl @@ -46,7 +48,7 @@ testcases3() { url="https://${UH}/${UB}/${base}.zarr#mode=${mode},s3" ${NCDUMP} $url > tmp_${base}_${zext}.cdl # Find the proper ref file - diff -b ${RESULTSDIR}/ref_${base}.cdl tmp_${base}_${zext}.cdl + diff -b ${ISOPATH}/ref_${base}.cdl tmp_${base}_${zext}.cdl } testallcases() { @@ -54,14 +56,12 @@ zext=$1 case "$zext" in file) # need to unpack - unzip ${srcdir}/ref_power_901_constants_orig.zip > /dev/null - mv ${RESULTSDIR}/ref_power_901_constants ${RESULTSDIR}/ref_power_901_constants.file + unzip ref_power_901_constants.zip >> tmp_ignore.txt + mv ${ISOPATH}/ref_power_901_constants ${ISOPATH}/ref_power_901_constants.file testcasefile power_901_constants zarr metaonly; # test xarray as default ;; zip) # Move into position - cp -f ${srcdir}/ref_power_901_constants_orig.zip ${RESULTSDIR}/ref_power_901_constants.zip - cp -f ${srcdir}/ref_quotes_orig.zip ${RESULTSDIR}/ref_quotes.zip testcasezip power_901_constants xarray metaonly # Test large constant interoperability testcasezip quotes zarr metaonly @@ -70,23 +70,23 @@ case "$zext" in # Read a test case created by netcdf-java zarr. # unpack # Use gunzip because it always appears to be available - gunzip -c ${srcdir}/ref_zarr_test_data.cdl.gz > ${RESULTSDIR}/ref_zarr_test_data.cdl + gunzip -c ${srcdir}/ref_zarr_test_data.cdl.gz > ${ISOPATH}/ref_zarr_test_data.cdl testcases3 zarr_test_data xarray ;; *) echo "unimplemented kind: $1" ; exit 1;; esac } -THISDIR=`pwd` -rm -fr ${RESULTSDIR} -mkdir -p ${RESULTSDIR} -cd ${RESULTSDIR} +# common setup +if ! test -f ${ISOPATH}/ref_power_901_constants.zip ; then + cp -f ${srcdir}/ref_power_901_constants_orig.zip ${ISOPATH}/ref_power_901_constants.zip +fi +if ! test -f ${ISOPATH}/ref_quotes.zip ; then + cp -f ${srcdir}/ref_quotes_orig.zip ${ISOPATH}/ref_quotes.zip +fi + testallcases file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testallcases zip; fi if test "x$FEATURE_S3TESTS" = xyes ; then testallcases s3; fi -cd ${THISDIR} - -# Cleanup -rm -fr ${RESULTSDIR} -exit 0 +if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_it_chunks1.sh b/nczarr_test/run_it_chunks1.sh index 56277329d4..c3efe9a61b 100755 --- a/nczarr_test/run_it_chunks1.sh +++ b/nczarr_test/run_it_chunks1.sh @@ -21,7 +21,7 @@ ${execdir}/tst_chunks2 -e $1 $CLOUD ittest file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then ittest zip; fi -if test "x$FEATURE_S3TESTS" = xyes ; then ittest s3 "${NCZARR_S3_TEST_URL}/netcdf-c'; fi +if test "x$FEATURE_S3TESTS" = xyes ; then ittest s3 "${NCZARR_S3_TEST_URL}/${S3TESTPATH}'; fi } diff --git a/nczarr_test/run_jsonconvention.sh b/nczarr_test/run_jsonconvention.sh index 0c9c72bd81..ad35ea8910 100755 --- a/nczarr_test/run_jsonconvention.sh +++ b/nczarr_test/run_jsonconvention.sh @@ -5,11 +5,15 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . "$srcdir/test_nczarr.sh" +set -e + +s3isolate "testdir_jsonconvention" +THISDIR=`pwd` +cd $ISOPATH + # This shell script tests support for: # read/write using json convention -set -e - testcase() { zext=$1 @@ -22,16 +26,17 @@ ${ZMD} -h $fileurl > tmp_jsonconvention_${zext}.txt # | sed -e 's/,key1=value1|key2=value2//' -e '/"_NCProperties"/ s/(378)/(354)/' # Clean up extraneous changes so comparisons work # remove '\n' from ref file before comparing -sed -e 's|\\n||g' < ${srcdir}/ref_jsonconvention.cdl > tmp_jsonconvention_clean.cdl -sed -e 's|\\n||g' < ${srcdir}/ref_jsonconvention.zmap > tmp_jsonconvention_clean.zmap +#sed -e 's|\\n||g' < ${srcdir}/ref_jsonconvention.cdl > tmp_jsonconvention_clean.cdl +cat < ${srcdir}/ref_jsonconvention.cdl > tmp_jsonconvention_clean.cdl cat < tmp_jsonconvention_${zext}.cdl > tmp_jsonconvention_clean_${zext}.cdl -sed -e 's|"_NCProperties": "version=2,netcdf=[^,]*,nczarr=2.0.0",||' < tmp_jsonconvention_${zext}.txt > tmp_jsonconvention_clean_${zext}.txt +sed -e 's|\(.z[a-z][a-z]*\) : ([0-9][0-9]*)|\1 : ()|g' < tmp_jsonconvention_${zext}.txt >tmp1.tmp +sed -e 's|"_NCProperties": "version=[0-9],[^"]*",||' tmp_jsonconvention_clean_${zext}.txt diff -b tmp_jsonconvention_clean.cdl tmp_jsonconvention_clean_${zext}.cdl -diff -b tmp_jsonconvention_clean.zmap tmp_jsonconvention_clean_${zext}.txt +diff -b ${srcdir}/ref_jsonconvention.zmap tmp_jsonconvention_clean_${zext}.txt } testcase file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi -exit 0 +if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_misc.sh b/nczarr_test/run_misc.sh index 1ffb122bfb..2d58cf72b9 100755 --- a/nczarr_test/run_misc.sh +++ b/nczarr_test/run_misc.sh @@ -5,17 +5,13 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . "$srcdir/test_nczarr.sh" -# This test uses a shared resource: the .rc files; so run in a special directory -# Create a special directory -# And enter it to execute tests -rm -fr rcmiscdir -mkdir rcmiscdir -cd rcmiscdir -WD=`pwd` +set -e -# This shell script provides a miscellaneous set of tests +s3isolate "testdir_misc" +THISDIR=`pwd` +cd $ISOPATH -set -e +# This shell script provides a miscellaneous set of tests cleanup() { resetrc @@ -24,7 +20,7 @@ cleanup() { # Setup the .rc files createrc() { - RCP="${WD}/.ncrc" + RCP="${ISOPATH}/.ncrc" echo "Creating rc file $RCP" echo "ZARR.DIMENSION_SEPARATOR=/" >>$RCP } @@ -65,4 +61,4 @@ if test "x$FEATURE_S3TESTS" = xyes ; then testcase2 s3 fi -exit 0 +if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_nccopyz.sh b/nczarr_test/run_nccopyz.sh index 76d1d10dc5..ae426a9d93 100755 --- a/nczarr_test/run_nccopyz.sh +++ b/nczarr_test/run_nccopyz.sh @@ -6,7 +6,10 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . "$srcdir/test_nczarr.sh" set -e -echo "" + +s3isolate "testdir_nccopyz" +THISDIR=`pwd` +cd $ISOPATH #chunkclean src dst chunkclean() { @@ -79,4 +82,5 @@ if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi echo "*** All nccopy nczarr tests passed!" -exit 0 + +if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_ncgen4.sh b/nczarr_test/run_ncgen4.sh index cbc9a9039d..bd56d288bb 100755 --- a/nczarr_test/run_ncgen4.sh +++ b/nczarr_test/run_ncgen4.sh @@ -10,8 +10,12 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . "$srcdir/test_nczarr.sh" -set -e +# Isolate both test and S3 +s3isolate "testdir_ncgen4" +THISDIR=`pwd` +cd $ISOPATH +set -e # To add a new test, # 1. put the .cdl file in the 'ncdump/cdl' directory @@ -36,7 +40,6 @@ ALLTESTS="$TESTS $FVTESTS" # Location constants cdl="$srcdir/../ncdump/cdl" expected="$srcdir/../ncdump/expected" -RESULTSDIR="./results" # Functions @@ -83,12 +86,7 @@ done runtestset() { extfor $1 echo "*** Testing nczarr X ncgen with zmap=${zext}" -rm -fr ${RESULTSDIR}.$zext -mkdir ${RESULTSDIR}.${zext} -WD=`pwd ` -cd ${RESULTSDIR}.${zext} difftest -cd $WD echo "*** PASSED: zext=${zext}" } @@ -97,3 +95,5 @@ if test "x$FEATURE_NCZARR_ZIP" = xyes ; then runtestset zip; fi if test "x$FEATURE_S3TESTS" = xyes ; then runtestset s3; fi echo "*** PASSED ***" + +if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_nczarr_fill.sh b/nczarr_test/run_nczarr_fill.sh index 4995f8dc80..c095fc0804 100755 --- a/nczarr_test/run_nczarr_fill.sh +++ b/nczarr_test/run_nczarr_fill.sh @@ -5,6 +5,10 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . "$srcdir/test_nczarr.sh" +s3isolate "testdir_nczarr_fill" +THISDIR=`pwd` +cd $ISOPATH + set -e echo "*** Test: Github issues #2063, #2062, #2059" @@ -26,7 +30,7 @@ testcase2062() { zext=$1 echo "*** Test: Github issue #2062" rm -fr ref_byte.zarr -unzip ${srcdir}/ref_byte.zarr.zip +unzip ref_byte.zarr.zip >> tmp_ignore.txt rm -fr tmp_nczfill.cdl ${ZMD} -h "file://ref_byte.zarr#mode=zarr,$zext" ${NCDUMP} -s "file://ref_byte.zarr#mode=zarr,$zext" > tmp_nczfill.cdl @@ -39,7 +43,7 @@ testcase2063() { zext=$1 echo "*** Test: Github issue #2063" rm -fr ref_byte_fill_value_null.zarr -unzip ${srcdir}/ref_byte_fill_value_null.zarr.zip +unzip ref_byte_fill_value_null.zarr.zip >> tmp_ignore.txt rm -fr tmp_nczfill.cdl ${ZMD} -h "file://ref_byte_fill_value_null.zarr#mode=zarr,$zext" ${NCDUMP} -s "file://ref_byte_fill_value_null.zarr#mode=zarr,$zext" > tmp_nczfill.cdl @@ -48,6 +52,10 @@ diff -wb ${srcdir}/ref_byte_fill_value_null.cdl tmp_byte_fill_value_null_$zext.c rm -fr ref_byte_fill_value_null.zarr } +if ! test -f ${ISOPATH}/ref_byte.zarr.zip ; then + cp -f ${srcdir}/ref_byte.zarr.zip ${ISOPATH}/ref_byte.zarr.zip + cp -f ${srcdir}/ref_byte_fill_value_null.zarr.zip ${ISOPATH}/ref_byte_fill_value_null.zarr.zip +fi testcase2062 file testcase2063 file @@ -61,4 +69,4 @@ if test "x$FEATURE_HDF5" = xyes ; then fi fi -exit 0 +if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_nczfilter.sh b/nczarr_test/run_nczfilter.sh index f6575fc18f..2d87cbc120 100755 --- a/nczarr_test/run_nczfilter.sh +++ b/nczarr_test/run_nczfilter.sh @@ -11,4 +11,9 @@ set -e pwd +s3isolate "testdir_nczfilter" +cd $ISOPATH + ${execdir}/tst_nczfilter + +if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_newformat.sh b/nczarr_test/run_newformat.sh index e4f945ab17..40eb0d8ef6 100755 --- a/nczarr_test/run_newformat.sh +++ b/nczarr_test/run_newformat.sh @@ -7,6 +7,10 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi set -e +s3isolate "testdir_newformat" +THISDIR=`pwd` +cd $ISOPATH + echo "" echo "*** Testing backward compatibilty between nczarr meta data format V1 vs V2" @@ -43,4 +47,4 @@ if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcasepure zip fi -exit 0 +if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_notzarr.sh b/nczarr_test/run_notzarr.sh index fc1b423a7b..f81e81dba8 100755 --- a/nczarr_test/run_notzarr.sh +++ b/nczarr_test/run_notzarr.sh @@ -5,13 +5,16 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . "$srcdir/test_nczarr.sh" +# Build both ISOPATH and S3ISOPATH +s3isolate "testdir_notzarr" +THISDIR=`pwd` +cd $ISOPATH + # Test ability to detect NCZarr/Zarr files URL="${NCZARR_S3_TEST_HOST}/${NCZARR_S3_TEST_BUCKET}" -KEY="/netcdf-c" +KEY="/${S3ISOPATH}" -THISDIR=`pwd` -RESULTSDIR=tmp_notzarr sometestfailed= testfailed() { @@ -21,25 +24,12 @@ testfailed() { fi } -rm -fr ${RESULTSDIR} -mkdir -p ${RESULTSDIR} -cd ${RESULTSDIR} - # Make test sets -mkdir empty.file # empty -mkdir notzarr.file # non-empty, non-zarr -echo "random data" >notzarr.file/notzarr.txt -if test "x$FEATURE_NCZARR_ZIP" = xyes ; then - mkdir empty - zip -r empty.zip empty - cp -r notzarr.file ./notzarr - zip -r notzarr.zip notzarr - rm -fr empty notzarr -fi +cp ${srcdir}/ref_notzarr.tar.gz . +gunzip ref_notzarr.tar.gz +tar -xf ref_notzarr.tar if test "x$FEATURE_S3TESTS" = xyes ; then - cat /dev/null > empty.txt - # not possible: ${execdir}/s3util -f notzarr.txt -u "https://${URL}" -k "/netcdf-c/empty.s3" upload - ${execdir}/s3util -f notzarr.file/notzarr.txt -u "https://${URL}" -k "/netcdf-c/notzarr.s3/notzarr.txt" upload + ${execdir}/s3util -f notzarr.file/notzarr.txt -u "https://${URL}" -k "/${S3ISOPATH}/notzarr.s3/notzarr.txt" upload fi echo "Test empty file" @@ -62,21 +52,13 @@ if test "x$FEATURE_S3TESTS" = xyes ; then if test 1 = 0 ; then # This test is NA for S3 echo "Test empty S3 file" - KEY="/netcdf-c/empty.s3" - RET=`${execdir}/tst_notzarr "https://$URL${KEY}#mode=zarr,s3"` + KEY2="${KEY}/empty.s3" + RET=`${execdir}/tst_notzarr "https://$URL${KEY2}#mode=zarr,s3"` testfailed "$RET" fi echo "Test non-zarr S3 file" -RET=`${execdir}/tst_notzarr "https://$URL/netcdf-c/notzarr.s3#mode=zarr,s3"` +RET=`${execdir}/tst_notzarr "https://$URL/${S3ISOPATH}/notzarr.s3#mode=zarr,s3"` testfailed "$RET" fi -cd ${THISDIR} - -# Cleanup -rm -fr ${RESULTSDIR} -if test "x$FEATURE_S3TESTS" = xyes ; then - awsdelete "/netcdf-c" -fi - -exit 0 +if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_nulls.sh b/nczarr_test/run_nulls.sh index 2dc7484bf2..796bf967f3 100755 --- a/nczarr_test/run_nulls.sh +++ b/nczarr_test/run_nulls.sh @@ -5,10 +5,14 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . "$srcdir/test_nczarr.sh" -# This shell script tests support for special cases of NC_CHAR - set -e +s3isolate "testdir_nulls" +THISDIR=`pwd` +cd $ISOPATH + +# This shell script tests support for special cases of NC_CHAR + testcase() { zext=$1 @@ -50,4 +54,4 @@ testcase file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi -exit 0 +if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_perf_chunks1.sh b/nczarr_test/run_perf_chunks1.sh index 778990732e..86775e19fb 100755 --- a/nczarr_test/run_perf_chunks1.sh +++ b/nczarr_test/run_perf_chunks1.sh @@ -11,6 +11,10 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh . "$srcdir/test_nczarr.sh" +s3isolate "testdir_perf_chunks1" +THISDIR=`pwd` +cd $ISOPATH + test1() { FMT=$1 DIMS=$2 @@ -35,4 +39,4 @@ testcases file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcases zip; fi if test "x$FEATURE_S3TESTS" = xyes ; then testcases s3; fi -exit 0 +if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_purezarr.sh b/nczarr_test/run_purezarr.sh index b068994c19..45cfc4f4da 100755 --- a/nczarr_test/run_purezarr.sh +++ b/nczarr_test/run_purezarr.sh @@ -5,12 +5,16 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . "$srcdir/test_nczarr.sh" +set -e + +s3isolate "testdir_purezarr" +THISDIR=`pwd` +cd $ISOPATH + # This shell script tests support for: # 1. pure zarr (noxarray) read/write # 2. xarray read/write -set -e - testcase() { zext=$1 @@ -41,4 +45,3 @@ testcase file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi -exit 0 diff --git a/nczarr_test/run_quantize.sh b/nczarr_test/run_quantize.sh index 8663eeb565..32af46df7f 100755 --- a/nczarr_test/run_quantize.sh +++ b/nczarr_test/run_quantize.sh @@ -5,6 +5,12 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . "$srcdir/test_nczarr.sh" +# Construct both ISOPATH and S3ISOPATH +s3isolate "testdir_quantize" + +THISDIR=`pwd` +cd $ISOPATH + # This shell script runs test_quantize set -e @@ -15,7 +21,7 @@ testcase() { case "$zext" in file) template="file://${execdir}/%s.zarr#mode=zarr,$zext" ;; zip) template="file://${execdir}/%s.zip#mode=zarr,$zext" ;; - s3) template="s3://${NCZARR_S3_TEST_BUCKET}/netcdf-c/%s.zarr#mode=zarr,$zext" ;; + s3) template="s3://${NCZARR_S3_TEST_BUCKET}/${S3TESTPATH}/%s.zarr#mode=zarr,$zext" ;; *) echo "unknown file type"; exit 1 ;; esac ${execdir}/test_quantize "$template" @@ -25,3 +31,4 @@ testcase file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi # There is a (currently) untraceable bug when using S3 #if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi +# if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_s3_cleanup.sh b/nczarr_test/run_s3_cleanup.sh index 6df8c9c476..542af09d5c 100755 --- a/nczarr_test/run_s3_cleanup.sh +++ b/nczarr_test/run_s3_cleanup.sh @@ -13,7 +13,7 @@ echo "*** Remove /netcdf-c from S3 repository" fileargs netcdf-c if test "x$FEATURE_S3TESTS" = xyes ; then - ${execdir}/s3util -u "${NCZARR_S3_TEST_URL}" -k "/netcdf-c" clear +${execdir}/s3util -u "${NCZARR_S3_TEST_URL}" -k "/netcdf-c" clear fi exit 0 diff --git a/nczarr_test/run_scalar.sh b/nczarr_test/run_scalar.sh index 694c513062..b1aff8e768 100755 --- a/nczarr_test/run_scalar.sh +++ b/nczarr_test/run_scalar.sh @@ -5,10 +5,14 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . "$top_srcdir/nczarr_test/test_nczarr.sh" -# This shell script tests support for the NC_STRING type - set -e +s3isolate "testdir_nczarr" +THISDIR=`pwd` +cd $ISOPATH + +# This shell script tests support for the NC_STRING type + zarrscalar() { rm -f $2 sed -e '/dimensions:/d' -e '/_scalar_ =/d' -e '/int v/ s|(_scalar_)||' <$1 >$2 @@ -57,4 +61,4 @@ testcase file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi -exit 0 +if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_strings.sh b/nczarr_test/run_strings.sh index ddca482413..69cd93b038 100755 --- a/nczarr_test/run_strings.sh +++ b/nczarr_test/run_strings.sh @@ -9,6 +9,10 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi set -e +s3isolate "testdir_strings" +THISDIR=`pwd` +cd $ISOPATH + testcase() { zext=$1 @@ -34,10 +38,10 @@ ${NCGEN} -4 -b -o "$nczarrurl" $srcdir/ref_string.cdl echo "*** read purezarr" ${NCDUMP} -n ref_string $zarrurl > tmp_string_zarr_${zext}.cdl -${ZMD} -h $zarrurl > tmp_string_zarr_${zext}.txt +${ZMD} -t 'string/6' $zarrurl > tmp_string_zarr_${zext}.txt echo "*** read nczarr" ${NCDUMP} -n ref_string $nczarrurl > tmp_string_nczarr_${zext}.cdl -${ZMD} -h $nczarrurl > tmp_string_nczarr_${zext}.txt +${ZMD} -t 'string/6' $nczarrurl > tmp_string_nczarr_${zext}.txt echo "*** verify zarr output" diff -bw ${srcdir}/ref_string_zarr.baseline tmp_string_zarr_${zext}.cdl @@ -50,4 +54,4 @@ testcase file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi -exit 0 +if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_ut_mapapi.sh b/nczarr_test/run_ut_mapapi.sh index a46935d8fd..1cf0459cc1 100755 --- a/nczarr_test/run_ut_mapapi.sh +++ b/nczarr_test/run_ut_mapapi.sh @@ -9,6 +9,10 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi set -e +s3isolate "testdir_mapapi" +THISDIR=`pwd` +cd $ISOPATH + # Test map implementations for consistency at the zmap API # level. This allows testing of implementations that do not admit # of easy examination of the actual storage. For example it is @@ -94,6 +98,7 @@ if test "x$FEATURE_S3TESTS" = xyes ; then echo ""; echo "*** Test zmap_s3sdk" export PROFILE="-p default" testmapcreate s3; testmapmeta s3; testmapdata s3; testmapsearch s3 + s3sdkdelete "/${S3ISOPATH}" # Cleanup fi } diff --git a/nczarr_test/s3util.c b/nczarr_test/s3util.c index 0852a45a0c..42562ae9f0 100644 --- a/nczarr_test/s3util.c +++ b/nczarr_test/s3util.c @@ -55,6 +55,7 @@ static struct S3ops { /* Command line options */ struct Dumpptions { int debug; + int verbose; S3op s3op; NCURI* url; char* key; /* via -k flag */ @@ -162,14 +163,14 @@ main(int argc, char** argv) ncuriparse(p,&dumpoptions.url); nullfree(p); if(dumpoptions.url == NULL) { - fprintf(stderr,"malformed -f option: %s",optarg); + fprintf(stderr,"malformed -u option: %s",optarg); stat = NC_EINVAL; goto done; } } break; case 'v': - usage(); - goto done; + dumpoptions.verbose = 1; + break; case 'T': nctracelevel(atoi(optarg)); break; @@ -185,7 +186,10 @@ main(int argc, char** argv) argv += optind; if (argc > 1) { - fprintf(stderr, "s3util: only one command argument permitted\n"); + int j; + fprintf(stderr, "s3util: only one command argument permitted:"); + for(j=0;j 0) { + if(nkeys > 0 && keys != NULL) { size_t i; /* Sort the list -- shortest first */ nczm_sortenvv(nkeys,keys); - printf("deleted keys:\n"); - for(i=0;i /dev/null ; then echo "delete failed: $1" mapstillexists=1 fi @@ -82,7 +105,7 @@ fileargs() { if test "x$frag" = x ; then frag="mode=nczarr,$zext" ; fi case "$zext" in s3) - S3PATH="${NCZARR_S3_TEST_URL}/netcdf-c" + S3PATH="${NCZARR_S3_TEST_URL}/${S3ISOPATH}" fileurl="${S3PATH}/${f}#${frag}" file=$fileurl S3HOST=`${execdir}/zs3parse -h $S3PATH` @@ -125,8 +148,6 @@ if test "x$NCAUTH_HOMETEST" != x ; then RCHOME=1; fi # Set plugin path -#cd ../plugins; make clean all >/dev/null; cd ../nczarr_test - if test "x$FP_USEPLUGINS" = xyes; then # Load the findplugins function . ${builddir}/findplugin.sh @@ -151,4 +172,19 @@ resetrc() { unset DAPRCFILE } +# Enforce cleanup +atexit() { + atexit_cleanup() { + if test "x$S3ISOPATH" != x ; then + if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup + fi + } + trap atexit_cleanup EXIT +} + +GDBB="gdb -batch -ex r -ex bt -ex q --args" + resetrc +atexit + +fi #TEST_NCZARR_SH diff --git a/nczarr_test/ut_mapapi.c b/nczarr_test/ut_mapapi.c index 16be6527f4..ffe0907bf2 100644 --- a/nczarr_test/ut_mapapi.c +++ b/nczarr_test/ut_mapapi.c @@ -88,8 +88,8 @@ simplecreate(void) title(__func__); switch(stat = nczmap_create(impl,url,0,0,NULL,&map)) { - case NC_NOERR: break; /* already exists */ - case NC_EEMPTY: break; /*created*/ + case NC_EOBJECT: break; /* already exists */ + case NC_NOERR: break; /*created*/ default: goto done; } diff --git a/nczarr_test/ut_util.c b/nczarr_test/ut_util.c index f8018b9f6c..52478efc22 100644 --- a/nczarr_test/ut_util.c +++ b/nczarr_test/ut_util.c @@ -239,7 +239,7 @@ parseintvector(const char* s0, int typelen, void** vectorp) void freedimdefs(NClist* defs) { - int i; + size_t i; for(i=0;iname); @@ -250,7 +250,7 @@ freedimdefs(NClist* defs) void freevardefs(NClist* defs) { - int i; + size_t i; for(i=0;iname); @@ -348,7 +348,7 @@ ut_typeforname(const char* tname) static Dimdef* finddim(const char* name, NClist* defs) { - int i; + size_t i; for(i=0;iname,name) == 0) @@ -421,7 +421,7 @@ void printoptions(struct UTOptions* opts) { char** p; - int i; + size_t i; printf("Options:"); #if 0 printf(" debug=%d",opts->debug); @@ -455,7 +455,7 @@ printoptions(struct UTOptions* opts) } printf(" -s "); - for(i=0;inslices;i++) { + for(i=0;i<(size_t)opts->nslices;i++) { NCZSlice* sl = &opts->slices[i]; printf("%s",nczprint_slicex(*sl,1)); } @@ -477,7 +477,8 @@ hasdriveletter(const char* f) void ut_sortlist(NClist* l) { - int i, switched; + int switched; + size_t i; if(nclistlength(l) <= 1) return; do { diff --git a/nczarr_test/zmapio.c b/nczarr_test/zmapio.c index 3131793e38..679ab2af27 100644 --- a/nczarr_test/zmapio.c +++ b/nczarr_test/zmapio.c @@ -286,6 +286,7 @@ rootpathfor(const char* path) case NCZM_ZIP: rootpath = strdup("/"); /*constant*/ break; +#ifdef ENABLE_S3 case NCZM_S3: /* Split the path part */ if((stat = nczm_split(uri->path,segments))) goto done; @@ -295,6 +296,7 @@ rootpathfor(const char* path) /* Put it back together */ if((stat = nczm_join(segments,&rootpath))) goto done; break; +#endif default: stat = NC_EINVAL; goto done; @@ -334,6 +336,8 @@ objdump(void) size64_t len = 0; OBJKIND kind = 0; int hascontent = 0; + nullfree(content); content = NULL; + nullfree(obj); obj = NULL; obj = nclistremove(stack,0); /* zero pos is always top of stack */ kind = keykind(obj); /* Now print info for this obj key */ @@ -343,14 +347,12 @@ objdump(void) case NC_EACCESS: hascontent = 0; len = 0; stat = NC_NOERR; break; default: goto done; } - if(!hascontent) goto next; /* ignore it */ + if(!hascontent) continue; /* ignore it */ if(len > 0) { size_t padlen = (len+dumpoptions.nctype->typesize); content = calloc(1,padlen+1); if((stat=nczmap_read(map,obj,0,len,content))) goto done; content[len] = '\0'; - } else { - content = NULL; } if(hascontent) { if(len > 0) { @@ -382,9 +384,6 @@ objdump(void) } else { printf("[%d] %s\n",depth,obj); } - nullfree(content); content = NULL; -next: - nullfree(obj); obj = NULL; } done: nullfree(obj); @@ -458,12 +457,12 @@ printcontent(size64_t len, const char* content, OBJKIND kind) format = dumpoptions.nctype->format; if(dumpoptions.format[0] != '\0') format = dumpoptions.format; + strlen = dumpoptions.strlen; + count = len; - if(dumpoptions.strlen > 0) { - strlen = dumpoptions.strlen; - count = ((len+strlen)-1)/strlen; - } else - count = len; +#ifdef DEBUG + printf("debug: len=%d strlen=%d count=%d\n",(int)len,(int)strlen,(int)count); fflush(stdout); +#endif for(i=0;iyytext,c); + ncbytesnull(lex->yytext); } #ifndef DAP2ENCODE diff --git a/oc2/ochttp.c b/oc2/ochttp.c index 16da166245..8fd7279991 100644 --- a/oc2/ochttp.c +++ b/oc2/ochttp.c @@ -127,7 +127,7 @@ ocfetchurl(CURL* curl, const char* url, NCbytes* buf, long* filetime) /* Null terminate the buffer*/ len = ncbyteslength(buf); - ncbytesappend(buf, '\0'); + ncbytesnull(buf); ncbytessetlength(buf, len); /* don't count null in buffer size*/ #ifdef OCDEBUG nclog(NCLOGNOTE,"buffersize: %lu bytes",(off_t)ncbyteslength(buf)); diff --git a/oc2/ocinternal.c b/oc2/ocinternal.c index fbaf49fba8..12daca6ecc 100644 --- a/oc2/ocinternal.c +++ b/oc2/ocinternal.c @@ -433,6 +433,7 @@ ocextractddsinfile(OCstate* state, OCtree* tree, OCflags flags) count = fread(chunk,1,sizeof(chunk),tree->data.file); if(count <= 0) break; /* EOF;*/ ncbytesappendn(state->packet,chunk,count); + ncbytesnull(state->packet); bodfound = ocfindbod(state->packet,&bod,&ddslen); } while(!bodfound); if(!bodfound) {/* No BOD; pretend */ diff --git a/plugins/H5Znoop.c b/plugins/H5Znoop.c index d50fe14233..03f62a3c45 100644 --- a/plugins/H5Znoop.c +++ b/plugins/H5Znoop.c @@ -230,16 +230,21 @@ NCZ_noop_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) int i,stat = NC_NOERR; char json[8192]; char value[1024]; + size_t jlen, count; if(nparams != 0 && params == NULL) {stat = NC_EINVAL; goto done;} - snprintf(json,sizeof(json),"{\"id\": \"%s\"",NCZ_noop_codec.codecid); + + jlen = sizeof(json); + count = snprintf(json,sizeof(json),"{\"id\": \"%s\"",NCZ_noop_codec.codecid); for(i=0;i count); + strcat(json,value); } - strlcat(json,"}",sizeof(json)); + count += 1; assert(jlen > count); + strcat(json,"}"); if(codecp) { if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} } diff --git a/plugins/Makefile.am b/plugins/Makefile.am index 96b3444e58..f7d86fd078 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -21,7 +21,7 @@ AM_LDFLAGS += $(plugin_version_info) endif !ISCYGWIN endif !ISMINGW -# Create an alternate directory if not installing or for noinst installs. +# Create an alternate directory if not installing. ALTPLUGINDIR = ${abs_top_builddir}/plugins/plugindir RPATH = -rpath $(abs_builddir)/.libs @@ -80,11 +80,11 @@ endif # ENABLE_NCZARR_FILTERS if ENABLE_PLUGINS -# The NCZarr codec libraries +# The NCZarr codec libraries (they need libnetcdf) lib__nczstdfilters_la_SOURCES = NCZstdfilters.c -lib__nczstdfilters_la_LIBADD = $(LIBADD) +lib__nczstdfilters_la_LIBADD = $(LIBADD) $(top_builddir)/liblib/libnetcdf.la lib__nczhdf5filters_la_SOURCES = NCZhdf5filters.c -lib__nczhdf5filters_la_LIBADD = $(LIBADD) +lib__nczhdf5filters_la_LIBADD = $(LIBADD) $(top_builddir)/liblib/libnetcdf.la plugins_to_install += lib__nczhdf5filters.la plugins_to_install += lib__nczstdfilters.la @@ -119,7 +119,6 @@ lib__nch5unknown_la_SOURCES = H5Zunknown.c lib__nch5unknown_la_LDFLAGS = $(AM_LDFLAGS) ${RPATH} check_LTLIBRARIES += lib__nch5noop.la lib__nch5noop1.la lib__nch5unknown.la -# findplugin.sh needs these plugins, and I want to see if these get built properly check_LTLIBRARIES += lib__nch5misc.la lib__nczmisc.la # Bzip2 is used to test more complex filters diff --git a/plugins/NCZmisc.c b/plugins/NCZmisc.c index 06c6879756..94883576e9 100644 --- a/plugins/NCZmisc.c +++ b/plugins/NCZmisc.c @@ -156,6 +156,7 @@ NCZ_misc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) int i,stat = NC_NOERR; char json[4096]; char value[1024]; + size_t count, jlen; if(nparams == 0 || params == NULL) {stat = NC_EINVAL; goto done;} @@ -164,12 +165,15 @@ NCZ_misc_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) stat = NC_EINVAL; goto done; } - snprintf(json,sizeof(json),"{\"id\": \"%s\"",NCZ_misc_codec.codecid); + jlen = sizeof(json); + count = snprintf(json,sizeof(json),"{\"id\": \"%s\"",NCZ_misc_codec.codecid); for(i=0;i<14;i++) { - snprintf(value,sizeof(value),", \"%s\": \"%u\"",fields[i],params[i]); - strlcat(json,value,sizeof(json)); + size_t len = snprintf(value,sizeof(value),", \"%s\": \"%u\"",fields[i],params[i]); + count += len; assert(jlen > count); + strcat(json,value); } - strlcat(json,"}",sizeof(json)); + count += 1; assert(jlen > count); + strcat(json,"}"); if(codecp) { if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} } diff --git a/test-driver-verbose b/test-driver-verbose index 37a5c81ab7..57ebda8759 100755 --- a/test-driver-verbose +++ b/test-driver-verbose @@ -1,7 +1,7 @@ #! /bin/sh # test-driver - basic testsuite driver script. -scriptversion=2022-07-12.01; # UTC +scriptversion=2023-04-15.21; # UTC # Copyright (C) 2011-2013 Free Software Foundation, Inc. # @@ -107,9 +107,9 @@ case $estatus:$expect_failure in *:*) col=$red res=FAIL recheck=yes gcopy=yes;; esac -echo "begin $log_file" -cat $log_file -echo "end $log_file" +echo "begin $test_name" +cat $log_file | sed -e "s|\(.*\)|[$test_name] \1|" +echo "end $test_name" # Report outcome to console. echo "${col}${res}${std}: $test_name" diff --git a/test_common.in b/test_common.in index 3ad81d187b..3eee3bd7a5 100644 --- a/test_common.in +++ b/test_common.in @@ -33,11 +33,14 @@ FEATURE_HDF5=@HAS_HDF5@ FEATURE_FILTERTESTS=@DO_FILTER_TESTS@ FEATURE_PLUGIN_INSTALL_DIR=@PLUGIN_INSTALL_DIR@ FEATURE_BYTERANGE=@HAS_BYTERANGE@ -FEATURE_S3_SDK=@HAS_S3_SDK@ +FEATURE_ROS3=@HAS_HDF5_ROS3@ +FEATURE_S3_AWS=@HAS_S3_AWS@ +FEATURE_S3_INTERNAL=@HAS_S3_INTERNAL@ FEATURE_S3=@HAS_S3@ FEATURE_NCZARR=@HAS_NCZARR@ -FEATURE_S3TESTS=@DO_NCZARR_S3_TESTS@ +FEATURE_S3TESTS=@DO_S3_TESTING@ FEATURE_NCZARR_ZIP=@DO_NCZARR_ZIP_TESTS@ +FEATURE_LARGE_TESTS=@DO_LARGE_TESTS@ # Thredds-test server is currently disabled #FEATURE_THREDDSTEST=1 @@ -183,10 +186,27 @@ avail() { if test yes = `${execdir}/../ncdump/ncfilteravail $1` ; then return 0 ; else echo "filter $1 not available" ; return 1; fi } -# Thredds-test is no longer available -#FEATURE_THREDDSTEST=1 # Make sure we are in builddir (not execdir) cd $builddir +# As a protection against parallel make inter-test dependencies and race conditions, +# Support the creation of an isolation directory in which created products are stored. + +isolate() { +local rnd +if test "x$ISOPATH" = x ; then + ISOPATH=${builddir} + ISODIR="$1" + if test "x$ISODIR" != x ; then + # Make sure the path is unique + rnd=`${execdir}/../libdispatch/ncrandom` + ISODIR="${ISODIR}_$rnd" + ISOPATH="${ISOPATH}/$ISODIR" + rm -fr $ISOPATH + mkdir $ISOPATH + fi +fi +} + fi #TEST_COMMON_SH diff --git a/unit_test/CMakeLists.txt b/unit_test/CMakeLists.txt index d9d4a41730..f186a86269 100644 --- a/unit_test/CMakeLists.txt +++ b/unit_test/CMakeLists.txt @@ -16,7 +16,7 @@ SET(UNIT_TESTS test_ncuri) IF(USE_X_GETOPT) -SET(XGETOPTSRC "${CMAKE_CURRENT_SOURCE_DIR}/../libdispatch/XGetopt.c") + SET(XGETOPTSRC "${CMAKE_CURRENT_SOURCE_DIR}/../libdispatch/XGetopt.c") ENDIF() IF(NOT MSVC) @@ -32,9 +32,13 @@ ENDFOREACH() # Path convert test(s) add_bin_test(unit_test test_pathcvt) -# Aws Tests -build_bin_test(test_aws) -ADD_SH_TEST(unit_test run_aws) +IF(BUILD_UTILITIES) +IF(ENABLE_S3 AND WITH_S3_TESTING) +# SDK Test +BUILD_BIN_TEST(test_s3sdk ${XGETOPTSRC}) +ADD_SH_TEST(unit_test run_s3sdk) +ENDIF() +ENDIF() # Performance tests add_bin_test(unit_test tst_exhash timer_utils.c) diff --git a/unit_test/Makefile.am b/unit_test/Makefile.am index 67d0d566e8..a0f730a409 100644 --- a/unit_test/Makefile.am +++ b/unit_test/Makefile.am @@ -8,14 +8,15 @@ # Ed Hartnett 8/9/19 -# Put together AM_CPPFLAGS and AM_LDFLAGS. -include $(top_srcdir)/lib_flags.am - #SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose #sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose #LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose #TESTS_ENVIRONMENT = export SETX=1; +# Put together AM_CPPFLAGS and AM_LDFLAGS. +include $(top_srcdir)/lib_flags.am +AM_CPPFLAGS += -I${top_srcdir} -I${top_srcdir}/libdispatch + # Find and link to the netcdf-c library. LDADD = ${top_builddir}/liblib/libnetcdf.la @@ -36,12 +37,14 @@ check_PROGRAMS += tst_nc4internal TESTS += tst_nc4internal endif # USE_NETCDF4 -if ENABLE_NCZARR_S3_TESTS -check_PROGRAMS += test_aws -TESTS += run_aws.sh +if ENABLE_S3 +if ENABLE_S3_TESTALL +check_PROGRAMS += test_s3sdk +TESTS += run_s3sdk.sh +endif endif -EXTRA_DIST = CMakeLists.txt run_aws.sh +EXTRA_DIST = CMakeLists.txt run_s3sdk.sh EXTRA_DIST += nctest_netcdf4_classic.nc # If valgrind is present, add valgrind targets. diff --git a/unit_test/run_aws.sh b/unit_test/run_aws.sh deleted file mode 100755 index 18ea3f1e69..0000000000 --- a/unit_test/run_aws.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh - -if test "x$srcdir" = x ; then srcdir=`pwd`; fi -. ../test_common.sh - -set -e - -export NC_TEST_AWS_DIR=`pwd` - -rm -fr ./.aws -mkdir .aws -cat >.aws/config < -#include -#include -#include -#include "netcdf.h" -#include "ncrc.h" -#include "ncpathmgr.h" - -#undef DEBUG - -typedef struct ProfileTest { - const char* profile; - const char* access_key; - const char* secret_key; - const char* region; -} ProfileTest; - -typedef struct URLTest { - const char* url; - const char* newurl; - const char* profile; - const char* region; - const char* bucket; -} URLTest; - -static ProfileTest PROFILETESTS[] = { -{"default", "ACCESSKEYDEFAULTXXXX", "DEFAULT/ef0ghijklmnopqr/defaultxxxxxxxxx",""}, -{"ncar", "ACCESSKEYNCARXXXXXXX", "NCAR/ef0ghijklmnopqr/ncarxxxxxxxxxxxxxxx",""}, -{"unidata", "ACCESSKEYUNIDATAXXXX", "UNIDATA/ef0ghijklmnopqr/unidataxxxxxxxxx", "us-west-1"}, -{NULL, NULL,NULL,NULL} -}; - -static URLTest URLTESTS[] = { -{"s3://simplebucket#mode=nczarr,s3&aws.region=us-west-1", - "https://s3.us-west-1.amazonaws.com/simplebucket#mode=nczarr,s3&aws.region=us-west-1","default","us-west-1","simplebucket"}, -#if 0 -{"s3://simplebucket#mode=nczarr,s3&aws.profile=unidata", - "https://s3.us-west-1.amazonaws.com/simplebucket#mode=nczarr,s3&aws.profile=unidata","unidata","us-west-1","simplebucket"}, -{"https://s3.eu-east-1.amazonaws.com/simplebucket#mode=nczarr,s3&aws.profile=none", - "https://s3.eu-east-1.amazonaws.com/simplebucket#mode=nczarr,s3&aws.profile=none","none","eu-east-1","simplebucket"}, -{"https://s3.eu-west-1.amazonaws.com/bucket2#mode=nczarr,s3", - "https://s3.eu-west-1.amazonaws.com/bucket2#mode=nczarr,s3","default","eu-west-1","bucket2"}, -#endif -{NULL, NULL,NULL,NULL,NULL} -}; - -static char* awstestdir0 = NULL; - -void -failurltest(URLTest* test) -{ - fprintf(stderr,"***FAIL: urL=%s\n",test->url); -#ifdef DEBUG - abort(); -#endif - exit(1); -} - -void -failprofiletest(ProfileTest* test) -{ - fprintf(stderr,"***FAIL: profile=%s\n",test->profile); -#ifdef DEBUG - abort(); -#endif - exit(1); -} - -static int -testprofiles(void) -{ - int stat = NC_NOERR; - ProfileTest* test; - int index; - - for(index=0,test=PROFILETESTS;test->profile;test++,index++) { - const char* accesskey = NULL; - const char* region = NULL; - - if((stat = NC_s3profilelookup(test->profile, "aws_access_key_id", &accesskey))) goto done; - if((stat = NC_s3profilelookup(test->profile, "aws_region", ®ion))) goto done; - if(region == NULL) region = ""; -#ifdef DEBUG - printf("profile=%s aws_access_key_id=%s region=%s\n", - test->profile, - (accesskey?accesskey:""), - (region?region:"")); -#endif - if(accesskey == NULL || strcasecmp(accesskey,test->access_key)!=0) failprofiletest(test); - if(region == NULL || strcasecmp(region,test->region)!=0) failprofiletest(test); - } -done: - return stat; -} - -static int -testurls(void) -{ - int stat = NC_NOERR; - URLTest* test; - int index; - NCURI* url = NULL; - NCURI* url2 = NULL; - const char* profile = NULL; - char* region = NULL; - char* bucket = NULL; - char* newurl = NULL; - - for(index=0,test=URLTESTS;test->url;test++,index++) { - ncuriparse(test->url,&url); - if(url == NULL) { - fprintf(stderr,"URI parse fail: %s\n",test->url); - goto done; - } - if((stat = NC_getactives3profile(url, &profile))) { - fprintf(stderr,"active profile fail: %s\n",test->url); - goto done; - } - if((stat = NC_s3urlrebuild(url, &url2, &bucket, ®ion))) { - fprintf(stderr,"url rebuild failed: %s\n",test->url); - goto done; - } - newurl = ncuribuild(url2,NULL,NULL,NCURIALL); -#ifdef DEBUG - printf("url=%s {url=%s bucket=%s region=%s profile=%s}\n", - test->url,newurl,bucket,region,profile); -#endif - if(strcasecmp(newurl,test->newurl)!=0) failurltest(test); - if(strcasecmp(profile,test->profile)!=0) failurltest(test); - if(strcasecmp(region,test->region)!=0) failurltest(test); - if(strcasecmp(bucket,test->bucket)!=0) failurltest(test); - ncurifree(url); url = NULL; - ncurifree(url2); url2 = NULL; - nullfree(newurl); newurl = NULL; - nullfree(bucket); bucket = NULL; - nullfree(region); region = NULL; - } -done: - return stat; -} - - -int -main(int argc, char** argv) -{ - int stat = NC_NOERR; - - awstestdir0 = getenv("NC_TEST_AWS_DIR"); - if(awstestdir0 == NULL) { - fprintf(stderr,"NC_TEST_AWS_DIR environment variable is undefined\n"); - goto done; - } - - /* Load RC and .aws/config */ - if((stat = nc_initialize())) goto done; - - printf("testprofiles:\n-------------\n"); - stat = testprofiles(); - - printf("testurls:\n--------\n"); - stat = testurls(); - - printf("***PASS test_aws\n"); - -done: - if(stat) printf("*** FAIL: %s(%d)\n",nc_strerror(stat),stat); - exit(stat?1:0); -} diff --git a/unit_test/test_s3sdk.c b/unit_test/test_s3sdk.c new file mode 100644 index 0000000000..4237396187 --- /dev/null +++ b/unit_test/test_s3sdk.c @@ -0,0 +1,521 @@ +/********************************************************************* + * Copyright 2018, UCAR/Unidata + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + *********************************************************************/ + +#include +#include +#include +#include +#include "netcdf.h" +#include "ncrc.h" +#include "ncpathmgr.h" +#include "ncs3sdk.h" +#include "ncuri.h" + +#ifdef HAVE_GETOPT_H +#include +#endif + +#if defined(_WIN32) && !defined(__MINGW32__) +#include "XGetopt.h" +#endif + +#undef DEBUG + +#define SELF_CLEAN + +/* Mnemonic(s) */ +#define FORCE 1 + +#define LONGCOUNT 1010 + +enum Actions {ERROR_ACTION, EXISTS_ACTION, SIZE_ACTION, READ_ACTION, WRITE_ACTION, DELETE_ACTION, LIST_ACTION, LONGLIST_ACTION, SEARCH_ACTION}; + +struct Options { + int debug; + int test; + enum Actions action; + const char* url; + const char* key; +} dumpoptions; + +/* Upload data */ +static const char* uploaddata = "line1\nline2\nline3"; + +//static const char* testurl = "https://s3.us-east-1.amazonaws.com/unidata-zarr-test-data"; + +/* Global values */ +NCURI* purl = NULL; +const char* activeprofile = NULL; +const char* accessid = NULL; +const char* accesskey = NULL; +const char* newurl = NULL; +NCS3INFO s3info; +void* s3client = NULL; + +char* testkeypath = NULL; + +/* Forward */ +static void cleanup(void); + +#define CHECK(code) do {stat = check(code,__func__,__LINE__); if(stat) {goto done;}} while(0) + +static int +check(int code, const char* fcn, int line) +{ + if(code == NC_NOERR) return code; + fprintf(stderr,"***FAIL: (%d) %s @ %s:%d\n",code,nc_strerror(code),fcn,line); +#ifdef DEBUG + abort(); +#endif + exit(1); +} + +static enum Actions +actionfor(const char* s) +{ + if(strcasecmp(s,"exists")==0) return EXISTS_ACTION; + else if(strcasecmp(s,"size")==0) return SIZE_ACTION; + else if(strcasecmp(s,"read")==0) return READ_ACTION; + else if(strcasecmp(s,"write")==0) return WRITE_ACTION; + else if(strcasecmp(s,"list")==0) return LIST_ACTION; + else if(strcasecmp(s,"longlist")==0) return LONGLIST_ACTION; + else if(strcasecmp(s,"search")==0) return SEARCH_ACTION; + else if(strcasecmp(s,"delete")==0) return DELETE_ACTION; + return ERROR_ACTION; +} + +static void +seturl(const char* url, const char* key, int force) +{ +if(0) { + if(force || dumpoptions.url == NULL) + dumpoptions.url = url; + if(force || dumpoptions.key == NULL) + dumpoptions.key = key; + printf("url=|%s| key=|%s|\n", + dumpoptions.url?dumpoptions.url:"", + dumpoptions.key?dumpoptions.key:""); +} +} + +static int +profilesetup(const char* url) +{ + int stat = NC_NOERR; + + ncuriparse(url,&purl); + if(purl == NULL) { + fprintf(stderr,"URI parse fail: %s\n",url); + goto done; + } + CHECK(NC_s3urlprocess(purl, &s3info)); + + CHECK(NC_getactives3profile(purl, &activeprofile)); + CHECK(NC_s3profilelookup(activeprofile, "aws_access_key_id", &accessid)); + CHECK(NC_s3profilelookup(activeprofile, "aws_secret_access_key", &accesskey)); + if(s3info.profile) free(s3info.profile); + s3info.profile = nulldup(activeprofile); + if(s3info.region == NULL) s3info.region = ""; + if(s3info.bucket == NULL) {stat = NC_ES3; goto done;} + +#ifdef DEBUG + printf("%s\n",NC_s3dumps3info(&s3info)); + printf("\taws_access_key_id=%s\n",(accesskey?accesskey:"")); +#endif + +done: + return stat; +} + +static int +testbucketexists(void) +{ + int stat = NC_NOERR; + int exists = 0; + + seturl("https://s3.us-east-1.amazonaws.com/unidata-zarr-test-data",NULL,!FORCE); + + CHECK(profilesetup(dumpoptions.url)); + newurl = ncuribuild(purl,NULL,NULL,NCURIALL); +#ifdef DEBUG + printf("url=%s {url=%s bucket=%s region=%s profile=%s}\n", + dumpoptions.url,newurl,s3info.bucket,s3info.region,activeprofile); +#endif + if((s3client = NC_s3sdkcreateclient(&s3info))==NULL) {CHECK(NC_ES3);} + CHECK(NC_s3sdkbucketexists(s3client, s3info.bucket, &exists, NULL)); + printf("testbucketexists: exists=%d\n",exists); + if(!exists) {stat = NC_EINVAL; goto done;} + +done: + cleanup(); + return stat; +} + +static int +testinfo(void) +{ + int stat = NC_NOERR; + unsigned long long size = 0; + + seturl("https://s3.us-east-1.amazonaws.com/unidata-zarr-test-data","/object_store/dir1/nested1/file1.txt",!FORCE); + + CHECK(profilesetup(dumpoptions.url)); + newurl = ncuribuild(purl,NULL,NULL,NCURIALL); +#ifdef DEBUG + printf("url=%s => {url=%s bucket=%s region=%s profile=%s}\n", + dumpoptions.url,newurl,s3info.bucket,s3info.region,activeprofile); +#endif + if((s3client = NC_s3sdkcreateclient(&s3info))==NULL) {CHECK(NC_ES3);} + CHECK(NC_s3sdkinfo(s3client, s3info.bucket, dumpoptions.key, &size, NULL)); + printf("testinfo: size=%llu\n",size); + +done: + cleanup(); + return stat; +} + +static int +testread(void) +{ + int stat = NC_NOERR; + unsigned long long size = 0; + void* content = NULL; + + seturl("https://s3.us-east-1.amazonaws.com/unidata-zarr-test-data", "/netcdf-c/test_s3.txt",!FORCE); + + CHECK(profilesetup(dumpoptions.url)); + newurl = ncuribuild(purl,NULL,NULL,NCURIALL); +#ifdef DEBUG + printf("url=%s {url=%s bucket=%s region=%s profile=%s}\n", + dumpoptions.url,newurl,s3info.bucket,s3info.region,activeprofile); +#endif + if((s3client = NC_s3sdkcreateclient(&s3info))==NULL) {CHECK(NC_ES3);} + CHECK(NC_s3sdkinfo(s3client, s3info.bucket, dumpoptions.key, &size, NULL)); + printf("testread: size=%llu\n",size); + content = calloc(1,size+1); + CHECK(NC_s3sdkread(s3client, s3info.bucket, dumpoptions.key, 0, size, content, NULL)); + ((char*)content)[size] = '\0'; + printf("testread: content=|%s|\n",(char*)content); + free(content); + +done: + cleanup(); + return stat; +} + +static int +testwrite(void) +{ + int stat = NC_NOERR; + size64_t size = 0; + void* content = NULL; + + seturl("https://s3.us-east-1.amazonaws.com/unidata-zarr-test-data", "/netcdf-c/test_s3.txt",!FORCE); + + CHECK(profilesetup(dumpoptions.url)); + newurl = ncuribuild(purl,NULL,NULL,NCURIALL); +#ifdef DEBUG + printf("url=%s {url=%s bucket=%s region=%s profile=%s}\n", + dumpoptions.url,newurl,s3info.bucket,s3info.region,activeprofile); +#endif + if((s3client = NC_s3sdkcreateclient(&s3info))==NULL) {CHECK(NC_ES3);} + CHECK(NC_s3sdkwriteobject(s3client, s3info.bucket, dumpoptions.key, strlen(uploaddata), uploaddata, NULL)); + + /* Verify existence and size */ + CHECK(NC_s3sdkinfo(s3client, s3info.bucket, dumpoptions.key, &size, NULL)); + printf("testwrite: size=%llu\n",size); + + content = calloc(1,size+1); /* allow for trailing nul */ + CHECK(NC_s3sdkread(s3client, s3info.bucket, dumpoptions.key, 0, size, content, NULL)); + ((char*)content)[size] = '\0'; + printf("testwrite: content=|%s|\n",(const char*)content); + free(content); + +done: + cleanup(); + return stat; +} + +static int +testgetkeys(void) +{ + int stat = NC_NOERR; + size_t i,nkeys = 0; + char** keys = NULL; + + seturl("https://s3.us-east-1.amazonaws.com/unidata-zarr-test-data", "/object_store/dir1",!FORCE); + + CHECK(profilesetup(dumpoptions.url)); + newurl = ncuribuild(purl,NULL,NULL,NCURIALL); +#ifdef DEBUG + printf("url=%s => info=%s\n",dumpoptions.url,NC_s3dumps3info(&s3info)); +#endif + if((s3client = NC_s3sdkcreateclient(&s3info))==NULL) {CHECK(NC_ES3);} + CHECK(NC_s3sdkgetkeys(s3client, s3info.bucket, dumpoptions.key, &nkeys, &keys, NULL)); + printf("testgetkeys: nkeys=%u; keys:\n",(unsigned)nkeys); + for(i=0;i info=%s\n",dumpoptions.url,NC_s3dumps3info(&s3info)); +#endif + if((s3client = NC_s3sdkcreateclient(&s3info))==NULL) {CHECK(NC_ES3);} + for(i=0;i GETKEYLEN && memcmp(suffix,GETKEY,GETKEYLEN)==0) { + sscanf(suffix,GETKEY"%ld",&index); + if(index >= 0) checklist[index] = 1; + } + } + printf("\n"); + for(i=0;i info=%s\n",dumpoptions.url,NC_s3dumps3info(&s3info)); +#endif + if((s3client = NC_s3sdkcreateclient(&s3info))==NULL) {CHECK(NC_ES3);} + CHECK(NC_s3sdksearch(s3client, s3info.bucket, dumpoptions.key, &nkeys, &keys, NULL)); + printf("testsearch: nkeys=%u; keys:\n",(unsigned)nkeys); + for(i=0;i][-k ] \n"); + goto done; + case 'k': + dumpoptions.key = strdup(optarg); + break; + case 't': + dumpoptions.test = 1; + break; + case 'u': + dumpoptions.url = strdup(optarg); + break; + case '?': + fprintf(stderr,"unknown option\n"); + stat = NC_EINVAL; + goto done; + } + } + + if(dumpoptions.url== NULL) { + fprintf(stderr,"no -u argument\n"); + stat = NC_EINVAL; + goto done; + } + + if(dumpoptions.test) { + /* Mimic run_s3sdk.sh test */ + printf("Test: testwrite\n"); + if((stat = testwrite())) goto done; + printf("Test: testread\n"); + if((stat = testread())) goto done; + printf("Test: testinfo\n"); + if((stat = testinfo())) goto done; + printf("Test: testgetkeys\n"); + if((stat = testgetkeys())) goto done; + printf("Test: testgetkeyslong\n"); + if((stat = testgetkeyslong())) goto done; + printf("Test: testsearch\n"); + if((stat = testsearch())) goto done; + printf("Test: testdeletekey\n"); + if((stat = testdeletekey())) goto done; + } else { + /* get action argument */ + argc -= optind; + argv += optind; + if (argc > 1) { + fprintf(stderr, "test_s3: only one action argument permitted\n"); + stat = NC_EINVAL; goto done; + } + if (argc == 0) { + fprintf(stderr, "test_s3: no action argument specified\n"); + stat = NC_EINVAL; goto done; + } + + dumpoptions.action = actionfor(argv[0]); + + if(dumpoptions.action != EXISTS_ACTION && dumpoptions.key == NULL) { + fprintf(stderr,"no -k argument\n"); + stat = NC_EINVAL; + goto done; + } + + switch (dumpoptions.action) { + case EXISTS_ACTION: stat = testbucketexists(); break; + case SIZE_ACTION: stat = testinfo(); break; + case READ_ACTION: stat = testread(); break; + case WRITE_ACTION: stat = testwrite(); break; + case LIST_ACTION: stat = testgetkeys(); break; + case LONGLIST_ACTION: stat = testgetkeyslong(); break; + case SEARCH_ACTION: stat = testsearch(); break; + case DELETE_ACTION: stat = testdeletekey(); break; + case ERROR_ACTION: /* fall thru */ + default: fprintf(stderr,"Illegal action\n"); exit(1); + } + } +done: + cleanup(); + if(stat) + printf("*** FAIL: %s(%d)\n",nc_strerror(stat),stat); + else + printf("***PASS\n"); + (void)NC_s3sdkfinalize(); + (void)nc_finalize(); + exit(stat?1:0); +}