From 1827a5876c03f5c3285a1cb6e9bf2504b3ddbff9 Mon Sep 17 00:00:00 2001 From: wkliao Date: Thu, 5 Sep 2024 13:51:12 -0500 Subject: [PATCH 1/6] move nonblockin examples to folder nonblocking --- examples/nonblocking/Makefile | 49 +++++++++++++ .../{ => nonblocking}/nonblocking_write.py | 28 ++++---- .../nonblocking_write_def.py | 28 ++++---- examples/nonblocking/parallel_run.sh | 72 +++++++++++++++++++ 4 files changed, 149 insertions(+), 28 deletions(-) create mode 100644 examples/nonblocking/Makefile rename examples/{ => nonblocking}/nonblocking_write.py (100%) rename examples/{ => nonblocking}/nonblocking_write_def.py (100%) create mode 100755 examples/nonblocking/parallel_run.sh diff --git a/examples/nonblocking/Makefile b/examples/nonblocking/Makefile new file mode 100644 index 0000000..494c168 --- /dev/null +++ b/examples/nonblocking/Makefile @@ -0,0 +1,49 @@ +# +# Copyright (C) 2024, Northwestern University and Argonne National Laboratory +# See COPYRIGHT notice in top-level directory. +# + +check_PROGRAMS = nonblocking_write_def.py \ + nonblocking_write.py + + +TESTS_ENVIRONMENT = export check_PROGRAMS="${check_PROGRAMS}"; +TESTS_ENVIRONMENT += export PNETCDF_DIR="${PNETCDF_DIR}"; + +OUTPUT_DIR = _tmp_output + +all: + +check: ptest4 + +ptests: ptest3 ptest4 ptest8 + +ptest3: + @mkdir -p ${OUTPUT_DIR} + @echo "======================================================================" + @echo " examples: Parallel testing on 3 MPI processes" + @echo "======================================================================" + @${TESTS_ENVIRONMENT} export NPROC=3; ./parallel_run.sh ${OUTPUT_DIR} || exit 1 + @echo "" + +ptest4: + @mkdir -p ${OUTPUT_DIR} + @echo "======================================================================" + @echo " examples: Parallel testing on 4 MPI processes" + @echo "======================================================================" + @${TESTS_ENVIRONMENT} export NPROC=4; ./parallel_run.sh ${OUTPUT_DIR} || exit 1 + @echo "" + +ptest8: + @mkdir -p ${OUTPUT_DIR} + @echo "======================================================================" + @echo " examples: Parallel testing on 8 MPI processes" + @echo "======================================================================" + @${TESTS_ENVIRONMENT} export NPROC=8; ./parallel_run.sh ${OUTPUT_DIR} || exit 1 + @echo "" + +clean: + rm -rf ${OUTPUT_DIR} + +.PHONY: all check ptests ptest3 ptest4 ptest8 clean + diff --git a/examples/nonblocking_write.py b/examples/nonblocking/nonblocking_write.py similarity index 100% rename from examples/nonblocking_write.py rename to examples/nonblocking/nonblocking_write.py index 671e962..858671b 100644 --- a/examples/nonblocking_write.py +++ b/examples/nonblocking/nonblocking_write.py @@ -36,20 +36,6 @@ from mpi4py import MPI import pnetcdf -def parse_help(): - help_flag = "-h" in sys.argv or "--help" in sys.argv - if help_flag and rank == 0: - help_text = ( - "Usage: {} [-h] | [-q] [file_name]\n" - " [-h] Print help\n" - " [-q] Quiet mode (reports when fail)\n" - " [-l len] size of each dimension of the local array\n" - " [filename] (Optional) output netCDF file name\n" - ).format(sys.argv[0]) - print(help_text) - return help_flag - - def pnetcdf_io(filename, length): NDIMS = 3 NUM_VARS = 10 @@ -146,6 +132,20 @@ def pnetcdf_io(filename, length): f.close() +def parse_help(): + help_flag = "-h" in sys.argv or "--help" in sys.argv + if help_flag and rank == 0: + help_text = ( + "Usage: {} [-h] | [-q] [file_name]\n" + " [-h] Print help\n" + " [-q] Quiet mode (reports when fail)\n" + " [-l len] size of each dimension of the local array\n" + " [filename] (Optional) output netCDF file name\n" + ).format(sys.argv[0]) + print(help_text) + return help_flag + + if __name__ == "__main__": comm = MPI.COMM_WORLD rank = comm.Get_rank() diff --git a/examples/nonblocking_write_def.py b/examples/nonblocking/nonblocking_write_def.py similarity index 100% rename from examples/nonblocking_write_def.py rename to examples/nonblocking/nonblocking_write_def.py index f9797ac..2003dc7 100644 --- a/examples/nonblocking_write_def.py +++ b/examples/nonblocking/nonblocking_write_def.py @@ -37,20 +37,6 @@ from mpi4py import MPI import pnetcdf -def parse_help(): - help_flag = "-h" in sys.argv or "--help" in sys.argv - if help_flag and rank == 0: - help_text = ( - "Usage: {} [-h] | [-q] [file_name]\n" - " [-h] Print help\n" - " [-q] Quiet mode (reports when fail)\n" - " [-l len] size of each dimension of the local array\n" - " [filename] (Optional) output netCDF file name\n" - ).format(sys.argv[0]) - print(help_text) - return help_flag - - def pnetcdf_io(file_name, length): NDIMS = 3 NUM_VARS = 10 @@ -140,6 +126,20 @@ def pnetcdf_io(file_name, length): f.close() +def parse_help(): + help_flag = "-h" in sys.argv or "--help" in sys.argv + if help_flag and rank == 0: + help_text = ( + "Usage: {} [-h] | [-q] [file_name]\n" + " [-h] Print help\n" + " [-q] Quiet mode (reports when fail)\n" + " [-l len] size of each dimension of the local array\n" + " [filename] (Optional) output netCDF file name\n" + ).format(sys.argv[0]) + print(help_text) + return help_flag + + if __name__ == "__main__": comm = MPI.COMM_WORLD rank = comm.Get_rank() diff --git a/examples/nonblocking/parallel_run.sh b/examples/nonblocking/parallel_run.sh new file mode 100755 index 0000000..cbbae44 --- /dev/null +++ b/examples/nonblocking/parallel_run.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# +# Copyright (C) 2024, Northwestern University and Argonne National Laboratory +# See COPYRIGHT notice in top-level directory. +# + +# Exit immediately if a command exits with a non-zero status. +set -e + +# Get the directory containing this script +if test "x$NPROC" = x ; then + NPROC=4 +fi + +# get output folder from command line +if test "$#" -gt 0 ; then + args=("$@") + OUT_DIR="${args[0]}" + # check if output folder exists + if ! test -d $OUT_DIR ; then + echo "Error: output folder \"$OUT_DIR\" does not exist." + exit 1 + fi +else + # output folder is not set at command line, use current folder + OUT_DIR="." +fi +# echo "OUT_DIR=$OUT_DIR" + +MPI4PY_VERSION=`python -c "import mpi4py; print(mpi4py.__version__)"` +MPI4PY_VERSION_MAJOR=`echo ${MPI4PY_VERSION} | cut -d. -f1` +# echo "MPI4PY_VERSION=$MPI4PY_VERSION" +# echo "MPI4PY_VERSION_MAJOR=$MPI4PY_VERSION_MAJOR" + +TEST_FLEXIBLE_API=no +if test "x$PNETCDF_DIR" != x ; then + PNETCDF_C_VERSION=`$PNETCDF_DIR/bin/pnetcdf-config --version | cut -d' ' -f2` + V_MAJOR=`echo ${PNETCDF_C_VERSION} | cut -d. -f1` + V_MINOR=`echo ${PNETCDF_C_VERSION} | cut -d. -f2` + V_SUB=`echo ${PNETCDF_C_VERSION} | cut -d. -f3` + VER_NUM=$((V_MAJOR*1000000 + V_MINOR*1000 + V_SUB)) + if test $VER_NUM -gt 1013000 ; then + TEST_FLEXIBLE_API=yes + fi +fi + +# test PnetCDF flexible APIs only when PnetCDF-C >= 1.13.1 or mpi4py < 4 +if test $MPI4PY_VERSION_MAJOR -lt 4 ; then + TEST_FLEXIBLE_API=yes +fi + +for prog in $check_PROGRAMS; do + if test "x$TEST_FLEXIBLE_API" = xno && + test "x$prog" = "xflexible_api.py" ; then + TETS_PROGS+=" flexible_api.py" + printf '%-60s' "Testing $prog" + echo " ---- SKIP" + continue + fi + + printf '%-60s' "Testing $prog" + + CMD="mpiexec -n $NPROC python $prog -q $OUT_DIR/${prog%.*}.nc" + $CMD + status=$? + if [ $status -ne 0 ]; then + echo " ---- FAIL" + else + echo " ---- PASS" + fi +done + From fe79b9b9ccfffe19acfc04e1b3a497406e7c103f Mon Sep 17 00:00:00 2001 From: wkliao Date: Thu, 5 Sep 2024 13:52:13 -0500 Subject: [PATCH 2/6] use python style for subarray with ghost cells --- examples/ghost_cell.py | 72 ++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 38 deletions(-) diff --git a/examples/ghost_cell.py b/examples/ghost_cell.py index d1ffae0..1dd710f 100644 --- a/examples/ghost_cell.py +++ b/examples/ghost_cell.py @@ -66,62 +66,38 @@ import pnetcdf -def parse_help(): - help_flag = "-h" in sys.argv or "--help" in sys.argv - if help_flag and rank == 0: - help_text = ( - "Usage: {} [-h] | [-q] [file_name]\n" - " [-h] Print help\n" - " [-q] Quiet mode (reports when fail)\n" - " [-k format] file format: 1 for CDF-1, 2 for CDF-2, 5 for CDF-5\n" - " [-l len] size of each dimension of the local array\n" - " [filename] (Optional) output netCDF file name\n" - ).format(sys.argv[0]) - print(help_text) - return help_flag - - def pnetcdf_io(filename, file_format, length): count = [length, length + 1] psizes = MPI.Compute_dims(nprocs, 2) + nghosts = 2 + if verbose and rank == 0: - print("psizes=", psizes[0], psizes[1]) + print("number of MPI processes =", nprocs) + print("number of ghost cells =", nghosts) + print("psizes=", psizes) # set global array sizes gsizes = np.zeros(2, dtype=np.int64) gsizes[0] = length * psizes[0] # global array size gsizes[1] = (length + 1) * psizes[1] if verbose and rank == 0: - print("global variable shape:", gsizes[0], gsizes[1]) + print("global variable shape:", gsizes) # find its local rank IDs along each dimension local_rank = np.zeros(2, dtype=np.int32) local_rank[0] = rank // psizes[1] local_rank[1] = rank % psizes[1] if verbose: - print("rank {}: dim rank= {} {}".format(rank, local_rank[0], local_rank[1])) + print("rank ",rank,": local_rank = ", local_rank) # set subarray access pattern count = np.array([length, length + 1], dtype=np.int64) start = np.array([local_rank[0] * count[0], local_rank[1] * count[1]], dtype=np.int64) if verbose: - print("start= {} {} count= {} {}".format(start[0], start[1], count[0], count[1])) - - # allocate and initialize buffer with ghost cells on both ends of each dim - nghosts = 2 - bufsize = (count[0] + 2 * nghosts) * (count[1] + 2 * nghosts) - buf = np.empty(bufsize, dtype=np.int32) - for i in range(count[0] + 2 * nghosts): - for j in range(count[1] + 2 * nghosts): - if nghosts <= i < count[0] + nghosts and \ - nghosts <= j < count[1] + nghosts: - buf[i * (count[1] + 2 * nghosts) + j] = rank - else: - # set values of all ghost cells to -8 - buf[i * (count[1] + 2 * nghosts) + j] = -8 + print("rank ",rank,": start = ",start," count =", count) # Create the file f = pnetcdf.File(filename = filename, @@ -140,19 +116,39 @@ def pnetcdf_io(filename, file_format, length): # Exit the define mode f.enddef() - # set imap pattern for local buffer - imap = np.zeros(2, dtype=np.int64) - imap[1] = 1 - imap[0] = count[1] + 2 * nghosts - buf_ptr = buf[nghosts * (count[1] + 2 * nghosts + 1):] + # allocate and initialize buffer with ghost cells on both ends of each dim + buf = np.empty([2*nghosts+count[0], 2*nghosts+count[1]], dtype=np.int32) + buf.fill(-8) + + # keep contents of ghost cells to -8, all others 'rank' + buf[nghosts:nghosts+count[0], nghosts:nghosts+count[1]].fill(rank) # Write data to the variable - var.put_var_all(buf_ptr, start = start, count = count, imap = imap) + end = np.add(start, count) + var[start[0]:end[0], start[1]:end[1]] = buf[nghosts:nghosts+count[0], nghosts:nghosts+count[1]] + + # Equivalently, below uses function call + var.put_var_all(buf[nghosts:nghosts+count[0], nghosts:nghosts+count[1]], start = start, count = count) # Close the file f.close() +def parse_help(): + help_flag = "-h" in sys.argv or "--help" in sys.argv + if help_flag and rank == 0: + help_text = ( + "Usage: {} [-h] | [-q] [file_name]\n" + " [-h] Print help\n" + " [-q] Quiet mode (reports when fail)\n" + " [-k format] file format: 1 for CDF-1, 2 for CDF-2, 5 for CDF-5\n" + " [-l len] size of each dimension of the local array\n" + " [filename] (Optional) output netCDF file name\n" + ).format(sys.argv[0]) + print(help_text) + return help_flag + + if __name__ == "__main__": comm = MPI.COMM_WORLD rank = comm.Get_rank() From 4be774a444533557e37e0a53f836e050884d782a Mon Sep 17 00:00:00 2001 From: wkliao Date: Thu, 5 Sep 2024 13:57:52 -0500 Subject: [PATCH 3/6] add python style of array I/O --- examples/collective_write.py | 42 +++++++++++++++----------- examples/fill_mode.py | 58 ++++++++++++++++++++++++------------ examples/get_vara.py | 52 +++++++++++++++++++++----------- examples/global_attribute.py | 37 +++++++++++++---------- examples/hints.py | 40 +++++++++++++++---------- examples/put_vara.py | 53 +++++++++++++++++++++----------- 6 files changed, 178 insertions(+), 104 deletions(-) diff --git a/examples/collective_write.py b/examples/collective_write.py index 77cbb10..f4115a3 100644 --- a/examples/collective_write.py +++ b/examples/collective_write.py @@ -36,20 +36,6 @@ from mpi4py import MPI import pnetcdf -def parse_help(): - help_flag = "-h" in sys.argv or "--help" in sys.argv - if help_flag and rank == 0: - help_text = ( - "Usage: {} [-h] | [-q] [file_name]\n" - " [-h] Print help\n" - " [-q] Quiet mode (reports when fail)\n" - " [-k format] file format: 1 for CDF-1, 2 for CDF-2, 5 for CDF-5\n" - " [-l len] size of each dimension of the local array\n" - " [filename] (Optional) output netCDF file name\n" - ).format(sys.argv[0]) - print(help_text) - return help_flag - def pnetcdf_io(filename, file_format, length): # number of dimensions NDIMS = 3 @@ -60,8 +46,8 @@ def pnetcdf_io(filename, file_format, length): print("Number of variables = ", NUM_VARS) print("Number of dimensions = ", NDIMS) - start = np.zeros(NDIMS, dtype=np.int32) - count = np.zeros(NDIMS, dtype=np.int32) + start = np.zeros(NDIMS, dtype=np.int32) + count = np.zeros(NDIMS, dtype=np.int32) gsizes = np.zeros(NDIMS, dtype=np.int32) buf = [] @@ -75,8 +61,10 @@ def pnetcdf_io(filename, file_format, length): for i in range(NDIMS): gsizes[i] = length * psizes[i] start[i] *= length - count[i] = length - bufsize *= length + count[i] = length + bufsize *= length + + end = np.add(start, count) # Allocate buffer and initialize with non-zero numbers for i in range(NUM_VARS): @@ -111,12 +99,30 @@ def pnetcdf_io(filename, file_format, length): # Collectively write one variable at a time for i in range(NUM_VARS): + # write using Python style subarray access + vars[i][start[0]:end[0], start[1]:end[1], start[2]:end[2]] = buf[i] + + # Equivalently, below uses function call vars[i].put_var_all(buf[i], start = start, count = count) # Close the file f.close() +def parse_help(): + help_flag = "-h" in sys.argv or "--help" in sys.argv + if help_flag and rank == 0: + help_text = ( + "Usage: {} [-h] | [-q] [file_name]\n" + " [-h] Print help\n" + " [-q] Quiet mode (reports when fail)\n" + " [-k format] file format: 1 for CDF-1, 2 for CDF-2, 5 for CDF-5\n" + " [-l len] size of each dimension of the local array\n" + " [filename] (Optional) output netCDF file name\n" + ).format(sys.argv[0]) + print(help_text) + return help_flag + if __name__ == "__main__": comm = MPI.COMM_WORLD diff --git a/examples/fill_mode.py b/examples/fill_mode.py index 1f9a86e..3135709 100644 --- a/examples/fill_mode.py +++ b/examples/fill_mode.py @@ -45,18 +45,6 @@ from mpi4py import MPI import pnetcdf -def parse_help(): - help_flag = "-h" in sys.argv or "--help" in sys.argv - if help_flag and rank == 0: - help_text = ( - "Usage: {} [-h] | [-q] [file_name]\n" - " [-h] Print help\n" - " [-q] Quiet mode (reports when fail)\n" - " [filename] (Optional) output netCDF file name\n" - ).format(sys.argv[0]) - print(help_text) - return help_flag - def pnetcdf_io(filename): NY = 3 @@ -77,13 +65,13 @@ def pnetcdf_io(filename): global_nx = NX * nprocs # define dimensions - dim_xu = f.def_dim('REC_DIM', -1) + time = f.def_dim('REC_DIM', -1) dim_x = f.def_dim('X',global_nx) dim_y = f.def_dim('Y',global_ny) # define 2D variables of integer type fix_var = f.def_var("fix_var", pnetcdf.NC_INT, (dim_y, dim_x)) - rec_var = f.def_var("rec_var", pnetcdf.NC_INT, (dim_xu, dim_x)) + rec_var = f.def_var("rec_var", pnetcdf.NC_INT, (time, dim_x)) # set the fill mode to NC_FILL for the entire file old_fillmode = f.set_fill(pnetcdf.NC_FILL) @@ -110,12 +98,24 @@ def pnetcdf_io(filename): start = np.array([0, NX * rank]) count = np.array([NY, NX]) + if verbose: + print("rank: ", rank," global_ny=",global_ny," global_nx=",global_nx," start=",start," count=",count) + # allocate user buffer buf = np.array([[rank] * NX] * NY).astype('i4') + for i in range(NY): + for j in range(NX): + buf[i, j] = 10 + rank*10 + i * NX + j # do not write the variable in full count[1] -= 1 - fix_var.put_var_all(buf, start = start, count = count) + + # write using Python style subarray access + end = np.add(start, count) + fix_var[start[0]:end[0], start[1]:end[1]] = buf[0:count[0], 0:count[1]] + + # Equivalently, below uses function call + fix_var.put_var_all(buf[0:count[0], 0:count[1]], start = start, count = count) # check fill value no_fill, fill_value = fix_var.inq_fill() @@ -126,20 +126,40 @@ def pnetcdf_io(filename): count[0] = 1 rec_var.fill_rec(start[0]) - # write to the 1st record - rec_var.put_var_all(buf, start = start, count = count) + # write to the 1st record, using Python style subarray access + end = np.add(start, count) + rec_var[start[0]:end[0], start[1]:end[1]] = buf[0:count[0], 0:count[1]] + + # Equivalently, below uses function call + rec_var.put_var_all(buf[0:count[0], 0:count[1]], start = start, count = count) # fill the 2nd record of the record variable start[0] = 1 rec_var.fill_rec(start[0]) - # write to the 2nd record - rec_var.put_var_all(buf, start = start, count = count) + # write to the 2nd record using Python style subarray access + end = np.add(start, count) + rec_var[start[0]:end[0], start[1]:end[1]] = buf[0:count[0], 0:count[1]] + + # Equivalently, below uses function call + rec_var.put_var_all(buf[0:count[0], 0:count[1]], start = start, count = count) # close file f.close() +def parse_help(): + help_flag = "-h" in sys.argv or "--help" in sys.argv + if help_flag and rank == 0: + help_text = ( + "Usage: {} [-h] | [-q] [file_name]\n" + " [-h] Print help\n" + " [-q] Quiet mode (reports when fail)\n" + " [filename] (Optional) output netCDF file name\n" + ).format(sys.argv[0]) + print(help_text) + return help_flag + if __name__ == "__main__": verbose = True diff --git a/examples/get_vara.py b/examples/get_vara.py index 361b004..7781d09 100644 --- a/examples/get_vara.py +++ b/examples/get_vara.py @@ -51,21 +51,6 @@ from mpi4py import MPI import pnetcdf -def parse_help(): - help_flag = "-h" in sys.argv or "--help" in sys.argv - if help_flag and rank == 0: - help_text = ( - "Usage: {} [-h] | [-q] [file_name]\n" - " [-h] Print help\n" - " [-q] Quiet mode (reports when fail)\n" - " [-k format] file format: 1 for CDF-1, 2 for CDF-2, 5 for CDF-5\n" - " [-l len] size of each dimension of the local array\n" - " [filename] input netCDF file name\n" - ).format(sys.argv[0]) - print(help_text) - return help_flag - - def pnetcdf_io(filename, file_format): # Open an existing file for reading @@ -87,15 +72,25 @@ def pnetcdf_io(filename, file_format): print("Y dimension size = ", global_ny) print("X dimension size = ", global_nx) - # get the variable of a 2D variable of integer type + # get the handler of variable named 'var', a 2D variable of integer type v = f.variables['var'] # Get the variable's attribute named "str_att_name" + str_att = v.str_att_name + + if rank == 0 and verbose: + print("Read variable attribute \"str_att_name\" of type text =", str_att) + + # Equivalently, below uses function call str_att = v.get_att("str_att_name") + if rank == 0 and verbose: - print("variable attribute \"str_att_name\" of type text =", str_att) + print("Read variable attribute \"str_att_name\" of type text =", str_att) # Get the variable's attribute named "float_att_name" + float_att = v.float_att_name + + # Equivalently, below uses function call float_att = v.get_att("float_att_name") # set access pattern for reading subarray @@ -103,15 +98,36 @@ def pnetcdf_io(filename, file_format): local_nx = global_nx // nprocs start = [0, local_nx * rank] count = [local_ny, local_nx] + end = np.add(start, count) - # Read a subarray in collective mode + # allocate read buffer r_buf = np.empty(tuple(count), v.dtype) + + # Read a subarray in collective mode + r_bufs = v[start[0]:end[0], start[1]:end[1]] + + # Equivalently, below uses function call v.get_var_all(r_buf, start = start, count = count) # close the file f.close() +def parse_help(): + help_flag = "-h" in sys.argv or "--help" in sys.argv + if help_flag and rank == 0: + help_text = ( + "Usage: {} [-h] | [-q] [file_name]\n" + " [-h] Print help\n" + " [-q] Quiet mode (reports when fail)\n" + " [-k format] file format: 1 for CDF-1, 2 for CDF-2, 5 for CDF-5\n" + " [-l len] size of each dimension of the local array\n" + " [filename] input netCDF file name\n" + ).format(sys.argv[0]) + print(help_text) + return help_flag + + if __name__ == "__main__": comm = MPI.COMM_WORLD rank = comm.Get_rank() diff --git a/examples/global_attribute.py b/examples/global_attribute.py index 0c75a57..f90dfb5 100644 --- a/examples/global_attribute.py +++ b/examples/global_attribute.py @@ -32,19 +32,6 @@ import pnetcdf -def parse_help(): - help_flag = "-h" in sys.argv or "--help" in sys.argv - if help_flag and rank == 0: - help_text = ( - "Usage: {} [-h] | [-q] [file_name]\n" - " [-h] Print help\n" - " [-q] Quiet mode (reports when fail)\n" - " [-k format] file format: 1 for CDF-1, 2 for CDF-2, 5 for CDF-5\n" - " [filename] (Optional) output netCDF file name\n" - ).format(sys.argv[0]) - print(help_text) - return help_flag - def pnetcdf_io(filename, file_format): digit = np.int16([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) @@ -67,15 +54,22 @@ def pnetcdf_io(filename, file_format): str_att = comm.bcast(str_att, root=0) # write a global attribute - f.put_att('history',str_att) + f.history = str_att + if rank == 0 and verbose: print(f'writing global attribute "history" of text {str_att}') + # Equivalently, below uses function call + f.put_att('history',str_att) + # add another global attribute named "digits": an array of short type - f.put_att('digits', digit) + f.digits = digit if rank == 0 and verbose: print("writing global attribute \"digits\" of 10 short integers") + # Equivalently, below uses function call + f.put_att('digits', digit) + # Close the file f.close() @@ -107,6 +101,19 @@ def pnetcdf_io(filename, file_format): f.close() +def parse_help(): + help_flag = "-h" in sys.argv or "--help" in sys.argv + if help_flag and rank == 0: + help_text = ( + "Usage: {} [-h] | [-q] [file_name]\n" + " [-h] Print help\n" + " [-q] Quiet mode (reports when fail)\n" + " [-k format] file format: 1 for CDF-1, 2 for CDF-2, 5 for CDF-5\n" + " [filename] (Optional) output netCDF file name\n" + ).format(sys.argv[0]) + print(help_text) + return help_flag + if __name__ == "__main__": comm = MPI.COMM_WORLD rank = comm.Get_rank() diff --git a/examples/hints.py b/examples/hints.py index d917fcf..2d02b48 100644 --- a/examples/hints.py +++ b/examples/hints.py @@ -30,18 +30,6 @@ from mpi4py import MPI import pnetcdf -def parse_help(): - help_flag = "-h" in sys.argv or "--help" in sys.argv - if help_flag and rank == 0: - help_text = ( - "Usage: {} [-h] | [-q] [file_name]\n" - " [-h] Print help\n" - " [-q] Quiet mode (reports when fail)\n" - " [filename] (Optional) output netCDF file name\n" - ).format(sys.argv[0]) - print(help_text) - return help_flag - def print_hints(nc_file, nc_var1, nc_var2): value = np.zeros(MPI.MAX_INFO_VAL, dtype='c') header_size, header_extent, var_zy_start, var_yx_start = -1, -1, -1, -1 @@ -131,9 +119,13 @@ def pnetcdf_io(filename): # set subarray access pattern start = [NZ * rank, 0] - count =[NZ, NY * nprocs] + count = [NZ, NY * nprocs] + end = [start[i] + count[i] for i in range(2)] + + # write to variable var_zy + var_zy[start[0]:end[0], start[1]:end[1]] = buf_zy - # write to variable + # Equivalently, below uses function call var_zy.put_var_all(buf_zy, start = start, count = count) # var_yx is partitioned along X dimension @@ -143,9 +135,13 @@ def pnetcdf_io(filename): # set subarray access pattern start = [0, NX*rank] - count =[NY * nprocs, NX] + count = [NY * nprocs, NX] + end = [start[i] + count[i] for i in range(2)] - # write to variable + # write to variable var_yx + var_yx[start[0]:end[0], start[1]:end[1]] = buf_yx + + # Equivalently, below uses function call var_yx.put_var_all(buf_yx, start = start, count = count) if verbose and rank == 0: @@ -156,6 +152,18 @@ def pnetcdf_io(filename): f.close() +def parse_help(): + help_flag = "-h" in sys.argv or "--help" in sys.argv + if help_flag and rank == 0: + help_text = ( + "Usage: {} [-h] | [-q] [file_name]\n" + " [-h] Print help\n" + " [-q] Quiet mode (reports when fail)\n" + " [filename] (Optional) output netCDF file name\n" + ).format(sys.argv[0]) + print(help_text) + return help_flag + if __name__ == "__main__": comm = MPI.COMM_WORLD rank = comm.Get_rank() diff --git a/examples/put_vara.py b/examples/put_vara.py index 1630d0b..2936cb4 100644 --- a/examples/put_vara.py +++ b/examples/put_vara.py @@ -56,20 +56,6 @@ import pnetcdf -def parse_help(): - help_flag = "-h" in sys.argv or "--help" in sys.argv - if help_flag and rank == 0: - help_text = ( - "Usage: {} [-h] | [-q] [file_name]\n" - " [-h] Print help\n" - " [-q] Quiet mode (reports when fail)\n" - " [-k format] file format: 1 for CDF-1, 2 for CDF-2, 5 for CDF-5\n" - " [filename] (Optional) output netCDF file name\n" - ).format(sys.argv[0]) - print(help_text) - return help_flag - - def pnetcdf_io(filename, file_format): NY = 10 NX = 4 @@ -99,6 +85,9 @@ def pnetcdf_io(filename, file_format): str_att = comm.bcast(str_att, root=0) # write a global attribute + f.history = str_att + + # Equivalently, below uses function call f.put_att('history',str_att) # Define dimensions @@ -108,14 +97,25 @@ def pnetcdf_io(filename, file_format): # Define a 2D variable of integer type var = f.def_var("var", pnetcdf.NC_INT, (dim_y, dim_x)) - # Add attributes to the variable + # Add an attribute of string type to the variable str_att = "example attribute of type text." + var.str_att_name = str_att + + # Equivalently, below uses function call var.put_att("str_att_name", str_att) + # Add an attribute of float type to the variable float_att = np.arange(8, dtype = 'f4') + var.float_att_name = float_att + + # Equivalently, below uses function call var.put_att("float_att_name", float_att) + # Add an attribute of int16 type to the variable short_att = np.int16(1000) + var.short_att_name = short_att + + # Equivalently, below uses function call var.put_att("short_att_name", short_att) # Exit the define mode @@ -124,14 +124,31 @@ def pnetcdf_io(filename, file_format): # initialize write buffer buf = np.zeros(shape = (NY, NX), dtype = "i4") + rank - # Write data to the variable - # var.put_var_all(buf, start = starts, count = counts) - var[0:NY, NX*rank:NX*rank+NX] = buf + # Write to the variable + end = [start[i] + count[i] for i in range(2)] + var[start[0]:end[0], start[1]:end[1]] = buf + + # Equivalently, below uses function call + var.put_var_all(buf, start = start, count = count) # Close the file f.close() +def parse_help(): + help_flag = "-h" in sys.argv or "--help" in sys.argv + if help_flag and rank == 0: + help_text = ( + "Usage: {} [-h] | [-q] [file_name]\n" + " [-h] Print help\n" + " [-q] Quiet mode (reports when fail)\n" + " [-k format] file format: 1 for CDF-1, 2 for CDF-2, 5 for CDF-5\n" + " [filename] (Optional) output netCDF file name\n" + ).format(sys.argv[0]) + print(help_text) + return help_flag + + if __name__ == "__main__": comm = MPI.COMM_WORLD rank = comm.Get_rank() From 64e8090f09d2b9f10eb64998c6264d42fd4c2e9e Mon Sep 17 00:00:00 2001 From: wkliao Date: Thu, 5 Sep 2024 13:59:50 -0500 Subject: [PATCH 4/6] make function of example codes appearing on top --- examples/create_open.py | 24 ++++++++++---------- examples/flexible_api.py | 47 ++++++++++++++++++++-------------------- examples/get_info.py | 26 +++++++++++----------- examples/put_varn_int.py | 28 ++++++++++++------------ examples/transpose.py | 30 ++++++++++++------------- examples/transpose2D.py | 28 ++++++++++++------------ 6 files changed, 92 insertions(+), 91 deletions(-) diff --git a/examples/create_open.py b/examples/create_open.py index e64e2e8..009dd59 100644 --- a/examples/create_open.py +++ b/examples/create_open.py @@ -21,18 +21,6 @@ from mpi4py import MPI import pnetcdf -def parse_help(): - help_flag = "-h" in sys.argv or "--help" in sys.argv - if help_flag and rank == 0: - help_text = ( - "Usage: {} [-h] | [-q] [file_name]\n" - " [-h] Print help\n" - " [-q] Quiet mode (reports when fail)\n" - " [filename] (Optional) output netCDF file name\n" - ).format(sys.argv[0]) - print(help_text) - return help_flag - def pnetcdf_io(filename): # create a new file using file clobber mode, i.e. flag "-w" @@ -51,6 +39,18 @@ def pnetcdf_io(filename): f.close() +def parse_help(): + help_flag = "-h" in sys.argv or "--help" in sys.argv + if help_flag and rank == 0: + help_text = ( + "Usage: {} [-h] | [-q] [file_name]\n" + " [-h] Print help\n" + " [-q] Quiet mode (reports when fail)\n" + " [filename] (Optional) output netCDF file name\n" + ).format(sys.argv[0]) + print(help_text) + return help_flag + if __name__ == "__main__": comm = MPI.COMM_WORLD rank = comm.Get_rank() diff --git a/examples/flexible_api.py b/examples/flexible_api.py index 8b46686..13f6d3e 100644 --- a/examples/flexible_api.py +++ b/examples/flexible_api.py @@ -72,19 +72,6 @@ from mpi4py import MPI import pnetcdf -def parse_help(): - help_flag = "-h" in sys.argv or "--help" in sys.argv - if help_flag and rank == 0: - help_text = ( - "Usage: {} [-h] | [-q] [file_name]\n" - " [-h] Print help\n" - " [-q] Quiet mode (reports when fail)\n" - " [-k format] file format: 1 for CDF-1, 2 for CDF-2, 5 for CDF-5\n" - " [filename] (Optional) output netCDF file name\n" - ).format(sys.argv[0]) - print(help_text) - return help_flag - def pnetcdf_io(filename, file_format): NY = 5 NX = 5 @@ -124,9 +111,9 @@ def pnetcdf_io(filename, file_format): array_of_subsizes = np.array([NZ, NY]) array_of_start = np.array([ghost_len, ghost_len]) - subarray = MPI.INT.Create_subarray(array_of_sizes, \ - array_of_subsizes, \ - array_of_start, \ + subarray = MPI.INT.Create_subarray(array_of_sizes, + array_of_subsizes, + array_of_start, order=MPI.ORDER_C) subarray.Commit() @@ -139,12 +126,13 @@ def pnetcdf_io(filename, file_format): count = np.array([NZ, NY]) # calling a blocking flexible API using put_var_all() - var_zy.put_var_all(buf_zy, start = start, \ - count = count, \ - bufcount = 1, \ + var_zy.put_var_all(buf_zy, start = start, + count = count, + bufcount = 1, buftype = subarray) + # check if write buffer's contents are altered (should not be). for i in range(buffer_len): if buf_zy[i] != rank: print(f"Error at line {sys._getframe().f_lineno} in {__file__}: put buffer[{i}] is altered") @@ -158,7 +146,7 @@ def pnetcdf_io(filename, file_format): bufcount = 1, buftype = subarray) - # check contents of the get buffer + # check whether contents of the get buffer are expected for i in range(array_of_sizes[0]): for j in range(array_of_sizes[1]): index = i*array_of_sizes[1] + j @@ -200,7 +188,7 @@ def pnetcdf_io(filename, file_format): status = [None] f.wait_all(1, [req_id], status = status) - # check request error msg for each unsuccessful requests + # check request error for each unsuccessful requests if pnetcdf.strerrno(status[0]) != "NC_NOERR": print(f"Error on request {i}:", pnetcdf.strerror(status[0])) @@ -216,11 +204,11 @@ def pnetcdf_io(filename, file_format): # commit posted pending nonblocking requests f.wait_all(1, [req_id], status = status) - # check request error msg for each unsuccessful requests + # check request error for each unsuccessful requests if pnetcdf.strerrno(status[0]) != "NC_NOERR": print(f"Error on request {i}:", pnetcdf.strerror(status[0])) - # check the contents of iget buffer + # check the contents of read buffer for i in range(array_of_sizes[0]): for j in range(array_of_sizes[1]): index = i * array_of_sizes[1] + j @@ -236,6 +224,19 @@ def pnetcdf_io(filename, file_format): f.close() +def parse_help(): + help_flag = "-h" in sys.argv or "--help" in sys.argv + if help_flag and rank == 0: + help_text = ( + "Usage: {} [-h] | [-q] [file_name]\n" + " [-h] Print help\n" + " [-q] Quiet mode (reports when fail)\n" + " [-k format] file format: 1 for CDF-1, 2 for CDF-2, 5 for CDF-5\n" + " [filename] (Optional) output netCDF file name\n" + ).format(sys.argv[0]) + print(help_text) + return help_flag + if __name__ == "__main__": comm = MPI.COMM_WORLD rank = comm.Get_rank() diff --git a/examples/get_info.py b/examples/get_info.py index 8b67349..e9d9d64 100644 --- a/examples/get_info.py +++ b/examples/get_info.py @@ -38,19 +38,6 @@ from mpi4py import MPI import pnetcdf -def parse_help(): - help_flag = "-h" in sys.argv or "--help" in sys.argv - if help_flag and rank == 0: - help_text = ( - "Usage: {} [-h] | [-q] [file_name]\n" - " [-h] Print help\n" - " [-q] Quiet mode (reports when fail)\n" - " [filename] (Optional) output netCDF file name\n" - ).format(sys.argv[0]) - print(help_text) - return help_flag - - def print_info(info_used): nkeys = info_used.Get_nkeys() print("MPI File Info: nkeys =", nkeys) @@ -84,6 +71,19 @@ def pnetcdf_io(filename): f.close() +def parse_help(): + help_flag = "-h" in sys.argv or "--help" in sys.argv + if help_flag and rank == 0: + help_text = ( + "Usage: {} [-h] | [-q] [file_name]\n" + " [-h] Print help\n" + " [-q] Quiet mode (reports when fail)\n" + " [filename] (Optional) output netCDF file name\n" + ).format(sys.argv[0]) + print(help_text) + return help_flag + + if __name__ == "__main__": comm = MPI.COMM_WORLD rank = comm.Get_rank() diff --git a/examples/put_varn_int.py b/examples/put_varn_int.py index 5a082de..bea82e2 100644 --- a/examples/put_varn_int.py +++ b/examples/put_varn_int.py @@ -42,20 +42,6 @@ import pnetcdf -def parse_help(): - help_flag = "-h" in sys.argv or "--help" in sys.argv - if help_flag and rank == 0: - help_text = ( - "Usage: {} [-h] | [-q] [file_name]\n" - " [-h] Print help\n" - " [-q] Quiet mode (reports when fail)\n" - " [-k format] file format: 1 for CDF-1, 2 for CDF-2, 5 for CDF-5\n" - " [filename] (Optional) output netCDF file name\n" - ).format(sys.argv[0]) - print(help_text) - return help_flag - - def pnetcdf_io(file_name, file_format): NY = 4 NX = 10 @@ -159,6 +145,20 @@ def pnetcdf_io(file_name, file_format): f.close() +def parse_help(): + help_flag = "-h" in sys.argv or "--help" in sys.argv + if help_flag and rank == 0: + help_text = ( + "Usage: {} [-h] | [-q] [file_name]\n" + " [-h] Print help\n" + " [-q] Quiet mode (reports when fail)\n" + " [-k format] file format: 1 for CDF-1, 2 for CDF-2, 5 for CDF-5\n" + " [filename] (Optional) output netCDF file name\n" + ).format(sys.argv[0]) + print(help_text) + return help_flag + + if __name__ == "__main__": comm = MPI.COMM_WORLD rank = comm.Get_rank() diff --git a/examples/transpose.py b/examples/transpose.py index 4428c14..328f566 100644 --- a/examples/transpose.py +++ b/examples/transpose.py @@ -29,21 +29,6 @@ from mpi4py import MPI import pnetcdf -def parse_help(): - help_flag = "-h" in sys.argv or "--help" in sys.argv - if help_flag and rank == 0: - help_text = ( - "Usage: {} [-h] | [-q] [file_name]\n" - " [-h] Print help\n" - " [-q] Quiet mode (reports when fail)\n" - " [-k format] file format: 1 for CDF-1, 2 for CDF-2, 5 for CDF-5\n" - " [-l len] size of each dimension of the local array\n" - " [filename] (Optional) output netCDF file name\n" - ).format(sys.argv[0]) - print(help_text) - return help_flag - - def pnetcdf_io(filename, file_format, length): NDIMS = 3 @@ -164,6 +149,21 @@ def pnetcdf_io(filename, file_format, length): f.close() +def parse_help(): + help_flag = "-h" in sys.argv or "--help" in sys.argv + if help_flag and rank == 0: + help_text = ( + "Usage: {} [-h] | [-q] [file_name]\n" + " [-h] Print help\n" + " [-q] Quiet mode (reports when fail)\n" + " [-k format] file format: 1 for CDF-1, 2 for CDF-2, 5 for CDF-5\n" + " [-l len] size of each dimension of the local array\n" + " [filename] (Optional) output netCDF file name\n" + ).format(sys.argv[0]) + print(help_text) + return help_flag + + if __name__ == "__main__": comm = MPI.COMM_WORLD rank = comm.Get_rank() diff --git a/examples/transpose2D.py b/examples/transpose2D.py index a7e7c10..4442d62 100644 --- a/examples/transpose2D.py +++ b/examples/transpose2D.py @@ -51,20 +51,6 @@ from mpi4py import MPI import pnetcdf -def parse_help(): - help_flag = "-h" in sys.argv or "--help" in sys.argv - if help_flag and rank == 0: - help_text = ( - "Usage: {} [-h] | [-q] [file_name]\n" - " [-h] Print help\n" - " [-q] Quiet mode (reports when fail)\n" - " [-k format] file format: 1 for CDF-1, 2 for CDF-2, 5 for CDF-5\n" - " [-l len] size of each dimension of the local array\n" - " [filename] (Optional) output netCDF file name\n" - ).format(sys.argv[0]) - print(help_text) - return help_flag - def pnetcdf_io(filename, file_format, length): NDIMS = 2 @@ -147,6 +133,20 @@ def pnetcdf_io(filename, file_format, length): f.close() +def parse_help(): + help_flag = "-h" in sys.argv or "--help" in sys.argv + if help_flag and rank == 0: + help_text = ( + "Usage: {} [-h] | [-q] [file_name]\n" + " [-h] Print help\n" + " [-q] Quiet mode (reports when fail)\n" + " [-k format] file format: 1 for CDF-1, 2 for CDF-2, 5 for CDF-5\n" + " [-l len] size of each dimension of the local array\n" + " [filename] (Optional) output netCDF file name\n" + ).format(sys.argv[0]) + print(help_text) + return help_flag + if __name__ == "__main__": comm = MPI.COMM_WORLD rank = comm.Get_rank() From ad72a65c212bc72fe0efb583a65746d12910ad95 Mon Sep 17 00:00:00 2001 From: wkliao Date: Thu, 5 Sep 2024 14:00:17 -0500 Subject: [PATCH 5/6] update README.md and Makefile --- examples/MNIST/Makefile | 1 + examples/Makefile | 5 +++-- examples/README.md | 30 ++++++++++++++++-------------- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/examples/MNIST/Makefile b/examples/MNIST/Makefile index 83bbac0..021f178 100644 --- a/examples/MNIST/Makefile +++ b/examples/MNIST/Makefile @@ -52,6 +52,7 @@ clean: rm -f mnist_main.py rm -f $(MNIST_DATASETS) rm -f $(MNIST_DATASETS_GZ) + rm -rf __pycache__ .PHONY: all check ptests clean diff --git a/examples/Makefile b/examples/Makefile index 45b0d32..b9d4808 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -11,8 +11,6 @@ check_PROGRAMS = collective_write.py \ ghost_cell.py \ global_attribute.py \ hints.py \ - nonblocking_write_def.py \ - nonblocking_write.py \ put_varn_int.py \ transpose2D.py \ transpose.py \ @@ -28,10 +26,12 @@ OUTPUT_DIR = _tmp_output all: check: ptest4 + cd nonblocking && make check cd Pytorch_DDP && make check cd MNIST && make check ptests: ptest3 ptest4 ptest8 + cd nonblocking && make ptests cd Pytorch_DDP && make ptests cd MNIST && make ptests @@ -61,6 +61,7 @@ ptest8: clean: rm -rf ${OUTPUT_DIR} + cd nonblocking && make clean cd Pytorch_DDP && make clean cd MNIST && make clean diff --git a/examples/README.md b/examples/README.md index 1a75257..a40f2c2 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,8 +1,8 @@ # PnetCDF-python examples This directory contains example python programs that make use of PnetCDF to -perform file I/O. Detailed description of each program and run instructions can -be found at the beginning of each file. +perform file I/O in parallel. Detailed description of each program and run +instructions can be found at the beginning of each file. --- ### Running individual example programs @@ -20,7 +20,7 @@ be found at the beginning of each file. ### Overview of Test Programs * [MNIST](./MNIST) - + This directory contains an example of + + A directory contains an example of [MNIST](https://github.com/pytorch/examples/tree/main/mnist), using Pytorch module `DistributedDataParallel` for parallel training and `PnetCDF-Python` for reading data from a NetCDF files. @@ -29,6 +29,19 @@ be found at the beginning of each file. + A directory contains examples that make use of Pytorch Distributed Data Parallel module to run python programs in parallel. +* [nonblocking](./nonblocking) + + A directory contains examples that make use of nonblocking I/O APIs. + * [nonblocking_write.py](./nonblocking/nonblocking_write.py]) -- + Similar to `collective_write.py`, this example uses nonblocking APIs + instead. It creates a netcdf file in CDF-5 format and writes a number of + 3D integer non-record variables. + + * [nonblocking_write_def.py](./nonblocking/nonblocking_write_def.py]) -- + This example is the same as `nonblocking_write.py` expect all nonblocking + write requests (calls to `iput` and `bput`) are posted in define mode. It + creates a netcdf file in CDF-5 format and writes a number of 3D integer + non-record variables. + * [collective_write.py](./collective_write.py) + This example writes multiple 3D subarrays to non-record variables of int type using collective I/O mode. @@ -43,17 +56,6 @@ be found at the beginning of each file. shows how to use to `Variable` method get_var() read a 2D 4-byte integer array in parallel. -* [nonblocking_write.py](./nonblocking_write.py) - + Similar to `collective_write.py`, this example uses nonblocking APIs - instead. It creates a netcdf file in CDF-5 format and writes a number of 3D - integer non-record variables. - -* [nonblocking_write_def.py](./nonblocking_write_def.py) - + This example is the same as `nonblocking_write.py` expect all nonblocking - write requests (calls to `iput` and `bput`) are posted in define mode. It - creates a netcdf file in CDF-5 format and writes a number of 3D integer - non-record variables. - * [create_open.py](./create_open.py) + This example shows how to use `File` class constructor to create a NetCDF file and to open the file for read only. From 40629e134f9da087f128de8e064b73088bcb5e1d Mon Sep 17 00:00:00 2001 From: wkliao Date: Thu, 5 Sep 2024 14:04:45 -0500 Subject: [PATCH 6/6] delet folder test/__pycache__ in make clean --- test/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/test/Makefile b/test/Makefile index 3050ece..3d64df1 100644 --- a/test/Makefile +++ b/test/Makefile @@ -83,6 +83,7 @@ ptest8: clean: rm -rf ${OUTPUT_DIR} + rm -rf __pycache__ .PHONY: all check ptests ptest3 ptest4 ptest8 clean