Add python style of subarray I/O to example programs

examples/MNIST/Makefile

@@ -52,6 +52,7 @@ clean:
 	rm -f mnist_main.py
 	rm -f $(MNIST_DATASETS)
 	rm -f $(MNIST_DATASETS_GZ)
+	rm -rf __pycache__
 
 .PHONY: all check ptests clean
 

examples/Makefile

@@ -11,8 +11,6 @@ check_PROGRAMS = collective_write.py \
                  ghost_cell.py \
                  global_attribute.py \
                  hints.py \
-                 nonblocking_write_def.py \
-                 nonblocking_write.py \
                  put_varn_int.py \
                  transpose2D.py \
                  transpose.py \
@@ -28,10 +26,12 @@ OUTPUT_DIR = _tmp_output
 all:
 
 check: ptest4
+	cd nonblocking && make check
 	cd Pytorch_DDP && make check
 	cd MNIST && make check
 
 ptests: ptest3 ptest4 ptest8
+	cd nonblocking && make ptests
 	cd Pytorch_DDP && make ptests
 	cd MNIST && make ptests
 
@@ -61,6 +61,7 @@ ptest8:
 
 clean:
 	rm -rf ${OUTPUT_DIR}
+	cd nonblocking && make clean
 	cd Pytorch_DDP && make clean
 	cd MNIST && make clean
 

examples/README.md

@@ -1,8 +1,8 @@
 # PnetCDF-python examples
 
 This directory contains example python programs that make use of PnetCDF to
-perform file I/O. Detailed description of each program and run instructions can
-be found at the beginning of each file.
+perform file I/O in parallel. Detailed description of each program and run
+instructions can be found at the beginning of each file.
 
 ---
 ### Running individual example programs
@@ -20,7 +20,7 @@ be found at the beginning of each file.
 ### Overview of Test Programs
 
 * [MNIST](./MNIST)
-  + This directory contains an example of
+  + A directory contains an example of
     [MNIST](https://github.com/pytorch/examples/tree/main/mnist),
     using Pytorch module `DistributedDataParallel` for parallel training and
     `PnetCDF-Python` for reading data from a NetCDF files.
@@ -29,6 +29,19 @@ be found at the beginning of each file.
   + A directory contains examples that make use of Pytorch Distributed Data
     Parallel module to run python programs in parallel.
 
+* [nonblocking](./nonblocking)
+  + A directory contains examples that make use of nonblocking I/O APIs.
+    * [nonblocking_write.py](./nonblocking/nonblocking_write.py]) --
+      Similar to `collective_write.py`, this example uses nonblocking APIs
+      instead. It creates a netcdf file in CDF-5 format and writes a number of
+      3D integer non-record variables.
+
+    * [nonblocking_write_def.py](./nonblocking/nonblocking_write_def.py]) --
+      This example is the same as `nonblocking_write.py` expect all nonblocking
+      write requests (calls to `iput` and `bput`) are posted in define mode. It
+      creates a netcdf file in CDF-5 format and writes a number of 3D integer
+      non-record variables.
+
 * [collective_write.py](./collective_write.py)
   + This example writes multiple 3D subarrays to non-record variables of int
     type using collective I/O mode.
@@ -43,17 +56,6 @@ be found at the beginning of each file.
     shows how to use to `Variable` method get_var() read a 2D 4-byte integer
     array in parallel.
 
-* [nonblocking_write.py](./nonblocking_write.py)
-  + Similar to `collective_write.py`, this example uses nonblocking APIs
-    instead. It creates a netcdf file in CDF-5 format and writes a number of 3D
-    integer non-record variables.
-
-* [nonblocking_write_def.py](./nonblocking_write_def.py)
-  + This example is the same as `nonblocking_write.py` expect all nonblocking
-    write requests (calls to `iput` and `bput`) are posted in define mode. It
-    creates a netcdf file in CDF-5 format and writes a number of 3D integer
-    non-record variables.
-
 * [create_open.py](./create_open.py)
   + This example shows how to use `File` class constructor to create a NetCDF
     file and to open the file for read only.

examples/collective_write.py

@@ -36,20 +36,6 @@
 from mpi4py import MPI
 import pnetcdf
 
-def parse_help():
-    help_flag = "-h" in sys.argv or "--help" in sys.argv
-    if help_flag and rank == 0:
-        help_text = (
-            "Usage: {} [-h] | [-q] [file_name]\n"
-            "       [-h] Print help\n"
-            "       [-q] Quiet mode (reports when fail)\n"
-            "       [-k format] file format: 1 for CDF-1, 2 for CDF-2, 5 for CDF-5\n"
-            "       [-l len] size of each dimension of the local array\n"
-            "       [filename] (Optional) output netCDF file name\n"
-        ).format(sys.argv[0])
-        print(help_text)
-    return help_flag
-
 def pnetcdf_io(filename, file_format, length):
     # number of dimensions
     NDIMS = 3
@@ -60,8 +46,8 @@ def pnetcdf_io(filename, file_format, length):
         print("Number of variables = ", NUM_VARS)
         print("Number of dimensions = ", NDIMS)
 
-    start = np.zeros(NDIMS, dtype=np.int32)
-    count = np.zeros(NDIMS, dtype=np.int32)
+    start  = np.zeros(NDIMS, dtype=np.int32)
+    count  = np.zeros(NDIMS, dtype=np.int32)
     gsizes = np.zeros(NDIMS, dtype=np.int32)
     buf = []
 
@@ -75,8 +61,10 @@ def pnetcdf_io(filename, file_format, length):
     for i in range(NDIMS):
         gsizes[i] = length * psizes[i]
         start[i] *= length
-        count[i] = length
-        bufsize *= length
+        count[i]  = length
+        bufsize  *= length
+
+    end = np.add(start, count)
 
     # Allocate buffer and initialize with non-zero numbers
     for i in range(NUM_VARS):
@@ -111,12 +99,30 @@ def pnetcdf_io(filename, file_format, length):
 
     # Collectively write one variable at a time
     for i in range(NUM_VARS):
+        # write using Python style subarray access
+        vars[i][start[0]:end[0], start[1]:end[1], start[2]:end[2]] = buf[i]
+
+        # Equivalently, below uses function call
         vars[i].put_var_all(buf[i], start = start, count = count)
 
     # Close the file
     f.close()
 
 
+def parse_help():
+    help_flag = "-h" in sys.argv or "--help" in sys.argv
+    if help_flag and rank == 0:
+        help_text = (
+            "Usage: {} [-h] | [-q] [file_name]\n"
+            "       [-h] Print help\n"
+            "       [-q] Quiet mode (reports when fail)\n"
+            "       [-k format] file format: 1 for CDF-1, 2 for CDF-2, 5 for CDF-5\n"
+            "       [-l len] size of each dimension of the local array\n"
+            "       [filename] (Optional) output netCDF file name\n"
+        ).format(sys.argv[0])
+        print(help_text)
+    return help_flag
+
 if __name__ == "__main__":
 
     comm = MPI.COMM_WORLD

examples/create_open.py

@@ -21,18 +21,6 @@
 from mpi4py import MPI
 import pnetcdf
 
-def parse_help():
-    help_flag = "-h" in sys.argv or "--help" in sys.argv
-    if help_flag and rank == 0:
-        help_text = (
-        "Usage: {} [-h] | [-q] [file_name]\n"
-            "       [-h] Print help\n"
-            "       [-q] Quiet mode (reports when fail)\n"
-            "       [filename] (Optional) output netCDF file name\n"
-        ).format(sys.argv[0])
-        print(help_text)
-    return help_flag
-
 def pnetcdf_io(filename):
 
     # create a new file using file clobber mode, i.e. flag "-w"
@@ -51,6 +39,18 @@ def pnetcdf_io(filename):
     f.close()
 
 
+def parse_help():
+    help_flag = "-h" in sys.argv or "--help" in sys.argv
+    if help_flag and rank == 0:
+        help_text = (
+        "Usage: {} [-h] | [-q] [file_name]\n"
+            "       [-h] Print help\n"
+            "       [-q] Quiet mode (reports when fail)\n"
+            "       [filename] (Optional) output netCDF file name\n"
+        ).format(sys.argv[0])
+        print(help_text)
+    return help_flag
+
 if __name__ == "__main__":
     comm = MPI.COMM_WORLD
     rank = comm.Get_rank()

examples/fill_mode.py

@@ -45,18 +45,6 @@
 from mpi4py import MPI
 import pnetcdf
 
-def parse_help():
-    help_flag = "-h" in sys.argv or "--help" in sys.argv
-    if help_flag and rank == 0:
-        help_text = (
-            "Usage: {} [-h] | [-q] [file_name]\n"
-            "       [-h] Print help\n"
-            "       [-q] Quiet mode (reports when fail)\n"
-            "       [filename] (Optional) output netCDF file name\n"
-        ).format(sys.argv[0])
-        print(help_text)
-    return help_flag
-
 def pnetcdf_io(filename):
 
     NY = 3
@@ -77,13 +65,13 @@ def pnetcdf_io(filename):
     global_nx = NX * nprocs
 
     # define dimensions
-    dim_xu = f.def_dim('REC_DIM', -1)
+    time = f.def_dim('REC_DIM', -1)
     dim_x = f.def_dim('X',global_nx)
     dim_y = f.def_dim('Y',global_ny)
 
     # define 2D variables of integer type
     fix_var = f.def_var("fix_var", pnetcdf.NC_INT, (dim_y, dim_x))
-    rec_var = f.def_var("rec_var", pnetcdf.NC_INT, (dim_xu, dim_x))
+    rec_var = f.def_var("rec_var", pnetcdf.NC_INT, (time, dim_x))
 
     # set the fill mode to NC_FILL for the entire file
     old_fillmode = f.set_fill(pnetcdf.NC_FILL)
@@ -110,12 +98,24 @@ def pnetcdf_io(filename):
     start = np.array([0, NX * rank])
     count = np.array([NY, NX])
 
+    if verbose:
+        print("rank: ", rank," global_ny=",global_ny," global_nx=",global_nx," start=",start," count=",count)
+
     # allocate user buffer
     buf = np.array([[rank] * NX] * NY).astype('i4')
+    for i in range(NY):
+        for j in range(NX):
+            buf[i, j] = 10 + rank*10 + i * NX + j
 
     # do not write the variable in full
     count[1] -= 1
-    fix_var.put_var_all(buf, start = start, count = count)
+
+    # write using Python style subarray access
+    end = np.add(start, count)
+    fix_var[start[0]:end[0], start[1]:end[1]] = buf[0:count[0], 0:count[1]]
+
+    # Equivalently, below uses function call
+    fix_var.put_var_all(buf[0:count[0], 0:count[1]], start = start, count = count)
 
     # check fill value
     no_fill, fill_value = fix_var.inq_fill()
@@ -126,20 +126,40 @@ def pnetcdf_io(filename):
     count[0] = 1
     rec_var.fill_rec(start[0])
 
-    # write to the 1st record
-    rec_var.put_var_all(buf, start = start, count = count)
+    # write to the 1st record, using Python style subarray access
+    end = np.add(start, count)
+    rec_var[start[0]:end[0], start[1]:end[1]] = buf[0:count[0], 0:count[1]]
+
+    # Equivalently, below uses function call
+    rec_var.put_var_all(buf[0:count[0], 0:count[1]], start = start, count = count)
 
     # fill the 2nd record of the record variable
     start[0] = 1
     rec_var.fill_rec(start[0])
 
-    # write to the 2nd record
-    rec_var.put_var_all(buf, start = start, count = count)
+    # write to the 2nd record using Python style subarray access
+    end = np.add(start, count)
+    rec_var[start[0]:end[0], start[1]:end[1]] = buf[0:count[0], 0:count[1]]
+
+    # Equivalently, below uses function call
+    rec_var.put_var_all(buf[0:count[0], 0:count[1]], start = start, count = count)
 
     # close file
     f.close()
 
 
+def parse_help():
+    help_flag = "-h" in sys.argv or "--help" in sys.argv
+    if help_flag and rank == 0:
+        help_text = (
+            "Usage: {} [-h] | [-q] [file_name]\n"
+            "       [-h] Print help\n"
+            "       [-q] Quiet mode (reports when fail)\n"
+            "       [filename] (Optional) output netCDF file name\n"
+        ).format(sys.argv[0])
+        print(help_text)
+    return help_flag
+
 if __name__ == "__main__":
     verbose = True