Rework memory management for integer arrays

src/MPI/Comm.pyx

@@ -1265,11 +1265,13 @@ cdef class Intracomm(Comm):
         """
         Create cartesian communicator
         """
-        cdef int ndims = 0, *idims = NULL
-        dims = getarray_int(dims, &ndims, &idims)
-        if periods is None: periods = [False] * ndims
+        cdef int ndims = 0
+        ndims = <int>len(dims)
+        cdef int *idims = NULL
+        dims = asarray_int(dims, ndims, &idims)
         cdef int *iperiods = NULL
-        periods = chkarray_int(periods, ndims, &iperiods)
+        if periods is None: periods = [False] * ndims
+        periods = asarray_int(periods, ndims, &iperiods)
         #
         cdef Cartcomm comm = <Cartcomm>Cartcomm.__new__(Cartcomm)
         with nogil: CHKERR( MPI_Cart_create(
@@ -1330,10 +1332,11 @@ cdef class Intracomm(Comm):
         cdef int nv = 0, ne = 0, i = 0
         cdef int *isource = NULL, *idegree = NULL,
         cdef int *idest = NULL, *iweight = MPI_UNWEIGHTED
-        sources = getarray_int(sources, &nv, &isource)
-        degrees = chkarray_int(degrees,  nv, &idegree)
+        nv = <int>len(sources)
+        sources = asarray_int(sources, nv, &isource)
+        degrees = asarray_int(degrees, nv, &idegree)
         for i from 0 <= i < nv: ne += idegree[i]
-        destinations = chkarray_int(destinations, ne, &idest)
+        destinations = asarray_int(destinations, ne, &idest)
         weights = asarray_weights(weights, ne, &iweight)
         cdef MPI_Info cinfo = arg_Info(info)
         #
@@ -1600,7 +1603,7 @@ cdef class Intracomm(Comm):
             if args is not None:
                 tmp1 = asarray_argv(args, &argv)
         if errcodes is not None:
-            tmp2 = newarray_int(maxprocs, &ierrcodes)
+            tmp2 = mkarray_int(maxprocs, &ierrcodes)
         #
         cdef Intercomm comm = <Intercomm>Intercomm.__new__(Intercomm)
         with nogil: CHKERR( MPI_Comm_spawn(
@@ -1627,7 +1630,7 @@ cdef class Intracomm(Comm):
         #
         cdef int rank = MPI_UNDEFINED
         CHKERR( MPI_Comm_rank(self.ob_mpi, &rank) )
-        cdef object tmp1, tmp2, tmp3, tmp4, tmp5
+        cdef tmp1, tmp2, tmp3, tmp4, tmp5
         cdef Py_ssize_t i=0, n=0
         if root == rank:
             count = <int>len(command)
@@ -1641,7 +1644,7 @@ cdef class Intracomm(Comm):
                 tmp3 = asarray_nprocs(maxprocs, count, &imaxprocs)
             for i from 0 <= i < count:
                 n += imaxprocs[i]
-            tmp5 = newarray_int(n, &ierrcodes)
+            tmp5 = mkarray_int(n, &ierrcodes)
         #
         cdef Intercomm comm = <Intercomm>Intercomm.__new__(Intercomm)
         with nogil: CHKERR( MPI_Comm_spawn_multiple(
@@ -1740,11 +1743,11 @@ cdef class Cartcomm(Intracomm):
         cdef int ndim = 0
         CHKERR( MPI_Cartdim_get(self.ob_mpi, &ndim) )
         cdef int *idims = NULL
-        cdef tmp1 = newarray_int(ndim, &idims)
+        cdef tmp1 = mkarray_int(ndim, &idims)
         cdef int *iperiods = NULL
-        cdef tmp2 = newarray_int(ndim, &iperiods)
+        cdef tmp2 = mkarray_int(ndim, &iperiods)
         cdef int *icoords = NULL
-        cdef tmp3 = newarray_int(ndim, &icoords)
+        cdef tmp3 = mkarray_int(ndim, &icoords)
         CHKERR( MPI_Cart_get(self.ob_mpi, ndim, idims, iperiods, icoords) )
         cdef int i = 0
         cdef object dims    = [idims[i]    for i from 0 <= i < ndim]
@@ -1782,7 +1785,7 @@ cdef class Cartcomm(Intracomm):
         """
         cdef int ndim = 0, *icoords = NULL
         CHKERR( MPI_Cartdim_get( self.ob_mpi, &ndim) )
-        coords = chkarray_int(coords, ndim, &icoords)
+        coords = asarray_int(coords, ndim, &icoords)
         cdef int rank = MPI_PROC_NULL
         CHKERR( MPI_Cart_rank(self.ob_mpi, icoords, &rank) )
         return rank
@@ -1791,10 +1794,11 @@ cdef class Cartcomm(Intracomm):
         """
         Translate ranks to logical coordinates
         """
-        cdef int ndim = 0, *icoords = NULL
+        cdef int i = 0, ndim = 0, *icoords = NULL
         CHKERR( MPI_Cartdim_get(self.ob_mpi, &ndim) )
-        cdef object coords = newarray_int(ndim, &icoords)
+        cdef tmp = mkarray_int(ndim, &icoords)
         CHKERR( MPI_Cart_coords(self.ob_mpi, rank, ndim, icoords) )
+        cdef object coords = [icoords[i] for i from 0 <= i < ndim]
         return coords
 
     # Cartesian Shift Function
@@ -1819,7 +1823,7 @@ cdef class Cartcomm(Intracomm):
         """
         cdef int ndim = 0, *iremdims = NULL
         CHKERR( MPI_Cartdim_get(self.ob_mpi, &ndim) )
-        remain_dims = chkarray_int(remain_dims, ndim, &iremdims)
+        remain_dims = asarray_int(remain_dims, ndim, &iremdims)
         cdef Cartcomm comm = <Cartcomm>Cartcomm.__new__(Cartcomm)
         with nogil: CHKERR( MPI_Cart_sub(self.ob_mpi, iremdims, &comm.ob_mpi) )
         return comm
@@ -1834,9 +1838,10 @@ cdef class Cartcomm(Intracomm):
         calling process on the physical machine
         """
         cdef int ndims = 0, *idims = NULL, *iperiods = NULL
-        dims = getarray_int(dims, &ndims, &idims)
+        ndims = <int>len(dims)
+        dims = asarray_int(dims, ndims, &idims)
         if periods is None: periods = [False] * ndims
-        periods = chkarray_int(periods, ndims, &iperiods)
+        periods = asarray_int(periods, ndims, &iperiods)
         cdef int rank = MPI_PROC_NULL
         CHKERR( MPI_Cart_map(self.ob_mpi, ndims, idims, iperiods, &rank) )
         return rank
@@ -1849,14 +1854,15 @@ def Compute_dims(int nnodes, dims):
     Return a balanced distribution of
     processes per coordinate direction
     """
-    cdef int ndims=0, *idims = NULL
+    cdef int i = 0, ndims = 0, *idims = NULL
     try:
         ndims = <int>len(dims)
     except:
         ndims = dims
         dims = [0] * ndims
-    dims = chkarray_int(dims, ndims, &idims)
+    cdef tmp = asarray_int(dims, ndims, &idims)
     CHKERR( MPI_Dims_create(nnodes, ndims, idims) )
+    dims = [idims[i] for i from 0 <= i < ndims]
     return dims
 
 
@@ -1906,9 +1912,9 @@ cdef class Graphcomm(Intracomm):
         cdef int nindex = 0, nedges = 0
         CHKERR( MPI_Graphdims_get( self.ob_mpi, &nindex, &nedges) )
         cdef int *iindex = NULL
-        cdef tmp1 = newarray_int(nindex, &iindex)
+        cdef tmp1 = mkarray_int(nindex, &iindex)
         cdef int *iedges = NULL
-        cdef tmp2 = newarray_int(nedges, &iedges)
+        cdef tmp2 = mkarray_int(nedges, &iedges)
         CHKERR( MPI_Graph_get(self.ob_mpi, nindex, nedges, iindex, iedges) )
         cdef int i = 0
         cdef object index = [iindex[i] for i from 0 <= i < nindex]
@@ -1951,14 +1957,12 @@ cdef class Graphcomm(Intracomm):
         """
         Return list of neighbors of a process
         """
-        cdef int nneighbors = 0
+        cdef int i = 0, nneighbors = 0, *ineighbors = NULL
         CHKERR( MPI_Graph_neighbors_count(
                 self.ob_mpi, rank, &nneighbors) )
-        cdef int *ineighbors = NULL
-        cdef tmp = newarray_int(nneighbors, &ineighbors)
+        cdef tmp = mkarray_int(nneighbors, &ineighbors)
         CHKERR( MPI_Graph_neighbors(
                 self.ob_mpi, rank, nneighbors, ineighbors) )
-        cdef int i = 0
         cdef object neighbors = [ineighbors[i] for i from 0 <= i < nneighbors]
         return neighbors
 
@@ -2027,13 +2031,13 @@ cdef class Distgraphcomm(Intracomm):
         cdef int *sourceweights = MPI_UNWEIGHTED
         cdef int *destweights   = MPI_UNWEIGHTED
         cdef tmp1, tmp2, tmp3, tmp4
-        tmp1 = newarray_int(maxindegree,  &sources)
-        tmp2 = newarray_int(maxoutdegree, &destinations)
+        tmp1 = mkarray_int(maxindegree,  &sources)
+        tmp2 = mkarray_int(maxoutdegree, &destinations)
         cdef int i = 0
         if weighted:
-            tmp3 = newarray_int(maxindegree,  &sourceweights)
+            tmp3 = mkarray_int(maxindegree,  &sourceweights)
             for i from 0 <= i < maxindegree:  sourceweights[i] = 1
-            tmp4 = newarray_int(maxoutdegree, &destweights)
+            tmp4 = mkarray_int(maxoutdegree, &destweights)
             for i from 0 <= i < maxoutdegree: destweights[i]   = 1
         #
         CHKERR( MPI_Dist_graph_neighbors(

src/MPI/Datatype.pyx

@@ -249,9 +249,9 @@ cdef class Datatype:
         """
         cdef int ndims = 0, *isizes = NULL
         cdef int *isubsizes = NULL, *istarts = NULL
-        sizes    = getarray_int(sizes,    &ndims, &isizes   )
-        subsizes = chkarray_int(subsizes,  ndims, &isubsizes)
-        starts   = chkarray_int(starts,    ndims, &istarts  )
+        sizes    = getarray_int(sizes,   &ndims, &isizes   )
+        subsizes = chkarray_int(subsizes, ndims, &isubsizes)
+        starts   = chkarray_int(starts,   ndims, &istarts  )
         cdef int iorder = MPI_ORDER_C
         if order is not None: iorder = order
         #
@@ -428,7 +428,7 @@ cdef class Datatype:
         # get the datatype envelope
         cdef int ni = 0, na = 0, nd = 0, combiner = MPI_UNDEFINED
         CHKERR( MPI_Type_get_envelope(self.ob_mpi, &ni, &na, &nd, &combiner) )
-        # return self immediatly for named datatypes
+        # return self immediately for named datatypes
         if combiner == MPI_COMBINER_NAMED: return self
         # get the datatype contents
         cdef int *i = NULL

src/MPI/Group.pyx

@@ -63,9 +63,9 @@ cdef class Group:
         """
         cdef MPI_Group grp1 = MPI_GROUP_NULL
         cdef MPI_Group grp2 = MPI_GROUP_NULL
-        cdef int n = 0, *iranks1 = NULL, *iranks2 = NULL
-        cdef object ranks_ = getarray_int(ranks1, &n, &iranks1)
-        cdef object ranks2 = newarray_int(n, &iranks2)
+        cdef int i = 0, n = 0, *iranks1 = NULL, *iranks2 = NULL
+        cdef tmp1 = getarray_int(ranks1, &n, &iranks1)
+        cdef tmp2 = newarray_int(n, &iranks2)
         #
         grp1 = group1.ob_mpi
         if group2 is not None:
@@ -79,6 +79,7 @@ cdef class Group:
             if group2 is None:
                 CHKERR( MPI_Group_free(&grp2) )
         #
+        cdef object ranks2 = [iranks2[i] for i from 0 <= i < n]
         return ranks2
 
     @classmethod

src/MPI/Request.pyx

@@ -154,44 +154,48 @@ cdef class Request:
         cdef int outcount = MPI_UNDEFINED, *iindices = NULL
         cdef MPI_Status *istatuses = MPI_STATUSES_IGNORE
         #
-        cdef tmp = acquire_rs(requests, statuses,
-                              &incount, &irequests, &istatuses)
-        cdef object indices = newarray_int(incount, &iindices)
+        cdef tmp1 = acquire_rs(requests, statuses,
+                               &incount, &irequests, &istatuses)
+        cdef tmp2 = mkarray_int(incount, &iindices)
         try:
             with nogil: CHKERR( MPI_Waitsome(
                 incount, irequests, &outcount, iindices, istatuses) )
         finally:
             release_rs(requests, statuses, incount, irequests, istatuses)
         #
+        cdef int i = 0
+        cdef object indices
         if outcount == MPI_UNDEFINED:
-            del indices[:]
+            indices = []
         else:
-            del indices[outcount:]
+            indices = [iindices[i] for i from 0 <= i < outcount]
         return (outcount, indices)
 
     @classmethod
     def Testsome(cls, requests, statuses=None):
         """
         Test for completion of some previously initiated requests
         """
-        cdef int incount = <int>len(requests)
+        cdef int incount = 0
         cdef MPI_Request *irequests = NULL
         cdef int outcount = MPI_UNDEFINED, *iindices = NULL
         cdef MPI_Status *istatuses = MPI_STATUSES_IGNORE
         #
-        cdef tmp = acquire_rs(requests, statuses,
-                              &incount, &irequests, &istatuses)
-        cdef object indices = newarray_int(incount, &iindices)
+        cdef tmp1 = acquire_rs(requests, statuses,
+                               &incount, &irequests, &istatuses)
+        cdef tmp2 = mkarray_int(incount, &iindices)
         try:
             with nogil: CHKERR( MPI_Testsome(
                 incount, irequests, &outcount, iindices, istatuses) )
         finally:
             release_rs(requests, statuses, incount, irequests, istatuses)
         #
+        cdef int i = 0
+        cdef object indices
         if outcount == MPI_UNDEFINED:
-            del indices[:]
+            indices = []
         else:
-            del indices[outcount:]
+            indices = [iindices[i] for i from 0 <= i < outcount]
         return (outcount, indices)
 
     # Cancel

src/MPI/asarray.pxi

@@ -1,39 +1,29 @@
 # -----------------------------------------------------------------------------
 
-cdef type arraytype
-from array import array as arraytype
-
 cdef inline object newarray_int(Py_ssize_t n, int **p):
-    cdef object ary = arraytype('i', [0])
-    if n < 1: del ary[0]
-    if n > 1: ary *= n
-    cdef int *base = NULL
-    cdef Py_ssize_t size = 0
-    if PYPY:
-        getbuffer_w(ary, <void**>&base, NULL)
-    else:
-        PyObject_AsWriteBuffer(ary, <void**>&base, &size)
-    if p != NULL: p[0] = base
-    return ary
+    return allocate(n, sizeof(int), <void**>p)
 
 cdef inline object getarray_int(object ob, int *n, int **p):
     cdef int *base = NULL
     cdef Py_ssize_t i = 0, size = len(ob)
-    cdef object ary = newarray_int(size, &base)
+    cdef object mem = newarray_int(size, &base)
     for i from 0 <= i < size: base[i] = ob[i]
-    if n != NULL: n[0] = <int> size # XXX overflow?
-    if p != NULL: p[0] = base
-    return ary
+    n[0] = <int>  size # XXX overflow?
+    p[0] = <int*> base
+    return mem
 
 cdef inline object chkarray_int(object ob, Py_ssize_t size, int **p):
     cdef int n = 0
-    cdef object ary = getarray_int(ob, &n, p)
+    cdef object mem = getarray_int(ob, &n, p)
     if size != <Py_ssize_t>n: raise ValueError(
         "expecting %d items, got %d" % (size, n))
-    return ary
+    return mem
 
 # -----------------------------------------------------------------------------
 
+cdef inline object mkarray_int(Py_ssize_t size, int **p):
+     return allocate(size, sizeof(int), <void**>p)
+
 cdef inline object asarray_int(object sequence,
                                Py_ssize_t size, int **p):
      cdef int *array = NULL

src/MPI/asmemory.pxi

@@ -3,11 +3,11 @@
 cdef extern from "Python.h":
     enum: PY_SSIZE_T_MAX
     void *PyMem_Malloc(size_t)
-    void *PyMem_Realloc(void *, size_t)
-    void PyMem_Free(void *)
+    void *PyMem_Realloc(void*, size_t)
+    void PyMem_Free(void*)
 
 cdef extern from "Python.h":
-    object PyLong_FromVoidPtr(void *)
+    object PyLong_FromVoidPtr(void*)
     void*  PyLong_AsVoidPtr(object)
 
 #------------------------------------------------------------------------------

src/MPI/commimpl.pxi

@@ -149,13 +149,13 @@ cdef object asarray_weights(object weights, int nweight, int **iweight):
             iweight[0] = MPI_WEIGHTS_EMPTY
             return None
         else:
-            return newarray_int(nweight, iweight)
+            return mkarray_int(nweight, iweight)
     #
     if weights is __UNWEIGHTED__:
         iweight[0] = MPI_UNWEIGHTED
         return None
     #
-    return chkarray_int(weights, nweight, iweight)
+    return asarray_int(weights, nweight, iweight)
 
 # -----------------------------------------------------------------------------