pystella/decomp.py
def share_halos(self, queue, fx):
        """
        Communicates halo data across all axes, imposing periodic boundary
        conditions.

        :arg queue: The :class:`pyopencl.CommandQueue` to enqueue kernels and copies.

        :arg fx: The :class:`pyopencl.array.Array` whose halo elements are to be
            synchronized across ranks.
            The number of halo layers on each face of the grid is fixed by
            :attr:`halo_shape`, while the shape of ``fx``
            (i.e., subtracting halo padding specified by :attr:`halo_shape`)
            is only fixed if ``rank_shape`` or ``grid_shape`` were
            passed at object creation).
        """

        h = self.halo_shape
        rank_shape = tuple(ni - 2 * hi for ni, hi in zip(fx.shape, h))

        if h[2] > 0:
            if self.proc_shape[2] == 1:
                evt, _ = self.pack_unpack_z_knl(queue, arr=fx)
            else:
                raise NotImplementedError("domain decomposition in z direction")

        if h[0] > 0:
            if self.proc_shape[0] == 1:
                evt, _ = self.pack_unpack_x_knl(queue, arr=fx)
            else:
                self._debug_barrier("all ranks beginning halo share in x")
                shape = (2, h[0], rank_shape[1] + 2*h[1], rank_shape[2] + 2*h[2])
                buf_dev, buf_send, buf_recv = self._get_or_create_halo_buffers(
                    queue, shape, fx.dtype)

                evt, _ = self.pack_x_knl(queue, arr=fx, buf=buf_dev)
                buf_dev.get(ary=buf_send)

                dest = self.rankID(self.rx-1, self.ry, self.rz)
                source = self.rankID(self.rx+1, self.ry, self.rz)
                self._debug(f"initiating first halo share in x: {dest=}, {source=}")
                self.comm.Sendrecv(sendbuf=buf_send[0], dest=dest,
                                   recvbuf=buf_recv[0], source=source)

                dest = self.rankID(self.rx+1, self.ry, self.rz)
                source = self.rankID(self.rx-1, self.ry, self.rz)
                self._debug(f"initiating second halo share in x: {dest=}, {source=}")
                self.comm.Sendrecv(sendbuf=buf_send[1], dest=dest,
                                   recvbuf=buf_recv[1], source=source)

                self._debug("finished halo share in x")
                buf_dev.set(buf_recv)
                evt, _ = self.unpack_x_knl(queue, arr=fx, buf=buf_dev)
                self._debug_barrier("all ranks finished halo share in x")

        if h[1] > 0:
            if self.proc_shape[1] == 1:
                evt, _ = self.pack_unpack_y_knl(queue, arr=fx)
            else:
                self._debug_barrier("all ranks beginning halo share in y")
                shape = (2, rank_shape[0] + 2*h[0], h[1], rank_shape[2] + 2*h[2])
                buf_dev, buf_send, buf_recv = self._get_or_create_halo_buffers(
                    queue, shape, fx.dtype)

                evt, _ = self.pack_y_knl(queue, arr=fx, buf=buf_dev)
                buf_dev.get(ary=buf_send)

                dest = self.rankID(self.rx, self.ry-1, self.rz)
                source = self.rankID(self.rx, self.ry+1, self.rz)
                self._debug(f"initiating first halo share in y: {dest=}, {source=}")
                self.comm.Sendrecv(sendbuf=buf_send[0], dest=dest,
                                   recvbuf=buf_recv[0], source=source)

                dest = self.rankID(self.rx, self.ry+1, self.rz)
                source = self.rankID(self.rx, self.ry-1, self.rz)
                self._debug(f"initiating second halo share in y: {dest=}, {source=}")
                self.comm.Sendrecv(sendbuf=buf_send[1], dest=dest,
                                   recvbuf=buf_recv[1], source=source)

                self._debug("finished halo share in y")
                buf_dev.set(buf_recv)
                evt, _ = self.unpack_y_knl(queue, arr=fx, buf=buf_dev)
                self._debug_barrier("all ranks finished halo share in y")

        Communicates halo data across all axes, imposing periodic boundary
        conditions.

        :arg queue: The :class:`pyopencl.CommandQueue` to enqueue kernels and copies.

        :arg fx: The :class:`pyopencl.array.Array` whose halo elements are to be
            synchronized across ranks.
            The number of halo layers on each face of the grid is fixed by
            :attr:`halo_shape`, while the shape of ``fx``
            (i.e., subtracting halo padding specified by :attr:`halo_shape`)
            is only fixed if ``rank_shape`` or ``grid_shape`` were
            passed at object creation).
        
Communicates halo data across all axes, imposing periodic boundary
conditions.
Communicates halo data across all axes imposing periodic boundary conditions .
def share_halos(self, queue, fx):
        h = self.halo_shape
        rank_shape = tuple(ni - 2 * hi for ni, hi in zip(fx.shape, h))
        if h[2] > 0:
            if self.proc_shape[2] == 1:
                evt, _ = self.pack_unpack_z_knl(queue, arr=fx)
            else:
                raise NotImplementedError("domain decomposition in z direction")
        if h[0] > 0:
            if self.proc_shape[0] == 1:
                evt, _ = self.pack_unpack_x_knl(queue, arr=fx)
            else:
                self._debug_barrier("all ranks beginning halo share in x")
                shape = (2, h[0], rank_shape[1] + 2*h[1], rank_shape[2] + 2*h[2])
                buf_dev, buf_send, buf_recv = self._get_or_create_halo_buffers(
                    queue, shape, fx.dtype)
                evt, _ = self.pack_x_knl(queue, arr=fx, buf=buf_dev)
                buf_dev.get(ary=buf_send)
                dest = self.rankID(self.rx-1, self.ry, self.rz)
                source = self.rankID(self.rx+1, self.ry, self.rz)
                self._debug(f"initiating first halo share in x: {dest=}, {source=}")
                self.comm.Sendrecv(sendbuf=buf_send[0], dest=dest,
                                   recvbuf=buf_recv[0], source=source)
                dest = self.rankID(self.rx+1, self.ry, self.rz)
                source = self.rankID(self.rx-1, self.ry, self.rz)
                self._debug(f"initiating second halo share in x: {dest=}, {source=}")
                self.comm.Sendrecv(sendbuf=buf_send[1], dest=dest,
                                   recvbuf=buf_recv[1], source=source)
                self._debug("finished halo share in x")
                buf_dev.set(buf_recv)
                evt, _ = self.unpack_x_knl(queue, arr=fx, buf=buf_dev)
                self._debug_barrier("all ranks finished halo share in x")
        if h[1] > 0:
            if self.proc_shape[1] == 1:
                evt, _ = self.pack_unpack_y_knl(queue, arr=fx)
            else:
                self._debug_barrier("all ranks beginning halo share in y")
                shape = (2, rank_shape[0] + 2*h[0], h[1], rank_shape[2] + 2*h[2])
                buf_dev, buf_send, buf_recv = self._get_or_create_halo_buffers(
                    queue, shape, fx.dtype)
                evt, _ = self.pack_y_knl(queue, arr=fx, buf=buf_dev)
                buf_dev.get(ary=buf_send)
                dest = self.rankID(self.rx, self.ry-1, self.rz)
                source = self.rankID(self.rx, self.ry+1, self.rz)
                self._debug(f"initiating first halo share in y: {dest=}, {source=}")
                self.comm.Sendrecv(sendbuf=buf_send[0], dest=dest,
                                   recvbuf=buf_recv[0], source=source)
                dest = self.rankID(self.rx, self.ry+1, self.rz)
                source = self.rankID(self.rx, self.ry-1, self.rz)
                self._debug(f"initiating second halo share in y: {dest=}, {source=}")
                self.comm.Sendrecv(sendbuf=buf_send[1], dest=dest,
                                   recvbuf=buf_recv[1], source=source)
                self._debug("finished halo share in y")
                buf_dev.set(buf_recv)
                evt, _ = self.unpack_y_knl(queue, arr=fx, buf=buf_dev)
                self._debug_barrier("all ranks finished halo share in y")
def share_halos ( self , queue , fx ) : h = self . halo_shape rank_shape = tuple ( ni - 2 * hi for ni , hi in zip ( fx . shape , h ) ) if h [ 2 ] > 0 : if self . proc_shape [ 2 ] == 1 : evt , _ = self . pack_unpack_z_knl ( queue , arr = fx ) else : raise NotImplementedError ( "domain decomposition in z direction" ) if h [ 0 ] > 0 : if self . proc_shape [ 0 ] == 1 : evt , _ = self . pack_unpack_x_knl ( queue , arr = fx ) else : self . _debug_barrier ( "all ranks beginning halo share in x" ) shape = ( 2 , h [ 0 ] , rank_shape [ 1 ] + 2 * h [ 1 ] , rank_shape [ 2 ] + 2 * h [ 2 ] ) buf_dev , buf_send , buf_recv = self . _get_or_create_halo_buffers ( queue , shape , fx . dtype ) evt , _ = self . pack_x_knl ( queue , arr = fx , buf = buf_dev ) buf_dev . get ( ary = buf_send ) dest = self . rankID ( self . rx - 1 , self . ry , self . rz ) source = self . rankID ( self . rx + 1 , self . ry , self . rz ) self . _debug ( f"initiating first halo share in x: {dest=}, {source=}" ) self . comm . Sendrecv ( sendbuf = buf_send [ 0 ] , dest = dest , recvbuf = buf_recv [ 0 ] , source = source ) dest = self . rankID ( self . rx + 1 , self . ry , self . rz ) source = self . rankID ( self . rx - 1 , self . ry , self . rz ) self . _debug ( f"initiating second halo share in x: {dest=}, {source=}" ) self . comm . Sendrecv ( sendbuf = buf_send [ 1 ] , dest = dest , recvbuf = buf_recv [ 1 ] , source = source ) self . _debug ( "finished halo share in x" ) buf_dev . set ( buf_recv ) evt , _ = self . unpack_x_knl ( queue , arr = fx , buf = buf_dev ) self . _debug_barrier ( "all ranks finished halo share in x" ) if h [ 1 ] > 0 : if self . proc_shape [ 1 ] == 1 : evt , _ = self . pack_unpack_y_knl ( queue , arr = fx ) else : self . _debug_barrier ( "all ranks beginning halo share in y" ) shape = ( 2 , rank_shape [ 0 ] + 2 * h [ 0 ] , h [ 1 ] , rank_shape [ 2 ] + 2 * h [ 2 ] ) buf_dev , buf_send , buf_recv = self . _get_or_create_halo_buffers ( queue , shape , fx . dtype ) evt , _ = self . pack_y_knl ( queue , arr = fx , buf = buf_dev ) buf_dev . get ( ary = buf_send ) dest = self . rankID ( self . rx , self . ry - 1 , self . rz ) source = self . rankID ( self . rx , self . ry + 1 , self . rz ) self . _debug ( f"initiating first halo share in y: {dest=}, {source=}" ) self . comm . Sendrecv ( sendbuf = buf_send [ 0 ] , dest = dest , recvbuf = buf_recv [ 0 ] , source = source ) dest = self . rankID ( self . rx , self . ry + 1 , self . rz ) source = self . rankID ( self . rx , self . ry - 1 , self . rz ) self . _debug ( f"initiating second halo share in y: {dest=}, {source=}" ) self . comm . Sendrecv ( sendbuf = buf_send [ 1 ] , dest = dest , recvbuf = buf_recv [ 1 ] , source = source ) self . _debug ( "finished halo share in y" ) buf_dev . set ( buf_recv ) evt , _ = self . unpack_y_knl ( queue , arr = fx , buf = buf_dev ) self . _debug_barrier ( "all ranks finished halo share in y" )
Communicates halo data across all axes, imposing periodic boundary
conditions.
Communicates halo data across all axes imposing periodic boundary conditions .
"""
        Communicates halo data across all axes, imposing periodic boundary
        conditions.

        :arg queue: The :class:`pyopencl.CommandQueue` to enqueue kernels and copies.

        :arg fx: The :class:`pyopencl.array.Array` whose halo elements are to be
            synchronized across ranks.
            The number of halo layers on each face of the grid is fixed by
            :attr:`halo_shape`, while the shape of ``fx``
            (i.e., subtracting halo padding specified by :attr:`halo_shape`)
            is only fixed if ``rank_shape`` or ``grid_shape`` were
            passed at object creation).
        """
self
queue
The :class:`pyopencl.CommandQueue` to enqueue kernels and copies.
The : class : ` pyopencl . CommandQueue ` to enqueue kernels and copies .
fx
The :class:`pyopencl.array.Array` whose halo elements are to be
synchronized across ranks.
The number of halo layers on each face of the grid is fixed by
:attr:`halo_shape`, while the shape of ``fx``

is only fixed if ``rank_shape`` or ``grid_shape`` were
passed at object creation).
The : class : ` pyopencl . array . Array ` whose halo elements are to be synchronized across ranks . The number of halo layers on each face of the grid is fixed by : attr : ` halo_shape ` while the shape of ` ` fx ` ` is only fixed if ` ` rank_shape ` ` or ` ` grid_shape ` ` were passed at object creation ) .