class_docstr: Implement Softmax on the gpu.
code: def gpu_kernels(self, node, nodename):
      dtype_x = node.inputs[0].dtype
      dtype_sm = node.outputs[0].dtype
      load_x = load_w(dtype_x)
      write_sm = write_w(node.outputs[0].dtype)
      work_sm = work_dtype(dtype_sm)
      flags = Kernel.get_flags(dtype_x, dtype_sm)
      type_x = gpuarray.dtype_to_ctype(dtype_x)
      type_sm = gpuarray.dtype_to_ctype(dtype_sm)
      type_acc = gpuarray.dtype_to_ctype(work_sm)
  
      ctype = gpuarray.dtype_to_ctype(work_sm)
  
      params = [
          gpuarray.SIZE,
          gpuarray.SIZE,
          gpuarray.GpuArray,
          gpuarray.SIZE,
          gpuarray.SSIZE,
          gpuarray.SSIZE,
          gpuarray.GpuArray,
          gpuarray.SIZE,
          gpuarray.SSIZE,
          gpuarray.SSIZE,
      ]
      kernels = []
      kname = "kSoftmax"
      k_var = "kSoftmax_" + nodename
      code = (
          """#include "cluda.h"
  
          KERNEL void %(kname)s (const ga_size M, const ga_size N,
                                 GLOBAL_MEM const %(type_x)s * x, const ga_size offset_x, const ga_ssize sx0, const ga_ssize sx1,
                                 GLOBAL_MEM %(type_sm)s * sm, const ga_size offset_sm, const ga_ssize sm_s0, const ga_ssize sm_s1 GA_DECL_SHARED_PARAM(%(type_acc)s, buf))
          {
              GA_DECL_SHARED_BODY(%(type_acc)s, buf);
              LOCAL_MEM_ARG %(type_acc)s * buf2 = buf + N;
              x = (GLOBAL_MEM const %(type_x)s *)(((GLOBAL_MEM char *)x)+offset_x);
              sm = (GLOBAL_MEM %(type_sm)s *)(((GLOBAL_MEM char *)sm)+offset_sm);
              for (ga_int blockIDX = GID_0; blockIDX < M; blockIDX += GDIM_0) {
                  for (ga_int tx = LID_0; tx< N; tx += LDIM_0) {
                      buf[tx] = %(load_x)s(x[blockIDX * sx0 + tx * sx1]);
                      buf2[tx] = buf[tx];
                  }
                  local_barrier();
                  {
                      // This function trashes buf[1..GA_WARP_SIZE],
                      // leaving the reduction result in buf[0].
                      if (LID_0 < GA_WARP_SIZE) {
                          for (ga_int i = LID_0 + GA_WARP_SIZE; i < N; i += GA_WARP_SIZE)
                          {
                              buf[LID_0] = max(buf[LID_0], buf[i]);
                          }
                      }
                      local_barrier();
                      //reduce so that LID_0 0 has the reduction of everything
                      for (ga_uint _n = GA_WARP_SIZE / 2; _n > 0; _n /= 2) {
                          if (LID_0 < _n && LID_0 + _n < N)
                              buf[LID_0] = max(buf[LID_0], buf[LID_0+_n]);
                          local_barrier();
                      }
                  }
                  %(ctype)s row_max = buf[0];
                  local_barrier();
                  for(ga_int __i=LID_0; __i<N; __i+=LDIM_0){
                      buf[__i] = exp(buf2[__i] - row_max);
                      buf2[__i] = buf[__i];
                  }
                  local_barrier();
                  {
                      // This function trashes buf[1..GA_WARP_SIZE],
                      // leaving the reduction result in buf[0].
                      if (LID_0 < GA_WARP_SIZE) {
                          for (ga_int i = LID_0 + GA_WARP_SIZE; i < N; i += GA_WARP_SIZE)
                          {
                              buf[LID_0] = buf[LID_0] + buf[i];
                          }
                      }
                      local_barrier();
                      //reduce so that LID_0 0 has the reduction of everything
                      for (ga_uint _n = GA_WARP_SIZE / 2; _n > 0; _n /= 2) {
                          if (LID_0 < _n && LID_0 + _n < N)
                              buf[LID_0] = buf[LID_0] + buf[LID_0+_n];
                          local_barrier();
                      }
                  }
                  %(ctype)s row_sum = buf[0];
                  local_barrier();
                  for(ga_int __i=LID_0; __i<N; __i+=LDIM_0) {
                      buf[__i] = buf2[__i] / row_sum;
                  }
                  local_barrier();
                  for (ga_int tx = LID_0; tx< N; tx += LDIM_0) {
                      sm[blockIDX * sm_s0 + tx * sm_s1] = %(write_sm)s(buf[tx]);
                  }
                  local_barrier();
              }
          }
          """
          % locals()
      )
      kernels.append(
          Kernel(
              code=code,
              name=kname,
              params=params,
              flags=flags,
              objvar=k_var,
          )
      )
      kname = "kSoftmax_fixed_shared"
      k_var = "kSoftmax_fixed_shared" + nodename
      code = (
          """#include "cluda.h"
  
          KERNEL void %(kname)s (const ga_size M, const ga_size N,
                                 GLOBAL_MEM const %(type_x)s * x, const ga_size offset_x, const ga_ssize sx0, const ga_ssize sx1,
                                 GLOBAL_MEM %(type_sm)s * sm, const ga_size offset_sm, const ga_ssize sm_s0, const ga_ssize sm_s1 GA_DECL_SHARED_PARAM(%(type_acc)s, buf))
          {
              GA_DECL_SHARED_BODY(%(type_acc)s, buf);
              x = (GLOBAL_MEM const %(type_x)s *)(((GLOBAL_MEM char *)x)+offset_x);
              sm = (GLOBAL_MEM %(type_sm)s *)(((GLOBAL_MEM char *)sm)+offset_sm);
              for (ga_int blockIDX = GID_0; blockIDX < M; blockIDX += GDIM_0){
                  GLOBAL_MEM const %(type_x)s *x_ptr = &x[blockIDX * sx0];
                  GLOBAL_MEM %(type_sm)s *sm_ptr = &sm[blockIDX * sm_s0];
                  {
                      // This function trashes buf[1..n_threads],
                      // leaving the reduction result in buf[0].
                      %(ctype)s red = %(load_x)s(x_ptr[LID_0 * sx1]);
                      #pragma unroll 16
                      for (ga_int i = LID_0 + LDIM_0; i<N; i += LDIM_0) {
                          red = max(red, %(load_x)s(x_ptr[i * sx1]));
                      }
                      buf[LID_0] = red;
                      local_barrier();
                      if (LID_0 < GA_WARP_SIZE) {
                          for (ga_int i = LID_0 + GA_WARP_SIZE; i < LDIM_0; i += GA_WARP_SIZE) {
                              buf[LID_0] = max(buf[LID_0], buf[i]);
                          }
                      }
                      local_barrier();
                      //reduce so that LID_0 0 has the reduction of everything
                      for (ga_uint _n = GA_WARP_SIZE / 2; _n > 0; _n /= 2) {
                          if (LID_0 < _n && LID_0 + _n < N)
                              buf[LID_0] = max(buf[LID_0], buf[LID_0+_n]);
                          local_barrier();
                      }
                  }
                  %(ctype)s row_max = buf[0];
                  local_barrier();
                  {
                      // This function trashes buf[1..n_threads],
                      // leaving the reduction result in buf[0].
                      %(ctype)s red = exp(%(load_x)s(x_ptr[LID_0 * sx1]) - row_max);
                      #pragma unroll 16
                      for (ga_int i = LID_0 + LDIM_0; i<N; i += LDIM_0) {
                          red = red + exp(%(load_x)s(x_ptr[i * sx1]) - row_max);
                      }
                      buf[LID_0] = red;
                      local_barrier();
                      if (LID_0 < GA_WARP_SIZE) {
                          for (ga_int i = LID_0 + GA_WARP_SIZE; i < LDIM_0; i += GA_WARP_SIZE) {
                              buf[LID_0] = buf[LID_0] + buf[i];
                          }
                      }
                      local_barrier();
                      //reduce so that LID_0 0 has the reduction of everything
                      for (ga_uint _n = GA_WARP_SIZE / 2; _n > 0; _n /= 2) {
                          if (LID_0 < _n && LID_0 + _n < N)
                              buf[LID_0] = buf[LID_0] + buf[LID_0+_n];
                          local_barrier();
                      }
                  }
                  %(ctype)s row_sum = buf[0];
                  local_barrier();
                  for (ga_int tx = LID_0; tx< N; tx += LDIM_0){
                      sm_ptr[tx * sm_s1] = %(write_sm)s(exp(%(load_x)s(x_ptr[tx * sx1]) - row_max) / row_sum);
                  }
                  local_barrier();
              }
          }
          """
          % locals()
      )
      kernels.append(
          Kernel(
              code=code,
              name=kname,
              params=params,
              flags=flags,
              objvar=k_var,
          )
      )
      return kernels
  
imports: from __future__ import absolute_import, print_function, division
  import numpy
  from theano import Op, Apply
  from six import StringIO
  from .basic_ops import as_gpuarray_variable, GpuKernelBase, Kernel, gpuarray_helper_inc_dir, infer_context_name
  from .type import GpuArrayType
  from .fp16_help import work_dtype, load_w, write_w
label_desc: Use image node_id 6 for calling the GpuSoftmax obj's underlying member method code with example usage: obj.gpu_kernels(node, nodename) and returns: kernels