title: Hybrid approach with OpenMP and MPI does not use same number of threads in cluster with different number of hosts
body: <p>I'm testing a hybrid approach by paralleling the friendly-numbers (CAPBenchmark) program with MPI and OpenMP.</p>
  
  <p>My cluster has 8 machines and each machine has a 4 core processor.</p>
  
  <p>The code:</p>
  
  <pre><code>/*
   * Copyright(C) 2014 Pedro H. Penna &lt;pedrohenriquepenna@gmail.com&gt;
   * 
   * friendly-numbers.c - Friendly numbers kernel.
   */
  
  #include &lt;global.h&gt;
  #include &lt;mpi.h&gt;
  #include &lt;omp.h&gt;
  #include &lt;stdio.h&gt;
  #include &lt;stdlib.h&gt;
  #include &lt;util.h&gt;
  #include "fn.h"
  
  /*
   * Computes the Greatest Common Divisor of two numbers.
   */
  static int gcd(int a, int b)
  {
    int c;
  
    /* Compute greatest common divisor. */
    while (a != 0)
    {
       c = a;
       a = b%a;
       b = c;
    }
  
    return (b);
  }
  
  /*
   * Some of divisors.
   */
  static int sumdiv(int n)
  {
      int sum;    /* Sum of divisors. */
      int factor; /* Working factor.  */
  
      sum = 1 + n;
  
      /* Compute sum of divisors. */
      for (factor = 2; factor &lt; n; factor++)
      {
          /* Divisor found. */
          if ((n%factor) == 0)
              sum += factor;
      }
  
      return (sum);
  }
  
  /*
   * Computes friendly numbers.
   */
  int friendly_numbers(int start, int end) 
  {
      int n;        /* Divisor.                    */
      int *num;     /* Numerator.                  */
      int *den;     /* Denominator.                */
      int *totalnum;
      int *totalden;
      int rcv_friends;
      int range;    /* Range of numbers.           */
      int i, j;     /* Loop indexes.               */
      int nfriends; /* Number of friendly numbers. */
      int slice;
  
      range = end - start + 1;
      slice = range / nthreads;
      if (rank == 0) {
  
          num = smalloc(sizeof(int)*range);
          den = smalloc(sizeof(int)*range);
          totalnum = smalloc(sizeof(int)*range);
          totalden = smalloc(sizeof(int)*range);
  
      } else {
  
          num = smalloc(sizeof(int) * slice);
          den = smalloc(sizeof(int) * slice);
          totalnum = smalloc(sizeof(int)*range);
          totalden = smalloc(sizeof(int)*range);
      }
  
      j = 0;
      omp_set_dynamic(0);    
      omp_set_num_threads(4);
      #pragma omp parallel for private(i, j, n) default(shared) 
      for (i = start + rank * slice; i &lt; start + (rank + 1) * slice; i++) { 
              j = i - (start + rank * slice);
              num[j] = sumdiv(i);
              den[j] = i;
  
              n = gcd(num[j], den[j]);
              num[j] /= n;
              den[j] /= n;
      }
      if (rank != 0) {
          MPI_Send(num, slice, MPI_INT, 0, 0, MPI_COMM_WORLD);
          MPI_Send(den, slice, MPI_INT, 0, 1, MPI_COMM_WORLD);
      } else {
          for (i = 1; i &lt; nthreads; i++)  {
              MPI_Recv(num + (i * (slice)), slice, MPI_INT, i, 0, MPI_COMM_WORLD, 0);
              MPI_Recv(den + (i * (slice)), slice, MPI_INT, i, 1, MPI_COMM_WORLD, 0);
          }
      }
  
      if (rank == 0) {
          for (i = 1; i &lt; nthreads; i++) {
              MPI_Send(num, range, MPI_INT, i, 2, MPI_COMM_WORLD);
              MPI_Send(den, range, MPI_INT, i, 3, MPI_COMM_WORLD);
          }
      } else {
          MPI_Recv(totalnum, range, MPI_INT, 0, 2, MPI_COMM_WORLD,0);
          MPI_Recv(totalden, range, MPI_INT, 0, 3, MPI_COMM_WORLD,0);
      }
  
      /* Check friendly numbers. */
      nfriends = 0;
      if (rank == 0) {
          omp_set_dynamic(0);
          omp_set_num_threads(4);
          #pragma omp parallel for private(i, j) default(shared) reduction(+:nfriends)
          for (i = rank; i &lt; range; i += nthreads) {
              for (j = 0; j &lt; i; j++) {
                  /* Friends. */
                  if ((num[i] == num[j]) &amp;&amp; (den[i] == den[j])) 
                      nfriends++;
              }
          }
      } else {
          omp_set_dynamic(0);
          omp_set_num_threads(4);
          #pragma omp parallel for private(i, j) default(shared) reduction(+:nfriends)
          for (i = rank; i &lt; range; i += nthreads) {
              for (j = 0; j &lt; i; j++) {
                  /* Friends. */
                  if ((totalnum[i] == totalnum[j]) &amp;&amp; (totalden[i] == totalden[j])) 
                      nfriends++;
              }
          }
  
      }
      if (rank == 0) {
          for (i = 1; i &lt; nthreads; i++) {
              MPI_Recv(&amp;rcv_friends, 1, MPI_INT, i, 4, MPI_COMM_WORLD, 0);
              nfriends += rcv_friends;
          }
      } else {
          MPI_Send(&amp;nfriends, 1, MPI_INT, 0, 4, MPI_COMM_WORLD);
      }
  
      free(num);
      free(den);
  
      return (nfriends);
  }
  </code></pre>
  
  <p>During the executions I observed the following behavior: </p>
  
  <p>When I run <code>mpirun</code> with 4 and 8 hosts, each of the hosts uses 4 threads for processing, as expected.</p>
  
  <p>However when running using only 2 hosts only 1 thread is used on each machine.
  What could cause this behavior? Is there any alternative to "force" the use of the 4 threads in the case of the 2 hosts?</p>