How to detect the direction of a rotary encoder?
I have a page about rotary encoders - in that I show how you can detect encoder changes (and direction) using one interrupt pin and another pin, like this:

// Wiring: Connect common pin of encoder to ground.
// Connect pin A (one of the outer ones) to a pin that can generate interrupts (eg. D2)
// Connect pin B (the other outer one) to another free pin (eg. D5)

volatile bool fired;
volatile bool up;

const byte encoderPinA = 2;
const byte encoderPinB = 5;

// Interrupt Service Routine for a change to encoder pin A
void isr ()
{
  if (digitalRead (encoderPinA))
    up = digitalRead (encoderPinB);
  else
    up = !digitalRead (encoderPinB);
  fired = true;
}  // end of isr


void setup ()
{
  pinMode (encoderPinA, INPUT_PULLUP);     // enable pull-ups
  pinMode (encoderPinB, INPUT_PULLUP); 
  attachInterrupt (digitalPinToInterrupt (encoderPinA), isr, CHANGE);   // interrupt 0 is pin 2

  Serial.begin (115200);
}  // end of setup

void loop ()
{
static long rotaryCount = 0;

  if (fired)
    {
    if (up)
      rotaryCount++;
    else
      rotaryCount--;
    fired = false;

    Serial.print ("Count = ");  
    Serial.println (rotaryCount);
    }  // end if fired

}  // end of loop


In the main loop you just need to test the boolean fired to see if the encoder changed, and up to see if it went up (otherwise it went down).




  I receive random of these values ...


I think with your code you need to detect a change - that is, if the encoder pin is now HIGH but was previously LOW.



Solution for 4 encoders


  The problem is I am working on four encoders ...


I have reworked my code to use pin-change interrupts instead of external interrupts.

// Wiring: Connect common pin of encoder to ground.

volatile bool fired;

const byte ENCODERS = 4;

typedef struct 
  {
  int aPin;     // which Arduino pin for the "A" side
  int bPin;     // which Arduino pin for the "B" side

  // values below are calculated at run-time
  volatile byte * aPort;  // which processor port the A side is plugged into
  volatile byte * bPort;  // which processor port the B side is plugged into
  byte aBitMask;   // which bit in the A port
  byte bBitMask;   // which bit in the B port
  byte whichInterrupt;  // which pin-change interrupt port (0, 1, 2)
  byte oldValue;   // old value of A port (to see if it changed)
  int count;       // current encoder counter
  } encoder;

 volatile encoder encoders [ENCODERS] =
  {

 // A   B  pins  (eg. D4 and D8, D5 and D9 and so on)

  { 4,  8 },
  { 5,  9 },
  { 6, 10 },
  { 7, 11 },

  };  // end of encoders


void checkForPinChange (const byte which)
  {
  for (byte i = 0; i &lt; ENCODERS; i++)
    {
    if (encoders [i].whichInterrupt == which)
      {
      byte newValue = *(encoders [i].aPort) &amp; encoders [i].aBitMask;
      if (newValue != encoders [i].oldValue)
        {
        bool up;
        byte bPort = *(encoders [i].bPort) &amp; encoders [i].bBitMask;
        if (newValue)
          up = bPort;
        else
          up = !bPort;
        fired = true;
        if (up)
          encoders [i].count++;
        else
          encoders [i].count--;
        encoders [i].oldValue = newValue;
        }  // end of if value has changed
      }   // end of if this is the right interrupt number
    }     // end of for each encoder
  } // end of checkForPinChange

// handle pin change interrupt for D8 to D13 here
ISR (PCINT0_vect)
 {
 checkForPinChange (PCIE0);
 }  // end of PCINT0_vect

// handle pin change interrupt for A0 to A5 here
ISR (PCINT1_vect)
 {
 checkForPinChange (PCIE1);
 }  // end of PCINT1_vect

// handle pin change interrupt for D0 to D7 here
ISR (PCINT2_vect)
 {
 checkForPinChange (PCIE2);
 }  // end of PCINT2_vect

void setup ()
{
 Serial.begin (115200);
 Serial.println ("Starting ...");

 PCIFR  |= bit (PCIF0) | bit (PCIF1) | bit (PCIF2);   // clear any outstanding interrupts

 for (byte i = 0; i &lt; ENCODERS; i++)
    {
    pinMode (encoders [i].aPin, INPUT_PULLUP);
    pinMode (encoders [i].bPin, INPUT_PULLUP);
    // convert pin number to port, mask, etc. for efficiency

    // Input port
    encoders [i].aPort = portInputRegister (digitalPinToPort (encoders [i].aPin));
    encoders [i].bPort = portInputRegister (digitalPinToPort (encoders [i].bPin));

    // Which bit in the port to test
    encoders [i].aBitMask = digitalPinToBitMask (encoders [i].aPin);
    encoders [i].bBitMask = digitalPinToBitMask (encoders [i].bPin);

    // Which interrupt number (0, 1, 2)
    encoders [i].whichInterrupt = digitalPinToPCICRbit (encoders [i].aPin);

    // activate this pin-change interrupt bit (eg. PCMSK0, PCMSK1, PCMSK2)
    volatile byte * ICRmaskPort = digitalPinToPCMSK (encoders [i].aPin);
    *ICRmaskPort  |= bit (digitalPinToPCMSKbit (encoders [i].aPin));

    // enable this pin-change interrupt
    PCICR |= bit (digitalPinToPCICRbit (encoders [i].aPin));

    } // end of for each encoder

}  // end of setup

void loop ()
{
  if (fired)
    {
    for (byte i = 0; i &lt; ENCODERS; i++)
      {
      char buf [10];
      sprintf (buf, "%5d", encoders [i].count);
      Serial.print (buf);
      }
    Serial.println ();
    fired = false;
    }  // end if fired

}  // end of loop


The code is general enough to handle 1 to x encoders (eg. 10 encoders) - within reason! You need to have 2 pins spare per encoder (the third pin goes to ground). One of the pins needs to be capable of pin-change interrupts (on the Uno that is all of them).

The code sets up to detect an interrupt on the "A" side of the encoder and tests the value on the "B" side to see if it was turned clockwise or counter-clockwise.

I am getting results like this (with 2 encoders connected):

   16  -19    1    1
   17  -19    1    1
   18  -19    1    1
   19  -19    1    1
   20  -19    1    1
   21  -19    1    1
   22  -19    1    1
   23  -19    1    1
   23  -20    1    1
   23  -21    1    1
   23  -22    1    1
   23  -23    1    1
   23  -24    1    1
   23  -25    1    1


The other two are showing as "1" (that is, not turned yet).