original_string: public function getTokens($string, &$needle = 0)
      {
          $exprTokenizer = new ExpressionTokenizer();
          $ret = array();
          $depth = 0;
          $needle = 0;
          $len = strlen($string);
          while ($needle < $len) {
              $before = $needle;
              $substr = substr($string, $needle);
              if ($depth === 0) {
                  // match either '$(' or '@(' and mark that as an EXPR_START token.
                  if (preg_match('/^([$@])\(/', $substr, $m)) {
                      $needle += strlen($m[0]);
                      $ret[] = new Token(Token::EXPR_START, $m[0]);
  
                      // record expression depth, to make sure the usage of parentheses inside the expression doesn't
                      // break tokenization (e.g. '$( ("foo") )'
                      $depth++;
                  } else {
                      // store the current token in a temp var for appending, in case it's a DATA token
                      $token = end($ret);
  
                      // handle escaping of the $( syntax, '$$(' becomes '$('
                      if (preg_match('/^\$\$\(/', $substr, $m)) {
                          $value = substr($m[0], 1);
                          $needle += strlen($m[0]);
                      } else {
                          $value = $string{$needle};
                          $needle += strlen($value);
                      }
  
                      // if the current token is DATA, and the previous token is DATA, append the value to the previous
                      // and ignore the current.
                      if ($token && $token->match(Token::DATA)) {
                          $token->value .= $value;
                          unset($token);
                      } else {
                          $ret[] = new Token(Token::DATA, $value);
                      }
                  }
              } else {
                  $ret = array_merge($ret, $exprTokenizer->getTokens($string, $needle));
                  $depth = 0;
              }
              if ($before === $needle) {
                  // safety net.
                  throw new \UnexpectedValueException(
                      "Unexpected input near token {$string{$needle}}, unsupported character"
                  );
              }
          }
          return $ret;
      }
code: public function getTokens($string, &$needle = 0)
      {
          $exprTokenizer = new ExpressionTokenizer();
          $ret = array();
          $depth = 0;
          $needle = 0;
          $len = strlen($string);
          while ($needle < $len) {
              $before = $needle;
              $substr = substr($string, $needle);
              if ($depth === 0) {
                  // match either '$(' or '@(' and mark that as an EXPR_START token.
                  if (preg_match('/^([$@])\(/', $substr, $m)) {
                      $needle += strlen($m[0]);
                      $ret[] = new Token(Token::EXPR_START, $m[0]);
  
                      // record expression depth, to make sure the usage of parentheses inside the expression doesn't
                      // break tokenization (e.g. '$( ("foo") )'
                      $depth++;
                  } else {
                      // store the current token in a temp var for appending, in case it's a DATA token
                      $token = end($ret);
  
                      // handle escaping of the $( syntax, '$$(' becomes '$('
                      if (preg_match('/^\$\$\(/', $substr, $m)) {
                          $value = substr($m[0], 1);
                          $needle += strlen($m[0]);
                      } else {
                          $value = $string{$needle};
                          $needle += strlen($value);
                      }
  
                      // if the current token is DATA, and the previous token is DATA, append the value to the previous
                      // and ignore the current.
                      if ($token && $token->match(Token::DATA)) {
                          $token->value .= $value;
                          unset($token);
                      } else {
                          $ret[] = new Token(Token::DATA, $value);
                      }
                  }
              } else {
                  $ret = array_merge($ret, $exprTokenizer->getTokens($string, $needle));
                  $depth = 0;
              }
              if ($before === $needle) {
                  // safety net.
                  throw new \UnexpectedValueException(
                      "Unexpected input near token {$string{$needle}}, unsupported character"
                  );
              }
          }
          return $ret;
      }
code_tokens: public function getTokens ( $ string , & $ needle = 0 ) { $ exprTokenizer = new ExpressionTokenizer ( ) ; $ ret = array ( ) ; $ depth = 0 ; $ needle = 0 ; $ len = strlen ( $ string ) ; while ( $ needle < $ len ) { $ before = $ needle ; $ substr = substr ( $ string , $ needle ) ; if ( $ depth === 0 ) { // match either '$(' or '@(' and mark that as an EXPR_START token. if ( preg_match ( '/^([$@])\(/' , $ substr , $ m ) ) { $ needle += strlen ( $ m [ 0 ] ) ; $ ret [ ] = new Token ( Token :: EXPR_START , $ m [ 0 ] ) ; // record expression depth, to make sure the usage of parentheses inside the expression doesn't // break tokenization (e.g. '$( ("foo") )' $ depth ++ ; } else { // store the current token in a temp var for appending, in case it's a DATA token $ token = end ( $ ret ) ; // handle escaping of the $( syntax, '$$(' becomes '$(' if ( preg_match ( '/^\$\$\(/' , $ substr , $ m ) ) { $ value = substr ( $ m [ 0 ] , 1 ) ; $ needle += strlen ( $ m [ 0 ] ) ; } else { $ value = $ string { $ needle } ; $ needle += strlen ( $ value ) ; } // if the current token is DATA, and the previous token is DATA, append the value to the previous // and ignore the current. if ( $ token && $ token -> match ( Token :: DATA ) ) { $ token -> value .= $ value ; unset ( $ token ) ; } else { $ ret [ ] = new Token ( Token :: DATA , $ value ) ; } } } else { $ ret = array_merge ( $ ret , $ exprTokenizer -> getTokens ( $ string , $ needle ) ) ; $ depth = 0 ; } if ( $ before === $ needle ) { // safety net. throw new \ UnexpectedValueException ( "Unexpected input near token {$string{$needle}}, unsupported character" ) ; } } return $ ret ; }
docstring: Returns an array of tokens
  
  @param string $string
  @param int &$needle
  @throws \UnexpectedValueException
  @return array
docstring_tokens: Returns an array of tokens