original_string: public function filter($token)
      {
          // Apply replacements
          if ($this->tokenFrom) {
              $token = strtr($token, $this->tokenFrom, $this->tokenTo);
          }
          
          // If not case sensitive, convert to lowercase
          if (! $this->tokenCaseSensitive) {
              $token = strtolower($token);
          }
          
          // Filter out invalid chars
          $tokenLength   = strlen($token);
          $filteredToken = '';
          
          for ($tokenPos = 0; ($tokenPos < $tokenLength); $tokenPos++) {
              if (strpos($this->tokenChars, $token[$tokenPos]) !== false) {
                  $filteredToken .= $token[$tokenPos];
              }
          }
  
          // Now check against the format for fixed chars
          $formatLength = strlen($this->tokenFormat);
          $tokenLength  = strlen($filteredToken);
          $tokenPos     = 0;
          $resultToken  = '';
          
          for ($formatPos = 0; ($formatPos < $formatLength) && ($tokenPos < $tokenLength); $formatPos++) {
              if ('\\' == $this->tokenFormat[$formatPos]) {
                  $formatPos++;
                  $resultToken .= $this->tokenFormat[$formatPos];
  
              } else {                
                  $resultToken .= $filteredToken[$tokenPos];
                  $tokenPos++;
              }
          }
          
          // Separate extra chars with a space and a questionmark
          if ($tokenPos < $tokenLength) {
              $resultToken .= ' ?' . substr($filteredToken, $tokenPos);
          }
  
          return $resultToken;
      }
code: public function filter($token)
      {
          // Apply replacements
          if ($this->tokenFrom) {
              $token = strtr($token, $this->tokenFrom, $this->tokenTo);
          }
          
          // If not case sensitive, convert to lowercase
          if (! $this->tokenCaseSensitive) {
              $token = strtolower($token);
          }
          
          // Filter out invalid chars
          $tokenLength   = strlen($token);
          $filteredToken = '';
          
          for ($tokenPos = 0; ($tokenPos < $tokenLength); $tokenPos++) {
              if (strpos($this->tokenChars, $token[$tokenPos]) !== false) {
                  $filteredToken .= $token[$tokenPos];
              }
          }
  
          // Now check against the format for fixed chars
          $formatLength = strlen($this->tokenFormat);
          $tokenLength  = strlen($filteredToken);
          $tokenPos     = 0;
          $resultToken  = '';
          
          for ($formatPos = 0; ($formatPos < $formatLength) && ($tokenPos < $tokenLength); $formatPos++) {
              if ('\\' == $this->tokenFormat[$formatPos]) {
                  $formatPos++;
                  $resultToken .= $this->tokenFormat[$formatPos];
  
              } else {                
                  $resultToken .= $filteredToken[$tokenPos];
                  $tokenPos++;
              }
          }
          
          // Separate extra chars with a space and a questionmark
          if ($tokenPos < $tokenLength) {
              $resultToken .= ' ?' . substr($filteredToken, $tokenPos);
          }
  
          return $resultToken;
      }
code_tokens: public function filter ( $ token ) { // Apply replacements if ( $ this -> tokenFrom ) { $ token = strtr ( $ token , $ this -> tokenFrom , $ this -> tokenTo ) ; } // If not case sensitive, convert to lowercase if ( ! $ this -> tokenCaseSensitive ) { $ token = strtolower ( $ token ) ; } // Filter out invalid chars $ tokenLength = strlen ( $ token ) ; $ filteredToken = '' ; for ( $ tokenPos = 0 ; ( $ tokenPos < $ tokenLength ) ; $ tokenPos ++ ) { if ( strpos ( $ this -> tokenChars , $ token [ $ tokenPos ] ) !== false ) { $ filteredToken .= $ token [ $ tokenPos ] ; } } // Now check against the format for fixed chars $ formatLength = strlen ( $ this -> tokenFormat ) ; $ tokenLength = strlen ( $ filteredToken ) ; $ tokenPos = 0 ; $ resultToken = '' ; for ( $ formatPos = 0 ; ( $ formatPos < $ formatLength ) && ( $ tokenPos < $ tokenLength ) ; $ formatPos ++ ) { if ( '\\' == $ this -> tokenFormat [ $ formatPos ] ) { $ formatPos ++ ; $ resultToken .= $ this -> tokenFormat [ $ formatPos ] ; } else { $ resultToken .= $ filteredToken [ $ tokenPos ] ; $ tokenPos ++ ; } } // Separate extra chars with a space and a questionmark if ( $ tokenPos < $ tokenLength ) { $ resultToken .= ' ?' . substr ( $ filteredToken , $ tokenPos ) ; } return $ resultToken ; }
docstring: Removes all unacceptable characters from the input token and inserts any fixed characters left out
  
  @param string $token
  @return string Reformatted token
docstring_tokens: Removes all unacceptable characters from the input token and inserts any fixed characters left out