Add comments explaining the word_chars and space_chars tables.
@@ -62,12 +62,18 @@ with noyywrap then we can remove this pragma.
#define snprintf _snprintf
#endif
+// Bitmap with 1 bit for each of the 256 characters in the ASCII table. The bit
+// is set to 1 if the corresponding character is alphanumeric or 0 if otherwise.
static uint8_t word_chars[] = {
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF, 0x03,
0xFE, 0xFF, 0xFF, 0x87, 0xFE, 0xFF, 0xFF, 0x07,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
+// Bitmap with 1 bit for each of the 256 characters in the ASCII table. The bit
+// is set to 1 if the corresponding character is considered a space. Space
+// characters include horizontal and vertical tabs, carriage return, new line
+// and form feed (\t, \v, \r, \n, \f).
static uint8_t space_chars[] = {
0x00, 0x3E, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
