text: #include "../stemmer.h"
  #include "../tokenize.h"
  #include "test_util.h"
  #include "../rmutil/alloc.h"
  #include <string.h>
  
  int testStemmer() {
  
    Stemmer *s = NewStemmer(SnowballStemmer, "en");
    ASSERT(s != NULL)
  
    size_t sl;
    const char *stem = s->Stem(s->ctx, "arbitrary", (size_t)strlen("arbitrary"), &sl);
    ASSERT(stem != NULL)
    ASSERT(!strcasecmp(stem, "+arbitrari"));
    ASSERT(sl == strlen(stem));
    printf("stem: %s\n", stem);
  
    // free((void*)stem);
    s->Free(s);
    return 0;
  }
  
  typedef struct {
    int num;
    const char **expectedTokens;
    const char **expectedStems;
  } tokenContext;
  
  int testTokenize() {
  
    char *txt = strdup("Hello? world... worlds going ? -WAZZ@UP? שלום");
  
    const char *expectedToks[] = {"hello", "world", "worlds", "going", "wazz", "up", "שלום"};
    const char *expectedStems[] = {NULL /*hello*/,
                                   NULL /*world/*/,
                                   "+world" /*worlds*/,
                                   "+go" /*going*/,
                                   NULL /*wazz*/,
                                   NULL /*up*/,
                                   NULL /*שלום*/};
    tokenContext ctx = {0};
    ctx.expectedTokens = expectedToks;
    ctx.expectedStems = expectedStems;
  
    Stemmer *s = NewStemmer(SnowballStemmer, "en");
    ASSERT(s != NULL)
  
    RSTokenizer *tk = NewSimpleTokenizer(s, DefaultStopWordList(), 0);
    Token t;
  
    tokenContext *tx = &ctx;
    tk->Start(tk, txt, strlen(txt), 0);
    while (tk->Next(tk, &t)) {
      printf("round %d\n", ctx.num);
      int ret = strncmp(t.tok, tx->expectedTokens[tx->num], t.tokLen);
      ASSERT(ret == 0);
      ASSERT(t.pos > 0);
  
      if (t.stem) {
        printf("Stem: %.*s, num=%d, orig=%.*s\n", (int)t.stemLen, t.stem, tx->num, (int)t.tokLen,
               t.tok);
        ASSERT(tx->expectedStems[tx->num]);
        ASSERT(strlen(tx->expectedStems[tx->num]) == t.stemLen);
        ASSERT(strncmp(tx->expectedStems[tx->num], t.stem, t.stemLen) == 0);
      } else {
        ASSERT(tx->expectedStems[tx->num] == NULL);
      }
      tx->num++;
    }
  
    ASSERT_EQUAL(ctx.num, 7);
  
    free(txt);
  
    return 0;
  }
  
  REDISMODULE_INIT_SYMBOLS();
  
  TEST_MAIN({
    RMUTil_InitAlloc();
    TESTFUNC(testStemmer);
    TESTFUNC(testTokenize);
  });