blob: d2632690d5107b53ee8a7ac4832cd85eb8c7bfc1 [file] [log] [blame]
 #include "cache.h" #include "levenshtein.h" /* * This function implements the Damerau-Levenshtein algorithm to * calculate a distance between strings. * * Basically, it says how many letters need to be swapped, substituted, * deleted from, or added to string1, at least, to get string2. * * The idea is to build a distance matrix for the substrings of both * strings. To avoid a large space complexity, only the last three rows * are kept in memory (if swaps had the same or higher cost as one deletion * plus one insertion, only two rows would be needed). * * At any stage, "i + 1" denotes the length of the current substring of * string1 that the distance is calculated for. * * row2 holds the current row, row1 the previous row (i.e. for the substring * of string1 of length "i"), and row0 the row before that. * * In other words, at the start of the big loop, row2[j + 1] contains the * Damerau-Levenshtein distance between the substring of string1 of length * "i" and the substring of string2 of length "j + 1". * * All the big loop does is determine the partial minimum-cost paths. * * It does so by calculating the costs of the path ending in characters * i (in string1) and j (in string2), respectively, given that the last * operation is a substitution, a swap, a deletion, or an insertion. * * This implementation allows the costs to be weighted: * * - w (as in "sWap") * - s (as in "Substitution") * - a (for insertion, AKA "Add") * - d (as in "Deletion") * * Note that this algorithm calculates a distance _iff_ d == a. */ int levenshtein(const char *string1, const char *string2, int w, int s, int a, int d) { int len1 = strlen(string1), len2 = strlen(string2); int *row0, *row1, *row2; int i, j; ALLOC_ARRAY(row0, len2 + 1); ALLOC_ARRAY(row1, len2 + 1); ALLOC_ARRAY(row2, len2 + 1); for (j = 0; j <= len2; j++) row1[j] = j * a; for (i = 0; i < len1; i++) { int *dummy; row2 = (i + 1) * d; for (j = 0; j < len2; j++) { /* substitution */ row2[j + 1] = row1[j] + s * (string1[i] != string2[j]); /* swap */ if (i > 0 && j > 0 && string1[i - 1] == string2[j] && string1[i] == string2[j - 1] && row2[j + 1] > row0[j - 1] + w) row2[j + 1] = row0[j - 1] + w; /* deletion */ if (row2[j + 1] > row1[j + 1] + d) row2[j + 1] = row1[j + 1] + d; /* insertion */ if (row2[j + 1] > row2[j] + a) row2[j + 1] = row2[j] + a; } dummy = row0; row0 = row1; row1 = row2; row2 = dummy; } i = row1[len2]; free(row0); free(row1); free(row2); return i; }
{"ft_lang_label":"__label__en","ft_lang_prob":0.72909117,"math_prob":0.99506205,"size":2524,"snap":"2021-31-2021-39","text_gpt3_token_len":867,"char_repetition_ratio":0.1388889,"word_repetition_ratio":0.012295082,"special_character_ratio":0.39976227,"punctuation_ratio":0.15337424,"nsfw_num_words":0,"has_unicode_error":false,"math_prob_llama3":0.996746,"pos_list":[0],"im_url_duplicate_count":[null],"WARC_HEADER":"{\"WARC-Type\":\"response\",\"WARC-Date\":\"2021-07-27T08:56:56Z\",\"WARC-Record-ID\":\"<urn:uuid:26aaf02c-32af-4e13-823b-a976134078f5>\",\"Content-Length\":\"36350\",\"Content-Type\":\"application/http; msgtype=response\",\"WARC-Warcinfo-ID\":\"<urn:uuid:a0d584f6-3529-4266-8afa-34da77a5d7f3>\",\"WARC-Concurrent-To\":\"<urn:uuid:098ded51-7317-41de-af9f-97ba4b3b9b9d>\",\"WARC-IP-Address\":\"173.194.205.82\",\"WARC-Target-URI\":\"https://code.googlesource.com/git/+/refs/tags/v2.16.0-rc1/levenshtein.c\",\"WARC-Payload-Digest\":\"sha1:KBKM36VBEPWPW7GMG67FPTNITY2HFYAE\",\"WARC-Block-Digest\":\"sha1:2Q7HFJJ3XVEULCMYYVPDNJERMQ4EFP5H\",\"WARC-Identified-Payload-Type\":\"text/html\",\"warc_filename\":\"/cc_download/warc_2021/CC-MAIN-2021-31/CC-MAIN-2021-31_segments_1627046153223.30_warc_CC-MAIN-20210727072531-20210727102531-00276.warc.gz\"}"}