src/libmetha/worker.c
static M_CODE
lm_worker_sort(worker_t *w)
{
    int        x, epeek;
    ulist_t    *list, *peek_list = 0;
    crawler_t  *cr;
    int        match, lookup;
    int        syn;
    uehandle_t *ue_h = w->ue_h;
    filetype_t *ft;
    url_t      *url;
    const char *mime;
    char       *c;
    int num_urls;
    int depth_limit, depth_counter;

    /* get the current list of URLs */
    if (!(list = lm_utable_top(&ue_h->primary)))
        return M_FAILED;

    int list_index = ue_h->primary.sz-1;

    cr     = w->crawler;
    epeek  = (lm_crawler_flag_isset(cr, LM_CRFLAG_EPEEK) && !ue_h->is_peeking)
              ? 1 : 0;
    syn    = w->io_h->io->synchronous;
    lookup = 0;

    url_t *current_url = w->ue_h->current;

    /* need to cache these here since invocation of epeek might change them during */
    depth_limit = ue_h->depth_limit;
    depth_counter = ue_h->depth_counter;

    num_urls = list->sz;

    for (x=0; x<list->sz; x++) {
        url   = lm_ulist_row(list, x);
        match = 0;

        /* first we try to match the URL by looking at the string */
        if ((ft = lm_ftindex_match_by_url(&cr->ftindex, url))) {
            if (ft == LM_FTINDEX_POSSIBLE_MATCH) {
                if (!syn) {
                    if (lm_multipeek_add(w->io_h, url, x) == M_OK) {
                        match = 1;
                        lookup ++;
                    }
                } else {
                    lm_io_head(w->io_h, url);
                    char *mime = w->io_h->transfer.headers.content_type;
                    if (mime) {
                        if ((c = strchr(mime, ';')))
                            *c = '\0';

                        if ((ft = lm_ftindex_match_by_mime(&cr->ftindex, mime))
                                && lm_worker_bind_url(w, url, ft, epeek, &peek_list) == 0)
                            match = 1;
                    }
                }
            } else {
                if (lm_worker_bind_url(w, url, ft, epeek, &peek_list) == 0) {
                    match = 1;
                }

                list = &w->ue_h->primary.row[list_index];
            }
        }

        if (!match) {
            lm_url_nullify(url);
        }
    }

    if (!lookup) {
        goto cleanup;
    }

    ioprivate_t *info;
    CURL        *h;

#ifdef DEBUG
    if (!syn)
        fprintf(stderr, "* worker:(%p) waiting for %d HTTP HEAD requests...\n", w, lookup);
#endif

    while ((info = lm_multipeek_wait(w->io_h))) {
        h   = info->handle;
        url = lm_ulist_row(list, info->identifier);
        match = 0;

        curl_easy_getinfo(h, CURLINFO_CONTENT_TYPE, &mime);

        if (mime) {
            if ((c = strchr(mime, ';')))
                *c = '\0';

            if ((ft = lm_ftindex_match_by_mime(&cr->ftindex, mime))
                    && lm_worker_bind_url(w, url, ft, epeek, &peek_list) == 0)
                match = 1;

            list = &w->ue_h->primary.row[list_index];
        }

        curl_easy_cleanup(h);

        if (!match)
            lm_url_nullify(url);
    }

cleanup:
    /* finally, remove everything that is honoring depth_limit */
    if (depth_limit) {
        if (depth_counter >= depth_limit) {
            for (x=0; x<list->sz; x++) {
                url   = lm_ulist_row(list, x);
                if (!url->sz) {
                    continue;
                }
                ft = w->m->filetypes[url->bind-1];
                if (!FT_FLAG_ISSET(ft, FT_FLAG_IGNORE_DEPTH)/* || !epeek*/) {
                    lm_url_nullify(url);
                }
            }
        }
    }

    /* remove nullified urls from the list */
    x = list->sz-1;
    while (x >= 0 && !(lm_ulist_row(list, x)->sz))
        x--;

    list->sz = x+1;

    for (x=0; x<list->sz; ) {
        if (!(lm_ulist_row(list, x)->sz)) {
            lm_url_swap(lm_ulist_row(list, x), lm_ulist_top(list));
            lm_ulist_dec(list);
        } else
            x++;
    }

    int discarded = num_urls - list->sz;
    w->num_discarded_urls += discarded;

    if (w->m->report_cb && current_url && (w->io_h->transfer.status_code < 300 || w->io_h->transfer.status_code >= 400)) {
        w->m->report_cb(w->m, current_url, LMREPORT_NUM_DISCARDED_URLS, discarded);
    }

    return M_OK;
}
/** 
 * Discard or sort URLs into filetypes depending 
 * on the current crawler's list of filetypes 
 * and rules.
 *
 * If we have reached the maximum depth we must also discard
 * URLs here to prevent them from being crawled.
 *
 * Returns M_OK unless an error occured.
 **/
Discard or sort URLs into filetypes depending
on the current crawler's list of filetypes
and rules.

If we have reached the maximum depth we must also discard
URLs here to prevent them from being crawled.

Returns M_OK unless an error occured.
Discard or sort URLs into filetypes depending on the current crawler ' s list of filetypes and rules . If we have reached the maximum depth we must also discard URLs here to prevent them from being crawled . Returns M_OK unless an error occured .
static M_CODE
lm_worker_sort(worker_t *w)
{
    int        x, epeek;
    ulist_t    *list, *peek_list = 0;
    crawler_t  *cr;
    int        match, lookup;
    int        syn;
    uehandle_t *ue_h = w->ue_h;
    filetype_t *ft;
    url_t      *url;
    const char *mime;
    char       *c;
    int num_urls;
    int depth_limit, depth_counter;
    if (!(list = lm_utable_top(&ue_h->primary)))
        return M_FAILED;
    int list_index = ue_h->primary.sz-1;
    cr     = w->crawler;
    epeek  = (lm_crawler_flag_isset(cr, LM_CRFLAG_EPEEK) && !ue_h->is_peeking)
              ? 1 : 0;
    syn    = w->io_h->io->synchronous;
    lookup = 0;
    url_t *current_url = w->ue_h->current;
    depth_limit = ue_h->depth_limit;
    depth_counter = ue_h->depth_counter;
    num_urls = list->sz;
    for (x=0; x<list->sz; x++) {
        url   = lm_ulist_row(list, x);
        match = 0;
        if ((ft = lm_ftindex_match_by_url(&cr->ftindex, url))) {
            if (ft == LM_FTINDEX_POSSIBLE_MATCH) {
                if (!syn) {
                    if (lm_multipeek_add(w->io_h, url, x) == M_OK) {
                        match = 1;
                        lookup ++;
                    }
                } else {
                    lm_io_head(w->io_h, url);
                    char *mime = w->io_h->transfer.headers.content_type;
                    if (mime) {
                        if ((c = strchr(mime, ';')))
                            *c = '\0';
                        if ((ft = lm_ftindex_match_by_mime(&cr->ftindex, mime))
                                && lm_worker_bind_url(w, url, ft, epeek, &peek_list) == 0)
                            match = 1;
                    }
                }
            } else {
                if (lm_worker_bind_url(w, url, ft, epeek, &peek_list) == 0) {
                    match = 1;
                }
                list = &w->ue_h->primary.row[list_index];
            }
        }
        if (!match) {
            lm_url_nullify(url);
        }
    }
    if (!lookup) {
        goto cleanup;
    }
    ioprivate_t *info;
    CURL        *h;
#ifdef DEBUG
    if (!syn)
        fprintf(stderr, "* worker:(%p) waiting for %d HTTP HEAD requests...\n", w, lookup);
#endif
    while ((info = lm_multipeek_wait(w->io_h))) {
        h   = info->handle;
        url = lm_ulist_row(list, info->identifier);
        match = 0;
        curl_easy_getinfo(h, CURLINFO_CONTENT_TYPE, &mime);
        if (mime) {
            if ((c = strchr(mime, ';')))
                *c = '\0';
            if ((ft = lm_ftindex_match_by_mime(&cr->ftindex, mime))
                    && lm_worker_bind_url(w, url, ft, epeek, &peek_list) == 0)
                match = 1;
            list = &w->ue_h->primary.row[list_index];
        }
        curl_easy_cleanup(h);
        if (!match)
            lm_url_nullify(url);
    }
cleanup:
    if (depth_limit) {
        if (depth_counter >= depth_limit) {
            for (x=0; x<list->sz; x++) {
                url   = lm_ulist_row(list, x);
                if (!url->sz) {
                    continue;
                }
                ft = w->m->filetypes[url->bind-1];
                if (!FT_FLAG_ISSET(ft, FT_FLAG_IGNORE_DEPTH)) {
                    lm_url_nullify(url);
                }
            }
        }
    }
    x = list->sz-1;
    while (x >= 0 && !(lm_ulist_row(list, x)->sz))
        x--;
    list->sz = x+1;
    for (x=0; x<list->sz; ) {
        if (!(lm_ulist_row(list, x)->sz)) {
            lm_url_swap(lm_ulist_row(list, x), lm_ulist_top(list));
            lm_ulist_dec(list);
        } else
            x++;
    }
    int discarded = num_urls - list->sz;
    w->num_discarded_urls += discarded;
    if (w->m->report_cb && current_url && (w->io_h->transfer.status_code < 300 || w->io_h->transfer.status_code >= 400)) {
        w->m->report_cb(w->m, current_url, LMREPORT_NUM_DISCARDED_URLS, discarded);
    }
    return M_OK;
}
static M_CODE lm_worker_sort ( worker_t * w ) { int x , epeek ; ulist_t * list , * peek_list = 0 ; crawler_t * cr ; int match , lookup ; int syn ; uehandle_t * ue_h = w -> ue_h ; filetype_t * ft ; url_t * url ; const char * mime ; char * c ; int num_urls ; int depth_limit , depth_counter ; if ( ! ( list = lm_utable_top ( & ue_h -> primary ) ) ) return M_FAILED ; int list_index = ue_h -> primary . sz - 1 ; cr = w -> crawler ; epeek = ( lm_crawler_flag_isset ( cr , LM_CRFLAG_EPEEK ) && ! ue_h -> is_peeking ) ? 1 : 0 ; syn = w -> io_h -> io -> synchronous ; lookup = 0 ; url_t * current_url = w -> ue_h -> current ; depth_limit = ue_h -> depth_limit ; depth_counter = ue_h -> depth_counter ; num_urls = list -> sz ; for ( x = 0 ; x < list -> sz ; x ++ ) { url = lm_ulist_row ( list , x ) ; match = 0 ; if ( ( ft = lm_ftindex_match_by_url ( & cr -> ftindex , url ) ) ) { if ( ft == LM_FTINDEX_POSSIBLE_MATCH ) { if ( ! syn ) { if ( lm_multipeek_add ( w -> io_h , url , x ) == M_OK ) { match = 1 ; lookup ++ ; } } else { lm_io_head ( w -> io_h , url ) ; char * mime = w -> io_h -> transfer . headers . content_type ; if ( mime ) { if ( ( c = strchr ( mime , ' ' ) ) ) * c = ' \0 ' ; if ( ( ft = lm_ftindex_match_by_mime ( & cr -> ftindex , mime ) ) && lm_worker_bind_url ( w , url , ft , epeek , & peek_list ) == 0 ) match = 1 ; } } } else { if ( lm_worker_bind_url ( w , url , ft , epeek , & peek_list ) == 0 ) { match = 1 ; } list = & w -> ue_h -> primary . row [ list_index ] ; } } if ( ! match ) { lm_url_nullify ( url ) ; } } if ( ! lookup ) { goto cleanup ; } ioprivate_t * info ; CURL * h ; #ifdef DEBUG if ( ! syn ) fprintf ( stderr , " \n " , w , lookup ) ; #endif while ( ( info = lm_multipeek_wait ( w -> io_h ) ) ) { h = info -> handle ; url = lm_ulist_row ( list , info -> identifier ) ; match = 0 ; curl_easy_getinfo ( h , CURLINFO_CONTENT_TYPE , & mime ) ; if ( mime ) { if ( ( c = strchr ( mime , ' ' ) ) ) * c = ' \0 ' ; if ( ( ft = lm_ftindex_match_by_mime ( & cr -> ftindex , mime ) ) && lm_worker_bind_url ( w , url , ft , epeek , & peek_list ) == 0 ) match = 1 ; list = & w -> ue_h -> primary . row [ list_index ] ; } curl_easy_cleanup ( h ) ; if ( ! match ) lm_url_nullify ( url ) ; } cleanup : if ( depth_limit ) { if ( depth_counter >= depth_limit ) { for ( x = 0 ; x < list -> sz ; x ++ ) { url = lm_ulist_row ( list , x ) ; if ( ! url -> sz ) { continue ; } ft = w -> m -> filetypes [ url -> bind - 1 ] ; if ( ! FT_FLAG_ISSET ( ft , FT_FLAG_IGNORE_DEPTH ) ) { lm_url_nullify ( url ) ; } } } } x = list -> sz - 1 ; while ( x >= 0 && ! ( lm_ulist_row ( list , x ) -> sz ) ) x -- ; list -> sz = x + 1 ; for ( x = 0 ; x < list -> sz ; ) { if ( ! ( lm_ulist_row ( list , x ) -> sz ) ) { lm_url_swap ( lm_ulist_row ( list , x ) , lm_ulist_top ( list ) ) ; lm_ulist_dec ( list ) ; } else x ++ ; } int discarded = num_urls - list -> sz ; w -> num_discarded_urls += discarded ; if ( w -> m -> report_cb && current_url && ( w -> io_h -> transfer . status_code < 300 || w -> io_h -> transfer . status_code >= 400 ) ) { w -> m -> report_cb ( w -> m , current_url , LMREPORT_NUM_DISCARDED_URLS , discarded ) ; } return M_OK ; }
Discard or sort URLs into filetypes depending
 on the current crawler's list of filetypes
 and rules.
Discard or sort URLs into filetypes depending on the current crawler ' s list of filetypes and rules .
/* get the current list of URLs */ /* need to cache these here since invocation of epeek might change them during */ /* first we try to match the URL by looking at the string */ /* finally, remove everything that is honoring depth_limit */ /* || !epeek*/ /* remove nullified urls from the list */