src/libmetha/worker.c
static M_CODE
lm_worker_perform(worker_t *w)
{
    M_CODE r;
    int x;

    if (w->ue_h->current->bind == 0) {
        LM_WARNING(w->m, "lm_worker_perform called on unbound URL");
        w->ue_h->current->bind = 1;//w->crawler->initial_filetype.ptr->id;
    }

    filetype_t *ft = w->m->filetypes[w->ue_h->current->bind-1];

    uint8_t ft_binding = w->ue_h->current->bind;

    if (lm_crawler_flag_isset(w->crawler, LM_CRFLAG_JAIL)) {
        url_t *url = w->ue_h->current;
        url_t *jail_url = &w->crawler->jail_url;
        if (url->file_o-url->host_o-url->host_l
                < jail_url->file_o-jail_url->host_o-jail_url->host_l
                || strncasecmp(jail_url->str+jail_url->host_o+jail_url->host_l,
                               url->str+url->host_o+url->host_l,
                               jail_url->file_o-jail_url->host_o-jail_url->host_l) != 0) {
            return M_OK;
        }
    }

    if (lm_filter_eval_url(&w->ue_h->host_ent->filter, w->ue_h->current) != LM_FILTER_ALLOW)
        return M_OK;

#ifdef DEBUG
    fprintf(stderr, "* worker:(%p) URL: %s\n", w, w->ue_h->current->str);
#endif

    if (ft->switch_to.ptr)
        lm_worker_set_crawler(w, ft->switch_to.ptr);

    w->io_h->timer_wait = (clock_t)(w->crawler->wait * (float)CLOCKS_PER_SEC);
    w->io_h->timer_wait_mp = 0;
    
    /* prepare the attributes list for this filetype, so
     * that our parsers can fill in values specifically for this
     * url */
    lm_attrlist_prepare(&w->attributes, (const char**)ft->attributes, ft->attr_count);

    /** 
     * call the handler for this filetype, the handler should download
     * the data before we go to the parser chain. If this filetype does
     * not have a handler set, then the default handler for the active
     * crawler is used.
     *
     * TODO: clear all values in 'this' before a javascript handler is
     *       called?
     **/
    wfunction_t *wf = 
        (ft->handler.wf?ft->handler.wf:
            (w->crawler->default_handler.wf?w->crawler->default_handler.wf:0));
    if (wf) {
#ifdef DEBUG
        fprintf(stderr, "* worker:(%p) calling handler '%s'\n", w, wf->name);
#endif
        switch (wf->type) {
            case LM_WFUNCTION_TYPE_NATIVE:
                r = wf->fn.native_handler(w->m, w, w->io_h, &w->io_h->buf, &w->io_h->transfer, w->ue_h->current);
                break;
            default:
                return M_ERROR;
        }
    } else
        r = lm_io_get(w->io_h, w->ue_h->current);

    if (r != M_OK) {
#ifdef DEBUG
        fprintf(stderr, "* worker:(%p) handler or lm_io_get failed: %s (%d)\n", w, lm_strerror(r), r);
#endif
        return r;
    }


    if (w->ue_h->current->bind != ft_binding) {
#ifdef DEBUG
        fprintf(stderr, "* worker:(%p) filetype binding changed from %d to %d\n", w, ft_binding, w->ue_h->current->bind);
#endif
        if (FT_FLAG_ISSET(ft, FT_FLAG_IGNORE_DEPTH)
            && !FT_FLAG_ISSET(w->m->filetypes[w->ue_h->current->bind-1], FT_FLAG_IGNORE_DEPTH)) {
            LM_ERROR(w->m, "switching between filetypes with mismatch ignore_depth setting not allowed, not parsing");
            return M_ERROR;
        }
        ft = w->m->filetypes[w->ue_h->current->bind-1];
        ft_binding = w->ue_h->current->bind;
    }

    w->m->status_cb(w->m, w, w->ue_h->current, &w->io_h->transfer);

    /** 
     * Handler is done, time to do our own logic. If the transfer was over HTTP and we got a 
     * redirect, we will add the URL given by the "Location" header to the URL engine.
     **/
    if (w->io_h->transfer.status_code >= 300
            && w->io_h->transfer.status_code < 400) {
        if (w->io_h->transfer.headers.location) {
#ifdef DEBUG
            fprintf(stderr, "* worker:(%p) %d redirect to '%s'\n",
                    w,
                    w->io_h->transfer.status_code,
                    w->io_h->transfer.headers.location);
#endif
            w->redirects++;
            if (w->redirects >= 20) {
                LM_WARNING(w->m, "breaking out of possible redirect loop");
                w->redirects = 0;
                return M_OK;
            }
            /* add the value of the location header to a 
             * temporary new URL structure, we'll compare it 
             * to the current host name and call ue_move_to_secondary
             * if it is external, otherwise redirect this worker
             * to the URL */
            url_t tmp;
            lm_url_init(&tmp);
            if (lm_url_set(&tmp,
                    w->io_h->transfer.headers.location,
                    strlen(w->io_h->transfer.headers.location)) == M_OK) {
                if (lm_url_hostcmp(&tmp, w->ue_h->current) == 0) {
                    ue_revert(w->ue_h, tmp.str, tmp.sz);
                } else {
                    ue_move_to_secondary(w->ue_h, &tmp);
                }
            }
            lm_url_uninit(&tmp);
            return M_OK;
        }
    }

    w->redirects = 0;

    /**
     * Call each parser in the parser chain, giving the first the
     * data downloaded by the handler. Since a parser might change the 
     * data, we need to make sure the new versions are sent to the 
     * next parser between each. 
     **/
    int last = LM_WFUNCTION_TYPE_NATIVE;
    for (x=0; x<ft->parser_chain.num_parsers; x++) {
        wfunction_t *p = ft->parser_chain.parsers[x];
#ifdef DEBUG
        fprintf(stderr, "* worker:(%p) parser %d of %d: '%s'\n", w, x+1,
                ft->parser_chain.num_parsers,
                p->name);
#endif
        /*TODO: error handling */
        switch (p->type) {
            case LM_WFUNCTION_TYPE_NATIVE:
                p->fn.native_parser(w->m, w, &w->io_h->buf, w->ue_h, w->ue_h->current, &w->attributes);
                break;
            default:
                return M_FAILED;
        }

        last = p->type;
    }

    /* parsing is done. if a parser set any of the filetype attributes
     * using lm_attribute_set, then this file is considered a match, 
     * so we send it to the LMOPT_TARGET_FUNCTION callback */
    if (w->attributes.changed)
        w->m->target_cb(w->m, w, w->ue_h->current, &w->attributes, ft);

    return M_OK;
}
/** 
 * Perform a transfer on one URL
 **/
Perform a transfer on one URL
Perform a transfer on one URL
static M_CODE
lm_worker_perform(worker_t *w)
{
    M_CODE r;
    int x;
    if (w->ue_h->current->bind == 0) {
        LM_WARNING(w->m, "lm_worker_perform called on unbound URL");
        w->ue_h->current->bind = 1;
    }
    filetype_t *ft = w->m->filetypes[w->ue_h->current->bind-1];
    uint8_t ft_binding = w->ue_h->current->bind;
    if (lm_crawler_flag_isset(w->crawler, LM_CRFLAG_JAIL)) {
        url_t *url = w->ue_h->current;
        url_t *jail_url = &w->crawler->jail_url;
        if (url->file_o-url->host_o-url->host_l
                < jail_url->file_o-jail_url->host_o-jail_url->host_l
                || strncasecmp(jail_url->str+jail_url->host_o+jail_url->host_l,
                               url->str+url->host_o+url->host_l,
                               jail_url->file_o-jail_url->host_o-jail_url->host_l) != 0) {
            return M_OK;
        }
    }
    if (lm_filter_eval_url(&w->ue_h->host_ent->filter, w->ue_h->current) != LM_FILTER_ALLOW)
        return M_OK;
#ifdef DEBUG
    fprintf(stderr, "* worker:(%p) URL: %s\n", w, w->ue_h->current->str);
#endif
    if (ft->switch_to.ptr)
        lm_worker_set_crawler(w, ft->switch_to.ptr);
    w->io_h->timer_wait = (clock_t)(w->crawler->wait * (float)CLOCKS_PER_SEC);
    w->io_h->timer_wait_mp = 0;
    lm_attrlist_prepare(&w->attributes, (const char**)ft->attributes, ft->attr_count);
    wfunction_t *wf = 
        (ft->handler.wf?ft->handler.wf:
            (w->crawler->default_handler.wf?w->crawler->default_handler.wf:0));
    if (wf) {
#ifdef DEBUG
        fprintf(stderr, "* worker:(%p) calling handler '%s'\n", w, wf->name);
#endif
        switch (wf->type) {
            case LM_WFUNCTION_TYPE_NATIVE:
                r = wf->fn.native_handler(w->m, w, w->io_h, &w->io_h->buf, &w->io_h->transfer, w->ue_h->current);
                break;
            default:
                return M_ERROR;
        }
    } else
        r = lm_io_get(w->io_h, w->ue_h->current);
    if (r != M_OK) {
#ifdef DEBUG
        fprintf(stderr, "* worker:(%p) handler or lm_io_get failed: %s (%d)\n", w, lm_strerror(r), r);
#endif
        return r;
    }
    if (w->ue_h->current->bind != ft_binding) {
#ifdef DEBUG
        fprintf(stderr, "* worker:(%p) filetype binding changed from %d to %d\n", w, ft_binding, w->ue_h->current->bind);
#endif
        if (FT_FLAG_ISSET(ft, FT_FLAG_IGNORE_DEPTH)
            && !FT_FLAG_ISSET(w->m->filetypes[w->ue_h->current->bind-1], FT_FLAG_IGNORE_DEPTH)) {
            LM_ERROR(w->m, "switching between filetypes with mismatch ignore_depth setting not allowed, not parsing");
            return M_ERROR;
        }
        ft = w->m->filetypes[w->ue_h->current->bind-1];
        ft_binding = w->ue_h->current->bind;
    }
    w->m->status_cb(w->m, w, w->ue_h->current, &w->io_h->transfer);
    if (w->io_h->transfer.status_code >= 300
            && w->io_h->transfer.status_code < 400) {
        if (w->io_h->transfer.headers.location) {
#ifdef DEBUG
            fprintf(stderr, "* worker:(%p) %d redirect to '%s'\n",
                    w,
                    w->io_h->transfer.status_code,
                    w->io_h->transfer.headers.location);
#endif
            w->redirects++;
            if (w->redirects >= 20) {
                LM_WARNING(w->m, "breaking out of possible redirect loop");
                w->redirects = 0;
                return M_OK;
            }
            url_t tmp;
            lm_url_init(&tmp);
            if (lm_url_set(&tmp,
                    w->io_h->transfer.headers.location,
                    strlen(w->io_h->transfer.headers.location)) == M_OK) {
                if (lm_url_hostcmp(&tmp, w->ue_h->current) == 0) {
                    ue_revert(w->ue_h, tmp.str, tmp.sz);
                } else {
                    ue_move_to_secondary(w->ue_h, &tmp);
                }
            }
            lm_url_uninit(&tmp);
            return M_OK;
        }
    }
    w->redirects = 0;
    int last = LM_WFUNCTION_TYPE_NATIVE;
    for (x=0; x<ft->parser_chain.num_parsers; x++) {
        wfunction_t *p = ft->parser_chain.parsers[x];
#ifdef DEBUG
        fprintf(stderr, "* worker:(%p) parser %d of %d: '%s'\n", w, x+1,
                ft->parser_chain.num_parsers,
                p->name);
#endif
        switch (p->type) {
            case LM_WFUNCTION_TYPE_NATIVE:
                p->fn.native_parser(w->m, w, &w->io_h->buf, w->ue_h, w->ue_h->current, &w->attributes);
                break;
            default:
                return M_FAILED;
        }
        last = p->type;
    }
    if (w->attributes.changed)
        w->m->target_cb(w->m, w, w->ue_h->current, &w->attributes, ft);
    return M_OK;
}
static M_CODE lm_worker_perform ( worker_t * w ) { M_CODE r ; int x ; if ( w -> ue_h -> current -> bind == 0 ) { LM_WARNING ( w -> m , " " ) ; w -> ue_h -> current -> bind = 1 ; } filetype_t * ft = w -> m -> filetypes [ w -> ue_h -> current -> bind - 1 ] ; uint8_t ft_binding = w -> ue_h -> current -> bind ; if ( lm_crawler_flag_isset ( w -> crawler , LM_CRFLAG_JAIL ) ) { url_t * url = w -> ue_h -> current ; url_t * jail_url = & w -> crawler -> jail_url ; if ( url -> file_o - url -> host_o - url -> host_l < jail_url -> file_o - jail_url -> host_o - jail_url -> host_l || strncasecmp ( jail_url -> str + jail_url -> host_o + jail_url -> host_l , url -> str + url -> host_o + url -> host_l , jail_url -> file_o - jail_url -> host_o - jail_url -> host_l ) != 0 ) { return M_OK ; } } if ( lm_filter_eval_url ( & w -> ue_h -> host_ent -> filter , w -> ue_h -> current ) != LM_FILTER_ALLOW ) return M_OK ; #ifdef DEBUG fprintf ( stderr , " \n " , w , w -> ue_h -> current -> str ) ; #endif if ( ft -> switch_to . ptr ) lm_worker_set_crawler ( w , ft -> switch_to . ptr ) ; w -> io_h -> timer_wait = ( clock_t ) ( w -> crawler -> wait * ( float ) CLOCKS_PER_SEC ) ; w -> io_h -> timer_wait_mp = 0 ; lm_attrlist_prepare ( & w -> attributes , ( const char * * ) ft -> attributes , ft -> attr_count ) ; wfunction_t * wf = ( ft -> handler . wf ? ft -> handler . wf : ( w -> crawler -> default_handler . wf ? w -> crawler -> default_handler . wf : 0 ) ) ; if ( wf ) { #ifdef DEBUG fprintf ( stderr , " \n " , w , wf -> name ) ; #endif switch ( wf -> type ) { case LM_WFUNCTION_TYPE_NATIVE : r = wf -> fn . native_handler ( w -> m , w , w -> io_h , & w -> io_h -> buf , & w -> io_h -> transfer , w -> ue_h -> current ) ; break ; default : return M_ERROR ; } } else r = lm_io_get ( w -> io_h , w -> ue_h -> current ) ; if ( r != M_OK ) { #ifdef DEBUG fprintf ( stderr , " \n " , w , lm_strerror ( r ) , r ) ; #endif return r ; } if ( w -> ue_h -> current -> bind != ft_binding ) { #ifdef DEBUG fprintf ( stderr , " \n " , w , ft_binding , w -> ue_h -> current -> bind ) ; #endif if ( FT_FLAG_ISSET ( ft , FT_FLAG_IGNORE_DEPTH ) && ! FT_FLAG_ISSET ( w -> m -> filetypes [ w -> ue_h -> current -> bind - 1 ] , FT_FLAG_IGNORE_DEPTH ) ) { LM_ERROR ( w -> m , " " ) ; return M_ERROR ; } ft = w -> m -> filetypes [ w -> ue_h -> current -> bind - 1 ] ; ft_binding = w -> ue_h -> current -> bind ; } w -> m -> status_cb ( w -> m , w , w -> ue_h -> current , & w -> io_h -> transfer ) ; if ( w -> io_h -> transfer . status_code >= 300 && w -> io_h -> transfer . status_code < 400 ) { if ( w -> io_h -> transfer . headers . location ) { #ifdef DEBUG fprintf ( stderr , " \n " , w , w -> io_h -> transfer . status_code , w -> io_h -> transfer . headers . location ) ; #endif w -> redirects ++ ; if ( w -> redirects >= 20 ) { LM_WARNING ( w -> m , " " ) ; w -> redirects = 0 ; return M_OK ; } url_t tmp ; lm_url_init ( & tmp ) ; if ( lm_url_set ( & tmp , w -> io_h -> transfer . headers . location , strlen ( w -> io_h -> transfer . headers . location ) ) == M_OK ) { if ( lm_url_hostcmp ( & tmp , w -> ue_h -> current ) == 0 ) { ue_revert ( w -> ue_h , tmp . str , tmp . sz ) ; } else { ue_move_to_secondary ( w -> ue_h , & tmp ) ; } } lm_url_uninit ( & tmp ) ; return M_OK ; } } w -> redirects = 0 ; int last = LM_WFUNCTION_TYPE_NATIVE ; for ( x = 0 ; x < ft -> parser_chain . num_parsers ; x ++ ) { wfunction_t * p = ft -> parser_chain . parsers [ x ] ; #ifdef DEBUG fprintf ( stderr , " \n " , w , x + 1 , ft -> parser_chain . num_parsers , p -> name ) ; #endif switch ( p -> type ) { case LM_WFUNCTION_TYPE_NATIVE : p -> fn . native_parser ( w -> m , w , & w -> io_h -> buf , w -> ue_h , w -> ue_h -> current , & w -> attributes ) ; break ; default : return M_FAILED ; } last = p -> type ; } if ( w -> attributes . changed ) w -> m -> target_cb ( w -> m , w , w -> ue_h -> current , & w -> attributes , ft ) ; return M_OK ; }
Perform a transfer on one URL
Perform a transfer on one URL
//w->crawler->initial_filetype.ptr->id; /* prepare the attributes list for this filetype, so
     * that our parsers can fill in values specifically for this
     * url */ /** 
     * call the handler for this filetype, the handler should download
     * the data before we go to the parser chain. If this filetype does
     * not have a handler set, then the default handler for the active
     * crawler is used.
     *
     * TODO: clear all values in 'this' before a javascript handler is
     *       called?
     **/ /** 
     * Handler is done, time to do our own logic. If the transfer was over HTTP and we got a 
     * redirect, we will add the URL given by the "Location" header to the URL engine.
     **/ /* add the value of the location header to a 
             * temporary new URL structure, we'll compare it 
             * to the current host name and call ue_move_to_secondary
             * if it is external, otherwise redirect this worker
             * to the URL */ /**
     * Call each parser in the parser chain, giving the first the
     * data downloaded by the handler. Since a parser might change the 
     * data, we need to make sure the new versions are sent to the 
     * next parser between each. 
     **/ /*TODO: error handling */ /* parsing is done. if a parser set any of the filetype attributes
     * using lm_attribute_set, then this file is considered a match, 
     * so we send it to the LMOPT_TARGET_FUNCTION callback */