src/libmetha/worker.c
static M_CODE
lm_worker_get_robotstxt(worker_t *w, struct host_ent *ent)
{
    char *url;
    char *s, *e;
    M_CODE status;
    size_t enable = 1; /* set enable if user agent matches */

    char *opt_s, *opt_e, *val_s, *val_e;

    if (!(url = malloc(ent->len+19))) 
        return M_OUT_OF_MEM;

    url_t u;
    u.sz=sprintf(url, "http://%s/robots.txt", ent->str);
    u.protocol = LM_PROTOCOL_HTTP;
    u.str=url;
    
#ifdef DEBUG
    fprintf(stderr, "* worker:(%p) updating filters (%s)\n", w, url);
#endif

    if ((status = lm_io_get(w->io_h, &u)) == M_OK) {
        for (s=w->io_h->buf.ptr, e=w->io_h->buf.ptr+w->io_h->buf.sz;s<e;s++) {
            while (isspace(*s) && s<e)
                s++; 
            if (*s == '#') {
                do s++; while (s < e && *s != '\n');
                continue;
            }
            opt_s = s;
            if (!(s = opt_e = memchr(s, ':', e-s)))
                break;
            s++;
            while (isspace(*s) && s<e)
                s++; 
            val_s = s;
            if (!(val_e = memchr(s, '\n', e-s)))
                val_e = e;

            int len = opt_e - opt_s;

            if (len == 10 && memcmp(opt_s, "User-agent", 10) == 0) {
                /* temporarily set val_e to \0 */

                if (val_e-val_s == 1 && *val_s == '*') {
                    enable = 1;
                } else {
                    char tmp = *val_e;
                    *val_e = '\0';
                    enable = (size_t)strstr(w->m->io.user_agent, val_s);
                    *val_e = tmp;
                }
            } else if (enable) {
                if (len == 8 && memcmp(opt_s, "Disallow", len) == 0) {
#ifdef DEBUG
                    char tmp = *val_e;
                    *val_e = '\0';
                    fprintf(stderr, "* worker:(%p) Disallow '%s' for '%s'\n", w, val_s, ent->str);
                    *val_e = tmp;
#endif
                    lm_filter_add_rule(&ent->filter, LM_FILTER_DENY,
                                       umex_explicit_strstart(val_s, val_e-val_s));
                } else if (len == 5 && memcmp(opt_s, "Allow", len) == 0) {
#ifdef DEBUG
                    char tmp = *val_e;
                    *val_e = '\0';
                    fprintf(stderr, "* worker:(%p) Allow '%s' for '%s'\n", w, val_s, ent->str);
                    *val_e = tmp;
#endif
                    lm_filter_add_rule(&ent->filter, LM_FILTER_ALLOW,
                                       umex_explicit_strstart(val_s, val_e-val_s));
                }
            }

            s=val_e;
        }
    }

    ent->rfetched = 1;
    free(url);
    return status;
}
/** 
 * Fetch and parse a robots.txt file for the given domain, 
 * update the host_ent's filter to reflect the rules in 
 * the robots.txt file.
 **/
Fetch and parse a robots.txt file for the given domain,
update the host_ent's filter to reflect the rules in
the robots.txt file.
Fetch and parse a robots . txt file for the given domain update the host_ent ' s filter to reflect the rules in the robots . txt file .
static M_CODE
lm_worker_get_robotstxt(worker_t *w, struct host_ent *ent)
{
    char *url;
    char *s, *e;
    M_CODE status;
    size_t enable = 1; 
    char *opt_s, *opt_e, *val_s, *val_e;
    if (!(url = malloc(ent->len+19))) 
        return M_OUT_OF_MEM;
    url_t u;
    u.sz=sprintf(url, "http://%s/robots.txt", ent->str);
    u.protocol = LM_PROTOCOL_HTTP;
    u.str=url;
#ifdef DEBUG
    fprintf(stderr, "* worker:(%p) updating filters (%s)\n", w, url);
#endif
    if ((status = lm_io_get(w->io_h, &u)) == M_OK) {
        for (s=w->io_h->buf.ptr, e=w->io_h->buf.ptr+w->io_h->buf.sz;s<e;s++) {
            while (isspace(*s) && s<e)
                s++; 
            if (*s == '#') {
                do s++; while (s < e && *s != '\n');
                continue;
            }
            opt_s = s;
            if (!(s = opt_e = memchr(s, ':', e-s)))
                break;
            s++;
            while (isspace(*s) && s<e)
                s++; 
            val_s = s;
            if (!(val_e = memchr(s, '\n', e-s)))
                val_e = e;
            int len = opt_e - opt_s;
            if (len == 10 && memcmp(opt_s, "User-agent", 10) == 0) {
                if (val_e-val_s == 1 && *val_s == '*') {
                    enable = 1;
                } else {
                    char tmp = *val_e;
                    *val_e = '\0';
                    enable = (size_t)strstr(w->m->io.user_agent, val_s);
                    *val_e = tmp;
                }
            } else if (enable) {
                if (len == 8 && memcmp(opt_s, "Disallow", len) == 0) {
#ifdef DEBUG
                    char tmp = *val_e;
                    *val_e = '\0';
                    fprintf(stderr, "* worker:(%p) Disallow '%s' for '%s'\n", w, val_s, ent->str);
                    *val_e = tmp;
#endif
                    lm_filter_add_rule(&ent->filter, LM_FILTER_DENY,
                                       umex_explicit_strstart(val_s, val_e-val_s));
                } else if (len == 5 && memcmp(opt_s, "Allow", len) == 0) {
#ifdef DEBUG
                    char tmp = *val_e;
                    *val_e = '\0';
                    fprintf(stderr, "* worker:(%p) Allow '%s' for '%s'\n", w, val_s, ent->str);
                    *val_e = tmp;
#endif
                    lm_filter_add_rule(&ent->filter, LM_FILTER_ALLOW,
                                       umex_explicit_strstart(val_s, val_e-val_s));
                }
            }
            s=val_e;
        }
    }
    ent->rfetched = 1;
    free(url);
    return status;
}
static M_CODE lm_worker_get_robotstxt ( worker_t * w , struct host_ent * ent ) { char * url ; char * s , * e ; M_CODE status ; size_t enable = 1 ; char * opt_s , * opt_e , * val_s , * val_e ; if ( ! ( url = malloc ( ent -> len + 19 ) ) ) return M_OUT_OF_MEM ; url_t u ; u . sz = sprintf ( url , " " , ent -> str ) ; u . protocol = LM_PROTOCOL_HTTP ; u . str = url ; #ifdef DEBUG fprintf ( stderr , " \n " , w , url ) ; #endif if ( ( status = lm_io_get ( w -> io_h , & u ) ) == M_OK ) { for ( s = w -> io_h -> buf . ptr , e = w -> io_h -> buf . ptr + w -> io_h -> buf . sz ; s < e ; s ++ ) { while ( isspace ( * s ) && s < e ) s ++ ; if ( * s == ' ' ) { do s ++ ; while ( s < e && * s != ' \n ' ) ; continue ; } opt_s = s ; if ( ! ( s = opt_e = memchr ( s , ' ' , e - s ) ) ) break ; s ++ ; while ( isspace ( * s ) && s < e ) s ++ ; val_s = s ; if ( ! ( val_e = memchr ( s , ' \n ' , e - s ) ) ) val_e = e ; int len = opt_e - opt_s ; if ( len == 10 && memcmp ( opt_s , " " , 10 ) == 0 ) { if ( val_e - val_s == 1 && * val_s == ' ' ) { enable = 1 ; } else { char tmp = * val_e ; * val_e = ' \0 ' ; enable = ( size_t ) strstr ( w -> m -> io . user_agent , val_s ) ; * val_e = tmp ; } } else if ( enable ) { if ( len == 8 && memcmp ( opt_s , " " , len ) == 0 ) { #ifdef DEBUG char tmp = * val_e ; * val_e = ' \0 ' ; fprintf ( stderr , " \n " , w , val_s , ent -> str ) ; * val_e = tmp ; #endif lm_filter_add_rule ( & ent -> filter , LM_FILTER_DENY , umex_explicit_strstart ( val_s , val_e - val_s ) ) ; } else if ( len == 5 && memcmp ( opt_s , " " , len ) == 0 ) { #ifdef DEBUG char tmp = * val_e ; * val_e = ' \0 ' ; fprintf ( stderr , " \n " , w , val_s , ent -> str ) ; * val_e = tmp ; #endif lm_filter_add_rule ( & ent -> filter , LM_FILTER_ALLOW , umex_explicit_strstart ( val_s , val_e - val_s ) ) ; } } s = val_e ; } } ent -> rfetched = 1 ; free ( url ) ; return status ; }
Fetch and parse a robots.txt file for the given domain,
 update the host_ent's filter to reflect the rules in
 the robots.txt file.
Fetch and parse a robots . txt file for the given domain update the host_ent ' s filter to reflect the rules in the robots . txt file .
/* set enable if user agent matches */ /* temporarily set val_e to \0 */