'real crawl API
checking kwargs, and repack them to each sub-dict'

def _crawl(self, url, **kwargs):

    task = {}
    assert (len(url) < 1024), 'Maximum    (1024)    URL    length    error.'
    if kwargs.get('callback'):
        callback = kwargs['callback']
        if (isinstance(callback, six.string_types) and hasattr(self, callback)):
            func = getattr(self, callback)
        elif (six.callable(callback) and (six.get_method_self(callback) is self)):
            func = callback
            kwargs['callback'] = func.__name__
        else:
            raise NotImplementedError(('self.%s()    not    implemented!' % callback))
        if hasattr(func, '_config'):
            for (k, v) in iteritems(func._config):
                if (isinstance(v, dict) and isinstance(kwargs.get(k), dict)):
                    kwargs[k].update(v)
                else:
                    kwargs.setdefault(k, v)
    url = quote_chinese(_build_url(url.strip(), kwargs.pop('params', None)))
    if kwargs.get('files'):
        assert isinstance(kwargs.get('data', {}), dict), 'data    must    be    a    dict    when    using    with    files!'
        (content_type, data) = _encode_multipart_formdata(kwargs.pop('data', {}), kwargs.pop('files', {}))
        kwargs.setdefault('headers', {})
        kwargs['headers']['Content-Type'] = content_type
        kwargs['data'] = data
    if kwargs.get('data'):
        kwargs['data'] = _encode_params(kwargs['data'])
    if kwargs.get('data'):
        kwargs.setdefault('method', 'POST')
    if kwargs.get('user_agent'):
        kwargs.setdefault('headers', {})
        kwargs['headers']['User-Agent'] = kwargs.get('user_agent')
    schedule = {}
    for key in self.schedule_fields:
        if (key in kwargs):
            schedule[key] = kwargs.pop(key)
        elif (key in self.crawl_config):
            schedule[key] = self.crawl_config[key]
    task['schedule'] = schedule
    fetch = {}
    for key in self.fetch_fields:
        if (key in kwargs):
            fetch[key] = kwargs.pop(key)
    task['fetch'] = fetch
    process = {}
    for key in self.process_fields:
        if (key in kwargs):
            process[key] = kwargs.pop(key)
    task['process'] = process
    task['project'] = self.project_name
    task['url'] = url
    if ('taskid' in kwargs):
        task['taskid'] = kwargs.pop('taskid')
    else:
        task['taskid'] = self.get_taskid(task)
    if kwargs:
        raise TypeError(('crawl()    got    unexpected    keyword    argument:    %s' % kwargs.keys()))
    if self.is_debugger():
        task = self.task_join_crawl_config(task, self.crawl_config)
    cache_key = ('%(project)s:%(taskid)s' % task)
    if (cache_key not in self._follows_keys):
        self._follows_keys.add(cache_key)
        self._follows.append(task)
    return task
