<gh_stars>100-1000
#!/usr/bin/python3

from .config_utils import get_base_config
from .log_utils import get_module_logger
from .string_utils import get_host_from_url
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse


import os
import requests
import sys


CDIR = os.path.dirname(os.path.realpath(__file__))
ROOTDIR = os.path.abspath(os.path.join(CDIR, os.pardir))
BASECONFIG = get_base_config(ROOTDIR)
LOGGING = get_module_logger(__name__)


BAD_CHARS = ['*', '?']
ALLOWED_EXTENSIONS = ['.doc', '.xls', '.docx', '.xlsx', '.pdf', '.js', '.vba', '.vbs', '.hta', '.lnk', '.wsf', '.bin', '.dll', '.exe', '.7z', '.gz', '.rar', '.zip', '.infected', '.virus']


def build_folder_map(base_url, proxies):
    """Request base_url and extract all relevant href values from the page.

   Target page must be a folder listing.

    Params:
    - base_url: (type: string) URL to query.
    - proxies: (type: JSON object) HTTP and HTTPS proxy values.

    Returns:
    - children: (type: string list) list of URLs.
    """
    try:
        LOGGING.info('Requesting: {0}'.format(base_url))

        user_agent = {'User-agent': BASECONFIG.user_agent}

        request = requests.get(base_url, headers=user_agent, proxies=proxies, timeout=(20, 20))

        if request.status_code == 200:
            LOGGING.info('Request OK. Parsing result...')

            children = []

            content = BeautifulSoup(request.text, 'html.parser')
            links = content.find_all('a', href=True)

            for link in links:
                if 'Parent Directory' in link:
                    continue

                href = link.get('href')

                if len(href) > 1 and not any(s in href for s in BAD_CHARS):
                    children.append(urljoin(base_url, href))

            return children

        else:
            LOGGING.warning(
                'Problem connecting to {0}. Status code: {1}. Aborting task.'.format(
                    base_url, request.status_code))

    except requests.exceptions.ConnectionError as e:
        LOGGING.warning(
            'Problem connecting to {0}. Error: {1}'.format(
                base_url, e))

    except Exception as e:
        LOGGING.warning(
            'Problem connecting to {0}. Aborting task.'.format(base_url))
        LOGGING.exception(sys.exc_info())
        LOGGING.exception(type(e))
        LOGGING.exception(e.args)
        LOGGING.exception(e)

    return []


def process_list(entity_list):
    """Split a list of URLs into folder and file lists.

    Params:
    - entity_url: (type: string list) raw list of URLs.

    Returns:
    - folder_list: (type: string list) list of folder URLs.
    - file_list: (type: string list) list of file URLs.
    """
    LOGGING.info('Processing returned entities...')

    folder_list = []
    file_list = []

    for entity in entity_list:
        entity_parts = os.path.split(urlparse(entity).path)

        if entity_parts[1] == '':
            folder_list.append(entity)

        else:
            if any(entity_parts[1].endswith(ext) for ext in ALLOWED_EXTENSIONS):
                file_list.append(entity)

    return folder_list, file_list


def get_file_url_list(root_url):
    """Initiates a spider across an open directory listing.

    Params:
    - root_url: (type: string list) (type: string) URL to query.

    Returns:
    - file_list: (type: string list) list of file URLs.
    """
    LOGGING.info('Starting spider...')

    proxies = {
        'http': '',
        'https': ''
    }

    if get_host_from_url(root_url).endswith('.onion'):
        if not BASECONFIG.use_tor:
            LOGGING.warning('.onion source requires Tor to be enabled.')
            return []

        LOGGING.info('Requests will be made over the Tor network.')

        proxies = {
            'http': 'socks5h://{0}:{1}'.format(BASECONFIG.tor_ip, int(BASECONFIG.tor_port)),
            'https': 'socks5h://{0}:{1}'.format(BASECONFIG.tor_ip, int(BASECONFIG.tor_port))
        }

    initial_list = build_folder_map(root_url, proxies)
    folder_list, file_list = process_list(initial_list)

    if len(folder_list) > 0:
        while len(folder_list) > 0:
            for folder in folder_list:
                child_list = build_folder_map(folder, proxies)
                child_folders, child_files = process_list(child_list)

                if len(child_files) > 0:
                    file_list.extend(child_files)

                if len(child_folders) > 0:
                    folder_list.extend(child_folders)

                folder_list.remove(folder)

    LOGGING.info('Spider completed. Discovered {0} files:'.format(len(file_list)))

    return file_list
