<filename>polityzer_tool/config.py
import os
import time

HEADERS = {
    "user-agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.106 Mobile Safari/537.36 (contact xxx at xxx dot com)",
    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
    "accept-encoding": "gzip, deflate, br",
    "accept-language": "en-US,en;q=0.9",
    "cache-control": "max-age=0",
    "sec-fetch-dest": "document",
    "sec-fetch-mode": "navigate",
    "sec-fetch-site": "none",
    "sec-fetch-user": "?1",
    "upgrade-insecure-requests": "1",
}

# chromedriver settings
CHROMEDRIVER_FOLDER = "chromedriver"
CHROMEDRIVER_PATH = os.path.join(CHROMEDRIVER_FOLDER, "chromedriver")  # path to the chromedriver executable

# crawler settings
MAX_DEPTH = 2  # depth to which candidate websites will be crawled by the crawler
DATABASE_FOLDER = "database"  # folder that contains the different database files including the input file with the list of candidate and their website links
CANDIDATE_OFFICE_WEBSITE = os.path.join(
    DATABASE_FOLDER, "candidate_office_website.csv"
)  # input file containing the list of the candidates, their offices and website links that are to be crawled and downloaded
HTML_FOLDER = "html"  # folder where the candidate website pages will be downloaded to

# normal logs settings
LOGS_FOLDER = "logs"  # logs produced during the crawling and analysis
DATABASE_FILE = os.path.join(
    DATABASE_FOLDER, "downloaded_websites.csv"
)  # csv containing the locations of the files that are crawled and saved by the crawler
CRAWLER_LOG_FILE = (
    "scrapy_run" + str(int(time.time())) + ".log"
)  # default name of the logs generated by the crawler, will be overwritten by the name set by the crawler or analysis script (whichever is run first)

# error logs settings
ERROR_FILENAME = "".join(["error", str(int(time.time())), ".csv"])  # name of the error log file
ERROR_FILEPATH = os.path.join(
    LOGS_FOLDER, ERROR_FILENAME
)  # path to the error logs generated during the crawling and analysis


# results settings
RESULTS_FOLDER = "results"  # folder containing the results produced by the different analyzer scripts


# Individual component settings
# Crawler/Downlader
DOWNLOAD_SITES = 1

# privacy_policy_analyzer settings
PRIVACY_POLICY_ANALYSIS = 1
PRIVACY_POLICY_RESULTS = os.path.join(
    RESULTS_FOLDER, "privacy_policy_result.json"
)  # result file that lists whether the candidates have privacy policies in their websites as well as potential privacy policy links found in their website
COPY_PRIVACY_POLICY_FILE = 1  # set this flag to copy the privacy policy file found during analysis to a separate folder to ease further manual analysis of the file
PRIVACY_POLICY_FOLDER = os.path.join(
    RESULTS_FOLDER, "privacy_policies"
)  # folder where the privacy policy files will be copied to if COPY_PRIVACY_POLICY_FILE flag is set above

# link_extractor settings
LINK_EXTRACTOR_ANALYSIS = 1
LINK_EXTRACTOR_RESULTS = os.path.join(RESULTS_FOLDER, "link_extractor_result.json")

# form_extractor settings
FORM_EXTRACTOR_ANALYSIS = 1
FORM_EXTRACTOR_RESULTS = os.path.join(RESULTS_FOLDER, "form_extractor_result.json")
