prospector/processing/nlp.py
def extract_path_tokens(text: str, strict_extensions: bool = False) -> List[str]:
    """
    Used to look for paths in the text (i.e. vulnerability description)

    Input:
        text (str)
        strict_extensions (bool): this function will always extract tokens with (back) slashes,
            but it will only match single file names if they have the correct extension, if this argument is True

    Returns:
        list: a list of paths that are found
    """
    tokens = re.split(r"\s+", text)  # split the text into words
    tokens = [
        token.strip(",.:;-+!?)]}'\"") for token in tokens
    ]  # removing common punctuation marks
    paths = []
    for token in tokens:
        contains_path_separators = ("\\" in token) or ("/" in token)
        separated_with_period = "." in token
        has_relevant_extension = token.split(".")[-1] in RELEVANT_EXTENSIONS
        is_xml_tag = token.startswith("<")
        is_property = token.endswith("=")

        is_path = contains_path_separators or (
            has_relevant_extension if strict_extensions else separated_with_period
        )
        probably_not_path = is_xml_tag or is_property
        if is_path and not probably_not_path:
            paths.append(token)
    return paths

    Used to look for paths in the text (i.e. vulnerability description)

    Input:
        text (str)
        strict_extensions (bool): this function will always extract tokens with (back) slashes,
            but it will only match single file names if they have the correct extension, if this argument is True

    Returns:
        list: a list of paths that are found
    
Used to look for paths in the text
Input:
text (str)
strict_extensions (bool): this function will always extract tokens with (back) slashes,
but it will only match single file names if they have the correct extension, if this argument is True
Used to look for paths in the text Input : text ( str ) strict_extensions ( bool ) : this function will always extract tokens with ( back ) slashes but it will only match single file names if they have the correct extension if this argument is True
def extract_path_tokens(text: str, strict_extensions: bool = False) -> List[str]:
    tokens = re.split(r"\s+", text)  
    tokens = [
        token.strip(",.:;-+!?)]}'\"") for token in tokens
    ]  
    paths = []
    for token in tokens:
        contains_path_separators = ("\\" in token) or ("/" in token)
        separated_with_period = "." in token
        has_relevant_extension = token.split(".")[-1] in RELEVANT_EXTENSIONS
        is_xml_tag = token.startswith("<")
        is_property = token.endswith("=")
        is_path = contains_path_separators or (
            has_relevant_extension if strict_extensions else separated_with_period
        )
        probably_not_path = is_xml_tag or is_property
        if is_path and not probably_not_path:
            paths.append(token)
    return paths
def extract_path_tokens ( text : str , strict_extensions : bool = False ) -> List [ str ] : tokens = re . split ( r"\s+" , text ) tokens = [ token . strip ( ",.:;-+!?)]}'\"" ) for token in tokens ] paths = [ ] for token in tokens : contains_path_separators = ( "\\" in token ) or ( "/" in token ) separated_with_period = "." in token has_relevant_extension = token . split ( "." ) [ - 1 ] in RELEVANT_EXTENSIONS is_xml_tag = token . startswith ( "<" ) is_property = token . endswith ( "=" ) is_path = contains_path_separators or ( has_relevant_extension if strict_extensions else separated_with_period ) probably_not_path = is_xml_tag or is_property if is_path and not probably_not_path : paths . append ( token ) return paths
Used to look for paths in the text (i.e.
Used to look for paths in the text ( i . e .
"""
    Used to look for paths in the text (i.e. vulnerability description)

    Input:
        text (str)
        strict_extensions (bool): this function will always extract tokens with (back) slashes,
            but it will only match single file names if they have the correct extension, if this argument is True

    Returns:
        list: a list of paths that are found
    """ # split the text into words # removing common punctuation marks
a list of paths that are found
a list of paths that are found
list
text
str
strict_extensions
bool