Tests/scripts/hook_validations/secrets.py
def search_potential_secrets(secrets_file_paths):
    """Returns potential secrets(sensitive data) found in committed and added files
    :param secrets_file_paths: paths of files that are being commited to git repo
    :return: dictionary(filename: (list)secrets) of strings sorted by file name for secrets found in files
    """
    secrets_found = {}
    for file_path in secrets_file_paths:
        # Get if file path in pack and pack name
        is_pack = is_file_path_in_pack(file_path)
        pack_name = get_pack_name(file_path)
        # Get generic/ioc/files white list sets based on if pack or not
        secrets_white_list, ioc_white_list, files_white_list = get_white_listed_items(is_pack, pack_name)
        # Skip white listed files
        if file_path in files_white_list:
            print("Skipping secrets detection for file: {} as it is white listed".format(file_path))
            continue
        # Init vars for current loop
        file_name = os.path.basename(file_path)
        high_entropy_strings = []
        secrets_found_with_regex = []
        _, file_extension = os.path.splitext(file_path)
        skip_secrets = {'skip_once': False, 'skip_multi': False}
        # get file contents
        file_contents = get_file_contents(file_path, file_extension)
        # in packs regard all items as regex as well, reset pack's whitelist in order to avoid repetition later
        if is_pack:
            file_contents = remove_white_list_regex(file_contents, secrets_white_list)
            secrets_white_list = set()
        yml_file_contents = get_related_yml_contents(file_path)
        # Add all context output paths keywords to whitelist temporary
        if file_extension == YML_FILE_EXTENSION or yml_file_contents:
            temp_white_list = create_temp_white_list(yml_file_contents if yml_file_contents else file_contents)
            secrets_white_list = secrets_white_list.union(temp_white_list)
        # Search by lines after strings with high entropy / IoCs regex as possibly suspicious
        for line in file_contents.split('\n'):
            # if detected disable-secrets comments, skip the line/s
            skip_secrets = is_secrets_disabled(line, skip_secrets)
            if skip_secrets['skip_once'] or skip_secrets['skip_multi']:
                skip_secrets['skip_once'] = False
                continue
            # REGEX scanning for IOCs and false positive groups
            regex_secrets, false_positives = regex_for_secrets(line)
            for regex_secret in regex_secrets:
                if not any(ioc.lower() in regex_secret.lower() for ioc in ioc_white_list):
                    secrets_found_with_regex.append(regex_secret)
            # added false positives into white list array before testing the strings in line
            secrets_white_list = secrets_white_list.union(false_positives)
            # due to nature of eml files, skip string by string secret detection - only regex
            if file_extension in SKIP_FILE_TYPE_ENTROPY_CHECKS or \
                    any(demisto_type in file_name for demisto_type in SKIP_DEMISTO_TYPE_ENTROPY_CHECKS):
                continue
            line = remove_false_positives(line)
            # calculate entropy for each string in the file
            for string_ in line.split():
                # compare the lower case of the string against both generic whitelist & temp white list
                if not any(white_list_string.lower() in string_.lower() for white_list_string in secrets_white_list):
                    entropy = calculate_shannon_entropy(string_)
                    if entropy >= ENTROPY_THRESHOLD:
                        high_entropy_strings.append(string_)

        if high_entropy_strings or secrets_found_with_regex:
            # uniquify identical matches between lists
            file_secrets = list(set(high_entropy_strings + secrets_found_with_regex))
            secrets_found[file_name] = file_secrets

    return secrets_found
Returns potential secrets(sensitive data) found in committed and added files
    :param secrets_file_paths: paths of files that are being commited to git repo
    :return: dictionary(filename: (list)secrets) of strings sorted by file name for secrets found in files
    
Returns potential secrets(sensitive data) found in committed and added files
Returns potential secrets ( sensitive data ) found in committed and added files
def search_potential_secrets(secrets_file_paths):
    secrets_found = {}
    for file_path in secrets_file_paths:
        is_pack = is_file_path_in_pack(file_path)
        pack_name = get_pack_name(file_path)
        secrets_white_list, ioc_white_list, files_white_list = get_white_listed_items(is_pack, pack_name)
        if file_path in files_white_list:
            print("Skipping secrets detection for file: {} as it is white listed".format(file_path))
            continue
        file_name = os.path.basename(file_path)
        high_entropy_strings = []
        secrets_found_with_regex = []
        _, file_extension = os.path.splitext(file_path)
        skip_secrets = {'skip_once': False, 'skip_multi': False}
        file_contents = get_file_contents(file_path, file_extension)
        if is_pack:
            file_contents = remove_white_list_regex(file_contents, secrets_white_list)
            secrets_white_list = set()
        yml_file_contents = get_related_yml_contents(file_path)
        if file_extension == YML_FILE_EXTENSION or yml_file_contents:
            temp_white_list = create_temp_white_list(yml_file_contents if yml_file_contents else file_contents)
            secrets_white_list = secrets_white_list.union(temp_white_list)
        for line in file_contents.split('\n'):
            skip_secrets = is_secrets_disabled(line, skip_secrets)
            if skip_secrets['skip_once'] or skip_secrets['skip_multi']:
                skip_secrets['skip_once'] = False
                continue
            regex_secrets, false_positives = regex_for_secrets(line)
            for regex_secret in regex_secrets:
                if not any(ioc.lower() in regex_secret.lower() for ioc in ioc_white_list):
                    secrets_found_with_regex.append(regex_secret)
            secrets_white_list = secrets_white_list.union(false_positives)
            if file_extension in SKIP_FILE_TYPE_ENTROPY_CHECKS or \
                    any(demisto_type in file_name for demisto_type in SKIP_DEMISTO_TYPE_ENTROPY_CHECKS):
                continue
            line = remove_false_positives(line)
            for string_ in line.split():
                if not any(white_list_string.lower() in string_.lower() for white_list_string in secrets_white_list):
                    entropy = calculate_shannon_entropy(string_)
                    if entropy >= ENTROPY_THRESHOLD:
                        high_entropy_strings.append(string_)
        if high_entropy_strings or secrets_found_with_regex:
            file_secrets = list(set(high_entropy_strings + secrets_found_with_regex))
            secrets_found[file_name] = file_secrets
    return secrets_found
def search_potential_secrets ( secrets_file_paths ) : secrets_found = { } for file_path in secrets_file_paths : is_pack = is_file_path_in_pack ( file_path ) pack_name = get_pack_name ( file_path ) secrets_white_list , ioc_white_list , files_white_list = get_white_listed_items ( is_pack , pack_name ) if file_path in files_white_list : print ( "Skipping secrets detection for file: {} as it is white listed" . format ( file_path ) ) continue file_name = os . path . basename ( file_path ) high_entropy_strings = [ ] secrets_found_with_regex = [ ] _ , file_extension = os . path . splitext ( file_path ) skip_secrets = { 'skip_once' : False , 'skip_multi' : False } file_contents = get_file_contents ( file_path , file_extension ) if is_pack : file_contents = remove_white_list_regex ( file_contents , secrets_white_list ) secrets_white_list = set ( ) yml_file_contents = get_related_yml_contents ( file_path ) if file_extension == YML_FILE_EXTENSION or yml_file_contents : temp_white_list = create_temp_white_list ( yml_file_contents if yml_file_contents else file_contents ) secrets_white_list = secrets_white_list . union ( temp_white_list ) for line in file_contents . split ( '\n' ) : skip_secrets = is_secrets_disabled ( line , skip_secrets ) if skip_secrets [ 'skip_once' ] or skip_secrets [ 'skip_multi' ] : skip_secrets [ 'skip_once' ] = False continue regex_secrets , false_positives = regex_for_secrets ( line ) for regex_secret in regex_secrets : if not any ( ioc . lower ( ) in regex_secret . lower ( ) for ioc in ioc_white_list ) : secrets_found_with_regex . append ( regex_secret ) secrets_white_list = secrets_white_list . union ( false_positives ) if file_extension in SKIP_FILE_TYPE_ENTROPY_CHECKS or any ( demisto_type in file_name for demisto_type in SKIP_DEMISTO_TYPE_ENTROPY_CHECKS ) : continue line = remove_false_positives ( line ) for string_ in line . split ( ) : if not any ( white_list_string . lower ( ) in string_ . lower ( ) for white_list_string in secrets_white_list ) : entropy = calculate_shannon_entropy ( string_ ) if entropy >= ENTROPY_THRESHOLD : high_entropy_strings . append ( string_ ) if high_entropy_strings or secrets_found_with_regex : file_secrets = list ( set ( high_entropy_strings + secrets_found_with_regex ) ) secrets_found [ file_name ] = file_secrets return secrets_found
Returns potential secrets(sensitive data) found in committed and added files
Returns potential secrets ( sensitive data ) found in committed and added files
"""Returns potential secrets(sensitive data) found in committed and added files
    :param secrets_file_paths: paths of files that are being commited to git repo
    :return: dictionary(filename: (list)secrets) of strings sorted by file name for secrets found in files
    """ # Get if file path in pack and pack name # Get generic/ioc/files white list sets based on if pack or not # Skip white listed files # Init vars for current loop # get file contents # in packs regard all items as regex as well, reset pack's whitelist in order to avoid repetition later # Add all context output paths keywords to whitelist temporary # Search by lines after strings with high entropy / IoCs regex as possibly suspicious # if detected disable-secrets comments, skip the line/s # REGEX scanning for IOCs and false positive groups # added false positives into white list array before testing the strings in line # due to nature of eml files, skip string by string secret detection - only regex # calculate entropy for each string in the file # compare the lower case of the string against both generic whitelist & temp white list # uniquify identical matches between lists
(list)secrets) of strings sorted by file name for secrets found in files
( list ) secrets ) of strings sorted by file name for secrets found in files
secrets_file_paths
paths of files that are being commited to git repo
paths of files that are being commited to git repo