def gather_file_data(config):
    """ Gather policy information from files
    """
    file_regex = re.compile(config['file_regex'])
    category_regex = re.compile(config['category_regex'])
    policies = {}

    for root, dirs, files in os.walk(config['c7n_policy_directory']):
        for file in files:
            if file_regex.match(file):
                file_path = root + '/' + file
                logging.debug('Processing file %s', file_path)
                with open(file_path, 'r') as stream:
                    try:
                        if category_regex.search(file_path):
                            category = 'Security & Governance'
                        else:
                            category = 'Cost Controls'

                        policies = yaml.load(stream)
                        for policy in policies['policies']:
                            logging.debug(
                                'Processing policy %s', policy['name'])
                            policy['file_url'] = get_file_url(
                                file_path, config)
                            resource_type = policy['resource']
                            if category not in c7n_data:
                                c7n_data[category] = {}
                            if resource_type not in c7n_data[category]:
                                c7n_data[category][resource_type] = []
                            c7n_data[category][resource_type].append(policy)
                    except yaml.YAMLError as exc:
                        logging.error(exc)
Gather policy information from files