<NME> scanner.py
<BEF> #!/usr/bin/env python3
#
# scanner.py - part of the FDroid server tools
# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os
import re
import sys
import json
import imghdr

import common
import metadata
from common import BuildException
from common import VCSException

config = None
options = None
from tempfile import TemporaryDirectory
from pathlib import Path
from datetime import datetime, timedelta

from . import _
from . import common
from . import metadata
from .exception import BuildException, VCSException, ConfigurationException
from . import scanner

options = None

DEFAULT_JSON_PER_BUILD = {'errors': [], 'warnings': [], 'infos': []}  # type: ignore
json_per_build = deepcopy(DEFAULT_JSON_PER_BUILD)

MAVEN_URL_REGEX = re.compile(r"""\smaven\s*(?:{.*?(?:setUrl|url)|\((?:url)?)\s*=?\s*(?:uri)?\(?\s*["']?([^\s"']+)["']?[^})]*[)}]""",
                             re.DOTALL)


SCANNER_CACHE_VERSION = 1


def get_gradle_compile_commands(build):
    compileCommands = ['compile',
                       'provided',
                       'apk',
                       'implementation',
                       'classpath',
                       'api',
                       'compileOnly',
                       'runtimeOnly']
    buildTypes = ['', 'release']
    flavors = ['']
    if build.gradle and build.gradle != ['yes']:
        flavors += build.gradle

    commands = [''.join(c) for c in itertools.product(flavors, buildTypes, compileCommands)]
    return [re.compile(r'\s*' + c, re.IGNORECASE) for c in commands]


def get_embedded_classes(apkfile, depth=0):
    """
    Get the list of Java classes embedded into all DEX files.

    :return: set of Java classes names as string
    """
    if depth > 10:  # zipbomb protection
        return {_('Max recursion depth in ZIP file reached: %s') % apkfile}

    archive_regex = re.compile(r'.*\.(aab|aar|apk|apks|jar|war|xapk|zip)$')
    class_regex = re.compile(r'classes.*\.dex')
    classes = set()

    try:
        with TemporaryDirectory() as tmp_dir, zipfile.ZipFile(apkfile, 'r') as apk_zip:
            for info in apk_zip.infolist():
                # apk files can contain apk files, again
                with apk_zip.open(info) as apk_fp:
                    if zipfile.is_zipfile(apk_fp):
                        classes = classes.union(get_embedded_classes(apk_fp, depth + 1))
                        if not archive_regex.search(info.filename):
                            classes.add(
                                'ZIP file without proper file extension: %s'
                                % info.filename
                            )
                        continue

                with apk_zip.open(info.filename) as fp:
                    file_magic = fp.read(3)
                if file_magic == b'dex':
                    if not class_regex.search(info.filename):
                        classes.add('DEX file with fake name: %s' % info.filename)
                    apk_zip.extract(info, tmp_dir)
                    run = common.SdkToolsPopen(
                        ["dexdump", '{}/{}'.format(tmp_dir, info.filename)],
                        output=False,
                    )
                    classes = classes.union(set(re.findall(r'[A-Z]+((?:\w+\/)+\w+)', run.output)))
    except zipfile.BadZipFile as ex:
        return {_('Problem with ZIP file: %s, error %s') % (apkfile, ex)}

    return classes


def _datetime_now():
    """Get datetime.now(), using this funciton allows mocking it for testing."""
    return datetime.utcnow()


def _scanner_cachedir():
    """Get `Path` to fdroidserver cache dir."""
    cfg = common.get_config()
    if not cfg:
        raise ConfigurationException('config not initialized')
    if "cachedir_scanner" not in cfg:
        raise ConfigurationException("could not load 'cachedir_scanner' from config")
    cachedir = Path(cfg["cachedir_scanner"])
    cachedir.mkdir(exist_ok=True, parents=True)
    return cachedir


class SignatureDataMalformedException(Exception):
    pass


class SignatureDataOutdatedException(Exception):
    pass


class SignatureDataCacheMissException(Exception):
    pass


class SignatureDataVersionMismatchException(Exception):
    pass


class SignatureDataController:
    def __init__(self, name, filename, url):
        self.name = name
        self.filename = filename
        self.url = url
        # by default we assume cache is valid indefinitely
        self.cache_duration = timedelta(days=999999)
        self.data = {}

    def check_data_version(self):
        if self.data.get("version") != SCANNER_CACHE_VERSION:
            raise SignatureDataVersionMismatchException()

    def check_last_updated(self):
        """
        Check if the last_updated value is ok and raise an exception if expired or inaccessible.

        :raises SignatureDataMalformedException: when timestamp value is
                                                 inaccessible or not parse-able
        :raises SignatureDataOutdatedException: when timestamp is older then
                                                `self.cache_duration`
        """
        last_updated = self.data.get("last_updated", None)
        if last_updated:
            try:
                last_updated = datetime.fromtimestamp(last_updated)
            except ValueError as e:
                raise SignatureDataMalformedException() from e
            except TypeError as e:
                raise SignatureDataMalformedException() from e
            delta = (last_updated + self.cache_duration) - scanner._datetime_now()
            if delta > timedelta(seconds=0):
                logging.debug(_('next {name} cache update due in {time}').format(
                    name=self.filename, time=delta
                ))
            else:
                raise SignatureDataOutdatedException()

    def fetch(self):
        try:
            self.fetch_signatures_from_web()
            self.write_to_cache()
        except Exception as e:
            raise Exception(_("downloading scanner signatures from '{}' failed").format(self.url)) from e

    def load(self):
        try:
            try:
                self.load_from_cache()
                self.verify_data()
                self.check_last_updated()
            except SignatureDataCacheMissException:
                self.load_from_defaults()
        except SignatureDataOutdatedException:
            self.fetch_signatures_from_web()
            self.write_to_cache()
        except (SignatureDataMalformedException, SignatureDataVersionMismatchException) as e:
            logging.critical(_("scanner cache is malformed! You can clear it with: '{clear}'").format(
                clear='rm -r {}'.format(common.get_config()['cachedir_scanner'])
            ))
            raise e

    def load_from_defaults(self):
        sig_file = (Path(__file__).parent / 'data' / 'scanner' / self.filename).resolve()
        with open(sig_file) as f:
            self.set_data(json.load(f))

    def load_from_cache(self):
        sig_file = scanner._scanner_cachedir() / self.filename
        if not sig_file.exists():
            raise SignatureDataCacheMissException()
        with open(sig_file) as f:
            self.set_data(json.load(f))

    def write_to_cache(self):
        sig_file = scanner._scanner_cachedir() / self.filename
        with open(sig_file, "w", encoding="utf-8") as f:
            json.dump(self.data, f, indent=2)
        logging.debug("write '{}' to cache".format(self.filename))

    def verify_data(self):
        """
        Clean and validate `self.data`.

        Right now this function does just a basic key sanitation.
        """
        self.check_data_version()
        valid_keys = ['timestamp', 'last_updated', 'version', 'signatures', 'cache_duration']

        for k in list(self.data.keys()):
            if k not in valid_keys:
                del self.data[k]

    def set_data(self, new_data):
        self.data = new_data
        if 'cache_duration' in new_data:
            self.cache_duration = timedelta(seconds=new_data['cache_duration'])

    def fetch_signatures_from_web(self):
        if not self.url.startswith("https://"):
            raise Exception(_("can't open non-https url: '{};".format(self.url)))
        logging.debug(_("downloading '{}'").format(self.url))
        with urllib.request.urlopen(self.url) as f:  # nosec B310 scheme filtered above
            self.set_data(json.load(f))
        self.data['last_updated'] = scanner._datetime_now().timestamp()


class ExodusSignatureDataController(SignatureDataController):
    def __init__(self):
        super().__init__('Exodus signatures', 'exodus.yml', 'https://reports.exodus-privacy.eu.org/api/trackers')
        self.cache_duration = timedelta(days=1)  # refresh exodus cache after one day

    def fetch_signatures_from_web(self):
        logging.debug(_("downloading '{}'").format(self.url))

        data = {
            "signatures": {},
            "timestamp": scanner._datetime_now().timestamp(),
            "last_updated": scanner._datetime_now().timestamp(),
            "version": SCANNER_CACHE_VERSION,
        }

        if not self.url.startswith("https://"):
            raise Exception(_("can't open non-https url: '{};".format(self.url)))
        with urllib.request.urlopen(self.url) as f:  # nosec B310 scheme filtered above
            d = json.load(f)
            for tracker in d["trackers"].values():
                if tracker.get('code_signature'):
                    data["signatures"][tracker["name"]] = {
                        "name": tracker["name"],
                        "warn_code_signatures": [tracker["code_signature"]],
                        # exodus also provides network signatures, unused atm.
                        # "network_signatures": [tracker["network_signature"]],
                        "AntiFeatures": ["Tracking"],
                        "license": "NonFree"  # We assume all trackers in exodus
                                              # are non-free, although free
                                              # trackers like piwik, acra,
                                              # etc. might be listed by exodus
                                              # too.
                    }
        self.set_data(data)


class SUSSDataController(SignatureDataController):
    def __init__(self):
        super().__init__(
            'SUSS',
            'suss.json',
            'https://fdroid.gitlab.io/fdroid-suss/suss.json'
        )

    def load_from_defaults(self):
        self.set_data(json.loads(SUSS_DEFAULT))


class ScannerTool():
    def __init__(self):
        self.sdcs = [
            SUSSDataController(),
        ]

        # we could add support for loading additional signature source
        # definitions from config.yml here

        self.load()
        self.compile_regexes()

    def load(self):
        for sdc in self.sdcs:
            sdc.load()

    def compile_regexes(self):
        self.regexs = {
            'err_code_signatures': {},
            'err_gradle_signatures': {},
            'warn_code_signatures': {},
            'warn_gradle_signatures': {},
        }
        for sdc in self.sdcs:
            for signame, sigdef in sdc.data.get('signatures', {}).items():
                for sig in sigdef.get('code_signatures', []):
                    self.regexs['err_code_signatures'][sig] = re.compile('.*' + sig, re.IGNORECASE)
                for sig in sigdef.get('gradle_signatures', []):
                    self.regexs['err_gradle_signatures'][sig] = re.compile('.*' + sig, re.IGNORECASE)
                for sig in sigdef.get('warn_code_signatures', []):
                    self.regexs['warn_code_signatures'][sig] = re.compile('.*' + sig, re.IGNORECASE)
                for sig in sigdef.get('warn_gradle_signatures', []):
                    self.regexs['warn_gradle_signatures'][sig] = re.compile('.*' + sig, re.IGNORECASE)

    def refresh(self):
        for sdc in self.sdcs:
            sdc.fetch_signatures_from_web()
            sdc.write_to_cache()

    def add(self, new_controller: SignatureDataController):
        self.sdcs.append(new_controller)
        self.compile_regexes()


# TODO: change this from singleton instance to dependency injection
# use `_get_tool()` instead of accessing this directly
_SCANNER_TOOL = None


def _get_tool():
    """
    Lazy loading function for getting a ScannerTool instance.

    ScannerTool initialization need to access `common.config` values. Those are only available after initialization through `common.read_config()`. So this factory assumes config was called at an erlier point in time.
    """
    if not scanner._SCANNER_TOOL:
        scanner._SCANNER_TOOL = ScannerTool()
    return scanner._SCANNER_TOOL


def scan_binary(apkfile):
    """Scan output of dexdump for known non-free classes."""
    logging.info(_('Scanning APK with dexdump for known non-free classes.'))
    result = get_embedded_classes(apkfile)
    problems, warnings = 0, 0
    for classname in result:
        for suspect, regexp in _get_tool().regexs['warn_code_signatures'].items():
            if regexp.match(classname):
                logging.debug("Warning: found class '%s'" % classname)
                warnings += 1
        for suspect, regexp in _get_tool().regexs['err_code_signatures'].items():
            if regexp.match(classname):
                logging.debug("Problem: found class '%s'" % classname)
                problems += 1
    if warnings:
        logging.warning(_("Found {count} warnings in {filename}").format(count=warnings, filename=apkfile))
    if problems:
        logging.critical(_("Found {count} problems in {filename}").format(count=problems, filename=apkfile))
    return problems


def scan_source(build_dir, build=metadata.Build()):
    """Scan the source code in the given directory (and all subdirectories).

    Returns
    -------
    the number of fatal problems encountered.
    """
    count = 0

    def suspects_found(s):
        for n, r in _get_tool().regexs['err_gradle_signatures'].items():
            if r.match(s):
                yield n

    allowed_repos = [re.compile(r'^https://' + re.escape(repo) + r'/*') for repo in [
        'repo1.maven.org/maven2',  # mavenCentral()
        'jcenter.bintray.com',     # jcenter()
        'jitpack.io',
        'www.jitpack.io',
        'repo.maven.apache.org/maven2',
        'oss.jfrog.org/artifactory/oss-snapshot-local',
        'oss.sonatype.org/content/repositories/snapshots',
        'oss.sonatype.org/content/repositories/releases',
        'oss.sonatype.org/content/groups/public',
        'clojars.org/repo',  # Clojure free software libs
        'repo.clojars.org',  # Clojure free software libs
        's3.amazonaws.com/repo.commonsware.com',  # CommonsWare
        'plugins.gradle.org/m2',  # Gradle plugin repo
        'maven.google.com',  # Google Maven Repo, https://developer.android.com/studio/build/dependencies.html#google-maven
        ]
    ] + [re.compile(r'^file://' + re.escape(repo) + r'/*') for repo in [
        '/usr/share/maven-repo',  # local repo on Debian installs
        ]
    ]

    scanignore = common.getpaths_map(build_dir, build.scanignore)
    scandelete = common.getpaths_map(build_dir, build.scandelete)

    scanignore_worked = set()
    scandelete_worked = set()

    def toignore(path_in_build_dir):
        for k, paths in scanignore.items():
            for p in paths:
                if path_in_build_dir.startswith(p):
                    scanignore_worked.add(k)
                    return True
        return False

    def todelete(path_in_build_dir):
        for k, paths in scandelete.items():
            for p in paths:
                if path_in_build_dir.startswith(p):
                    scandelete_worked.add(k)
                    return True
        return False

    def ignoreproblem(what, path_in_build_dir):
        """No summary.

        Parameters
        ----------
        what: string
          describing the problem, will be printed in log messages
        path_in_build_dir
          path to the file relative to `build`-dir

        Returns
        -------
        0 as we explicitly ignore the file, so don't count an error
        """
        msg = ('Ignoring %s at %s' % (what, path_in_build_dir))
        logging.info(msg)
        if json_per_build is not None:
            json_per_build['infos'].append([msg, path_in_build_dir])
        return 0

    def removeproblem(what, path_in_build_dir, filepath):
        """No summary.

        Parameters
        ----------
        what: string
          describing the problem, will be printed in log messages
        path_in_build_dir
          path to the file relative to `build`-dir
        filepath
          Path (relative to our current path) to the file

        Returns
        -------
        0 as we deleted the offending file
        """
        msg = ('Removing %s at %s' % (what, path_in_build_dir))
        logging.info(msg)
        if json_per_build is not None:
            json_per_build['infos'].append([msg, path_in_build_dir])
        try:
            os.remove(filepath)
        except FileNotFoundError:
            # File is already gone, nothing to do.
            # This can happen if we find multiple problems in one file that is setup for scandelete
            # I.e. build.gradle files containig multiple unknown maven repos.
            pass
        return 0

    def warnproblem(what, path_in_build_dir):
        """No summary.

        Parameters
        ----------
        what: string
          describing the problem, will be printed in log messages
        path_in_build_dir
          path to the file relative to `build`-dir

        Returns
        -------
        0, as warnings don't count as errors
        """
        if toignore(path_in_build_dir):
            return 0
        logging.warning('Found %s at %s' % (what, path_in_build_dir))
        if json_per_build is not None:
            json_per_build['warnings'].append([what, path_in_build_dir])
        return 0

    def handleproblem(what, path_in_build_dir, filepath):
        """Dispatches to problem handlers (ignore, delete, warn).

        Or returns 1 for increasing the error count.

        Parameters
        ----------
        what: string
          describing the problem, will be printed in log messages
        path_in_build_dir
          path to the file relative to `build`-dir
        filepath
          Path (relative to our current path) to the file

        Returns
        -------
        0 if the problem was ignored/deleted/is only a warning, 1 otherwise
        """
        if toignore(path_in_build_dir):
            return ignoreproblem(what, path_in_build_dir)
        if todelete(path_in_build_dir):
            return removeproblem(what, path_in_build_dir, filepath)
        if 'src/test' in path_in_build_dir or '/test/' in path_in_build_dir:
            return warnproblem(what, path_in_build_dir)
        if options and 'json' in vars(options) and options.json:
            json_per_build['errors'].append([what, path_in_build_dir])
        if options and (options.verbose or not ('json' in vars(options) and options.json)):
            logging.error('Found %s at %s' % (what, path_in_build_dir))
        return 1

    def is_executable(path):
        return os.path.exists(path) and os.access(path, os.X_OK)

    textchars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7f})

    def is_binary(path):
        d = None
        with open(path, 'rb') as f:
            d = f.read(1024)
        return bool(d.translate(None, textchars))

    # False positives patterns for files that are binary and executable.
    safe_paths = [re.compile(r) for r in [
        r".*/drawable[^/]*/.*\.png$",  # png drawables
        r".*/mipmap[^/]*/.*\.png$",    # png mipmaps
        ]
    ]

    def is_image_file(path):
        if imghdr.what(path) is not None:
            return True

    def safe_path(path_in_build_dir):
        for sp in safe_paths:
            if sp.match(path_in_build_dir):
                return True
        return False

    gradle_compile_commands = get_gradle_compile_commands(build)

    def is_used_by_gradle(line):
        return any(command.match(line) for command in gradle_compile_commands)

    # Iterate through all files in the source code
    for root, dirs, files in os.walk(build_dir, topdown=True):

        # It's topdown, so checking the basename is enough
        for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
            if ignoredir in dirs:
                dirs.remove(ignoredir)

        for curfile in files:

            if curfile in ['.DS_Store']:
                continue

            # Path (relative) to the file
            filepath = os.path.join(root, curfile)

            if os.path.islink(filepath):
                continue

            path_in_build_dir = os.path.relpath(filepath, build_dir)

            if curfile in ('gradle-wrapper.jar', 'gradlew', 'gradlew.bat'):
                removeproblem(curfile, path_in_build_dir, filepath)
            elif curfile.endswith('.apk'):
                removeproblem(_('Android APK file'), path_in_build_dir, filepath)

            elif curfile.endswith('.a'):
                count += handleproblem(_('static library'), path_in_build_dir, filepath)
            elif curfile.endswith('.aar'):
                count += handleproblem(_('Android AAR library'), path_in_build_dir, filepath)
            elif curfile.endswith('.class'):
                count += handleproblem(_('Java compiled class'), path_in_build_dir, filepath)
            elif curfile.endswith('.dex'):
                count += handleproblem(_('Android DEX code'), path_in_build_dir, filepath)
            elif curfile.endswith('.gz'):
                count += handleproblem(_('gzip file archive'), path_in_build_dir, filepath)
            # We use a regular expression here to also match versioned shared objects like .so.0.0.0
            elif re.match(r'.*\.so(\..+)*$', curfile):
                count += handleproblem(_('shared library'), path_in_build_dir, filepath)
            elif curfile.endswith('.zip'):
                count += handleproblem(_('ZIP file archive'), path_in_build_dir, filepath)
            elif curfile.endswith('.jar'):
                for name in suspects_found(curfile):
                    count += handleproblem('usual suspect \'%s\'' % name, path_in_build_dir, filepath)
                count += handleproblem(_('Java JAR file'), path_in_build_dir, filepath)

            elif curfile.endswith('.java'):
                if not os.path.isfile(filepath):
                    continue
                with open(filepath, 'r', errors='replace') as f:
                    for line in f:
                        if 'DexClassLoader' in line:
                            count += handleproblem('DexClassLoader', path_in_build_dir, filepath)
                            break

            elif curfile.endswith('.gradle') or curfile.endswith('.gradle.kts'):
                if not os.path.isfile(filepath):
                    continue
                with open(filepath, 'r', errors='replace') as f:
                    lines = f.readlines()
                for i, line in enumerate(lines):
                    if is_used_by_gradle(line):
                        for name in suspects_found(line):
                            count += handleproblem("usual suspect \'%s\'" % (name),
                                                   path_in_build_dir, filepath)
                noncomment_lines = [line for line in lines if not common.gradle_comment.match(line)]
                no_comments = re.sub(r'/\*.*?\*/', '', ''.join(noncomment_lines), flags=re.DOTALL)
                for url in MAVEN_URL_REGEX.findall(no_comments):
                    if not any(r.match(url) for r in allowed_repos):
                        count += handleproblem('unknown maven repo \'%s\'' % url, path_in_build_dir, filepath)

            elif os.path.splitext(path_in_build_dir)[1] in ['', '.bin', '.out', '.exe']:
                if is_binary(filepath):
                    count += handleproblem('binary', path_in_build_dir, filepath)

            elif is_executable(filepath):
                if is_binary(filepath) and not (safe_path(path_in_build_dir) or is_image_file(filepath)):
                    warnproblem(_('executable binary, possibly code'), path_in_build_dir)

    for p in scanignore:
        if p not in scanignore_worked:
            logging.error(_('Unused scanignore path: %s') % p)
            count += 1

    for p in scandelete:
        if p not in scandelete_worked:
            logging.error(_('Unused scandelete path: %s') % p)
            count += 1

    return count


def main():
    global options, json_per_build

    # Parse command line...
    parser = ArgumentParser(
        usage="%(prog)s [options] [(APPID[:VERCODE] | path/to.apk) ...]"
    )
    common.setup_global_opts(parser)
    parser.add_argument("appid", nargs='*', help=_("application ID with optional versionCode in the form APPID[:VERCODE]"))
    parser.add_argument(
        "--exodus",
        action="store_true",
        help="Use tracker scanner from Exodus project (requires internet)",
    )
    parser.add_argument("-f", "--force", action="store_true", default=False,
                        help=_("Force scan of disabled apps and builds."))
    parser.add_argument("--json", action="store_true", default=False,
                        help=_("Output JSON to stdout."))
    parser.add_argument("--refresh", "-r", action="store_true", default=False,
                        help=_("fetch the latest version of signatures from the web"))
    metadata.add_metadata_arguments(parser)
    options = parser.parse_args()
    metadata.warnings_action = options.W

    json_output = dict()
    if options.json:
        if options.verbose:
            logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
        else:
            logging.getLogger().setLevel(logging.ERROR)

    # initialize/load configuration values
    common.get_config(opts=options)

    if options.refresh:
        scanner._get_tool().refresh()
    if options.exodus:
        c = ExodusSignatureDataController()
        if options.refresh:
            c.fetch_signatures_from_web()
        else:
            c.fetch()
        scanner._get_tool().add(c)

    probcount = 0

    appids = []
    for apk in options.appid:
        if os.path.isfile(apk):
            count = scanner.scan_binary(apk)
            if count > 0:
                logging.warning(
                    _('Scanner found {count} problems in {apk}').format(
                        count=count, apk=apk
                    )
                )
                probcount += count
        else:
            appids.append(apk)

    if not appids:
        return

    # Read all app and srclib metadata

    allapps = metadata.read_metadata()
    apps = common.read_app_args(appids, allapps, True)

    build_dir = 'build'
    if not os.path.isdir(build_dir):
        logging.info("Creating build directory")
        os.makedirs(build_dir)
    srclib_dir = os.path.join(build_dir, 'srclib')
    extlib_dir = os.path.join(build_dir, 'extlib')

    for appid, app in apps.items():

        json_per_appid = dict()

        if app.Disabled and not options.force:
            logging.info(_("Skipping {appid}: disabled").format(appid=appid))
            json_per_appid['disabled'] = json_per_build['infos'].append('Skipping: disabled')
            continue

        try:
            if app.RepoType == 'srclib':
                build_dir = os.path.join('build', 'srclib', app.Repo)
            else:
                build_dir = os.path.join('build', appid)

            if app.get('Builds'):
                logging.info(_("Processing {appid}").format(appid=appid))
                # Set up vcs interface and make sure we have the latest code...
                vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
            else:
                logging.info(_("{appid}: no builds specified, running on current source state")
                             .format(appid=appid))
                json_per_build = deepcopy(DEFAULT_JSON_PER_BUILD)
                json_per_appid['current-source-state'] = json_per_build
                count = scan_source(build_dir)
                if count > 0:
                    logging.warning(_('Scanner found {count} problems in {appid}:')
                                    .format(count=count, appid=appid))
                    probcount += count
                app['Builds'] = []

            for build in app.get('Builds', []):
                json_per_build = deepcopy(DEFAULT_JSON_PER_BUILD)
                json_per_appid[build.versionCode] = json_per_build

                if build.disable and not options.force:
                    logging.info("...skipping version %s - %s" % (
                        build.versionName, build.get('disable', build.commit[1:])))
                    continue

                logging.info("...scanning version " + build.versionName)
                # Prepare the source code...
                common.prepare_source(vcs, app, build,
                                      build_dir, srclib_dir,
                                      extlib_dir, False)

                count = scan_source(build_dir, build)
                if count > 0:
                    logging.warning(_('Scanner found {count} problems in {appid}:{versionCode}:')
                                    .format(count=count, appid=appid, versionCode=build.versionCode))
                    probcount += count

        except BuildException as be:
            logging.warning('Could not scan app %s due to BuildException: %s' % (
                appid, be))
            probcount += 1
        except VCSException as vcse:
            logging.warning('VCS error while scanning app %s: %s' % (appid, vcse))
            probcount += 1
        except Exception:
            logging.warning('Could not scan app %s due to unknown error: %s' % (
                appid, traceback.format_exc()))
            probcount += 1

        for k, v in json_per_appid.items():
            if len(v['errors']) or len(v['warnings']) or len(v['infos']):
                json_output[appid] = json_per_appid
                break

    logging.info(_("Finished"))
    if options.json:
        print(json.dumps(json_output))
    else:
        print(_("%d problems found") % probcount)


if __name__ == "__main__":
    main()


SUSS_DEFAULT = '''{
  "cache_duration": 86400,
  "signatures": {
    "admob": {
      "gradle_signatures": [
        "admob.*sdk.*android"
      ],
      "license": "NonFree"
    },
    "androidx": {
      "gradle_signatures": [
        "androidx.navigation:navigation-dynamic-features",
        "androidx.work:work-gcm"
      ],
      "license": "NonFree"
    },
    "appcenter-push": {
      "gradle_signatures": [
        "appcenter-push"
      ],
      "license": "NonFree"
    },
    "bugsense": {
      "gradle_signatures": [
        "bugsense"
      ],
      "license": "NonFree"
    },
    "cloudrail": {
      "gradle_signatures": [
        "cloudrail"
      ],
      "license": "NonFree"
    },
    "com.android.billing": {
      "code_signatures": [
        "com/android/billing"
      ],
      "license": "NonFree"
    },
    "com.android.billingclient": {
      "gradle_signatures": [
        "com.android.billingclient"
      ],
      "license": "NonFree"
    },
    "com.anjlab.android.iab.v3": {
      "gradle_signatures": [
        "com.anjlab.android.iab.v3:library"
      ],
      "license": "NonFree"
    },
    "com.cloudinary": {
      "gradle_signatures": [
        "com.cloudinary:cloudinary-android"
      ],
      "license": "NonFree"
    },
    "com.evernote": {
      "gradle_signatures": [
        "com.evernote:android-job"
      ],
      "license": "NonFree"
    },
    "com.facebook": {
      "gradle_signatures": [
        "[\\"']com.facebook.android['\\":]"
      ],
      "license": "NonFree"
    },
    "com.github.junrar": {
      "gradle_signatures": [
        "com.github.junrar:junrar"
      ],
      "license": "NonFree"
    },
    "com.github.penn5": {
      "gradle_signatures": [
        "com.github.penn5:donations"
      ],
      "license": "NonFree"
    },
    "com.google.analytics": {
      "code_signatures": [
        "com/google/analytics"
      ],
      "license": "NonFree"
    },
    "com.google.android.exoplayer": {
      "gradle_signatures": [
        "com.google.android.exoplayer:extension-cast",
        "com.google.android.exoplayer:extension-cronet"
      ],
      "license": "NonFree"
    },
    "com.google.android.gms": {
      "code_signatures": [
        "com/google/android/gms"
      ],
      "license": "NonFree"
    },
    "com.google.android.libraries.places": {
      "gradle_signatures": [
        "com.google.android.libraries.places:places"
      ],
      "license": "NonFree"
    },
    "com.google.android.play": {
      "gradle_signatures": [
        "com.google.android.play:app-update",
        "com.google.android.play:core.*"
      ],
      "license": "NonFree"
    },
    "com.google.android.play.core": {
      "code_signatures": [
        "com/google/android/play/core"
      ],
      "license": "NonFree"
    },
    "com.google.firebase": {
      "code_signatures": [
        "com/google/firebase"
      ],
      "license": "NonFree"
    },
    "com.google.mlkit": {
      "gradle_signatures": [
        "com.google.mlkit"
      ],
      "license": "NonFree"
    },
    "com.google.tagmanager": {
      "code_signatures": [
        "com/google/tagmanager"
      ],
      "license": "NonFree"
    },
    "com.hypertrack": {
      "gradle_signatures": [
        "com\\\\.hypertrack(?!:hyperlog)"
      ],
      "license": "NonFree"
    },
    "com.mapbox": {
      "MaintainerNotes": "com.mapbox.mapboxsdk:mapbox-sdk-services seems to be fully under this license:\\nhttps://github.com/mapbox/mapbox-java/blob/main/LICENSE\\n",
      "gradle_signatures": [
        "com\\\\.mapbox(?!\\\\.mapboxsdk:mapbox-sdk-services)"
      ],
      "license": "NonFree"
    },
    "com.onesignal": {
      "gradle_signatures": [
        "com.onesignal:OneSignal"
      ],
      "license": "NonFree"
    },
    "com.tencent.bugly": {
      "gradle_signatures": [
        "com.tencent.bugly"
      ],
      "license": "NonFree"
    },
    "com.umeng.umsdk": {
      "gradle_signatures": [
        "com.umeng.umsdk"
      ],
      "license": "NonFree"
    },
    "com.yandex.android": {
      "gradle_signatures": [
        "com\\\\.yandex\\\\.android(?!:authsdk)"
      ],
      "license": "NonFree"
    },
    "com.yayandroid": {
      "gradle_signatures": [
        "com.yayandroid:LocationManager"
      ],
      "license": "NonFree"
    },
    "crashlytics": {
      "gradle_signatures": [
        "crashlytics"
      ],
      "license": "NonFree"
    },
    "crittercism": {
      "gradle_signatures": [
        "crittercism"
      ],
      "license": "NonFree"
    },
    "firebase": {
      "gradle_signatures": [
        "com(\\\\.google)?\\\\.firebase[.:](?!firebase-jobdispatcher|geofire-java)"
      ],
      "license": "NonFree"
    },
    "flurryagent": {
      "gradle_signatures": [
        "flurryagent"
      ],
      "license": "NonFree"
    },
    "google-ad": {
      "gradle_signatures": [
        "google.*ad.*view"
      ],
      "license": "NonFree"
    },
    "google.admob": {
      "gradle_signatures": [
        "google.*admob"
      ],
      "license": "NonFree"
    },
    "google.play.services": {
      "gradle_signatures": [
        "google.*play.*services"
      ],
      "license": "NonFree"
    },
    "heyzap": {
      "gradle_signatures": [
        "heyzap"
      ],
      "license": "NonFree"
    },
    "io.github.sinaweibosdk": {
      "gradle_signatures": [
        "io.github.sinaweibosdk"
      ],
      "license": "NonFree"
    },
    "io.objectbox": {
      "gradle_signatures": [
        "io.objectbox:objectbox-gradle-plugin"
      ],
      "license": "NonFree"
    },
    "jpct": {
      "gradle_signatures": [
        "jpct.*ae"
      ],
      "license": "NonFree"
    },
    "libspen23": {
      "gradle_signatures": [
        "libspen23"
      ],
      "license": "NonFree"
    },
    "me.pushy": {
      "gradle_signatures": [
        "me.pushy:sdk"
      ],
      "license": "NonFree"
    },
    "org.jetbrains.kotlinx": {
      "gradle_signatures": [
        "org.jetbrains.kotlinx:kotlinx-coroutines-play-services"
      ],
      "license": "NonFree"
    },
    "ouya": {
      "gradle_signatures": [
        "ouya.*sdk"
      ],
      "license": "NonFree"
    },
    "paypal": {
      "gradle_signatures": [
        "paypal.*mpl"
      ],
      "license": "NonFree"
    },
    "xyz.belvi.mobilevision": {
      "gradle_signatures": [
        "xyz.belvi.mobilevision:barcodescanner"
      ],
      "license": "NonFree"
    },
    "youtube": {
      "gradle_signatures": [
        "youtube.*android.*player.*api"
      ],
      "license": "NonFree"
    }
  },
  "timestamp": 1664480104.875586,
  "version": 1,
  "last_updated": 1664480104.875586
}'''

<MSG> Exception handling improvements

* Replace some prints with proper logging messages
* Make VCSException as verbose as BuildException, including error output

<DFF> @@ -24,8 +24,7 @@ import logging
 
 import common
 import metadata
-from common import BuildException
-from common import VCSException
+from common import BuildException, VCSException
 
 config = None
 options = None

{'commit': 'diff --git a/fdroidserver/build.py b/fdroidserver/build.py\nindex 3870d7e3..3c3b96f9 100644\n--- a/fdroidserver/build.py\n+++ b/fdroidserver/build.py\n@@ -1050,20 +1050,23 @@ def main():\n                 logfile.write(str(be))\n                 logfile.close()\n                 reason = str(be).split(\'\\n\', 1)[0] if options.verbose else str(be)\n-                print("Could not build app %s due to BuildException: %s" % (\n+                logging.error("Could not build app %s due to BuildException: %s" % (\n                     app[\'id\'], reason))\n                 if options.stop:\n                     sys.exit(1)\n                 failed_apps[app[\'id\']] = be\n                 wikilog = be.get_wikitext()\n             except VCSException as vcse:\n-                print("VCS error while building app %s: %s" % (app[\'id\'], vcse))\n+                reason = str(vcse).split(\'\\n\', 1)[0] if options.verbose else str(vcse)\n+                logging.error("VCS error while building app %s: %s" % (\n+                    app[\'id\'], reason))\n                 if options.stop:\n                     sys.exit(1)\n                 failed_apps[app[\'id\']] = vcse\n                 wikilog = str(vcse)\n             except Exception as e:\n-                print("Could not build app %s due to unknown error: %s" % (app[\'id\'], traceback.format_exc()))\n+                logging.error("Could not build app %s due to unknown error: %s" % (\n+                    app[\'id\'], traceback.format_exc()))\n                 if options.stop:\n                     sys.exit(1)\n                 failed_apps[app[\'id\']] = e\ndiff --git a/fdroidserver/checkupdates.py b/fdroidserver/checkupdates.py\nindex f154b4a3..6ab4e236 100644\n--- a/fdroidserver/checkupdates.py\n+++ b/fdroidserver/checkupdates.py\n@@ -32,8 +32,7 @@ import logging\n \n import common\n import metadata\n-from common import BuildException\n-from common import VCSException\n+from common import BuildException, VCSException\n from metadata import MetaDataException\n \n \ndiff --git a/fdroidserver/common.py b/fdroidserver/common.py\nindex 1db91797..2cb7de5a 100644\n--- a/fdroidserver/common.py\n+++ b/fdroidserver/common.py\n@@ -433,42 +433,42 @@ class vcs_git(vcs):\n             # Brand new checkout\n             p = FDroidPopen([\'git\', \'clone\', self.remote, self.local])\n             if p.returncode != 0:\n-                raise VCSException("Git clone failed")\n+                raise VCSException("Git clone failed", p.output)\n             self.checkrepo()\n         else:\n             self.checkrepo()\n             # Discard any working tree changes\n             p = SilentPopen([\'git\', \'reset\', \'--hard\'], cwd=self.local)\n             if p.returncode != 0:\n-                raise VCSException("Git reset failed")\n+                raise VCSException("Git reset failed", p.output)\n             # Remove untracked files now, in case they\'re tracked in the target\n             # revision (it happens!)\n             p = SilentPopen([\'git\', \'clean\', \'-dffx\'], cwd=self.local)\n             if p.returncode != 0:\n-                raise VCSException("Git clean failed")\n+                raise VCSException("Git clean failed", p.output)\n             if not self.refreshed:\n                 # Get latest commits and tags from remote\n                 p = FDroidPopen([\'git\', \'fetch\', \'origin\'], cwd=self.local)\n                 if p.returncode != 0:\n-                    raise VCSException("Git fetch failed")\n+                    raise VCSException("Git fetch failed", p.output)\n                 p = SilentPopen([\'git\', \'fetch\', \'--prune\', \'--tags\', \'origin\'], cwd=self.local)\n                 if p.returncode != 0:\n-                    raise VCSException("Git fetch failed")\n+                    raise VCSException("Git fetch failed", p.output)\n                 # Recreate origin/HEAD as git clone would do it, in case it disappeared\n                 p = SilentPopen([\'git\', \'remote\', \'set-head\', \'origin\', \'--auto\'], cwd=self.local)\n                 if p.returncode != 0:\n-                    raise VCSException("Git remote set-head failed")\n+                    raise VCSException("Git remote set-head failed", p.output)\n                 self.refreshed = True\n         # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on\n         # a github repo. Most of the time this is the same as origin/master.\n         rev = str(rev if rev else \'origin/HEAD\')\n         p = SilentPopen([\'git\', \'checkout\', \'-f\', rev], cwd=self.local)\n         if p.returncode != 0:\n-            raise VCSException("Git checkout of \'%s\' failed" % rev)\n+            raise VCSException("Git checkout of \'%s\' failed" % rev, p.output)\n         # Get rid of any uncontrolled files left behind\n         p = SilentPopen([\'git\', \'clean\', \'-dffx\'], cwd=self.local)\n         if p.returncode != 0:\n-            raise VCSException("Git clean failed")\n+            raise VCSException("Git clean failed", p.output)\n \n     def initsubmodules(self):\n         self.checkrepo()\n@@ -491,13 +491,13 @@ class vcs_git(vcs):\n                 ]:\n             p = SilentPopen([\'git\', \'submodule\', \'foreach\', \'--recursive\'] + cmd, cwd=self.local)\n             if p.returncode != 0:\n-                raise VCSException("Git submodule reset failed")\n+                raise VCSException("Git submodule reset failed", p.output)\n         p = FDroidPopen([\'git\', \'submodule\', \'sync\'], cwd=self.local)\n         if p.returncode != 0:\n-            raise VCSException("Git submodule sync failed")\n+            raise VCSException("Git submodule sync failed", p.output)\n         p = FDroidPopen([\'git\', \'submodule\', \'update\', \'--init\', \'--force\', \'--recursive\'], cwd=self.local)\n         if p.returncode != 0:\n-            raise VCSException("Git submodule update failed")\n+            raise VCSException("Git submodule update failed", p.output)\n \n     def gettags(self):\n         self.checkrepo()\n@@ -550,23 +550,23 @@ class vcs_gitsvn(vcs):\n                         gitsvn_cmd += \' -b %s\' % i[9:]\n                 p = SilentPopen([gitsvn_cmd + " %s %s" % (remote_split[0], self.local)], shell=True)\n                 if p.returncode != 0:\n-                    raise VCSException("Git clone failed")\n+                    raise VCSException("Git clone failed", p.output)\n             else:\n                 p = SilentPopen([gitsvn_cmd + " %s %s" % (self.remote, self.local)], shell=True)\n                 if p.returncode != 0:\n-                    raise VCSException("Git clone failed")\n+                    raise VCSException("Git clone failed", p.output)\n             self.checkrepo()\n         else:\n             self.checkrepo()\n             # Discard any working tree changes\n             p = SilentPopen([\'git\', \'reset\', \'--hard\'], cwd=self.local)\n             if p.returncode != 0:\n-                raise VCSException("Git reset failed")\n+                raise VCSException("Git reset failed", p.output)\n             # Remove untracked files now, in case they\'re tracked in the target\n             # revision (it happens!)\n             p = SilentPopen([\'git\', \'clean\', \'-dffx\'], cwd=self.local)\n             if p.returncode != 0:\n-                raise VCSException("Git clean failed")\n+                raise VCSException("Git clean failed", p.output)\n             if not self.refreshed:\n                 # Get new commits, branches and tags from repo\n                 p = SilentPopen([\'%sgit svn fetch %s\' % self.userargs()], cwd=self.local, shell=True)\n@@ -574,7 +574,7 @@ class vcs_gitsvn(vcs):\n                     raise VCSException("Git svn fetch failed")\n                 p = SilentPopen([\'%sgit svn rebase %s\' % self.userargs()], cwd=self.local, shell=True)\n                 if p.returncode != 0:\n-                    raise VCSException("Git svn rebase failed")\n+                    raise VCSException("Git svn rebase failed", p.output)\n                 self.refreshed = True\n \n         rev = str(rev if rev else \'master\')\n@@ -603,17 +603,17 @@ class vcs_gitsvn(vcs):\n                     # Try a plain git checkout as a last resort\n                     p = SilentPopen([\'git\', \'checkout\', rev], cwd=self.local)\n                     if p.returncode != 0:\n-                        raise VCSException("No git treeish found and direct git checkout of \'%s\' failed" % rev)\n+                        raise VCSException("No git treeish found and direct git checkout of \'%s\' failed" % rev, p.output)\n                 else:\n                     # Check out the git rev equivalent to the svn rev\n                     p = SilentPopen([\'git\', \'checkout\', git_rev], cwd=self.local)\n                     if p.returncode != 0:\n-                        raise VCSException("Git svn checkout of \'%s\' failed" % rev)\n+                        raise VCSException("Git svn checkout of \'%s\' failed" % rev, p.output)\n \n         # Get rid of any uncontrolled files left behind\n         p = SilentPopen([\'git\', \'clean\', \'-dffx\'], cwd=self.local)\n         if p.returncode != 0:\n-            raise VCSException("Git clean failed")\n+            raise VCSException("Git clean failed", p.output)\n \n     def gettags(self):\n         self.checkrepo()\n@@ -643,24 +643,24 @@ class vcs_svn(vcs):\n         if not os.path.exists(self.local):\n             p = SilentPopen([\'svn\', \'checkout\', self.remote, self.local] + self.userargs())\n             if p.returncode != 0:\n-                raise VCSException("Svn checkout of \'%s\' failed" % rev)\n+                raise VCSException("Svn checkout of \'%s\' failed" % rev, p.output)\n         else:\n             for svncommand in (\n                     \'svn revert -R .\',\n                     r"svn status | awk \'/\\?/ {print $2}\' | xargs rm -rf"):\n                 p = SilentPopen([svncommand], cwd=self.local, shell=True)\n                 if p.returncode != 0:\n-                    raise VCSException("Svn reset ({0}) failed in {1}".format(svncommand, self.local))\n+                    raise VCSException("Svn reset ({0}) failed in {1}".format(svncommand, self.local), p.output)\n             if not self.refreshed:\n                 p = SilentPopen([\'svn\', \'update\'] + self.userargs(), cwd=self.local)\n                 if p.returncode != 0:\n-                    raise VCSException("Svn update failed")\n+                    raise VCSException("Svn update failed", p.output)\n                 self.refreshed = True\n \n         revargs = list([\'-r\', rev] if rev else [])\n         p = SilentPopen([\'svn\', \'update\', \'--force\'] + revargs + self.userargs(), cwd=self.local)\n         if p.returncode != 0:\n-            raise VCSException("Svn update failed")\n+            raise VCSException("Svn update failed", p.output)\n \n     def getref(self):\n         p = SilentPopen([\'svn\', \'info\'], cwd=self.local)\n@@ -679,15 +679,15 @@ class vcs_hg(vcs):\n         if not os.path.exists(self.local):\n             p = SilentPopen([\'hg\', \'clone\', self.remote, self.local])\n             if p.returncode != 0:\n-                raise VCSException("Hg clone failed")\n+                raise VCSException("Hg clone failed", p.output)\n         else:\n             p = SilentPopen([\'hg status -uS | xargs rm -rf\'], cwd=self.local, shell=True)\n             if p.returncode != 0:\n-                raise VCSException("Hg clean failed")\n+                raise VCSException("Hg clean failed", p.output)\n             if not self.refreshed:\n                 p = SilentPopen([\'hg\', \'pull\'], cwd=self.local)\n                 if p.returncode != 0:\n-                    raise VCSException("Hg pull failed")\n+                    raise VCSException("Hg pull failed", p.output)\n                 self.refreshed = True\n \n         rev = str(rev if rev else \'default\')\n@@ -695,7 +695,7 @@ class vcs_hg(vcs):\n             return\n         p = SilentPopen([\'hg\', \'update\', \'-C\', rev], cwd=self.local)\n         if p.returncode != 0:\n-            raise VCSException("Hg checkout of \'%s\' failed" % rev)\n+            raise VCSException("Hg checkout of \'%s\' failed" % rev, p.output)\n         p = SilentPopen([\'hg\', \'purge\', \'--all\'], cwd=self.local)\n         # Also delete untracked files, we have to enable purge extension for that:\n         if "\'purge\' is provided by the following extension" in p.output:\n@@ -703,9 +703,9 @@ class vcs_hg(vcs):\n                 myfile.write("\\n[extensions]\\nhgext.purge=\\n")\n             p = SilentPopen([\'hg\', \'purge\', \'--all\'], cwd=self.local)\n             if p.returncode != 0:\n-                raise VCSException("HG purge failed")\n+                raise VCSException("HG purge failed", p.output)\n         elif p.returncode != 0:\n-            raise VCSException("HG purge failed")\n+            raise VCSException("HG purge failed", p.output)\n \n     def gettags(self):\n         p = SilentPopen([\'hg\', \'tags\', \'-q\'], cwd=self.local)\n@@ -721,21 +721,21 @@ class vcs_bzr(vcs):\n         if not os.path.exists(self.local):\n             p = SilentPopen([\'bzr\', \'branch\', self.remote, self.local])\n             if p.returncode != 0:\n-                raise VCSException("Bzr branch failed")\n+                raise VCSException("Bzr branch failed", p.output)\n         else:\n             p = SilentPopen([\'bzr\', \'clean-tree\', \'--force\', \'--unknown\', \'--ignored\'], cwd=self.local)\n             if p.returncode != 0:\n-                raise VCSException("Bzr revert failed")\n+                raise VCSException("Bzr revert failed", p.output)\n             if not self.refreshed:\n                 p = SilentPopen([\'bzr\', \'pull\'], cwd=self.local)\n                 if p.returncode != 0:\n-                    raise VCSException("Bzr update failed")\n+                    raise VCSException("Bzr update failed", p.output)\n                 self.refreshed = True\n \n         revargs = list([\'-r\', rev] if rev else [])\n         p = SilentPopen([\'bzr\', \'revert\'] + revargs, cwd=self.local)\n         if p.returncode != 0:\n-            raise VCSException("Bzr revert of \'%s\' failed" % rev)\n+            raise VCSException("Bzr revert of \'%s\' failed" % rev, p.output)\n \n     def gettags(self):\n         p = SilentPopen([\'bzr\', \'tags\'], cwd=self.local)\n@@ -943,7 +943,7 @@ def parse_androidmanifests(paths, ignoreversions=None):\n     return (max_version, max_vercode, max_package)\n \n \n-class BuildException(Exception):\n+class _FDroidException(Exception):\n     def __init__(self, value, detail=None):\n         self.value = value\n         self.detail = detail\n@@ -965,12 +965,12 @@ class BuildException(Exception):\n         return ret\n \n \n-class VCSException(Exception):\n-    def __init__(self, value):\n-        self.value = value\n+class VCSException(_FDroidException):\n+    pass\n \n-    def __str__(self):\n-        return self.value\n+\n+class BuildException(_FDroidException):\n+    pass\n \n \n # Get the specified source library.\ndiff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py\nindex 47bf119a..d35f3ae7 100644\n--- a/fdroidserver/scanner.py\n+++ b/fdroidserver/scanner.py\n@@ -24,8 +24,7 @@ import logging\n \n import common\n import metadata\n-from common import BuildException\n-from common import VCSException\n+from common import BuildException, VCSException\n \n config = None\n options = None\n', 'commit_subject': 'Exception handling improvements', 'repo_name': 'f-droid/fdroidserver', 'committer': {'name': 'Daniel Martí', 'email': '29bf28b42a938ace12a5819da4ed4ba03d82f315@mvdan.cc', 'time_sec': '1404308686', 'tz_offset': '120', 'date': {'seconds': '1404308686'}}, 'commit_tree': '3c5a0aaabe79ec054b4dfa59a41ef7ea6fde4fbb', 'commit_message': 'Exception handling improvements\n\n* Replace some prints with proper logging messages\n* Make VCSException as verbose as BuildException, including error output\n', 'language_name': 'Python', 'license': 'agpl-3.0', 'hunks': '@@ -24,8 +24,7 @@ import logging\n \n import common\n import metadata\n-from common import BuildException\n-from common import VCSException\n+from common import BuildException, VCSException\n \n config = None\n options = None\n', 'addition_count': 1, 'deletion_count': 2, 'src_file': 'a/fdroidserver/scanner.py', 'tgt_file': 'b/fdroidserver/scanner.py', 'file_extension': '.py', 'before_file': '#!/usr/bin/env python3\n#\n# scanner.py - part of the FDroid server tools\n# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU Affero General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n# GNU Affero General Public License for more details.\n#\n# You should have received a copy of the GNU Affero General Public License\n# along with this program.  If not, see <http://www.gnu.org/licenses/>.\n\nimport os\nimport re\nimport sys\nimport json\nimport imghdr\n\nimport common\nimport metadata\nfrom common import BuildException\nfrom common import VCSException\n\nconfig = None\noptions = None\nfrom tempfile import TemporaryDirectory\nfrom pathlib import Path\nfrom datetime import datetime, timedelta\n\nfrom . import _\nfrom . import common\nfrom . import metadata\nfrom .exception import BuildException, VCSException, ConfigurationException\nfrom . import scanner\n\noptions = None\n\nDEFAULT_JSON_PER_BUILD = {\'errors\': [], \'warnings\': [], \'infos\': []}  # type: ignore\njson_per_build = deepcopy(DEFAULT_JSON_PER_BUILD)\n\nMAVEN_URL_REGEX = re.compile(r"""\\smaven\\s*(?:{.*?(?:setUrl|url)|\\((?:url)?)\\s*=?\\s*(?:uri)?\\(?\\s*["\']?([^\\s"\']+)["\']?[^})]*[)}]""",\n                             re.DOTALL)\n\n\nSCANNER_CACHE_VERSION = 1\n\n\ndef get_gradle_compile_commands(build):\n    compileCommands = [\'compile\',\n                       \'provided\',\n                       \'apk\',\n                       \'implementation\',\n                       \'classpath\',\n                       \'api\',\n                       \'compileOnly\',\n                       \'runtimeOnly\']\n    buildTypes = [\'\', \'release\']\n    flavors = [\'\']\n    if build.gradle and build.gradle != [\'yes\']:\n        flavors += build.gradle\n\n    commands = [\'\'.join(c) for c in itertools.product(flavors, buildTypes, compileCommands)]\n    return [re.compile(r\'\\s*\' + c, re.IGNORECASE) for c in commands]\n\n\ndef get_embedded_classes(apkfile, depth=0):\n    """\n    Get the list of Java classes embedded into all DEX files.\n\n    :return: set of Java classes names as string\n    """\n    if depth > 10:  # zipbomb protection\n        return {_(\'Max recursion depth in ZIP file reached: %s\') % apkfile}\n\n    archive_regex = re.compile(r\'.*\\.(aab|aar|apk|apks|jar|war|xapk|zip)$\')\n    class_regex = re.compile(r\'classes.*\\.dex\')\n    classes = set()\n\n    try:\n        with TemporaryDirectory() as tmp_dir, zipfile.ZipFile(apkfile, \'r\') as apk_zip:\n            for info in apk_zip.infolist():\n                # apk files can contain apk files, again\n                with apk_zip.open(info) as apk_fp:\n                    if zipfile.is_zipfile(apk_fp):\n                        classes = classes.union(get_embedded_classes(apk_fp, depth + 1))\n                        if not archive_regex.search(info.filename):\n                            classes.add(\n                                \'ZIP file without proper file extension: %s\'\n                                % info.filename\n                            )\n                        continue\n\n                with apk_zip.open(info.filename) as fp:\n                    file_magic = fp.read(3)\n                if file_magic == b\'dex\':\n                    if not class_regex.search(info.filename):\n                        classes.add(\'DEX file with fake name: %s\' % info.filename)\n                    apk_zip.extract(info, tmp_dir)\n                    run = common.SdkToolsPopen(\n                        ["dexdump", \'{}/{}\'.format(tmp_dir, info.filename)],\n                        output=False,\n                    )\n                    classes = classes.union(set(re.findall(r\'[A-Z]+((?:\\w+\\/)+\\w+)\', run.output)))\n    except zipfile.BadZipFile as ex:\n        return {_(\'Problem with ZIP file: %s, error %s\') % (apkfile, ex)}\n\n    return classes\n\n\ndef _datetime_now():\n    """Get datetime.now(), using this funciton allows mocking it for testing."""\n    return datetime.utcnow()\n\n\ndef _scanner_cachedir():\n    """Get `Path` to fdroidserver cache dir."""\n    cfg = common.get_config()\n    if not cfg:\n        raise ConfigurationException(\'config not initialized\')\n    if "cachedir_scanner" not in cfg:\n        raise ConfigurationException("could not load \'cachedir_scanner\' from config")\n    cachedir = Path(cfg["cachedir_scanner"])\n    cachedir.mkdir(exist_ok=True, parents=True)\n    return cachedir\n\n\nclass SignatureDataMalformedException(Exception):\n    pass\n\n\nclass SignatureDataOutdatedException(Exception):\n    pass\n\n\nclass SignatureDataCacheMissException(Exception):\n    pass\n\n\nclass SignatureDataVersionMismatchException(Exception):\n    pass\n\n\nclass SignatureDataController:\n    def __init__(self, name, filename, url):\n        self.name = name\n        self.filename = filename\n        self.url = url\n        # by default we assume cache is valid indefinitely\n        self.cache_duration = timedelta(days=999999)\n        self.data = {}\n\n    def check_data_version(self):\n        if self.data.get("version") != SCANNER_CACHE_VERSION:\n            raise SignatureDataVersionMismatchException()\n\n    def check_last_updated(self):\n        """\n        Check if the last_updated value is ok and raise an exception if expired or inaccessible.\n\n        :raises SignatureDataMalformedException: when timestamp value is\n                                                 inaccessible or not parse-able\n        :raises SignatureDataOutdatedException: when timestamp is older then\n                                                `self.cache_duration`\n        """\n        last_updated = self.data.get("last_updated", None)\n        if last_updated:\n            try:\n                last_updated = datetime.fromtimestamp(last_updated)\n            except ValueError as e:\n                raise SignatureDataMalformedException() from e\n            except TypeError as e:\n                raise SignatureDataMalformedException() from e\n            delta = (last_updated + self.cache_duration) - scanner._datetime_now()\n            if delta > timedelta(seconds=0):\n                logging.debug(_(\'next {name} cache update due in {time}\').format(\n                    name=self.filename, time=delta\n                ))\n            else:\n                raise SignatureDataOutdatedException()\n\n    def fetch(self):\n        try:\n            self.fetch_signatures_from_web()\n            self.write_to_cache()\n        except Exception as e:\n            raise Exception(_("downloading scanner signatures from \'{}\' failed").format(self.url)) from e\n\n    def load(self):\n        try:\n            try:\n                self.load_from_cache()\n                self.verify_data()\n                self.check_last_updated()\n            except SignatureDataCacheMissException:\n                self.load_from_defaults()\n        except SignatureDataOutdatedException:\n            self.fetch_signatures_from_web()\n            self.write_to_cache()\n        except (SignatureDataMalformedException, SignatureDataVersionMismatchException) as e:\n            logging.critical(_("scanner cache is malformed! You can clear it with: \'{clear}\'").format(\n                clear=\'rm -r {}\'.format(common.get_config()[\'cachedir_scanner\'])\n            ))\n            raise e\n\n    def load_from_defaults(self):\n        sig_file = (Path(__file__).parent / \'data\' / \'scanner\' / self.filename).resolve()\n        with open(sig_file) as f:\n            self.set_data(json.load(f))\n\n    def load_from_cache(self):\n        sig_file = scanner._scanner_cachedir() / self.filename\n        if not sig_file.exists():\n            raise SignatureDataCacheMissException()\n        with open(sig_file) as f:\n            self.set_data(json.load(f))\n\n    def write_to_cache(self):\n        sig_file = scanner._scanner_cachedir() / self.filename\n        with open(sig_file, "w", encoding="utf-8") as f:\n            json.dump(self.data, f, indent=2)\n        logging.debug("write \'{}\' to cache".format(self.filename))\n\n    def verify_data(self):\n        """\n        Clean and validate `self.data`.\n\n        Right now this function does just a basic key sanitation.\n        """\n        self.check_data_version()\n        valid_keys = [\'timestamp\', \'last_updated\', \'version\', \'signatures\', \'cache_duration\']\n\n        for k in list(self.data.keys()):\n            if k not in valid_keys:\n                del self.data[k]\n\n    def set_data(self, new_data):\n        self.data = new_data\n        if \'cache_duration\' in new_data:\n            self.cache_duration = timedelta(seconds=new_data[\'cache_duration\'])\n\n    def fetch_signatures_from_web(self):\n        if not self.url.startswith("https://"):\n            raise Exception(_("can\'t open non-https url: \'{};".format(self.url)))\n        logging.debug(_("downloading \'{}\'").format(self.url))\n        with urllib.request.urlopen(self.url) as f:  # nosec B310 scheme filtered above\n            self.set_data(json.load(f))\n        self.data[\'last_updated\'] = scanner._datetime_now().timestamp()\n\n\nclass ExodusSignatureDataController(SignatureDataController):\n    def __init__(self):\n        super().__init__(\'Exodus signatures\', \'exodus.yml\', \'https://reports.exodus-privacy.eu.org/api/trackers\')\n        self.cache_duration = timedelta(days=1)  # refresh exodus cache after one day\n\n    def fetch_signatures_from_web(self):\n        logging.debug(_("downloading \'{}\'").format(self.url))\n\n        data = {\n            "signatures": {},\n            "timestamp": scanner._datetime_now().timestamp(),\n            "last_updated": scanner._datetime_now().timestamp(),\n            "version": SCANNER_CACHE_VERSION,\n        }\n\n        if not self.url.startswith("https://"):\n            raise Exception(_("can\'t open non-https url: \'{};".format(self.url)))\n        with urllib.request.urlopen(self.url) as f:  # nosec B310 scheme filtered above\n            d = json.load(f)\n            for tracker in d["trackers"].values():\n                if tracker.get(\'code_signature\'):\n                    data["signatures"][tracker["name"]] = {\n                        "name": tracker["name"],\n                        "warn_code_signatures": [tracker["code_signature"]],\n                        # exodus also provides network signatures, unused atm.\n                        # "network_signatures": [tracker["network_signature"]],\n                        "AntiFeatures": ["Tracking"],\n                        "license": "NonFree"  # We assume all trackers in exodus\n                                              # are non-free, although free\n                                              # trackers like piwik, acra,\n                                              # etc. might be listed by exodus\n                                              # too.\n                    }\n        self.set_data(data)\n\n\nclass SUSSDataController(SignatureDataController):\n    def __init__(self):\n        super().__init__(\n            \'SUSS\',\n            \'suss.json\',\n            \'https://fdroid.gitlab.io/fdroid-suss/suss.json\'\n        )\n\n    def load_from_defaults(self):\n        self.set_data(json.loads(SUSS_DEFAULT))\n\n\nclass ScannerTool():\n    def __init__(self):\n        self.sdcs = [\n            SUSSDataController(),\n        ]\n\n        # we could add support for loading additional signature source\n        # definitions from config.yml here\n\n        self.load()\n        self.compile_regexes()\n\n    def load(self):\n        for sdc in self.sdcs:\n            sdc.load()\n\n    def compile_regexes(self):\n        self.regexs = {\n            \'err_code_signatures\': {},\n            \'err_gradle_signatures\': {},\n            \'warn_code_signatures\': {},\n            \'warn_gradle_signatures\': {},\n        }\n        for sdc in self.sdcs:\n            for signame, sigdef in sdc.data.get(\'signatures\', {}).items():\n                for sig in sigdef.get(\'code_signatures\', []):\n                    self.regexs[\'err_code_signatures\'][sig] = re.compile(\'.*\' + sig, re.IGNORECASE)\n                for sig in sigdef.get(\'gradle_signatures\', []):\n                    self.regexs[\'err_gradle_signatures\'][sig] = re.compile(\'.*\' + sig, re.IGNORECASE)\n                for sig in sigdef.get(\'warn_code_signatures\', []):\n                    self.regexs[\'warn_code_signatures\'][sig] = re.compile(\'.*\' + sig, re.IGNORECASE)\n                for sig in sigdef.get(\'warn_gradle_signatures\', []):\n                    self.regexs[\'warn_gradle_signatures\'][sig] = re.compile(\'.*\' + sig, re.IGNORECASE)\n\n    def refresh(self):\n        for sdc in self.sdcs:\n            sdc.fetch_signatures_from_web()\n            sdc.write_to_cache()\n\n    def add(self, new_controller: SignatureDataController):\n        self.sdcs.append(new_controller)\n        self.compile_regexes()\n\n\n# TODO: change this from singleton instance to dependency injection\n# use `_get_tool()` instead of accessing this directly\n_SCANNER_TOOL = None\n\n\ndef _get_tool():\n    """\n    Lazy loading function for getting a ScannerTool instance.\n\n    ScannerTool initialization need to access `common.config` values. Those are only available after initialization through `common.read_config()`. So this factory assumes config was called at an erlier point in time.\n    """\n    if not scanner._SCANNER_TOOL:\n        scanner._SCANNER_TOOL = ScannerTool()\n    return scanner._SCANNER_TOOL\n\n\ndef scan_binary(apkfile):\n    """Scan output of dexdump for known non-free classes."""\n    logging.info(_(\'Scanning APK with dexdump for known non-free classes.\'))\n    result = get_embedded_classes(apkfile)\n    problems, warnings = 0, 0\n    for classname in result:\n        for suspect, regexp in _get_tool().regexs[\'warn_code_signatures\'].items():\n            if regexp.match(classname):\n                logging.debug("Warning: found class \'%s\'" % classname)\n                warnings += 1\n        for suspect, regexp in _get_tool().regexs[\'err_code_signatures\'].items():\n            if regexp.match(classname):\n                logging.debug("Problem: found class \'%s\'" % classname)\n                problems += 1\n    if warnings:\n        logging.warning(_("Found {count} warnings in {filename}").format(count=warnings, filename=apkfile))\n    if problems:\n        logging.critical(_("Found {count} problems in {filename}").format(count=problems, filename=apkfile))\n    return problems\n\n\ndef scan_source(build_dir, build=metadata.Build()):\n    """Scan the source code in the given directory (and all subdirectories).\n\n    Returns\n    -------\n    the number of fatal problems encountered.\n    """\n    count = 0\n\n    def suspects_found(s):\n        for n, r in _get_tool().regexs[\'err_gradle_signatures\'].items():\n            if r.match(s):\n                yield n\n\n    allowed_repos = [re.compile(r\'^https://\' + re.escape(repo) + r\'/*\') for repo in [\n        \'repo1.maven.org/maven2\',  # mavenCentral()\n        \'jcenter.bintray.com\',     # jcenter()\n        \'jitpack.io\',\n        \'www.jitpack.io\',\n        \'repo.maven.apache.org/maven2\',\n        \'oss.jfrog.org/artifactory/oss-snapshot-local\',\n        \'oss.sonatype.org/content/repositories/snapshots\',\n        \'oss.sonatype.org/content/repositories/releases\',\n        \'oss.sonatype.org/content/groups/public\',\n        \'clojars.org/repo\',  # Clojure free software libs\n        \'repo.clojars.org\',  # Clojure free software libs\n        \'s3.amazonaws.com/repo.commonsware.com\',  # CommonsWare\n        \'plugins.gradle.org/m2\',  # Gradle plugin repo\n        \'maven.google.com\',  # Google Maven Repo, https://developer.android.com/studio/build/dependencies.html#google-maven\n        ]\n    ] + [re.compile(r\'^file://\' + re.escape(repo) + r\'/*\') for repo in [\n        \'/usr/share/maven-repo\',  # local repo on Debian installs\n        ]\n    ]\n\n    scanignore = common.getpaths_map(build_dir, build.scanignore)\n    scandelete = common.getpaths_map(build_dir, build.scandelete)\n\n    scanignore_worked = set()\n    scandelete_worked = set()\n\n    def toignore(path_in_build_dir):\n        for k, paths in scanignore.items():\n            for p in paths:\n                if path_in_build_dir.startswith(p):\n                    scanignore_worked.add(k)\n                    return True\n        return False\n\n    def todelete(path_in_build_dir):\n        for k, paths in scandelete.items():\n            for p in paths:\n                if path_in_build_dir.startswith(p):\n                    scandelete_worked.add(k)\n                    return True\n        return False\n\n    def ignoreproblem(what, path_in_build_dir):\n        """No summary.\n\n        Parameters\n        ----------\n        what: string\n          describing the problem, will be printed in log messages\n        path_in_build_dir\n          path to the file relative to `build`-dir\n\n        Returns\n        -------\n        0 as we explicitly ignore the file, so don\'t count an error\n        """\n        msg = (\'Ignoring %s at %s\' % (what, path_in_build_dir))\n        logging.info(msg)\n        if json_per_build is not None:\n            json_per_build[\'infos\'].append([msg, path_in_build_dir])\n        return 0\n\n    def removeproblem(what, path_in_build_dir, filepath):\n        """No summary.\n\n        Parameters\n        ----------\n        what: string\n          describing the problem, will be printed in log messages\n        path_in_build_dir\n          path to the file relative to `build`-dir\n        filepath\n          Path (relative to our current path) to the file\n\n        Returns\n        -------\n        0 as we deleted the offending file\n        """\n        msg = (\'Removing %s at %s\' % (what, path_in_build_dir))\n        logging.info(msg)\n        if json_per_build is not None:\n            json_per_build[\'infos\'].append([msg, path_in_build_dir])\n        try:\n            os.remove(filepath)\n        except FileNotFoundError:\n            # File is already gone, nothing to do.\n            # This can happen if we find multiple problems in one file that is setup for scandelete\n            # I.e. build.gradle files containig multiple unknown maven repos.\n            pass\n        return 0\n\n    def warnproblem(what, path_in_build_dir):\n        """No summary.\n\n        Parameters\n        ----------\n        what: string\n          describing the problem, will be printed in log messages\n        path_in_build_dir\n          path to the file relative to `build`-dir\n\n        Returns\n        -------\n        0, as warnings don\'t count as errors\n        """\n        if toignore(path_in_build_dir):\n            return 0\n        logging.warning(\'Found %s at %s\' % (what, path_in_build_dir))\n        if json_per_build is not None:\n            json_per_build[\'warnings\'].append([what, path_in_build_dir])\n        return 0\n\n    def handleproblem(what, path_in_build_dir, filepath):\n        """Dispatches to problem handlers (ignore, delete, warn).\n\n        Or returns 1 for increasing the error count.\n\n        Parameters\n        ----------\n        what: string\n          describing the problem, will be printed in log messages\n        path_in_build_dir\n          path to the file relative to `build`-dir\n        filepath\n          Path (relative to our current path) to the file\n\n        Returns\n        -------\n        0 if the problem was ignored/deleted/is only a warning, 1 otherwise\n        """\n        if toignore(path_in_build_dir):\n            return ignoreproblem(what, path_in_build_dir)\n        if todelete(path_in_build_dir):\n            return removeproblem(what, path_in_build_dir, filepath)\n        if \'src/test\' in path_in_build_dir or \'/test/\' in path_in_build_dir:\n            return warnproblem(what, path_in_build_dir)\n        if options and \'json\' in vars(options) and options.json:\n            json_per_build[\'errors\'].append([what, path_in_build_dir])\n        if options and (options.verbose or not (\'json\' in vars(options) and options.json)):\n            logging.error(\'Found %s at %s\' % (what, path_in_build_dir))\n        return 1\n\n    def is_executable(path):\n        return os.path.exists(path) and os.access(path, os.X_OK)\n\n    textchars = bytearray({7, 8, 9, 10, 12, 13, 27} | set(range(0x20, 0x100)) - {0x7f})\n\n    def is_binary(path):\n        d = None\n        with open(path, \'rb\') as f:\n            d = f.read(1024)\n        return bool(d.translate(None, textchars))\n\n    # False positives patterns for files that are binary and executable.\n    safe_paths = [re.compile(r) for r in [\n        r".*/drawable[^/]*/.*\\.png$",  # png drawables\n        r".*/mipmap[^/]*/.*\\.png$",    # png mipmaps\n        ]\n    ]\n\n    def is_image_file(path):\n        if imghdr.what(path) is not None:\n            return True\n\n    def safe_path(path_in_build_dir):\n        for sp in safe_paths:\n            if sp.match(path_in_build_dir):\n                return True\n        return False\n\n    gradle_compile_commands = get_gradle_compile_commands(build)\n\n    def is_used_by_gradle(line):\n        return any(command.match(line) for command in gradle_compile_commands)\n\n    # Iterate through all files in the source code\n    for root, dirs, files in os.walk(build_dir, topdown=True):\n\n        # It\'s topdown, so checking the basename is enough\n        for ignoredir in (\'.hg\', \'.git\', \'.svn\', \'.bzr\'):\n            if ignoredir in dirs:\n                dirs.remove(ignoredir)\n\n        for curfile in files:\n\n            if curfile in [\'.DS_Store\']:\n                continue\n\n            # Path (relative) to the file\n            filepath = os.path.join(root, curfile)\n\n            if os.path.islink(filepath):\n                continue\n\n            path_in_build_dir = os.path.relpath(filepath, build_dir)\n\n            if curfile in (\'gradle-wrapper.jar\', \'gradlew\', \'gradlew.bat\'):\n                removeproblem(curfile, path_in_build_dir, filepath)\n            elif curfile.endswith(\'.apk\'):\n                removeproblem(_(\'Android APK file\'), path_in_build_dir, filepath)\n\n            elif curfile.endswith(\'.a\'):\n                count += handleproblem(_(\'static library\'), path_in_build_dir, filepath)\n            elif curfile.endswith(\'.aar\'):\n                count += handleproblem(_(\'Android AAR library\'), path_in_build_dir, filepath)\n            elif curfile.endswith(\'.class\'):\n                count += handleproblem(_(\'Java compiled class\'), path_in_build_dir, filepath)\n            elif curfile.endswith(\'.dex\'):\n                count += handleproblem(_(\'Android DEX code\'), path_in_build_dir, filepath)\n            elif curfile.endswith(\'.gz\'):\n                count += handleproblem(_(\'gzip file archive\'), path_in_build_dir, filepath)\n            # We use a regular expression here to also match versioned shared objects like .so.0.0.0\n            elif re.match(r\'.*\\.so(\\..+)*$\', curfile):\n                count += handleproblem(_(\'shared library\'), path_in_build_dir, filepath)\n            elif curfile.endswith(\'.zip\'):\n                count += handleproblem(_(\'ZIP file archive\'), path_in_build_dir, filepath)\n            elif curfile.endswith(\'.jar\'):\n                for name in suspects_found(curfile):\n                    count += handleproblem(\'usual suspect \\\'%s\\\'\' % name, path_in_build_dir, filepath)\n                count += handleproblem(_(\'Java JAR file\'), path_in_build_dir, filepath)\n\n            elif curfile.endswith(\'.java\'):\n                if not os.path.isfile(filepath):\n                    continue\n                with open(filepath, \'r\', errors=\'replace\') as f:\n                    for line in f:\n                        if \'DexClassLoader\' in line:\n                            count += handleproblem(\'DexClassLoader\', path_in_build_dir, filepath)\n                            break\n\n            elif curfile.endswith(\'.gradle\') or curfile.endswith(\'.gradle.kts\'):\n                if not os.path.isfile(filepath):\n                    continue\n                with open(filepath, \'r\', errors=\'replace\') as f:\n                    lines = f.readlines()\n                for i, line in enumerate(lines):\n                    if is_used_by_gradle(line):\n                        for name in suspects_found(line):\n                            count += handleproblem("usual suspect \\\'%s\\\'" % (name),\n                                                   path_in_build_dir, filepath)\n                noncomment_lines = [line for line in lines if not common.gradle_comment.match(line)]\n                no_comments = re.sub(r\'/\\*.*?\\*/\', \'\', \'\'.join(noncomment_lines), flags=re.DOTALL)\n                for url in MAVEN_URL_REGEX.findall(no_comments):\n                    if not any(r.match(url) for r in allowed_repos):\n                        count += handleproblem(\'unknown maven repo \\\'%s\\\'\' % url, path_in_build_dir, filepath)\n\n            elif os.path.splitext(path_in_build_dir)[1] in [\'\', \'.bin\', \'.out\', \'.exe\']:\n                if is_binary(filepath):\n                    count += handleproblem(\'binary\', path_in_build_dir, filepath)\n\n            elif is_executable(filepath):\n                if is_binary(filepath) and not (safe_path(path_in_build_dir) or is_image_file(filepath)):\n                    warnproblem(_(\'executable binary, possibly code\'), path_in_build_dir)\n\n    for p in scanignore:\n        if p not in scanignore_worked:\n            logging.error(_(\'Unused scanignore path: %s\') % p)\n            count += 1\n\n    for p in scandelete:\n        if p not in scandelete_worked:\n            logging.error(_(\'Unused scandelete path: %s\') % p)\n            count += 1\n\n    return count\n\n\ndef main():\n    global options, json_per_build\n\n    # Parse command line...\n    parser = ArgumentParser(\n        usage="%(prog)s [options] [(APPID[:VERCODE] | path/to.apk) ...]"\n    )\n    common.setup_global_opts(parser)\n    parser.add_argument("appid", nargs=\'*\', help=_("application ID with optional versionCode in the form APPID[:VERCODE]"))\n    parser.add_argument(\n        "--exodus",\n        action="store_true",\n        help="Use tracker scanner from Exodus project (requires internet)",\n    )\n    parser.add_argument("-f", "--force", action="store_true", default=False,\n                        help=_("Force scan of disabled apps and builds."))\n    parser.add_argument("--json", action="store_true", default=False,\n                        help=_("Output JSON to stdout."))\n    parser.add_argument("--refresh", "-r", action="store_true", default=False,\n                        help=_("fetch the latest version of signatures from the web"))\n    metadata.add_metadata_arguments(parser)\n    options = parser.parse_args()\n    metadata.warnings_action = options.W\n\n    json_output = dict()\n    if options.json:\n        if options.verbose:\n            logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)\n        else:\n            logging.getLogger().setLevel(logging.ERROR)\n\n    # initialize/load configuration values\n    common.get_config(opts=options)\n\n    if options.refresh:\n        scanner._get_tool().refresh()\n    if options.exodus:\n        c = ExodusSignatureDataController()\n        if options.refresh:\n            c.fetch_signatures_from_web()\n        else:\n            c.fetch()\n        scanner._get_tool().add(c)\n\n    probcount = 0\n\n    appids = []\n    for apk in options.appid:\n        if os.path.isfile(apk):\n            count = scanner.scan_binary(apk)\n            if count > 0:\n                logging.warning(\n                    _(\'Scanner found {count} problems in {apk}\').format(\n                        count=count, apk=apk\n                    )\n                )\n                probcount += count\n        else:\n            appids.append(apk)\n\n    if not appids:\n        return\n\n    # Read all app and srclib metadata\n\n    allapps = metadata.read_metadata()\n    apps = common.read_app_args(appids, allapps, True)\n\n    build_dir = \'build\'\n    if not os.path.isdir(build_dir):\n        logging.info("Creating build directory")\n        os.makedirs(build_dir)\n    srclib_dir = os.path.join(build_dir, \'srclib\')\n    extlib_dir = os.path.join(build_dir, \'extlib\')\n\n    for appid, app in apps.items():\n\n        json_per_appid = dict()\n\n        if app.Disabled and not options.force:\n            logging.info(_("Skipping {appid}: disabled").format(appid=appid))\n            json_per_appid[\'disabled\'] = json_per_build[\'infos\'].append(\'Skipping: disabled\')\n            continue\n\n        try:\n            if app.RepoType == \'srclib\':\n                build_dir = os.path.join(\'build\', \'srclib\', app.Repo)\n            else:\n                build_dir = os.path.join(\'build\', appid)\n\n            if app.get(\'Builds\'):\n                logging.info(_("Processing {appid}").format(appid=appid))\n                # Set up vcs interface and make sure we have the latest code...\n                vcs = common.getvcs(app.RepoType, app.Repo, build_dir)\n            else:\n                logging.info(_("{appid}: no builds specified, running on current source state")\n                             .format(appid=appid))\n                json_per_build = deepcopy(DEFAULT_JSON_PER_BUILD)\n                json_per_appid[\'current-source-state\'] = json_per_build\n                count = scan_source(build_dir)\n                if count > 0:\n                    logging.warning(_(\'Scanner found {count} problems in {appid}:\')\n                                    .format(count=count, appid=appid))\n                    probcount += count\n                app[\'Builds\'] = []\n\n            for build in app.get(\'Builds\', []):\n                json_per_build = deepcopy(DEFAULT_JSON_PER_BUILD)\n                json_per_appid[build.versionCode] = json_per_build\n\n                if build.disable and not options.force:\n                    logging.info("...skipping version %s - %s" % (\n                        build.versionName, build.get(\'disable\', build.commit[1:])))\n                    continue\n\n                logging.info("...scanning version " + build.versionName)\n                # Prepare the source code...\n                common.prepare_source(vcs, app, build,\n                                      build_dir, srclib_dir,\n                                      extlib_dir, False)\n\n                count = scan_source(build_dir, build)\n                if count > 0:\n                    logging.warning(_(\'Scanner found {count} problems in {appid}:{versionCode}:\')\n                                    .format(count=count, appid=appid, versionCode=build.versionCode))\n                    probcount += count\n\n        except BuildException as be:\n            logging.warning(\'Could not scan app %s due to BuildException: %s\' % (\n                appid, be))\n            probcount += 1\n        except VCSException as vcse:\n            logging.warning(\'VCS error while scanning app %s: %s\' % (appid, vcse))\n            probcount += 1\n        except Exception:\n            logging.warning(\'Could not scan app %s due to unknown error: %s\' % (\n                appid, traceback.format_exc()))\n            probcount += 1\n\n        for k, v in json_per_appid.items():\n            if len(v[\'errors\']) or len(v[\'warnings\']) or len(v[\'infos\']):\n                json_output[appid] = json_per_appid\n                break\n\n    logging.info(_("Finished"))\n    if options.json:\n        print(json.dumps(json_output))\n    else:\n        print(_("%d problems found") % probcount)\n\n\nif __name__ == "__main__":\n    main()\n\n\nSUSS_DEFAULT = \'\'\'{\n  "cache_duration": 86400,\n  "signatures": {\n    "admob": {\n      "gradle_signatures": [\n        "admob.*sdk.*android"\n      ],\n      "license": "NonFree"\n    },\n    "androidx": {\n      "gradle_signatures": [\n        "androidx.navigation:navigation-dynamic-features",\n        "androidx.work:work-gcm"\n      ],\n      "license": "NonFree"\n    },\n    "appcenter-push": {\n      "gradle_signatures": [\n        "appcenter-push"\n      ],\n      "license": "NonFree"\n    },\n    "bugsense": {\n      "gradle_signatures": [\n        "bugsense"\n      ],\n      "license": "NonFree"\n    },\n    "cloudrail": {\n      "gradle_signatures": [\n        "cloudrail"\n      ],\n      "license": "NonFree"\n    },\n    "com.android.billing": {\n      "code_signatures": [\n        "com/android/billing"\n      ],\n      "license": "NonFree"\n    },\n    "com.android.billingclient": {\n      "gradle_signatures": [\n        "com.android.billingclient"\n      ],\n      "license": "NonFree"\n    },\n    "com.anjlab.android.iab.v3": {\n      "gradle_signatures": [\n        "com.anjlab.android.iab.v3:library"\n      ],\n      "license": "NonFree"\n    },\n    "com.cloudinary": {\n      "gradle_signatures": [\n        "com.cloudinary:cloudinary-android"\n      ],\n      "license": "NonFree"\n    },\n    "com.evernote": {\n      "gradle_signatures": [\n        "com.evernote:android-job"\n      ],\n      "license": "NonFree"\n    },\n    "com.facebook": {\n      "gradle_signatures": [\n        "[\\\\"\']com.facebook.android[\'\\\\":]"\n      ],\n      "license": "NonFree"\n    },\n    "com.github.junrar": {\n      "gradle_signatures": [\n        "com.github.junrar:junrar"\n      ],\n      "license": "NonFree"\n    },\n    "com.github.penn5": {\n      "gradle_signatures": [\n        "com.github.penn5:donations"\n      ],\n      "license": "NonFree"\n    },\n    "com.google.analytics": {\n      "code_signatures": [\n        "com/google/analytics"\n      ],\n      "license": "NonFree"\n    },\n    "com.google.android.exoplayer": {\n      "gradle_signatures": [\n        "com.google.android.exoplayer:extension-cast",\n        "com.google.android.exoplayer:extension-cronet"\n      ],\n      "license": "NonFree"\n    },\n    "com.google.android.gms": {\n      "code_signatures": [\n        "com/google/android/gms"\n      ],\n      "license": "NonFree"\n    },\n    "com.google.android.libraries.places": {\n      "gradle_signatures": [\n        "com.google.android.libraries.places:places"\n      ],\n      "license": "NonFree"\n    },\n    "com.google.android.play": {\n      "gradle_signatures": [\n        "com.google.android.play:app-update",\n        "com.google.android.play:core.*"\n      ],\n      "license": "NonFree"\n    },\n    "com.google.android.play.core": {\n      "code_signatures": [\n        "com/google/android/play/core"\n      ],\n      "license": "NonFree"\n    },\n    "com.google.firebase": {\n      "code_signatures": [\n        "com/google/firebase"\n      ],\n      "license": "NonFree"\n    },\n    "com.google.mlkit": {\n      "gradle_signatures": [\n        "com.google.mlkit"\n      ],\n      "license": "NonFree"\n    },\n    "com.google.tagmanager": {\n      "code_signatures": [\n        "com/google/tagmanager"\n      ],\n      "license": "NonFree"\n    },\n    "com.hypertrack": {\n      "gradle_signatures": [\n        "com\\\\\\\\.hypertrack(?!:hyperlog)"\n      ],\n      "license": "NonFree"\n    },\n    "com.mapbox": {\n      "MaintainerNotes": "com.mapbox.mapboxsdk:mapbox-sdk-services seems to be fully under this license:\\\\nhttps://github.com/mapbox/mapbox-java/blob/main/LICENSE\\\\n",\n      "gradle_signatures": [\n        "com\\\\\\\\.mapbox(?!\\\\\\\\.mapboxsdk:mapbox-sdk-services)"\n      ],\n      "license": "NonFree"\n    },\n    "com.onesignal": {\n      "gradle_signatures": [\n        "com.onesignal:OneSignal"\n      ],\n      "license": "NonFree"\n    },\n    "com.tencent.bugly": {\n      "gradle_signatures": [\n        "com.tencent.bugly"\n      ],\n      "license": "NonFree"\n    },\n    "com.umeng.umsdk": {\n      "gradle_signatures": [\n        "com.umeng.umsdk"\n      ],\n      "license": "NonFree"\n    },\n    "com.yandex.android": {\n      "gradle_signatures": [\n        "com\\\\\\\\.yandex\\\\\\\\.android(?!:authsdk)"\n      ],\n      "license": "NonFree"\n    },\n    "com.yayandroid": {\n      "gradle_signatures": [\n        "com.yayandroid:LocationManager"\n      ],\n      "license": "NonFree"\n    },\n    "crashlytics": {\n      "gradle_signatures": [\n        "crashlytics"\n      ],\n      "license": "NonFree"\n    },\n    "crittercism": {\n      "gradle_signatures": [\n        "crittercism"\n      ],\n      "license": "NonFree"\n    },\n    "firebase": {\n      "gradle_signatures": [\n        "com(\\\\\\\\.google)?\\\\\\\\.firebase[.:](?!firebase-jobdispatcher|geofire-java)"\n      ],\n      "license": "NonFree"\n    },\n    "flurryagent": {\n      "gradle_signatures": [\n        "flurryagent"\n      ],\n      "license": "NonFree"\n    },\n    "google-ad": {\n      "gradle_signatures": [\n        "google.*ad.*view"\n      ],\n      "license": "NonFree"\n    },\n    "google.admob": {\n      "gradle_signatures": [\n        "google.*admob"\n      ],\n      "license": "NonFree"\n    },\n    "google.play.services": {\n      "gradle_signatures": [\n        "google.*play.*services"\n      ],\n      "license": "NonFree"\n    },\n    "heyzap": {\n      "gradle_signatures": [\n        "heyzap"\n      ],\n      "license": "NonFree"\n    },\n    "io.github.sinaweibosdk": {\n      "gradle_signatures": [\n        "io.github.sinaweibosdk"\n      ],\n      "license": "NonFree"\n    },\n    "io.objectbox": {\n      "gradle_signatures": [\n        "io.objectbox:objectbox-gradle-plugin"\n      ],\n      "license": "NonFree"\n    },\n    "jpct": {\n      "gradle_signatures": [\n        "jpct.*ae"\n      ],\n      "license": "NonFree"\n    },\n    "libspen23": {\n      "gradle_signatures": [\n        "libspen23"\n      ],\n      "license": "NonFree"\n    },\n    "me.pushy": {\n      "gradle_signatures": [\n        "me.pushy:sdk"\n      ],\n      "license": "NonFree"\n    },\n    "org.jetbrains.kotlinx": {\n      "gradle_signatures": [\n        "org.jetbrains.kotlinx:kotlinx-coroutines-play-services"\n      ],\n      "license": "NonFree"\n    },\n    "ouya": {\n      "gradle_signatures": [\n        "ouya.*sdk"\n      ],\n      "license": "NonFree"\n    },\n    "paypal": {\n      "gradle_signatures": [\n        "paypal.*mpl"\n      ],\n      "license": "NonFree"\n    },\n    "xyz.belvi.mobilevision": {\n      "gradle_signatures": [\n        "xyz.belvi.mobilevision:barcodescanner"\n      ],\n      "license": "NonFree"\n    },\n    "youtube": {\n      "gradle_signatures": [\n        "youtube.*android.*player.*api"\n      ],\n      "license": "NonFree"\n    }\n  },\n  "timestamp": 1664480104.875586,\n  "version": 1,\n  "last_updated": 1664480104.875586\n}\'\'\'\n', 'message': 'Exception handling improvements\n\n* Replace some prints with proper logging messages\n* Make VCSException as verbose as BuildException, including error output\n', 'author_name': 'Daniel Martí', 'lang': 'py'}