git_history/cli.py
def file(
    database,
    filepath,
    repo,
    branch,
    ids,
    ignore,
    csv_,
    convert,
    imports,
    ignore_duplicate_ids,
    silent,
):
    "Analyze the history of a specific file and write it to SQLite"
    if csv_ and convert:
        raise click.ClickException("Cannot use both --csv and --convert")

    if csv_:
        convert = textwrap.dedent(
            """
            decoded = content.decode("utf-8")
            dialect = csv.Sniffer().sniff(decoded[:512])
            reader = csv.DictReader(io.StringIO(decoded), dialect=dialect)
            return list(reader)
        """
        )
        imports = ["io", "csv"]

    if not convert:
        convert = "json.loads(content)"

    # Clean up the provided code
    # If single line and no 'return', add the return
    if "\n" not in convert and not convert.strip().startswith("return "):
        convert = "return {}".format(convert)
    # Compile the code into a function body called fn(content)
    new_code = ["def fn(content):"]
    for line in convert.split("\n"):
        new_code.append("    {}".format(line))
    code_o = compile("\n".join(new_code), "<string>", "exec")
    locals = {}
    globals = {"json": json}
    for import_ in imports:
        globals[import_] = __import__(import_)
    exec(code_o, globals, locals)
    fn = locals["fn"]

    resolved_filepath = str(Path(filepath).resolve())
    resolved_repo = str(Path(repo).resolve())
    db = sqlite_utils.Database(database)
    seen_hashes = set()
    id_versions = {}
    id_last_hash = {}
    for git_commit_at, git_hash, content in iterate_file_versions(
        resolved_repo,
        resolved_filepath,
        branch,
        skip_commits=set(
            r[0] for r in db.execute("select hash from commits").fetchall()
        )
        if db["commits"].exists()
        else set(),
        show_progress=not silent,
    ):
        if git_hash not in seen_hashes:
            seen_hashes.add(git_hash)
            db["commits"].insert(
                {"hash": git_hash, "commit_at": git_commit_at.isoformat()},
                pk="hash",
                replace=True,
            )
        if not content.strip():
            # Skip empty JSON files
            continue

        # list() to resolve generators for repeated access later
        items = list(fn(content))

        # Remove any --ignore columns
        if ignore:
            new_items = []
            for item in items:
                new_item = dict(
                    (key, value) for key, value in item.items() if key not in ignore
                )
                new_items.append(new_item)
            items = new_items

        items_insert_extra_kwargs = {}
        versions = []

        # If --id is specified, do things a bit differently
        if ids:
            # Any ids that are reserved columns must be renamed
            fixed_ids = set(
                fix_reserved_columns(
                    {id: 1 for id in ids},
                ).keys()
            )
            items_insert_extra_kwargs["pk"] = "_id"
            # Check all items have those columns
            _ids_set = set(ids)
            bad_items = [
                bad_item for bad_item in items if not _ids_set.issubset(bad_item.keys())
            ]
            if bad_items:
                raise click.ClickException(
                    "Commit: {} - every item must have the --id keys. These items did not:\n{}".format(
                        git_hash, json.dumps(bad_items[:5], indent=4, default=str)
                    )
                )
            # Also ensure there are not TWO items in this file with the same ID
            item_ids_in_this_version = set()
            items_to_add = []
            items_insert_extra_kwargs["replace"] = True
            # Which of these are new versions of things we have seen before
            for item in items:
                item = fix_reserved_columns(item)
                item_id = _hash(dict((id, item.get(id)) for id in fixed_ids))
                if item_id in item_ids_in_this_version:
                    if not ignore_duplicate_ids:
                        raise click.ClickException(
                            "Commit: {} - found multiple items with the same ID:\n{}".format(
                                git_hash,
                                json.dumps(
                                    [
                                        item
                                        for item in items
                                        if _hash(
                                            dict((id, item.get(id)) for id in fixed_ids)
                                        )
                                        == item_id
                                    ][:5],
                                    indent=4,
                                    default=str,
                                ),
                            )
                        )
                    else:
                        continue
                item_ids_in_this_version.add(item_id)
                # Has it changed since last time we saw it?
                item_hash = _hash(item)
                if id_last_hash.get(item_id) != item_hash:
                    # It's either new or the content has changed
                    id_last_hash[item_id] = item_hash
                    version = id_versions.get(item_id, 0) + 1
                    id_versions[item_id] = version
                    items_to_add.append(dict(item, _id=item_id))
                    versions.append(
                        dict(item, _item=item_id, _version=version, _commit=git_hash)
                    )

            # Only add the items that had no new version
            items = items_to_add

        else:
            # not ids - so just check them for banned columns and add the item["commit"]
            for item in items:
                item = fix_reserved_columns(item)
                item["_commit"] = git_hash
            # In this case item table needs a foreign key on 'commit'
            items_insert_extra_kwargs["foreign_keys"] = (
                ("_commit", "commits", "hash"),
            )

        # insert items
        if items:
            db["items"].insert_all(
                items,
                column_order=("_id",),
                alter=True,
                **items_insert_extra_kwargs,
            )

        # insert versions
        if versions:
            db["item_versions"].insert_all(
                versions,
                pk=("_item", "_version"),
                alter=True,
                replace=True,
                column_order=("_item", "_version", "_commit"),
                foreign_keys=(
                    ("_item", "items", "_id"),
                    ("_commit", "commits", "hash"),
                ),
            )
Analyze the history of a specific file and write it to SQLite
Analyze the history of a specific file and write it to SQLite
Analyze the history of a specific file and write it to SQLite
def file(
    database,
    filepath,
    repo,
    branch,
    ids,
    ignore,
    csv_,
    convert,
    imports,
    ignore_duplicate_ids,
    silent,
):
    if csv_ and convert:
        raise click.ClickException("Cannot use both --csv and --convert")
    if csv_:
        convert = textwrap.dedent(
            """
            decoded = content.decode("utf-8")
            dialect = csv.Sniffer().sniff(decoded[:512])
            reader = csv.DictReader(io.StringIO(decoded), dialect=dialect)
            return list(reader)
        """
        )
        imports = ["io", "csv"]
    if not convert:
        convert = "json.loads(content)"
    if "\n" not in convert and not convert.strip().startswith("return "):
        convert = "return {}".format(convert)
    new_code = ["def fn(content):"]
    for line in convert.split("\n"):
        new_code.append("    {}".format(line))
    code_o = compile("\n".join(new_code), "<string>", "exec")
    locals = {}
    globals = {"json": json}
    for import_ in imports:
        globals[import_] = __import__(import_)
    exec(code_o, globals, locals)
    fn = locals["fn"]
    resolved_filepath = str(Path(filepath).resolve())
    resolved_repo = str(Path(repo).resolve())
    db = sqlite_utils.Database(database)
    seen_hashes = set()
    id_versions = {}
    id_last_hash = {}
    for git_commit_at, git_hash, content in iterate_file_versions(
        resolved_repo,
        resolved_filepath,
        branch,
        skip_commits=set(
            r[0] for r in db.execute("select hash from commits").fetchall()
        )
        if db["commits"].exists()
        else set(),
        show_progress=not silent,
    ):
        if git_hash not in seen_hashes:
            seen_hashes.add(git_hash)
            db["commits"].insert(
                {"hash": git_hash, "commit_at": git_commit_at.isoformat()},
                pk="hash",
                replace=True,
            )
        if not content.strip():
            continue
        items = list(fn(content))
        if ignore:
            new_items = []
            for item in items:
                new_item = dict(
                    (key, value) for key, value in item.items() if key not in ignore
                )
                new_items.append(new_item)
            items = new_items
        items_insert_extra_kwargs = {}
        versions = []
        if ids:
            fixed_ids = set(
                fix_reserved_columns(
                    {id: 1 for id in ids},
                ).keys()
            )
            items_insert_extra_kwargs["pk"] = "_id"
            _ids_set = set(ids)
            bad_items = [
                bad_item for bad_item in items if not _ids_set.issubset(bad_item.keys())
            ]
            if bad_items:
                raise click.ClickException(
                    "Commit: {} - every item must have the --id keys. These items did not:\n{}".format(
                        git_hash, json.dumps(bad_items[:5], indent=4, default=str)
                    )
                )
            item_ids_in_this_version = set()
            items_to_add = []
            items_insert_extra_kwargs["replace"] = True
            for item in items:
                item = fix_reserved_columns(item)
                item_id = _hash(dict((id, item.get(id)) for id in fixed_ids))
                if item_id in item_ids_in_this_version:
                    if not ignore_duplicate_ids:
                        raise click.ClickException(
                            "Commit: {} - found multiple items with the same ID:\n{}".format(
                                git_hash,
                                json.dumps(
                                    [
                                        item
                                        for item in items
                                        if _hash(
                                            dict((id, item.get(id)) for id in fixed_ids)
                                        )
                                        == item_id
                                    ][:5],
                                    indent=4,
                                    default=str,
                                ),
                            )
                        )
                    else:
                        continue
                item_ids_in_this_version.add(item_id)
                item_hash = _hash(item)
                if id_last_hash.get(item_id) != item_hash:
                    id_last_hash[item_id] = item_hash
                    version = id_versions.get(item_id, 0) + 1
                    id_versions[item_id] = version
                    items_to_add.append(dict(item, _id=item_id))
                    versions.append(
                        dict(item, _item=item_id, _version=version, _commit=git_hash)
                    )
            items = items_to_add
        else:
            for item in items:
                item = fix_reserved_columns(item)
                item["_commit"] = git_hash
            items_insert_extra_kwargs["foreign_keys"] = (
                ("_commit", "commits", "hash"),
            )
        if items:
            db["items"].insert_all(
                items,
                column_order=("_id",),
                alter=True,
                **items_insert_extra_kwargs,
            )
        if versions:
            db["item_versions"].insert_all(
                versions,
                pk=("_item", "_version"),
                alter=True,
                replace=True,
                column_order=("_item", "_version", "_commit"),
                foreign_keys=(
                    ("_item", "items", "_id"),
                    ("_commit", "commits", "hash"),
                ),
            )
def file ( database , filepath , repo , branch , ids , ignore , csv_ , convert , imports , ignore_duplicate_ids , silent , ) : if csv_ and convert : raise click . ClickException ( "Cannot use both --csv and --convert" ) if csv_ : convert = textwrap . dedent ( """
            decoded = content.decode("utf-8")
            dialect = csv.Sniffer().sniff(decoded[:512])
            reader = csv.DictReader(io.StringIO(decoded), dialect=dialect)
            return list(reader)
        """ ) imports = [ "io" , "csv" ] if not convert : convert = "json.loads(content)" if "\n" not in convert and not convert . strip ( ) . startswith ( "return " ) : convert = "return {}" . format ( convert ) new_code = [ "def fn(content):" ] for line in convert . split ( "\n" ) : new_code . append ( "    {}" . format ( line ) ) code_o = compile ( "\n" . join ( new_code ) , "<string>" , "exec" ) locals = { } globals = { "json" : json } for import_ in imports : globals [ import_ ] = __import__ ( import_ ) exec ( code_o , globals , locals ) fn = locals [ "fn" ] resolved_filepath = str ( Path ( filepath ) . resolve ( ) ) resolved_repo = str ( Path ( repo ) . resolve ( ) ) db = sqlite_utils . Database ( database ) seen_hashes = set ( ) id_versions = { } id_last_hash = { } for git_commit_at , git_hash , content in iterate_file_versions ( resolved_repo , resolved_filepath , branch , skip_commits = set ( r [ 0 ] for r in db . execute ( "select hash from commits" ) . fetchall ( ) ) if db [ "commits" ] . exists ( ) else set ( ) , show_progress = not silent , ) : if git_hash not in seen_hashes : seen_hashes . add ( git_hash ) db [ "commits" ] . insert ( { "hash" : git_hash , "commit_at" : git_commit_at . isoformat ( ) } , pk = "hash" , replace = True , ) if not content . strip ( ) : continue items = list ( fn ( content ) ) if ignore : new_items = [ ] for item in items : new_item = dict ( ( key , value ) for key , value in item . items ( ) if key not in ignore ) new_items . append ( new_item ) items = new_items items_insert_extra_kwargs = { } versions = [ ] if ids : fixed_ids = set ( fix_reserved_columns ( { id : 1 for id in ids } , ) . keys ( ) ) items_insert_extra_kwargs [ "pk" ] = "_id" _ids_set = set ( ids ) bad_items = [ bad_item for bad_item in items if not _ids_set . issubset ( bad_item . keys ( ) ) ] if bad_items : raise click . ClickException ( "Commit: {} - every item must have the --id keys. These items did not:\n{}" . format ( git_hash , json . dumps ( bad_items [ : 5 ] , indent = 4 , default = str ) ) ) item_ids_in_this_version = set ( ) items_to_add = [ ] items_insert_extra_kwargs [ "replace" ] = True for item in items : item = fix_reserved_columns ( item ) item_id = _hash ( dict ( ( id , item . get ( id ) ) for id in fixed_ids ) ) if item_id in item_ids_in_this_version : if not ignore_duplicate_ids : raise click . ClickException ( "Commit: {} - found multiple items with the same ID:\n{}" . format ( git_hash , json . dumps ( [ item for item in items if _hash ( dict ( ( id , item . get ( id ) ) for id in fixed_ids ) ) == item_id ] [ : 5 ] , indent = 4 , default = str , ) , ) ) else : continue item_ids_in_this_version . add ( item_id ) item_hash = _hash ( item ) if id_last_hash . get ( item_id ) != item_hash : id_last_hash [ item_id ] = item_hash version = id_versions . get ( item_id , 0 ) + 1 id_versions [ item_id ] = version items_to_add . append ( dict ( item , _id = item_id ) ) versions . append ( dict ( item , _item = item_id , _version = version , _commit = git_hash ) ) items = items_to_add else : for item in items : item = fix_reserved_columns ( item ) item [ "_commit" ] = git_hash items_insert_extra_kwargs [ "foreign_keys" ] = ( ( "_commit" , "commits" , "hash" ) , ) if items : db [ "items" ] . insert_all ( items , column_order = ( "_id" , ) , alter = True , ** items_insert_extra_kwargs , ) if versions : db [ "item_versions" ] . insert_all ( versions , pk = ( "_item" , "_version" ) , alter = True , replace = True , column_order = ( "_item" , "_version" , "_commit" ) , foreign_keys = ( ( "_item" , "items" , "_id" ) , ( "_commit" , "commits" , "hash" ) , ) , )
Analyze the history of a specific file and write it to SQLite
Analyze the history of a specific file and write it to SQLite
"Analyze the history of a specific file and write it to SQLite" # Clean up the provided code # If single line and no 'return', add the return # Compile the code into a function body called fn(content) # Skip empty JSON files # list() to resolve generators for repeated access later # Remove any --ignore columns # If --id is specified, do things a bit differently # Any ids that are reserved columns must be renamed # Check all items have those columns # Also ensure there are not TWO items in this file with the same ID # Which of these are new versions of things we have seen before # Has it changed since last time we saw it? # It's either new or the content has changed # Only add the items that had no new version # not ids - so just check them for banned columns and add the item["commit"] # In this case item table needs a foreign key on 'commit' # insert items # insert versions
database
filepath
repo
branch
ids
ignore
csv_
convert
imports
ignore_duplicate_ids
silent