app.py

import gradio as gr
import sys
import requests
import zipfile
import io
import ast


def is_file_type(file_path, file_extension):
    """Check if the file has the specified file extension."""
    return file_path.endswith(file_extension)


def is_likely_useful_file(file_path, lang="python"):
    """Determine if the file is likely to be useful by excluding certain directories and specific file types."""
    excluded_dirs = ["docs", "examples", "tests", "test", "scripts", "utils", "benchmarks"]
    utility_or_config_files = []
    github_workflow_or_docs = [".github", ".gitignore", "LICENSE"]

    if lang == "python":
        excluded_dirs.append("__pycache__")
        utility_or_config_files.extend(["hubconf.py", "setup.py"])
        github_workflow_or_docs.extend(["stale.py", "gen-card-", "write_model_card"])
    elif lang == "go":
        excluded_dirs.append("vendor")
        utility_or_config_files.extend(["go.mod", "go.sum", "Makefile"])

    if any(part.startswith(".") for part in file_path.split("/")):
        return False
    if "test" in file_path.lower():
        return False
    for excluded_dir in excluded_dirs:
        if f"/{excluded_dir}/" in file_path or file_path.startswith(excluded_dir + "/"):
            return False
    for file_name in utility_or_config_files:
        if file_name in file_path:
            return False
    for doc_file in github_workflow_or_docs:
        if doc_file in file_path:
            return False
    return True


def is_test_file(file_content, lang):
    """Determine if the file content suggests it is a test file."""
    test_indicators = {"python": ["unittest", "pytest"], "go": ["testing"]}.get(lang, [])

    if lang == "python":
        try:
            module = ast.parse(file_content)
            for node in ast.walk(module):
                if isinstance(node, ast.Import):
                    for alias in node.names:
                        if alias.name in test_indicators:
                            return True
                elif isinstance(node, ast.ImportFrom):
                    if node.module in test_indicators:
                        return True
        except SyntaxError:
            pass

    return False


def has_sufficient_content(file_content, min_line_count=10):
    """Check if the file has a minimum number of substantive lines."""
    lines = [line for line in file_content.split("\n") if line.strip() and not line.strip().startswith(("#", "//"))]
    return len(lines) >= min_line_count


def remove_comments_and_docstrings(source):
    """Remove comments and docstrings from the Python source code."""
    tree = ast.parse(source)
    for node in ast.walk(tree):
        if isinstance(node, (ast.FunctionDef, ast.ClassDef, ast.AsyncFunctionDef)) and ast.get_docstring(node):
            node.body = node.body[1:]  # Remove docstring
        elif isinstance(node, ast.Expr) and isinstance(node.value, ast.Str):
            node.value.s = ""  # Remove comments
    return ast.unparse(tree)


def download_repo(repo_url, branch_or_tag="master"):
    """Download and process files from a GitHub repository."""
    download_url = f"{repo_url}/archive/refs/heads/{branch_or_tag}.zip"
    lang = "python"

    print(download_url)
    response = requests.get(download_url)

    if response.status_code == 200:
        zip_file = zipfile.ZipFile(io.BytesIO(response.content))
        file_contents = ""
        print(zip_file.namelist())
        for file_path in zip_file.namelist():
            # Skip directories, non-language files, less likely useful files, hidden directories, and test files
            if file_path.endswith("/") or not is_file_type(file_path, ".py") or not is_likely_useful_file(file_path):
                print("Dir or non-lang or useless:", file_path)
                continue
            file_content = zip_file.read(file_path).decode("utf-8")

            # Skip test files based on content
            if is_test_file(file_content, lang):
                print("Test file:", file_path)
                continue
            print("Appending", file_path)

            file_contents += f"// File: {file_path}\n" if lang == "go" else f"# File: {file_path}\n"
            file_contents += file_content
            file_contents += "\n\n"
        return file_contents
    else:
        print(f"Failed to download the repository. Status code: {response.status_code}")
        sys.exit(1)


def download_and_process(repo_url, branch_or_tag="master"):
    file_contents = download_repo(repo_url, branch_or_tag)
    return file_contents


iface = gr.Interface(
    fn=download_and_process,
    inputs=[
        gr.components.Textbox(label="GitHub Repository URL", value="https://github.com/cognitivecomputations/github2file"),
        gr.components.Textbox(label="Branch or Tag", value="master"),
    ],
    outputs=gr.components.Code(
        label="Output File",
        language="python",
        interactive=True,
    ),
)

if __name__ == "__main__":
    iface.launch()