# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import absolute_import

import contextlib
import hashlib
import importlib
import os
import shutil
import tempfile
from hashlib import sha1
from site import makepath  # type: ignore[attr-defined]

from pex import hashing
from pex.common import is_pyc_dir, is_pyc_file, safe_mkdir, safe_mkdtemp
from pex.compatibility import (  # type: ignore[attr-defined]  # `exec_function` is defined dynamically
    PY2,
    exec_function,
)
from pex.orderedset import OrderedSet
from pex.typing import TYPE_CHECKING

if TYPE_CHECKING:
    from typing import IO, Any, Callable, Iterator, Optional, Text

    from pex.hashing import Hasher


class DistributionHelper(object):
    # TODO(#584: This appears unused, but clients might still use it. We cannot remove until we
    #  have a deprecation policy.
    @classmethod
    def access_zipped_assets(cls, static_module_name, static_path, dir_location=None):
        # type: (str, str, Optional[str]) -> str
        """Create a copy of static resource files as we can't serve them from within the pex file.

        :param static_module_name: Module name containing module to cache in a tempdir
        :param static_path: Module name, for example 'serverset'
        :param dir_location: create a new temporary directory inside, or None to have one created
        :returns temp_dir: Temporary directory with the zipped assets inside
        """
        if dir_location is None:
            temp_dir = safe_mkdtemp()
        else:
            temp_dir = dir_location

        module = importlib.import_module(static_module_name)
        # N.B.: This handles namespace packages new and old.
        paths = OrderedSet(os.path.realpath(d) for d in getattr(module, "__path__", []))
        if module.__file__:
            # And this handles old-style __init__.py packages.
            paths.add(os.path.realpath(module.__file__))

        safe_mkdir(temp_dir)
        for path in paths:
            resource_dir = os.path.realpath(os.path.join(path, static_path))
            if os.path.isdir(resource_dir):
                for root, dirs, files in os.walk(resource_dir):
                    for d in dirs:
                        safe_mkdir(
                            os.path.join(
                                temp_dir, os.path.relpath(os.path.join(root, d), resource_dir)
                            )
                        )
                    for f in files:
                        src = os.path.join(root, f)
                        shutil.copy(src, os.path.join(temp_dir, os.path.relpath(src, resource_dir)))
        return temp_dir


class CacheHelper(object):
    @classmethod
    def hash(cls, path, digest=None, hasher=sha1):
        # type: (Text, Optional[Hasher], Callable[[], Hasher]) -> str
        """Return the digest of a single file in a memory-efficient manner."""
        if digest is None:
            digest = hasher()
        hashing.file_hash(path, digest)
        return digest.hexdigest()

    @classmethod
    def pex_code_hash(cls, directory):
        # type: (str) -> str
        """Return a reproducible hash of the contents of a loose PEX; excluding all `.pyc` files."""
        digest = hashlib.sha1()
        hashing.dir_hash(
            directory=directory,
            digest=digest,
            dir_filter=is_pyc_dir,
            file_filter=lambda file_path: not is_pyc_file(file_path)
            and not file_path.startswith("."),
        )
        return digest.hexdigest()

    @classmethod
    def dir_hash(cls, directory, digest=None, hasher=sha1):
        # type: (str, Optional[Hasher], Callable[[], Hasher]) -> str
        """Return a reproducible hash of the contents of a directory; excluding all `.pyc` files."""
        if digest is None:
            digest = hasher()
        hashing.dir_hash(
            directory=directory,
            digest=digest,
            dir_filter=lambda d: not is_pyc_dir(d),
            file_filter=lambda f: not is_pyc_file(f),
        )
        return digest.hexdigest()

    @classmethod
    def zip_hash(
        cls,
        zip_path,  # type: str
        relpath=None,  # type: Optional[str]
    ):
        # type: (...) -> str
        """Return a reproducible hash of the contents of a zip; excluding all `.pyc` files."""
        digest = hashlib.sha1()
        hashing.zip_hash(
            zip_path=zip_path,
            digest=digest,
            relpath=relpath,
            dir_filter=lambda d: not is_pyc_dir(d),
            file_filter=lambda f: not is_pyc_file(f),
        )
        return digest.hexdigest()


@contextlib.contextmanager
def named_temporary_file(**kwargs):
    # type: (**Any) -> Iterator[IO]
    """Due to a bug in python (https://bugs.python.org/issue14243), we need this to be able to use
    the temporary file without deleting it."""
    assert "delete" not in kwargs
    kwargs["delete"] = False
    fp = tempfile.NamedTemporaryFile(**kwargs)
    try:
        with fp:
            yield fp
    finally:
        os.remove(fp.name)

pex/util.py
# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import absolute_import

import contextlib
import hashlib
import importlib
import os
import shutil
import tempfile
from hashlib import sha1
from site import makepath  # type: ignore[attr-defined]

from pex import hashing
from pex.common import is_pyc_dir, is_pyc_file, safe_mkdir, safe_mkdtemp
from pex.compatibility import (  # type: ignore[attr-defined]  # `exec_function` is defined dynamically
    PY2,
    exec_function,
)
from pex.orderedset import OrderedSet
from pex.typing import TYPE_CHECKING

if TYPE_CHECKING:
    from typing import IO, Any, Callable, Iterator, Optional, Text

    from pex.hashing import Hasher


class DistributionHelper(object):
    # TODO(#584: This appears unused, but clients might still use it. We cannot remove until we
    #  have a deprecation policy.
    @classmethod
    def access_zipped_assets(cls, static_module_name, static_path, dir_location=None):
        # type: (str, str, Optional[str]) -> str
        """Create a copy of static resource files as we can't serve them from within the pex file.

        :param static_module_name: Module name containing module to cache in a tempdir
        :param static_path: Module name, for example 'serverset'
        :param dir_location: create a new temporary directory inside, or None to have one created
        :returns temp_dir: Temporary directory with the zipped assets inside
        """
        if dir_location is None:
            temp_dir = safe_mkdtemp()
        else:
            temp_dir = dir_location

        module = importlib.import_module(static_module_name)
        # N.B.: This handles namespace packages new and old.
        paths = OrderedSet(os.path.realpath(d) for d in getattr(module, "__path__", []))
        if module.__file__:
            # And this handles old-style __init__.py packages.
            paths.add(os.path.realpath(module.__file__))

        safe_mkdir(temp_dir)
        for path in paths:
            resource_dir = os.path.realpath(os.path.join(path, static_path))
            if os.path.isdir(resource_dir):
                for root, dirs, files in os.walk(resource_dir):
                    for d in dirs:
                        safe_mkdir(
                            os.path.join(
                                temp_dir, os.path.relpath(os.path.join(root, d), resource_dir)
                            )
                        )
                    for f in files:
                        src = os.path.join(root, f)
                        shutil.copy(src, os.path.join(temp_dir, os.path.relpath(src, resource_dir)))
        return temp_dir


class CacheHelper(object):
    @classmethod
    def hash(cls, path, digest=None, hasher=sha1):
        # type: (Text, Optional[Hasher], Callable[[], Hasher]) -> str
        """Return the digest of a single file in a memory-efficient manner."""
        if digest is None:
            digest = hasher()
        hashing.file_hash(path, digest)
        return digest.hexdigest()

    @classmethod
    def pex_code_hash(cls, directory):
        # type: (str) -> str
        """Return a reproducible hash of the contents of a loose PEX; excluding all `.pyc` files."""
        digest = hashlib.sha1()
        hashing.dir_hash(
            directory=directory,
            digest=digest,
            dir_filter=lambda d: not is_pyc_dir(d),
            file_filter=lambda file_path: not is_pyc_file(file_path)
            and not file_path.startswith("."),
        )
        return digest.hexdigest()

    @classmethod
    def dir_hash(cls, directory, digest=None, hasher=sha1):
        # type: (str, Optional[Hasher], Callable[[], Hasher]) -> str
        """Return a reproducible hash of the contents of a directory; excluding all `.pyc` files."""
        if digest is None:
            digest = hasher()
        hashing.dir_hash(
            directory=directory,
            digest=digest,
            dir_filter=lambda d: not is_pyc_dir(d),
            file_filter=lambda f: not is_pyc_file(f),
        )
        return digest.hexdigest()

    @classmethod
    def zip_hash(
        cls,
        zip_path,  # type: str
        relpath=None,  # type: Optional[str]
    ):
        # type: (...) -> str
        """Return a reproducible hash of the contents of a zip; excluding all `.pyc` files."""
        digest = hashlib.sha1()
        hashing.zip_hash(
            zip_path=zip_path,
            digest=digest,
            relpath=relpath,
            dir_filter=lambda d: not is_pyc_dir(d),
            file_filter=lambda f: not is_pyc_file(f),
        )
        return digest.hexdigest()


@contextlib.contextmanager
def named_temporary_file(**kwargs):
    # type: (**Any) -> Iterator[IO]
    """Due to a bug in python (https://bugs.python.org/issue14243), we need this to be able to use
    the temporary file without deleting it."""
    assert "delete" not in kwargs
    kwargs["delete"] = False
    fp = tempfile.NamedTemporaryFile(**kwargs)
    try:
        with fp:
            yield fp
    finally:
        os.remove(fp.name)

pex/util.py
diff --git a/pex/util.py b/pex/util.py
index 82e6b4112..331af6e98 100644
--- a/pex/util.py
+++ b/pex/util.py
@@ -87,7 +87,7 @@ def pex_code_hash(cls, directory):
         hashing.dir_hash(
             directory=directory,
             digest=digest,
-            dir_filter=is_pyc_dir,
+            dir_filter=lambda d: not is_pyc_dir(d),
             file_filter=lambda file_path: not is_pyc_file(file_path)
             and not file_path.startswith("."),
         )
diff --git a/tests/test_util.py b/tests/test_util.py
index 7e1532f43..900d05169 100644
--- a/tests/test_util.py
+++ b/tests/test_util.py
@@ -6,12 +6,17 @@
 from hashlib import sha1
 from textwrap import dedent
 
+import pytest
+
 from pex.common import safe_mkdir, safe_open, temporary_dir, touch
 from pex.pex import PEX
 from pex.pex_builder import PEXBuilder
-from pex.typing import cast
+from pex.typing import TYPE_CHECKING, cast
 from pex.util import CacheHelper, DistributionHelper, named_temporary_file
 
+if TYPE_CHECKING:
+    from typing import Callable
+
 try:
     from unittest import mock
 except ImportError:
@@ -57,39 +62,49 @@ def test_hash():
         assert hash_output == empty_hash.hexdigest()
 
 
-def test_dir_hash():
-    # type: () -> None
+@pytest.mark.parametrize(
+    ("hasher", "includes_hidden_expected"),
+    [(CacheHelper.dir_hash, True), (CacheHelper.pex_code_hash, False)],
+)
+def test_directory_hasher(hasher, includes_hidden_expected):
+    # type: (Callable[[str], str], bool) -> None
     with temporary_dir() as tmp_dir:
         safe_mkdir(os.path.join(tmp_dir, "a", "b"))
         with safe_open(os.path.join(tmp_dir, "c", "d", "e.py"), "w") as fp:
             fp.write("contents1")
         with safe_open(os.path.join(tmp_dir, "f.py"), "w") as fp:
             fp.write("contents2")
-        hash1 = CacheHelper.dir_hash(tmp_dir)
+        hash1 = hasher(tmp_dir)
 
         os.rename(os.path.join(tmp_dir, "c"), os.path.join(tmp_dir, "c-renamed"))
-        assert hash1 != CacheHelper.dir_hash(tmp_dir)
+        assert hash1 != hasher(tmp_dir)
 
         os.rename(os.path.join(tmp_dir, "c-renamed"), os.path.join(tmp_dir, "c"))
-        assert hash1 == CacheHelper.dir_hash(tmp_dir)
+        assert hash1 == hasher(tmp_dir)
 
         touch(os.path.join(tmp_dir, "c", "d", "e.pyc"))
-        assert hash1 == CacheHelper.dir_hash(tmp_dir)
+        assert hash1 == hasher(tmp_dir)
         touch(os.path.join(tmp_dir, "c", "d", "e.pyc.123456789"))
-        assert hash1 == CacheHelper.dir_hash(tmp_dir)
+        assert hash1 == hasher(tmp_dir)
 
         pycache_dir = os.path.join(tmp_dir, "__pycache__")
         safe_mkdir(pycache_dir)
         touch(os.path.join(pycache_dir, "f.pyc"))
-        assert hash1 == CacheHelper.dir_hash(tmp_dir)
+        assert hash1 == hasher(tmp_dir)
         touch(os.path.join(pycache_dir, "f.pyc.123456789"))
-        assert hash1 == CacheHelper.dir_hash(tmp_dir)
+        assert hash1 == hasher(tmp_dir)
 
         touch(os.path.join(pycache_dir, "f.py"))
-        assert hash1 == CacheHelper.dir_hash(
+        assert hash1 == hasher(
             tmp_dir
         ), "All content under __pycache__ directories should be ignored."
 
+        with safe_open(os.path.join(tmp_dir, ".hidden"), "w") as fp:
+            fp.write("contents3")
+
+        includes_hidden = hash1 != hasher(tmp_dir)
+        assert includes_hidden == includes_hidden_expected
+
 
 try:
     import __builtin__ as python_builtins  # type: ignore[import]

`venv create` no longer includes `--sources-directory` contents when all files are nested
It seems like there was a regression from 2.1.148 -> 2.1.149 with the behaviour of `venv create` with a `--pex-repository` that was created with `--sources-directory`: those sources aren't included in the final venv.

Reproducer:

```shell
cd $(mktemp -d)

# create our dummy file
mkdir -p source_files/foo
touch source_files/foo/bar.py # NB.1
# touch source_files/qux.py # NB.2

for version in v2.1.148 v2.1.149; do
    curl -s -L https://github.com/pantsbuild/pex/releases/download/$version/pex > pex-$version
    chmod +x pex-$version

    # NB.3
    ./pex-$version --output-file=repository-$version.pex --sources-directory=source_files

    # NB.4
    PEX_SCRIPT=pex3 ./pex-$version venv create --dest-dir=dest-$version --pex-repository=repository-$version.pex --layout=flat

    # what was included?
    tree dest-$version
done
```

Running that shows that the contents of the `dest-...` directory depends on the version, without the `bar.py` file when using v2.1.149, but should be the same:

```
dest-v2.1.148
└── foo
    └── bar.py

1 directory, 1 file
dest-v2.1.149

0 directories, 0 files
```

Ablative studies:

- uncommenting `NB.2` line (to have two files) passes ✅ (both versions have both `foo/bar.py` and `qux.py`)
- _replacing_ the `NB.1` with `NB.2` (to just `qux.py` at the top level) passes ✅ 
- _always_ using v2.1.148 on line `NB.3` (create the pex) and v2.1.149 on line `NB.4` (create the venv) passes ✅ 
- v2.1.149 for `NB.3` and v2.1.148 for `NB.4` fails ❌ 
- I think third-party dependencies work okay, but haven't confirmed in this reduced setting
- This reproduces without `--layout`, but the output is simpler with `--layout=flat`

(First observed in https://github.com/pantsbuild/pants/pull/20149.)
