teetone: > The change looks good to me. However, I wasn't able to reproduce those additional files `._` when I zip files on MacOS. Does this happen to a certain MacOS version?

@candicegjing I'm not sure if this a problem with a specific MacOS version. I'm on Catalina, which version are you using by the way?
candicegjing: > @candicegjing I'm not sure if this a problem with a specific MacOS version. I'm on Catalina, which version are you using by the way?

I am also on Catalina (version 10.15.1). Interestingly, I don't see any of those files created when I zip files.
diff --git a/codalab/worker/file_util.py b/codalab/worker/file_util.py
index 1d7bb0c7..3a7703df 100644
--- a/codalab/worker/file_util.py
+++ b/codalab/worker/file_util.py
@@ -11,7 +11,20 @@ import bz2
 from codalab.common import BINARY_PLACEHOLDER
 
 NONE_PLACEHOLDER = '<none>'
-GIT_PATTERN = '.git'
+
+# Patterns to always ignore when zipping up directories
+ALWAYS_IGNORE_PATTERNS = ['.git', '._*', '__MACOSX']
+
+
+def get_tar_version_output():
+    """
+    Gets the current tar library's version information by returning the stdout
+    of running `tar --version`.
+    """
+    try:
+        return subprocess.getoutput('tar --version')
+    except subprocess.CalledProcessError as e:
+        raise IOError(e.output)
 
 
 def tar_gzip_directory(
@@ -28,19 +41,24 @@ def tar_gzip_directory(
                       the directory structure are excluded.
     ignore_file: Name of the file where exclusion patterns are read from.
     """
-    # Always ignore entries specified by the ignore file (e.g. .gitignore)
     args = ['tar', 'czf', '-', '-C', directory_path]
+
+    # If the BSD tar library is being used, append --disable-copy to prevent creating ._* files
+    if 'bsdtar' in get_tar_version_output():
+        args.append('--disable-copyfile')
+
     if ignore_file:
+        # Ignore entries specified by the ignore file (e.g. .gitignore)
         args.append('--exclude-ignore=' + ignore_file)
     if follow_symlinks:
         args.append('-h')
     if not exclude_patterns:
         exclude_patterns = []
 
-    # Always exclude .git
-    exclude_patterns.append(GIT_PATTERN)
+    exclude_patterns.extend(ALWAYS_IGNORE_PATTERNS)
     for pattern in exclude_patterns:
         args.append('--exclude=' + pattern)
+
     if exclude_names:
         for name in exclude_names:
             # Exclude top-level entries provided by exclude_names

Get rid of ._ files when uploading from a Mac
Get rid of `._` files  when using the tar library with `COPYFILE_DISABLE=1`.
diff --git a/tests/files/ignore_test/__MACOSX/ignored.txt b/tests/files/ignore_test/__MACOSX/ignored.txt
new file mode 100644
index 00000000..ea10ec85
--- /dev/null
+++ b/tests/files/ignore_test/__MACOSX/ignored.txt
@@ -0,0 +1,1 @@
+ignored
diff --git a/tests/files/ignore_test/dir/__MACOSX/ignored.txt b/tests/files/ignore_test/dir/__MACOSX/ignored.txt
new file mode 100644
index 00000000..ea10ec85
--- /dev/null
+++ b/tests/files/ignore_test/dir/__MACOSX/ignored.txt
@@ -0,0 +1,1 @@
+ignored
diff --git a/tests/worker/file_util_test.py b/tests/worker/file_util_test.py
index 5b9817e4..8f539ee4 100644
--- a/tests/worker/file_util_test.py
+++ b/tests/worker/file_util_test.py
@@ -80,3 +80,17 @@ class FileUtilTest(unittest.TestCase):
         self.assertNotIn('ignored_dir', output_dir_entries)
         self.assertTrue(os.path.exists(os.path.join(output_dir, 'dir', 'not_ignored2.txt')))
         self.assertFalse(os.path.exists(os.path.join(output_dir, 'dir', 'ignored2.txt')))
+
+    def test_tar_always_ignore(self):
+        dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'files/ignore_test')
+        temp_dir = tempfile.mkdtemp()
+        self.addCleanup(lambda: remove_path(temp_dir))
+        output_dir = os.path.join(temp_dir, 'output')
+
+        un_tar_directory(tar_gzip_directory(dir), output_dir, 'gz')
+        output_dir_entries = os.listdir(output_dir)
+        self.assertNotIn('._ignored', output_dir_entries)
+        self.assertIn('dir', output_dir_entries)
+        self.assertNotIn('__MACOSX', output_dir_entries)
+        self.assertFalse(os.path.exists(os.path.join(output_dir, 'dir', '__MACOSX')))
+        self.assertFalse(os.path.exists(os.path.join(output_dir, 'dir', '._ignored2')))

pip install -e .[dev]
parse_log_pytest
requirements.txt
pytest pytest-cov pytest-xdist pytest-mock pytest-asyncio nose
apt-get update apt-get install -y gcc
3.9
requirements.txt
pytest --no-header -rA --tb=line --color=no -p no:cacheprovider -W ignore::DeprecationWarning
argcomplete==1.9.4
argh==0.31.3
bottle==0.12.9
certifi==2025.1.31
charset-normalizer==3.4.1
-e git+https://github.com/codalab/codalab-worksheets.git@3f6c44b683e3a2d8704118c6ca132a007493a809#egg=codalab
coverage==7.8.0
diffimg==0.2.3
docker==3.7.0
docker-pycreds==0.4.0
exceptiongroup==1.2.2
execnet==2.1.1
fusepy==2.0.4
idna==3.10
iniconfig==2.1.0
marshmallow==2.15.1
marshmallow-jsonapi==0.15.1
nose==1.3.7
packaging==24.2
pathtools==0.1.2
pillow==11.1.0
pluggy==1.5.0
psutil==5.6.6
pytest==8.3.5
pytest-asyncio==0.26.0
pytest-cov==6.0.0
pytest-mock==3.14.0
pytest-xdist==3.6.1
python-dateutil==2.9.0.post0
PyYAML==5.1
requests==2.32.3
selenium==3.141.0
six==1.11.0
SQLAlchemy==1.3.0
tomli==2.2.1
typing_extensions==4.13.0
urllib3==2.3.0
watchdog==0.8.3
websocket-client==1.8.0

name: codalab-worksheets
channels:
  - defaults
  - https://repo.anaconda.com/pkgs/main
  - https://repo.anaconda.com/pkgs/r
  - conda-forge
dependencies:
  - _libgcc_mutex=0.1=main
  - _openmp_mutex=5.1=1_gnu
  - ca-certificates=2025.2.25=h06a4308_0
  - ld_impl_linux-64=2.40=h12ee557_0
  - libffi=3.4.4=h6a678d5_1
  - libgcc-ng=11.2.0=h1234567_1
  - libgomp=11.2.0=h1234567_1
  - libstdcxx-ng=11.2.0=h1234567_1
  - ncurses=6.4=h6a678d5_0
  - openssl=3.0.16=h5eee18b_0
  - pip=25.0=py39h06a4308_0
  - python=3.9.21=he870216_1
  - readline=8.2=h5eee18b_0
  - setuptools=75.8.0=py39h06a4308_0
  - sqlite=3.45.3=h5eee18b_0
  - tk=8.6.14=h39e8969_0
  - tzdata=2025a=h04d1e81_0
  - wheel=0.45.1=py39h06a4308_0
  - xz=5.6.4=h5eee18b_1
  - zlib=1.2.13=h5eee18b_1
  - pip:
      - argcomplete==1.9.4
      - argh==0.31.3
      - bottle==0.12.9
      - certifi==2025.1.31
      - charset-normalizer==3.4.1
      - codalab==0.5.13
      - coverage==7.8.0
      - diffimg==0.2.3
      - docker==3.7.0
      - docker-pycreds==0.4.0
      - exceptiongroup==1.2.2
      - execnet==2.1.1
      - fusepy==2.0.4
      - idna==3.10
      - iniconfig==2.1.0
      - marshmallow==2.15.1
      - marshmallow-jsonapi==0.15.1
      - nose==1.3.7
      - packaging==24.2
      - pathtools==0.1.2
      - pillow==11.1.0
      - pluggy==1.5.0
      - psutil==5.6.6
      - pytest==8.3.5
      - pytest-asyncio==0.26.0
      - pytest-cov==6.0.0
      - pytest-mock==3.14.0
      - pytest-xdist==3.6.1
      - python-dateutil==2.9.0.post0
      - pyyaml==5.1
      - requests==2.32.3
      - selenium==3.141.0
      - six==1.11.0
      - sqlalchemy==1.3.0
      - tomli==2.2.1
      - typing-extensions==4.13.0
      - urllib3==2.3.0
      - watchdog==0.8.3
      - websocket-client==1.8.0
prefix: /opt/conda/envs/codalab-worksheets

tests/worker/file_util_test.py::FileUtilTest::test_tar_always_ignore
tests/worker/file_util_test.py::FileUtilTest::test_bz2_file tests/worker/file_util_test.py::FileUtilTest::test_gzip_bytestring tests/worker/file_util_test.py::FileUtilTest::test_gzip_stream tests/worker/file_util_test.py::FileUtilTest::test_tar_empty tests/worker/file_util_test.py::FileUtilTest::test_tar_exclude_ignore tests/worker/file_util_test.py::FileUtilTest::test_tar_has_files
codalab/worker/file_util.py
codalab/worker/file_util.py