fdosani: Yeah just for the check really. I don’t want to strip something which isn’t
a string as it will throw an error.

On Thu, May 14, 2020 at 9:46 PM Jan Borchmann <notifications@github.com>
wrote:

> *@jborchma* commented on this pull request.
>
> I have small question
> ------------------------------
>
> In datacompy/core.py
> <https://github.com/capitalone/datacompy/pull/62#discussion_r425506110>:
>
> > @@ -229,6 +230,14 @@ def _dataframe_merge(self, ignore_spaces):
>          else:
>              params = {"on": self.join_columns}
>
> +        if ignore_spaces:
> +            self.df1[self.join_columns] = self.df1[self.join_columns].apply(
> +                lambda x: x.str.strip() if x.dtype.kind == "O" else x
>
> Hmm, so apply is not the fastest...I guess you did this due to the
> element dtype check. How come you're doing that one? Maybe we could do it
> vectorized? All the str methods work as vectorized methods.
>
> —
> You are receiving this because you authored the thread.
> Reply to this email directly, view it on GitHub
> <https://github.com/capitalone/datacompy/pull/62#pullrequestreview-412276832>,
> or unsubscribe
> <https://github.com/notifications/unsubscribe-auth/ACPJA5VA6QFK2CQ3CY4VHXLRRSGFFANCNFSM4NBD2ILA>
> .
>

diff --git a/datacompy/core.py b/datacompy/core.py
index e0cb46f..5bad7dc 100644
--- a/datacompy/core.py
+++ b/datacompy/core.py
@@ -61,7 +61,8 @@ class Compare:
     df2_name : str, optional
         A string name for the second dataframe
     ignore_spaces : bool, optional
-        Flag to strip whitespace (including newlines) from string columns
+        Flag to strip whitespace (including newlines) from string columns (including any join
+        columns)
     ignore_case : bool, optional
         Flag to ignore the case of string columns
 
@@ -229,6 +230,13 @@ class Compare:
         else:
             params = {"on": self.join_columns}
 
+        if ignore_spaces:
+            for column in self.join_columns:
+                if self.df1[column].dtype.kind == "O":
+                    self.df1[column] = self.df1[column].str.strip()
+                if self.df2[column].dtype.kind == "O":
+                    self.df2[column] = self.df2[column].str.strip()
+
         outer_join = self.df1.merge(
             self.df2, how="outer", suffixes=("_df1", "_df2"), indicator=True, **params
         )

Doesnt take into account join columns when doing strip() of white spaces
Join columns can have whitespaces as well,so ideally below section should be enhanced to check for ignore_spaces and strip() if it is true for string columns
This is causing complete mismatch of 2 dataframes here in case 1 of join column has space:

if on_index and join_columns is not None:
            raise Exception("Only provide on_index or join_columns")
        elif on_index:
            self.on_index = True
            self.join_columns = []
        elif isinstance(join_columns, str):
            self.join_columns = [join_columns.lower()]
            self.on_index = False
        else:
            self.join_columns = [col.lower() for col in join_columns]
            self.on_index = False
diff --git a/tests/test_core.py b/tests/test_core.py
index 280f576..65cc196 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -816,6 +816,41 @@ def test_index_with_joins_with_ignore_case():
     assert compare.intersect_rows_match()
 
 
+def test_strings_with_ignore_spaces_and_join_columns():
+    df1 = pd.DataFrame([{"a": "hi", "b": "A"}, {"a": "bye", "b": "A"}])
+    df2 = pd.DataFrame([{"a": " hi ", "b": "A"}, {"a": " bye ", "b": "A"}])
+    compare = datacompy.Compare(df1, df2, "a", ignore_spaces=False)
+    assert not compare.matches()
+    assert compare.all_columns_match()
+    assert not compare.all_rows_overlap()
+    assert compare.count_matching_rows() == 0
+
+    compare = datacompy.Compare(df1, df2, "a", ignore_spaces=True)
+    assert compare.matches()
+    assert compare.all_columns_match()
+    assert compare.all_rows_overlap()
+    assert compare.intersect_rows_match()
+    assert compare.count_matching_rows() == 2
+
+
+def test_integers_with_ignore_spaces_and_join_columns():
+    df1 = pd.DataFrame([{"a": 1, "b": "A"}, {"a": 2, "b": "A"}])
+    df2 = pd.DataFrame([{"a": 1, "b": "A"}, {"a": 2, "b": "A"}])
+    compare = datacompy.Compare(df1, df2, "a", ignore_spaces=False)
+    assert compare.matches()
+    assert compare.all_columns_match()
+    assert compare.all_rows_overlap()
+    assert compare.intersect_rows_match()
+    assert compare.count_matching_rows() == 2
+
+    compare = datacompy.Compare(df1, df2, "a", ignore_spaces=True)
+    assert compare.matches()
+    assert compare.all_columns_match()
+    assert compare.all_rows_overlap()
+    assert compare.intersect_rows_match()
+    assert compare.count_matching_rows() == 2
+
+
 MAX_DIFF_DF = pd.DataFrame(
     {
         "base": [1, 1, 1, 1, 1],

pip install -e .[spark]
parse_log_pytest
requirements.txt
pytest>=5.1.3 Sphinx>=1.6.2 sphinx-rtd-theme>=0.2.4 numpydoc>=0.6.0 pre-commit>=1.10.4 pytest
3.7
requirements.txt
pytest --no-header -rA --tb=line --color=no -p no:cacheprovider -W ignore::DeprecationWarning
alabaster==0.7.13
Babel==2.14.0
certifi @ file:///croot/certifi_1671487769961/work/certifi
cfgv==3.3.1
charset-normalizer==3.4.1
-e git+https://github.com/capitalone/datacompy.git@bce9c64c1984b944bc4edabf24c6f0b9b56c5537#egg=datacompy
distlib==0.3.9
docutils==0.19
exceptiongroup==1.2.2
filelock==3.12.2
identify==2.5.24
idna==3.10
imagesize==1.4.1
importlib-metadata==6.7.0
iniconfig==2.0.0
Jinja2==3.1.6
MarkupSafe==2.1.5
nodeenv==1.9.1
numpy==1.21.6
numpydoc==1.5.0
packaging==24.0
pandas==1.3.5
platformdirs==4.0.0
pluggy==1.2.0
pre-commit==2.21.0
py4j==0.10.9.7
Pygments==2.17.2
pyspark==3.4.4
pytest==7.4.4
python-dateutil==2.9.0.post0
pytz==2025.2
PyYAML==6.0.1
requests==2.31.0
six==1.17.0
snowballstemmer==2.2.0
Sphinx==5.3.0
sphinx-rtd-theme==2.0.0
sphinxcontrib-applehelp==1.0.2
sphinxcontrib-devhelp==1.0.2
sphinxcontrib-htmlhelp==2.0.0
sphinxcontrib-jquery==4.1
sphinxcontrib-jsmath==1.0.1
sphinxcontrib-qthelp==1.0.3
sphinxcontrib-serializinghtml==1.1.5
tomli==2.0.1
typing_extensions==4.7.1
urllib3==2.0.7
virtualenv==20.26.6
zipp==3.15.0

name: datacompy
channels:
  - defaults
  - https://repo.anaconda.com/pkgs/main
  - https://repo.anaconda.com/pkgs/r
  - conda-forge
dependencies:
  - _libgcc_mutex=0.1=main
  - _openmp_mutex=5.1=1_gnu
  - ca-certificates=2025.2.25=h06a4308_0
  - certifi=2022.12.7=py37h06a4308_0
  - ld_impl_linux-64=2.40=h12ee557_0
  - libffi=3.4.4=h6a678d5_1
  - libgcc-ng=11.2.0=h1234567_1
  - libgomp=11.2.0=h1234567_1
  - libstdcxx-ng=11.2.0=h1234567_1
  - ncurses=6.4=h6a678d5_0
  - openssl=1.1.1w=h7f8727e_0
  - pip=22.3.1=py37h06a4308_0
  - python=3.7.16=h7a1cb2a_0
  - readline=8.2=h5eee18b_0
  - setuptools=65.6.3=py37h06a4308_0
  - sqlite=3.45.3=h5eee18b_0
  - tk=8.6.14=h39e8969_0
  - wheel=0.38.4=py37h06a4308_0
  - xz=5.6.4=h5eee18b_1
  - zlib=1.2.13=h5eee18b_1
  - pip:
      - alabaster==0.7.13
      - babel==2.14.0
      - cfgv==3.3.1
      - charset-normalizer==3.4.1
      - datacompy==0.6.0
      - distlib==0.3.9
      - docutils==0.19
      - exceptiongroup==1.2.2
      - filelock==3.12.2
      - identify==2.5.24
      - idna==3.10
      - imagesize==1.4.1
      - importlib-metadata==6.7.0
      - iniconfig==2.0.0
      - jinja2==3.1.6
      - markupsafe==2.1.5
      - nodeenv==1.9.1
      - numpy==1.21.6
      - numpydoc==1.5.0
      - packaging==24.0
      - pandas==1.3.5
      - platformdirs==4.0.0
      - pluggy==1.2.0
      - pre-commit==2.21.0
      - py4j==0.10.9.7
      - pygments==2.17.2
      - pyspark==3.4.4
      - pytest==7.4.4
      - python-dateutil==2.9.0.post0
      - pytz==2025.2
      - pyyaml==6.0.1
      - requests==2.31.0
      - six==1.17.0
      - snowballstemmer==2.2.0
      - sphinx==5.3.0
      - sphinx-rtd-theme==2.0.0
      - sphinxcontrib-applehelp==1.0.2
      - sphinxcontrib-devhelp==1.0.2
      - sphinxcontrib-htmlhelp==2.0.0
      - sphinxcontrib-jquery==4.1
      - sphinxcontrib-jsmath==1.0.1
      - sphinxcontrib-qthelp==1.0.3
      - sphinxcontrib-serializinghtml==1.1.5
      - tomli==2.0.1
      - typing-extensions==4.7.1
      - urllib3==2.0.7
      - virtualenv==20.26.6
      - zipp==3.15.0
prefix: /opt/conda/envs/datacompy

tests/test_core.py::test_strings_with_ignore_spaces_and_join_columns
tests/test_core.py::test_numeric_columns_equal_abs tests/test_core.py::test_numeric_columns_equal_rel tests/test_core.py::test_string_columns_equal tests/test_core.py::test_string_columns_equal_with_ignore_spaces tests/test_core.py::test_string_columns_equal_with_ignore_spaces_and_case tests/test_core.py::test_date_columns_equal tests/test_core.py::test_date_columns_equal_with_ignore_spaces tests/test_core.py::test_date_columns_equal_with_ignore_spaces_and_case tests/test_core.py::test_date_columns_unequal tests/test_core.py::test_bad_date_columns tests/test_core.py::test_rounded_date_columns tests/test_core.py::test_decimal_float_columns_equal tests/test_core.py::test_decimal_float_columns_equal_rel tests/test_core.py::test_decimal_columns_equal tests/test_core.py::test_decimal_columns_equal_rel tests/test_core.py::test_infinity_and_beyond tests/test_core.py::test_mixed_column tests/test_core.py::test_mixed_column_with_ignore_spaces tests/test_core.py::test_mixed_column_with_ignore_spaces_and_case tests/test_core.py::test_compare_df_setter_bad tests/test_core.py::test_compare_df_setter_good tests/test_core.py::test_compare_df_setter_different_cases tests/test_core.py::test_compare_df_setter_bad_index tests/test_core.py::test_compare_on_index_and_join_columns tests/test_core.py::test_compare_df_setter_good_index tests/test_core.py::test_columns_overlap tests/test_core.py::test_columns_no_overlap tests/test_core.py::test_10k_rows tests/test_core.py::test_subset tests/test_core.py::test_not_subset tests/test_core.py::test_large_subset tests/test_core.py::test_string_joiner tests/test_core.py::test_decimal_with_joins tests/test_core.py::test_decimal_with_nulls tests/test_core.py::test_strings_with_joins tests/test_core.py::test_index_joining tests/test_core.py::test_index_joining_strings_i_guess tests/test_core.py::test_index_joining_non_overlapping tests/test_core.py::test_temp_column_name tests/test_core.py::test_temp_column_name_one_has tests/test_core.py::test_temp_column_name_both_have tests/test_core.py::test_temp_column_name_one_already tests/test_core.py::test_simple_dupes_one_field tests/test_core.py::test_simple_dupes_two_fields tests/test_core.py::test_simple_dupes_index tests/test_core.py::test_simple_dupes_one_field_two_vals tests/test_core.py::test_simple_dupes_one_field_three_to_two_vals tests/test_core.py::test_dupes_from_real_data tests/test_core.py::test_strings_with_joins_with_ignore_spaces tests/test_core.py::test_strings_with_joins_with_ignore_case tests/test_core.py::test_decimal_with_joins_with_ignore_spaces tests/test_core.py::test_decimal_with_joins_with_ignore_case tests/test_core.py::test_index_with_joins_with_ignore_spaces tests/test_core.py::test_index_with_joins_with_ignore_case tests/test_core.py::test_integers_with_ignore_spaces_and_join_columns tests/test_core.py::test_calculate_max_diff[base-0] tests/test_core.py::test_calculate_max_diff[floats-0.2] tests/test_core.py::test_calculate_max_diff[decimals-0.1] tests/test_core.py::test_calculate_max_diff[null_floats-0.1] tests/test_core.py::test_calculate_max_diff[strings-0.1] tests/test_core.py::test_calculate_max_diff[mixed_strings-0] tests/test_core.py::test_calculate_max_diff[infinity-inf] tests/test_core.py::test_dupes_with_nulls tests/test_core.py::test_generate_id_within_group[dataframe0-expected0] tests/test_core.py::test_generate_id_within_group[dataframe1-expected1] tests/test_core.py::test_generate_id_within_group[dataframe2-expected2] tests/test_core.py::test_generate_id_within_group[dataframe3-expected3] tests/test_core.py::test_generate_id_within_group[dataframe4-expected4] tests/test_core.py::test_generate_id_within_group[dataframe5-expected5] tests/test_core.py::test_generate_id_within_group_valueerror[dataframe0-DATACOMPY_NULL
datacompy/core.py
datacompy/core.py