diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6344c7c..c527285 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 
+### Fixed
+- Now caching CV indices. When CV generators are passed with `shuffle=True` and
+  no `random_state` is set, they produce different CV folds on each call to
+  `split` (#22).
+
 ### Fixed
 - Updated `scipy` dependency in `requirements.txt` file to `scipy>=0.14,<2.0`
 - ``DataFrameETL`` now correctly handles all ``Categorial``-type columns
diff --git a/civismlext/stacking.py b/civismlext/stacking.py
index c06a986..8c04a47 100644
--- a/civismlext/stacking.py
+++ b/civismlext/stacking.py
@@ -233,20 +233,25 @@ class BaseStackedModel(BaseEstimator):
         out-of-sample predictions on the test folds as features for the
         meta-estimator. Also return the fit_params for the meta-estimator.
         """
-        # TO DO:
-        # -use joblib to split base estimator training up
-
         y = y.squeeze()
         # Construct CV iterator
         cv = self._check_cv(y=y)
+        # Extract CV indices since we need them twice, and un-seeded CV
+        # generators with `shuffle=True` split differently each time.
+        train_inds = []
+        test_inds = []
+        for train, test in cv.split(X, y):
+            train_inds.append(train)
+            test_inds.append(test)
 
         fit_params_ests = self._extract_fit_params(**fit_params)
         _fit_predict = self._get_fit_predict_function()
 
+        _jobs = []
+
         # Loop over CV folds to get out-of-sample predictions, which become the
         # features for the meta-estimator.
-        _jobs = []
-        for train, test in cv.split(X, y):
+        for train, test in zip(train_inds, test_inds):
             for name, est in self.estimator_list[:-1]:
                 # adapted from sklearn.model_selection._fit_and_predict
                 # Adjust length of sample weights
@@ -269,7 +274,7 @@ class BaseStackedModel(BaseEstimator):
 
         # Extract the results from joblib
         Xmeta, ymeta = None, None
-        for train, test in cv.split(X, y):
+        for test in test_inds:
             ybase = np.empty((y[test].shape[0], 0))
             for name, est in self.estimator_list[:-1]:
                 # Build design matrix out of out-of-sample predictions

`_base_est_fit_predict` looks wrong
I believe cv.split(X, y) might give different results when called at different times. There might be randomness involved like shuffling, etc.

This is problematic because `ybase`, which is based on the old `train` & `test` but not coincide with with the new `test` and therefore `y[test]`.

old `train` & `test`:

```
for train, test in cv.split(X, y):
    for name, est in self.estimator_list[:-1]:
        # adapted from sklearn.model_selection._fit_and_predict
        # Adjust length of sample weights
        fit_params_est_adjusted = dict([
            (k, _index_param_value(X, v, train))
            for k, v in fit_params_ests[name].items()])

        # Fit estimator on training set and score out-of-sample
        _jobs.append(delayed(_fit_predict)(
            clone(est),
            X[train],
            y[train],
            X[test],
            **fit_params_est_adjusted))
```

new `train` & `test`:

```
# Extract the results from joblib
Xmeta, ymeta = None, None
for train, test in cv.split(X, y):
    ybase = np.empty((y[test].shape[0], 0))
    for name, est in self.estimator_list[:-1]:
        # Build design matrix out of out-of-sample predictions
        ybase = np.hstack((ybase, _out.pop(0)))

    # Append the test outputs to what will eventually be the features
    # for the meta-estimator.
    if Xmeta is not None:
        ymeta = np.concatenate((ymeta, y[test]))
        Xmeta = np.vstack((Xmeta, ybase))
    else:
        Xmeta = ybase
        ymeta = y[test]
```
diff --git a/civismlext/test/test_stacking.py b/civismlext/test/test_stacking.py
index eefd2dc..39741c6 100644
--- a/civismlext/test/test_stacking.py
+++ b/civismlext/test/test_stacking.py
@@ -8,7 +8,7 @@ from ..nonnegative import NonNegativeLinearRegression
 import pytest
 import numpy as np
 from sklearn.tree import DecisionTreeClassifier
-from sklearn.model_selection import train_test_split
+from sklearn.model_selection import KFold, train_test_split
 from sklearn.metrics import roc_curve, auc, mean_squared_error
 from sklearn.linear_model import LogisticRegression, LinearRegression
 from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
@@ -729,6 +729,32 @@ def test_fit_regression():
     assert sr.meta_estimator.fit_params == {'bar': 'b'}
 
 
+def test_cv_shuffle_indices():
+    """Make sure xmeta and ymeta retain the correct order, even when the CV
+    generator is shuffling. This is checking for the bug reported in issue #16.
+    """
+    estlist = [('be1', PassThruReg()),
+               ('be2', PassThruReg()),
+               ('meta', PassThruReg())]
+    sr = StackedRegressor(estlist, cv=KFold(n_splits=2, shuffle=True))
+    x = np.arange(6)
+    y = np.arange(6)
+
+    # Suppose the train indices of a 2-fold CV are:
+    #  [a, b, c]  and  [d, e, f]
+    # Then the test indices are:
+    #  [d, e, f]  and  [a, b, c]
+    # Since xmeta is just a pass-through of x[train] (horizontally stacked
+    # twice, due to the two base estimators) and ymeta is a pass-through of
+    # y[test], and since x = y, we should expect that:
+    #  xmeta[inds, 0] == ymeta
+    #
+    inds = np.array([3, 4, 5, 0, 1, 2])
+
+    xmeta, ymeta, _ = sr._base_est_fit_predict(x, y)
+    np.testing.assert_equal(xmeta[inds, 0], ymeta)
+
+
 @mock.patch('civismlext.stacking.clone', lambda x: x)
 def test_base_est_fit_predict_multiout_regression():
     estlist = [('be1', PassThruReg()),

pip install -e .[dev]
parse_log_pytest
requirements.txt
pytest
3.6
requirements.txt
pytest --no-header -rA --tb=line --color=no -p no:cacheprovider -W ignore::DeprecationWarning
attrs==22.2.0
certifi==2021.5.30
-e git+https://github.com/civisanalytics/civisml-extensions.git@8f4b19e074bace3172269bdce3c16bd3dd242256#egg=civisml_extensions
importlib-metadata==4.8.3
iniconfig==1.1.1
numpy==1.19.5
packaging==21.3
pandas==0.25.3
pluggy==1.0.0
py==1.11.0
pyparsing==3.1.4
pytest==7.0.1
python-dateutil==2.9.0.post0
pytz==2025.2
scikit-learn==0.19.2
scipy==1.5.4
six==1.17.0
tomli==1.2.3
typing_extensions==4.1.1
zipp==3.6.0

name: civisml-extensions
channels:
  - defaults
  - https://repo.anaconda.com/pkgs/main
  - https://repo.anaconda.com/pkgs/r
  - conda-forge
dependencies:
  - _libgcc_mutex=0.1=main
  - _openmp_mutex=5.1=1_gnu
  - ca-certificates=2025.2.25=h06a4308_0
  - certifi=2021.5.30=py36h06a4308_0
  - ld_impl_linux-64=2.40=h12ee557_0
  - libffi=3.3=he6710b0_2
  - libgcc-ng=11.2.0=h1234567_1
  - libgomp=11.2.0=h1234567_1
  - libstdcxx-ng=11.2.0=h1234567_1
  - ncurses=6.4=h6a678d5_0
  - openssl=1.1.1w=h7f8727e_0
  - pip=21.2.2=py36h06a4308_0
  - python=3.6.13=h12debd9_1
  - readline=8.2=h5eee18b_0
  - setuptools=58.0.4=py36h06a4308_0
  - sqlite=3.45.3=h5eee18b_0
  - tk=8.6.14=h39e8969_0
  - wheel=0.37.1=pyhd3eb1b0_0
  - xz=5.6.4=h5eee18b_1
  - zlib=1.2.13=h5eee18b_1
  - pip:
      - attrs==22.2.0
      - importlib-metadata==4.8.3
      - iniconfig==1.1.1
      - numpy==1.19.5
      - packaging==21.3
      - pandas==0.25.3
      - pluggy==1.0.0
      - py==1.11.0
      - pyparsing==3.1.4
      - pytest==7.0.1
      - python-dateutil==2.9.0.post0
      - pytz==2025.2
      - scikit-learn==0.19.2
      - scipy==1.5.4
      - six==1.17.0
      - tomli==1.2.3
      - typing-extensions==4.1.1
      - zipp==3.6.0
prefix: /opt/conda/envs/civisml-extensions

civismlext/test/test_stacking.py::test_cv_shuffle_indices
civismlext/test/test_stacking.py::test_init[StackedClassifier] civismlext/test/test_stacking.py::test_init[StackedRegressor] civismlext/test/test_stacking.py::test_get_params[StackedClassifier] civismlext/test/test_stacking.py::test_get_params[StackedRegressor] civismlext/test/test_stacking.py::test_get_params_estimator_list_empty[StackedClassifier] civismlext/test/test_stacking.py::test_get_params_estimator_list_empty[StackedRegressor] civismlext/test/test_stacking.py::test_set_params[StackedClassifier] civismlext/test/test_stacking.py::test_set_params[StackedRegressor] civismlext/test/test_stacking.py::test_properties_clf civismlext/test/test_stacking.py::test_properties_regression civismlext/test/test_stacking.py::test_smoke_clf_methods[1] civismlext/test/test_stacking.py::test_smoke_clf_methods[3] civismlext/test/test_stacking.py::test_smoke_multiclass_clf_methods[1] civismlext/test/test_stacking.py::test_smoke_multiclass_clf_methods[3] civismlext/test/test_stacking.py::test_smoke_regression_methods[1] civismlext/test/test_stacking.py::test_smoke_regression_methods[3] civismlext/test/test_stacking.py::test_smoke_multiout_regression_methods[1] civismlext/test/test_stacking.py::test_smoke_multiout_regression_methods[3] civismlext/test/test_stacking.py::test_validate_estimators[StackedClassifier] civismlext/test/test_stacking.py::test_validate_estimators[StackedRegressor] civismlext/test/test_stacking.py::test_validate_clf_estimators civismlext/test/test_stacking.py::test_check_clf_methods civismlext/test/test_stacking.py::test_validate_estimators_in_fit[StackedClassifier] civismlext/test/test_stacking.py::test_validate_estimators_in_fit[StackedRegressor] civismlext/test/test_stacking.py::test_validate_names[StackedClassifier] civismlext/test/test_stacking.py::test_validate_names[StackedRegressor] civismlext/test/test_stacking.py::test_validate_names_in_fit[StackedClassifier] civismlext/test/test_stacking.py::test_validate_names_in_fit[StackedRegressor] civismlext/test/test_stacking.py::test_validate_at_least_2_estimators[StackedClassifier] civismlext/test/test_stacking.py::test_validate_at_least_2_estimators[StackedRegressor] civismlext/test/test_stacking.py::test_extract_fit_params[StackedClassifier] civismlext/test/test_stacking.py::test_extract_fit_params[StackedRegressor] civismlext/test/test_stacking.py::test_base_est_fit[StackedClassifier] civismlext/test/test_stacking.py::test_base_est_fit[StackedRegressor] civismlext/test/test_stacking.py::test_base_est_predict_clf civismlext/test/test_stacking.py::test_base_est_predict_reg civismlext/test/test_stacking.py::test_base_est_fit_predict_clf civismlext/test/test_stacking.py::test_base_est_fit_predict_multiout_clf civismlext/test/test_stacking.py::test_fit_clf civismlext/test/test_stacking.py::test_fit_multiclass_clf civismlext/test/test_stacking.py::test_fit_multiout_clf civismlext/test/test_stacking.py::test_base_est_fit_predict_regression civismlext/test/test_stacking.py::test_fit_regression civismlext/test/test_stacking.py::test_base_est_fit_predict_multiout_regression civismlext/test/test_stacking.py::test_fit_multiout_regression civismlext/test/test_stacking.py::test_fit_pred_simple_regression civismlext/test/test_stacking.py::test_fit_pred_simple_clf civismlext/test/test_stacking.py::test_fit_params_regression civismlext/test/test_stacking.py::test_fit_params_clf civismlext/test/test_stacking.py::test_integration_regression[1] civismlext/test/test_stacking.py::test_integration_regression[3] civismlext/test/test_stacking.py::test_integration_clf[1] civismlext/test/test_stacking.py::test_integration_clf[3]
civismlext/stacking.py CHANGELOG.md
civismlext/stacking.py CHANGELOG.md