file pattern complete, recipe not working yet
MODIFY
docs/tutorials/cmip6-recipe.ipynb
docs/tutorials/cmip6-recipe.ipynb
"output_type": "stream",
"text": [
"<class 'pandas.core.frame.DataFrame'>\n",
- "RangeIndex: 949541 entries, 0 to 949540\n",
+ "RangeIndex: 956306 entries, 0 to 956305\n",
"Data columns (total 13 columns):\n",
" # Column Non-Null Count Dtype \n",
"--- ------ -------------- ----- \n",
- " 0 project 949541 non-null object\n",
- " 1 institute 949541 non-null object\n",
- " 2 model 949541 non-null object\n",
- " 3 experiment_id 949541 non-null object\n",
- " 4 frequency 461707 non-null object\n",
- " 5 modeling_realm 461707 non-null object\n",
- " 6 mip_table 949541 non-null object\n",
- " 7 ensemble_member 949541 non-null object\n",
- " 8 grid_label 949541 non-null object\n",
- " 9 variable 949541 non-null object\n",
- " 10 temporal subset 920892 non-null object\n",
- " 11 version 949541 non-null object\n",
- " 12 path 949541 non-null object\n",
+ " 0 project 956306 non-null object\n",
+ " 1 institute 956306 non-null object\n",
+ " 2 model 956306 non-null object\n",
+ " 3 experiment_id 956306 non-null object\n",
+ " 4 frequency 467990 non-null object\n",
+ " 5 modeling_realm 467990 non-null object\n",
+ " 6 mip_table 956306 non-null object\n",
+ " 7 ensemble_member 956306 non-null object\n",
+ " 8 grid_label 956306 non-null object\n",
+ " 9 variable 956306 non-null object\n",
+ " 10 temporal subset 927657 non-null object\n",
+ " 11 version 956306 non-null object\n",
+ " 12 path 956306 non-null object\n",
"dtypes: object(13)\n",
- "memory usage: 94.2+ MB\n"
+ "memory usage: 94.8+ MB\n"
]
}
],
}
],
"source": [
- "# So there are 949,541 entries, one for each netcdf file. We can see the first five here:\n",
+ "# So there are 956,306 entries, one for each netcdf file. We can see the first five here:\n",
"# The 'path' column is the most important - you may need to scroll the window to see it!\n",
"\n",
"df_s3.head()"
{
"data": {
"text/plain": [
- "241675"
+ "241944"
]
},
"execution_count": 4,
}
],
"source": [
- "# We will add a new column which is our short name for the datasets (may take a moment for all 949541 rows)\n",
- "df_s3['dataset'] = df_s3['dataset'] = df_s3.apply(lambda row: '.'.join(row.path.split('/')[6:12]),axis=1)\n",
+ "# We will add a new column which is our short name for the datasets (may take a moment for all 956306 rows)\n",
+ "df_s3['dataset'] = df_s3.apply(lambda row: '.'.join(row.path.split('/')[6:12]),axis=1)\n",
"# the number of unique dataset names can be found using the 'nunique' method\n",
"df_s3.dataset.nunique()"
]
"output_type": "stream",
"text": [
"MRI-ESM2-0.piClim-SO2.r1i1p1f1.Amon.rlds.gn ['v20190912' 'v20200114']\n",
- "CanESM5.historical.r23i1p1f1.Lmon.gpp.gn ['v20190306' 'v20190429']\n",
- "IPSL-CM6A-LR.abrupt-4xCO2.r1i1p1f1.Lmon.cLeaf.gr ['v20180727' 'v20181005' 'v20190118']\n",
- "IPSL-CM6A-LR.piControl.r1i1p1f1.CFsubhr.rsdscs.gn ['v20180802' 'v20181022']\n",
- "CESM2.amip.r1i1p1f1.AERmon.mmrbc.gn ['v20190218' 'v20190319']\n",
- "GFDL-CM4.abrupt-4xCO2.r1i1p1f1.Lmon.mrro.gr1 ['v20180319' 'v20180701']\n",
- "CanESM5.hist-aer.r8i1p1f1.Amon.va.gn ['v20190306' 'v20190429']\n",
- "CanESM5.hist-volc.r9i1p1f1.Amon.va.gn ['v20190306' 'v20190429']\n",
- "CanESM5.ssp245-GHG.r9i1p1f1.Omon.fgo2.gn ['v20190306' 'v20190429']\n",
- "CanESM5.ssp119.r1i1p1f1.Omon.no3.gn ['v20190306' 'v20190429']\n",
- "CanESM5.ssp370.r1i1p1f1.Amon.uas.gn ['v20190306' 'v20190429']\n",
- "CanESM5.ssp434.r2i1p1f1.Omon.tauvo.gn ['v20190306' 'v20190429']\n",
- "CanESM5.ssp534-over.r1i1p1f1.Lmon.nbp.gn ['v20190306' 'v20190429']\n",
- "CNRM-ESM2-1.ssp585.r1i1p1f2.Emon.loadss.gr ['v20190328' 'v20191021']\n"
+ "CanESM5.historical.r1i1p1f1.Omon.no3.gn ['v20190306' 'v20190429']\n",
+ "CanESM5.historical.r6i1p1f1.Amon.ps.gn ['v20190306' 'v20190429']\n",
+ "IPSL-CM6A-LR.abrupt-4xCO2.r1i1p1f1.CFday.rsdscs.gr ['v20181005' 'v20190118']\n",
+ "MIROC6.abrupt-4xCO2.r1i1p1f1.Amon.tauv.gn ['v20190311' 'v20190705']\n",
+ "CESM2.abrupt-4xCO2.r1i1p1f1.Lmon.cSoilSlow.gn ['v20190425' 'v20190828' 'v20190927']\n",
+ "CanESM5.hist-CO2.r8i1p1f1.Omon.tauuo.gn ['v20190306' 'v20190429']\n",
+ "CanESM5.hist-nat.r5i1p1f1.Amon.vas.gn ['v20190306' 'v20190429']\n",
+ "CanESM5.ssp245-aer.r7i1p1f1.Omon.no3.gn ['v20190306' 'v20190429']\n",
+ "CanESM5.ssp245-stratO3.r8i1p1f1.Omon.wo.gn ['v20190306' 'v20190429']\n",
+ "IPSL-CM6A-LR.piClim-anthro.r1i1p1f1.Amon.rsut.gr ['v20190118' 'v20190419']\n",
+ "IPSL-CM6A-LR.piClim-spAer-anthro.r1i1p1f1.Lmon.nbp.gr ['v20190118' 'v20190507']\n",
+ "MRI-ESM2-0.piClim-lu.r1i1p1f1.Lmon.mrro.gn ['v20190603' 'v20200114']\n",
+ "CanESM5.ssp245.r4i1p1f1.Amon.va.gn ['v20190306' 'v20190429']\n",
+ "EC-Earth3-Veg.ssp245.r1i1p1f1.LImon.sbl.gr ['v20190629' 'v20200225']\n",
+ "IPSL-CM6A-LR.ssp126.r1i1p1f1.Omon.si.gn ['v20190121' 'v20190903']\n",
+ "IPSL-CM6A-LR.ssp245.r2i1p1f1.Omon.si.gn ['v20190119' 'v20190516']\n",
+ "IPSL-CM6A-LR.ssp585.r1i1p1f1.Omon.talkos.gn ['v20190119' 'v20190903']\n"
]
}
],
" </thead>\n",
" <tbody>\n",
" <tr>\n",
- " <th>460531</th>\n",
+ " <th>463275</th>\n",
" <td>CMIP6</td>\n",
" <td>NOAA-GFDL</td>\n",
" <td>GFDL-CM4</td>\n",
" <td>GFDL-CM4.historical.r1i1p1f1.Amon.tas.gr1</td>\n",
" </tr>\n",
" <tr>\n",
- " <th>460532</th>\n",
+ " <th>463276</th>\n",
" <td>CMIP6</td>\n",
" <td>NOAA-GFDL</td>\n",
" <td>GFDL-CM4</td>\n",
" <td>GFDL-CM4.historical.r1i1p1f1.Amon.tas.gr1</td>\n",
" </tr>\n",
" <tr>\n",
- " <th>460533</th>\n",
+ " <th>463277</th>\n",
" <td>CMIP6</td>\n",
" <td>NOAA-GFDL</td>\n",
" <td>GFDL-CM4</td>\n",
],
"text/plain": [
" project institute model experiment_id frequency modeling_realm \\\n",
- "460531 CMIP6 NOAA-GFDL GFDL-CM4 historical mon atmos \n",
- "460532 CMIP6 NOAA-GFDL GFDL-CM4 historical mon atmos \n",
- "460533 CMIP6 NOAA-GFDL GFDL-CM4 historical mon atmos \n",
+ "463275 CMIP6 NOAA-GFDL GFDL-CM4 historical mon atmos \n",
+ "463276 CMIP6 NOAA-GFDL GFDL-CM4 historical mon atmos \n",
+ "463277 CMIP6 NOAA-GFDL GFDL-CM4 historical mon atmos \n",
"\n",
" mip_table ensemble_member grid_label variable temporal subset \\\n",
- "460531 Amon r1i1p1f1 gr1 tas 185001-185412 \n",
- "460532 Amon r1i1p1f1 gr1 tas 185001-194912 \n",
- "460533 Amon r1i1p1f1 gr1 tas 195001-201412 \n",
+ "463275 Amon r1i1p1f1 gr1 tas 185001-185412 \n",
+ "463276 Amon r1i1p1f1 gr1 tas 185001-194912 \n",
+ "463277 Amon r1i1p1f1 gr1 tas 195001-201412 \n",
"\n",
" version path \\\n",
- "460531 v20180301 s3://esgf-world/CMIP6/CMIP/NOAA-GFDL/GFDL-CM4/... \n",
- "460532 v20180701 s3://esgf-world/CMIP6/CMIP/NOAA-GFDL/GFDL-CM4/... \n",
- "460533 v20180701 s3://esgf-world/CMIP6/CMIP/NOAA-GFDL/GFDL-CM4/... \n",
+ "463275 v20180301 s3://esgf-world/CMIP6/CMIP/NOAA-GFDL/GFDL-CM4/... \n",
+ "463276 v20180701 s3://esgf-world/CMIP6/CMIP/NOAA-GFDL/GFDL-CM4/... \n",
+ "463277 v20180701 s3://esgf-world/CMIP6/CMIP/NOAA-GFDL/GFDL-CM4/... \n",
"\n",
" dataset \n",
- "460531 GFDL-CM4.historical.r1i1p1f1.Amon.tas.gr1 \n",
- "460532 GFDL-CM4.historical.r1i1p1f1.Amon.tas.gr1 \n",
- "460533 GFDL-CM4.historical.r1i1p1f1.Amon.tas.gr1 "
+ "463275 GFDL-CM4.historical.r1i1p1f1.Amon.tas.gr1 \n",
+ "463276 GFDL-CM4.historical.r1i1p1f1.Amon.tas.gr1 \n",
+ "463277 GFDL-CM4.historical.r1i1p1f1.Amon.tas.gr1 "
]
},
"execution_count": 9,
" return ds"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Step 4: Create a recipe\n",
+ "- A `FilePattern` is the starting place for all recipes. These Python objects are the \"raw ingredients\" upon which the recipe will act. They describe how many individual source files are organized logically as part of a larger dataset. (Further reading is available in [the file pattern documention](https://pangeo-forge.readthedocs.io/en/latest/file_patterns.html#create-a-filepattern).) To create a file pattern, the first step is to define a function which takes any variable components of the dataset file path as inputs, and returns full file path strings.\n",
+ "- Revisting our input urls, we see that the only variable components of these paths are the 13-character numerical strings which immediatly precede the .nc file extension:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "s3://esgf-world/CMIP6/CMIP/NOAA-GFDL/GFDL-CM4/historical/r1i1p1f1/Amon/tas/gr1/v20180701/tas_Amon_GFDL-CM4_historical_r1i1p1f1_gr1_185001-194912.nc\n",
+ " \n",
+ "s3://esgf-world/CMIP6/CMIP/NOAA-GFDL/GFDL-CM4/historical/r1i1p1f1/Amon/tas/gr1/v20180701/tas_Amon_GFDL-CM4_historical_r1i1p1f1_gr1_195001-201412.nc\n",
+ " \n"
+ ]
+ }
+ ],
+ "source": [
+ "for url in input_urls:\n",
+ " print(f'''{url}\n",
+ " ''')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**What do these strings refer to?**\n",
+ "- If it was not immediately apparent, comparison to our dataset coordinates makes it clear that these numerical strings are time ranges; the string `'185001-194912'` from the first url, e.g., represents a time range from Jan 1850 through Dec 1949:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Coordinates:\n",
+ " * bnds (bnds) float64 1.0 2.0\n",
+ " height float64 ...\n",
+ " * lat (lat) float64 -89.5 -88.5 -87.5 -86.5 -85.5 ... 86.5 87.5 88.5 89.5\n",
+ " * lon (lon) float64 0.625 1.875 3.125 4.375 ... 355.6 356.9 358.1 359.4\n",
+ " * time (time) object 1850-01-16 12:00:00 ... 1949-12-16 12:00:00\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(ds.coords)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Let's define a function which variabizes these strings**\n",
+ "- ... and returns full files paths!"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def make_full_path(time_range):\n",
+ " '''\n",
+ " Parameters\n",
+ " ----------\n",
+ " time_range : str\n",
+ " \n",
+ " A 13-character string, comprised of two 6-character dates delimited by a dash. \n",
+ " The first four characters of each date are the year, and the final two are the month.\n",
+ " \n",
+ " e.g. The time_range for Jan 1850 through Dec 1949 is expressed as '185001-194912'.\n",
+ " \n",
+ " '''\n",
+ " base_url = 's3://esgf-world/CMIP6/CMIP/NOAA-GFDL/GFDL-CM4/historical/r1i1p1f1/Amon/tas/gr1/v20180701/'\n",
+ " return base_url + f'tas_Amon_GFDL-CM4_historical_r1i1p1f1_gr1_{time_range}.nc'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "s3://esgf-world/CMIP6/CMIP/NOAA-GFDL/GFDL-CM4/historical/r1i1p1f1/Amon/tas/gr1/v20180701/tas_Amon_GFDL-CM4_historical_r1i1p1f1_gr1_185001-194912.nc\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "True"
+ ]
+ },
+ "execution_count": 17,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# And let's test out our function before moving on.\n",
+ "\n",
+ "test_url = make_full_path('185001-194912')\n",
+ "print(test_url)\n",
+ "\n",
+ "# If our function works, inputting '185001-194912' should have returned a url which is identical to\n",
+ "# the first of the two urls in the list named `input_urls` defined in cell 10, above:\n",
+ "\n",
+ "test_url == input_urls[0]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Combining dimensions**\n",
+ "- Before we initialize our file pattern, we need to define how we want files to be combined in our eventual zarr store\n",
+ "- As outlined in [the file pattern documentation](https://pangeo-forge.readthedocs.io/en/latest/file_patterns.html#create-a-filepattern), we have two options:\n",
+ " 1. Concatenating dimensions with a `ConcatDim` instance\n",
+ " 2. Merging dimensions with a `MergeDim` instance\n",
+ " \n",
+ " \n",
+ "- Our current dataset requires only concatenation, which we can achieve by instantiating `ConcatDim` with our variable name (`\"time_range\"`) as a positional argument, followed by a `keys` kwarg, which should be an iterable (in this case, a list) containing all of the ways which this variable appears in our set of file paths.\n",
+ "\n",
+ "> **Note:** This simple example has only two variations of this variable, so we've chosen to simply copy-and-paste them into a list. Were there to be a very large number of files, finding a way to generate this list programatically might be a good idea."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from pangeo_forge_recipes.patterns import ConcatDim\n",
+ "time_concat_dim = ConcatDim(\"time_range\", keys=['185001-194912', '195001-201412'])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Instantiating the file pattern**\n",
+ "- Now that we have a file path function and our \"combine dimensions\" object, we can move on to instantiating to file pattern, passing these two objects as arguments:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "<FilePattern {'time_range': 2}>"
+ ]
+ },
+ "execution_count": 19,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "from pangeo_forge.patterns import FilePattern\n",
+ "pattern = FilePattern(make_full_path, time_concat_dim)\n",
+ "pattern"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "> As described in [the filepattern documentation](https://pangeo-forge.readthedocs.io/en/latest/file_patterns.html#inspect-a-filepattern), inspecting our instantiated pattern (while not a required step of this workflow) may be helpful for gaining a sense of what this object is \"doing\". Specifically, we see that our pattern has indexed our two files chronologically according to the concatenation key we provided it, and assigned the correct url to each file using the file path function:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "(0,) s3://esgf-world/CMIP6/CMIP/NOAA-GFDL/GFDL-CM4/historical/r1i1p1f1/Amon/tas/gr1/v20180701/tas_Amon_GFDL-CM4_historical_r1i1p1f1_gr1_185001-194912.nc\n",
+ "(1,) s3://esgf-world/CMIP6/CMIP/NOAA-GFDL/GFDL-CM4/historical/r1i1p1f1/Amon/tas/gr1/v20180701/tas_Amon_GFDL-CM4_historical_r1i1p1f1_gr1_195001-201412.nc\n"
+ ]
+ }
+ ],
+ "source": [
+ "for index, fname in pattern.items():\n",
+ " print(index, fname)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Time to make the recipe!**\n",
+ "- Earlier versions of `pangeo-forge-recipes` contained a range of different recipes classes for different use cases. Nowadays, however, there is only one recipe, the `XarrayZarrRecipe`, which is what we'll be using here.\n",
+ "- In it's most basic form, `XarrayZarrRecipe` can be instantiated using a file pattern as the only argument. Here we'll be using some of the optional arguments to specify our targeted chunk size, chunk preprocessing function, etc.:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "metadata": {},
+ "outputs": [
+ {
+ "ename": "IndexError",
+ "evalue": "list index out of range",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)",
+ "\u001b[0;32m<ipython-input-21-b7662d2a343b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mpangeo_forge_recipes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecipes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mxarray_zarr\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mXarrayZarrRecipe\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m recipe = XarrayZarrRecipe(\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0mpattern\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0mtarget_chunks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtarget_chunks\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;32m~/.pyenv/versions/anaconda3-2019.10/envs/pangeo-forge3.8/lib/python3.8/site-packages/pangeo_forge_recipes-0.0.0-py3.8.egg/pangeo_forge_recipes/recipes/xarray_zarr.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, file_pattern, inputs_per_chunk, target_chunks, target, input_cache, metadata_cache, cache_inputs, copy_input_to_local_file, consolidate_zarr, xarray_open_kwargs, xarray_concat_kwargs, delete_input_encoding, fsspec_open_kwargs, process_input, process_chunk, _concat_dim, _concat_dim_chunks)\u001b[0m\n",
+ "\u001b[0;32m~/.pyenv/versions/anaconda3-2019.10/envs/pangeo-forge3.8/lib/python3.8/site-packages/pangeo_forge_recipes-0.0.0-py3.8.egg/pangeo_forge_recipes/recipes/xarray_zarr.py\u001b[0m in \u001b[0;36m__post_init__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 178\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_validate_file_pattern\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 179\u001b[0m \u001b[0;31m# from here on we know there is at most one merge dim and one concat dim\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 180\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_concat_dim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfile_pattern\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcat_dims\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 181\u001b[0m self._cache_metadata = any(\n\u001b[1;32m 182\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mv\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfile_pattern\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconcat_sequence_lens\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+ "\u001b[0;31mIndexError\u001b[0m: list index out of range"
+ ]
+ }
+ ],
+ "source": [
+ "from pangeo_forge_recipes.recipes.xarray_zarr import XarrayZarrRecipe\n",
+ "\n",
+ "recipe = XarrayZarrRecipe(\n",
+ " pattern, \n",
+ " target_chunks = target_chunks,\n",
+ " process_chunk = set_bnds_as_coords,\n",
+ " #xarray_open_kwargs = {'decode_coords':False},\n",
+ " xarray_concat_kwargs = {'join':'exact'}\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "(ConcatDim(name='time_range', nitems_per_file=None),)"
+ ]
+ },
+ "execution_count": 22,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "pattern.combine_dims"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# TODO: Delete the next two cells once the new Step 4 (above) is completed"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
"metadata": {},
"outputs": [],
"source": [
- "# Choose a Recipe:\n",
- "from pangeo_forge_recipes.recipe import NetCDFtoZarrSequentialRecipe\n",
+ "from pangeo_forge_recipes.recipes.xarray_zarr import XarrayZarrRecipe\n",
"\n",
- "recipe = NetCDFtoZarrSequentialRecipe(\n",
+ "recipe = XarrayZarrRecipe(\n",
" input_urls = input_urls,\n",
" sequence_dim = \"time\", \n",
" target_chunks = target_chunks,\n",

file pattern complete, recipe not working yet