laliberte: @jameshiebert do you give your thumbs up for this to be merged?
jameshiebert: Yeah, I like the way that you split out the utility functions. Makes the code more clear. In any case, it looks good from my perspective.
laliberte: Ok, I'll go ahead and merge!
diff --git a/setup.py b/setup.py
index 0cfe3cc..d6134fa 100644
--- a/setup.py
+++ b/setup.py
@@ -44,6 +44,7 @@ tests_require = functions_extras + cas_extras + server_extras + [
 
 testing_extras = tests_require + [
     'nose',
+    'mock',
     'coverage',
     'requests'
 ]
diff --git a/src/pydap/handlers/dap.py b/src/pydap/handlers/dap.py
index d1b66c7..5cb9b4c 100644
--- a/src/pydap/handlers/dap.py
+++ b/src/pydap/handlers/dap.py
@@ -139,43 +139,9 @@ class BaseProxy(object):
         dds, data = r.body.split(b'\nData:\n', 1)
         dds = dds.decode(r.content_encoding or 'ascii')
 
-        if self.shape:
-            # skip size packing
-            if self.dtype.char in 'SU':
-                data = data[4:]
-            else:
-                data = data[8:]
-
-        # calculate array size
-        shape = tuple(
-            int(np.ceil((s.stop-s.start)/float(s.step))) for s in index)
-        size = int(np.prod(shape))
-
-        if self.dtype == np.byte:
-            return np.fromstring(data[:size], 'B').reshape(shape)
-        elif self.dtype.char in 'SU':
-            out = []
-            for word in range(size):
-                n = np.asscalar(np.fromstring(data[:4], '>I'))  # read length
-                data = data[4:]
-                out.append(data[:n])
-                data = data[n + (-n % 4):]
-            return np.array([text_type(x.decode('ascii'))
-                             for x in out], 'S').reshape(shape)
-        else:
-            try:
-                return np.fromstring(data, self.dtype).reshape(shape)
-            except ValueError as e:
-                if str(e) == 'total size of new array must be unchanged':
-                    # server-side failure.
-                    # it is expected that the user should be mindful of this:
-                    raise RuntimeError(
-                                ('variable {0} could not be properly '
-                                 'retrieved. To avoid this '
-                                 'error consider using open_url(..., '
-                                 'output_grid=False).').format(quote(self.id)))
-                else:
-                    raise
+        # Parse received dataset:
+        dataset = build_dataset(dds, data=data)
+        return dataset[self.id].data
 
     def __len__(self):
         return self.shape[0]
diff --git a/src/pydap/handlers/lib.py b/src/pydap/handlers/lib.py
index 438ac89..72222dc 100644
--- a/src/pydap/handlers/lib.py
+++ b/src/pydap/handlers/lib.py
@@ -196,12 +196,18 @@ def apply_selection(selection, dataset):
     return dataset
 
 
+def degenerate_grid_to_structure(candidate):
+    if isinstance(candidate, GridType):
+        candidate = StructureType(
+            candidate.name, candidate.attributes)
+    return candidate
+
+
 def apply_projection(projection, dataset):
     """Apply a given projection to a dataset.
 
     This function builds and returns a new dataset by adding those variables
     that were requested on the projection.
-
     """
     out = DatasetType(name=dataset.name, attributes=dataset.attributes)
 
@@ -218,9 +224,7 @@ def apply_projection(projection, dataset):
                         # if there are more children to add we need to clear
                         # the candidate so it has only explicitly added
                         # children; also, Grids are degenerated into Structures
-                        if isinstance(candidate, GridType):
-                            candidate = StructureType(
-                                candidate.name, candidate.attributes)
+                        candidate = degenerate_grid_to_structure(candidate)
                         candidate._keys = []
                     target[name] = candidate
                 target, template = target[name], template[name]
diff --git a/src/pydap/model.py b/src/pydap/model.py
index 83180d8..9c11b6e 100644
--- a/src/pydap/model.py
+++ b/src/pydap/model.py
@@ -322,8 +322,18 @@ class StructureType(DapType):
         item.id = '%s.%s' % (self.id, item.name)
 
     def __getitem__(self, key):
-        key = quote(key)
-        return self._dict[key]
+        try:
+            return self._dict[quote(key)]
+        except KeyError:
+            splitted = key.split('.')
+            if len(splitted) > 1:
+                try:
+                    return (self
+                            .__getitem__(splitted[0])['.'.join(splitted[1:])])
+                except KeyError:
+                    return self.__getitem__('.'.join(splitted[1:]))
+            else:
+                raise
 
     def __delitem__(self, key):
         self._dict.__delitem__(key)
diff --git a/src/pydap/parsers/dds.py b/src/pydap/parsers/dds.py
index 7c85d64..19560b4 100644
--- a/src/pydap/parsers/dds.py
+++ b/src/pydap/parsers/dds.py
@@ -3,6 +3,7 @@
 import re
 
 import numpy as np
+from six import text_type
 
 from . import SimpleParser
 from ..model import (DatasetType, BaseType,
@@ -10,7 +11,6 @@ from ..model import (DatasetType, BaseType,
                      GridType)
 from ..lib import quote, STRING
 
-
 typemap = {
     'byte':    np.dtype("B"),
     'int':     np.dtype(">i4"),
@@ -32,9 +32,10 @@ class DDSParser(SimpleParser):
 
     """A parser for the DDS."""
 
-    def __init__(self, dds):
+    def __init__(self, dds, data=None):
         super(DDSParser, self).__init__(dds, re.IGNORECASE)
         self.dds = dds
+        self.data = data
 
     def consume(self, regexp):
         """Consume and return a token."""
@@ -81,6 +82,9 @@ class DDSParser(SimpleParser):
         self.consume(';')
 
         data = DummyData(dtype, shape)
+        if self.data is not None:
+            data, self.data = convert_data_to_array(self.data, shape,
+                                                    dtype, name)
         var = BaseType(name, data, dimensions=dimensions)
 
         return var
@@ -156,12 +160,54 @@ class DDSParser(SimpleParser):
         return grid
 
 
-def build_dataset(dds):
+def build_dataset(dds, data=None):
     """Return a dataset object from a DDS representation."""
-    return DDSParser(dds).parse()
+    return DDSParser(dds, data=data).parse()
 
 
 class DummyData(object):
     def __init__(self, dtype, shape):
         self.dtype = dtype
         self.shape = shape
+
+
+def convert_data_to_array(data, shape, dtype, id):
+    if len(shape) > 0:
+        if dtype.char in 'SU':
+            data = data[4:]
+        else:
+            data = data[8:]
+
+    # calculate array size
+    size = 1
+    if len(shape) > 0:
+        size = int(np.prod(shape))
+
+    nitems = dtype.itemsize * size
+
+    if dtype == np.byte:
+        return np.fromstring(data[:nitems], 'B').reshape(shape), data[nitems:]
+    elif dtype.char in 'SU':
+        out = []
+        for word in range(size):
+            n = np.asscalar(np.fromstring(data[:4], '>I'))  # read itemsize
+            data = data[4:]
+            out.append(data[:n])
+            data = data[n + (-n % 4):]
+        return np.array([text_type(x.decode('ascii'))
+                         for x in out], 'S').reshape(shape), data
+    else:
+        try:
+            return (np.fromstring(data[:nitems], dtype).reshape(shape),
+                    data[nitems:])
+        except ValueError as e:
+            if str(e) == 'total size of new array must be unchanged':
+                # server-side failure.
+                # it is expected that the user should be mindful of this:
+                raise RuntimeError(
+                            ('variable {0} could not be properly '
+                             'retrieved. To avoid this '
+                             'error consider using open_url(..., '
+                             'output_grid=False).').format(quote(id)))
+            else:
+                raise

Test Erddap responses
Some Erddap servers return invalid DDS responses when a deep object is returned. In that case, we should parse the DDS from the DODS response and extract the data using it.

diff --git a/src/pydap/tests/test_handlers_dap.py b/src/pydap/tests/test_handlers_dap.py
index f6cfe31..34abe45 100644
--- a/src/pydap/tests/test_handlers_dap.py
+++ b/src/pydap/tests/test_handlers_dap.py
@@ -8,6 +8,10 @@ from pydap.tests.datasets import (
     SimpleSequence, SimpleGrid, SimpleArray, VerySimpleSequence)
 
 import unittest
+try:
+    from unittest.mock import patch
+except ImportError:
+    from mock import patch
 
 
 class TestDapHandler(unittest.TestCase):
@@ -50,6 +54,67 @@ class TestDapHandler(unittest.TestCase):
         self.assertEqual(
             dataset.SimpleGrid.SimpleGrid.data.slice,
             (slice(None), slice(None)))
+        self.assertEqual(
+                repr(dataset.SimpleGrid[:]),
+                "<GridType with array 'SimpleGrid' and maps 'x', 'y'>")
+
+    def test_grid_erddap(self):
+        """Test that dataset has the correct data proxies for grids
+           with the ERDDAP behavior."""
+        with patch('pydap.handlers.lib.degenerate_grid_to_structure',
+                   side_effect=(lambda x: x)) as mock_degenerate:
+            dataset = DAPHandler("http://localhost:8001/", self.app1).dataset
+            self.assertEqual(
+                    repr(dataset.SimpleGrid[:]),
+                    "<GridType with array 'SimpleGrid' and maps 'x', 'y'>")
+            assert mock_degenerate.called
+
+    def test_grid_output_grid_false(self):
+        """Test that dataset has the correct data proxies for grids with
+           option output_grid set to False."""
+        dataset = DAPHandler("http://localhost:8001/", self.app1,
+                             output_grid=False).dataset
+
+        self.assertEqual(dataset.keys(), ["SimpleGrid", "x", "y"])
+        self.assertEqual(
+            dataset.SimpleGrid.keys(), ["SimpleGrid", "x", "y"])
+
+        # test one of the axis
+        self.assertIsInstance(dataset.SimpleGrid.x.data, BaseProxy)
+        self.assertEqual(
+            dataset.SimpleGrid.x.data.baseurl, "http://localhost:8001/")
+        self.assertEqual(dataset.SimpleGrid.x.data.id, "SimpleGrid.x")
+        self.assertEqual(dataset.SimpleGrid.x.data.dtype, np.dtype('>i4'))
+        self.assertEqual(dataset.SimpleGrid.x.data.shape, (3,))
+        self.assertEqual(
+            dataset.SimpleGrid.x.data.slice, (slice(None),))
+
+        # test the grid
+        self.assertIsInstance(dataset.SimpleGrid.SimpleGrid.data, BaseProxy)
+        self.assertEqual(
+            dataset.SimpleGrid.SimpleGrid.data.baseurl,
+            "http://localhost:8001/")
+        self.assertEqual(
+            dataset.SimpleGrid.SimpleGrid.data.id, "SimpleGrid.SimpleGrid")
+        self.assertEqual(
+            dataset.SimpleGrid.SimpleGrid.data.dtype, np.dtype('>i4'))
+        self.assertEqual(dataset.SimpleGrid.SimpleGrid.data.shape, (2, 3))
+        self.assertEqual(
+            dataset.SimpleGrid.SimpleGrid.data.slice,
+            (slice(None), slice(None)))
+        np.testing.assert_array_equal(dataset.SimpleGrid[:],
+                                      [[0, 1, 2], [3, 4, 5]])
+
+    def test_grid_erddap_output_grid_false(self):
+        """Test that dataset has the correct data proxies for grids with
+           option output_grid set to False and with the ERDDAP behavior."""
+        with patch('pydap.handlers.lib.degenerate_grid_to_structure',
+                   side_effect=(lambda x: x)) as mock_degenerate:
+            dataset = DAPHandler("http://localhost:8001/", self.app1,
+                                 output_grid=False).dataset
+            np.testing.assert_array_equal(dataset.SimpleGrid[:],
+                                          [[0, 1, 2], [3, 4, 5]])
+            assert mock_degenerate.called
 
     def test_grid_with_projection(self):
         """Test that a sliced proxy can be created for grids."""

pip install -e .[testing]
parse_log_pytest
pytest
nose pytest
apt-get update apt-get install -y gfortran libblas-dev liblapack-dev
3.6
pytest --no-header -rA --tb=line --color=no -p no:cacheprovider -W ignore::DeprecationWarning
attrs @ file:///opt/conda/conda-bld/attrs_1642510447205/work
beautifulsoup4==4.12.3
certifi==2021.5.30
charset-normalizer==2.0.12
coards==1.0.5
coverage==6.2
docopt==0.6.2
flake8==5.0.4
gsw==3.4.0
gunicorn==21.2.0
idna==3.10
importlib-metadata==4.2.0
iniconfig @ file:///home/linux1/recipes/ci/iniconfig_1610983019677/work
Jinja2==3.0.3
lxml==5.3.1
MarkupSafe==2.0.1
mccabe==0.7.0
MechanicalSoup==1.3.0
more-itertools @ file:///tmp/build/80754af9/more-itertools_1637733554872/work
nose==1.3.7
numpy==1.19.5
packaging @ file:///tmp/build/80754af9/packaging_1637314298585/work
pluggy @ file:///tmp/build/80754af9/pluggy_1615976315926/work
py @ file:///opt/conda/conda-bld/py_1644396412707/work
pycodestyle==2.9.1
-e git+https://github.com/pydap/pydap.git@84273bf521ac625c19c2f603529cebd6523e9d61#egg=Pydap
pyflakes==2.5.0
pyparsing @ file:///tmp/build/80754af9/pyparsing_1635766073266/work
pytest==6.2.4
requests==2.27.1
scipy==1.5.4
six==1.17.0
soupsieve==2.3.2.post1
toml @ file:///tmp/build/80754af9/toml_1616166611790/work
typing_extensions @ file:///opt/conda/conda-bld/typing_extensions_1647553014482/work
urllib3==1.26.20
waitress==2.0.0
WebOb==1.8.9
WebTest==3.0.0
zipp @ file:///tmp/build/80754af9/zipp_1633618647012/work

name: pydap
channels:
  - defaults
  - https://repo.anaconda.com/pkgs/main
  - https://repo.anaconda.com/pkgs/r
  - conda-forge
dependencies:
  - _libgcc_mutex=0.1=main
  - _openmp_mutex=5.1=1_gnu
  - attrs=21.4.0=pyhd3eb1b0_0
  - ca-certificates=2025.2.25=h06a4308_0
  - certifi=2021.5.30=py36h06a4308_0
  - importlib_metadata=4.8.1=hd3eb1b0_0
  - iniconfig=1.1.1=pyhd3eb1b0_0
  - ld_impl_linux-64=2.40=h12ee557_0
  - libffi=3.3=he6710b0_2
  - libgcc-ng=11.2.0=h1234567_1
  - libgomp=11.2.0=h1234567_1
  - libstdcxx-ng=11.2.0=h1234567_1
  - more-itertools=8.12.0=pyhd3eb1b0_0
  - ncurses=6.4=h6a678d5_0
  - openssl=1.1.1w=h7f8727e_0
  - packaging=21.3=pyhd3eb1b0_0
  - pip=21.2.2=py36h06a4308_0
  - pluggy=0.13.1=py36h06a4308_0
  - py=1.11.0=pyhd3eb1b0_0
  - pyparsing=3.0.4=pyhd3eb1b0_0
  - pytest=6.2.4=py36h06a4308_2
  - python=3.6.13=h12debd9_1
  - readline=8.2=h5eee18b_0
  - setuptools=58.0.4=py36h06a4308_0
  - sqlite=3.45.3=h5eee18b_0
  - tk=8.6.14=h39e8969_0
  - toml=0.10.2=pyhd3eb1b0_0
  - typing_extensions=4.1.1=pyh06a4308_0
  - wheel=0.37.1=pyhd3eb1b0_0
  - xz=5.6.4=h5eee18b_1
  - zipp=3.6.0=pyhd3eb1b0_0
  - zlib=1.2.13=h5eee18b_1
  - pip:
      - beautifulsoup4==4.12.3
      - charset-normalizer==2.0.12
      - coards==1.0.5
      - coverage==6.2
      - docopt==0.6.2
      - flake8==5.0.4
      - gsw==3.4.0
      - gunicorn==21.2.0
      - idna==3.10
      - importlib-metadata==4.2.0
      - jinja2==3.0.3
      - lxml==5.3.1
      - markupsafe==2.0.1
      - mccabe==0.7.0
      - mechanicalsoup==1.3.0
      - nose==1.3.7
      - numpy==1.19.5
      - pycodestyle==2.9.1
      - pyflakes==2.5.0
      - requests==2.27.1
      - scipy==1.5.4
      - six==1.17.0
      - soupsieve==2.3.2.post1
      - urllib3==1.26.20
      - waitress==2.0.0
      - webob==1.8.9
      - webtest==3.0.0
prefix: /opt/conda/envs/pydap

src/pydap/tests/test_handlers_dap.py::TestDapHandler::test_grid_erddap src/pydap/tests/test_handlers_dap.py::TestDapHandler::test_grid_erddap_output_grid_false
src/pydap/tests/test_handlers_dap.py::TestDapHandler::test_base_type_with_projection src/pydap/tests/test_handlers_dap.py::TestDapHandler::test_grid src/pydap/tests/test_handlers_dap.py::TestDapHandler::test_grid_array_with_projection src/pydap/tests/test_handlers_dap.py::TestDapHandler::test_grid_map_with_projection src/pydap/tests/test_handlers_dap.py::TestDapHandler::test_grid_output_grid_false src/pydap/tests/test_handlers_dap.py::TestDapHandler::test_grid_with_projection src/pydap/tests/test_handlers_dap.py::TestDapHandler::test_sequence src/pydap/tests/test_handlers_dap.py::TestDapHandler::test_sequence_with_projection src/pydap/tests/test_handlers_dap.py::TestBaseProxy::test_comparisons src/pydap/tests/test_handlers_dap.py::TestBaseProxy::test_getitem src/pydap/tests/test_handlers_dap.py::TestBaseProxy::test_getitem_out_of_bound src/pydap/tests/test_handlers_dap.py::TestBaseProxy::test_inexact_division src/pydap/tests/test_handlers_dap.py::TestBaseProxy::test_iteration src/pydap/tests/test_handlers_dap.py::TestBaseProxy::test_len src/pydap/tests/test_handlers_dap.py::TestBaseProxy::test_repr src/pydap/tests/test_handlers_dap.py::TestBaseProxyShort::test_getitem src/pydap/tests/test_handlers_dap.py::TestBaseProxyString::test_getitem src/pydap/tests/test_handlers_dap.py::TestSequenceProxy::test_attributes src/pydap/tests/test_handlers_dap.py::TestSequenceProxy::test_comparisons src/pydap/tests/test_handlers_dap.py::TestSequenceProxy::test_getitem src/pydap/tests/test_handlers_dap.py::TestSequenceProxy::test_iter src/pydap/tests/test_handlers_dap.py::TestSequenceProxy::test_iter_child src/pydap/tests/test_handlers_dap.py::TestSequenceProxy::test_repr src/pydap/tests/test_handlers_dap.py::TestSequenceProxy::test_url src/pydap/tests/test_handlers_dap.py::TestSequenceWithString::test_filtering src/pydap/tests/test_handlers_dap.py::TestSequenceWithString::test_iter src/pydap/tests/test_handlers_dap.py::TestSequenceWithString::test_iter_child src/pydap/tests/test_handlers_dap.py::TestSequenceWithString::test_projection src/pydap/tests/test_handlers_dap.py::TestStringBaseType::test_getitem src/pydap/tests/test_handlers_dap.py::TestArrayStringBaseType::test_getitem
src/pydap/handlers/lib.py setup.py src/pydap/parsers/dds.py src/pydap/model.py src/pydap/handlers/dap.py
src/pydap/handlers/lib.py setup.py src/pydap/parsers/dds.py src/pydap/model.py src/pydap/handlers/dap.py