<h4 dir="auto">A small, complete example of the issue</h4>
<div class="highlight highlight-source-python notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="&quot;&quot;&quot;Example of Pandas bug.&quot;&quot;&quot;
from numpy.random import randn
from pandas import DataFrame, read_msgpack

POWER = 6
ROWS = 10 ** POWER
COLS = 280
SIZE = ROWS * COLS
# COL_NAMES = ['Columns']

print('rows: {:,}; cols: {}; data points: {:,}; float memory (bytes): {:,}; '
      'float memory (GiB): {:.3}'.format(ROWS, COLS, SIZE, SIZE * 8,
                                         float(SIZE * 8) / 1024 / 1024 / 1024))

DTF = DataFrame(randn(ROWS, COLS))  # , columns=COL_NAMES
print(DTF)
DTF.to_msgpack('foobar_{}.msg'.format(POWER))
print('Reading exercise')
DTF2 = read_msgpack('foobar_{}.msg'.format(POWER))
print(DTF2)"><pre class="notranslate"><span class="pl-s">"""Example of Pandas bug."""</span>
<span class="pl-k">from</span> <span class="pl-s1">numpy</span>.<span class="pl-s1">random</span> <span class="pl-k">import</span> <span class="pl-s1">randn</span>
<span class="pl-k">from</span> <span class="pl-s1">pandas</span> <span class="pl-k">import</span> <span class="pl-v">DataFrame</span>, <span class="pl-s1">read_msgpack</span>

<span class="pl-v">POWER</span> <span class="pl-c1">=</span> <span class="pl-c1">6</span>
<span class="pl-v">ROWS</span> <span class="pl-c1">=</span> <span class="pl-c1">10</span> <span class="pl-c1">**</span> <span class="pl-v">POWER</span>
<span class="pl-v">COLS</span> <span class="pl-c1">=</span> <span class="pl-c1">280</span>
<span class="pl-v">SIZE</span> <span class="pl-c1">=</span> <span class="pl-v">ROWS</span> <span class="pl-c1">*</span> <span class="pl-v">COLS</span>
<span class="pl-c"># COL_NAMES = ['Columns']</span>

<span class="pl-en">print</span>(<span class="pl-s">'rows: {:,}; cols: {}; data points: {:,}; float memory (bytes): {:,}; '</span>
      <span class="pl-s">'float memory (GiB): {:.3}'</span>.<span class="pl-en">format</span>(<span class="pl-v">ROWS</span>, <span class="pl-v">COLS</span>, <span class="pl-v">SIZE</span>, <span class="pl-v">SIZE</span> <span class="pl-c1">*</span> <span class="pl-c1">8</span>,
                                         <span class="pl-en">float</span>(<span class="pl-v">SIZE</span> <span class="pl-c1">*</span> <span class="pl-c1">8</span>) <span class="pl-c1">/</span> <span class="pl-c1">1024</span> <span class="pl-c1">/</span> <span class="pl-c1">1024</span> <span class="pl-c1">/</span> <span class="pl-c1">1024</span>))

<span class="pl-v">DTF</span> <span class="pl-c1">=</span> <span class="pl-v">DataFrame</span>(<span class="pl-en">randn</span>(<span class="pl-v">ROWS</span>, <span class="pl-v">COLS</span>))  <span class="pl-c"># , columns=COL_NAMES</span>
<span class="pl-en">print</span>(<span class="pl-v">DTF</span>)
<span class="pl-v">DTF</span>.<span class="pl-en">to_msgpack</span>(<span class="pl-s">'foobar_{}.msg'</span>.<span class="pl-en">format</span>(<span class="pl-v">POWER</span>))
<span class="pl-en">print</span>(<span class="pl-s">'Reading exercise'</span>)
<span class="pl-v">DTF2</span> <span class="pl-c1">=</span> <span class="pl-en">read_msgpack</span>(<span class="pl-s">'foobar_{}.msg'</span>.<span class="pl-en">format</span>(<span class="pl-v">POWER</span>))
<span class="pl-en">print</span>(<span class="pl-v">DTF2</span>)</pre></div>
<h4 dir="auto">Current Output</h4>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="rows: 1,000,000; cols: 280; data points: 280,000,000; float memory (bytes): 2,240,000,000; float memory (GiB): 2.09
        0         1         2         3         4         5         6    \
0 -0.026045  1.102096  0.702563 -1.508039  0.308464  0.504351  2.887272   

        7        8         9      ...          270      271       272    273  \
0 -0.700034 -0.11521 -0.447198    ...    -0.566173  0.16131 -0.581333 -0.362   

        274      275       276       277       278       279  
0 -1.433336  0.60287  0.799355  0.628572  0.542312 -0.410359  

[1 rows x 280 columns]
Reading exercise
Traceback (most recent call last):
  File &quot;../bug/pandas_bug6.py&quot;, line 20, in &lt;module&gt;
    print(DTF2.head(1))
AttributeError: 'list' object has no attribute 'head'"><pre class="notranslate"><code class="notranslate">rows: 1,000,000; cols: 280; data points: 280,000,000; float memory (bytes): 2,240,000,000; float memory (GiB): 2.09
        0         1         2         3         4         5         6    \
0 -0.026045  1.102096  0.702563 -1.508039  0.308464  0.504351  2.887272   

        7        8         9      ...          270      271       272    273  \
0 -0.700034 -0.11521 -0.447198    ...    -0.566173  0.16131 -0.581333 -0.362   

        274      275       276       277       278       279  
0 -1.433336  0.60287  0.799355  0.628572  0.542312 -0.410359  

[1 rows x 280 columns]
Reading exercise
Traceback (most recent call last):
  File "../bug/pandas_bug6.py", line 20, in &lt;module&gt;
    print(DTF2.head(1))
AttributeError: 'list' object has no attribute 'head'
</code></pre></div>
<h4 dir="auto">Expected Output</h4>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="rows: 1,000,000; cols: 280; data points: 280,000,000; float memory (bytes): 2,240,000,000; float memory (GiB): 2.09
        0         1         2         3         4         5         6    \
0 -0.026045  1.102096  0.702563 -1.508039  0.308464  0.504351  2.887272   

        7        8         9      ...          270      271       272    273  \
0 -0.700034 -0.11521 -0.447198    ...    -0.566173  0.16131 -0.581333 -0.362   

        274      275       276       277       278       279  
0 -1.433336  0.60287  0.799355  0.628572  0.542312 -0.410359  

[1 rows x 280 columns]
Reading exercise
        0         1         2         3         4         5         6    \
0 -0.026045  1.102096  0.702563 -1.508039  0.308464  0.504351  2.887272   

        7        8         9      ...          270      271       272    273  \
0 -0.700034 -0.11521 -0.447198    ...    -0.566173  0.16131 -0.581333 -0.362   

        274      275       276       277       278       279  
0 -1.433336  0.60287  0.799355  0.628572  0.542312 -0.410359  

[1 rows x 280 columns]"><pre class="notranslate"><code class="notranslate">rows: 1,000,000; cols: 280; data points: 280,000,000; float memory (bytes): 2,240,000,000; float memory (GiB): 2.09
        0         1         2         3         4         5         6    \
0 -0.026045  1.102096  0.702563 -1.508039  0.308464  0.504351  2.887272   

        7        8         9      ...          270      271       272    273  \
0 -0.700034 -0.11521 -0.447198    ...    -0.566173  0.16131 -0.581333 -0.362   

        274      275       276       277       278       279  
0 -1.433336  0.60287  0.799355  0.628572  0.542312 -0.410359  

[1 rows x 280 columns]
Reading exercise
        0         1         2         3         4         5         6    \
0 -0.026045  1.102096  0.702563 -1.508039  0.308464  0.504351  2.887272   

        7        8         9      ...          270      271       272    273  \
0 -0.700034 -0.11521 -0.447198    ...    -0.566173  0.16131 -0.581333 -0.362   

        274      275       276       277       278       279  
0 -1.433336  0.60287  0.799355  0.628572  0.542312 -0.410359  

[1 rows x 280 columns]
</code></pre></div>
<h4 dir="auto">Output of <code class="notranslate">pd.show_versions()</code></h4>
<details>
## INSTALLED VERSIONS
<p dir="auto">commit: None<br>
python: 2.7.10.final.0<br>
python-bits: 64<br>
OS: Darwin<br>
OS-release: 16.0.0<br>
machine: x86_64<br>
processor: i386<br>
byteorder: little<br>
LC_ALL: None<br>
LANG: None</p>
<p dir="auto">pandas: 0.18.1<br>
nose: None<br>
pip: 8.1.2<br>
setuptools: 18.5<br>
Cython: None<br>
numpy: 1.8.0rc1<br>
scipy: 0.13.0b1<br>
statsmodels: None<br>
xarray: None<br>
IPython: None<br>
sphinx: None<br>
patsy: None<br>
dateutil: 1.5<br>
pytz: 2013.7<br>
blosc: None<br>
bottleneck: None<br>
tables: None<br>
numexpr: None<br>
matplotlib: 1.3.1<br>
openpyxl: None<br>
xlrd: None<br>
xlwt: None<br>
xlsxwriter: None<br>
lxml: None<br>
bs4: None<br>
html5lib: None<br>
httplib2: None<br>
apiclient: None<br>
sqlalchemy: None<br>
pymysql: None<br>
psycopg2: None<br>
jinja2: None<br>
boto: None<br>
pandas_datareader: None<br>
None</p>
</details>
<p dir="auto">Hi guys,</p>
<p dir="auto">I am happy to help you improve <code class="notranslate">msgpack.</code><br>
I tried to export my massive dataframe this morning using <code class="notranslate">msgpack</code>, and I got this error.</p>
<p dir="auto"><code class="notranslate">ValueError: EXT data is too large</code></p>
<p dir="auto">What does that mean? Is there a size limit?</p>