Python binary data reading
Question: A urllib2 request receives binary response as below:

    
    
    00 00 00 01 00 04 41 4D 54 44 00 00 00 00 02 41
    97 33 33 41 99 5C 29 41 90 3D 71 41 91 D7 0A 47
    0F C6 14 00 00 01 16 6A E0 68 80 41 93 B4 05 41
    97 1E B8 41 90 7A E1 41 96 8F 57 46 E6 2E 80 00
    00 01 16 7A 53 7C 80 FF FF
    

Its structure is:

    
    
    DATA, TYPE, DESCRIPTION
    
    00 00 00 01, 4 bytes, Symbol Count =1
    
    00 04, 2 bytes, Symbol Length = 4
    
    41 4D 54 44, 6 bytes, Symbol = AMTD
    
    00, 1 byte, Error code = 0 (OK)
    
    00 00 00 02, 4 bytes, Bar Count =  2
    
    FIRST BAR
    
    41 97 33 33, 4 bytes, Close = 18.90
    
    41 99 5C 29, 4 bytes, High = 19.17
    
    41 90 3D 71, 4 bytes, Low = 18.03
    
    41 91 D7 0A, 4 bytes, Open = 18.23
    
    47 0F C6 14, 4 bytes, Volume = 3,680,608
    
    00 00 01 16 6A E0 68 80, 8 bytes, Timestamp = November 23,2007
    
    SECOND BAR
    
    41 93 B4 05, 4 bytes, Close = 18.4629
    
    41 97 1E B8, 4 bytes, High = 18.89
    
    41 90 7A E1, 4 bytes, Low = 18.06
    
    41 96 8F 57, 4 bytes, Open = 18.82
    
    46 E6 2E 80, 4 bytes, Volume = 2,946,325
    
    00 00 01 16 7A 53 7C 80, 8 bytes, Timestamp = November 26,2007
    
    TERMINATOR
    
    FF FF, 2 bytes,
    

How to read binary data like this?

Thanks in advance.

### Update:

I tried struct module on first 6 bytes with following code:

    
    
    struct.unpack('ih', response.read(6))
    

(16777216, 1024)

But it should output (1, 4). I take a look at the manual but have no clue what
was wrong.


Answer: So here's my best shot at interpreting the data you're giving...:

    
    
    import datetime
    import struct
    
    class Printable(object):
      specials = ()
      def __str__(self):
        resultlines = []
        for pair in self.__dict__.items():
          if pair[0] in self.specials: continue
          resultlines.append('%10s %s' % pair)
        return '\n'.join(resultlines)
    
    head_fmt = '>IH6sBH'
    head_struct = struct.Struct(head_fmt)
    class Header(Printable):
      specials = ('bars',)
      def __init__(self, symbol_count, symbol_length,
                   symbol, error_code, bar_count):
        self.__dict__.update(locals())
        self.bars = []
        del self.self
    
    bar_fmt = '>5fQ'
    bar_struct = struct.Struct(bar_fmt)
    class Bar(Printable):
      specials = ('header',)
      def __init__(self, header, close, high, low,
                   open, volume, timestamp):
        self.__dict__.update(locals())
        self.header.bars.append(self)
        del self.self
        self.timestamp /= 1000.0
        self.timestamp = datetime.date.fromtimestamp(self.timestamp)
    
    def showdata(data):
      terminator = '\xff' * 2
      assert data[-2:] == terminator
      head_data = head_struct.unpack(data[:head_struct.size])
      try:
        assert head_data[4] * bar_struct.size + head_struct.size == \
               len(data) - len(terminator)
      except AssertionError:
        print 'data length is %d' % len(data)
        print 'head struct size is %d' % head_struct.size
        print 'bar struct size is %d' % bar_struct.size
        print 'number of bars is %d' % head_data[4]
        print 'head data:', head_data
        print 'terminator:', terminator
        print 'so, something is wrong, since',
        print head_data[4] * bar_struct.size + head_struct.size, '!=',
        print len(data) - len(terminator)
        raise
    
      head = Header(*head_data)
      for i in range(head.bar_count):
        bar_substr = data[head_struct.size + i * bar_struct.size:
                          head_struct.size + (i+1) * bar_struct.size]
        bar_data = bar_struct.unpack(bar_substr)
        Bar(head, *bar_data)
      assert len(head.bars) == head.bar_count
      print head
      for i, x in enumerate(head.bars):
        print 'Bar #%s' % i
        print x
    
    datas = '''
    00 00 00 01 00 04 41 4D 54 44 00 00 00 00 02 41
    97 33 33 41 99 5C 29 41 90 3D 71 41 91 D7 0A 47
    0F C6 14 00 00 01 16 6A E0 68 80 41 93 B4 05 41
    97 1E B8 41 90 7A E1 41 96 8F 57 46 E6 2E 80 00
    00 01 16 7A 53 7C 80 FF FF
    '''
    
    data = ''.join(chr(int(x, 16)) for x in datas.split())
    showdata(data)
    

this emits:

    
    
    symbol_count 1
     bar_count 2
        symbol AMTD
    error_code 0
    symbol_length 4
    Bar #0
        volume 36806.078125
     timestamp 2007-11-22
          high 19.1700000763
           low 18.0300006866
         close 18.8999996185
          open 18.2299995422
    Bar #1
        volume 29463.25
     timestamp 2007-11-25
          high 18.8899993896
           low 18.0599994659
         close 18.4629001617
          open 18.8199901581
    

...which seems to be pretty close to what you want, net of some output
formatting details. Hope this helps!-)

