swift/proxy/controllers/obj.py
def chunk_transformer(policy):
    """
    A generator to transform a source chunk to erasure coded chunks for each
    `send` call. The number of erasure coded chunks is as
    policy.ec_n_unique_fragments.
    """
    segment_size = policy.ec_segment_size

    buf = collections.deque()  # deque()定义了双端队列，可以从头/尾两端添加或删除元素
    total_buf_len = 0

    # 第一次调用时，在这里暂停
    # 下次调用 .send(msg) 时，chunk=msg，然后往后运行
    chunk = yield
    while chunk:
        buf.append(chunk)   #头部插入
        total_buf_len += len(chunk)
        if total_buf_len >= segment_size:
            chunks_to_encode = []
            # extract as many chunks as we can from the input buffer
            while total_buf_len >= segment_size:
                to_take = segment_size
                pieces = []
                while to_take > 0:
                    piece = buf.popleft()  #尾部取出
                    if len(piece) > to_take:
                        # 只取一个 segment_size 大小的数据，把多余的从尾部插入
                        buf.appendleft(piece[to_take:])
                        # piece 是从 buf 尾部取出的 segment_size 大小的数据
                        piece = piece[:to_take]
                    pieces.append(piece)
                    to_take -= len(piece)
                    total_buf_len -= len(piece)
                # str.join(sequence)将序列中的元素以指定的字符连接生成一个新的字符串
                # pieces 中所有数据的总和是一个 segment_size 大小
                chunks_to_encode.append(''.join(pieces))

            # .encode() 的返回值是一个列表，对一个 chunk 编码会生成多个块(比如3+1的编码会生成4个块)
            # .encode(): Encode an arbitrary-sized string
            #            param data_bytes: the buffer to encode
            #            returns: a list of buffers (first k entries are data and the last m are parity)
            # frags_by_byte_order 最终保存的是一个列表的列表
            frags_by_byte_order = []
            for chunk_to_encode in chunks_to_encode:
                frags_by_byte_order.append(
                    policy.pyeclib_driver.encode(chunk_to_encode))
            # Sequential calls to encode() have given us a list that
            # looks like this:
            #
            # [[frag_A1, frag_B1, frag_C1, ...],
            #  [frag_A2, frag_B2, frag_C2, ...], ...]
            #
            # What we need is a list like this:
            #
            # [(frag_A1 + frag_A2 + ...),  # destined for node A
            #  (frag_B1 + frag_B2 + ...),  # destined for node B
            #  (frag_C1 + frag_C2 + ...),  # destined for node C
            #  ...]
            # obj_data 是一个列表，列表的第一个元素是 A1,A2,.. 拼接而成的，
            #                          第二个元素是 B1,B2,...拼接而成的，...
            # obj_data 中的每一个元素都对应存到一个 node 上
            obj_data = [''.join(frags)
                        for frags in zip(*frags_by_byte_order)]
            # 迭代到这里时，返回 obj_data
            # 下次调用 .send(msg) 时，chunk = msg,然后继续往下运行
            chunk = yield obj_data
        else:
            # didn't have enough data to encode
            chunk = yield None

    # Now we've gotten an empty chunk, which indicates end-of-input.
    # Take any leftover bytes and encode them.
    # 对最后不足一个 segment_size 大小的数据进行编码
    last_bytes = ''.join(buf)
    if last_bytes:
        last_frags = policy.pyeclib_driver.encode(last_bytes)
        yield last_frags
    else:
        # 如果数据大小刚好是 segment_size 的整数倍，则返回 ['','','','',..] 表示结束
        # policy.ec_n_unique_fragments = data fragments + parity fragments
        yield [''] * policy.ec_n_unique_fragments

    A generator to transform a source chunk to erasure coded chunks for each
    `send` call. The number of erasure coded chunks is as
    policy.ec_n_unique_fragments.
    
A generator to transform a source chunk to erasure coded chunks for each
`send` call. The number of erasure coded chunks is as
policy.ec_n_unique_fragments.
A generator to transform a source chunk to erasure coded chunks for each ` send ` call . The number of erasure coded chunks is as policy . ec_n_unique_fragments .
def chunk_transformer(policy):
    segment_size = policy.ec_segment_size
    buf = collections.deque()  
    total_buf_len = 0
    chunk = yield
    while chunk:
        buf.append(chunk)   
        total_buf_len += len(chunk)
        if total_buf_len >= segment_size:
            chunks_to_encode = []
            while total_buf_len >= segment_size:
                to_take = segment_size
                pieces = []
                while to_take > 0:
                    piece = buf.popleft()  
                    if len(piece) > to_take:
                        buf.appendleft(piece[to_take:])
                        piece = piece[:to_take]
                    pieces.append(piece)
                    to_take -= len(piece)
                    total_buf_len -= len(piece)
                chunks_to_encode.append(''.join(pieces))
            frags_by_byte_order = []
            for chunk_to_encode in chunks_to_encode:
                frags_by_byte_order.append(
                    policy.pyeclib_driver.encode(chunk_to_encode))
             [[frag_A1, frag_B1, frag_C1, ...],
              [frag_A2, frag_B2, frag_C2, ...], ...]
             What we need is a list like this:
             [(frag_A1 + frag_A2 + ...),   destined for node A
              (frag_B1 + frag_B2 + ...),   destined for node B
              (frag_C1 + frag_C2 + ...),   destined for node C
              ...]
             obj_data 是一个列表，列表的第一个元素是 A1,A2,.. 拼接而成的，
                                      第二个元素是 B1,B2,...拼接而成的，...
             obj_data 中的每一个元素都对应存到一个 node 上
            obj_data = [''.join(frags)
                        for frags in zip(*frags_by_byte_order)]
             迭代到这里时，返回 obj_data
             下次调用 .send(msg) 时，chunk = msg,然后继续往下运行
            chunk = yield obj_data
        else:
             didn't have enough data to encode
            chunk = yield None
     Now we've gotten an empty chunk, which indicates end-of-input.
     Take any leftover bytes and encode them.
     对最后不足一个 segment_size 大小的数据进行编码
    last_bytes = ''.join(buf)
    if last_bytes:
        last_frags = policy.pyeclib_driver.encode(last_bytes)
        yield last_frags
    else:
         如果数据大小刚好是 segment_size 的整数倍，则返回 ['','','','',..] 表示结束
         policy.ec_n_unique_fragments = data fragments + parity fragments
        yield [''] * policy.ec_n_unique_fragments
def chunk_transformer ( policy ) : segment_size = policy . ec_segment_size buf = collections . deque ( ) total_buf_len = 0 chunk = yield while chunk : buf . append ( chunk ) total_buf_len += len ( chunk ) if total_buf_len >= segment_size : chunks_to_encode = [ ] while total_buf_len >= segment_size : to_take = segment_size pieces = [ ] while to_take > 0 : piece = buf . popleft ( ) if len ( piece ) > to_take : buf . appendleft ( piece [ to_take : ] ) piece = piece [ : to_take ] pieces . append ( piece ) to_take -= len ( piece ) total_buf_len -= len ( piece ) chunks_to_encode . append ( '' . join ( pieces ) ) frags_by_byte_order = [ ] for chunk_to_encode in chunks_to_encode : frags_by_byte_order . append ( policy . pyeclib_driver . encode ( chunk_to_encode ) ) obj_data = [ '' . join ( frags ) for frags in zip ( * frags_by_byte_order ) ] chunk = yield obj_data else : chunk = yield None last_bytes = '' . join ( buf ) if last_bytes : last_frags = policy . pyeclib_driver . encode ( last_bytes ) yield last_frags else : yield [ '' ] * policy . ec_n_unique_fragments
A generator to transform a source chunk to erasure coded chunks for each
`send` call.
A generator to transform a source chunk to erasure coded chunks for each ` send ` call .
"""
    A generator to transform a source chunk to erasure coded chunks for each
    `send` call. The number of erasure coded chunks is as
    policy.ec_n_unique_fragments.
    """ # deque()定义了双端队列，可以从头/尾两端添加或删除元素 # 第一次调用时，在这里暂停 # 下次调用 .send(msg) 时，chunk=msg，然后往后运行 #头部插入 # extract as many chunks as we can from the input buffer #尾部取出 # 只取一个 segment_size 大小的数据，把多余的从尾部插入 # piece 是从 buf 尾部取出的 segment_size 大小的数据 # str.join(sequence)将序列中的元素以指定的字符连接生成一个新的字符串 # pieces 中所有数据的总和是一个 segment_size 大小 # .encode() 的返回值是一个列表，对一个 chunk 编码会生成多个块(比如3+1的编码会生成4个块) # .encode(): Encode an arbitrary-sized string #            param data_bytes: the buffer to encode #            returns: a list of buffers (first k entries are data and the last m are parity) # frags_by_byte_order 最终保存的是一个列表的列表 # Sequential calls to encode() have given us a list that # looks like this: # # [[frag_A1, frag_B1, frag_C1, ...], #  [frag_A2, frag_B2, frag_C2, ...], ...] # # What we need is a list like this: # # [(frag_A1 + frag_A2 + ...),  # destined for node A #  (frag_B1 + frag_B2 + ...),  # destined for node B #  (frag_C1 + frag_C2 + ...),  # destined for node C #  ...] # obj_data 是一个列表，列表的第一个元素是 A1,A2,.. 拼接而成的， #                          第二个元素是 B1,B2,...拼接而成的，... # obj_data 中的每一个元素都对应存到一个 node 上 # 迭代到这里时，返回 obj_data # 下次调用 .send(msg) 时，chunk = msg,然后继续往下运行 # didn't have enough data to encode # Now we've gotten an empty chunk, which indicates end-of-input. # Take any leftover bytes and encode them. # 对最后不足一个 segment_size 大小的数据进行编码 # 如果数据大小刚好是 segment_size 的整数倍，则返回 ['','','','',..] 表示结束 # policy.ec_n_unique_fragments = data fragments + parity fragments
policy