pub(crate) fn iter_raw<'a: 'b, 'b>(
        &'b self,
        delete_bitset: Option<&'a DeleteBitSet>,
    ) -> impl Iterator<Item = crate::Result<OwnedBytes>> + 'b {
        let last_docid = self
            .block_checkpoints()
            .last()
            .map(|checkpoint| checkpoint.doc_range.end)
            .unwrap_or(0);
        let mut checkpoint_block_iter = self.block_checkpoints();
        let mut curr_checkpoint = checkpoint_block_iter.next();
        let mut curr_block = curr_checkpoint
            .as_ref()
            .map(|checkpoint| self.read_block(&checkpoint).map_err(|e| e.kind())); // map error in order to enable cloning
        let mut block_start_pos = 0;
        let mut num_skipped = 0;
        (0..last_docid)
            .filter_map(move |doc_id| {
                // filter_map is only used to resolve lifetime issues between the two closures on
                // the outer variables
                let alive = delete_bitset.map_or(true, |bitset| bitset.is_alive(doc_id));
                if !alive {
                    // we keep the number of skipped documents to move forward in the map block
                    num_skipped += 1;
                }
                // check move to next checkpoint
                let mut reset_block_pos = false;
                if doc_id >= curr_checkpoint.as_ref().unwrap().doc_range.end {
                    curr_checkpoint = checkpoint_block_iter.next();
                    curr_block = curr_checkpoint
                        .as_ref()
                        .map(|checkpoint| self.read_block(&checkpoint).map_err(|e| e.kind()));
                    reset_block_pos = true;
                    num_skipped = 0;
                }

                if alive {
                    let ret = Some((curr_block.clone(), num_skipped, reset_block_pos));
                    // the map block will move over the num_skipped, so we reset to 0
                    num_skipped = 0;
                    ret
                } else {
                    None
                }
            })
            .map(move |(block, num_skipped, reset_block_pos)| {
                let block = block
                    .ok_or_else(|| {
                        DataCorruption::comment_only(
                            "the current checkpoint in the doc store iterator is none, this should never happen",
                        )
                    })?
                    .map_err(|error_kind| {
                        std::io::Error::new(error_kind, "error when reading block in doc store")
                    })?;
                // this flag is set, when filter_map moved to the next block
                if reset_block_pos {
                    block_start_pos = 0;
                }
                let mut cursor = &block[block_start_pos..];
                let mut pos = 0;
                // move forward 1 doc + num_skipped in block and return length of current doc
                let doc_length = loop {
                    let doc_length = VInt::deserialize(&mut cursor)?.val() as usize;
                    let num_bytes_read = block[block_start_pos..].len() - cursor.len();
                    block_start_pos += num_bytes_read;

                    pos += 1;
                    if pos == num_skipped + 1 {
                        break doc_length;
                    } else {
                        block_start_pos += doc_length;
                        cursor = &block[block_start_pos..];
                    }
                };
                let end_pos = block_start_pos + doc_length;
                let doc_bytes = block.slice(block_start_pos..end_pos);
                block_start_pos = end_pos;
                Ok(doc_bytes)
            })
    }
/// Iterator over all RawDocuments in their order as they are stored in the doc store.
/// Use this, if you want to extract all Documents from the doc store.
/// The delete_bitset has to be forwarded from the `SegmentReader` or the results maybe wrong.
Iterator over all RawDocuments in their order as they are stored in the doc store.
Use this, if you want to extract all Documents from the doc store.
The delete_bitset has to be forwarded from the `SegmentReader` or the results maybe wrong.
Iterator over all RawDocuments in their order as they are stored in the doc store . Use this if you want to extract all Documents from the doc store . The delete_bitset has to be forwarded from the ` SegmentReader ` or the results maybe wrong .
pub(crate) fn iter_raw<'a: 'b, 'b>(
        &'b self,
        delete_bitset: Option<&'a DeleteBitSet>,
    ) -> impl Iterator<Item = crate::Result<OwnedBytes>> + 'b {
        let last_docid = self
            .block_checkpoints()
            .last()
            .map(|checkpoint| checkpoint.doc_range.end)
            .unwrap_or(0);
        let mut checkpoint_block_iter = self.block_checkpoints();
        let mut curr_checkpoint = checkpoint_block_iter.next();
        let mut curr_block = curr_checkpoint
            .as_ref()
            .map(|checkpoint| self.read_block(&checkpoint).map_err(|e| e.kind())); // map error in order to enable cloning
        let mut block_start_pos = 0;
        let mut num_skipped = 0;
        (0..last_docid)
            .filter_map(move |doc_id| {
                // filter_map is only used to resolve lifetime issues between the two closures on
                // the outer variables
                let alive = delete_bitset.map_or(true, |bitset| bitset.is_alive(doc_id));
                if !alive {
                    // we keep the number of skipped documents to move forward in the map block
                    num_skipped += 1;
                }
                // check move to next checkpoint
                let mut reset_block_pos = false;
                if doc_id >= curr_checkpoint.as_ref().unwrap().doc_range.end {
                    curr_checkpoint = checkpoint_block_iter.next();
                    curr_block = curr_checkpoint
                        .as_ref()
                        .map(|checkpoint| self.read_block(&checkpoint).map_err(|e| e.kind()));
                    reset_block_pos = true;
                    num_skipped = 0;
                }
                if alive {
                    let ret = Some((curr_block.clone(), num_skipped, reset_block_pos));
                    // the map block will move over the num_skipped, so we reset to 0
                    num_skipped = 0;
                    ret
                } else {
                    None
                }
            })
            .map(move |(block, num_skipped, reset_block_pos)| {
                let block = block
                    .ok_or_else(|| {
                        DataCorruption::comment_only(
                            "the current checkpoint in the doc store iterator is none, this should never happen",
                        )
                    })?
                    .map_err(|error_kind| {
                        std::io::Error::new(error_kind, "error when reading block in doc store")
                    })?;
                // this flag is set, when filter_map moved to the next block
                if reset_block_pos {
                    block_start_pos = 0;
                }
                let mut cursor = &block[block_start_pos..];
                let mut pos = 0;
                // move forward 1 doc + num_skipped in block and return length of current doc
                let doc_length = loop {
                    let doc_length = VInt::deserialize(&mut cursor)?.val() as usize;
                    let num_bytes_read = block[block_start_pos..].len() - cursor.len();
                    block_start_pos += num_bytes_read;
                    pos += 1;
                    if pos == num_skipped + 1 {
                        break doc_length;
                    } else {
                        block_start_pos += doc_length;
                        cursor = &block[block_start_pos..];
                    }
                };
                let end_pos = block_start_pos + doc_length;
                let doc_bytes = block.slice(block_start_pos..end_pos);
                block_start_pos = end_pos;
                Ok(doc_bytes)
            })
    }
pub ( crate ) fn iter_raw < ' a : ' b , ' b > ( & ' b self , delete_bitset : Option < & ' a DeleteBitSet > , ) -> impl Iterator < Item = crate :: Result < OwnedBytes > > + ' b { let last_docid = self . block_checkpoints ( ) . last ( ) . map ( | checkpoint | checkpoint . doc_range . end ) . unwrap_or ( 0 ) ; let mut checkpoint_block_iter = self . block_checkpoints ( ) ; let mut curr_checkpoint = checkpoint_block_iter . next ( ) ; let mut curr_block = curr_checkpoint . as_ref ( ) . map ( | checkpoint | self . read_block ( & checkpoint ) . map_err ( | e | e . kind ( ) ) ) ; // map error in order to enable cloning let mut block_start_pos = 0 ; let mut num_skipped = 0 ; ( 0 .. last_docid ) . filter_map ( move | doc_id | { // filter_map is only used to resolve lifetime issues between the two closures on // the outer variables let alive = delete_bitset . map_or ( true , | bitset | bitset . is_alive ( doc_id ) ) ; if ! alive { // we keep the number of skipped documents to move forward in the map block num_skipped += 1 ; } // check move to next checkpoint let mut reset_block_pos = false ; if doc_id >= curr_checkpoint . as_ref ( ) . unwrap ( ) . doc_range . end { curr_checkpoint = checkpoint_block_iter . next ( ) ; curr_block = curr_checkpoint . as_ref ( ) . map ( | checkpoint | self . read_block ( & checkpoint ) . map_err ( | e | e . kind ( ) ) ) ; reset_block_pos = true ; num_skipped = 0 ; } if alive { let ret = Some ( ( curr_block . clone ( ) , num_skipped , reset_block_pos ) ) ; // the map block will move over the num_skipped, so we reset to 0 num_skipped = 0 ; ret } else { None } } ) . map ( move | ( block , num_skipped , reset_block_pos ) | { let block = block . ok_or_else ( | | { DataCorruption :: comment_only ( " " , ) } ) ? . map_err ( | error_kind | { std :: io :: Error :: new ( error_kind , " " ) } ) ? ; // this flag is set, when filter_map moved to the next block if reset_block_pos { block_start_pos = 0 ; } let mut cursor = & block [ block_start_pos .. ] ; let mut pos = 0 ; // move forward 1 doc + num_skipped in block and return length of current doc let doc_length = loop { let doc_length = VInt :: deserialize ( & mut cursor ) ? . val ( ) as usize ; let num_bytes_read = block [ block_start_pos .. ] . len ( ) - cursor . len ( ) ; block_start_pos += num_bytes_read ; pos += 1 ; if pos == num_skipped + 1 { break doc_length ; } else { block_start_pos += doc_length ; cursor = & block [ block_start_pos .. ] ; } } ; let end_pos = block_start_pos + doc_length ; let doc_bytes = block . slice ( block_start_pos .. end_pos ) ; block_start_pos = end_pos ; Ok ( doc_bytes ) } ) }
Iterator over all RawDocuments in their order as they are stored in the doc store.
Iterator over all RawDocuments in their order as they are stored in the doc store .
&'b self
delete_bitset
Option<&'a DeleteBitSet>
&'b self
delete_bitset
Option<&'a DeleteBitSet>