fn bin_glyph_matcher<'a>(histogram: &[Bin], matcher: &'a dyn Matcher) -> Vec<Match<'a>> {
    let mut i: usize = 0; // index into the histogram.
    let mut res: Vec<Match<'a>> = Vec::new();

    fn _pattern_matches(pattern: &[u8], to_match: &[Bin]) -> bool {
        let min = std::cmp::min(pattern.len(), to_match.len());
        let a = pattern[0..min].iter().map(|x| *x);
        let b = to_match[0..min].iter().map(|x| x.count as u8);
        a.eq(b)
    }

    // Boolean to keep track of whether we are using stripped values or non stripped values
    // to compare.
    let mut use_stripped = true;

    while i < histogram.len() - 1 {
        // If we are using stripped symbols, remove the padding from the left, this will be very
        // fast.
        if use_stripped {
            if histogram[i].count == 0 {
                // This checks if the last entry in the current matches is a whitespace token,
                // if it is, we will add one to it, otherwise, we push a new whitespace token.
                // CONSIDER: this is less than ideal, may want to do something smart here
                // run through it once to identify whitespace jumps at the top to prepare
                // then here just do a single jump if within one of the whitespace intervals.

                // https://stackoverflow.com/a/32554326
                let mut last = res.last_mut();
                if last.is_some()
                    && std::mem::discriminant(&last.as_ref().unwrap().token)
                        == std::mem::discriminant(&Token::WhiteSpace(0))
                {
                    last.as_mut().unwrap().width += 1;
                    if let Token::WhiteSpace(ref mut z) = last.unwrap().token {
                        *z += 1;
                    }
                } else {
                    res.push(Match {
                        token: Token::WhiteSpace(1),
                        position: i as u32,
                        width: 1,
                    });
                }
                i += 1;
                continue;
            }
        }

        // CONSIDER: Splitting the histogram by labels at the start, then match on the labels.
        // Next, make sure we only match the label found in the first bin.
        // This is problematic... Since space between letters may not have the label set.
        // Disabled this for now. See CONSIDER_MATCH_LABEL.
        // let max_index = remainder.iter().position(|x| x.label != remainder[0].label);
        // let remainder = &histogram[i..i + max_index.unwrap_or(remainder.len())];

        /*
        // This was the old linear search over all glyphs.
        // It should be re-evaluated when we can order the glyphs by occurence rate
        // it may be more performant than the more random access of the matcher.

        // Get the first glyph that matches with zero cost:
        let mut index_of_min: Option<usize> = None;

        // We got a non zero entry in the first bin now.
        let remainder = &histogram[i..];

        for (zz, glyph) in set.entries.iter().enumerate() {
            let hist_to_use = if use_stripped {
                glyph.lstrip_hist()
            } else {
                glyph.hist()
            };

          let exactly_equal = _pattern_matches(hist_to_use, remainder);
          if exactly_equal {
            index_of_min = Some(zz);
            break;
          }
        }
        */

        let remainder = &histogram[i..];
        let glyph_search: Option<&glyphs::Glyph>;
        if !use_stripped {
            glyph_search = matcher.find_match(&remainder);
        } else {
            glyph_search = matcher.lstrip_find_match(&remainder);
        }

        if let Some(found_glyph) = glyph_search {
            // Calculate the true position, depends on whether we used stripped values.
            let first_non_zero = found_glyph.first_non_zero();
            let position =
                (i as u32).saturating_sub(if use_stripped { first_non_zero } else { 0 } as u32);
            // Position where histogram where this letter has the first non-zero;
            let label_position = position as usize + first_non_zero;

            // Add the newly detected glyph
            res.push(Match {
                position: position,
                token: Token::Glyph {
                    glyph: found_glyph,
                    label: histogram[label_position].label,
                },
                width: found_glyph.hist().len() as u32,
            });

            // Advance the cursor by the width of the glyph we just matched.
            i += if use_stripped {
                found_glyph
                    .lstrip_hist()
                    .expect("must have had a lstrip histogram to find it")
                    .len()
            } else {
                found_glyph.hist().len()
            };

            use_stripped = false;
        } else {
            i += 1;
            use_stripped = true; // Switch to using stripped, we didn't get a perfect match.
        }
    }
    res
}
/// This function takes a histogram made up of bins and a matcher, with this it creates a set of
/// matches in this particular histogram. These 1D matches hold tokens, either whitespace or
/// glyphs at a particular position.
This function takes a histogram made up of bins and a matcher, with this it creates a set of
matches in this particular histogram. These 1D matches hold tokens, either whitespace or
glyphs at a particular position.
This function takes a histogram made up of bins and a matcher with this it creates a set of matches in this particular histogram . These 1D matches hold tokens either whitespace or glyphs at a particular position .
fn bin_glyph_matcher<'a>(histogram: &[Bin], matcher: &'a dyn Matcher) -> Vec<Match<'a>> {
    let mut i: usize = 0; // index into the histogram.
    let mut res: Vec<Match<'a>> = Vec::new();
    fn _pattern_matches(pattern: &[u8], to_match: &[Bin]) -> bool {
        let min = std::cmp::min(pattern.len(), to_match.len());
        let a = pattern[0..min].iter().map(|x| *x);
        let b = to_match[0..min].iter().map(|x| x.count as u8);
        a.eq(b)
    }
    // Boolean to keep track of whether we are using stripped values or non stripped values
    // to compare.
    let mut use_stripped = true;
    while i < histogram.len() - 1 {
        // If we are using stripped symbols, remove the padding from the left, this will be very
        // fast.
        if use_stripped {
            if histogram[i].count == 0 {
                // This checks if the last entry in the current matches is a whitespace token,
                // if it is, we will add one to it, otherwise, we push a new whitespace token.
                // CONSIDER: this is less than ideal, may want to do something smart here
                // run through it once to identify whitespace jumps at the top to prepare
                // then here just do a single jump if within one of the whitespace intervals.
                // https://stackoverflow.com/a/32554326
                let mut last = res.last_mut();
                if last.is_some()
                    && std::mem::discriminant(&last.as_ref().unwrap().token)
                        == std::mem::discriminant(&Token::WhiteSpace(0))
                {
                    last.as_mut().unwrap().width += 1;
                    if let Token::WhiteSpace(ref mut z) = last.unwrap().token {
                        *z += 1;
                    }
                } else {
                    res.push(Match {
                        token: Token::WhiteSpace(1),
                        position: i as u32,
                        width: 1,
                    });
                }
                i += 1;
                continue;
            }
        }
        // CONSIDER: Splitting the histogram by labels at the start, then match on the labels.
        // Next, make sure we only match the label found in the first bin.
        // This is problematic... Since space between letters may not have the label set.
        // Disabled this for now. See CONSIDER_MATCH_LABEL.
        // let max_index = remainder.iter().position(|x| x.label != remainder[0].label);
        // let remainder = &histogram[i..i + max_index.unwrap_or(remainder.len())];
        /*
        // This was the old linear search over all glyphs.
        // It should be re-evaluated when we can order the glyphs by occurence rate
        // it may be more performant than the more random access of the matcher.
        // Get the first glyph that matches with zero cost:
        let mut index_of_min: Option<usize> = None;
        // We got a non zero entry in the first bin now.
        let remainder = &histogram[i..];
        for (zz, glyph) in set.entries.iter().enumerate() {
            let hist_to_use = if use_stripped {
                glyph.lstrip_hist()
            } else {
                glyph.hist()
            };
          let exactly_equal = _pattern_matches(hist_to_use, remainder);
          if exactly_equal {
            index_of_min = Some(zz);
            break;
          }
        }
        */
        let remainder = &histogram[i..];
        let glyph_search: Option<&glyphs::Glyph>;
        if !use_stripped {
            glyph_search = matcher.find_match(&remainder);
        } else {
            glyph_search = matcher.lstrip_find_match(&remainder);
        }
        if let Some(found_glyph) = glyph_search {
            // Calculate the true position, depends on whether we used stripped values.
            let first_non_zero = found_glyph.first_non_zero();
            let position =
                (i as u32).saturating_sub(if use_stripped { first_non_zero } else { 0 } as u32);
            // Position where histogram where this letter has the first non-zero;
            let label_position = position as usize + first_non_zero;
            // Add the newly detected glyph
            res.push(Match {
                position: position,
                token: Token::Glyph {
                    glyph: found_glyph,
                    label: histogram[label_position].label,
                },
                width: found_glyph.hist().len() as u32,
            });
            // Advance the cursor by the width of the glyph we just matched.
            i += if use_stripped {
                found_glyph
                    .lstrip_hist()
                    .expect("must have had a lstrip histogram to find it")
                    .len()
            } else {
                found_glyph.hist().len()
            };
            use_stripped = false;
        } else {
            i += 1;
            use_stripped = true; // Switch to using stripped, we didn't get a perfect match.
        }
    }
    res
}
fn bin_glyph_matcher < ' a > ( histogram : & [ Bin ] , matcher : & ' a dyn Matcher ) -> Vec < Match < ' a > > { let mut i : usize = 0 ; // index into the histogram. let mut res : Vec < Match < ' a > > = Vec :: new ( ) ; fn _pattern_matches ( pattern : & [ u8 ] , to_match : & [ Bin ] ) -> bool { let min = std :: cmp :: min ( pattern . len ( ) , to_match . len ( ) ) ; let a = pattern [ 0 .. min ] . iter ( ) . map ( | x | * x ) ; let b = to_match [ 0 .. min ] . iter ( ) . map ( | x | x . count as u8 ) ; a . eq ( b ) } // Boolean to keep track of whether we are using stripped values or non stripped values // to compare. let mut use_stripped = true ; while i < histogram . len ( ) - 1 { // If we are using stripped symbols, remove the padding from the left, this will be very // fast. if use_stripped { if histogram [ i ] . count == 0 { // This checks if the last entry in the current matches is a whitespace token, // if it is, we will add one to it, otherwise, we push a new whitespace token. // CONSIDER: this is less than ideal, may want to do something smart here // run through it once to identify whitespace jumps at the top to prepare // then here just do a single jump if within one of the whitespace intervals. // https://stackoverflow.com/a/32554326 let mut last = res . last_mut ( ) ; if last . is_some ( ) && std :: mem :: discriminant ( & last . as_ref ( ) . unwrap ( ) . token ) == std :: mem :: discriminant ( & Token :: WhiteSpace ( 0 ) ) { last . as_mut ( ) . unwrap ( ) . width += 1 ; if let Token :: WhiteSpace ( ref mut z ) = last . unwrap ( ) . token { * z += 1 ; } } else { res . push ( Match { token : Token :: WhiteSpace ( 1 ) , position : i as u32 , width : 1 , } ) ; } i += 1 ; continue ; } } // CONSIDER: Splitting the histogram by labels at the start, then match on the labels. // Next, make sure we only match the label found in the first bin. // This is problematic... Since space between letters may not have the label set. // Disabled this for now. See CONSIDER_MATCH_LABEL. // let max_index = remainder.iter().position(|x| x.label != remainder[0].label); // let remainder = &histogram[i..i + max_index.unwrap_or(remainder.len())]; /*
        // This was the old linear search over all glyphs.
        // It should be re-evaluated when we can order the glyphs by occurence rate
        // it may be more performant than the more random access of the matcher.

        // Get the first glyph that matches with zero cost:
        let mut index_of_min: Option<usize> = None;

        // We got a non zero entry in the first bin now.
        let remainder = &histogram[i..];

        for (zz, glyph) in set.entries.iter().enumerate() {
            let hist_to_use = if use_stripped {
                glyph.lstrip_hist()
            } else {
                glyph.hist()
            };

          let exactly_equal = _pattern_matches(hist_to_use, remainder);
          if exactly_equal {
            index_of_min = Some(zz);
            break;
          }
        }
        */ let remainder = & histogram [ i .. ] ; let glyph_search : Option < & glyphs :: Glyph > ; if ! use_stripped { glyph_search = matcher . find_match ( & remainder ) ; } else { glyph_search = matcher . lstrip_find_match ( & remainder ) ; } if let Some ( found_glyph ) = glyph_search { // Calculate the true position, depends on whether we used stripped values. let first_non_zero = found_glyph . first_non_zero ( ) ; let position = ( i as u32 ) . saturating_sub ( if use_stripped { first_non_zero } else { 0 } as u32 ) ; // Position where histogram where this letter has the first non-zero; let label_position = position as usize + first_non_zero ; // Add the newly detected glyph res . push ( Match { position : position , token : Token :: Glyph { glyph : found_glyph , label : histogram [ label_position ] . label , } , width : found_glyph . hist ( ) . len ( ) as u32 , } ) ; // Advance the cursor by the width of the glyph we just matched. i += if use_stripped { found_glyph . lstrip_hist ( ) . expect ( " " ) . len ( ) } else { found_glyph . hist ( ) . len ( ) } ; use_stripped = false ; } else { i += 1 ; use_stripped = true ; // Switch to using stripped, we didn't get a perfect match. } } res }
This function takes a histogram made up of bins and a matcher, with this it creates a set of
matches in this particular histogram.
This function takes a histogram made up of bins and a matcher with this it creates a set of matches in this particular histogram .
histogram
&[Bin]
matcher
&'a dyn Matcher