fn index_log(log_id:&str, conn:&mut PooledConn, errs:&mut Vec<(u64, String)>, dryrun:bool) {
    let query = format!("SELECT `id`, `content` FROM `voxes` WHERE `log_id` = \"{log_id}\"");
    let voxes = conn.query_map(query,
        |(new_id, new_content)| {
            VoxEntry { 
                id: new_id,
                author: String::new(),
                log_id: String::new(),
                date: String::new(),
                content:new_content,
            }
        },).unwrap();
    if voxes.len() == 0 && dryrun {
        println_dry_run_log(format!("No entry in DB found for {log_id}, can not index"), true);
    }

    let mut vox_index_data : Vec<VoxIndexData> = Vec::new();
    // Get the db access sorted before we start looping
    let opts = Opts::from_url(&get_db_path()).unwrap();
    let pool = Pool::new(opts).unwrap();
    let mut conn = pool.get_conn().unwrap();
    for vox in voxes {
        //println!("-- VoxEntry -- {}", vox.content.to_string());
        let has_song :bool = vox.content.contains("^s");
        let has_morshu :bool = vox.content.contains("^m") | vox.content.contains("^morshu");
        let has_grant :bool = vox.content.contains("^g") | vox.content.contains("^grant") | vox.content.contains("^dk");
        // ^v ix ignored

        // Perform filtering
        let cleaned_vox = filters::cleanup(
                            filters::pad_short_words(
                            filters::remap_note_shorthand(
                            filters::contractions(
                            filters::control_codes(
                            filters::pitch(
                            filters::pause(
                            filters::trunc( 
                            filters::commands(
                            filters::sanatize( vox.content.to_lowercase() ))))))))));
        let content_arr : Vec<&str> = cleaned_vox.split(' ').collect();
        let mut indexed_content = String::new();
        let mut used_words = HashSet::new();
        for word in content_arr {
            let trimmed = word.trim();

            if trimmed.len() > 0 && !used_words.contains(trimmed) {
                if validators::valid(&trimmed) {
                    used_words.insert(trimmed);
                    indexed_content.push_str(&(format!("{trimmed} ")));
                }
                else {
                    errs.push((vox.id, trimmed.to_string()));
                    if dryrun {
                        println_dry_run_log(format!("-- Vox entry [{}] has word [{}] that is not in the vocab.  Dropping...", vox.id, trimmed), true);
                    }
                    else {
                        println!("-- Vox entry [{}] has word [{}] that is not in the vocab.  Dropping...", vox.id, trimmed);
                    }
                }
            }
        }

        vox_index_data.push(VoxIndexData { 
            id: vox.id,
            indexed_content,
            has_song,
            has_morshu,
            has_grant,
        });
    }

    println!("Index data for [{log_id}] compiled, sending to server...");
    if dryrun {
        for vox_index_entry in vox_index_data {
            println_dry_run_log(vox_index_entry.to_string(), false);
        }
    }
    else {
        conn.exec_batch(
        r"REPLACE INTO vox_meta (id, indexed_content, has_song, has_morshu, has_grant)
        VALUES (:author, :indexed_content, :has_song, :has_morshu, :has_grant)",
        vox_index_data.iter().map(|p| params!{
            "author" => p.id,
            "indexed_content" => p.indexed_content.clone(),
            "has_song" => p.has_song,
            "has_morshu" => p.has_morshu,
            "has_grant" => p.has_grant,
         })).unwrap();
    }
}
// This builds the indexed data off of the main data from the DB.
This builds the indexed data off of the main data from the DB.
This builds the indexed data off of the main data from the DB .
fn index_log(log_id:&str, conn:&mut PooledConn, errs:&mut Vec<(u64, String)>, dryrun:bool) {
    let query = format!("SELECT `id`, `content` FROM `voxes` WHERE `log_id` = \"{log_id}\"");
    let voxes = conn.query_map(query,
        |(new_id, new_content)| {
            VoxEntry { 
                id: new_id,
                author: String::new(),
                log_id: String::new(),
                date: String::new(),
                content:new_content,
            }
        },).unwrap();
    if voxes.len() == 0 && dryrun {
        println_dry_run_log(format!("No entry in DB found for {log_id}, can not index"), true);
    }
    let mut vox_index_data : Vec<VoxIndexData> = Vec::new();
    // Get the db access sorted before we start looping
    let opts = Opts::from_url(&get_db_path()).unwrap();
    let pool = Pool::new(opts).unwrap();
    let mut conn = pool.get_conn().unwrap();
    for vox in voxes {
        //println!("-- VoxEntry -- {}", vox.content.to_string());
        let has_song :bool = vox.content.contains("^s");
        let has_morshu :bool = vox.content.contains("^m") | vox.content.contains("^morshu");
        let has_grant :bool = vox.content.contains("^g") | vox.content.contains("^grant") | vox.content.contains("^dk");
        // ^v ix ignored
        // Perform filtering
        let cleaned_vox = filters::cleanup(
                            filters::pad_short_words(
                            filters::remap_note_shorthand(
                            filters::contractions(
                            filters::control_codes(
                            filters::pitch(
                            filters::pause(
                            filters::trunc( 
                            filters::commands(
                            filters::sanatize( vox.content.to_lowercase() ))))))))));
        let content_arr : Vec<&str> = cleaned_vox.split(' ').collect();
        let mut indexed_content = String::new();
        let mut used_words = HashSet::new();
        for word in content_arr {
            let trimmed = word.trim();
            if trimmed.len() > 0 && !used_words.contains(trimmed) {
                if validators::valid(&trimmed) {
                    used_words.insert(trimmed);
                    indexed_content.push_str(&(format!("{trimmed} ")));
                }
                else {
                    errs.push((vox.id, trimmed.to_string()));
                    if dryrun {
                        println_dry_run_log(format!("-- Vox entry [{}] has word [{}] that is not in the vocab.  Dropping...", vox.id, trimmed), true);
                    }
                    else {
                        println!("-- Vox entry [{}] has word [{}] that is not in the vocab.  Dropping...", vox.id, trimmed);
                    }
                }
            }
        }
        vox_index_data.push(VoxIndexData { 
            id: vox.id,
            indexed_content,
            has_song,
            has_morshu,
            has_grant,
        });
    }
    println!("Index data for [{log_id}] compiled, sending to server...");
    if dryrun {
        for vox_index_entry in vox_index_data {
            println_dry_run_log(vox_index_entry.to_string(), false);
        }
    }
    else {
        conn.exec_batch(
        r"REPLACE INTO vox_meta (id, indexed_content, has_song, has_morshu, has_grant)
        VALUES (:author, :indexed_content, :has_song, :has_morshu, :has_grant)",
        vox_index_data.iter().map(|p| params!{
            "author" => p.id,
            "indexed_content" => p.indexed_content.clone(),
            "has_song" => p.has_song,
            "has_morshu" => p.has_morshu,
            "has_grant" => p.has_grant,
         })).unwrap();
    }
}
fn index_log ( log_id : & str , conn : & mut PooledConn , errs : & mut Vec < ( u64 , String ) > , dryrun : bool ) { let query = format ! ( " \" \" " ) ; let voxes = conn . query_map ( query , | ( new_id , new_content ) | { VoxEntry { id : new_id , author : String :: new ( ) , log_id : String :: new ( ) , date : String :: new ( ) , content : new_content , } } , ) . unwrap ( ) ; if voxes . len ( ) == 0 && dryrun { println_dry_run_log ( format ! ( " " ) , true ) ; } let mut vox_index_data : Vec < VoxIndexData > = Vec :: new ( ) ; // Get the db access sorted before we start looping let opts = Opts :: from_url ( & get_db_path ( ) ) . unwrap ( ) ; let pool = Pool :: new ( opts ) . unwrap ( ) ; let mut conn = pool . get_conn ( ) . unwrap ( ) ; for vox in voxes { //println!("-- VoxEntry -- {}", vox.content.to_string()); let has_song : bool = vox . content . contains ( " " ) ; let has_morshu : bool = vox . content . contains ( " " ) | vox . content . contains ( " " ) ; let has_grant : bool = vox . content . contains ( " " ) | vox . content . contains ( " " ) | vox . content . contains ( " " ) ; // ^v ix ignored // Perform filtering let cleaned_vox = filters :: cleanup ( filters :: pad_short_words ( filters :: remap_note_shorthand ( filters :: contractions ( filters :: control_codes ( filters :: pitch ( filters :: pause ( filters :: trunc ( filters :: commands ( filters :: sanatize ( vox . content . to_lowercase ( ) ) ) ) ) ) ) ) ) ) ) ; let content_arr : Vec < & str > = cleaned_vox . split ( ' ' ) . collect ( ) ; let mut indexed_content = String :: new ( ) ; let mut used_words = HashSet :: new ( ) ; for word in content_arr { let trimmed = word . trim ( ) ; if trimmed . len ( ) > 0 && ! used_words . contains ( trimmed ) { if validators :: valid ( & trimmed ) { used_words . insert ( trimmed ) ; indexed_content . push_str ( & ( format ! ( " " ) ) ) ; } else { errs . push ( ( vox . id , trimmed . to_string ( ) ) ) ; if dryrun { println_dry_run_log ( format ! ( " " vox id trimmed ) , true ) ; } else { println ! ( " " vox id trimmed ) ; } } } } vox_index_data . push ( VoxIndexData { id : vox . id , indexed_content , has_song , has_morshu , has_grant , } ) ; } println ! ( " " ) ; if dryrun { for vox_index_entry in vox_index_data { println_dry_run_log ( vox_index_entry . to_string ( ) , false ) ; } } else { conn . exec_batch ( r"REPLACE INTO vox_meta (id, indexed_content, has_song, has_morshu, has_grant)
        VALUES (:author, :indexed_content, :has_song, :has_morshu, :has_grant)" , vox_index_data . iter ( ) . map ( | p | params ! { " " p id " " p indexed_content clone ( ) " " p has_song " " p has_morshu " " p has_grant } ) ) . unwrap ( ) ; } }
This builds the indexed data off of the main data from the DB.
This builds the indexed data off of the main data from the DB .
log_id
&str
conn
&mut PooledConn
errs
&mut Vec<(u64, String)>
dryrun
bool