pub fn from_memmap_files(
        data_dim: usize,
        labels_scheme: LabelScheme,
        data_path: &[PathBuf],
        labels_path: &[PathBuf],
        ram: bool,
    ) -> PointCloudResult<PointCloud<M>> {
        if data_path.len() != labels_path.len() {
            panic!(
                "Mismatch of label and data paths Data: {:?}, Labels: {:?}",
                data_path, labels_path
            );
        }
        let mut addresses = IndexMap::new();
        let mut names_to_indexes: IndexMap<PointName, PointIndex> = IndexMap::new();
        let mut indexes_to_names: IndexMap<PointIndex, PointName> = IndexMap::new();
        let mut current_count: u64 = 0;
        let mut data_sources = Vec::new();
        let mut label_sources = Vec::new();
        for (i,(dp,lp)) in data_path.iter().zip(labels_path).enumerate() {
            let new_data: Box<dyn DataSource>;
            if ram {
                new_data = Box::new((DataMemmap::new(data_dim, &dp)?).convert_to_ram());
            } else {
                new_data = Box::new(DataMemmap::new(data_dim, &dp)?);
            }
            let new_labels = labels_scheme.open(&lp)?;
            if new_data.len() != new_labels.len() {
                panic!("The data count {:?} differs from the label count {:?} for the {}th data and label files", new_data.len(), new_labels.len(), i);
            }
            for j in 0..new_data.len() {
                let x = (i, j);
                let name = new_labels
                    .get_name(j)
                    .unwrap_or_else(|| format!("{}", current_count));

                if names_to_indexes.contains_key(&name) {
                    println!(
                        "Duplicate {:?} on line {} of file {:?}",
                        &name, j, labels_path[i]
                    );
                } else {
                    names_to_indexes.insert(name.clone(), current_count);
                    indexes_to_names.insert(current_count, name.clone());
                    addresses.insert(current_count, x);
                }
                current_count += 1;
            }
            data_sources.push(new_data);
            label_sources.push(new_labels);
        }

        // This could possibly be improved to be architecture specific. It depends on the CPU cache size
        let chunk = min(15000/data_dim,20);
        Ok(PointCloud {
            data_sources: data_sources,
            label_sources: label_sources,
            names_to_indexes: names_to_indexes,
            indexes_to_names: indexes_to_names,
            addresses: addresses,
            data_dim,
            labels_scheme,
            loaded_centers: Mutex::new(IndexMap::new()),
            chunk,
            metric: PhantomData,
        })
    }
/// Builds the point cloud from a collection of memmaps. If you want it to load all the data
/// into ram, pass a true for the ram.
Builds the point cloud from a collection of memmaps. If you want it to load all the data
into ram, pass a true for the ram.
Builds the point cloud from a collection of memmaps . If you want it to load all the data into ram pass a true for the ram .
pub fn from_memmap_files(
        data_dim: usize,
        labels_scheme: LabelScheme,
        data_path: &[PathBuf],
        labels_path: &[PathBuf],
        ram: bool,
    ) -> PointCloudResult<PointCloud<M>> {
        if data_path.len() != labels_path.len() {
            panic!(
                "Mismatch of label and data paths Data: {:?}, Labels: {:?}",
                data_path, labels_path
            );
        }
        let mut addresses = IndexMap::new();
        let mut names_to_indexes: IndexMap<PointName, PointIndex> = IndexMap::new();
        let mut indexes_to_names: IndexMap<PointIndex, PointName> = IndexMap::new();
        let mut current_count: u64 = 0;
        let mut data_sources = Vec::new();
        let mut label_sources = Vec::new();
        for (i,(dp,lp)) in data_path.iter().zip(labels_path).enumerate() {
            let new_data: Box<dyn DataSource>;
            if ram {
                new_data = Box::new((DataMemmap::new(data_dim, &dp)?).convert_to_ram());
            } else {
                new_data = Box::new(DataMemmap::new(data_dim, &dp)?);
            }
            let new_labels = labels_scheme.open(&lp)?;
            if new_data.len() != new_labels.len() {
                panic!("The data count {:?} differs from the label count {:?} for the {}th data and label files", new_data.len(), new_labels.len(), i);
            }
            for j in 0..new_data.len() {
                let x = (i, j);
                let name = new_labels
                    .get_name(j)
                    .unwrap_or_else(|| format!("{}", current_count));
                if names_to_indexes.contains_key(&name) {
                    println!(
                        "Duplicate {:?} on line {} of file {:?}",
                        &name, j, labels_path[i]
                    );
                } else {
                    names_to_indexes.insert(name.clone(), current_count);
                    indexes_to_names.insert(current_count, name.clone());
                    addresses.insert(current_count, x);
                }
                current_count += 1;
            }
            data_sources.push(new_data);
            label_sources.push(new_labels);
        }
        // This could possibly be improved to be architecture specific. It depends on the CPU cache size
        let chunk = min(15000/data_dim,20);
        Ok(PointCloud {
            data_sources: data_sources,
            label_sources: label_sources,
            names_to_indexes: names_to_indexes,
            indexes_to_names: indexes_to_names,
            addresses: addresses,
            data_dim,
            labels_scheme,
            loaded_centers: Mutex::new(IndexMap::new()),
            chunk,
            metric: PhantomData,
        })
    }
pub fn from_memmap_files ( data_dim : usize , labels_scheme : LabelScheme , data_path : & [ PathBuf ] , labels_path : & [ PathBuf ] , ram : bool , ) -> PointCloudResult < PointCloud < M > > { if data_path . len ( ) != labels_path . len ( ) { panic ! ( " " data_path labels_path ) ; } let mut addresses = IndexMap :: new ( ) ; let mut names_to_indexes : IndexMap < PointName , PointIndex > = IndexMap :: new ( ) ; let mut indexes_to_names : IndexMap < PointIndex , PointName > = IndexMap :: new ( ) ; let mut current_count : u64 = 0 ; let mut data_sources = Vec :: new ( ) ; let mut label_sources = Vec :: new ( ) ; for ( i , ( dp , lp ) ) in data_path . iter ( ) . zip ( labels_path ) . enumerate ( ) { let new_data : Box < dyn DataSource > ; if ram { new_data = Box :: new ( ( DataMemmap :: new ( data_dim , & dp ) ? ) . convert_to_ram ( ) ) ; } else { new_data = Box :: new ( DataMemmap :: new ( data_dim , & dp ) ? ) ; } let new_labels = labels_scheme . open ( & lp ) ? ; if new_data . len ( ) != new_labels . len ( ) { panic ! ( " " new_data len ( ) new_labels len ( ) i ) ; } for j in 0 .. new_data . len ( ) { let x = ( i , j ) ; let name = new_labels . get_name ( j ) . unwrap_or_else ( | | format ! ( " " current_count ) ) ; if names_to_indexes . contains_key ( & name ) { println ! ( " " name j labels_path [ i ] ) ; } else { names_to_indexes . insert ( name . clone ( ) , current_count ) ; indexes_to_names . insert ( current_count , name . clone ( ) ) ; addresses . insert ( current_count , x ) ; } current_count += 1 ; } data_sources . push ( new_data ) ; label_sources . push ( new_labels ) ; } // This could possibly be improved to be architecture specific. It depends on the CPU cache size let chunk = min ( 15000 / data_dim , 20 ) ; Ok ( PointCloud { data_sources : data_sources , label_sources : label_sources , names_to_indexes : names_to_indexes , indexes_to_names : indexes_to_names , addresses : addresses , data_dim , labels_scheme , loaded_centers : Mutex :: new ( IndexMap :: new ( ) ) , chunk , metric : PhantomData , } ) }
Builds the point cloud from a collection of memmaps.
Builds the point cloud from a collection of memmaps .
data_dim
usize
labels_scheme
LabelScheme
data_path
&[PathBuf]
labels_path
&[PathBuf]
ram
bool
data_dim
usize
labels_scheme
LabelScheme
data_path
&[PathBuf]
labels_path
&[PathBuf]
ram
bool