<NME> HashStore.pm
<BEF> =head1 NAME

Bio::JBrowse::HashStore - on-disk 2-level hash table

=head1 DESCRIPTION

Makes an on-disk hash table, designed to be easily read as static files over HTTP.

Makes a set of JSON-encoded files at paths based on a hash of the key.
For example:

  path/to/dir/29c/c14/fc.json
  path/to/dir/23f/5ad/11.json
  path/to/dir/711/7c1/29.json
  path/to/dir/5ec/b24/6a.json
  path/to/dir/4de/9ac/c6.json
  path/to/dir/41b/c43/27.json
  path/to/dir/28c/d86/e9.json

Where each file contains a JSON object containing data items like:

  { foo: "bar", ... }

Where "foo" is the original key, and "bar" is the JSON-encoded data.

=cut

package Bio::JBrowse::HashStore;
use strict;
use warnings;

use Carp;

use Storable ();
use JSON 2;

use File::Next ();
use File::Path ();
use File::Spec ();

use Digest::Crc32 ();
use DB_File ();
use IO::File ();
use POSIX ();

my $bucket_class = 'Bio::JBrowse::HashStore::Bucket';


=head2 open( dir => "/path/to/dir", hash_bits => 16, mem => 256 * 2**20 )

=cut

sub open {
    my $class = shift;

    # source of data: defaults, overridden by open args, overridden by meta.json contents
    my $self = bless {
        max_filehandles => 1000,
        mem => 256*2**20,
        @_
    }, $class;

    $self->{dir} or croak "dir option required";

    $self->empty if $self->{empty};

    $self->{meta} = $self->_read_meta;

    $self->{crc} = Digest::Crc32->new;

    # compress, format, and hash_bits all use the value in the
    # meta.json if present, or the value passed in by the constructor,
    # or the default, in order of priority
    my %defaults = ( compress => 0, format => 'json', hash_bits => 16 );
    for (qw( compress format hash_bits )) {
        $self->{$_} = $self->{meta}{$_} = (
            defined $self->{meta}{$_}  ?  $self->{meta}{$_}  :
            defined $self->{$_}        ?  $self->{$_}        :
                                          $defaults{$_}
        );
    }

    # check that hash_bits is a multiple of 4
    if( $self->{hash_bits} % 4 ) {
        die "Invalid hash bits value $self->{hash_bits}, must be a multiple of 4.\n";
    }

    $self->{hash_mask} = 2**($self->{hash_bits}) - 1;
    $self->{hash_sprintf_pattern} = '%0'.int( $self->{hash_bits}/4 ).'x';
    $self->{file_extension} = '.'.$self->{format};

    $self->{cache_size} = int( $self->{mem} / 50000 / 2 );
    print "Hash store cache size: $self->{cache_size} buckets\n" if $self->{verbose};

    File::Path::mkpath( $self->{dir} );

    return $self;
}

sub _make_cache {
    my ( $self, @args ) = @_;
    return Bio::JBrowse::HashStore::FIFOCache->new( @args );
}

# write out meta.json file when the store itself is destroyed
sub DESTROY {
    my ( $self ) = @_;
    File::Path::mkpath( $self->{dir} );
    {
        my $meta_path = $self->_meta_path;
        CORE::open my $out, '>', $meta_path or die "$! writing $meta_path";
        $out->print( JSON::to_json( $self->{meta} ) )
            or die "$! writing $meta_path";
    }

    # free everything to flush buckets
    %$self = ();
}
sub _meta_path {
    File::Spec->catfile( shift->{dir}, 'meta.json' );
}
sub _read_meta {
    my ( $self ) = @_;
    my $meta_path = $self->_meta_path;
    return {} unless -r $meta_path;
    CORE::open my $meta, '<', $meta_path or die "$! reading $meta_path";
    local $/;
    my $d = eval { JSON->new->relaxed->decode( scalar <$meta> ) } || {};
    warn $@ if $@;
    $d->{compress} = 0 unless defined $d->{compress};
    return $d;
}

=head2 meta

return a hashref of metadata about this hash store

=cut

sub meta {
    ( shift->{meta} ||= {} )
}

=head2 get( $key )

=cut

sub get {
    my ( $self, $key ) = @_;

    my $bucket = $self->_getBucket( $key );
    return $bucket->{data}{$key};
}

=head2 stream_do( $arg_stream, $operation_callback )

=cut

sub stream_do {
    my ( $self, $op_stream, $do_operation, $estimated_op_count ) = @_;

    # clean up any stale log files
    { my $log_iterator = $self->_file_iterator( sub { /\.log$/ } );
      while( my $stale_logfile = $log_iterator->() ) {
          unlink $stale_logfile;
      }
    }

    # make log files for each bucket, log the operations that happen
    # on that bucket, but don't actually do them yet
    my $ops_written = 0;
    my $gzip = $self->{compress} ? ':gzip' : '';
    {
        my $hash_chars = $self->{hash_bits}/4;
        my $sort_log_chars = $hash_chars - int( log($self->{cache_size} )/log(16) );
        my $max_sort_log_chars = int( log( $self->{max_filehandles} )/log(16) );
        $sort_log_chars = 1 unless $sort_log_chars > 1;
        $sort_log_chars = $max_sort_log_chars unless $sort_log_chars <= $max_sort_log_chars;

        $hash_chars -= $sort_log_chars;
        my $zeroes = "0"x$hash_chars;

        print "Using $sort_log_chars chars for sort log names (".(16**$sort_log_chars)." sort logs)\n" if $self->{verbose};
        my $filehandle_cache = $self->_make_cache( size => $self->{max_filehandles} );
        my $progressbar = $estimated_op_count && $self->_make_progressbar( 'Sorting operations', $estimated_op_count );
        my $progressbar_next_update = 0;
        while ( my $op = $op_stream->() ) {
            my $hex = $self->_hex( $self->_hash( $op->[0] ) );

            substr( (my $log_hex = $hex), 0, $hash_chars, $zeroes );

            my $log_handle = $filehandle_cache->compute( $log_hex, sub {
                my ( $h ) = @_;
                my $pathinfo = $self->_hexToPath( $h );
                File::Path::mkpath( $pathinfo->{workdir} ) unless -d $pathinfo->{workdir};
                #warn "writing $pathinfo->{fullpath}.log\n";
                CORE::open( my $f, ">>$gzip", "$pathinfo->{workpath}.log" )
                    or die "$! opening bucket log $pathinfo->{workpath}.log";
                return $f;
            });

            Storable::store_fd( [$hex,$op], $log_handle );

            $ops_written++;
            if ( $progressbar && $ops_written >= $progressbar_next_update && $ops_written < $estimated_op_count ) {
                $progressbar_next_update = $progressbar->update( $ops_written );
            }
        }
        if ( $progressbar && $ops_written > $progressbar_next_update ) {
            $progressbar->update( $estimated_op_count );
        }
    }

    # play back the operations, feeding the $do_operation sub with the
    # bucket and the operation to be done
    {
        my $progressbar = $ops_written && $self->_make_progressbar( 'Executing operations', $ops_written );
        my $progressbar_next_update = 0;
        my $ops_played_back = 0;
        my $log_iterator = $self->_file_iterator( sub { /\.log$/ } );
        while ( my $log_path = $log_iterator->() ) {
            CORE::open( my $log_fh, "<$gzip", $log_path ) or die "$! reading $log_path";
            #warn "reading $log_path\n";
            while ( my $rec = eval { Storable::fd_retrieve( $log_fh ) } ) {
                my ( $hex, $op ) = @$rec;
                my $bucket = $self->_getBucketFromHex( $hex );
                $bucket->{data}{$op->[0]} = $do_operation->( $op, $bucket->{data}{$op->[0]} );
                $bucket->{dirty} = 1;

                if ( $progressbar && ++$ops_played_back > $progressbar_next_update ) {
                    $progressbar_next_update = $progressbar->update( $ops_played_back );
                }
            }
            unlink $log_path;
        }

        if ( $progressbar && $ops_played_back > $progressbar_next_update ) {
            $progressbar->update( $ops_written );
        }
    }
}

sub _file_iterator {
    my ( $self, $filter ) = @_;
    return File::Next::files( { file_filter => $filter }, $self->{work_dir}||$self->{dir} );
}
    return $value;
}

=head2 stream_set( $kv_stream, $key_count, \&merge_sub )

Given a stream that returns ( $key, $value ) when called repeatedly,
set all those values in the hash.

If $key_count is provided, Term::ProgressBar is installed, and
C<verbose> is set on this HashStore, will print progress bars.

If $merge_sub is provided, it will be used to merge values in the
input stream with any values for the same key already in the hash
store like: C<$new_value = $merge_sub-E<gt>( $old_value, $input_value )>.
If not provided, the values will just be overwritten.

=cut

sub stream_set {
    my $self = shift;

    require File::Temp;
    my $tempfile = File::Temp->new( TEMPLATE => 'names-hash-tmp-XXXXXXXX', UNLINK => 1,
                                    DIR => $self->{work_dir} || $self->{dir} );
    $tempfile->close;
    print "Temporary rehashing DBM file: $tempfile\n" if $self->{verbose};

    # convert the input key => value stream to a temp hash of
    # bucket_hex => { key => value, key => value }
    my $bucket_count = $self->_stream_set_build_buckets( shift, $tempfile, shift );

    # write out the bucket files
    my $mergeSub = shift;
    $self->_stream_set_write_bucket_files( $tempfile, $bucket_count, $mergeSub );

    print "Hash store bulk load finished.\n" if $self->{verbose};
}

=head2 db_open( $filename, $flags, $mode, { cachesize => 2**24, ... } )

Open and tie a DB_File in BTREE mode, with the given config options.
Returns a hashref tied to the DB_File.

=cut

sub db_open {
    my ( $self, $file, $flags, $mode, $conf ) = @_;
    my $db_conf = DB_File::BTREEINFO->new;
    $db_conf->{$_} = $conf->{$_} for keys %{ $conf || {} };
    tie( my %store, 'DB_File', "$file", $flags, $mode || 0666, $db_conf);
    return \%store;
}

sub _stream_set_write_bucket_files {
    my ( $self, $tempfile, $bucket_count, $mergeSub ) = @_;

    # reopen the database, this is better because for iterating we
    # don't need the big cache memory used in hashing the keys
    my $buckets = $self->db_open( $tempfile, &POSIX::O_RDONLY, 0666 );

    my $progressbar = $bucket_count && $self->_make_progressbar('Writing/updating index bucket files', $bucket_count );
    my $progressbar_next_update = 0;

    print "Writing buckets to ".$self->{format}."...\n" if $self->{verbose} && ! $progressbar;

    my $buckets_written = 0;
    while( my ( $hex, $contents ) = each %$buckets ) {
        my $bucket = $self->_readBucket( $self->_hexToPath( $hex ) );
        if( my $bucketData = $bucket->{data} ) {
            my $d = Storable::thaw( $contents ) || {};
            if( $mergeSub ) {
                for my $k ( keys %$d ) {
                    if( exists $bucketData->{$k} ) {
                        $bucketData->{$k} = $mergeSub->( $bucketData->{$k}, $d->{$k} );
                    }
                    else {
                        $bucketData->{$k} = $d->{$k};
                    }
                }
            }
            else {
                $bucket->{data} = { %{$bucket->{data}}, %$d };
            }
        }
        else {
            $bucket->{data} = Storable::thaw( $contents );
        }
        $bucket->{dirty} = 1;
        $buckets_written++;

        if ( $progressbar && $buckets_written > $progressbar_next_update ) {
            $progressbar_next_update = $progressbar->update( $buckets_written );
        }
    }
    if ( $progressbar && $buckets_written >= $progressbar_next_update ) {
        $progressbar->update( $bucket_count );
    }
}

sub _stream_set_build_buckets {
    my ( $self, $kv_stream, $tempfile, $key_count ) = @_;

    require Storable;

    my $buckets = $self->db_open( $tempfile, &POSIX::O_CREAT|&POSIX::O_RDWR, 0666,
                                       { flags => 0x1, cachesize => $self->{mem} } );

    my $progressbar = $key_count && $self->_make_progressbar( 'Rehashing to final buckets', $key_count );
    my $progressbar_next_update = 0;
    my $bucket_count = 0;
    my $keys_processed = 0;

    print "Rehashing to final HashStore buckets...\n" if $self->{verbose} && ! $progressbar;

    while ( my ( $k, $v ) = $kv_stream->() ) {
        my $hex = $self->_hexHash( $k );
        my $b = $buckets->{$hex};
        if( $b ) {
            $b = Storable::thaw( $b );
        } else {
            $b = {};
            $bucket_count++;
        }
        $b->{$k} = $v;
        $buckets->{$hex} = Storable::freeze( $b );

        $keys_processed++;
        if ( $progressbar && $keys_processed > $progressbar_next_update ) {
            $progressbar_next_update = $progressbar->update( $keys_processed );
        }
    }
    if ( $progressbar && $key_count > $progressbar_next_update ) {
        $progressbar->update( $key_count );
    }

    return $bucket_count;
}

sub _make_progressbar {
    my ( $self, $description, $total_count ) = @_;

    my ( $self, $key, $value ) = @_;

    my $bucket = $self->_getBucket( $key );
    $bucket->{data}{$key} = $value;
    $bucket->{dirty} = 1;
    $self->{meta}{last_changed_entry} = $key;

    return $value;
}

sub _make_progressbar {
    my ( $self, $description, $total_count ) = @_;

    # do not initialize a progress bar if no tty
    if ( ! -t STDERR) {
       return;
    }

    return unless $self->{verbose};

    eval { require Term::ProgressBar };
    return if $@;

    my $progressbar = Term::ProgressBar->new({ name  => $description,
                                               count => $total_count,
                                               ETA   => 'linear'       });
    $progressbar->max_update_rate(1);
    return $progressbar;
}


=head2 empty

Clear the store of all contents.  Deletes all files and directories
from the store directory.  
Fix #563, don't destroy workdir, if specified

=cut

sub empty {
    my ( $self ) = @_;
    print "Removing existing contents of target dir $self->{dir}\n" if $self->{verbose};
    File::Path::rmtree( $self->{dir} );
#    File::Path::rmtree( $self->{work_dir} ) if defined $self->{work_dir};
    File::Path::mkpath( $self->{dir} );
    File::Path::mkpath( $self->{work_dir} ) if defined $self->{work_dir};
}


########## tied-hash support ########

sub TIEHASH {
    return shift->open( @_ );
}
sub FETCH {
    return shift->get(@_);
}
sub STORE {
    return shift->set(@_);
}
sub DELETE {
    die 'DELETE not implemented';
}
sub CLEAR {
    die 'CLEAR not implemented';
}
sub EXISTS {
    return !! shift->get(@_);
}
sub FIRSTKEY {
    die 'FIRSTKEY not implemented';
}
sub NEXTKEY {
    die 'NEXTKEY not implemented';
}
sub SCALAR {
    die 'SCALAR not implemented';
}
sub UNTIE {
    die 'UNTIE not implemented';
}

########## helper methods ###########

# cached combination hash and print as hex
sub _hexHash {
    my ( $self, $key ) = @_;
    my $cache = $self->{hex_hash_cache} ||= $self->_make_cache( size => 300 );
    return $cache->compute( $key, sub {
        my ($k) = @_;
        return $self->_hex( $self->_hash( $key ) );
    });
}

sub _hash {
    $_[0]->{crc}->strcrc32( $_[1] ) & $_[0]->{hash_mask}
}

sub _hex {
    sprintf( $_[0]->{hash_sprintf_pattern}, $_[1] );
}

sub _hexToPath {
    my ( $self, $hex ) = @_;
    my @dir = ( $self->{dir}, $hex =~ /(.{1,3})/g );
    my @workdir = ( $self->{work_dir}||$self->{dir}, $hex =~ /(.{1,3})/g );
    my $file = (pop @dir).$self->{file_extension};
    my $workfile = (pop @workdir).$self->{file_extension};
    my $dir = File::Spec->catdir(@dir);
    my $workdir = File::Spec->catdir(@workdir);
    #warn "crc: $crc, fullpath: ".File::Spec->catfile( $dir, $file )."\n";
    return { dir => $dir, fullpath => File::Spec->catfile( $dir, $file ), workdir => $workdir, workpath => File::Spec->catfile( $workdir, $file ) };
}

sub _getBucket {
    my ( $self, $key ) = @_;
    return $self->_getBucketFromHex( $self->_hexHash( $key ) );
}

sub _flushAllCaches {
    my ( $self ) = @_;
    delete $self->{$_} for (
        qw(
              bucket_cache
              bucket_log_filehandle_cache
              hex_hash_cache
              bucket_path_cache_by_hex
          ));
}

sub _getBucketFromHex {
    my ( $self, $hex ) = @_;
    my $bucket_cache = $self->{bucket_cache} ||= $self->_make_cache( size => $self->{cache_size} );
    return $bucket_cache->compute( $hex, sub {
        return $self->_readBucket( $self->_getBucketPath( $hex ) )
    });
}

sub _getBucketPath {
    my ( $self, $hex ) = @_;
    my $path_cache = $self->{bucket_path_cache_by_hex} ||= $self->_make_cache( size => $self->{cache_size} );
    return $path_cache->compute( $hex, sub { $self->_hexToPath( $hex ) });
}

sub _readBucket {
    my ( $self, $pathinfo ) = @_;

    my $path = $pathinfo->{fullpath}.( $self->{compress} ? 'z' : '' );
    my $dir = $pathinfo->{dir};
    my $gzip = $self->{compress} ? ':gzip' : '';

    return $bucket_class->new(
        format => $self->{format},
        compress => $self->{compress},
        dir => $dir,
        fullpath => $path,
        ( -f $path
            ? (
                data => eval {
                    if ( $self->{format} eq 'storable' ) {
                        Storable::retrieve( $path )
                      } else {
                          CORE::open my $in, "<$gzip", $path or die "$! reading $path";
                          local $/;
                          JSON::from_json( scalar <$in> )
                        }
                } || {}
              )
            : ( data => {}, dirty => 1 )
        ));
}


######## inner class for on-disk hash buckets ##########

package Bio::JBrowse::HashStore::Bucket;

sub new {
    my $class = shift;
    bless { @_ }, $class;
}

# when a bucket is deleted, flush it to disk
sub DESTROY {
    my ( $self ) = @_;

    if( $self->{dirty} && %{$self->{data}} ) {
        File::Path::mkpath( $self->{dir} ) unless -d $self->{dir};
        if( $self->{format} eq 'storable' ) {
            Storable::store( $self->{data}, $self->{fullpath} );
        } else {
            my $gzip = $self->{compress} ? ':gzip' : '';
            my $out = IO::File->new( $self->{fullpath}, ">$gzip" )
                or die "$! writing $self->{fullpath}";
            $out->print( JSON::to_json( $self->{data} ) ) or die "$! writing to $self->{fullpath}";
        }
    }
}


##### inner cache for FIFO caching ###
package Bio::JBrowse::HashStore::FIFOCache;

sub new {
    my $class = shift;
    return bless {
        fifo  => [],
        bykey => {},
        size  => 100,
        @_
    }, $class;
}

sub compute {
    my ( $self, $key, $callback ) = @_;
    return exists $self->{bykey}{$key} ? $self->{bykey}{$key} : do {
        my $fifo = $self->{fifo};
        if( @$fifo >= $self->{size} ) {
            delete $self->{bykey}{ shift @$fifo };
        }
        push @$fifo, $key;
        return $self->{bykey}{$key} = $callback->($key);
    };
}


1;

<MSG> throw out HashStore stream_set and related code

<DFF> @@ -247,141 +247,6 @@ sub set {
     return $value;
 }
 
-=head2 stream_set( $kv_stream, $key_count, \&merge_sub )
-
-Given a stream that returns ( $key, $value ) when called repeatedly,
-set all those values in the hash.
-
-If $key_count is provided, Term::ProgressBar is installed, and
-C<verbose> is set on this HashStore, will print progress bars.
-
-If $merge_sub is provided, it will be used to merge values in the
-input stream with any values for the same key already in the hash
-store like: C<$new_value = $merge_sub-E<gt>( $old_value, $input_value )>.
-If not provided, the values will just be overwritten.
-
-=cut
-
-sub stream_set {
-    my $self = shift;
-
-    require File::Temp;
-    my $tempfile = File::Temp->new( TEMPLATE => 'names-hash-tmp-XXXXXXXX', UNLINK => 1,
-                                    DIR => $self->{work_dir} || $self->{dir} );
-    $tempfile->close;
-    print "Temporary rehashing DBM file: $tempfile\n" if $self->{verbose};
-
-    # convert the input key => value stream to a temp hash of
-    # bucket_hex => { key => value, key => value }
-    my $bucket_count = $self->_stream_set_build_buckets( shift, $tempfile, shift );
-
-    # write out the bucket files
-    my $mergeSub = shift;
-    $self->_stream_set_write_bucket_files( $tempfile, $bucket_count, $mergeSub );
-
-    print "Hash store bulk load finished.\n" if $self->{verbose};
-}
-
-=head2 db_open( $filename, $flags, $mode, { cachesize => 2**24, ... } )
-
-Open and tie a DB_File in BTREE mode, with the given config options.
-Returns a hashref tied to the DB_File.
-
-=cut
-
-sub db_open {
-    my ( $self, $file, $flags, $mode, $conf ) = @_;
-    my $db_conf = DB_File::BTREEINFO->new;
-    $db_conf->{$_} = $conf->{$_} for keys %{ $conf || {} };
-    tie( my %store, 'DB_File', "$file", $flags, $mode || 0666, $db_conf);
-    return \%store;
-}
-
-sub _stream_set_write_bucket_files {
-    my ( $self, $tempfile, $bucket_count, $mergeSub ) = @_;
-
-    # reopen the database, this is better because for iterating we
-    # don't need the big cache memory used in hashing the keys
-    my $buckets = $self->db_open( $tempfile, &POSIX::O_RDONLY, 0666 );
-
-    my $progressbar = $bucket_count && $self->_make_progressbar('Writing/updating index bucket files', $bucket_count );
-    my $progressbar_next_update = 0;
-
-    print "Writing buckets to ".$self->{format}."...\n" if $self->{verbose} && ! $progressbar;
-
-    my $buckets_written = 0;
-    while( my ( $hex, $contents ) = each %$buckets ) {
-        my $bucket = $self->_readBucket( $self->_hexToPath( $hex ) );
-        if( my $bucketData = $bucket->{data} ) {
-            my $d = Storable::thaw( $contents ) || {};
-            if( $mergeSub ) {
-                for my $k ( keys %$d ) {
-                    if( exists $bucketData->{$k} ) {
-                        $bucketData->{$k} = $mergeSub->( $bucketData->{$k}, $d->{$k} );
-                    }
-                    else {
-                        $bucketData->{$k} = $d->{$k};
-                    }
-                }
-            }
-            else {
-                $bucket->{data} = { %{$bucket->{data}}, %$d };
-            }
-        }
-        else {
-            $bucket->{data} = Storable::thaw( $contents );
-        }
-        $bucket->{dirty} = 1;
-        $buckets_written++;
-
-        if ( $progressbar && $buckets_written > $progressbar_next_update ) {
-            $progressbar_next_update = $progressbar->update( $buckets_written );
-        }
-    }
-    if ( $progressbar && $buckets_written >= $progressbar_next_update ) {
-        $progressbar->update( $bucket_count );
-    }
-}
-
-sub _stream_set_build_buckets {
-    my ( $self, $kv_stream, $tempfile, $key_count ) = @_;
-
-    require Storable;
-
-    my $buckets = $self->db_open( $tempfile, &POSIX::O_CREAT|&POSIX::O_RDWR, 0666,
-                                       { flags => 0x1, cachesize => $self->{mem} } );
-
-    my $progressbar = $key_count && $self->_make_progressbar( 'Rehashing to final buckets', $key_count );
-    my $progressbar_next_update = 0;
-    my $bucket_count = 0;
-    my $keys_processed = 0;
-
-    print "Rehashing to final HashStore buckets...\n" if $self->{verbose} && ! $progressbar;
-
-    while ( my ( $k, $v ) = $kv_stream->() ) {
-        my $hex = $self->_hexHash( $k );
-        my $b = $buckets->{$hex};
-        if( $b ) {
-            $b = Storable::thaw( $b );
-        } else {
-            $b = {};
-            $bucket_count++;
-        }
-        $b->{$k} = $v;
-        $buckets->{$hex} = Storable::freeze( $b );
-
-        $keys_processed++;
-        if ( $progressbar && $keys_processed > $progressbar_next_update ) {
-            $progressbar_next_update = $progressbar->update( $keys_processed );
-        }
-    }
-    if ( $progressbar && $key_count > $progressbar_next_update ) {
-        $progressbar->update( $key_count );
-    }
-
-    return $bucket_count;
-}
-
 sub _make_progressbar {
     my ( $self, $description, $total_count ) = @_;
 

{'commit': 'diff --git a/src/perl5/Bio/JBrowse/HashStore.pm b/src/perl5/Bio/JBrowse/HashStore.pm\nindex 1127a595d..8aaccfada 100644\n--- a/src/perl5/Bio/JBrowse/HashStore.pm\n+++ b/src/perl5/Bio/JBrowse/HashStore.pm\n@@ -247,141 +247,6 @@ sub set {\n     return $value;\n }\n \n-=head2 stream_set( $kv_stream, $key_count, \\&merge_sub )\n-\n-Given a stream that returns ( $key, $value ) when called repeatedly,\n-set all those values in the hash.\n-\n-If $key_count is provided, Term::ProgressBar is installed, and\n-C<verbose> is set on this HashStore, will print progress bars.\n-\n-If $merge_sub is provided, it will be used to merge values in the\n-input stream with any values for the same key already in the hash\n-store like: C<$new_value = $merge_sub-E<gt>( $old_value, $input_value )>.\n-If not provided, the values will just be overwritten.\n-\n-=cut\n-\n-sub stream_set {\n-    my $self = shift;\n-\n-    require File::Temp;\n-    my $tempfile = File::Temp->new( TEMPLATE => \'names-hash-tmp-XXXXXXXX\', UNLINK => 1,\n-                                    DIR => $self->{work_dir} || $self->{dir} );\n-    $tempfile->close;\n-    print "Temporary rehashing DBM file: $tempfile\\n" if $self->{verbose};\n-\n-    # convert the input key => value stream to a temp hash of\n-    # bucket_hex => { key => value, key => value }\n-    my $bucket_count = $self->_stream_set_build_buckets( shift, $tempfile, shift );\n-\n-    # write out the bucket files\n-    my $mergeSub = shift;\n-    $self->_stream_set_write_bucket_files( $tempfile, $bucket_count, $mergeSub );\n-\n-    print "Hash store bulk load finished.\\n" if $self->{verbose};\n-}\n-\n-=head2 db_open( $filename, $flags, $mode, { cachesize => 2**24, ... } )\n-\n-Open and tie a DB_File in BTREE mode, with the given config options.\n-Returns a hashref tied to the DB_File.\n-\n-=cut\n-\n-sub db_open {\n-    my ( $self, $file, $flags, $mode, $conf ) = @_;\n-    my $db_conf = DB_File::BTREEINFO->new;\n-    $db_conf->{$_} = $conf->{$_} for keys %{ $conf || {} };\n-    tie( my %store, \'DB_File\', "$file", $flags, $mode || 0666, $db_conf);\n-    return \\%store;\n-}\n-\n-sub _stream_set_write_bucket_files {\n-    my ( $self, $tempfile, $bucket_count, $mergeSub ) = @_;\n-\n-    # reopen the database, this is better because for iterating we\n-    # don\'t need the big cache memory used in hashing the keys\n-    my $buckets = $self->db_open( $tempfile, &POSIX::O_RDONLY, 0666 );\n-\n-    my $progressbar = $bucket_count && $self->_make_progressbar(\'Writing/updating index bucket files\', $bucket_count );\n-    my $progressbar_next_update = 0;\n-\n-    print "Writing buckets to ".$self->{format}."...\\n" if $self->{verbose} && ! $progressbar;\n-\n-    my $buckets_written = 0;\n-    while( my ( $hex, $contents ) = each %$buckets ) {\n-        my $bucket = $self->_readBucket( $self->_hexToPath( $hex ) );\n-        if( my $bucketData = $bucket->{data} ) {\n-            my $d = Storable::thaw( $contents ) || {};\n-            if( $mergeSub ) {\n-                for my $k ( keys %$d ) {\n-                    if( exists $bucketData->{$k} ) {\n-                        $bucketData->{$k} = $mergeSub->( $bucketData->{$k}, $d->{$k} );\n-                    }\n-                    else {\n-                        $bucketData->{$k} = $d->{$k};\n-                    }\n-                }\n-            }\n-            else {\n-                $bucket->{data} = { %{$bucket->{data}}, %$d };\n-            }\n-        }\n-        else {\n-            $bucket->{data} = Storable::thaw( $contents );\n-        }\n-        $bucket->{dirty} = 1;\n-        $buckets_written++;\n-\n-        if ( $progressbar && $buckets_written > $progressbar_next_update ) {\n-            $progressbar_next_update = $progressbar->update( $buckets_written );\n-        }\n-    }\n-    if ( $progressbar && $buckets_written >= $progressbar_next_update ) {\n-        $progressbar->update( $bucket_count );\n-    }\n-}\n-\n-sub _stream_set_build_buckets {\n-    my ( $self, $kv_stream, $tempfile, $key_count ) = @_;\n-\n-    require Storable;\n-\n-    my $buckets = $self->db_open( $tempfile, &POSIX::O_CREAT|&POSIX::O_RDWR, 0666,\n-                                       { flags => 0x1, cachesize => $self->{mem} } );\n-\n-    my $progressbar = $key_count && $self->_make_progressbar( \'Rehashing to final buckets\', $key_count );\n-    my $progressbar_next_update = 0;\n-    my $bucket_count = 0;\n-    my $keys_processed = 0;\n-\n-    print "Rehashing to final HashStore buckets...\\n" if $self->{verbose} && ! $progressbar;\n-\n-    while ( my ( $k, $v ) = $kv_stream->() ) {\n-        my $hex = $self->_hexHash( $k );\n-        my $b = $buckets->{$hex};\n-        if( $b ) {\n-            $b = Storable::thaw( $b );\n-        } else {\n-            $b = {};\n-            $bucket_count++;\n-        }\n-        $b->{$k} = $v;\n-        $buckets->{$hex} = Storable::freeze( $b );\n-\n-        $keys_processed++;\n-        if ( $progressbar && $keys_processed > $progressbar_next_update ) {\n-            $progressbar_next_update = $progressbar->update( $keys_processed );\n-        }\n-    }\n-    if ( $progressbar && $key_count > $progressbar_next_update ) {\n-        $progressbar->update( $key_count );\n-    }\n-\n-    return $bucket_count;\n-}\n-\n sub _make_progressbar {\n     my ( $self, $description, $total_count ) = @_;\n \ndiff --git a/tests/perl_tests/hashstore.t b/tests/perl_tests/hashstore.t\ndeleted file mode 100644\nindex feff05354..000000000\n--- a/tests/perl_tests/hashstore.t\n+++ /dev/null\n@@ -1,44 +0,0 @@\n-use strict;\n-use warnings;\n-\n-use lib \'tests/perl_tests/lib\';\n-use JBlibs;\n-\n-use Test::More;\n-use File::Temp \'tempdir\';\n-\n-use FileSlurping qw( slurp md5sum_tree );\n-\n-\n-my $tempdir = tempdir( CLEANUP => 1 );\n-diag $tempdir;\n-\n-use_ok( \'Bio::JBrowse::HashStore\');\n-\n-my $store = Bio::JBrowse::HashStore->open(\n-    dir => $tempdir,\n-    empty => 1,\n-    nosync => 1,\n-    hash_bits => 16\n-);\n-\n-my $key_counter = 0;\n-$store->stream_set( sub {\n-  if( $key_counter < 5_000 ) {\n-      return ( "key".$key_counter++ => [ map $key_counter+$_, 40..1 ] );\n-  }\n-  return;\n-});\n-\n-#system "rm -rf tests/data/hash_store/test1; cp -r $tempdir tests/data/hash_store/test1";\n-my $output_sums = md5sum_tree( $tempdir );\n-# {\n-#     use JSON;\n-#     open my $f, \'>\', \'tests/data/hash_store/test1_sums.json\' or die $!;\n-#     $f->print( JSON->new->encode( $output_sums ) );\n-# }\n-\n-is_deeply( $output_sums, slurp( \'tests/data/hash_store/test1_sums.json\' ), \'got right output\' );\n-\n-done_testing;\n-\n', 'commit_subject': 'throw out HashStore stream_set and related code', 'repo_name': 'GMOD/jbrowse', 'committer': {'name': 'Robert Buels', 'email': '38bfb6640636de28997b5b52488be1fb0e57a2f4@cornell.edu', 'time_sec': '1387251267', 'tz_offset': '-300', 'date': {'seconds': '1387251267'}}, 'commit_tree': '69abb81814f3ca0b3348203adfa3d25860121228', 'commit_message': 'throw out HashStore stream_set and related code\n', 'language_name': 'JavaScript', 'license': 'lgpl-2.1', 'hunks': '@@ -247,141 +247,6 @@ sub set {\n     return $value;\n }\n \n-=head2 stream_set( $kv_stream, $key_count, \\&merge_sub )\n-\n-Given a stream that returns ( $key, $value ) when called repeatedly,\n-set all those values in the hash.\n-\n-If $key_count is provided, Term::ProgressBar is installed, and\n-C<verbose> is set on this HashStore, will print progress bars.\n-\n-If $merge_sub is provided, it will be used to merge values in the\n-input stream with any values for the same key already in the hash\n-store like: C<$new_value = $merge_sub-E<gt>( $old_value, $input_value )>.\n-If not provided, the values will just be overwritten.\n-\n-=cut\n-\n-sub stream_set {\n-    my $self = shift;\n-\n-    require File::Temp;\n-    my $tempfile = File::Temp->new( TEMPLATE => \'names-hash-tmp-XXXXXXXX\', UNLINK => 1,\n-                                    DIR => $self->{work_dir} || $self->{dir} );\n-    $tempfile->close;\n-    print "Temporary rehashing DBM file: $tempfile\\n" if $self->{verbose};\n-\n-    # convert the input key => value stream to a temp hash of\n-    # bucket_hex => { key => value, key => value }\n-    my $bucket_count = $self->_stream_set_build_buckets( shift, $tempfile, shift );\n-\n-    # write out the bucket files\n-    my $mergeSub = shift;\n-    $self->_stream_set_write_bucket_files( $tempfile, $bucket_count, $mergeSub );\n-\n-    print "Hash store bulk load finished.\\n" if $self->{verbose};\n-}\n-\n-=head2 db_open( $filename, $flags, $mode, { cachesize => 2**24, ... } )\n-\n-Open and tie a DB_File in BTREE mode, with the given config options.\n-Returns a hashref tied to the DB_File.\n-\n-=cut\n-\n-sub db_open {\n-    my ( $self, $file, $flags, $mode, $conf ) = @_;\n-    my $db_conf = DB_File::BTREEINFO->new;\n-    $db_conf->{$_} = $conf->{$_} for keys %{ $conf || {} };\n-    tie( my %store, \'DB_File\', "$file", $flags, $mode || 0666, $db_conf);\n-    return \\%store;\n-}\n-\n-sub _stream_set_write_bucket_files {\n-    my ( $self, $tempfile, $bucket_count, $mergeSub ) = @_;\n-\n-    # reopen the database, this is better because for iterating we\n-    # don\'t need the big cache memory used in hashing the keys\n-    my $buckets = $self->db_open( $tempfile, &POSIX::O_RDONLY, 0666 );\n-\n-    my $progressbar = $bucket_count && $self->_make_progressbar(\'Writing/updating index bucket files\', $bucket_count );\n-    my $progressbar_next_update = 0;\n-\n-    print "Writing buckets to ".$self->{format}."...\\n" if $self->{verbose} && ! $progressbar;\n-\n-    my $buckets_written = 0;\n-    while( my ( $hex, $contents ) = each %$buckets ) {\n-        my $bucket = $self->_readBucket( $self->_hexToPath( $hex ) );\n-        if( my $bucketData = $bucket->{data} ) {\n-            my $d = Storable::thaw( $contents ) || {};\n-            if( $mergeSub ) {\n-                for my $k ( keys %$d ) {\n-                    if( exists $bucketData->{$k} ) {\n-                        $bucketData->{$k} = $mergeSub->( $bucketData->{$k}, $d->{$k} );\n-                    }\n-                    else {\n-                        $bucketData->{$k} = $d->{$k};\n-                    }\n-                }\n-            }\n-            else {\n-                $bucket->{data} = { %{$bucket->{data}}, %$d };\n-            }\n-        }\n-        else {\n-            $bucket->{data} = Storable::thaw( $contents );\n-        }\n-        $bucket->{dirty} = 1;\n-        $buckets_written++;\n-\n-        if ( $progressbar && $buckets_written > $progressbar_next_update ) {\n-            $progressbar_next_update = $progressbar->update( $buckets_written );\n-        }\n-    }\n-    if ( $progressbar && $buckets_written >= $progressbar_next_update ) {\n-        $progressbar->update( $bucket_count );\n-    }\n-}\n-\n-sub _stream_set_build_buckets {\n-    my ( $self, $kv_stream, $tempfile, $key_count ) = @_;\n-\n-    require Storable;\n-\n-    my $buckets = $self->db_open( $tempfile, &POSIX::O_CREAT|&POSIX::O_RDWR, 0666,\n-                                       { flags => 0x1, cachesize => $self->{mem} } );\n-\n-    my $progressbar = $key_count && $self->_make_progressbar( \'Rehashing to final buckets\', $key_count );\n-    my $progressbar_next_update = 0;\n-    my $bucket_count = 0;\n-    my $keys_processed = 0;\n-\n-    print "Rehashing to final HashStore buckets...\\n" if $self->{verbose} && ! $progressbar;\n-\n-    while ( my ( $k, $v ) = $kv_stream->() ) {\n-        my $hex = $self->_hexHash( $k );\n-        my $b = $buckets->{$hex};\n-        if( $b ) {\n-            $b = Storable::thaw( $b );\n-        } else {\n-            $b = {};\n-            $bucket_count++;\n-        }\n-        $b->{$k} = $v;\n-        $buckets->{$hex} = Storable::freeze( $b );\n-\n-        $keys_processed++;\n-        if ( $progressbar && $keys_processed > $progressbar_next_update ) {\n-            $progressbar_next_update = $progressbar->update( $keys_processed );\n-        }\n-    }\n-    if ( $progressbar && $key_count > $progressbar_next_update ) {\n-        $progressbar->update( $key_count );\n-    }\n-\n-    return $bucket_count;\n-}\n-\n sub _make_progressbar {\n     my ( $self, $description, $total_count ) = @_;\n \n', 'addition_count': 0, 'deletion_count': 135, 'src_file': 'a/src/perl5/Bio/JBrowse/HashStore.pm', 'tgt_file': 'b/src/perl5/Bio/JBrowse/HashStore.pm', 'file_extension': '.pm', 'before_file': '=head1 NAME\n\nBio::JBrowse::HashStore - on-disk 2-level hash table\n\n=head1 DESCRIPTION\n\nMakes an on-disk hash table, designed to be easily read as static files over HTTP.\n\nMakes a set of JSON-encoded files at paths based on a hash of the key.\nFor example:\n\n  path/to/dir/29c/c14/fc.json\n  path/to/dir/23f/5ad/11.json\n  path/to/dir/711/7c1/29.json\n  path/to/dir/5ec/b24/6a.json\n  path/to/dir/4de/9ac/c6.json\n  path/to/dir/41b/c43/27.json\n  path/to/dir/28c/d86/e9.json\n\nWhere each file contains a JSON object containing data items like:\n\n  { foo: "bar", ... }\n\nWhere "foo" is the original key, and "bar" is the JSON-encoded data.\n\n=cut\n\npackage Bio::JBrowse::HashStore;\nuse strict;\nuse warnings;\n\nuse Carp;\n\nuse Storable ();\nuse JSON 2;\n\nuse File::Next ();\nuse File::Path ();\nuse File::Spec ();\n\nuse Digest::Crc32 ();\nuse DB_File ();\nuse IO::File ();\nuse POSIX ();\n\nmy $bucket_class = \'Bio::JBrowse::HashStore::Bucket\';\n\n\n=head2 open( dir => "/path/to/dir", hash_bits => 16, mem => 256 * 2**20 )\n\n=cut\n\nsub open {\n    my $class = shift;\n\n    # source of data: defaults, overridden by open args, overridden by meta.json contents\n    my $self = bless {\n        max_filehandles => 1000,\n        mem => 256*2**20,\n        @_\n    }, $class;\n\n    $self->{dir} or croak "dir option required";\n\n    $self->empty if $self->{empty};\n\n    $self->{meta} = $self->_read_meta;\n\n    $self->{crc} = Digest::Crc32->new;\n\n    # compress, format, and hash_bits all use the value in the\n    # meta.json if present, or the value passed in by the constructor,\n    # or the default, in order of priority\n    my %defaults = ( compress => 0, format => \'json\', hash_bits => 16 );\n    for (qw( compress format hash_bits )) {\n        $self->{$_} = $self->{meta}{$_} = (\n            defined $self->{meta}{$_}  ?  $self->{meta}{$_}  :\n            defined $self->{$_}        ?  $self->{$_}        :\n                                          $defaults{$_}\n        );\n    }\n\n    # check that hash_bits is a multiple of 4\n    if( $self->{hash_bits} % 4 ) {\n        die "Invalid hash bits value $self->{hash_bits}, must be a multiple of 4.\\n";\n    }\n\n    $self->{hash_mask} = 2**($self->{hash_bits}) - 1;\n    $self->{hash_sprintf_pattern} = \'%0\'.int( $self->{hash_bits}/4 ).\'x\';\n    $self->{file_extension} = \'.\'.$self->{format};\n\n    $self->{cache_size} = int( $self->{mem} / 50000 / 2 );\n    print "Hash store cache size: $self->{cache_size} buckets\\n" if $self->{verbose};\n\n    File::Path::mkpath( $self->{dir} );\n\n    return $self;\n}\n\nsub _make_cache {\n    my ( $self, @args ) = @_;\n    return Bio::JBrowse::HashStore::FIFOCache->new( @args );\n}\n\n# write out meta.json file when the store itself is destroyed\nsub DESTROY {\n    my ( $self ) = @_;\n    File::Path::mkpath( $self->{dir} );\n    {\n        my $meta_path = $self->_meta_path;\n        CORE::open my $out, \'>\', $meta_path or die "$! writing $meta_path";\n        $out->print( JSON::to_json( $self->{meta} ) )\n            or die "$! writing $meta_path";\n    }\n\n    # free everything to flush buckets\n    %$self = ();\n}\nsub _meta_path {\n    File::Spec->catfile( shift->{dir}, \'meta.json\' );\n}\nsub _read_meta {\n    my ( $self ) = @_;\n    my $meta_path = $self->_meta_path;\n    return {} unless -r $meta_path;\n    CORE::open my $meta, \'<\', $meta_path or die "$! reading $meta_path";\n    local $/;\n    my $d = eval { JSON->new->relaxed->decode( scalar <$meta> ) } || {};\n    warn $@ if $@;\n    $d->{compress} = 0 unless defined $d->{compress};\n    return $d;\n}\n\n=head2 meta\n\nreturn a hashref of metadata about this hash store\n\n=cut\n\nsub meta {\n    ( shift->{meta} ||= {} )\n}\n\n=head2 get( $key )\n\n=cut\n\nsub get {\n    my ( $self, $key ) = @_;\n\n    my $bucket = $self->_getBucket( $key );\n    return $bucket->{data}{$key};\n}\n\n=head2 stream_do( $arg_stream, $operation_callback )\n\n=cut\n\nsub stream_do {\n    my ( $self, $op_stream, $do_operation, $estimated_op_count ) = @_;\n\n    # clean up any stale log files\n    { my $log_iterator = $self->_file_iterator( sub { /\\.log$/ } );\n      while( my $stale_logfile = $log_iterator->() ) {\n          unlink $stale_logfile;\n      }\n    }\n\n    # make log files for each bucket, log the operations that happen\n    # on that bucket, but don\'t actually do them yet\n    my $ops_written = 0;\n    my $gzip = $self->{compress} ? \':gzip\' : \'\';\n    {\n        my $hash_chars = $self->{hash_bits}/4;\n        my $sort_log_chars = $hash_chars - int( log($self->{cache_size} )/log(16) );\n        my $max_sort_log_chars = int( log( $self->{max_filehandles} )/log(16) );\n        $sort_log_chars = 1 unless $sort_log_chars > 1;\n        $sort_log_chars = $max_sort_log_chars unless $sort_log_chars <= $max_sort_log_chars;\n\n        $hash_chars -= $sort_log_chars;\n        my $zeroes = "0"x$hash_chars;\n\n        print "Using $sort_log_chars chars for sort log names (".(16**$sort_log_chars)." sort logs)\\n" if $self->{verbose};\n        my $filehandle_cache = $self->_make_cache( size => $self->{max_filehandles} );\n        my $progressbar = $estimated_op_count && $self->_make_progressbar( \'Sorting operations\', $estimated_op_count );\n        my $progressbar_next_update = 0;\n        while ( my $op = $op_stream->() ) {\n            my $hex = $self->_hex( $self->_hash( $op->[0] ) );\n\n            substr( (my $log_hex = $hex), 0, $hash_chars, $zeroes );\n\n            my $log_handle = $filehandle_cache->compute( $log_hex, sub {\n                my ( $h ) = @_;\n                my $pathinfo = $self->_hexToPath( $h );\n                File::Path::mkpath( $pathinfo->{workdir} ) unless -d $pathinfo->{workdir};\n                #warn "writing $pathinfo->{fullpath}.log\\n";\n                CORE::open( my $f, ">>$gzip", "$pathinfo->{workpath}.log" )\n                    or die "$! opening bucket log $pathinfo->{workpath}.log";\n                return $f;\n            });\n\n            Storable::store_fd( [$hex,$op], $log_handle );\n\n            $ops_written++;\n            if ( $progressbar && $ops_written >= $progressbar_next_update && $ops_written < $estimated_op_count ) {\n                $progressbar_next_update = $progressbar->update( $ops_written );\n            }\n        }\n        if ( $progressbar && $ops_written > $progressbar_next_update ) {\n            $progressbar->update( $estimated_op_count );\n        }\n    }\n\n    # play back the operations, feeding the $do_operation sub with the\n    # bucket and the operation to be done\n    {\n        my $progressbar = $ops_written && $self->_make_progressbar( \'Executing operations\', $ops_written );\n        my $progressbar_next_update = 0;\n        my $ops_played_back = 0;\n        my $log_iterator = $self->_file_iterator( sub { /\\.log$/ } );\n        while ( my $log_path = $log_iterator->() ) {\n            CORE::open( my $log_fh, "<$gzip", $log_path ) or die "$! reading $log_path";\n            #warn "reading $log_path\\n";\n            while ( my $rec = eval { Storable::fd_retrieve( $log_fh ) } ) {\n                my ( $hex, $op ) = @$rec;\n                my $bucket = $self->_getBucketFromHex( $hex );\n                $bucket->{data}{$op->[0]} = $do_operation->( $op, $bucket->{data}{$op->[0]} );\n                $bucket->{dirty} = 1;\n\n                if ( $progressbar && ++$ops_played_back > $progressbar_next_update ) {\n                    $progressbar_next_update = $progressbar->update( $ops_played_back );\n                }\n            }\n            unlink $log_path;\n        }\n\n        if ( $progressbar && $ops_played_back > $progressbar_next_update ) {\n            $progressbar->update( $ops_written );\n        }\n    }\n}\n\nsub _file_iterator {\n    my ( $self, $filter ) = @_;\n    return File::Next::files( { file_filter => $filter }, $self->{work_dir}||$self->{dir} );\n}\n    return $value;\n}\n\n=head2 stream_set( $kv_stream, $key_count, \\&merge_sub )\n\nGiven a stream that returns ( $key, $value ) when called repeatedly,\nset all those values in the hash.\n\nIf $key_count is provided, Term::ProgressBar is installed, and\nC<verbose> is set on this HashStore, will print progress bars.\n\nIf $merge_sub is provided, it will be used to merge values in the\ninput stream with any values for the same key already in the hash\nstore like: C<$new_value = $merge_sub-E<gt>( $old_value, $input_value )>.\nIf not provided, the values will just be overwritten.\n\n=cut\n\nsub stream_set {\n    my $self = shift;\n\n    require File::Temp;\n    my $tempfile = File::Temp->new( TEMPLATE => \'names-hash-tmp-XXXXXXXX\', UNLINK => 1,\n                                    DIR => $self->{work_dir} || $self->{dir} );\n    $tempfile->close;\n    print "Temporary rehashing DBM file: $tempfile\\n" if $self->{verbose};\n\n    # convert the input key => value stream to a temp hash of\n    # bucket_hex => { key => value, key => value }\n    my $bucket_count = $self->_stream_set_build_buckets( shift, $tempfile, shift );\n\n    # write out the bucket files\n    my $mergeSub = shift;\n    $self->_stream_set_write_bucket_files( $tempfile, $bucket_count, $mergeSub );\n\n    print "Hash store bulk load finished.\\n" if $self->{verbose};\n}\n\n=head2 db_open( $filename, $flags, $mode, { cachesize => 2**24, ... } )\n\nOpen and tie a DB_File in BTREE mode, with the given config options.\nReturns a hashref tied to the DB_File.\n\n=cut\n\nsub db_open {\n    my ( $self, $file, $flags, $mode, $conf ) = @_;\n    my $db_conf = DB_File::BTREEINFO->new;\n    $db_conf->{$_} = $conf->{$_} for keys %{ $conf || {} };\n    tie( my %store, \'DB_File\', "$file", $flags, $mode || 0666, $db_conf);\n    return \\%store;\n}\n\nsub _stream_set_write_bucket_files {\n    my ( $self, $tempfile, $bucket_count, $mergeSub ) = @_;\n\n    # reopen the database, this is better because for iterating we\n    # don\'t need the big cache memory used in hashing the keys\n    my $buckets = $self->db_open( $tempfile, &POSIX::O_RDONLY, 0666 );\n\n    my $progressbar = $bucket_count && $self->_make_progressbar(\'Writing/updating index bucket files\', $bucket_count );\n    my $progressbar_next_update = 0;\n\n    print "Writing buckets to ".$self->{format}."...\\n" if $self->{verbose} && ! $progressbar;\n\n    my $buckets_written = 0;\n    while( my ( $hex, $contents ) = each %$buckets ) {\n        my $bucket = $self->_readBucket( $self->_hexToPath( $hex ) );\n        if( my $bucketData = $bucket->{data} ) {\n            my $d = Storable::thaw( $contents ) || {};\n            if( $mergeSub ) {\n                for my $k ( keys %$d ) {\n                    if( exists $bucketData->{$k} ) {\n                        $bucketData->{$k} = $mergeSub->( $bucketData->{$k}, $d->{$k} );\n                    }\n                    else {\n                        $bucketData->{$k} = $d->{$k};\n                    }\n                }\n            }\n            else {\n                $bucket->{data} = { %{$bucket->{data}}, %$d };\n            }\n        }\n        else {\n            $bucket->{data} = Storable::thaw( $contents );\n        }\n        $bucket->{dirty} = 1;\n        $buckets_written++;\n\n        if ( $progressbar && $buckets_written > $progressbar_next_update ) {\n            $progressbar_next_update = $progressbar->update( $buckets_written );\n        }\n    }\n    if ( $progressbar && $buckets_written >= $progressbar_next_update ) {\n        $progressbar->update( $bucket_count );\n    }\n}\n\nsub _stream_set_build_buckets {\n    my ( $self, $kv_stream, $tempfile, $key_count ) = @_;\n\n    require Storable;\n\n    my $buckets = $self->db_open( $tempfile, &POSIX::O_CREAT|&POSIX::O_RDWR, 0666,\n                                       { flags => 0x1, cachesize => $self->{mem} } );\n\n    my $progressbar = $key_count && $self->_make_progressbar( \'Rehashing to final buckets\', $key_count );\n    my $progressbar_next_update = 0;\n    my $bucket_count = 0;\n    my $keys_processed = 0;\n\n    print "Rehashing to final HashStore buckets...\\n" if $self->{verbose} && ! $progressbar;\n\n    while ( my ( $k, $v ) = $kv_stream->() ) {\n        my $hex = $self->_hexHash( $k );\n        my $b = $buckets->{$hex};\n        if( $b ) {\n            $b = Storable::thaw( $b );\n        } else {\n            $b = {};\n            $bucket_count++;\n        }\n        $b->{$k} = $v;\n        $buckets->{$hex} = Storable::freeze( $b );\n\n        $keys_processed++;\n        if ( $progressbar && $keys_processed > $progressbar_next_update ) {\n            $progressbar_next_update = $progressbar->update( $keys_processed );\n        }\n    }\n    if ( $progressbar && $key_count > $progressbar_next_update ) {\n        $progressbar->update( $key_count );\n    }\n\n    return $bucket_count;\n}\n\nsub _make_progressbar {\n    my ( $self, $description, $total_count ) = @_;\n\n    my ( $self, $key, $value ) = @_;\n\n    my $bucket = $self->_getBucket( $key );\n    $bucket->{data}{$key} = $value;\n    $bucket->{dirty} = 1;\n    $self->{meta}{last_changed_entry} = $key;\n\n    return $value;\n}\n\nsub _make_progressbar {\n    my ( $self, $description, $total_count ) = @_;\n\n    # do not initialize a progress bar if no tty\n    if ( ! -t STDERR) {\n       return;\n    }\n\n    return unless $self->{verbose};\n\n    eval { require Term::ProgressBar };\n    return if $@;\n\n    my $progressbar = Term::ProgressBar->new({ name  => $description,\n                                               count => $total_count,\n                                               ETA   => \'linear\'       });\n    $progressbar->max_update_rate(1);\n    return $progressbar;\n}\n\n\n=head2 empty\n\nClear the store of all contents.  Deletes all files and directories\nfrom the store directory.  \nFix #563, don\'t destroy workdir, if specified\n\n=cut\n\nsub empty {\n    my ( $self ) = @_;\n    print "Removing existing contents of target dir $self->{dir}\\n" if $self->{verbose};\n    File::Path::rmtree( $self->{dir} );\n#    File::Path::rmtree( $self->{work_dir} ) if defined $self->{work_dir};\n    File::Path::mkpath( $self->{dir} );\n    File::Path::mkpath( $self->{work_dir} ) if defined $self->{work_dir};\n}\n\n\n########## tied-hash support ########\n\nsub TIEHASH {\n    return shift->open( @_ );\n}\nsub FETCH {\n    return shift->get(@_);\n}\nsub STORE {\n    return shift->set(@_);\n}\nsub DELETE {\n    die \'DELETE not implemented\';\n}\nsub CLEAR {\n    die \'CLEAR not implemented\';\n}\nsub EXISTS {\n    return !! shift->get(@_);\n}\nsub FIRSTKEY {\n    die \'FIRSTKEY not implemented\';\n}\nsub NEXTKEY {\n    die \'NEXTKEY not implemented\';\n}\nsub SCALAR {\n    die \'SCALAR not implemented\';\n}\nsub UNTIE {\n    die \'UNTIE not implemented\';\n}\n\n########## helper methods ###########\n\n# cached combination hash and print as hex\nsub _hexHash {\n    my ( $self, $key ) = @_;\n    my $cache = $self->{hex_hash_cache} ||= $self->_make_cache( size => 300 );\n    return $cache->compute( $key, sub {\n        my ($k) = @_;\n        return $self->_hex( $self->_hash( $key ) );\n    });\n}\n\nsub _hash {\n    $_[0]->{crc}->strcrc32( $_[1] ) & $_[0]->{hash_mask}\n}\n\nsub _hex {\n    sprintf( $_[0]->{hash_sprintf_pattern}, $_[1] );\n}\n\nsub _hexToPath {\n    my ( $self, $hex ) = @_;\n    my @dir = ( $self->{dir}, $hex =~ /(.{1,3})/g );\n    my @workdir = ( $self->{work_dir}||$self->{dir}, $hex =~ /(.{1,3})/g );\n    my $file = (pop @dir).$self->{file_extension};\n    my $workfile = (pop @workdir).$self->{file_extension};\n    my $dir = File::Spec->catdir(@dir);\n    my $workdir = File::Spec->catdir(@workdir);\n    #warn "crc: $crc, fullpath: ".File::Spec->catfile( $dir, $file )."\\n";\n    return { dir => $dir, fullpath => File::Spec->catfile( $dir, $file ), workdir => $workdir, workpath => File::Spec->catfile( $workdir, $file ) };\n}\n\nsub _getBucket {\n    my ( $self, $key ) = @_;\n    return $self->_getBucketFromHex( $self->_hexHash( $key ) );\n}\n\nsub _flushAllCaches {\n    my ( $self ) = @_;\n    delete $self->{$_} for (\n        qw(\n              bucket_cache\n              bucket_log_filehandle_cache\n              hex_hash_cache\n              bucket_path_cache_by_hex\n          ));\n}\n\nsub _getBucketFromHex {\n    my ( $self, $hex ) = @_;\n    my $bucket_cache = $self->{bucket_cache} ||= $self->_make_cache( size => $self->{cache_size} );\n    return $bucket_cache->compute( $hex, sub {\n        return $self->_readBucket( $self->_getBucketPath( $hex ) )\n    });\n}\n\nsub _getBucketPath {\n    my ( $self, $hex ) = @_;\n    my $path_cache = $self->{bucket_path_cache_by_hex} ||= $self->_make_cache( size => $self->{cache_size} );\n    return $path_cache->compute( $hex, sub { $self->_hexToPath( $hex ) });\n}\n\nsub _readBucket {\n    my ( $self, $pathinfo ) = @_;\n\n    my $path = $pathinfo->{fullpath}.( $self->{compress} ? \'z\' : \'\' );\n    my $dir = $pathinfo->{dir};\n    my $gzip = $self->{compress} ? \':gzip\' : \'\';\n\n    return $bucket_class->new(\n        format => $self->{format},\n        compress => $self->{compress},\n        dir => $dir,\n        fullpath => $path,\n        ( -f $path\n            ? (\n                data => eval {\n                    if ( $self->{format} eq \'storable\' ) {\n                        Storable::retrieve( $path )\n                      } else {\n                          CORE::open my $in, "<$gzip", $path or die "$! reading $path";\n                          local $/;\n                          JSON::from_json( scalar <$in> )\n                        }\n                } || {}\n              )\n            : ( data => {}, dirty => 1 )\n        ));\n}\n\n\n######## inner class for on-disk hash buckets ##########\n\npackage Bio::JBrowse::HashStore::Bucket;\n\nsub new {\n    my $class = shift;\n    bless { @_ }, $class;\n}\n\n# when a bucket is deleted, flush it to disk\nsub DESTROY {\n    my ( $self ) = @_;\n\n    if( $self->{dirty} && %{$self->{data}} ) {\n        File::Path::mkpath( $self->{dir} ) unless -d $self->{dir};\n        if( $self->{format} eq \'storable\' ) {\n            Storable::store( $self->{data}, $self->{fullpath} );\n        } else {\n            my $gzip = $self->{compress} ? \':gzip\' : \'\';\n            my $out = IO::File->new( $self->{fullpath}, ">$gzip" )\n                or die "$! writing $self->{fullpath}";\n            $out->print( JSON::to_json( $self->{data} ) ) or die "$! writing to $self->{fullpath}";\n        }\n    }\n}\n\n\n##### inner cache for FIFO caching ###\npackage Bio::JBrowse::HashStore::FIFOCache;\n\nsub new {\n    my $class = shift;\n    return bless {\n        fifo  => [],\n        bykey => {},\n        size  => 100,\n        @_\n    }, $class;\n}\n\nsub compute {\n    my ( $self, $key, $callback ) = @_;\n    return exists $self->{bykey}{$key} ? $self->{bykey}{$key} : do {\n        my $fifo = $self->{fifo};\n        if( @$fifo >= $self->{size} ) {\n            delete $self->{bykey}{ shift @$fifo };\n        }\n        push @$fifo, $key;\n        return $self->{bykey}{$key} = $callback->($key);\n    };\n}\n\n\n1;\n', 'message': 'throw out HashStore stream_set and related code\n', 'author_name': 'Robert Buels', 'lang': 'pm'}