require 'digest'
require 'yajl'
require 'yajl/gzip'

class Admin::SnapshotsController < ApplicationController
  include RepositoryManager

  def create
    file = params[:file]
    snapshot = nil

    File.open(file) do |file|
      parse_metadata(file) do |parsed|

        case parsed
        when Hash
          attributes = filter_header(parsed)
          snapshot = Snapshot.create!(attributes)
        when Array
          parsed.each do |metadata|
            id = Digest::MD5.hexdigest(metadata["id"] + snapshot.id)
            attributes = { id: id, record: metadata }
            metadata = MetadataRecord.create!(attributes)
            snapshot.metadata_records << metadata
          end
        else
          raise TypeError, "Unknown type #{parsed.class}"
        end

      end
      snapshot.save!
    end

    render nothing: true
  end

  def destroy
    @snapshot.delete
    render nothing: true
  end

  def parse_metadata(file)
    gz = Zlib::GzipReader.new(file)
    parser = Yajl::Parser.new
    parser.on_parse_complete = Proc.new { |obj| yield(obj) }
    parser << gz.read
  end

  def filter_header(header)
    fields = Snapshot.fields.keys.map(&:to_sym)
    header.keys.inject({}) do |filtered, key|
      filtered[key] = header[key] if fields.include?(key)
      filtered
    end
  end

end

require 'digest'
require 'yajl'
require 'yajl/gzip'

class Admin::SnapshotsController < ApplicationController
  include RepositoryManager

  def create
    file = params[:file]
    snapshot = nil

    File.open(file) do |file|
      parse_metadata(file) do |parsed|

        case parsed
        when Hash
          attributes = filter_header(parsed)
          snapshot = Snapshot.create!(attributes)
        when Array
          records = parsed.map do |metadata|
            id = Digest::MD5.hexdigest(metadata["id"] + snapshot.id)
            attributes = { id: id, record: metadata, snaphot_id: snapshot.id }
          end
          MetadataRecord.collection.insert(records)
        else
          raise TypeError, "Unknown type #{parsed.class}"
        end

      end
      snapshot.save!
    end

    render nothing: true
  end

  def destroy
    @snapshot.delete
    render nothing: true
  end

  def parse_metadata(file)
    gz = Zlib::GzipReader.new(file)
    parser = Yajl::Parser.new
    parser.on_parse_complete = Proc.new { |obj| yield(obj) }
    while not gz.eof?
      parser << gz.readchar
    end
  end

  def filter_header(header)
    fields = Snapshot.fields.keys
    header.keys.inject({}) do |filtered, key|
      filtered[key] = header[key] if fields.include?(key)
      filtered
    end
  end

end

Change input parsing to one character at a time
Change input parsing to one character at a time

Another reason for the horrendous memory consumption was the fact that I
read the complete file before starting to process it. For now, I solved
this by reading one character at at time.

This again, the absolute trade-off between memory and speed, is too slow
again. I finally need to start using line breaks in my JSON dumps, then
I can read one line at a time.

Signed-off-by: Konrad Reiche <4ff0213f56e2082fcd8c882d04c546881c579ce4@gmail.com>
