lib/rocket_job/sliced/input.rb
def upload_mongo_query(criteria, *column_names, &block)
        create_indexes
        options = criteria.options

        # Without a block extract the fields from the supplied criteria
        if block
          # Criteria is returning old school :fields instead of :projections
          options[:projection] = options.delete(:fields) if options.key?(:fields)
        else
          column_names = column_names.collect(&:to_s)
          column_names << '_id' if column_names.size.zero?

          fields = options.delete(:fields) || {}
          column_names.each { |col| fields[col] = 1 }
          options[:projection] = fields

          block =
            if column_names.size == 1
              column = column_names.first
              ->(document) { document[column] }
            else
              ->(document) { column_names.collect { |c| document[c] } }
            end
        end

        Writer::Input.collect(self) do |records|
          # Drop down to the mongo driver level to avoid constructing a Model for each document returned
          criteria.klass.collection.find(criteria.selector, options).each do |document|
            records << block.call(document)
          end
        end
      end
# Upload the result of a MongoDB query to the input collection for processing
# Useful when an entire MongoDB collection, or part thereof needs to be
# processed by a job.
#
# Returns [Integer] the number of records uploaded
#
# If a Block is supplied it is passed the document returned from the
# database and should return a record for processing
#
# If no Block is supplied then the record will be the :fields returned
# from MongoDB
#
# Note:
#   This method uses the collection and not the MongoMapper document to
#   avoid the overhead of constructing a Model with every document returned
#   by the query
#
# Note:
#   The Block must return types that can be serialized to BSON.
#   Valid Types: Hash | Array | String | Integer | Float | Symbol | Regexp | Time
#   Invalid: Date, etc.
#
# Example: Upload document ids
#   criteria = User.where(state: 'FL')
#   job.record_count = job.upload_mongo_query(criteria)
#
# Example: Upload just the supplied column
#   criteria = User.where(state: 'FL')
#   job.record_count = job.upload_mongo_query(criteria, :zip_code)
Upload the result of a MongoDB query to the input collection for processing
Useful when an entire MongoDB collection, or part thereof needs to be
processed by a job.

Returns [Integer] the number of records uploaded

If a Block is supplied it is passed the document returned from the
database and should return a record for processing

If no Block is supplied then the record will be the :fields returned
from MongoDB

This method uses the collection and not the MongoMapper document to
avoid the overhead of constructing a Model with every document returned
by the query

The Block must return types that can be serialized to BSON.




Upload the result of a MongoDB query to the input collection for processing Useful when an entire MongoDB collection or part thereof needs to be processed by a job . Returns [ Integer ] the number of records uploaded If a Block is supplied it is passed the document returned from the database and should return a record for processing If no Block is supplied then the record will be the : fields returned from MongoDB This method uses the collection and not the MongoMapper document to avoid the overhead of constructing a Model with every document returned by the query The Block must return types that can be serialized to BSON .
def upload_mongo_query(criteria, *column_names, &block)
        create_indexes
        options = criteria.options
        if block
          options[:projection] = options.delete(:fields) if options.key?(:fields)
        else
          column_names = column_names.collect(&:to_s)
          column_names << '_id' if column_names.size.zero?
          fields = options.delete(:fields) || {}
          column_names.each { |col| fields[col] = 1 }
          options[:projection] = fields
          block =
            if column_names.size == 1
              column = column_names.first
              ->(document) { document[column] }
            else
              ->(document) { column_names.collect { |c| document[c] } }
            end
        end
        Writer::Input.collect(self) do |records|
          criteria.klass.collection.find(criteria.selector, options).each do |document|
            records << block.call(document)
          end
        end
      end
def upload_mongo_query ( criteria , * column_names , & block ) create_indexes options = criteria . options if block options [ :projection ] = options . delete ( :fields ) if options . key? ( :fields ) else column_names = column_names . collect ( & :to_s ) column_names << '_id' if column_names . size . zero? fields = options . delete ( :fields ) || { } column_names . each { | col | fields [ col ] = 1 } options [ :projection ] = fields block = if column_names . size == 1 column = column_names . first -> ( document ) { document [ column ] } else -> ( document ) { column_names . collect { | c | document [ c ] } } end end Writer :: Input . collect ( self ) do | records | criteria . klass . collection . find ( criteria . selector , options ) . each do | document | records << block . call ( document ) end end end
Upload the result of a MongoDB query to the input collection for processing
Useful when an entire MongoDB collection, or part thereof needs to be
processed by a job.
Upload the result of a MongoDB query to the input collection for processing Useful when an entire MongoDB collection or part thereof needs to be processed by a job .
# Without a block extract the fields from the supplied criteria # Criteria is returning old school :fields instead of :projections # Drop down to the mongo driver level to avoid constructing a Model for each document returned
criteria
column_names
block