labpack/databases/google/datastore.py
def _update_indices(self, batch=10):

        ''' a method to update indexing of fields in all records 

        NOTE:   datastore does not automatically index old records, this method
                scans each index and updates the indexing of fields which are
                not indexed. RUN THIS METHOD CAUTIOUSLY
        '''

        title = '%s._update_indices' % self.__class__.__name__

        # validate inputs
        args = {
            'batch': batch
        }
        for key, value in args.items():
            titlex = '%s(%s=%s)' % (title, key, str(value))
            self.fields.validate(value, '.%s' % key, titlex)

        # define empty key sets
        ids = set()
        indices = {}
        for index in self.store_indices:
            indices[index] = set()

        # retrieve all keys
        self.printer('Retrieving ids for table %s' % self.table_name, flush=True)
        query = self.client.query(kind=self.kind)
        query.keys_only()
        next_cursor = True
        kwargs = {
            'limit': 100
        }
        while next_cursor:
            query_iter = query.fetch(**kwargs)
            page = next(query_iter.pages)
            for entity in page:
                ids.add(entity.key._flat_path[-1])
                self.printer('.', flush=True)
            next_cursor = query_iter.next_page_token
            if next_cursor:
                kwargs['start_cursor'] = next_cursor
        self.printer(' done.')

        # for all indices, retrieve index keys
        for index in indices.keys():
            self.printer('Retrieving ids for index %s' % index, flush=True)
            index_query = self.client.query(kind=self.kind)
            index_query.keys_only()
            index_query.order = [index]
            next_cursor = True
            kwargs = {
                'limit': 100
            }
            while next_cursor:
                query_iter = index_query.fetch(**kwargs)
                page = next(query_iter.pages)
                for entity in page:
                    indices[index].add(entity.key._flat_path[-1])
                    self.printer('.', flush=True)
                next_cursor = query_iter.next_page_token
                if next_cursor:
                    kwargs['start_cursor'] = next_cursor
            self.printer(' done.')

        # determine records with all indices
        complete = ids
        for key in indices.keys():
            complete = complete.intersection(indices[key])
        incomplete = list(ids - complete)
        count = len(incomplete)

        # conduct multi get and update
        while True:
            i = batch * -1
            subset = incomplete[i or None:]

            # retrieve a batch of records
            keys = []
            for record_id in subset:
                keys.append(self.client.key(self.kind, record_id))
            records = self.client.get_multi(keys)

            # add indexes to fields
            entities = []
            for entity in records:
                exclusions = set(entity.keys()) - self.store_indices
                kwargs = {'key': entity.key}
                if exclusions:
                    kwargs['exclude_from_indexes'] = list(exclusions)
                updated = datastore.Entity(**kwargs)
                fields = {}
                for key, value in entity.items():
                    fields[key] = value
                updated.update(fields)
                entities.append(updated)
                self.printer('Updating indices for %s.' % entity.key._flat_path[-1])

            # send update request
            self.client.put_multi(entities)

            # pop processed records and break off loop
            incomplete = incomplete[:i or None]
            if len(subset) < batch or not incomplete:
                break

        # report completion and return count
        self.printer('Table %s indices updated.' % self.table_name)
        return count
 a method to update indexing of fields in all records 

        NOTE:   datastore does not automatically index old records, this method
                scans each index and updates the indexing of fields which are
                not indexed. RUN THIS METHOD CAUTIOUSLY
        
a method to update indexing of fields in all records
NOTE:   datastore does not automatically index old records, this method
scans each index and updates the indexing of fields which are
not indexed. RUN THIS METHOD CAUTIOUSLY
a method to update indexing of fields in all records NOTE : datastore does not automatically index old records this method scans each index and updates the indexing of fields which are not indexed . RUN THIS METHOD CAUTIOUSLY
def _update_indices(self, batch=10):
        title = '%s._update_indices' % self.__class__.__name__
        args = {
            'batch': batch
        }
        for key, value in args.items():
            titlex = '%s(%s=%s)' % (title, key, str(value))
            self.fields.validate(value, '.%s' % key, titlex)
        ids = set()
        indices = {}
        for index in self.store_indices:
            indices[index] = set()
        self.printer('Retrieving ids for table %s' % self.table_name, flush=True)
        query = self.client.query(kind=self.kind)
        query.keys_only()
        next_cursor = True
        kwargs = {
            'limit': 100
        }
        while next_cursor:
            query_iter = query.fetch(**kwargs)
            page = next(query_iter.pages)
            for entity in page:
                ids.add(entity.key._flat_path[-1])
                self.printer('.', flush=True)
            next_cursor = query_iter.next_page_token
            if next_cursor:
                kwargs['start_cursor'] = next_cursor
        self.printer(' done.')
        for index in indices.keys():
            self.printer('Retrieving ids for index %s' % index, flush=True)
            index_query = self.client.query(kind=self.kind)
            index_query.keys_only()
            index_query.order = [index]
            next_cursor = True
            kwargs = {
                'limit': 100
            }
            while next_cursor:
                query_iter = index_query.fetch(**kwargs)
                page = next(query_iter.pages)
                for entity in page:
                    indices[index].add(entity.key._flat_path[-1])
                    self.printer('.', flush=True)
                next_cursor = query_iter.next_page_token
                if next_cursor:
                    kwargs['start_cursor'] = next_cursor
            self.printer(' done.')
        complete = ids
        for key in indices.keys():
            complete = complete.intersection(indices[key])
        incomplete = list(ids - complete)
        count = len(incomplete)
        while True:
            i = batch * -1
            subset = incomplete[i or None:]
            keys = []
            for record_id in subset:
                keys.append(self.client.key(self.kind, record_id))
            records = self.client.get_multi(keys)
            entities = []
            for entity in records:
                exclusions = set(entity.keys()) - self.store_indices
                kwargs = {'key': entity.key}
                if exclusions:
                    kwargs['exclude_from_indexes'] = list(exclusions)
                updated = datastore.Entity(**kwargs)
                fields = {}
                for key, value in entity.items():
                    fields[key] = value
                updated.update(fields)
                entities.append(updated)
                self.printer('Updating indices for %s.' % entity.key._flat_path[-1])
            self.client.put_multi(entities)
            incomplete = incomplete[:i or None]
            if len(subset) < batch or not incomplete:
                break
        self.printer('Table %s indices updated.' % self.table_name)
        return count
def _update_indices ( self , batch = 10 ) : title = '%s._update_indices' % self . __class__ . __name__ args = { 'batch' : batch } for key , value in args . items ( ) : titlex = '%s(%s=%s)' % ( title , key , str ( value ) ) self . fields . validate ( value , '.%s' % key , titlex ) ids = set ( ) indices = { } for index in self . store_indices : indices [ index ] = set ( ) self . printer ( 'Retrieving ids for table %s' % self . table_name , flush = True ) query = self . client . query ( kind = self . kind ) query . keys_only ( ) next_cursor = True kwargs = { 'limit' : 100 } while next_cursor : query_iter = query . fetch ( ** kwargs ) page = next ( query_iter . pages ) for entity in page : ids . add ( entity . key . _flat_path [ - 1 ] ) self . printer ( '.' , flush = True ) next_cursor = query_iter . next_page_token if next_cursor : kwargs [ 'start_cursor' ] = next_cursor self . printer ( ' done.' ) for index in indices . keys ( ) : self . printer ( 'Retrieving ids for index %s' % index , flush = True ) index_query = self . client . query ( kind = self . kind ) index_query . keys_only ( ) index_query . order = [ index ] next_cursor = True kwargs = { 'limit' : 100 } while next_cursor : query_iter = index_query . fetch ( ** kwargs ) page = next ( query_iter . pages ) for entity in page : indices [ index ] . add ( entity . key . _flat_path [ - 1 ] ) self . printer ( '.' , flush = True ) next_cursor = query_iter . next_page_token if next_cursor : kwargs [ 'start_cursor' ] = next_cursor self . printer ( ' done.' ) complete = ids for key in indices . keys ( ) : complete = complete . intersection ( indices [ key ] ) incomplete = list ( ids - complete ) count = len ( incomplete ) while True : i = batch * - 1 subset = incomplete [ i or None : ] keys = [ ] for record_id in subset : keys . append ( self . client . key ( self . kind , record_id ) ) records = self . client . get_multi ( keys ) entities = [ ] for entity in records : exclusions = set ( entity . keys ( ) ) - self . store_indices kwargs = { 'key' : entity . key } if exclusions : kwargs [ 'exclude_from_indexes' ] = list ( exclusions ) updated = datastore . Entity ( ** kwargs ) fields = { } for key , value in entity . items ( ) : fields [ key ] = value updated . update ( fields ) entities . append ( updated ) self . printer ( 'Updating indices for %s.' % entity . key . _flat_path [ - 1 ] ) self . client . put_multi ( entities ) incomplete = incomplete [ : i or None ] if len ( subset ) < batch or not incomplete : break self . printer ( 'Table %s indices updated.' % self . table_name ) return count
a method to update indexing of fields in all records
NOTE:   datastore does not automatically index old records, this method
scans each index and updates the indexing of fields which are
not indexed.
a method to update indexing of fields in all records NOTE : datastore does not automatically index old records this method scans each index and updates the indexing of fields which are not indexed .
''' a method to update indexing of fields in all records 

        NOTE:   datastore does not automatically index old records, this method
                scans each index and updates the indexing of fields which are
                not indexed. RUN THIS METHOD CAUTIOUSLY
        ''' # validate inputs # define empty key sets # retrieve all keys # for all indices, retrieve index keys # determine records with all indices # conduct multi get and update # retrieve a batch of records # add indexes to fields # send update request # pop processed records and break off loop # report completion and return count
self
batch