ggnn_preprocessed_sparse.py
def make_minibatch_iterator(self, data: Any, is_training: bool):
        """Create minibatches by flattening adjacency matrices into a single adjacency matrix with
        multiple disconnected components."""
        if is_training:
            np.random.shuffle(data)
        # Pack until we cannot fit more graphs in the batch
        state_dropout_keep_prob = self.params['graph_state_dropout_keep_prob'] if is_training else 1.
        edge_weights_dropout_keep_prob = self.params['edge_weight_dropout_keep_prob'] if is_training else 1.
        num_graphs = 0
        while len(data)-num_graphs >= self.params['batch_size']:
            num_graphs_in_batch = 0
            batch_node_labels = []
            batch_node_types = []
            batch_candidates = []
            batch_slot = []
            batch_num_candidates = []
            batch_target_task_values = []
            # batch_target_task_mask = []
            batch_adjacency_lists = [[] for _ in range(self.num_edge_types)]
            batch_num_incoming_edges_per_type = []
            batch_graph_nodes_list = []
            node_offset = 0

            while num_graphs < len(data) and num_graphs_in_batch < self.params['batch_size']:
                cur_graph = data[num_graphs]
                num_nodes_in_graph = len(cur_graph['node_labels'])
                num_candidates_in_graph = cur_graph['num_candidates']

                #pad the word and type ids
                for l in cur_graph['node_labels']:
                    padded_word_ids = np.pad(l, (0, self.params['max_node_length']-len(l)), 'constant',
                                             constant_values=len(self.vocabs)+1)
                    batch_node_labels.append(padded_word_ids)
                for l in cur_graph['node_types']:
                    padded_type_ids = np.pad(l, (0, self.params['max_node_length']-len(l)), 'constant',
                                             constant_values=len(self.type_hierarchy[0]['types'])+1)
                    batch_node_types.append(padded_type_ids)
                # batch_node_labels.extend(cur_graph['node_labels'])
                # batch_node_types.extend(cur_graph['node_types'])
                batch_candidates.append(cur_graph['candidates'])
                batch_slot.append(
                    np.full(shape=[num_candidates_in_graph], fill_value=cur_graph['slot'] + node_offset,
                            dtype=np.int32))
                batch_num_candidates.append(num_candidates_in_graph)
                batch_graph_nodes_list.append(
                    np.full(shape=[num_nodes_in_graph], fill_value=num_graphs_in_batch, dtype=np.int32))
                for i in range(self.num_edge_types):
                    if str(i) in cur_graph['adjacency_lists']:
                        batch_adjacency_lists[i].append(np.asarray(cur_graph['adjacency_lists'][str(i)]) + node_offset)

                # Turn counters for incoming edges into np array:
                num_incoming_edges_per_type = np.zeros((num_nodes_in_graph, self.num_edge_types))
                for (e_type, num_incoming_edges_per_type_dict) in cur_graph['num_incoming_edge_per_type'].items():
                    for (node_id, edge_count) in num_incoming_edges_per_type_dict.items():
                        num_incoming_edges_per_type[int(node_id), int(e_type)] = edge_count
                batch_num_incoming_edges_per_type.append(num_incoming_edges_per_type)

                target_one_hot = np.zeros([num_candidates_in_graph], dtype=np.int32)
                if cur_graph['target'] is None:  # This is one of the examples we didn't sample...
                    batch_target_task_values.append(target_one_hot)
                    # batch_target_task_mask.append(0.)
                else:
                    target_one_hot[cur_graph['target']] = 1
                    batch_target_task_values.append(target_one_hot)
                    # batch_target_task_mask.append(1.)
                num_graphs += 1
                num_graphs_in_batch += 1
                node_offset += num_nodes_in_graph
            batch_feed_dict = {
                self.placeholders['initial_word_ids']: np.reshape(np.concatenate(batch_node_labels),
                                                                  [node_offset, self.params['max_node_length']]),
                self.placeholders['initial_type_ids']: np.reshape(np.concatenate(batch_node_types),
                                                                  [node_offset, self.params['max_node_length']]),
                self.placeholders['candidates']: np.reshape(np.concatenate(batch_candidates), [node_offset, 1]),
                self.placeholders['slots']: np.concatenate(batch_slot),
                self.placeholders['num_candidates_per_graph']: np.array(batch_num_candidates, dtype=np.int32),
                self.placeholders['num_incoming_edges_per_type']: np.concatenate(batch_num_incoming_edges_per_type,
                                                                                 axis=0),
                self.placeholders['graph_nodes_list']: np.concatenate(batch_graph_nodes_list),
                self.placeholders['target_values']: np.concatenate(batch_target_task_values, axis=0),
                # self.placeholders['target_mask']: np.array(batch_target_task_mask),
                self.placeholders['num_graphs']: num_graphs_in_batch,
                self.placeholders['graph_state_keep_prob']: state_dropout_keep_prob,
                self.placeholders['edge_weight_dropout_keep_prob']: edge_weights_dropout_keep_prob
            }

            # Merge adjacency lists and information about incoming nodes:
            for i in range(self.num_edge_types):
                if len(batch_adjacency_lists[i]) > 0:
                    adj_list = np.concatenate(batch_adjacency_lists[i])
                else:
                    adj_list = np.zeros((0, 2), dtype=np.int32)
                batch_feed_dict[self.placeholders['adjacency_lists'][i]] = adj_list

            yield batch_feed_dict
Create minibatches by flattening adjacency matrices into a single adjacency matrix with
        multiple disconnected components.
Create minibatches by flattening adjacency matrices into a single adjacency matrix with
multiple disconnected components.
Create minibatches by flattening adjacency matrices into a single adjacency matrix with multiple disconnected components .
def make_minibatch_iterator(self, data: Any, is_training: bool):
        if is_training:
            np.random.shuffle(data)
        state_dropout_keep_prob = self.params['graph_state_dropout_keep_prob'] if is_training else 1.
        edge_weights_dropout_keep_prob = self.params['edge_weight_dropout_keep_prob'] if is_training else 1.
        num_graphs = 0
        while len(data)-num_graphs >= self.params['batch_size']:
            num_graphs_in_batch = 0
            batch_node_labels = []
            batch_node_types = []
            batch_candidates = []
            batch_slot = []
            batch_num_candidates = []
            batch_target_task_values = []
            batch_adjacency_lists = [[] for _ in range(self.num_edge_types)]
            batch_num_incoming_edges_per_type = []
            batch_graph_nodes_list = []
            node_offset = 0
            while num_graphs < len(data) and num_graphs_in_batch < self.params['batch_size']:
                cur_graph = data[num_graphs]
                num_nodes_in_graph = len(cur_graph['node_labels'])
                num_candidates_in_graph = cur_graph['num_candidates']
                for l in cur_graph['node_labels']:
                    padded_word_ids = np.pad(l, (0, self.params['max_node_length']-len(l)), 'constant',
                                             constant_values=len(self.vocabs)+1)
                    batch_node_labels.append(padded_word_ids)
                for l in cur_graph['node_types']:
                    padded_type_ids = np.pad(l, (0, self.params['max_node_length']-len(l)), 'constant',
                                             constant_values=len(self.type_hierarchy[0]['types'])+1)
                    batch_node_types.append(padded_type_ids)
                batch_candidates.append(cur_graph['candidates'])
                batch_slot.append(
                    np.full(shape=[num_candidates_in_graph], fill_value=cur_graph['slot'] + node_offset,
                            dtype=np.int32))
                batch_num_candidates.append(num_candidates_in_graph)
                batch_graph_nodes_list.append(
                    np.full(shape=[num_nodes_in_graph], fill_value=num_graphs_in_batch, dtype=np.int32))
                for i in range(self.num_edge_types):
                    if str(i) in cur_graph['adjacency_lists']:
                        batch_adjacency_lists[i].append(np.asarray(cur_graph['adjacency_lists'][str(i)]) + node_offset)
                num_incoming_edges_per_type = np.zeros((num_nodes_in_graph, self.num_edge_types))
                for (e_type, num_incoming_edges_per_type_dict) in cur_graph['num_incoming_edge_per_type'].items():
                    for (node_id, edge_count) in num_incoming_edges_per_type_dict.items():
                        num_incoming_edges_per_type[int(node_id), int(e_type)] = edge_count
                batch_num_incoming_edges_per_type.append(num_incoming_edges_per_type)
                target_one_hot = np.zeros([num_candidates_in_graph], dtype=np.int32)
                if cur_graph['target'] is None:  
                    batch_target_task_values.append(target_one_hot)
                else:
                    target_one_hot[cur_graph['target']] = 1
                    batch_target_task_values.append(target_one_hot)
                num_graphs += 1
                num_graphs_in_batch += 1
                node_offset += num_nodes_in_graph
            batch_feed_dict = {
                self.placeholders['initial_word_ids']: np.reshape(np.concatenate(batch_node_labels),
                                                                  [node_offset, self.params['max_node_length']]),
                self.placeholders['initial_type_ids']: np.reshape(np.concatenate(batch_node_types),
                                                                  [node_offset, self.params['max_node_length']]),
                self.placeholders['candidates']: np.reshape(np.concatenate(batch_candidates), [node_offset, 1]),
                self.placeholders['slots']: np.concatenate(batch_slot),
                self.placeholders['num_candidates_per_graph']: np.array(batch_num_candidates, dtype=np.int32),
                self.placeholders['num_incoming_edges_per_type']: np.concatenate(batch_num_incoming_edges_per_type,
                                                                                 axis=0),
                self.placeholders['graph_nodes_list']: np.concatenate(batch_graph_nodes_list),
                self.placeholders['target_values']: np.concatenate(batch_target_task_values, axis=0),
                self.placeholders['num_graphs']: num_graphs_in_batch,
                self.placeholders['graph_state_keep_prob']: state_dropout_keep_prob,
                self.placeholders['edge_weight_dropout_keep_prob']: edge_weights_dropout_keep_prob
            }
            for i in range(self.num_edge_types):
                if len(batch_adjacency_lists[i]) > 0:
                    adj_list = np.concatenate(batch_adjacency_lists[i])
                else:
                    adj_list = np.zeros((0, 2), dtype=np.int32)
                batch_feed_dict[self.placeholders['adjacency_lists'][i]] = adj_list
            yield batch_feed_dict
def make_minibatch_iterator ( self , data : Any , is_training : bool ) : if is_training : np . random . shuffle ( data ) state_dropout_keep_prob = self . params [ 'graph_state_dropout_keep_prob' ] if is_training else 1. edge_weights_dropout_keep_prob = self . params [ 'edge_weight_dropout_keep_prob' ] if is_training else 1. num_graphs = 0 while len ( data ) - num_graphs >= self . params [ 'batch_size' ] : num_graphs_in_batch = 0 batch_node_labels = [ ] batch_node_types = [ ] batch_candidates = [ ] batch_slot = [ ] batch_num_candidates = [ ] batch_target_task_values = [ ] batch_adjacency_lists = [ [ ] for _ in range ( self . num_edge_types ) ] batch_num_incoming_edges_per_type = [ ] batch_graph_nodes_list = [ ] node_offset = 0 while num_graphs < len ( data ) and num_graphs_in_batch < self . params [ 'batch_size' ] : cur_graph = data [ num_graphs ] num_nodes_in_graph = len ( cur_graph [ 'node_labels' ] ) num_candidates_in_graph = cur_graph [ 'num_candidates' ] for l in cur_graph [ 'node_labels' ] : padded_word_ids = np . pad ( l , ( 0 , self . params [ 'max_node_length' ] - len ( l ) ) , 'constant' , constant_values = len ( self . vocabs ) + 1 ) batch_node_labels . append ( padded_word_ids ) for l in cur_graph [ 'node_types' ] : padded_type_ids = np . pad ( l , ( 0 , self . params [ 'max_node_length' ] - len ( l ) ) , 'constant' , constant_values = len ( self . type_hierarchy [ 0 ] [ 'types' ] ) + 1 ) batch_node_types . append ( padded_type_ids ) batch_candidates . append ( cur_graph [ 'candidates' ] ) batch_slot . append ( np . full ( shape = [ num_candidates_in_graph ] , fill_value = cur_graph [ 'slot' ] + node_offset , dtype = np . int32 ) ) batch_num_candidates . append ( num_candidates_in_graph ) batch_graph_nodes_list . append ( np . full ( shape = [ num_nodes_in_graph ] , fill_value = num_graphs_in_batch , dtype = np . int32 ) ) for i in range ( self . num_edge_types ) : if str ( i ) in cur_graph [ 'adjacency_lists' ] : batch_adjacency_lists [ i ] . append ( np . asarray ( cur_graph [ 'adjacency_lists' ] [ str ( i ) ] ) + node_offset ) num_incoming_edges_per_type = np . zeros ( ( num_nodes_in_graph , self . num_edge_types ) ) for ( e_type , num_incoming_edges_per_type_dict ) in cur_graph [ 'num_incoming_edge_per_type' ] . items ( ) : for ( node_id , edge_count ) in num_incoming_edges_per_type_dict . items ( ) : num_incoming_edges_per_type [ int ( node_id ) , int ( e_type ) ] = edge_count batch_num_incoming_edges_per_type . append ( num_incoming_edges_per_type ) target_one_hot = np . zeros ( [ num_candidates_in_graph ] , dtype = np . int32 ) if cur_graph [ 'target' ] is None : batch_target_task_values . append ( target_one_hot ) else : target_one_hot [ cur_graph [ 'target' ] ] = 1 batch_target_task_values . append ( target_one_hot ) num_graphs += 1 num_graphs_in_batch += 1 node_offset += num_nodes_in_graph batch_feed_dict = { self . placeholders [ 'initial_word_ids' ] : np . reshape ( np . concatenate ( batch_node_labels ) , [ node_offset , self . params [ 'max_node_length' ] ] ) , self . placeholders [ 'initial_type_ids' ] : np . reshape ( np . concatenate ( batch_node_types ) , [ node_offset , self . params [ 'max_node_length' ] ] ) , self . placeholders [ 'candidates' ] : np . reshape ( np . concatenate ( batch_candidates ) , [ node_offset , 1 ] ) , self . placeholders [ 'slots' ] : np . concatenate ( batch_slot ) , self . placeholders [ 'num_candidates_per_graph' ] : np . array ( batch_num_candidates , dtype = np . int32 ) , self . placeholders [ 'num_incoming_edges_per_type' ] : np . concatenate ( batch_num_incoming_edges_per_type , axis = 0 ) , self . placeholders [ 'graph_nodes_list' ] : np . concatenate ( batch_graph_nodes_list ) , self . placeholders [ 'target_values' ] : np . concatenate ( batch_target_task_values , axis = 0 ) , self . placeholders [ 'num_graphs' ] : num_graphs_in_batch , self . placeholders [ 'graph_state_keep_prob' ] : state_dropout_keep_prob , self . placeholders [ 'edge_weight_dropout_keep_prob' ] : edge_weights_dropout_keep_prob } for i in range ( self . num_edge_types ) : if len ( batch_adjacency_lists [ i ] ) > 0 : adj_list = np . concatenate ( batch_adjacency_lists [ i ] ) else : adj_list = np . zeros ( ( 0 , 2 ) , dtype = np . int32 ) batch_feed_dict [ self . placeholders [ 'adjacency_lists' ] [ i ] ] = adj_list yield batch_feed_dict
Create minibatches by flattening adjacency matrices into a single adjacency matrix with
multiple disconnected components.
Create minibatches by flattening adjacency matrices into a single adjacency matrix with multiple disconnected components .
"""Create minibatches by flattening adjacency matrices into a single adjacency matrix with
        multiple disconnected components.""" # Pack until we cannot fit more graphs in the batch # batch_target_task_mask = [] #pad the word and type ids # batch_node_labels.extend(cur_graph['node_labels']) # batch_node_types.extend(cur_graph['node_types']) # Turn counters for incoming edges into np array: # This is one of the examples we didn't sample... # batch_target_task_mask.append(0.) # batch_target_task_mask.append(1.) # self.placeholders['target_mask']: np.array(batch_target_task_mask), # Merge adjacency lists and information about incoming nodes:
self
data
Any
is_training
bool