Spaces:
Running
Running
| import math | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| import torch.utils.model_zoo as model_zoo | |
| from torch.nn.parameter import Parameter | |
| import numpy as np | |
| from collections import OrderedDict | |
| from torch.nn import Parameter | |
| from networks import deeplab_xception,gcn, deeplab_xception_synBN | |
| import pdb | |
| ####################### | |
| # base model | |
| ####################### | |
| class deeplab_xception_transfer_basemodel(deeplab_xception.DeepLabv3_plus): | |
| def __init__(self,nInputChannels=3, n_classes=7, os=16,input_channels=256,hidden_layers=128,out_channels=256): | |
| super(deeplab_xception_transfer_basemodel, self).__init__(nInputChannels=nInputChannels, n_classes=n_classes, | |
| os=os,) | |
| ### source graph | |
| # self.source_featuremap_2_graph = gcn.Featuremaps_to_Graph(input_channels=input_channels, hidden_layers=hidden_layers, | |
| # nodes=n_classes) | |
| # self.source_graph_conv1 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| # self.source_graph_conv2 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| # self.source_graph_conv3 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| # | |
| # self.source_graph_2_fea = gcn.Graph_to_Featuremaps(input_channels=input_channels, output_channels=out_channels, | |
| # hidden_layers=hidden_layers, nodes=n_classes | |
| # ) | |
| # self.source_skip_conv = nn.Sequential(*[nn.Conv2d(input_channels, input_channels, kernel_size=1), | |
| # nn.ReLU(True)]) | |
| ### target graph | |
| self.target_featuremap_2_graph = gcn.Featuremaps_to_Graph(input_channels=input_channels, hidden_layers=hidden_layers, | |
| nodes=n_classes) | |
| self.target_graph_conv1 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| self.target_graph_conv2 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| self.target_graph_conv3 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| self.target_graph_2_fea = gcn.Graph_to_Featuremaps(input_channels=input_channels, output_channels=out_channels, | |
| hidden_layers=hidden_layers, nodes=n_classes | |
| ) | |
| self.target_skip_conv = nn.Sequential(*[nn.Conv2d(input_channels, input_channels, kernel_size=1), | |
| nn.ReLU(True)]) | |
| def load_source_model(self,state_dict): | |
| own_state = self.state_dict() | |
| # for name inshop_cos own_state: | |
| # print name | |
| new_state_dict = OrderedDict() | |
| for name, param in state_dict.items(): | |
| name = name.replace('module.', '') | |
| if 'graph' in name and 'source' not in name and 'target' not in name and 'fc_graph' not in name and 'transpose_graph' not in name: | |
| if 'featuremap_2_graph' in name: | |
| name = name.replace('featuremap_2_graph','source_featuremap_2_graph') | |
| else: | |
| name = name.replace('graph','source_graph') | |
| new_state_dict[name] = 0 | |
| if name not in own_state: | |
| if 'num_batch' in name: | |
| continue | |
| print('unexpected key "{}" in state_dict' | |
| .format(name)) | |
| continue | |
| # if isinstance(param, own_state): | |
| if isinstance(param, Parameter): | |
| # backwards compatibility for serialized parameters | |
| param = param.data | |
| try: | |
| own_state[name].copy_(param) | |
| except: | |
| print('While copying the parameter named {}, whose dimensions in the model are' | |
| ' {} and whose dimensions in the checkpoint are {}, ...'.format( | |
| name, own_state[name].size(), param.size())) | |
| continue # i add inshop_cos 2018/02/01 | |
| own_state[name].copy_(param) | |
| # print 'copying %s' %name | |
| missing = set(own_state.keys()) - set(new_state_dict.keys()) | |
| if len(missing) > 0: | |
| print('missing keys in state_dict: "{}"'.format(missing)) | |
| def get_target_parameter(self): | |
| l = [] | |
| other = [] | |
| for name, k in self.named_parameters(): | |
| if 'target' in name or 'semantic' in name: | |
| l.append(k) | |
| else: | |
| other.append(k) | |
| return l, other | |
| def get_semantic_parameter(self): | |
| l = [] | |
| for name, k in self.named_parameters(): | |
| if 'semantic' in name: | |
| l.append(k) | |
| return l | |
| def get_source_parameter(self): | |
| l = [] | |
| for name, k in self.named_parameters(): | |
| if 'source' in name: | |
| l.append(k) | |
| return l | |
| def forward(self, input,adj1_target=None, adj2_source=None,adj3_transfer=None ): | |
| x, low_level_features = self.xception_features(input) | |
| # print(x.size()) | |
| x1 = self.aspp1(x) | |
| x2 = self.aspp2(x) | |
| x3 = self.aspp3(x) | |
| x4 = self.aspp4(x) | |
| x5 = self.global_avg_pool(x) | |
| x5 = F.upsample(x5, size=x4.size()[2:], mode='bilinear', align_corners=True) | |
| x = torch.cat((x1, x2, x3, x4, x5), dim=1) | |
| x = self.concat_projection_conv1(x) | |
| x = self.concat_projection_bn1(x) | |
| x = self.relu(x) | |
| # print(x.size()) | |
| x = F.upsample(x, size=low_level_features.size()[2:], mode='bilinear', align_corners=True) | |
| low_level_features = self.feature_projection_conv1(low_level_features) | |
| low_level_features = self.feature_projection_bn1(low_level_features) | |
| low_level_features = self.relu(low_level_features) | |
| # print(low_level_features.size()) | |
| # print(x.size()) | |
| x = torch.cat((x, low_level_features), dim=1) | |
| x = self.decoder(x) | |
| ### add graph | |
| # target graph | |
| # print('x size',x.size(),adj1.size()) | |
| graph = self.target_featuremap_2_graph(x) | |
| # graph combine | |
| # print(graph.size(),source_2_target_graph.size()) | |
| # graph = self.fc_graph.forward(graph,relu=True) | |
| # print(graph.size()) | |
| graph = self.target_graph_conv1.forward(graph, adj=adj1_target, relu=True) | |
| graph = self.target_graph_conv2.forward(graph, adj=adj1_target, relu=True) | |
| graph = self.target_graph_conv3.forward(graph, adj=adj1_target, relu=True) | |
| # print(graph.size(),x.size()) | |
| # graph = self.gcn_encode.forward(graph,relu=True) | |
| # graph = self.graph_conv2.forward(graph,adj=adj2,relu=True) | |
| # graph = self.gcn_decode.forward(graph,relu=True) | |
| graph = self.target_graph_2_fea.forward(graph, x) | |
| x = self.target_skip_conv(x) | |
| x = x + graph | |
| ### | |
| x = self.semantic(x) | |
| x = F.upsample(x, size=input.size()[2:], mode='bilinear', align_corners=True) | |
| return x | |
| class deeplab_xception_transfer_basemodel_savememory(deeplab_xception.DeepLabv3_plus): | |
| def __init__(self,nInputChannels=3, n_classes=7, os=16,input_channels=256,hidden_layers=128,out_channels=256): | |
| super(deeplab_xception_transfer_basemodel_savememory, self).__init__(nInputChannels=nInputChannels, n_classes=n_classes, | |
| os=os,) | |
| ### source graph | |
| ### target graph | |
| self.target_featuremap_2_graph = gcn.Featuremaps_to_Graph(input_channels=input_channels, hidden_layers=hidden_layers, | |
| nodes=n_classes) | |
| self.target_graph_conv1 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| self.target_graph_conv2 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| self.target_graph_conv3 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| self.target_graph_2_fea = gcn.Graph_to_Featuremaps_savemem(input_channels=input_channels, output_channels=out_channels, | |
| hidden_layers=hidden_layers, nodes=n_classes | |
| ) | |
| self.target_skip_conv = nn.Sequential(*[nn.Conv2d(input_channels, input_channels, kernel_size=1), | |
| nn.ReLU(True)]) | |
| def load_source_model(self,state_dict): | |
| own_state = self.state_dict() | |
| # for name inshop_cos own_state: | |
| # print name | |
| new_state_dict = OrderedDict() | |
| for name, param in state_dict.items(): | |
| name = name.replace('module.', '') | |
| if 'graph' in name and 'source' not in name and 'target' not in name and 'fc_graph' not in name and 'transpose_graph' not in name: | |
| if 'featuremap_2_graph' in name: | |
| name = name.replace('featuremap_2_graph','source_featuremap_2_graph') | |
| else: | |
| name = name.replace('graph','source_graph') | |
| new_state_dict[name] = 0 | |
| if name not in own_state: | |
| if 'num_batch' in name: | |
| continue | |
| print('unexpected key "{}" in state_dict' | |
| .format(name)) | |
| continue | |
| # if isinstance(param, own_state): | |
| if isinstance(param, Parameter): | |
| # backwards compatibility for serialized parameters | |
| param = param.data | |
| try: | |
| own_state[name].copy_(param) | |
| except: | |
| print('While copying the parameter named {}, whose dimensions in the model are' | |
| ' {} and whose dimensions in the checkpoint are {}, ...'.format( | |
| name, own_state[name].size(), param.size())) | |
| continue # i add inshop_cos 2018/02/01 | |
| own_state[name].copy_(param) | |
| # print 'copying %s' %name | |
| missing = set(own_state.keys()) - set(new_state_dict.keys()) | |
| if len(missing) > 0: | |
| print('missing keys in state_dict: "{}"'.format(missing)) | |
| def get_target_parameter(self): | |
| l = [] | |
| other = [] | |
| for name, k in self.named_parameters(): | |
| if 'target' in name or 'semantic' in name: | |
| l.append(k) | |
| else: | |
| other.append(k) | |
| return l, other | |
| def get_semantic_parameter(self): | |
| l = [] | |
| for name, k in self.named_parameters(): | |
| if 'semantic' in name: | |
| l.append(k) | |
| return l | |
| def get_source_parameter(self): | |
| l = [] | |
| for name, k in self.named_parameters(): | |
| if 'source' in name: | |
| l.append(k) | |
| return l | |
| def forward(self, input,adj1_target=None, adj2_source=None,adj3_transfer=None ): | |
| x, low_level_features = self.xception_features(input) | |
| # print(x.size()) | |
| x1 = self.aspp1(x) | |
| x2 = self.aspp2(x) | |
| x3 = self.aspp3(x) | |
| x4 = self.aspp4(x) | |
| x5 = self.global_avg_pool(x) | |
| x5 = F.upsample(x5, size=x4.size()[2:], mode='bilinear', align_corners=True) | |
| x = torch.cat((x1, x2, x3, x4, x5), dim=1) | |
| x = self.concat_projection_conv1(x) | |
| x = self.concat_projection_bn1(x) | |
| x = self.relu(x) | |
| # print(x.size()) | |
| x = F.upsample(x, size=low_level_features.size()[2:], mode='bilinear', align_corners=True) | |
| low_level_features = self.feature_projection_conv1(low_level_features) | |
| low_level_features = self.feature_projection_bn1(low_level_features) | |
| low_level_features = self.relu(low_level_features) | |
| # print(low_level_features.size()) | |
| # print(x.size()) | |
| x = torch.cat((x, low_level_features), dim=1) | |
| x = self.decoder(x) | |
| ### add graph | |
| # target graph | |
| # print('x size',x.size(),adj1.size()) | |
| graph = self.target_featuremap_2_graph(x) | |
| # graph combine | |
| # print(graph.size(),source_2_target_graph.size()) | |
| # graph = self.fc_graph.forward(graph,relu=True) | |
| # print(graph.size()) | |
| graph = self.target_graph_conv1.forward(graph, adj=adj1_target, relu=True) | |
| graph = self.target_graph_conv2.forward(graph, adj=adj1_target, relu=True) | |
| graph = self.target_graph_conv3.forward(graph, adj=adj1_target, relu=True) | |
| # print(graph.size(),x.size()) | |
| # graph = self.gcn_encode.forward(graph,relu=True) | |
| # graph = self.graph_conv2.forward(graph,adj=adj2,relu=True) | |
| # graph = self.gcn_decode.forward(graph,relu=True) | |
| graph = self.target_graph_2_fea.forward(graph, x) | |
| x = self.target_skip_conv(x) | |
| x = x + graph | |
| ### | |
| x = self.semantic(x) | |
| x = F.upsample(x, size=input.size()[2:], mode='bilinear', align_corners=True) | |
| return x | |
| class deeplab_xception_transfer_basemodel_synBN(deeplab_xception_synBN.DeepLabv3_plus): | |
| def __init__(self,nInputChannels=3, n_classes=7, os=16,input_channels=256,hidden_layers=128,out_channels=256): | |
| super(deeplab_xception_transfer_basemodel_synBN, self).__init__(nInputChannels=nInputChannels, n_classes=n_classes, | |
| os=os,) | |
| ### source graph | |
| # self.source_featuremap_2_graph = gcn.Featuremaps_to_Graph(input_channels=input_channels, hidden_layers=hidden_layers, | |
| # nodes=n_classes) | |
| # self.source_graph_conv1 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| # self.source_graph_conv2 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| # self.source_graph_conv3 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| # | |
| # self.source_graph_2_fea = gcn.Graph_to_Featuremaps(input_channels=input_channels, output_channels=out_channels, | |
| # hidden_layers=hidden_layers, nodes=n_classes | |
| # ) | |
| # self.source_skip_conv = nn.Sequential(*[nn.Conv2d(input_channels, input_channels, kernel_size=1), | |
| # nn.ReLU(True)]) | |
| ### target graph | |
| self.target_featuremap_2_graph = gcn.Featuremaps_to_Graph(input_channels=input_channels, hidden_layers=hidden_layers, | |
| nodes=n_classes) | |
| self.target_graph_conv1 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| self.target_graph_conv2 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| self.target_graph_conv3 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| self.target_graph_2_fea = gcn.Graph_to_Featuremaps(input_channels=input_channels, output_channels=out_channels, | |
| hidden_layers=hidden_layers, nodes=n_classes | |
| ) | |
| self.target_skip_conv = nn.Sequential(*[nn.Conv2d(input_channels, input_channels, kernel_size=1), | |
| nn.ReLU(True)]) | |
| def load_source_model(self,state_dict): | |
| own_state = self.state_dict() | |
| # for name inshop_cos own_state: | |
| # print name | |
| new_state_dict = OrderedDict() | |
| for name, param in state_dict.items(): | |
| name = name.replace('module.', '') | |
| if 'graph' in name and 'source' not in name and 'target' not in name: | |
| if 'featuremap_2_graph' in name: | |
| name = name.replace('featuremap_2_graph','source_featuremap_2_graph') | |
| else: | |
| name = name.replace('graph','source_graph') | |
| new_state_dict[name] = 0 | |
| if name not in own_state: | |
| if 'num_batch' in name: | |
| continue | |
| print('unexpected key "{}" in state_dict' | |
| .format(name)) | |
| continue | |
| # if isinstance(param, own_state): | |
| if isinstance(param, Parameter): | |
| # backwards compatibility for serialized parameters | |
| param = param.data | |
| try: | |
| own_state[name].copy_(param) | |
| except: | |
| print('While copying the parameter named {}, whose dimensions in the model are' | |
| ' {} and whose dimensions in the checkpoint are {}, ...'.format( | |
| name, own_state[name].size(), param.size())) | |
| continue # i add inshop_cos 2018/02/01 | |
| own_state[name].copy_(param) | |
| # print 'copying %s' %name | |
| missing = set(own_state.keys()) - set(new_state_dict.keys()) | |
| if len(missing) > 0: | |
| print('missing keys in state_dict: "{}"'.format(missing)) | |
| def get_target_parameter(self): | |
| l = [] | |
| other = [] | |
| for name, k in self.named_parameters(): | |
| if 'target' in name or 'semantic' in name: | |
| l.append(k) | |
| else: | |
| other.append(k) | |
| return l, other | |
| def get_semantic_parameter(self): | |
| l = [] | |
| for name, k in self.named_parameters(): | |
| if 'semantic' in name: | |
| l.append(k) | |
| return l | |
| def get_source_parameter(self): | |
| l = [] | |
| for name, k in self.named_parameters(): | |
| if 'source' in name: | |
| l.append(k) | |
| return l | |
| def forward(self, input,adj1_target=None, adj2_source=None,adj3_transfer=None ): | |
| x, low_level_features = self.xception_features(input) | |
| # print(x.size()) | |
| x1 = self.aspp1(x) | |
| x2 = self.aspp2(x) | |
| x3 = self.aspp3(x) | |
| x4 = self.aspp4(x) | |
| x5 = self.global_avg_pool(x) | |
| x5 = F.upsample(x5, size=x4.size()[2:], mode='bilinear', align_corners=True) | |
| x = torch.cat((x1, x2, x3, x4, x5), dim=1) | |
| x = self.concat_projection_conv1(x) | |
| x = self.concat_projection_bn1(x) | |
| x = self.relu(x) | |
| # print(x.size()) | |
| x = F.upsample(x, size=low_level_features.size()[2:], mode='bilinear', align_corners=True) | |
| low_level_features = self.feature_projection_conv1(low_level_features) | |
| low_level_features = self.feature_projection_bn1(low_level_features) | |
| low_level_features = self.relu(low_level_features) | |
| # print(low_level_features.size()) | |
| # print(x.size()) | |
| x = torch.cat((x, low_level_features), dim=1) | |
| x = self.decoder(x) | |
| ### add graph | |
| # target graph | |
| # print('x size',x.size(),adj1.size()) | |
| graph = self.target_featuremap_2_graph(x) | |
| # graph combine | |
| # print(graph.size(),source_2_target_graph.size()) | |
| # graph = self.fc_graph.forward(graph,relu=True) | |
| # print(graph.size()) | |
| graph = self.target_graph_conv1.forward(graph, adj=adj1_target, relu=True) | |
| graph = self.target_graph_conv2.forward(graph, adj=adj1_target, relu=True) | |
| graph = self.target_graph_conv3.forward(graph, adj=adj1_target, relu=True) | |
| # print(graph.size(),x.size()) | |
| # graph = self.gcn_encode.forward(graph,relu=True) | |
| # graph = self.graph_conv2.forward(graph,adj=adj2,relu=True) | |
| # graph = self.gcn_decode.forward(graph,relu=True) | |
| graph = self.target_graph_2_fea.forward(graph, x) | |
| x = self.target_skip_conv(x) | |
| x = x + graph | |
| ### | |
| x = self.semantic(x) | |
| x = F.upsample(x, size=input.size()[2:], mode='bilinear', align_corners=True) | |
| return x | |
| class deeplab_xception_transfer_basemodel_synBN_savememory(deeplab_xception_synBN.DeepLabv3_plus): | |
| def __init__(self,nInputChannels=3, n_classes=7, os=16,input_channels=256,hidden_layers=128,out_channels=256): | |
| super(deeplab_xception_transfer_basemodel_synBN_savememory, self).__init__(nInputChannels=nInputChannels, n_classes=n_classes, | |
| os=os, ) | |
| ### source graph | |
| # self.source_featuremap_2_graph = gcn.Featuremaps_to_Graph(input_channels=input_channels, hidden_layers=hidden_layers, | |
| # nodes=n_classes) | |
| # self.source_graph_conv1 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| # self.source_graph_conv2 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| # self.source_graph_conv3 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| # | |
| # self.source_graph_2_fea = gcn.Graph_to_Featuremaps(input_channels=input_channels, output_channels=out_channels, | |
| # hidden_layers=hidden_layers, nodes=n_classes | |
| # ) | |
| # self.source_skip_conv = nn.Sequential(*[nn.Conv2d(input_channels, input_channels, kernel_size=1), | |
| # nn.ReLU(True)]) | |
| ### target graph | |
| self.target_featuremap_2_graph = gcn.Featuremaps_to_Graph(input_channels=input_channels, hidden_layers=hidden_layers, | |
| nodes=n_classes) | |
| self.target_graph_conv1 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| self.target_graph_conv2 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| self.target_graph_conv3 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| self.target_graph_2_fea = gcn.Graph_to_Featuremaps_savemem(input_channels=input_channels, output_channels=out_channels, | |
| hidden_layers=hidden_layers, nodes=n_classes | |
| ) | |
| self.target_skip_conv = nn.Sequential(*[nn.Conv2d(input_channels, input_channels, kernel_size=1), | |
| nn.BatchNorm2d(input_channels), | |
| nn.ReLU(True)]) | |
| def load_source_model(self,state_dict): | |
| own_state = self.state_dict() | |
| # for name inshop_cos own_state: | |
| # print name | |
| new_state_dict = OrderedDict() | |
| for name, param in state_dict.items(): | |
| name = name.replace('module.', '') | |
| if 'graph' in name and 'source' not in name and 'target' not in name: | |
| if 'featuremap_2_graph' in name: | |
| name = name.replace('featuremap_2_graph','source_featuremap_2_graph') | |
| else: | |
| name = name.replace('graph','source_graph') | |
| new_state_dict[name] = 0 | |
| if name not in own_state: | |
| if 'num_batch' in name: | |
| continue | |
| print('unexpected key "{}" in state_dict' | |
| .format(name)) | |
| continue | |
| # if isinstance(param, own_state): | |
| if isinstance(param, Parameter): | |
| # backwards compatibility for serialized parameters | |
| param = param.data | |
| try: | |
| own_state[name].copy_(param) | |
| except: | |
| print('While copying the parameter named {}, whose dimensions in the model are' | |
| ' {} and whose dimensions in the checkpoint are {}, ...'.format( | |
| name, own_state[name].size(), param.size())) | |
| continue # i add inshop_cos 2018/02/01 | |
| own_state[name].copy_(param) | |
| # print 'copying %s' %name | |
| missing = set(own_state.keys()) - set(new_state_dict.keys()) | |
| if len(missing) > 0: | |
| print('missing keys in state_dict: "{}"'.format(missing)) | |
| def get_target_parameter(self): | |
| l = [] | |
| other = [] | |
| for name, k in self.named_parameters(): | |
| if 'target' in name or 'semantic' in name: | |
| l.append(k) | |
| else: | |
| other.append(k) | |
| return l, other | |
| def get_semantic_parameter(self): | |
| l = [] | |
| for name, k in self.named_parameters(): | |
| if 'semantic' in name: | |
| l.append(k) | |
| return l | |
| def get_source_parameter(self): | |
| l = [] | |
| for name, k in self.named_parameters(): | |
| if 'source' in name: | |
| l.append(k) | |
| return l | |
| def forward(self, input,adj1_target=None, adj2_source=None,adj3_transfer=None ): | |
| x, low_level_features = self.xception_features(input) | |
| # print(x.size()) | |
| x1 = self.aspp1(x) | |
| x2 = self.aspp2(x) | |
| x3 = self.aspp3(x) | |
| x4 = self.aspp4(x) | |
| x5 = self.global_avg_pool(x) | |
| x5 = F.upsample(x5, size=x4.size()[2:], mode='bilinear', align_corners=True) | |
| x = torch.cat((x1, x2, x3, x4, x5), dim=1) | |
| x = self.concat_projection_conv1(x) | |
| x = self.concat_projection_bn1(x) | |
| x = self.relu(x) | |
| # print(x.size()) | |
| x = F.upsample(x, size=low_level_features.size()[2:], mode='bilinear', align_corners=True) | |
| low_level_features = self.feature_projection_conv1(low_level_features) | |
| low_level_features = self.feature_projection_bn1(low_level_features) | |
| low_level_features = self.relu(low_level_features) | |
| # print(low_level_features.size()) | |
| # print(x.size()) | |
| x = torch.cat((x, low_level_features), dim=1) | |
| x = self.decoder(x) | |
| ### add graph | |
| # target graph | |
| # print('x size',x.size(),adj1.size()) | |
| graph = self.target_featuremap_2_graph(x) | |
| # graph combine | |
| # print(graph.size(),source_2_target_graph.size()) | |
| # graph = self.fc_graph.forward(graph,relu=True) | |
| # print(graph.size()) | |
| graph = self.target_graph_conv1.forward(graph, adj=adj1_target, relu=True) | |
| graph = self.target_graph_conv2.forward(graph, adj=adj1_target, relu=True) | |
| graph = self.target_graph_conv3.forward(graph, adj=adj1_target, relu=True) | |
| # print(graph.size(),x.size()) | |
| # graph = self.gcn_encode.forward(graph,relu=True) | |
| # graph = self.graph_conv2.forward(graph,adj=adj2,relu=True) | |
| # graph = self.gcn_decode.forward(graph,relu=True) | |
| graph = self.target_graph_2_fea.forward(graph, x) | |
| x = self.target_skip_conv(x) | |
| x = x + graph | |
| ### | |
| x = self.semantic(x) | |
| x = F.upsample(x, size=input.size()[2:], mode='bilinear', align_corners=True) | |
| return x | |
| ####################### | |
| # transfer model | |
| ####################### | |
| class deeplab_xception_transfer_projection(deeplab_xception_transfer_basemodel): | |
| def __init__(self, nInputChannels=3, n_classes=7, os=16,input_channels=256,hidden_layers=128,out_channels=256, | |
| transfer_graph=None, source_classes=20): | |
| super(deeplab_xception_transfer_projection, self).__init__(nInputChannels=nInputChannels, n_classes=n_classes, | |
| os=os, input_channels=input_channels, | |
| hidden_layers=hidden_layers, out_channels=out_channels, ) | |
| self.source_featuremap_2_graph = gcn.Featuremaps_to_Graph(input_channels=input_channels, hidden_layers=hidden_layers, | |
| nodes=source_classes) | |
| self.source_graph_conv1 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| self.source_graph_conv2 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| self.source_graph_conv3 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| self.transpose_graph = gcn.Graph_trans(in_features=hidden_layers,out_features=hidden_layers,adj=transfer_graph, | |
| begin_nodes=source_classes,end_nodes=n_classes) | |
| self.fc_graph = gcn.GraphConvolution(hidden_layers*3, hidden_layers) | |
| def forward(self, input,adj1_target=None, adj2_source=None,adj3_transfer=None ): | |
| x, low_level_features = self.xception_features(input) | |
| # print(x.size()) | |
| x1 = self.aspp1(x) | |
| x2 = self.aspp2(x) | |
| x3 = self.aspp3(x) | |
| x4 = self.aspp4(x) | |
| x5 = self.global_avg_pool(x) | |
| x5 = F.upsample(x5, size=x4.size()[2:], mode='bilinear', align_corners=True) | |
| x = torch.cat((x1, x2, x3, x4, x5), dim=1) | |
| x = self.concat_projection_conv1(x) | |
| x = self.concat_projection_bn1(x) | |
| x = self.relu(x) | |
| # print(x.size()) | |
| x = F.upsample(x, size=low_level_features.size()[2:], mode='bilinear', align_corners=True) | |
| low_level_features = self.feature_projection_conv1(low_level_features) | |
| low_level_features = self.feature_projection_bn1(low_level_features) | |
| low_level_features = self.relu(low_level_features) | |
| # print(low_level_features.size()) | |
| # print(x.size()) | |
| x = torch.cat((x, low_level_features), dim=1) | |
| x = self.decoder(x) | |
| ### add graph | |
| # source graph | |
| source_graph = self.source_featuremap_2_graph(x) | |
| source_graph1 = self.source_graph_conv1.forward(source_graph,adj=adj2_source, relu=True) | |
| source_graph2 = self.source_graph_conv2.forward(source_graph1, adj=adj2_source, relu=True) | |
| source_graph3 = self.source_graph_conv2.forward(source_graph2, adj=adj2_source, relu=True) | |
| source_2_target_graph1_v5 = self.transpose_graph.forward(source_graph1, adj=adj3_transfer, relu=True) | |
| source_2_target_graph2_v5 = self.transpose_graph.forward(source_graph2, adj=adj3_transfer, relu=True) | |
| source_2_target_graph3_v5 = self.transpose_graph.forward(source_graph3, adj=adj3_transfer, relu=True) | |
| # target graph | |
| # print('x size',x.size(),adj1.size()) | |
| graph = self.target_featuremap_2_graph(x) | |
| source_2_target_graph1 = self.similarity_trans(source_graph1, graph) | |
| # graph combine 1 | |
| # print(graph.size()) | |
| # print(source_2_target_graph1.size()) | |
| # print(source_2_target_graph1_v5.size()) | |
| graph = torch.cat((graph,source_2_target_graph1.squeeze(0), source_2_target_graph1_v5.squeeze(0)),dim=-1) | |
| graph = self.fc_graph.forward(graph,relu=True) | |
| graph = self.target_graph_conv1.forward(graph, adj=adj1_target, relu=True) | |
| source_2_target_graph2 = self.similarity_trans(source_graph2, graph) | |
| # graph combine 2 | |
| graph = torch.cat((graph, source_2_target_graph2, source_2_target_graph2_v5), dim=-1) | |
| graph = self.fc_graph.forward(graph, relu=True) | |
| graph = self.target_graph_conv2.forward(graph, adj=adj1_target, relu=True) | |
| source_2_target_graph3 = self.similarity_trans(source_graph3, graph) | |
| # graph combine 3 | |
| graph = torch.cat((graph, source_2_target_graph3, source_2_target_graph3_v5), dim=-1) | |
| graph = self.fc_graph.forward(graph, relu=True) | |
| graph = self.target_graph_conv3.forward(graph, adj=adj1_target, relu=True) | |
| # print(graph.size(),x.size()) | |
| graph = self.target_graph_2_fea.forward(graph, x) | |
| x = self.target_skip_conv(x) | |
| x = x + graph | |
| ### | |
| x = self.semantic(x) | |
| x = F.upsample(x, size=input.size()[2:], mode='bilinear', align_corners=True) | |
| return x | |
| def similarity_trans(self,source,target): | |
| sim = torch.matmul(F.normalize(target, p=2, dim=-1), F.normalize(source, p=2, dim=-1).transpose(-1, -2)) | |
| sim = F.softmax(sim, dim=-1) | |
| return torch.matmul(sim, source) | |
| def load_source_model(self,state_dict): | |
| own_state = self.state_dict() | |
| # for name inshop_cos own_state: | |
| # print name | |
| new_state_dict = OrderedDict() | |
| for name, param in state_dict.items(): | |
| name = name.replace('module.', '') | |
| if 'graph' in name and 'source' not in name and 'target' not in name and 'fc_' not in name and 'transpose_graph' not in name: | |
| if 'featuremap_2_graph' in name: | |
| name = name.replace('featuremap_2_graph','source_featuremap_2_graph') | |
| else: | |
| name = name.replace('graph','source_graph') | |
| new_state_dict[name] = 0 | |
| if name not in own_state: | |
| if 'num_batch' in name: | |
| continue | |
| print('unexpected key "{}" in state_dict' | |
| .format(name)) | |
| continue | |
| # if isinstance(param, own_state): | |
| if isinstance(param, Parameter): | |
| # backwards compatibility for serialized parameters | |
| param = param.data | |
| try: | |
| own_state[name].copy_(param) | |
| except: | |
| print('While copying the parameter named {}, whose dimensions in the model are' | |
| ' {} and whose dimensions in the checkpoint are {}, ...'.format( | |
| name, own_state[name].size(), param.size())) | |
| continue # i add inshop_cos 2018/02/01 | |
| own_state[name].copy_(param) | |
| # print 'copying %s' %name | |
| missing = set(own_state.keys()) - set(new_state_dict.keys()) | |
| if len(missing) > 0: | |
| print('missing keys in state_dict: "{}"'.format(missing)) | |
| class deeplab_xception_transfer_projection_savemem(deeplab_xception_transfer_basemodel_savememory): | |
| def __init__(self, nInputChannels=3, n_classes=7, os=16,input_channels=256,hidden_layers=128,out_channels=256, | |
| transfer_graph=None, source_classes=20): | |
| super(deeplab_xception_transfer_projection_savemem, self).__init__(nInputChannels=nInputChannels, n_classes=n_classes, | |
| os=os, input_channels=input_channels, | |
| hidden_layers=hidden_layers, out_channels=out_channels, ) | |
| self.source_featuremap_2_graph = gcn.Featuremaps_to_Graph(input_channels=input_channels, hidden_layers=hidden_layers, | |
| nodes=source_classes) | |
| self.source_graph_conv1 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| self.source_graph_conv2 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| self.source_graph_conv3 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| self.transpose_graph = gcn.Graph_trans(in_features=hidden_layers,out_features=hidden_layers,adj=transfer_graph, | |
| begin_nodes=source_classes,end_nodes=n_classes) | |
| self.fc_graph = gcn.GraphConvolution(hidden_layers*3, hidden_layers) | |
| def forward(self, input,adj1_target=None, adj2_source=None,adj3_transfer=None ): | |
| x, low_level_features = self.xception_features(input) | |
| # print(x.size()) | |
| x1 = self.aspp1(x) | |
| x2 = self.aspp2(x) | |
| x3 = self.aspp3(x) | |
| x4 = self.aspp4(x) | |
| x5 = self.global_avg_pool(x) | |
| x5 = F.upsample(x5, size=x4.size()[2:], mode='bilinear', align_corners=True) | |
| x = torch.cat((x1, x2, x3, x4, x5), dim=1) | |
| x = self.concat_projection_conv1(x) | |
| x = self.concat_projection_bn1(x) | |
| x = self.relu(x) | |
| # print(x.size()) | |
| x = F.upsample(x, size=low_level_features.size()[2:], mode='bilinear', align_corners=True) | |
| low_level_features = self.feature_projection_conv1(low_level_features) | |
| low_level_features = self.feature_projection_bn1(low_level_features) | |
| low_level_features = self.relu(low_level_features) | |
| # print(low_level_features.size()) | |
| # print(x.size()) | |
| x = torch.cat((x, low_level_features), dim=1) | |
| x = self.decoder(x) | |
| ### add graph | |
| # source graph | |
| source_graph = self.source_featuremap_2_graph(x) | |
| source_graph1 = self.source_graph_conv1.forward(source_graph,adj=adj2_source, relu=True) | |
| source_graph2 = self.source_graph_conv2.forward(source_graph1, adj=adj2_source, relu=True) | |
| source_graph3 = self.source_graph_conv2.forward(source_graph2, adj=adj2_source, relu=True) | |
| source_2_target_graph1_v5 = self.transpose_graph.forward(source_graph1, adj=adj3_transfer, relu=True) | |
| source_2_target_graph2_v5 = self.transpose_graph.forward(source_graph2, adj=adj3_transfer, relu=True) | |
| source_2_target_graph3_v5 = self.transpose_graph.forward(source_graph3, adj=adj3_transfer, relu=True) | |
| # target graph | |
| # print('x size',x.size(),adj1.size()) | |
| graph = self.target_featuremap_2_graph(x) | |
| source_2_target_graph1 = self.similarity_trans(source_graph1, graph) | |
| # graph combine 1 | |
| graph = torch.cat((graph,source_2_target_graph1.squeeze(0), source_2_target_graph1_v5.squeeze(0)),dim=-1) | |
| graph = self.fc_graph.forward(graph,relu=True) | |
| graph = self.target_graph_conv1.forward(graph, adj=adj1_target, relu=True) | |
| source_2_target_graph2 = self.similarity_trans(source_graph2, graph) | |
| # graph combine 2 | |
| graph = torch.cat((graph, source_2_target_graph2, source_2_target_graph2_v5), dim=-1) | |
| graph = self.fc_graph.forward(graph, relu=True) | |
| graph = self.target_graph_conv2.forward(graph, adj=adj1_target, relu=True) | |
| source_2_target_graph3 = self.similarity_trans(source_graph3, graph) | |
| # graph combine 3 | |
| graph = torch.cat((graph, source_2_target_graph3, source_2_target_graph3_v5), dim=-1) | |
| graph = self.fc_graph.forward(graph, relu=True) | |
| graph = self.target_graph_conv3.forward(graph, adj=adj1_target, relu=True) | |
| # print(graph.size(),x.size()) | |
| graph = self.target_graph_2_fea.forward(graph, x) | |
| x = self.target_skip_conv(x) | |
| x = x + graph | |
| ### | |
| x = self.semantic(x) | |
| x = F.upsample(x, size=input.size()[2:], mode='bilinear', align_corners=True) | |
| return x | |
| def similarity_trans(self,source,target): | |
| sim = torch.matmul(F.normalize(target, p=2, dim=-1), F.normalize(source, p=2, dim=-1).transpose(-1, -2)) | |
| sim = F.softmax(sim, dim=-1) | |
| return torch.matmul(sim, source) | |
| def load_source_model(self,state_dict): | |
| own_state = self.state_dict() | |
| # for name inshop_cos own_state: | |
| # print name | |
| new_state_dict = OrderedDict() | |
| for name, param in state_dict.items(): | |
| name = name.replace('module.', '') | |
| if 'graph' in name and 'source' not in name and 'target' not in name and 'fc_' not in name and 'transpose_graph' not in name: | |
| if 'featuremap_2_graph' in name: | |
| name = name.replace('featuremap_2_graph','source_featuremap_2_graph') | |
| else: | |
| name = name.replace('graph','source_graph') | |
| new_state_dict[name] = 0 | |
| if name not in own_state: | |
| if 'num_batch' in name: | |
| continue | |
| print('unexpected key "{}" in state_dict' | |
| .format(name)) | |
| continue | |
| # if isinstance(param, own_state): | |
| if isinstance(param, Parameter): | |
| # backwards compatibility for serialized parameters | |
| param = param.data | |
| try: | |
| own_state[name].copy_(param) | |
| except: | |
| print('While copying the parameter named {}, whose dimensions in the model are' | |
| ' {} and whose dimensions in the checkpoint are {}, ...'.format( | |
| name, own_state[name].size(), param.size())) | |
| continue # i add inshop_cos 2018/02/01 | |
| own_state[name].copy_(param) | |
| # print 'copying %s' %name | |
| missing = set(own_state.keys()) - set(new_state_dict.keys()) | |
| if len(missing) > 0: | |
| print('missing keys in state_dict: "{}"'.format(missing)) | |
| class deeplab_xception_transfer_projection_synBN_savemem(deeplab_xception_transfer_basemodel_synBN_savememory): | |
| def __init__(self, nInputChannels=3, n_classes=7, os=16,input_channels=256,hidden_layers=128,out_channels=256, | |
| transfer_graph=None, source_classes=20): | |
| super(deeplab_xception_transfer_projection_synBN_savemem, self).__init__(nInputChannels=nInputChannels, n_classes=n_classes, | |
| os=os, input_channels=input_channels, | |
| hidden_layers=hidden_layers, out_channels=out_channels, ) | |
| self.source_featuremap_2_graph = gcn.Featuremaps_to_Graph(input_channels=input_channels, hidden_layers=hidden_layers, | |
| nodes=source_classes) | |
| self.source_graph_conv1 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| self.source_graph_conv2 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| self.source_graph_conv3 = gcn.GraphConvolution(hidden_layers, hidden_layers) | |
| self.transpose_graph = gcn.Graph_trans(in_features=hidden_layers,out_features=hidden_layers,adj=transfer_graph, | |
| begin_nodes=source_classes,end_nodes=n_classes) | |
| self.fc_graph = gcn.GraphConvolution(hidden_layers*3 ,hidden_layers) | |
| def forward(self, input,adj1_target=None, adj2_source=None,adj3_transfer=None ): | |
| x, low_level_features = self.xception_features(input) | |
| # print(x.size()) | |
| x1 = self.aspp1(x) | |
| x2 = self.aspp2(x) | |
| x3 = self.aspp3(x) | |
| x4 = self.aspp4(x) | |
| x5 = self.global_avg_pool(x) | |
| x5 = F.upsample(x5, size=x4.size()[2:], mode='bilinear', align_corners=True) | |
| x = torch.cat((x1, x2, x3, x4, x5), dim=1) | |
| x = self.concat_projection_conv1(x) | |
| x = self.concat_projection_bn1(x) | |
| x = self.relu(x) | |
| # print(x.size()) | |
| x = F.upsample(x, size=low_level_features.size()[2:], mode='bilinear', align_corners=True) | |
| low_level_features = self.feature_projection_conv1(low_level_features) | |
| low_level_features = self.feature_projection_bn1(low_level_features) | |
| low_level_features = self.relu(low_level_features) | |
| # print(low_level_features.size()) | |
| # print(x.size()) | |
| x = torch.cat((x, low_level_features), dim=1) | |
| x = self.decoder(x) | |
| ### add graph | |
| # source graph | |
| source_graph = self.source_featuremap_2_graph(x) | |
| source_graph1 = self.source_graph_conv1.forward(source_graph,adj=adj2_source, relu=True) | |
| source_graph2 = self.source_graph_conv2.forward(source_graph1, adj=adj2_source, relu=True) | |
| source_graph3 = self.source_graph_conv2.forward(source_graph2, adj=adj2_source, relu=True) | |
| source_2_target_graph1_v5 = self.transpose_graph.forward(source_graph1, adj=adj3_transfer, relu=True) | |
| source_2_target_graph2_v5 = self.transpose_graph.forward(source_graph2, adj=adj3_transfer, relu=True) | |
| source_2_target_graph3_v5 = self.transpose_graph.forward(source_graph3, adj=adj3_transfer, relu=True) | |
| # target graph | |
| # print('x size',x.size(),adj1.size()) | |
| graph = self.target_featuremap_2_graph(x) | |
| source_2_target_graph1 = self.similarity_trans(source_graph1, graph) | |
| # graph combine 1 | |
| graph = torch.cat((graph,source_2_target_graph1.squeeze(0), source_2_target_graph1_v5.squeeze(0)),dim=-1) | |
| graph = self.fc_graph.forward(graph,relu=True) | |
| graph = self.target_graph_conv1.forward(graph, adj=adj1_target, relu=True) | |
| source_2_target_graph2 = self.similarity_trans(source_graph2, graph) | |
| # graph combine 2 | |
| graph = torch.cat((graph, source_2_target_graph2, source_2_target_graph2_v5), dim=-1) | |
| graph = self.fc_graph.forward(graph, relu=True) | |
| graph = self.target_graph_conv2.forward(graph, adj=adj1_target, relu=True) | |
| source_2_target_graph3 = self.similarity_trans(source_graph3, graph) | |
| # graph combine 3 | |
| graph = torch.cat((graph, source_2_target_graph3, source_2_target_graph3_v5), dim=-1) | |
| graph = self.fc_graph.forward(graph, relu=True) | |
| graph = self.target_graph_conv3.forward(graph, adj=adj1_target, relu=True) | |
| # print(graph.size(),x.size()) | |
| graph = self.target_graph_2_fea.forward(graph, x) | |
| x = self.target_skip_conv(x) | |
| x = x + graph | |
| ### | |
| x = self.semantic(x) | |
| x = F.upsample(x, size=input.size()[2:], mode='bilinear', align_corners=True) | |
| return x | |
| def similarity_trans(self,source,target): | |
| sim = torch.matmul(F.normalize(target, p=2, dim=-1), F.normalize(source, p=2, dim=-1).transpose(-1, -2)) | |
| sim = F.softmax(sim, dim=-1) | |
| return torch.matmul(sim, source) | |
| def load_source_model(self,state_dict): | |
| own_state = self.state_dict() | |
| # for name inshop_cos own_state: | |
| # print name | |
| new_state_dict = OrderedDict() | |
| for name, param in state_dict.items(): | |
| name = name.replace('module.', '') | |
| if 'graph' in name and 'source' not in name and 'target' not in name and 'fc_' not in name and 'transpose_graph' not in name: | |
| if 'featuremap_2_graph' in name: | |
| name = name.replace('featuremap_2_graph','source_featuremap_2_graph') | |
| else: | |
| name = name.replace('graph','source_graph') | |
| new_state_dict[name] = 0 | |
| if name not in own_state: | |
| if 'num_batch' in name: | |
| continue | |
| print('unexpected key "{}" in state_dict' | |
| .format(name)) | |
| continue | |
| # if isinstance(param, own_state): | |
| if isinstance(param, Parameter): | |
| # backwards compatibility for serialized parameters | |
| param = param.data | |
| try: | |
| own_state[name].copy_(param) | |
| except: | |
| print('While copying the parameter named {}, whose dimensions in the model are' | |
| ' {} and whose dimensions in the checkpoint are {}, ...'.format( | |
| name, own_state[name].size(), param.size())) | |
| continue # i add inshop_cos 2018/02/01 | |
| own_state[name].copy_(param) | |
| # print 'copying %s' %name | |
| missing = set(own_state.keys()) - set(new_state_dict.keys()) | |
| if len(missing) > 0: | |
| print('missing keys in state_dict: "{}"'.format(missing)) | |
| # if __name__ == '__main__': | |
| # net = deeplab_xception_transfer_projection_v3v5_more_savemem() | |
| # img = torch.rand((2,3,128,128)) | |
| # net.eval() | |
| # a = torch.rand((1,1,7,7)) | |
| # net.forward(img, adj1_target=a) |