Create new file

transformer.py

@@ -0,0 +1,207 @@
+import torch.nn as nn
+import torch
+from einops import rearrange
+import numpy
+def index_points(device, points, idx):
+    """
+
+    Input:
+        points: input points data, [B, N, C]
+        idx: sample index data, [B, S]
+    Return:
+        new_points:, indexed points data, [B, S, C]
+    """
+    B = points.shape[0]
+    view_shape = list(idx.shape)
+    view_shape[1:] = [1] * (len(view_shape) - 1)
+    repeat_shape = list(idx.shape)
+    repeat_shape[0] = 1
+    # batch_indices = torch.arange(B, dtype=torch.long).to(device).view(view_shape).repeat(repeat_shape)
+    batch_indices = torch.arange(B, dtype=torch.long).cuda().view(view_shape).repeat(repeat_shape)
+    new_points = points[batch_indices, idx, :]
+    return new_points
+
+def knn_l2(device, net, k, u):
+    '''
+    Input:
+        k: int32, number of k in k-nn search
+        net: (batch_size, npoint, c) float32 array, points
+        u: int32, block size
+    Output:
+        idx: (batch_size, npoint, k) int32 array, indices to input points
+    '''
+    INF = 1e8
+    batch_size = net.size(0)
+    npoint = net.size(1)
+    n_channel = net.size(2)
+
+    square = torch.pow(torch.norm(net, dim=2,keepdim=True),2)
+
+    def u_block(batch_size, npoint, u):
+        block = numpy.zeros([batch_size, npoint, npoint])
+        n = npoint // u
+        for i in range(n):
+            block[:, (i*u):(i*u+u), (i*u):(i*u+u)] = numpy.ones([batch_size, u, u]) * (-INF)
+        return block
+
+    # minus_distance = 2 * torch.matmul(net, net.transpose(2,1)) - square - square.transpose(2,1) + torch.Tensor(u_block(batch_size, npoint, u)).to(device)
+    minus_distance = 2 * torch.matmul(net, net.transpose(2,1)) - square - square.transpose(2,1) + torch.Tensor(u_block(batch_size, npoint, u)).cuda()
+    _, indices = torch.topk(minus_distance, k, largest=True, sorted=False)
+    
+    return indices
+
+class Residual(nn.Module):
+    def __init__(self, fn):
+        super().__init__()
+        self.fn = fn
+    def forward(self, x, **kwargs):
+        return self.fn(x, **kwargs) + x
+
+class PreNorm(nn.Module):
+    def __init__(self, dim, fn):
+        super().__init__()
+        self.norm = nn.LayerNorm(dim)
+        self.fn = fn
+    def forward(self, x, **kwargs):
+        return self.fn(self.norm(x), **kwargs)
+
+class FeedForward(nn.Module):
+    def __init__(self, dim, hidden_dim, dropout = 0.):
+        super().__init__()
+        self.net = nn.Sequential(
+            nn.Linear(dim, hidden_dim),
+            nn.GELU(),
+            nn.Dropout(dropout),
+            nn.Linear(hidden_dim, dim),
+            nn.Dropout(dropout)
+        )
+    def forward(self, x):
+        return self.net(x)
+
+class Attention(nn.Module):
+    def __init__(self, dim, heads = 4, dropout = 0.):
+        super().__init__()
+        self.heads = heads
+        self.scale = dim ** -0.5
+
+        self.to_qkv = nn.Linear(dim, dim * 3, bias = False)
+        self.to_out = nn.Sequential(
+            nn.Linear(dim, dim),
+            nn.Dropout(dropout)
+        )
+
+    def forward(self, x, mask = None):
+        b, n, _, h = *x.shape, self.heads
+        qkv = self.to_qkv(x).chunk(3, dim = -1)
+        q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = h), qkv)
+
+        dots = torch.einsum('bhid,bhjd->bhij', q, k) * self.scale
+
+        if mask is not None:
+            mask = F.pad(mask.flatten(1), (1, 0), value = True)
+            assert mask.shape[-1] == dots.shape[-1], 'mask has incorrect dimensions'
+            mask = mask[:, None, :] * mask[:, :, None]
+            dots.masked_fill_(~mask, float('-inf'))
+            del mask
+
+        attn = dots.softmax(dim=-1)
+
+        out = torch.einsum('bhij,bhjd->bhid', attn, v)
+        out = rearrange(out, 'b h n d -> b n (h d)')
+        out =  self.to_out(out)
+        return out
+        
+class Local_Attention(nn.Module):
+    def __init__(self, dim, heads = 4,knn=4, dropout = 0.):
+        super().__init__()
+        self.heads = heads
+        self.scale = dim ** -0.5
+
+        #self.to_qkv = nn.Linear(dim, dim * 3, bias = False)
+        self.q=nn.Linear(dim,dim,bias=False)
+        self.k=nn.Linear(dim,dim,bias=False)
+        self.v=nn.Linear(dim,dim,bias=False)
+        
+        self.to_out = nn.Sequential(
+            nn.Linear(dim, dim),
+            nn.Dropout(dropout)
+        )
+        self.knn=knn
+  
+    def forward(self, x, mask = None):
+        b, n, _, h = *x.shape, self.heads
+        
+        point=x*1
+        X=x*1
+        
+        idx = knn_l2(0, point.permute(0,2,1), 4, 1)
+        feat=idx
+        new_point = index_points(0, point.permute(0,2,1),idx)
+
+        group_point = new_point.permute(0, 3, 2, 1)
+        
+        _1,_2,_3,_4=group_point.shape
+        
+        q=self.q(X.reshape(_1*_2,1,_4))
+        k=self.k(torch.cat([group_point.reshape(_1*_2,self.knn,_4),X.reshape(_1*_2,1,_4)],dim=1))
+        v=self.v(torch.cat([group_point.reshape(_1*_2,self.knn,_4),X.reshape(_1*_2,1,_4)],dim=1))
+        q, k, v = rearrange(q, 'b n (h d) -> b h n d', h = h),rearrange(k, 'b n (h d) -> b h n d', h = h),rearrange(v, 'b n (h d) -> b h n d', h = h)
+        
+        [email protected](0,1,3,2)*self.scale
+        attn_map=attn_map.softmax(dim=-1)
+        
+        out=attn_map@v
+        out=out.view(b,out.shape[0]//b,out.shape[1],out.shape[3]).permute(0,2,1,3)
+        
+        out = rearrange(out, 'b h n d -> b n (h d)')
+        out =  self.to_out(out)
+        return out
+    
+    
+class Transformer(nn.Module):
+    def __init__(self, dim, depth, heads, mlp_dim, group=5, dropout=0.):
+        super().__init__()
+        self.layers = nn.ModuleList([])
+        for _ in range(depth):
+            self.layers.append(nn.ModuleList([
+                Residual(PreNorm(dim, Attention(dim, heads = heads, dropout = dropout))),
+                Residual(PreNorm(dim, FeedForward(dim, mlp_dim, dropout = dropout)))
+            ]))
+        self.group=group
+    def forward(self, x, mask = None):
+        bs_gp,dim,wid,hei=x.shape[0],x.shape[1],x.shape[2],x.shape[3]
+        bs=bs_gp//self.group
+        gp=self.group
+        x=x.reshape(bs,gp,dim,wid,hei)
+        x=x.permute(0,1,3,4,2).reshape(bs,gp*wid*hei,dim)
+        for attn, ff in self.layers:
+            x = attn(x, mask = mask)
+            x = ff(x)
+
+        x=x.reshape(bs,gp,wid,hei,dim).permute(0,1,4,2,3).reshape(bs_gp,dim,wid,hei)
+        
+        return x
+        
+class Transformer__local(nn.Module):
+    def __init__(self, dim, depth, heads, mlp_dim,knn_k=4, group=5, dropout=0.):
+        super().__init__()
+        self.layers = nn.ModuleList([])
+        for _ in range(depth):
+            self.layers.append(nn.ModuleList([
+                Residual(PreNorm(dim, Local_Attention(dim, heads = heads,knn=knn_k, dropout = dropout))),
+                Residual(PreNorm(dim, FeedForward(dim, mlp_dim, dropout = dropout)))
+            ]))
+        self.group=group
+    def forward(self, x, mask = None):
+        bs_gp,dim,wid,hei=x.shape[0],x.shape[1],x.shape[2],x.shape[3]
+        bs=bs_gp//self.group
+        gp=self.group
+        x=x.reshape(bs,gp,dim,wid,hei)
+        x=x.permute(0,1,3,4,2).reshape(bs,gp*wid*hei,dim)
+        for attn, ff in self.layers:
+            x = attn(x, mask = mask)
+            x = ff(x)
+
+        x=x.reshape(bs,gp,wid,hei,dim).permute(0,1,4,2,3).reshape(bs_gp,dim,wid,hei)
+        
+        return x