From 53f992723cd36498814ffa6d608cb16f9faa6d77 Mon Sep 17 00:00:00 2001
From: Fredo Guan <fredo.guan@hotmail.com>
Date: Wed, 18 Jan 2023 12:25:07 -0800
Subject: [PATCH] Update metaformers.py

---
 timm/models/metaformers.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/timm/models/metaformers.py b/timm/models/metaformers.py
index 7a41cf2a..5b98cba8 100644
--- a/timm/models/metaformers.py
+++ b/timm/models/metaformers.py
@@ -175,12 +175,16 @@ class Attention(nn.Module):
 class RandomMixing(nn.Module):
     def __init__(self, num_tokens=196, **kwargs):
         super().__init__()
+        '''
         self.random_matrix = nn.parameter.Parameter(
             data=torch.softmax(torch.rand(num_tokens, num_tokens), dim=-1), 
             requires_grad=False)
+        '''
+        self.random_matrix = torch.softmax(torch.rand(num_tokens, num_tokens)
     def forward(self, x):
         B, H, W, C = x.shape
         x = x.reshape(B, H*W, C)
+        # FIXME change to work with arbitrary input sizes
         x = torch.einsum('mn, bnc -> bmc', self.random_matrix, x)
         x = x.reshape(B, H, W, C)
         return x