|
|
@ -18,27 +18,20 @@ class StdConv2d(nn.Conv2d):
|
|
|
|
https://arxiv.org/abs/1903.10520v2
|
|
|
|
https://arxiv.org/abs/1903.10520v2
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
def __init__(
|
|
|
|
def __init__(
|
|
|
|
self, in_channel, out_channels, kernel_size, stride=1, padding=None, dilation=1,
|
|
|
|
self, in_channel, out_channels, kernel_size, stride=1, padding=None,
|
|
|
|
groups=1, bias=False, eps=1e-5, use_layernorm=True):
|
|
|
|
dilation=1, groups=1, bias=False, eps=1e-6):
|
|
|
|
if padding is None:
|
|
|
|
if padding is None:
|
|
|
|
padding = get_padding(kernel_size, stride, dilation)
|
|
|
|
padding = get_padding(kernel_size, stride, dilation)
|
|
|
|
super().__init__(
|
|
|
|
super().__init__(
|
|
|
|
in_channel, out_channels, kernel_size, stride=stride,
|
|
|
|
in_channel, out_channels, kernel_size, stride=stride,
|
|
|
|
padding=padding, dilation=dilation, groups=groups, bias=bias)
|
|
|
|
padding=padding, dilation=dilation, groups=groups, bias=bias)
|
|
|
|
self.eps = eps
|
|
|
|
self.eps = eps
|
|
|
|
self.use_layernorm = use_layernorm
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_weight(self):
|
|
|
|
|
|
|
|
if self.use_layernorm:
|
|
|
|
|
|
|
|
# NOTE F.layer_norm is being used to compute (self.weight - mean) / (sqrt(var) + self.eps) in one op
|
|
|
|
|
|
|
|
weight = F.layer_norm(self.weight, self.weight.shape[1:], eps=self.eps)
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
std, mean = torch.std_mean(self.weight, dim=[1, 2, 3], keepdim=True, unbiased=False)
|
|
|
|
|
|
|
|
weight = (self.weight - mean) / (std + self.eps)
|
|
|
|
|
|
|
|
return weight
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def forward(self, x):
|
|
|
|
def forward(self, x):
|
|
|
|
x = F.conv2d(x, self.get_weight(), self.bias, self.stride, self.padding, self.dilation, self.groups)
|
|
|
|
weight = F.batch_norm(
|
|
|
|
|
|
|
|
self.weight.view(1, self.out_channels, -1), None, None,
|
|
|
|
|
|
|
|
eps=self.eps, training=True, momentum=0.).reshape_as(self.weight)
|
|
|
|
|
|
|
|
x = F.conv2d(x, weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
|
|
|
|
return x
|
|
|
|
return x
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -49,29 +42,22 @@ class StdConv2dSame(nn.Conv2d):
|
|
|
|
https://arxiv.org/abs/1903.10520v2
|
|
|
|
https://arxiv.org/abs/1903.10520v2
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
def __init__(
|
|
|
|
def __init__(
|
|
|
|
self, in_channel, out_channels, kernel_size, stride=1, padding='SAME', dilation=1,
|
|
|
|
self, in_channel, out_channels, kernel_size, stride=1, padding='SAME',
|
|
|
|
groups=1, bias=False, eps=1e-5, use_layernorm=True):
|
|
|
|
dilation=1, groups=1, bias=False, eps=1e-6):
|
|
|
|
padding, is_dynamic = get_padding_value(padding, kernel_size, stride=stride, dilation=dilation)
|
|
|
|
padding, is_dynamic = get_padding_value(padding, kernel_size, stride=stride, dilation=dilation)
|
|
|
|
super().__init__(
|
|
|
|
super().__init__(
|
|
|
|
in_channel, out_channels, kernel_size, stride=stride, padding=padding, dilation=dilation,
|
|
|
|
in_channel, out_channels, kernel_size, stride=stride, padding=padding, dilation=dilation,
|
|
|
|
groups=groups, bias=bias)
|
|
|
|
groups=groups, bias=bias)
|
|
|
|
self.same_pad = is_dynamic
|
|
|
|
self.same_pad = is_dynamic
|
|
|
|
self.eps = eps
|
|
|
|
self.eps = eps
|
|
|
|
self.use_layernorm = use_layernorm
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_weight(self):
|
|
|
|
|
|
|
|
if self.use_layernorm:
|
|
|
|
|
|
|
|
# NOTE F.layer_norm is being used to compute (self.weight - mean) / (sqrt(var) + self.eps) in one op
|
|
|
|
|
|
|
|
weight = F.layer_norm(self.weight, self.weight.shape[1:], eps=self.eps)
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
std, mean = torch.std_mean(self.weight, dim=[1, 2, 3], keepdim=True, unbiased=False)
|
|
|
|
|
|
|
|
weight = (self.weight - mean) / (std + self.eps)
|
|
|
|
|
|
|
|
return weight
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def forward(self, x):
|
|
|
|
def forward(self, x):
|
|
|
|
if self.same_pad:
|
|
|
|
if self.same_pad:
|
|
|
|
x = pad_same(x, self.kernel_size, self.stride, self.dilation)
|
|
|
|
x = pad_same(x, self.kernel_size, self.stride, self.dilation)
|
|
|
|
x = F.conv2d(x, self.get_weight(), self.bias, self.stride, self.padding, self.dilation, self.groups)
|
|
|
|
weight = F.batch_norm(
|
|
|
|
|
|
|
|
self.weight.view(1, self.out_channels, -1), None, None,
|
|
|
|
|
|
|
|
eps=self.eps, training=True, momentum=0.).reshape_as(self.weight)
|
|
|
|
|
|
|
|
x = F.conv2d(x, weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
|
|
|
|
return x
|
|
|
|
return x
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -85,8 +71,8 @@ class ScaledStdConv2d(nn.Conv2d):
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
|
def __init__(
|
|
|
|
def __init__(
|
|
|
|
self, in_channels, out_channels, kernel_size, stride=1, padding=None, dilation=1, groups=1,
|
|
|
|
self, in_channels, out_channels, kernel_size, stride=1, padding=None,
|
|
|
|
bias=True, gamma=1.0, eps=1e-5, gain_init=1.0, use_layernorm=True):
|
|
|
|
dilation=1, groups=1, bias=True, gamma=1.0, eps=1e-6, gain_init=1.0):
|
|
|
|
if padding is None:
|
|
|
|
if padding is None:
|
|
|
|
padding = get_padding(kernel_size, stride, dilation)
|
|
|
|
padding = get_padding(kernel_size, stride, dilation)
|
|
|
|
super().__init__(
|
|
|
|
super().__init__(
|
|
|
@ -95,19 +81,13 @@ class ScaledStdConv2d(nn.Conv2d):
|
|
|
|
self.gain = nn.Parameter(torch.full((self.out_channels, 1, 1, 1), gain_init))
|
|
|
|
self.gain = nn.Parameter(torch.full((self.out_channels, 1, 1, 1), gain_init))
|
|
|
|
self.scale = gamma * self.weight[0].numel() ** -0.5 # gamma * 1 / sqrt(fan-in)
|
|
|
|
self.scale = gamma * self.weight[0].numel() ** -0.5 # gamma * 1 / sqrt(fan-in)
|
|
|
|
self.eps = eps
|
|
|
|
self.eps = eps
|
|
|
|
self.use_layernorm = use_layernorm # experimental, slightly faster/less GPU memory to hijack LN kernel
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_weight(self):
|
|
|
|
|
|
|
|
if self.use_layernorm:
|
|
|
|
|
|
|
|
# NOTE F.layer_norm is being used to compute (self.weight - mean) / (sqrt(var) + self.eps) in one op
|
|
|
|
|
|
|
|
weight = F.layer_norm(self.weight, self.weight.shape[1:], eps=self.eps)
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
std, mean = torch.std_mean(self.weight, dim=[1, 2, 3], keepdim=True, unbiased=False)
|
|
|
|
|
|
|
|
weight = (self.weight - mean) / (std + self.eps)
|
|
|
|
|
|
|
|
return weight.mul_(self.gain * self.scale)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def forward(self, x):
|
|
|
|
def forward(self, x):
|
|
|
|
return F.conv2d(x, self.get_weight(), self.bias, self.stride, self.padding, self.dilation, self.groups)
|
|
|
|
weight = F.batch_norm(
|
|
|
|
|
|
|
|
self.weight.view(1, self.out_channels, -1), None, None,
|
|
|
|
|
|
|
|
weight=(self.gain * self.scale).view(-1),
|
|
|
|
|
|
|
|
eps=self.eps, training=True, momentum=0.).reshape_as(self.weight)
|
|
|
|
|
|
|
|
return F.conv2d(x, weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class ScaledStdConv2dSame(nn.Conv2d):
|
|
|
|
class ScaledStdConv2dSame(nn.Conv2d):
|
|
|
@ -120,8 +100,8 @@ class ScaledStdConv2dSame(nn.Conv2d):
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
|
def __init__(
|
|
|
|
def __init__(
|
|
|
|
self, in_channels, out_channels, kernel_size, stride=1, padding='SAME', dilation=1, groups=1,
|
|
|
|
self, in_channels, out_channels, kernel_size, stride=1, padding='SAME',
|
|
|
|
bias=True, gamma=1.0, eps=1e-5, gain_init=1.0, use_layernorm=True):
|
|
|
|
dilation=1, groups=1, bias=True, gamma=1.0, eps=1e-6, gain_init=1.0):
|
|
|
|
padding, is_dynamic = get_padding_value(padding, kernel_size, stride=stride, dilation=dilation)
|
|
|
|
padding, is_dynamic = get_padding_value(padding, kernel_size, stride=stride, dilation=dilation)
|
|
|
|
super().__init__(
|
|
|
|
super().__init__(
|
|
|
|
in_channels, out_channels, kernel_size, stride=stride, padding=padding, dilation=dilation,
|
|
|
|
in_channels, out_channels, kernel_size, stride=stride, padding=padding, dilation=dilation,
|
|
|
@ -130,18 +110,12 @@ class ScaledStdConv2dSame(nn.Conv2d):
|
|
|
|
self.scale = gamma * self.weight[0].numel() ** -0.5
|
|
|
|
self.scale = gamma * self.weight[0].numel() ** -0.5
|
|
|
|
self.same_pad = is_dynamic
|
|
|
|
self.same_pad = is_dynamic
|
|
|
|
self.eps = eps
|
|
|
|
self.eps = eps
|
|
|
|
self.use_layernorm = use_layernorm # experimental, slightly faster/less GPU memory to hijack LN kernel
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_weight(self):
|
|
|
|
|
|
|
|
if self.use_layernorm:
|
|
|
|
|
|
|
|
# NOTE F.layer_norm is being used to compute (self.weight - mean) / (sqrt(var) + self.eps) in one op
|
|
|
|
|
|
|
|
weight = F.layer_norm(self.weight, self.weight.shape[1:], eps=self.eps)
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
|
|
|
std, mean = torch.std_mean(self.weight, dim=[1, 2, 3], keepdim=True, unbiased=False)
|
|
|
|
|
|
|
|
weight = (self.weight - mean) / (std + self.eps)
|
|
|
|
|
|
|
|
return weight.mul_(self.gain * self.scale)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def forward(self, x):
|
|
|
|
def forward(self, x):
|
|
|
|
if self.same_pad:
|
|
|
|
if self.same_pad:
|
|
|
|
x = pad_same(x, self.kernel_size, self.stride, self.dilation)
|
|
|
|
x = pad_same(x, self.kernel_size, self.stride, self.dilation)
|
|
|
|
return F.conv2d(x, self.get_weight(), self.bias, self.stride, self.padding, self.dilation, self.groups)
|
|
|
|
weight = F.batch_norm(
|
|
|
|
|
|
|
|
self.weight.view(1, self.out_channels, -1), None, None,
|
|
|
|
|
|
|
|
weight=(self.gain * self.scale).view(-1),
|
|
|
|
|
|
|
|
eps=self.eps, training=True, momentum=0.).reshape_as(self.weight)
|
|
|
|
|
|
|
|
return F.conv2d(x, weight, self.bias, self.stride, self.padding, self.dilation, self.groups)
|
|
|
|