commit
f54612f648
@ -0,0 +1,77 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
import numpy as np
|
||||
import math
|
||||
|
||||
|
||||
def drop_block_2d(x, drop_prob=0.1, block_size=7, gamma_scale=1.0, drop_with_noise=False):
|
||||
_, _, height, width = x.shape
|
||||
total_size = width * height
|
||||
clipped_block_size = min(block_size, min(width, height))
|
||||
# seed_drop_rate, the gamma parameter
|
||||
seed_drop_rate = gamma_scale * drop_prob * total_size / clipped_block_size ** 2 / (
|
||||
(width - block_size + 1) *
|
||||
(height - block_size + 1))
|
||||
|
||||
# Forces the block to be inside the feature map.
|
||||
w_i, h_i = torch.meshgrid(torch.arange(width).to(x.device), torch.arange(height).to(x.device))
|
||||
valid_block = ((w_i >= clipped_block_size // 2) & (w_i < width - (clipped_block_size - 1) // 2)) & \
|
||||
((h_i >= clipped_block_size // 2) & (h_i < height - (clipped_block_size - 1) // 2))
|
||||
valid_block = torch.reshape(valid_block, (1, 1, height, width)).float()
|
||||
|
||||
uniform_noise = torch.rand_like(x, dtype=torch.float32)
|
||||
block_mask = ((2 - seed_drop_rate - valid_block + uniform_noise) >= 1).float()
|
||||
block_mask = -F.max_pool2d(
|
||||
-block_mask,
|
||||
kernel_size=clipped_block_size, # block_size,
|
||||
stride=1,
|
||||
padding=clipped_block_size // 2)
|
||||
|
||||
if drop_with_noise:
|
||||
normal_noise = torch.randn_like(x)
|
||||
x = x * block_mask + normal_noise * (1 - block_mask)
|
||||
else:
|
||||
normalize_scale = block_mask.numel() / (torch.sum(block_mask) + 1e-7)
|
||||
x = x * block_mask * normalize_scale
|
||||
return x
|
||||
|
||||
|
||||
class DropBlock2d(nn.Module):
|
||||
""" DropBlock. See https://arxiv.org/pdf/1810.12890.pdf
|
||||
"""
|
||||
def __init__(self,
|
||||
drop_prob=0.1,
|
||||
block_size=7,
|
||||
gamma_scale=1.0,
|
||||
with_noise=False):
|
||||
super(DropBlock2d, self).__init__()
|
||||
self.drop_prob = drop_prob
|
||||
self.gamma_scale = gamma_scale
|
||||
self.block_size = block_size
|
||||
self.with_noise = with_noise
|
||||
|
||||
def forward(self, x):
|
||||
if not self.training or not self.drop_prob:
|
||||
return x
|
||||
return drop_block_2d(x, self.drop_prob, self.block_size, self.gamma_scale, self.with_noise)
|
||||
|
||||
|
||||
def drop_path(x, drop_prob=0.):
|
||||
"""Drop paths (Stochastic Depth) per sample (when applied in residual blocks)."""
|
||||
keep_prob = 1 - drop_prob
|
||||
random_tensor = keep_prob + torch.rand((x.size()[0], 1, 1, 1), dtype=x.dtype, device=x.device)
|
||||
random_tensor.floor_() # binarize
|
||||
output = x.div(keep_prob) * random_tensor
|
||||
return output
|
||||
|
||||
|
||||
class DropPath(nn.ModuleDict):
|
||||
def __init__(self, drop_prob=None):
|
||||
super(DropPath, self).__init__()
|
||||
self.drop_prob = drop_prob
|
||||
|
||||
def forward(self, x):
|
||||
if not self.training or not self.drop_prob:
|
||||
return x
|
||||
return drop_path(x, self.drop_prob)
|
@ -0,0 +1,242 @@
|
||||
import math
|
||||
|
||||
from torch import nn as nn
|
||||
|
||||
from timm.models.registry import register_model
|
||||
from timm.models.helpers import load_pretrained
|
||||
from timm.models.conv2d_layers import SelectiveKernelConv, ConvBnAct
|
||||
from timm.models.resnet import ResNet, SEModule
|
||||
from timm.data import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD
|
||||
|
||||
|
||||
def _cfg(url='', **kwargs):
|
||||
return {
|
||||
'url': url,
|
||||
'num_classes': 1000, 'input_size': (3, 224, 224), 'pool_size': (7, 7),
|
||||
'crop_pct': 0.875, 'interpolation': 'bilinear',
|
||||
'mean': IMAGENET_DEFAULT_MEAN, 'std': IMAGENET_DEFAULT_STD,
|
||||
'first_conv': 'conv1', 'classifier': 'fc',
|
||||
**kwargs
|
||||
}
|
||||
|
||||
|
||||
default_cfgs = {
|
||||
'skresnet18': _cfg(url=''),
|
||||
'skresnet26d': _cfg(),
|
||||
'skresnet50': _cfg(),
|
||||
'skresnet50d': _cfg(),
|
||||
'skresnext50_32x4d': _cfg(),
|
||||
}
|
||||
|
||||
|
||||
class SelectiveKernelBasic(nn.Module):
|
||||
expansion = 1
|
||||
|
||||
def __init__(self, inplanes, planes, stride=1, downsample=None, cardinality=1, base_width=64,
|
||||
use_se=False, sk_kwargs=None, reduce_first=1, dilation=1, first_dilation=None,
|
||||
drop_block=None, drop_path=None, act_layer=nn.ReLU, norm_layer=nn.BatchNorm2d):
|
||||
super(SelectiveKernelBasic, self).__init__()
|
||||
|
||||
sk_kwargs = sk_kwargs or {}
|
||||
conv_kwargs = dict(drop_block=drop_block, act_layer=act_layer, norm_layer=norm_layer)
|
||||
assert cardinality == 1, 'BasicBlock only supports cardinality of 1'
|
||||
assert base_width == 64, 'BasicBlock doest not support changing base width'
|
||||
first_planes = planes // reduce_first
|
||||
out_planes = planes * self.expansion
|
||||
first_dilation = first_dilation or dilation
|
||||
|
||||
_selective_first = True # FIXME temporary, for experiments
|
||||
if _selective_first:
|
||||
self.conv1 = SelectiveKernelConv(
|
||||
inplanes, first_planes, stride=stride, dilation=first_dilation, **conv_kwargs, **sk_kwargs)
|
||||
conv_kwargs['act_layer'] = None
|
||||
self.conv2 = ConvBnAct(
|
||||
first_planes, out_planes, kernel_size=3, dilation=dilation, **conv_kwargs)
|
||||
else:
|
||||
self.conv1 = ConvBnAct(
|
||||
inplanes, first_planes, kernel_size=3, stride=stride, dilation=first_dilation, **conv_kwargs)
|
||||
conv_kwargs['act_layer'] = None
|
||||
self.conv2 = SelectiveKernelConv(
|
||||
first_planes, out_planes, dilation=dilation, **conv_kwargs, **sk_kwargs)
|
||||
self.se = SEModule(out_planes, planes // 4) if use_se else None
|
||||
self.act = act_layer(inplace=True)
|
||||
self.downsample = downsample
|
||||
self.stride = stride
|
||||
self.dilation = dilation
|
||||
self.drop_block = drop_block
|
||||
self.drop_path = drop_path
|
||||
|
||||
def zero_init_last_bn(self):
|
||||
nn.init.zeros_(self.conv2.bn.weight)
|
||||
|
||||
def forward(self, x):
|
||||
residual = x
|
||||
x = self.conv1(x)
|
||||
x = self.conv2(x)
|
||||
if self.se is not None:
|
||||
x = self.se(x)
|
||||
if self.drop_path is not None:
|
||||
x = self.drop_path(x)
|
||||
if self.downsample is not None:
|
||||
residual = self.downsample(residual)
|
||||
x += residual
|
||||
x = self.act(x)
|
||||
return x
|
||||
|
||||
|
||||
class SelectiveKernelBottleneck(nn.Module):
|
||||
expansion = 4
|
||||
|
||||
def __init__(self, inplanes, planes, stride=1, downsample=None,
|
||||
cardinality=1, base_width=64, use_se=False, sk_kwargs=None,
|
||||
reduce_first=1, dilation=1, first_dilation=None,
|
||||
drop_block=None, drop_path=None,
|
||||
act_layer=nn.ReLU, norm_layer=nn.BatchNorm2d):
|
||||
super(SelectiveKernelBottleneck, self).__init__()
|
||||
|
||||
sk_kwargs = sk_kwargs or {}
|
||||
conv_kwargs = dict(drop_block=drop_block, act_layer=act_layer, norm_layer=norm_layer)
|
||||
width = int(math.floor(planes * (base_width / 64)) * cardinality)
|
||||
first_planes = width // reduce_first
|
||||
out_planes = planes * self.expansion
|
||||
first_dilation = first_dilation or dilation
|
||||
|
||||
self.conv1 = ConvBnAct(inplanes, first_planes, kernel_size=1, **conv_kwargs)
|
||||
self.conv2 = SelectiveKernelConv(
|
||||
first_planes, width, stride=stride, dilation=first_dilation, groups=cardinality,
|
||||
**conv_kwargs, **sk_kwargs)
|
||||
conv_kwargs['act_layer'] = None
|
||||
self.conv3 = ConvBnAct(width, out_planes, kernel_size=1, **conv_kwargs)
|
||||
self.se = SEModule(out_planes, planes // 4) if use_se else None
|
||||
self.act = act_layer(inplace=True)
|
||||
self.downsample = downsample
|
||||
self.stride = stride
|
||||
self.dilation = dilation
|
||||
self.drop_block = drop_block
|
||||
self.drop_path = drop_path
|
||||
|
||||
def zero_init_last_bn(self):
|
||||
nn.init.zeros_(self.conv3.bn.weight)
|
||||
|
||||
def forward(self, x):
|
||||
residual = x
|
||||
x = self.conv1(x)
|
||||
x = self.conv2(x)
|
||||
x = self.conv3(x)
|
||||
if self.se is not None:
|
||||
x = self.se(x)
|
||||
if self.drop_path is not None:
|
||||
x = self.drop_path(x)
|
||||
if self.downsample is not None:
|
||||
residual = self.downsample(residual)
|
||||
x += residual
|
||||
x = self.act(x)
|
||||
return x
|
||||
|
||||
|
||||
@register_model
|
||||
def skresnet18(pretrained=False, num_classes=1000, in_chans=3, **kwargs):
|
||||
"""Constructs a ResNet-18 model.
|
||||
"""
|
||||
default_cfg = default_cfgs['skresnet18']
|
||||
sk_kwargs = dict(
|
||||
min_attn_channels=16,
|
||||
)
|
||||
model = ResNet(
|
||||
SelectiveKernelBasic, [2, 2, 2, 2], num_classes=num_classes, in_chans=in_chans,
|
||||
block_args=dict(sk_kwargs=sk_kwargs), **kwargs)
|
||||
model.default_cfg = default_cfg
|
||||
if pretrained:
|
||||
load_pretrained(model, default_cfg, num_classes, in_chans)
|
||||
return model
|
||||
|
||||
|
||||
@register_model
|
||||
def sksresnet18(pretrained=False, num_classes=1000, in_chans=3, **kwargs):
|
||||
"""Constructs a ResNet-18 model.
|
||||
"""
|
||||
default_cfg = default_cfgs['skresnet18']
|
||||
sk_kwargs = dict(
|
||||
min_attn_channels=16,
|
||||
attn_reduction=8,
|
||||
split_input=True
|
||||
)
|
||||
model = ResNet(
|
||||
SelectiveKernelBasic, [2, 2, 2, 2], num_classes=num_classes, in_chans=in_chans,
|
||||
block_args=dict(sk_kwargs=sk_kwargs), zero_init_last_bn=False, **kwargs)
|
||||
model.default_cfg = default_cfg
|
||||
if pretrained:
|
||||
load_pretrained(model, default_cfg, num_classes, in_chans)
|
||||
return model
|
||||
|
||||
|
||||
@register_model
|
||||
def skresnet26d(pretrained=False, num_classes=1000, in_chans=3, **kwargs):
|
||||
"""Constructs a ResNet-26 model.
|
||||
"""
|
||||
default_cfg = default_cfgs['skresnet26d']
|
||||
sk_kwargs = dict(
|
||||
keep_3x3=False,
|
||||
)
|
||||
model = ResNet(
|
||||
SelectiveKernelBottleneck, [2, 2, 2, 2], stem_width=32, stem_type='deep', avg_down=True,
|
||||
num_classes=num_classes, in_chans=in_chans, block_args=dict(sk_kwargs=sk_kwargs), zero_init_last_bn=False
|
||||
**kwargs)
|
||||
model.default_cfg = default_cfg
|
||||
if pretrained:
|
||||
load_pretrained(model, default_cfg, num_classes, in_chans)
|
||||
return model
|
||||
|
||||
|
||||
@register_model
|
||||
def skresnet50(pretrained=False, num_classes=1000, in_chans=3, **kwargs):
|
||||
"""Constructs a Select Kernel ResNet-50 model.
|
||||
Based on config in "Compounding the Performance Improvements of Assembled Techniques in a
|
||||
Convolutional Neural Network"
|
||||
"""
|
||||
sk_kwargs = dict(
|
||||
attn_reduction=2,
|
||||
)
|
||||
default_cfg = default_cfgs['skresnet50']
|
||||
model = ResNet(
|
||||
SelectiveKernelBottleneck, [3, 4, 6, 3], num_classes=num_classes, in_chans=in_chans,
|
||||
block_args=dict(sk_kwargs=sk_kwargs), zero_init_last_bn=False, **kwargs)
|
||||
model.default_cfg = default_cfg
|
||||
if pretrained:
|
||||
load_pretrained(model, default_cfg, num_classes, in_chans)
|
||||
return model
|
||||
|
||||
|
||||
@register_model
|
||||
def skresnet50d(pretrained=False, num_classes=1000, in_chans=3, **kwargs):
|
||||
"""Constructs a Select Kernel ResNet-50-D model.
|
||||
Based on config in "Compounding the Performance Improvements of Assembled Techniques in a
|
||||
Convolutional Neural Network"
|
||||
"""
|
||||
sk_kwargs = dict(
|
||||
attn_reduction=2,
|
||||
)
|
||||
default_cfg = default_cfgs['skresnet50d']
|
||||
model = ResNet(
|
||||
SelectiveKernelBottleneck, [3, 4, 6, 3], stem_width=32, stem_type='deep', avg_down=True,
|
||||
num_classes=num_classes, in_chans=in_chans, block_args=dict(sk_kwargs=sk_kwargs),
|
||||
zero_init_last_bn=False, **kwargs)
|
||||
model.default_cfg = default_cfg
|
||||
if pretrained:
|
||||
load_pretrained(model, default_cfg, num_classes, in_chans)
|
||||
return model
|
||||
|
||||
|
||||
@register_model
|
||||
def skresnext50_32x4d(pretrained=False, num_classes=1000, in_chans=3, **kwargs):
|
||||
"""Constructs a Select Kernel ResNeXt50-32x4d model. This should be equivalent to
|
||||
the SKNet50 model in the Select Kernel Paper
|
||||
"""
|
||||
default_cfg = default_cfgs['skresnext50_32x4d']
|
||||
model = ResNet(
|
||||
SelectiveKernelBottleneck, [3, 4, 6, 3], cardinality=32, base_width=4,
|
||||
num_classes=num_classes, in_chans=in_chans, zero_init_last_bn=False, **kwargs)
|
||||
model.default_cfg = default_cfg
|
||||
if pretrained:
|
||||
load_pretrained(model, default_cfg, num_classes, in_chans)
|
||||
return model
|
Loading…
Reference in new issue