From 995e2691d6aa71fee1bc8f6c20412d38609c384b Mon Sep 17 00:00:00 2001 From: Ross Wightman Date: Wed, 24 Aug 2022 12:20:26 -0700 Subject: [PATCH 01/18] Update config.yml --- .github/ISSUE_TEMPLATE/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index e44fb40c..d87f5621 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -2,4 +2,4 @@ blank_issues_enabled: false contact_links: - name: Community Discussions url: https://github.com/rwightman/pytorch-image-models/discussions - about: Issues are for features and bugs. Questions can be asked in Discussions. + about: *H-param request in issues will be ignored!* Issues are for features and bugs. Questions can be asked in Discussions. From e018253accdf03c25bf7d9ff069b3ee6f8467916 Mon Sep 17 00:00:00 2001 From: Ross Wightman Date: Wed, 24 Aug 2022 12:21:03 -0700 Subject: [PATCH 02/18] Update config.yml --- .github/ISSUE_TEMPLATE/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index d87f5621..a178cd62 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -2,4 +2,4 @@ blank_issues_enabled: false contact_links: - name: Community Discussions url: https://github.com/rwightman/pytorch-image-models/discussions - about: *H-param request in issues will be ignored!* Issues are for features and bugs. Questions can be asked in Discussions. + about: Hparam request in issues will be ignored! Issues are for features and bugs. Questions can be asked in Discussions. From 2a5b5b2a7b5d5158dc48a868cb3618cbee8def75 Mon Sep 17 00:00:00 2001 From: Ross Wightman Date: Wed, 24 Aug 2022 12:24:32 -0700 Subject: [PATCH 03/18] Update feature_request.md --- .github/ISSUE_TEMPLATE/feature_request.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index d751adc8..ea51beb7 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,8 +1,7 @@ --- name: Feature request -about: Suggest an idea for this project. Issues are for reporting bugs or requesting - features, the discussion forum is available for asking questions or seeking help - from the community. +about: Suggest an idea for this project. Hparam requests, training help are not feature requests. + The discussion forum is available for asking questions or seeking help from the community. title: "[FEATURE] Feature title..." labels: enhancement assignees: '' From 527f9a4cb22f784e80d8b046ba083000feae081e Mon Sep 17 00:00:00 2001 From: Ross Wightman Date: Wed, 24 Aug 2022 12:42:11 -0700 Subject: [PATCH 04/18] Updated to correct maxvit_nano weights... --- timm/models/maxxvit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/timm/models/maxxvit.py b/timm/models/maxxvit.py index 898e1685..7f4ebf59 100644 --- a/timm/models/maxxvit.py +++ b/timm/models/maxxvit.py @@ -108,7 +108,7 @@ default_cfgs = { # Experimental configs 'maxvit_pico_rw_256': _cfg(url='', input_size=(3, 256, 256), pool_size=(8, 8)), 'maxvit_nano_rw_256': _cfg( - url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_nano_rw_256_sw-3e790ce3.pth', + url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_nano_rw_256_sw-fb127241.pth', input_size=(3, 256, 256), pool_size=(8, 8)), 'maxvit_tiny_rw_224': _cfg(url=''), 'maxvit_tiny_rw_256': _cfg(url='', input_size=(3, 256, 256), pool_size=(8, 8)), From 1d8d6f6072659e905d91a2b297d53e927853457d Mon Sep 17 00:00:00 2001 From: Ross Wightman Date: Thu, 25 Aug 2022 15:00:35 -0700 Subject: [PATCH 05/18] Fix two default args in DenseNet blocks... fix #1427 --- timm/models/densenet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/timm/models/densenet.py b/timm/models/densenet.py index a46b86ad..1afdfd7b 100644 --- a/timm/models/densenet.py +++ b/timm/models/densenet.py @@ -115,7 +115,7 @@ class DenseBlock(nn.ModuleDict): _version = 2 def __init__( - self, num_layers, num_input_features, bn_size, growth_rate, norm_layer=nn.ReLU, + self, num_layers, num_input_features, bn_size, growth_rate, norm_layer=BatchNormAct2d, drop_rate=0., memory_efficient=False): super(DenseBlock, self).__init__() for i in range(num_layers): @@ -138,7 +138,7 @@ class DenseBlock(nn.ModuleDict): class DenseTransition(nn.Sequential): - def __init__(self, num_input_features, num_output_features, norm_layer=nn.BatchNorm2d, aa_layer=None): + def __init__(self, num_input_features, num_output_features, norm_layer=BatchNormAct2d, aa_layer=None): super(DenseTransition, self).__init__() self.add_module('norm', norm_layer(num_input_features)) self.add_module('conv', nn.Conv2d( From 7c2660576d565b7441922265456ec8b050608da3 Mon Sep 17 00:00:00 2001 From: Ross Wightman Date: Thu, 25 Aug 2022 15:30:59 -0700 Subject: [PATCH 06/18] Tweak init for convnext block using maxxvit/coatnext. --- timm/models/maxxvit.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/timm/models/maxxvit.py b/timm/models/maxxvit.py index 7f4ebf59..82840523 100644 --- a/timm/models/maxxvit.py +++ b/timm/models/maxxvit.py @@ -259,8 +259,6 @@ def _rw_max_cfg( # - mbconv expansion calculated from input instead of output chs # - mbconv shortcut and final 1x1 conv did not have a bias # - mbconv uses silu in timm, not gelu - # - avg pool with kernel_size=2 favoured downsampling (instead of maxpool for coat) - # - default to avg pool for mbconv downsample instead of 1x1 or dw conv # - expansion in attention block done via output proj, not input proj return dict( conv_cfg=MaxxVitConvCfg( @@ -411,18 +409,19 @@ model_cfgs = dict( rel_pos_dim=384, # was supposed to be 512, woops ), ), - coatnext_nano_rw_224=MaxxVitCfg( + coatnet_nano_cc_224=MaxxVitCfg( embed_dim=(64, 128, 256, 512), depths=(3, 4, 6, 3), stem_width=(32, 64), - **_next_cfg(), + block_type=('C', 'C', ('C', 'T'), ('C', 'T')), + **_rw_coat_cfg(), ), - coatnet_nano_cc_224=MaxxVitCfg( + coatnext_nano_rw_224=MaxxVitCfg( embed_dim=(64, 128, 256, 512), depths=(3, 4, 6, 3), stem_width=(32, 64), - block_type=('C', 'C', ('C', 'T'), ('C', 'T')), - **_rw_coat_cfg(), + weight_init='normal', + **_next_cfg(), ), # Trying to be like the CoAtNet paper configs @@ -498,6 +497,7 @@ model_cfgs = dict( depths=(1, 2, 3, 1), block_type=('M',) * 4, stem_width=(32, 64), + weight_init='normal', **_next_cfg(window_size=8), ), From a54008bd97057149e6c110ddc8887481508ee595 Mon Sep 17 00:00:00 2001 From: Ross Wightman Date: Thu, 25 Aug 2022 15:56:56 -0700 Subject: [PATCH 07/18] Update README.md for merge --- README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/README.md b/README.md index 019fdae2..fdf83853 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,25 @@ And a big thanks to all GitHub sponsors who helped with some of my costs before ## What's New +### Aug 26, 2022 +* CoAtNet (https://arxiv.org/abs/2106.04803) and MaxVit (https://arxiv.org/abs/2204.01697) `timm` original models + * both found in [`maxxvit.py`](https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/maxxvit.py) model def, contains numerous experiments outside scope of original papers + * an unfinished Tensorflow version from MaxVit authors can be found https://github.com/google-research/maxvit +* Initial CoAtNet and MaxVit timm pretrained weights (working on more): + * `coatnet_nano_rw_224` - 81.7 @ 224 (T) + * `coatnet_rmlp_nano_rw_224` - 82.0 @ 224, 82.8 @ 320 (T) + * `coatnet_0_rw_224` - 82.4 (T) -- NOTE timm '0' coatnets have 2 more 3rd stage blocks + * `coatnet_bn_0_rw_224` - 82.4 (T) + * `maxvit_nano_rw_256` - 82.9 @ 256 (T) + * `coatnet_rmlp_1_rw_224` - 83.4 @ 224, 84 @ 320 (T) + * `coatnet_1_rw_224` - 83.6 @ 224 (G) +* GCVit (weights adapted from https://github.com/NVlabs/GCVit, code 100% `timm` re-write for license purposes) +* MViT-V2 (multi-scale vit, adapted from https://github.com/facebookresearch/mvit) +* EfficientFormer (adapted from https://github.com/snap-research/EfficientFormer) +* PyramidVisionTransformer-V2 (adapted from https://github.com/whai362/PVT) +* 'Fast Norm' support for LayerNorm and GroupNorm that avoids float32 upcast w/ AMP (uses APEX LN if available for further boost) + + ### Aug 15, 2022 * ConvNeXt atto weights added * `convnext_atto` - 75.7 @ 224, 77.0 @ 288 @@ -229,6 +248,7 @@ A full version of the list below with source links can be found in the [document * Bottleneck Transformers - https://arxiv.org/abs/2101.11605 * CaiT (Class-Attention in Image Transformers) - https://arxiv.org/abs/2103.17239 * CoaT (Co-Scale Conv-Attentional Image Transformers) - https://arxiv.org/abs/2104.06399 +* CoAtNet (Convolution and Attention) - https://arxiv.org/abs/2106.04803 * ConvNeXt - https://arxiv.org/abs/2201.03545 * ConViT (Soft Convolutional Inductive Biases Vision Transformers)- https://arxiv.org/abs/2103.10697 * CspNet (Cross-Stage Partial Networks) - https://arxiv.org/abs/1911.11929 @@ -238,6 +258,7 @@ A full version of the list below with source links can be found in the [document * DLA - https://arxiv.org/abs/1707.06484 * DPN (Dual-Path Network) - https://arxiv.org/abs/1707.01629 * EdgeNeXt - https://arxiv.org/abs/2206.10589 +* EfficientFormer - https://arxiv.org/abs/2206.01191 * EfficientNet (MBConvNet Family) * EfficientNet NoisyStudent (B0-B7, L2) - https://arxiv.org/abs/1911.04252 * EfficientNet AdvProp (B0-B8) - https://arxiv.org/abs/1911.09665 @@ -259,6 +280,7 @@ A full version of the list below with source links can be found in the [document * Inception-ResNet-V2 and Inception-V4 - https://arxiv.org/abs/1602.07261 * Lambda Networks - https://arxiv.org/abs/2102.08602 * LeViT (Vision Transformer in ConvNet's Clothing) - https://arxiv.org/abs/2104.01136 +* MaxViT (Multi-Axis Vision Transformer) - https://arxiv.org/abs/2204.01697 * MLP-Mixer - https://arxiv.org/abs/2105.01601 * MobileNet-V3 (MBConvNet w/ Efficient Head) - https://arxiv.org/abs/1905.02244 * FBNet-V3 - https://arxiv.org/abs/2006.02049 @@ -266,6 +288,7 @@ A full version of the list below with source links can be found in the [document * LCNet - https://arxiv.org/abs/2109.15099 * MobileViT - https://arxiv.org/abs/2110.02178 * MobileViT-V2 - https://arxiv.org/abs/2206.02680 +* MViT-V2 (Improved Multiscale Vision Transformer) - https://arxiv.org/abs/2112.01526 * NASNet-A - https://arxiv.org/abs/1707.07012 * NesT - https://arxiv.org/abs/2105.12723 * NFNet-F - https://arxiv.org/abs/2102.06171 @@ -273,6 +296,7 @@ A full version of the list below with source links can be found in the [document * PNasNet - https://arxiv.org/abs/1712.00559 * PoolFormer (MetaFormer) - https://arxiv.org/abs/2111.11418 * Pooling-based Vision Transformer (PiT) - https://arxiv.org/abs/2103.16302 +* PVT-V2 (Improved Pyramid Vision Transformer) - https://arxiv.org/abs/2106.13797 * RegNet - https://arxiv.org/abs/2003.13678 * RegNetZ - https://arxiv.org/abs/2103.06877 * RepVGG - https://arxiv.org/abs/2101.03697 From 99ee61e245e4a7652fd25043d4102fef7e480131 Mon Sep 17 00:00:00 2001 From: Ross Wightman Date: Thu, 25 Aug 2022 15:58:57 -0700 Subject: [PATCH 08/18] Add T/G legend to README.md maxvit list --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fdf83853..9912d40d 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ And a big thanks to all GitHub sponsors who helped with some of my costs before * `maxvit_nano_rw_256` - 82.9 @ 256 (T) * `coatnet_rmlp_1_rw_224` - 83.4 @ 224, 84 @ 320 (T) * `coatnet_1_rw_224` - 83.6 @ 224 (G) + * (T) = TPU trained with `bits_and_tpu` branch training code, (G) = GPU trained * GCVit (weights adapted from https://github.com/NVlabs/GCVit, code 100% `timm` re-write for license purposes) * MViT-V2 (multi-scale vit, adapted from https://github.com/facebookresearch/mvit) * EfficientFormer (adapted from https://github.com/snap-research/EfficientFormer) From 48e1df8b3789bda43e4942513c896677e1b1e8dc Mon Sep 17 00:00:00 2001 From: Ross Wightman Date: Thu, 25 Aug 2022 16:29:34 -0700 Subject: [PATCH 09/18] Add norm/norm_act header comments --- timm/models/layers/fast_norm.py | 10 ++++++++++ timm/models/layers/norm.py | 4 ++++ timm/models/layers/norm_act.py | 12 ++++++++++++ 3 files changed, 26 insertions(+) diff --git a/timm/models/layers/fast_norm.py b/timm/models/layers/fast_norm.py index 9a34a15e..fb35e47d 100644 --- a/timm/models/layers/fast_norm.py +++ b/timm/models/layers/fast_norm.py @@ -1,3 +1,11 @@ +""" 'Fast' Normalization Functions + +For GroupNorm and LayerNorm these functions bypass typical AMP upcast to float32. + +Additionally, for LayerNorm, the APEX fused LN is used if available (which also does not upcast) + +Hacked together by / Copyright 2022 Ross Wightman +""" from typing import List, Optional import torch @@ -37,6 +45,7 @@ def fast_group_norm( if torch.is_autocast_enabled(): # normally native AMP casts GN inputs to float32 # here we use the low precision autocast dtype + # FIXME what to do re CPU autocast? dt = torch.get_autocast_gpu_dtype() x, weight, bias = x.to(dt), weight.to(dt), bias.to(dt) @@ -62,6 +71,7 @@ def fast_layer_norm( # normally native AMP casts LN inputs to float32 # apex LN does not, this is behaving like Apex dt = torch.get_autocast_gpu_dtype() + # FIXME what to do re CPU autocast? x, weight, bias = x.to(dt), weight.to(dt), bias.to(dt) with torch.cuda.amp.autocast(enabled=False): diff --git a/timm/models/layers/norm.py b/timm/models/layers/norm.py index 2ff8fc08..42445a49 100644 --- a/timm/models/layers/norm.py +++ b/timm/models/layers/norm.py @@ -1,4 +1,8 @@ """ Normalization layers and wrappers + +Norm layer definitions that support fast norm and consistent channel arg order (always first arg). + +Hacked together by / Copyright 2022 Ross Wightman """ import torch diff --git a/timm/models/layers/norm_act.py b/timm/models/layers/norm_act.py index dc077160..ff075fbc 100644 --- a/timm/models/layers/norm_act.py +++ b/timm/models/layers/norm_act.py @@ -1,4 +1,16 @@ """ Normalization + Activation Layers + +Provides Norm+Act fns for standard PyTorch norm layers such as +* BatchNorm +* GroupNorm +* LayerNorm + +This allows swapping with alternative layers that are natively both norm + act such as +* EvoNorm (evo_norm.py) +* FilterResponseNorm (filter_response_norm.py) +* InplaceABN (inplace_abn.py) + +Hacked together by / Copyright 2022 Ross Wightman """ from typing import Union, List, Optional, Any From 769ab4b98a51f18ce8b1fa37deab3a8f1f9032aa Mon Sep 17 00:00:00 2001 From: Ross Wightman Date: Thu, 25 Aug 2022 16:29:52 -0700 Subject: [PATCH 10/18] Clean up no_grad for trunc normal weight inits --- timm/models/layers/weight_init.py | 48 ++++++++++++++++--------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/timm/models/layers/weight_init.py b/timm/models/layers/weight_init.py index 4a160931..943e4f4c 100644 --- a/timm/models/layers/weight_init.py +++ b/timm/models/layers/weight_init.py @@ -5,7 +5,7 @@ import warnings from torch.nn.init import _calculate_fan_in_and_fan_out -def _no_grad_trunc_normal_(tensor, mean, std, a, b): +def _trunc_normal_(tensor, mean, std, a, b): # Cut & paste from PyTorch official master until it's in a few official releases - RW # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf def norm_cdf(x): @@ -17,28 +17,27 @@ def _no_grad_trunc_normal_(tensor, mean, std, a, b): "The distribution of values may be incorrect.", stacklevel=2) - with torch.no_grad(): - # Values are generated by using a truncated uniform distribution and - # then using the inverse CDF for the normal distribution. - # Get upper and lower cdf values - l = norm_cdf((a - mean) / std) - u = norm_cdf((b - mean) / std) + # Values are generated by using a truncated uniform distribution and + # then using the inverse CDF for the normal distribution. + # Get upper and lower cdf values + l = norm_cdf((a - mean) / std) + u = norm_cdf((b - mean) / std) - # Uniformly fill tensor with values from [l, u], then translate to - # [2l-1, 2u-1]. - tensor.uniform_(2 * l - 1, 2 * u - 1) + # Uniformly fill tensor with values from [l, u], then translate to + # [2l-1, 2u-1]. + tensor.uniform_(2 * l - 1, 2 * u - 1) - # Use inverse cdf transform for normal distribution to get truncated - # standard normal - tensor.erfinv_() + # Use inverse cdf transform for normal distribution to get truncated + # standard normal + tensor.erfinv_() - # Transform to proper mean, std - tensor.mul_(std * math.sqrt(2.)) - tensor.add_(mean) + # Transform to proper mean, std + tensor.mul_(std * math.sqrt(2.)) + tensor.add_(mean) - # Clamp to ensure it's in the proper range - tensor.clamp_(min=a, max=b) - return tensor + # Clamp to ensure it's in the proper range + tensor.clamp_(min=a, max=b) + return tensor def trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.): @@ -64,7 +63,8 @@ def trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.): >>> w = torch.empty(3, 5) >>> nn.init.trunc_normal_(w) """ - return _no_grad_trunc_normal_(tensor, mean, std, a, b) + with torch.no_grad(): + return _trunc_normal_(tensor, mean, std, a, b) def trunc_normal_tf_(tensor, mean=0., std=1., a=-2., b=2.): @@ -90,8 +90,8 @@ def trunc_normal_tf_(tensor, mean=0., std=1., a=-2., b=2.): >>> w = torch.empty(3, 5) >>> nn.init.trunc_normal_(w) """ - _no_grad_trunc_normal_(tensor, 0, 1.0, a, b) with torch.no_grad(): + _trunc_normal_(tensor, 0, 1.0, a, b) tensor.mul_(std).add_(mean) return tensor @@ -111,10 +111,12 @@ def variance_scaling_(tensor, scale=1.0, mode='fan_in', distribution='normal'): # constant is stddev of standard normal truncated to (-2, 2) trunc_normal_tf_(tensor, std=math.sqrt(variance) / .87962566103423978) elif distribution == "normal": - tensor.normal_(std=math.sqrt(variance)) + with torch.no_grad(): + tensor.normal_(std=math.sqrt(variance)) elif distribution == "uniform": bound = math.sqrt(3 * variance) - tensor.uniform_(-bound, bound) + with torch.no_grad(): + tensor.uniform_(-bound, bound) else: raise ValueError(f"invalid distribution {distribution}") From ff6a919cf5f0a325236cf57c07548f779123173f Mon Sep 17 00:00:00 2001 From: Ross Wightman Date: Thu, 25 Aug 2022 17:00:54 -0700 Subject: [PATCH 11/18] Add --fast-norm arg to benchmark.py, train.py, validate.py --- benchmark.py | 8 ++++++-- timm/models/__init__.py | 1 + train.py | 6 +++++- validate.py | 6 +++++- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/benchmark.py b/benchmark.py index 4679a009..4a89441b 100755 --- a/benchmark.py +++ b/benchmark.py @@ -19,7 +19,7 @@ import torch.nn as nn import torch.nn.parallel from timm.data import resolve_data_config -from timm.models import create_model, is_model, list_models +from timm.models import create_model, is_model, list_models, set_fast_norm from timm.optim import create_optimizer_v2 from timm.utils import setup_default_logging, set_jit_fuser, decay_batch_step, check_batch_size_retry @@ -109,7 +109,8 @@ scripting_group.add_argument('--torchscript', dest='torchscript', action='store_ help='convert model torchscript for inference') scripting_group.add_argument('--aot-autograd', default=False, action='store_true', help="Enable AOT Autograd support. (It's recommended to use this option with `--fuser nvfuser` together)") - +scripting_group.add_argument('--fast-norm', default=False, action='store_true', + help='enable experimental fast-norm') # train optimizer parameters parser.add_argument('--opt', default='sgd', type=str, metavar='OPTIMIZER', @@ -598,6 +599,9 @@ def main(): model_cfgs = [] model_names = [] + if args.fast_norm: + set_fast_norm() + if args.model_list: args.model = '' with open(args.model_list) as f: diff --git a/timm/models/__init__.py b/timm/models/__init__.py index 51a38d0c..5ff79595 100644 --- a/timm/models/__init__.py +++ b/timm/models/__init__.py @@ -69,5 +69,6 @@ from .helpers import load_checkpoint, resume_checkpoint, model_parameters from .layers import TestTimePoolHead, apply_test_time_pool from .layers import convert_splitbn_model, convert_sync_batchnorm from .layers import is_scriptable, is_exportable, set_scriptable, set_exportable, is_no_jit, set_no_jit +from .layers import set_fast_norm from .registry import register_model, model_entrypoint, list_models, is_model, list_modules, is_model_in_modules,\ is_model_pretrained, get_pretrained_cfg, has_pretrained_cfg_key, is_pretrained_cfg_key, get_pretrained_cfg_value diff --git a/train.py b/train.py index e5d40566..ee137217 100755 --- a/train.py +++ b/train.py @@ -33,7 +33,7 @@ from timm.data import create_dataset, create_loader, resolve_data_config, Mixup, from timm.loss import JsdCrossEntropy, SoftTargetCrossEntropy, BinaryCrossEntropy, \ LabelSmoothingCrossEntropy from timm.models import create_model, safe_model_name, resume_checkpoint, load_checkpoint, \ - convert_splitbn_model, convert_sync_batchnorm, model_parameters + convert_splitbn_model, convert_sync_batchnorm, model_parameters, set_fast_norm from timm.optim import create_optimizer_v2, optimizer_kwargs from timm.scheduler import create_scheduler from timm.utils import ApexScaler, NativeScaler @@ -135,6 +135,8 @@ scripting_group.add_argument('--aot-autograd', default=False, action='store_true help="Enable AOT Autograd support. (It's recommended to use this option with `--fuser nvfuser` together)") group.add_argument('--fuser', default='', type=str, help="Select jit fuser. One of ('', 'te', 'old', 'nvfuser')") +group.add_argument('--fast-norm', default=False, action='store_true', + help='enable experimental fast-norm') group.add_argument('--grad-checkpointing', action='store_true', default=False, help='Enable gradient checkpointing through model blocks/stages') @@ -395,6 +397,8 @@ def main(): if args.fuser: utils.set_jit_fuser(args.fuser) + if args.fast_norm: + set_fast_norm() model = create_model( args.model, diff --git a/validate.py b/validate.py index a4d41868..6244f052 100755 --- a/validate.py +++ b/validate.py @@ -20,7 +20,7 @@ import torch.nn.parallel from collections import OrderedDict from contextlib import suppress -from timm.models import create_model, apply_test_time_pool, load_checkpoint, is_model, list_models +from timm.models import create_model, apply_test_time_pool, load_checkpoint, is_model, list_models, set_fast_norm from timm.data import create_dataset, create_loader, resolve_data_config, RealLabelsImagenet from timm.utils import accuracy, AverageMeter, natural_key, setup_default_logging, set_jit_fuser,\ decay_batch_step, check_batch_size_retry @@ -117,6 +117,8 @@ scripting_group.add_argument('--aot-autograd', default=False, action='store_true help="Enable AOT Autograd support. (It's recommended to use this option with `--fuser nvfuser` together)") parser.add_argument('--fuser', default='', type=str, help="Select jit fuser. One of ('', 'te', 'old', 'nvfuser')") +parser.add_argument('--fast-norm', default=False, action='store_true', + help='enable experimental fast-norm') parser.add_argument('--results-file', default='', type=str, metavar='FILENAME', help='Output csv file for validation results (summary)') parser.add_argument('--real-labels', default='', type=str, metavar='FILENAME', @@ -150,6 +152,8 @@ def validate(args): if args.fuser: set_jit_fuser(args.fuser) + if args.fast_norm: + set_fast_norm() # create model model = create_model( From eca6f0a25c439014ad80e2d5cb776e27b9f4a494 Mon Sep 17 00:00:00 2001 From: Ross Wightman Date: Fri, 26 Aug 2022 11:29:09 -0700 Subject: [PATCH 12/18] Fix syntax error (extra dataclass comma) in maxxvit.py --- timm/models/maxxvit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/timm/models/maxxvit.py b/timm/models/maxxvit.py index 82840523..2475e809 100644 --- a/timm/models/maxxvit.py +++ b/timm/models/maxxvit.py @@ -129,7 +129,7 @@ class MaxxVitTransformerCfg: dim_head: int = 32 expand_ratio: float = 4.0 expand_first: bool = True - shortcut_bias: bool = True, + shortcut_bias: bool = True attn_bias: bool = True attn_drop: float = 0. proj_drop: float = 0. From f1d2160d857c1d98942e16ad3b59d7b74ecb2255 Mon Sep 17 00:00:00 2001 From: Ross Wightman Date: Fri, 26 Aug 2022 12:53:49 -0700 Subject: [PATCH 13/18] Update a few maxxvit comments, rename PartitionAttention -> PartitionAttenionCl for consistency with other blocks --- timm/models/maxxvit.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/timm/models/maxxvit.py b/timm/models/maxxvit.py index 2475e809..de2c9fb8 100644 --- a/timm/models/maxxvit.py +++ b/timm/models/maxxvit.py @@ -651,7 +651,11 @@ class LayerScale2d(nn.Module): class Downsample2d(nn.Module): - """ A downsample pooling module for Coat that handles 2d <-> 1d conversion + """ A downsample pooling module supporting several maxpool and avgpool modes + * 'max' - MaxPool2d w/ kernel_size 3, stride 2, padding 1 + * 'max2' - MaxPool2d w/ kernel_size = stride = 2 + * 'avg' - AvgPool2d w/ kernel_size 3, stride 2, padding 1 + * 'avg2' - AvgPool2d w/ kernel_size = stride = 2 """ def __init__( @@ -710,6 +714,11 @@ def _init_transformer(module, name, scheme=''): class TransformerBlock2d(nn.Module): """ Transformer block with 2D downsampling '2D' NCHW tensor layout + + Some gains can be seen on GPU using a 1D / CL block, BUT w/ the need to switch back/forth to NCHW + for spatial pooling, the benefit is minimal so ended up using just this variant for CoAt configs. + + This impl was faster on TPU w/ PT XLA than the 1D experiment. """ def __init__( @@ -1011,9 +1020,9 @@ def get_rel_pos_cls(cfg: MaxxVitTransformerCfg, window_size): return rel_pos_cls -class PartitionAttention(nn.Module): +class PartitionAttentionCl(nn.Module): """ Grid or Block partition + Attn + FFN. - NxC tensor layout. + NxC 'channels last' tensor layout. """ def __init__( @@ -1183,6 +1192,7 @@ def grid_reverse_nchw(windows, grid_size: List[int], img_size: List[int]): class PartitionAttention2d(nn.Module): """ Grid or Block partition + Attn + FFN + '2D' NCHW tensor layout. """ @@ -1245,7 +1255,7 @@ class PartitionAttention2d(nn.Module): class MaxxVitBlock(nn.Module): - """ + """ MaxVit conv, window partition + FFN , grid partition + FFN """ def __init__( @@ -1264,7 +1274,7 @@ class MaxxVitBlock(nn.Module): self.conv = conv_cls(dim, dim_out, stride=stride, cfg=conv_cfg, drop_path=drop_path) attn_kwargs = dict(dim=dim_out, cfg=transformer_cfg, drop_path=drop_path) - partition_layer = PartitionAttention2d if use_nchw_attn else PartitionAttention + partition_layer = PartitionAttention2d if use_nchw_attn else PartitionAttentionCl self.nchw_attn = use_nchw_attn self.attn_block = partition_layer(**attn_kwargs) self.attn_grid = partition_layer(partition_type='grid', **attn_kwargs) @@ -1288,7 +1298,8 @@ class MaxxVitBlock(nn.Module): class ParallelMaxxVitBlock(nn.Module): - """ + """ MaxVit block with parallel cat(window + grid), one FF + Experimental timm block. """ def __init__( @@ -1427,7 +1438,9 @@ class Stem(nn.Module): class MaxxVit(nn.Module): - """ + """ CoaTNet + MaxVit base model. + + Highly configurable for different block compositions, tensor layouts, pooling types. """ def __init__( From b94b7cea6577a628272cfe4f3991d7417b2d64a5 Mon Sep 17 00:00:00 2001 From: Ross Wightman Date: Sun, 28 Aug 2022 15:22:28 -0700 Subject: [PATCH 14/18] Missed GCVit in README paper links --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9912d40d..9c45ed89 100644 --- a/README.md +++ b/README.md @@ -272,6 +272,7 @@ A full version of the list below with source links can be found in the [document * MobileNet-V2 - https://arxiv.org/abs/1801.04381 * Single-Path NAS - https://arxiv.org/abs/1904.02877 * TinyNet - https://arxiv.org/abs/2010.14819 +* GCViT (Global Context Vision Transformer) - https://arxiv.org/abs/2206.09959 * GhostNet - https://arxiv.org/abs/1911.11907 * gMLP - https://arxiv.org/abs/2105.08050 * GPU-Efficient Networks - https://arxiv.org/abs/2006.14090 From f66e5f0e35c83f086f5fe788af7d6ef5f1804001 Mon Sep 17 00:00:00 2001 From: Ross Wightman Date: Sun, 28 Aug 2022 15:24:04 -0700 Subject: [PATCH 15/18] Fix class token support in MViT-V2, add small_class variant to ensure it's tested. Fix #1443 --- timm/models/mvitv2.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/timm/models/mvitv2.py b/timm/models/mvitv2.py index 002225c6..76e70135 100644 --- a/timm/models/mvitv2.py +++ b/timm/models/mvitv2.py @@ -135,6 +135,11 @@ model_cfgs = dict( num_heads=2, expand_attn=False, ), + + mvitv2_small_cls=MultiScaleVitCfg( + depths=(1, 2, 11, 2), + use_cls_token=True, + ), ) @@ -641,7 +646,7 @@ class MultiScaleBlock(nn.Module): if self.shortcut_pool_attn is None: return x if self.has_cls_token: - cls_tok, x = x[:, :, :1, :], x[:, :, 1:, :] + cls_tok, x = x[:, :1, :], x[:, 1:, :] else: cls_tok = None B, L, C = x.shape @@ -650,7 +655,7 @@ class MultiScaleBlock(nn.Module): x = self.shortcut_pool_attn(x) x = x.reshape(B, C, -1).transpose(1, 2) if cls_tok is not None: - x = torch.cat((cls_tok, x), dim=2) + x = torch.cat((cls_tok, x), dim=1) return x def forward(self, x, feat_size: List[int]): @@ -996,3 +1001,8 @@ def mvitv2_large(pretrained=False, **kwargs): # @register_model # def mvitv2_huge_in21k(pretrained=False, **kwargs): # return _create_mvitv2('mvitv2_huge_in21k', pretrained=pretrained, **kwargs) + + +@register_model +def mvitv2_small_cls(pretrained=False, **kwargs): + return _create_mvitv2('mvitv2_small_cls', pretrained=pretrained, **kwargs) From e6a43613063670439b01d5a6300f799c5da9261c Mon Sep 17 00:00:00 2001 From: Ross Wightman Date: Sun, 28 Aug 2022 15:27:01 -0700 Subject: [PATCH 16/18] pretrained_cfg entry for mvitv2_small_cls --- timm/models/mvitv2.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/timm/models/mvitv2.py b/timm/models/mvitv2.py index 76e70135..c5aaa09e 100644 --- a/timm/models/mvitv2.py +++ b/timm/models/mvitv2.py @@ -57,6 +57,8 @@ default_cfgs = dict( mvitv2_huge_in21k=_cfg( url='https://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_H_in21k.pyth', num_classes=19168), + + mvitv2_small_cls=_cfg(url=''), ) From 7f1b223c02a8af5273918e447a05a5b73a4ed54d Mon Sep 17 00:00:00 2001 From: Ross Wightman Date: Mon, 29 Aug 2022 15:49:32 -0700 Subject: [PATCH 17/18] Add maxvit_rmlp_nano_rw_256 model def & weights, make window/grid size dynamic wrt img_size by default --- timm/models/maxxvit.py | 62 ++++++++++++++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 15 deletions(-) diff --git a/timm/models/maxxvit.py b/timm/models/maxxvit.py index de2c9fb8..495f682b 100644 --- a/timm/models/maxxvit.py +++ b/timm/models/maxxvit.py @@ -39,7 +39,7 @@ Hacked together by / Copyright 2022, Ross Wightman import math from collections import OrderedDict -from dataclasses import dataclass +from dataclasses import dataclass, replace from functools import partial from typing import Callable, Optional, Union, Tuple, List @@ -112,6 +112,9 @@ default_cfgs = { input_size=(3, 256, 256), pool_size=(8, 8)), 'maxvit_tiny_rw_224': _cfg(url=''), 'maxvit_tiny_rw_256': _cfg(url='', input_size=(3, 256, 256), pool_size=(8, 8)), + 'maxvit_rmlp_nano_rw_256': _cfg( + url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_rmlp_nano_rw_256_sw-c17bb0d6.pth', + input_size=(3, 256, 256), pool_size=(8, 8)), 'maxvit_tiny_pm_256': _cfg(url='', input_size=(3, 256, 256), pool_size=(8, 8)), 'maxxvit_nano_rw_256': _cfg(url='', input_size=(3, 256, 256), pool_size=(8, 8)), @@ -136,14 +139,23 @@ class MaxxVitTransformerCfg: pool_type: str = 'avg2' rel_pos_type: str = 'bias' rel_pos_dim: int = 512 # for relative position types w/ MLP - window_size: Tuple[int, int] = (7, 7) - grid_size: Tuple[int, int] = (7, 7) + partition_stride: int = 32 + window_size: Optional[Tuple[int, int]] = None + grid_size: Optional[Tuple[int, int]] = None init_values: Optional[float] = None act_layer: str = 'gelu' norm_layer: str = 'layernorm2d' norm_layer_cl: str = 'layernorm' norm_eps: float = 1e-6 + def __post_init__(self): + if self.grid_size is not None: + self.grid_size = to_2tuple(self.grid_size) + if self.window_size is not None: + self.window_size = to_2tuple(self.window_size) + if self.grid_size is None: + self.grid_size = self.window_size + @dataclass class MaxxVitConvCfg: @@ -249,7 +261,7 @@ def _rw_max_cfg( conv_norm_layer='', transformer_norm_layer='layernorm2d', transformer_norm_layer_cl='layernorm', - window_size=7, + window_size=None, dim_head=32, rel_pos_type='bias', rel_pos_dim=512, @@ -274,8 +286,7 @@ def _rw_max_cfg( expand_first=False, pool_type=pool_type, dim_head=dim_head, - window_size=to_2tuple(window_size), - grid_size=to_2tuple(window_size), + window_size=window_size, norm_layer=transformer_norm_layer, norm_layer_cl=transformer_norm_layer_cl, rel_pos_type=rel_pos_type, @@ -291,7 +302,7 @@ def _next_cfg( conv_norm_layer_cl='layernorm', transformer_norm_layer='layernorm2d', transformer_norm_layer_cl='layernorm', - window_size=7, + window_size=None, rel_pos_type='bias', rel_pos_dim=512, ): @@ -308,8 +319,7 @@ def _next_cfg( transformer_cfg=MaxxVitTransformerCfg( expand_first=False, pool_type=pool_type, - window_size=to_2tuple(window_size), - grid_size=to_2tuple(window_size), + window_size=window_size, norm_layer=transformer_norm_layer, norm_layer_cl=transformer_norm_layer_cl, rel_pos_type=rel_pos_type, @@ -462,14 +472,14 @@ model_cfgs = dict( depths=(2, 2, 5, 2), block_type=('M',) * 4, stem_width=(24, 32), - **_rw_max_cfg(window_size=8), + **_rw_max_cfg(), ), maxvit_nano_rw_256=MaxxVitCfg( embed_dim=(64, 128, 256, 512), depths=(1, 2, 3, 1), block_type=('M',) * 4, stem_width=(32, 64), - **_rw_max_cfg(window_size=8), + **_rw_max_cfg(), ), maxvit_tiny_rw_224=MaxxVitCfg( embed_dim=(64, 128, 256, 512), @@ -483,14 +493,21 @@ model_cfgs = dict( depths=(2, 2, 5, 2), block_type=('M',) * 4, stem_width=(32, 64), - **_rw_max_cfg(window_size=8), + **_rw_max_cfg(), + ), + maxvit_rmlp_nano_rw_256=MaxxVitCfg( + embed_dim=(64, 128, 256, 512), + depths=(1, 2, 3, 1), + block_type=('M',) * 4, + stem_width=(32, 64), + **_rw_max_cfg(rel_pos_type='mlp'), ), maxvit_tiny_pm_256=MaxxVitCfg( embed_dim=(64, 128, 256, 512), depths=(2, 2, 5, 2), block_type=('PM',) * 4, stem_width=(32, 64), - **_rw_max_cfg(window_size=8), + **_rw_max_cfg(), ), maxxvit_nano_rw_256=MaxxVitCfg( embed_dim=(64, 128, 256, 512), @@ -498,7 +515,7 @@ model_cfgs = dict( block_type=('M',) * 4, stem_width=(32, 64), weight_init='normal', - **_next_cfg(window_size=8), + **_next_cfg(), ), # Trying to be like the MaxViT paper configs @@ -1437,6 +1454,15 @@ class Stem(nn.Module): return x +def cfg_window_size(cfg: MaxxVitTransformerCfg, img_size: Tuple[int, int]): + if cfg.window_size is not None: + assert cfg.grid_size + return cfg + partition_size = img_size[0] // cfg.partition_stride, img_size[1] // cfg.partition_stride + cfg = replace(cfg, window_size=partition_size, grid_size=partition_size) + return cfg + + class MaxxVit(nn.Module): """ CoaTNet + MaxVit base model. @@ -1455,6 +1481,7 @@ class MaxxVit(nn.Module): ): super().__init__() img_size = to_2tuple(img_size) + transformer_cfg = cfg_window_size(cfg.transformer_cfg, img_size) self.num_classes = num_classes self.global_pool = global_pool self.num_features = cfg.embed_dim[-1] @@ -1488,7 +1515,7 @@ class MaxxVit(nn.Module): depth=cfg.depths[i], block_types=cfg.block_type[i], conv_cfg=cfg.conv_cfg, - transformer_cfg=cfg.transformer_cfg, + transformer_cfg=transformer_cfg, feat_size=feat_size, drop_path=dpr[i], )] @@ -1671,6 +1698,11 @@ def maxvit_tiny_rw_256(pretrained=False, **kwargs): return _create_maxxvit('maxvit_tiny_rw_256', pretrained=pretrained, **kwargs) +@register_model +def maxvit_rmlp_nano_rw_256(pretrained=False, **kwargs): + return _create_maxxvit('maxvit_rmlp_nano_rw_256', pretrained=pretrained, **kwargs) + + @register_model def maxvit_tiny_pm_256(pretrained=False, **kwargs): return _create_maxxvit('maxvit_tiny_pm_256', pretrained=pretrained, **kwargs) From c45c6ee8e406861898acccb1818f91d0c9240e48 Mon Sep 17 00:00:00 2001 From: Ross Wightman Date: Mon, 29 Aug 2022 15:52:24 -0700 Subject: [PATCH 18/18] Update README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 9c45ed89..b5945b19 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,10 @@ And a big thanks to all GitHub sponsors who helped with some of my costs before ## What's New +### Aug 29, 2022 +* MaxVit window size scales with img_size by default. Add new RelPosMlp MaxViT weight that leverages this: + * `maxvit_rmlp_nano_rw_256` - 83.0 @ 256, 83.6 @ 320 (T) + ### Aug 26, 2022 * CoAtNet (https://arxiv.org/abs/2106.04803) and MaxVit (https://arxiv.org/abs/2204.01697) `timm` original models * both found in [`maxxvit.py`](https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/maxxvit.py) model def, contains numerous experiments outside scope of original papers