From dd0b9b0e92233dc0a83f7c6c04b1dc916a6ed6e0 Mon Sep 17 00:00:00 2001 From: Ross Wightman Date: Fri, 14 Jan 2022 15:39:31 -0800 Subject: [PATCH] Update REAMDE.md. Sneak in g/G (giant / gigantic?) ViT defs from scaling paper --- README.md | 11 +++++++---- tests/test_models.py | 8 ++++---- timm/models/vision_transformer.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index c464454c..51844ed1 100644 --- a/README.md +++ b/README.md @@ -23,11 +23,14 @@ I'm fortunate to be able to dedicate significant time and money of my own suppor ## What's New -### Jan 6, 2022 -* Version 0.5.2 w/ release to be pushed to pypi. It's been a while since last pypi update and riskier changes will be merged to main branch soon.... -* Tried training a few small / mobile optimized models, a few are good so far, more on the way... +### Jan 14, 2022 +* Version 0.5.4 w/ release to be pushed to pypi. It's been a while since last pypi update and riskier changes will be merged to main branch soon.... +* Add ConvNeXT models /w weights from official impl (https://github.com/facebookresearch/ConvNeXt), a few perf tweaks, compatible with timm features +* Tried training a few small (~1.8-3M param) / mobile optimized models, a few are good so far, more on the way... * `mnasnet_small` - 65.6 top-1 - * `lcnet_100` - 72.1 top-1 + * `mobilenetv2_050` - 65.9 + * `lcnet_100/075/050` - 72.1 / 68.8 / 63.1 + * `semnasnet_075` - 73 * `fbnetv3_b/d/g` - 79.1 / 79.7 / 82.0 * TinyNet models added by [rsomani95](https://github.com/rsomani95) * LCNet added via MobileNetV3 architecture diff --git a/tests/test_models.py b/tests/test_models.py index 73c3e0b6..38847dcb 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -28,12 +28,12 @@ NON_STD_FILTERS = [ NUM_NON_STD = len(NON_STD_FILTERS) # exclude models that cause specific test failures -if 'GITHUB_ACTIONS' in os.environ: # and 'Linux' in platform.system(): +if 'GITHUB_ACTIONS' in os.environ: # GitHub Linux runner is slower and hits memory limits sooner than MacOS, exclude bigger models EXCLUDE_FILTERS = [ '*efficientnet_l2*', '*resnext101_32x48d', '*in21k', '*152x4_bitm', '*101x3_bitm', '*50x3_bitm', '*nfnet_f3*', '*nfnet_f4*', '*nfnet_f5*', '*nfnet_f6*', '*nfnet_f7*', '*efficientnetv2_xl*', - '*resnetrs350*', '*resnetrs420*', 'xcit_large_24_p8*'] + '*resnetrs350*', '*resnetrs420*', 'xcit_large_24_p8*', 'vit_gi*'] else: EXCLUDE_FILTERS = [] @@ -255,7 +255,7 @@ if 'GITHUB_ACTIONS' not in os.environ: EXCLUDE_JIT_FILTERS = [ '*iabn*', 'tresnet*', # models using inplace abn unlikely to ever be scriptable 'dla*', 'hrnet*', 'ghostnet*', # hopefully fix at some point - 'vit_large_*', 'vit_huge_*', + 'vit_large_*', 'vit_huge_*', 'vit_gi*', ] @@ -334,7 +334,7 @@ def _create_fx_model(model, train=False): return fx_model -EXCLUDE_FX_FILTERS = [] +EXCLUDE_FX_FILTERS = ['vit_gi*'] # not enough memory to run fx on more models than other tests if 'GITHUB_ACTIONS' in os.environ: EXCLUDE_FX_FILTERS += [ diff --git a/timm/models/vision_transformer.py b/timm/models/vision_transformer.py index fec44334..5c2346ce 100644 --- a/timm/models/vision_transformer.py +++ b/timm/models/vision_transformer.py @@ -105,6 +105,10 @@ default_cfgs = { 'L_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_384.npz', input_size=(3, 384, 384), crop_pct=1.0), + 'vit_huge_patch14_224': _cfg(url=''), + 'vit_giant_patch14_224': _cfg(url=''), + 'vit_gigantic_patch14_224': _cfg(url=''), + # patch models, imagenet21k (weights from official Google JAX impl) 'vit_tiny_patch16_224_in21k': _cfg( url='https://storage.googleapis.com/vit_models/augreg/Ti_16-i21k-300ep-lr_0.001-aug_none-wd_0.03-do_0.0-sd_0.0.npz', @@ -715,6 +719,33 @@ def vit_base_patch32_sam_224(pretrained=False, **kwargs): return model +@register_model +def vit_huge_patch14_224(pretrained=False, **kwargs): + """ ViT-Huge model (ViT-H/14) from original paper (https://arxiv.org/abs/2010.11929). + """ + model_kwargs = dict(patch_size=14, embed_dim=1280, depth=32, num_heads=16, **kwargs) + model = _create_vision_transformer('vit_huge_patch14_224', pretrained=pretrained, **model_kwargs) + return model + + +@register_model +def vit_giant_patch14_224(pretrained=False, **kwargs): + """ ViT-Giant model (ViT-g/14) from `Scaling Vision Transformers` - https://arxiv.org/abs/2106.04560 + """ + model_kwargs = dict(patch_size=14, embed_dim=1408, mlp_ratio=48/11, depth=40, num_heads=16, **kwargs) + model = _create_vision_transformer('vit_giant_patch14_224', pretrained=pretrained, **model_kwargs) + return model + + +@register_model +def vit_gigantic_patch14_224(pretrained=False, **kwargs): + """ ViT-Gigantic model (ViT-G/14) from `Scaling Vision Transformers` - https://arxiv.org/abs/2106.04560 + """ + model_kwargs = dict(patch_size=14, embed_dim=1664, mlp_ratio=64/13, depth=48, num_heads=16, **kwargs) + model = _create_vision_transformer('vit_gigantic_patch14_224', pretrained=pretrained, **model_kwargs) + return model + + @register_model def vit_tiny_patch16_224_in21k(pretrained=False, **kwargs): """ ViT-Tiny (Vit-Ti/16).