Just leave it float for now, will look at fp16 later. Remove unused reference code.

timm/models/nn_ops.py

@@ -4,74 +4,6 @@ import torch.nn.functional as F
 import numpy as np
 import math
 
-## Assembled CNN Tensorflow Impl
-#
-# def _bernoulli(shape, mean, seed=None, dtype=tf.float32):
-#     return tf.nn.relu(tf.sign(mean - tf.random_uniform(shape, minval=0, maxval=1, dtype=dtype, seed=seed)))
-#
-#
-# def dropblock(x, keep_prob, block_size, gamma_scale=1.0, seed=None, name=None,
-#               data_format='channels_last', is_training=True):  # pylint: disable=invalid-name
-#     """
-#     Dropblock layer. For more details, refer to https://arxiv.org/abs/1810.12890
-#     :param x: A floating point tensor.
-#     :param keep_prob: A scalar Tensor with the same type as x. The probability that each element is kept.
-#     :param block_size: The block size to drop
-#     :param gamma_scale: The multiplier to gamma.
-#     :param seed:  Python integer. Used to create random seeds.
-#     :param name: A name for this operation (optional)
-#     :param data_format: 'channels_last' or 'channels_first'
-#     :param is_training: If False, do nothing.
-#     :return: A Tensor of the same shape of x.
-#     """
-#     if not is_training:
-#         return x
-#
-#     # Early return if nothing needs to be dropped.
-#     if (isinstance(keep_prob, float) and keep_prob == 1) or gamma_scale == 0:
-#         return x
-#
-#     with tf.name_scope(name, "dropblock", [x]) as name:
-#         if not x.dtype.is_floating:
-#             raise ValueError("x has to be a floating point tensor since it's going to"
-#                              " be scaled. Got a %s tensor instead." % x.dtype)
-#         if isinstance(keep_prob, float) and not 0 < keep_prob <= 1:
-#             raise ValueError("keep_prob must be a scalar tensor or a float in the "
-#                              "range (0, 1], got %g" % keep_prob)
-#
-#         br = (block_size - 1) // 2
-#         tl = (block_size - 1) - br
-#         if data_format == 'channels_last':
-#             _, h, w, c = x.shape.as_list()
-#             sampling_mask_shape = tf.stack([1, h - block_size + 1, w - block_size + 1, c])
-#             pad_shape = [[0, 0], [tl, br], [tl, br], [0, 0]]
-#         elif data_format == 'channels_first':
-#             _, c, h, w = x.shape.as_list()
-#             sampling_mask_shape = tf.stack([1, c, h - block_size + 1, w - block_size + 1])
-#             pad_shape = [[0, 0], [0, 0], [tl, br], [tl, br]]
-#         else:
-#             raise NotImplementedError
-#
-#         gamma = (1. - keep_prob) * (w * h) / (block_size ** 2) / ((w - block_size + 1) * (h - block_size + 1))
-#         gamma = gamma_scale * gamma
-#         mask = _bernoulli(sampling_mask_shape, gamma, seed, tf.float32)
-#         mask = tf.pad(mask, pad_shape)
-#
-#         xdtype_mask = tf.cast(mask, x.dtype)
-#         xdtype_mask = tf.layers.max_pooling2d(
-#             inputs=xdtype_mask, pool_size=block_size,
-#             strides=1, padding='SAME',
-#             data_format=data_format)
-#
-#         xdtype_mask = 1 - xdtype_mask
-#         fp32_mask = tf.cast(xdtype_mask, tf.float32)
-#         ret = tf.multiply(x, xdtype_mask)
-#         float32_mask_size = tf.cast(tf.size(fp32_mask), tf.float32)
-#         float32_mask_reduce_sum = tf.reduce_sum(fp32_mask)
-#         normalize_factor = tf.cast(float32_mask_size / (float32_mask_reduce_sum + 1e-8), x.dtype)
-#         ret = ret * normalize_factor
-#         return ret
-
 
 def drop_block_2d(x, drop_prob=0.1, block_size=7, gamma_scale=1.0, drop_with_noise=False):
     _, _, height, width = x.shape
@@ -89,7 +21,7 @@ def drop_block_2d(x, drop_prob=0.1, block_size=7, gamma_scale=1.0, drop_with_noi
     valid_block = torch.reshape(valid_block, (1, 1, height, width)).float()
 
     uniform_noise = torch.rand_like(x, dtype=torch.float32)
-    block_mask = ((2 - seed_drop_rate - valid_block + uniform_noise) >= 1).to(dtype=x.dtype)
+    block_mask = ((2 - seed_drop_rate - valid_block + uniform_noise) >= 1).float()
     block_mask = -F.max_pool2d(
         -block_mask,
         kernel_size=clipped_block_size,  # block_size,
@@ -100,7 +32,7 @@ def drop_block_2d(x, drop_prob=0.1, block_size=7, gamma_scale=1.0, drop_with_noi
         normal_noise = torch.randn_like(x)
         x = x * block_mask + normal_noise * (1 - block_mask)
     else:
-        normalize_scale = block_mask.numel() / (torch.sum(block_mask, dtype=torch.float32) + 1e-7)
+        normalize_scale = block_mask.numel() / (torch.sum(block_mask) + 1e-7)
         x = x * block_mask * normalize_scale
     return x