From 62fea452a369cdcf2b473839eeea258252d26310 Mon Sep 17 00:00:00 2001 From: bonlime Date: Mon, 25 May 2020 12:15:36 +0300 Subject: [PATCH] Update activations.py This minor change makes swish and mish inplace by default and saves up to 20% of memory in my tests. --- timm/models/layers/activations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/timm/models/layers/activations.py b/timm/models/layers/activations.py index 6f8d2f89..81b397c5 100644 --- a/timm/models/layers/activations.py +++ b/timm/models/layers/activations.py @@ -18,7 +18,7 @@ if _USE_MEM_EFFICIENT_ISH: # recomputing torch.sigmoid(x) in backward instead of saving it. @torch.jit.script def swish_jit_fwd(x): - return x.mul(torch.sigmoid(x)) + return x.mul_(torch.sigmoid(x)) @torch.jit.script @@ -50,7 +50,7 @@ if _USE_MEM_EFFICIENT_ISH: @torch.jit.script def mish_jit_fwd(x): - return x.mul(torch.tanh(F.softplus(x))) + return x.mul_(torch.tanh(F.softplus(x))) @torch.jit.script