From b5831e3accdd3340bb1b443d4ba5661ee5f95f72 Mon Sep 17 00:00:00 2001 From: goutamyg Date: Mon, 31 Jul 2023 12:20:42 -0400 Subject: [PATCH] fixed the missing normalization of tokens for cross-attention computation --- cvnets/modules/transformer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cvnets/modules/transformer.py b/cvnets/modules/transformer.py index f78e878..190e57c 100644 --- a/cvnets/modules/transformer.py +++ b/cvnets/modules/transformer.py @@ -255,6 +255,7 @@ def forward( # cross-attention res = x x = self.pre_norm_attn[0](x) # norm + x_prev = self.pre_norm_attn[0](x_prev) # normalize x_prev as well! x = self.pre_norm_attn[1](x, x_prev) # attn x = self.pre_norm_attn[2](x) # drop x = x + res # residual