From 213b436538eb905c261d49775f7973eb7a0669d2 Mon Sep 17 00:00:00 2001 From: Dave Date: Tue, 31 Dec 2019 09:48:23 -0500 Subject: [PATCH] Remove masking on decoding attention --- setup.py | 2 +- text2vec/models/transformer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 7531754..e7572f3 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name="text2vec", - version="0.1", + version="0.1.1", description="Building blocks for text vectorization and embedding", author="Dave Hollander", author_url="https://github.com/brainsqueeze", diff --git a/text2vec/models/transformer.py b/text2vec/models/transformer.py index ecbf11a..e9c3d20 100644 --- a/text2vec/models/transformer.py +++ b/text2vec/models/transformer.py @@ -62,7 +62,7 @@ def __call__(self, x_enc, enc_mask, x_dec, dec_mask, context, attention, embeddi x_dec = self.h_dropout(mha([x_dec] * 3, mask_future=True, training=training), training=training) + x_dec x_dec = self.layer_norm(x_dec) - cross_context = attention(encoded=x_enc * enc_mask, decoded=x_dec * dec_mask) + cross_context = attention(encoded=x_enc, decoded=x_dec) x_dec = self.h_dropout(self.projection(x_dec, projection_vector=cross_context), training=training) + x_dec x_dec = self.layer_norm(x_dec)