fix sigmoidal gate

lucidrains · Jan 6, 2021 · e889fa8 · e889fa8
1 parent 73fa28f
commit e889fa8
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/esbn_pytorch/esbn_pytorch.py b/esbn_pytorch/esbn_pytorch.py
@@ -79,7 +79,7 @@ def forward(self, images):
 
                 # concat confidence to memory keys
                 # then weighted sum of all memory keys by attention of memory values
-                kx = g * (wk * torch.cat((Mk, ck), dim = -1)).sum(dim = 1)
+                kx = g.sigmoid() * (wk * torch.cat((Mk, ck), dim = -1)).sum(dim = 1)
 
             kw, z = map_fn(rearrange, 'b d -> b () d')(kw, z)
             Mk = safe_cat(Mk, kw, dim = 1)

diff --git a/setup.py b/setup.py
@@ -3,7 +3,7 @@
 setup(
   name = 'esbn-pytorch',
   packages = find_packages(),
-  version = '0.0.3',
+  version = '0.0.4',
   license='MIT',
   description = 'Emergent Symbol Binding Network - Pytorch',
   author = 'Phil Wang',