correct non_uniform_interpolation

jshuadvd · Jun 13, 2024 · e9992b3 · e9992b3
1 parent bc209b2
commit e9992b3
Showing 1 changed file with 4 additions and 2 deletions.
diff --git a/src/main.py b/src/main.py
@@ -43,9 +43,11 @@ def non_uniform_interpolation(pos_embed, extension_ratio, lambda_factors, n_hat)
     interpolated_pos = pos_embed.clone()
 
     for i in range(d_model // 2):
-        mask = torch.arange(pos_embed.shape[-2]) < n_hat
+        mask = torch.arange(pos_embed.shape[-2], device=pos_embed.device) < n_hat
         scale = torch.where(
-            mask, torch.ones_like(pos_embed[..., 0]), 1 / lambda_factors[i]
+            mask,
+            torch.ones_like(pos_embed[..., 0], device=pos_embed.device),
+            1 / (lambda_factors[i] * extension_ratio),
         )
         interpolated_pos[..., i * 2] *= scale
         if i * 2 + 1 < d_model:  # Check if the index is within bounds