zhangyiss/mlx
1
0
Fork 0
mirror of https://github.com/ml-explore/mlx.git synced 2025-10-19 00:04:41 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Added ALiBi implementation (#232)

Browse Source
This commit is contained in:
Hazem Essam
2023-12-22 00:36:38 +02:00
committed by GitHub
parent 794feb83df
commit 0aa65c7a6b
3 changed files with 50 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
python/tests/test_nn.py
View File

@@ -570,6 +570,18 @@ class TestNN(mlx_tests.MLXTestCase):
y = rope(x.astype(mx.float16))
self.assertTrue(y.dtype, mx.float16)
def test_alibi(self):
for kwargs in [{"num_heads": 8}]:
alibi = nn.ALibi(**kwargs)
shape = [1, 8, 20, 20]
x = mx.random.uniform(shape=shape)
y = alibi(x)
self.assertTrue(y.shape, shape)
self.assertTrue(y.dtype, mx.float32)
y = alibi(x.astype(mx.float16))
self.assertTrue(y.dtype, mx.float16)
if __name__ == "__main__":
unittest.main()