zhangyiss/mlx
1
0
Fork 0
mirror of https://github.com/ml-explore/mlx.git synced 2025-07-21 00:31:12 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix normalization check_input (#1452)

Browse Source
This commit is contained in:
Angelos Katharopoulos 2024-10-03 13:26:56 -07:00 committed by GitHub
parent 5900e3249f
commit d878015228
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 147 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
mlx/backend/metal/conv.cpp
View File

@ -914,10 +914,12 @@ void Convolution::eval_gpu(const std::vector<array>& inputs, array& out) {
} }
// Clear copies // Clear copies
if (copies.size() > 0) { if (!copies.empty()) {
auto command_buffer = d.get_command_buffer(s.index); auto command_buffer = d.get_command_buffer(s.index);
command_buffer->addCompletedHandler( command_buffer->addCompletedHandler(
[copies](MTL::CommandBuffer*) mutable { copies.clear(); }); [copies = std::move(copies)](MTL::CommandBuffer*) mutable {
copies.clear();
});
} }
} }

4
mlx/backend/metal/custom_kernel.cpp
View File

@ -81,7 +81,9 @@ void CustomKernel::eval_gpu(
if (!copies.empty()) { if (!copies.empty()) {
d.get_command_buffer(s.index)->addCompletedHandler( d.get_command_buffer(s.index)->addCompletedHandler(
[copies](MTL::CommandBuffer*) mutable { copies.clear(); }); [copies = std::move(copies)](MTL::CommandBuffer*) mutable {
copies.clear();
});
} }
} }

13
mlx/backend/metal/fft.cpp
View File

@ -576,7 +576,9 @@ void fft_op(
if (plan.four_step) { if (plan.four_step) {
four_step_fft(in, out, axis, inverse, real, plan, copies, s); four_step_fft(in, out, axis, inverse, real, plan, copies, s);
d.get_command_buffer(s.index)->addCompletedHandler( d.get_command_buffer(s.index)->addCompletedHandler(
[copies](MTL::CommandBuffer*) mutable { copies.clear(); }); [copies = std::move(copies)](MTL::CommandBuffer*) mutable {
copies.clear();
});
return; return;
} }
@ -741,8 +743,13 @@ void fft_op(
MTL::Size(batch_size, threadgroup_batch_size, threads_per_fft); MTL::Size(batch_size, threadgroup_batch_size, threads_per_fft);
compute_encoder->dispatchThreads(grid_dims, group_dims); compute_encoder->dispatchThreads(grid_dims, group_dims);
} }
d.get_command_buffer(s.index)->addCompletedHandler(
[copies](MTL::CommandBuffer*) mutable { copies.clear(); }); if (!copies.empty()) {
d.get_command_buffer(s.index)->addCompletedHandler(
[copies = std::move(copies)](MTL::CommandBuffer*) mutable {
copies.clear();
});
}
} }
void fft_op( void fft_op(

8
mlx/backend/metal/hadamard.cpp
View File

@ -196,8 +196,12 @@ void Hadamard::eval_gpu(const std::vector<array>& inputs, array& out) {
s); s);
} }
d.get_command_buffer(s.index)->addCompletedHandler( if (!copies.empty()) {
[copies](MTL::CommandBuffer*) mutable { copies.clear(); }); d.get_command_buffer(s.index)->addCompletedHandler(
[copies = std::move(copies)](MTL::CommandBuffer*) mutable {
copies.clear();
});
}
} }
} // namespace mlx::core } // namespace mlx::core

93
mlx/backend/metal/matmul.cpp
View File

@ -227,9 +227,12 @@ void steel_matmul_conv_groups(
compute_encoder.dispatchThreadgroups(grid_dims, group_dims); compute_encoder.dispatchThreadgroups(grid_dims, group_dims);
// Clear copies // Clear copies
d.get_command_buffer(s.index)->addCompletedHandler( if (!copies.empty()) {
[copies](MTL::CommandBuffer*) mutable { copies.clear(); }); d.get_command_buffer(s.index)->addCompletedHandler(
return; [copies = std::move(copies)](MTL::CommandBuffer*) mutable {
copies.clear();
});
}
} }
void steel_matmul( void steel_matmul(
@ -379,8 +382,12 @@ void steel_matmul(
compute_encoder.dispatchThreads(grid_dims, group_dims); compute_encoder.dispatchThreads(grid_dims, group_dims);
} }
d.get_command_buffer(s.index)->addCompletedHandler( if (!copies.empty()) {
[copies](MTL::CommandBuffer*) mutable { copies.clear(); }); d.get_command_buffer(s.index)->addCompletedHandler(
[copies = std::move(copies)](MTL::CommandBuffer*) mutable {
copies.clear();
});
}
return; return;
} }
@ -507,9 +514,12 @@ void steel_matmul(
compute_encoder.dispatchThreadgroups(grid_dims, group_dims); compute_encoder.dispatchThreadgroups(grid_dims, group_dims);
// Clear copies // Clear copies
d.get_command_buffer(s.index)->addCompletedHandler( if (!copies.empty()) {
[copies](MTL::CommandBuffer*) mutable { copies.clear(); }); d.get_command_buffer(s.index)->addCompletedHandler(
return; [copies = std::move(copies)](MTL::CommandBuffer*) mutable {
copies.clear();
});
}
} }
void Matmul::eval_gpu(const std::vector<array>& inputs, array& out) { void Matmul::eval_gpu(const std::vector<array>& inputs, array& out) {
@ -680,8 +690,12 @@ void Matmul::eval_gpu(const std::vector<array>& inputs, array& out) {
compute_encoder.dispatchThreadgroups(grid_dims, group_dims); compute_encoder.dispatchThreadgroups(grid_dims, group_dims);
d.get_command_buffer(s.index)->addCompletedHandler( if (!copies.empty()) {
[copies](MTL::CommandBuffer*) mutable { copies.clear(); }); d.get_command_buffer(s.index)->addCompletedHandler(
[copies = std::move(copies)](MTL::CommandBuffer*) mutable {
copies.clear();
});
}
return; return;
} }
///////////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////
@ -886,8 +900,12 @@ void AddMM::eval_gpu(const std::vector<array>& inputs, array& out) {
compute_encoder.dispatchThreadgroups(grid_dims, group_dims); compute_encoder.dispatchThreadgroups(grid_dims, group_dims);
d.get_command_buffer(s.index)->addCompletedHandler( if (!copies.empty()) {
[copies](MTL::CommandBuffer*) mutable { copies.clear(); }); d.get_command_buffer(s.index)->addCompletedHandler(
[copies = std::move(copies)](MTL::CommandBuffer*) mutable {
copies.clear();
});
}
return; return;
} }
@ -1000,8 +1018,12 @@ void AddMM::eval_gpu(const std::vector<array>& inputs, array& out) {
compute_encoder.dispatchThreads(grid_dims, group_dims); compute_encoder.dispatchThreads(grid_dims, group_dims);
} }
d.get_command_buffer(s.index)->addCompletedHandler( if (!copies.empty()) {
[copies](MTL::CommandBuffer*) mutable { copies.clear(); }); d.get_command_buffer(s.index)->addCompletedHandler(
[copies = std::move(copies)](MTL::CommandBuffer*) mutable {
copies.clear();
});
}
return; return;
} }
@ -1136,9 +1158,12 @@ void AddMM::eval_gpu(const std::vector<array>& inputs, array& out) {
compute_encoder.dispatchThreadgroups(grid_dims, group_dims); compute_encoder.dispatchThreadgroups(grid_dims, group_dims);
d.get_command_buffer(s.index)->addCompletedHandler( if (!copies.empty()) {
[copies](MTL::CommandBuffer*) mutable { copies.clear(); }); d.get_command_buffer(s.index)->addCompletedHandler(
return; [copies = std::move(copies)](MTL::CommandBuffer*) mutable {
copies.clear();
});
}
} }
void BlockMaskedMM::eval_gpu(const std::vector<array>& inputs, array& out) { void BlockMaskedMM::eval_gpu(const std::vector<array>& inputs, array& out) {
@ -1433,8 +1458,12 @@ void BlockMaskedMM::eval_gpu(const std::vector<array>& inputs, array& out) {
compute_encoder.dispatchThreadgroups(grid_dims, group_dims); compute_encoder.dispatchThreadgroups(grid_dims, group_dims);
d.get_command_buffer(s.index)->addCompletedHandler( if (!copies.empty()) {
[copies](MTL::CommandBuffer*) mutable { copies.clear(); }); d.get_command_buffer(s.index)->addCompletedHandler(
[copies = std::move(copies)](MTL::CommandBuffer*) mutable {
copies.clear();
});
}
return; return;
} }
@ -1545,9 +1574,12 @@ void BlockMaskedMM::eval_gpu(const std::vector<array>& inputs, array& out) {
compute_encoder.dispatchThreadgroups(grid_dims, group_dims); compute_encoder.dispatchThreadgroups(grid_dims, group_dims);
// Clear copies // Clear copies
d.get_command_buffer(s.index)->addCompletedHandler( if (!copies.empty()) {
[copies](MTL::CommandBuffer*) mutable { copies.clear(); }); d.get_command_buffer(s.index)->addCompletedHandler(
return; [copies = std::move(copies)](MTL::CommandBuffer*) mutable {
copies.clear();
});
}
} }
void GatherMM::eval_gpu(const std::vector<array>& inputs, array& out) { void GatherMM::eval_gpu(const std::vector<array>& inputs, array& out) {
@ -1773,8 +1805,12 @@ void GatherMM::eval_gpu(const std::vector<array>& inputs, array& out) {
compute_encoder.dispatchThreadgroups(grid_dims, group_dims); compute_encoder.dispatchThreadgroups(grid_dims, group_dims);
d.get_command_buffer(s.index)->addCompletedHandler( if (!copies.empty()) {
[copies](MTL::CommandBuffer*) mutable { copies.clear(); }); d.get_command_buffer(s.index)->addCompletedHandler(
[copies = std::move(copies)](MTL::CommandBuffer*) mutable {
copies.clear();
});
}
return; return;
} }
@ -1914,9 +1950,12 @@ void GatherMM::eval_gpu(const std::vector<array>& inputs, array& out) {
compute_encoder.dispatchThreadgroups(grid_dims, group_dims); compute_encoder.dispatchThreadgroups(grid_dims, group_dims);
// Clear copies // Clear copies
d.get_command_buffer(s.index)->addCompletedHandler( if (!copies.empty()) {
[copies](MTL::CommandBuffer*) mutable { copies.clear(); }); d.get_command_buffer(s.index)->addCompletedHandler(
return; [copies = std::move(copies)](MTL::CommandBuffer*) mutable {
copies.clear();
});
}
} }
} // namespace mlx::core } // namespace mlx::core

36
mlx/backend/metal/normalization.cpp
View File

@ -91,8 +91,12 @@ void RMSNorm::eval_gpu(
compute_encoder->setThreadgroupMemoryLength(simd_size * sizeof(float), 1); compute_encoder->setThreadgroupMemoryLength(simd_size * sizeof(float), 1);
compute_encoder.dispatchThreads(grid_dims, group_dims); compute_encoder.dispatchThreads(grid_dims, group_dims);
} }
d.get_command_buffer(s.index)->addCompletedHandler( if (!copies.empty()) {
[copies](MTL::CommandBuffer*) mutable { copies.clear(); }); d.get_command_buffer(s.index)->addCompletedHandler(
[copies = std::move(copies)](MTL::CommandBuffer*) mutable {
copies.clear();
});
}
} }
void RMSNormVJP::eval_gpu( void RMSNormVJP::eval_gpu(
@ -109,6 +113,12 @@ void RMSNormVJP::eval_gpu(
if (x.flags().row_contiguous) { if (x.flags().row_contiguous) {
return x; return x;
} }
// Make sure we 'll only ever allocate once. The point of that goes beyond
// the minor optimization. We need to ensure that there will be no
// reallocation such that the references won't change when we
// push_back(...). So tl;dr 3 possible copies x, g and gw_temp.
copies.reserve(3);
copies.push_back(array(x.shape(), x.dtype(), nullptr, {})); copies.push_back(array(x.shape(), x.dtype(), nullptr, {}));
copy_gpu(x, copies.back(), CopyType::General, s); copy_gpu(x, copies.back(), CopyType::General, s);
return copies.back(); return copies.back();
@ -195,7 +205,9 @@ void RMSNormVJP::eval_gpu(
gw_temp, gw, "sum", plan, {0}, compute_encoder, d, s); gw_temp, gw, "sum", plan, {0}, compute_encoder, d, s);
d.get_command_buffer(s.index)->addCompletedHandler( d.get_command_buffer(s.index)->addCompletedHandler(
[copies](MTL::CommandBuffer*) mutable { copies.clear(); }); [copies = std::move(copies)](MTL::CommandBuffer*) mutable {
copies.clear();
});
} }
void LayerNorm::eval_gpu( void LayerNorm::eval_gpu(
@ -280,8 +292,12 @@ void LayerNorm::eval_gpu(
compute_encoder->setBytes(&b_stride, sizeof(uint32_t), 7); compute_encoder->setBytes(&b_stride, sizeof(uint32_t), 7);
compute_encoder.dispatchThreads(grid_dims, group_dims); compute_encoder.dispatchThreads(grid_dims, group_dims);
} }
d.get_command_buffer(s.index)->addCompletedHandler( if (!copies.empty()) {
[copies](MTL::CommandBuffer*) mutable { copies.clear(); }); d.get_command_buffer(s.index)->addCompletedHandler(
[copies = std::move(copies)](MTL::CommandBuffer*) mutable {
copies.clear();
});
}
} }
void LayerNormVJP::eval_gpu( void LayerNormVJP::eval_gpu(
@ -298,6 +314,12 @@ void LayerNormVJP::eval_gpu(
if (x.flags().row_contiguous) { if (x.flags().row_contiguous) {
return x; return x;
} }
// Make sure we 'll only ever allocate once. The point of that goes beyond
// the minor optimization. We need to ensure that there will be no
// reallocation such that the references won't change when we
// push_back(...). So tl;dr 3 possible copies x, g and gw_temp.
copies.reserve(3);
copies.push_back(array(x.shape(), x.dtype(), nullptr, {})); copies.push_back(array(x.shape(), x.dtype(), nullptr, {}));
copy_gpu(x, copies.back(), CopyType::General, s); copy_gpu(x, copies.back(), CopyType::General, s);
return copies.back(); return copies.back();
@ -404,7 +426,9 @@ void LayerNormVJP::eval_gpu(
} }
d.get_command_buffer(s.index)->addCompletedHandler( d.get_command_buffer(s.index)->addCompletedHandler(
[copies](MTL::CommandBuffer*) mutable { copies.clear(); }); [copies = std::move(copies)](MTL::CommandBuffer*) mutable {
copies.clear();
});
} }
} // namespace mlx::core::fast } // namespace mlx::core::fast

16
mlx/backend/metal/quantized.cpp
View File

@ -209,8 +209,12 @@ void QuantizedMatmul::eval_gpu(const std::vector<array>& inputs, array& out) {
} }
} }
d.get_command_buffer(s.index)->addCompletedHandler( if (!copies.empty()) {
[copies](MTL::CommandBuffer*) mutable { copies.clear(); }); d.get_command_buffer(s.index)->addCompletedHandler(
[copies = std::move(copies)](MTL::CommandBuffer*) mutable {
copies.clear();
});
}
} }
void GatherQMM::eval_gpu(const std::vector<array>& inputs, array& out) { void GatherQMM::eval_gpu(const std::vector<array>& inputs, array& out) {
@ -599,8 +603,12 @@ void fast::AffineQuantize::eval_gpu(
: MTL::Size(nthreads, 1, 1); : MTL::Size(nthreads, 1, 1);
compute_encoder.dispatchThreads(grid_dims, group_dims); compute_encoder.dispatchThreads(grid_dims, group_dims);
d.get_command_buffer(s.index)->addCompletedHandler( if (!copies.empty()) {
[copies](MTL::CommandBuffer*) mutable { copies.clear(); }); d.get_command_buffer(s.index)->addCompletedHandler(
[copies = std::move(copies)](MTL::CommandBuffer*) mutable {
copies.clear();
});
}
} }
} // namespace mlx::core } // namespace mlx::core

4
mlx/backend/metal/reduce.cpp
View File

@ -662,7 +662,9 @@ void Reduce::eval_gpu(const std::vector<array>& inputs, array& out) {
if (!copies.empty()) { if (!copies.empty()) {
d.get_command_buffer(s.index)->addCompletedHandler( d.get_command_buffer(s.index)->addCompletedHandler(
[copies](MTL::CommandBuffer*) mutable { copies.clear(); }); [copies = std::move(copies)](MTL::CommandBuffer*) mutable {
copies.clear();
});
} }
} }

6
mlx/backend/metal/scan.cpp
View File

@ -107,10 +107,12 @@ void Scan::eval_gpu(const std::vector<array>& inputs, array& out) {
compute_encoder.dispatchThreads(grid_dims, group_dims); compute_encoder.dispatchThreads(grid_dims, group_dims);
} }
if (copies.size() > 0) { if (!copies.empty()) {
auto command_buffer = d.get_command_buffer(s.index); auto command_buffer = d.get_command_buffer(s.index);
command_buffer->addCompletedHandler( command_buffer->addCompletedHandler(
[copies](MTL::CommandBuffer*) mutable { copies.clear(); }); [copies = std::move(copies)](MTL::CommandBuffer*) mutable {
copies.clear();
});
} }
} }

8
mlx/backend/metal/softmax.cpp
View File

@ -88,8 +88,12 @@ void Softmax::eval_gpu(const std::vector<array>& inputs, array& out) {
compute_encoder->setBytes(&axis_size, sizeof(int), 2); compute_encoder->setBytes(&axis_size, sizeof(int), 2);
compute_encoder.dispatchThreads(grid_dims, group_dims); compute_encoder.dispatchThreads(grid_dims, group_dims);
} }
d.get_command_buffer(s.index)->addCompletedHandler( if (!copies.empty()) {
[copies](MTL::CommandBuffer*) mutable { copies.clear(); }); d.get_command_buffer(s.index)->addCompletedHandler(
[copies = std::move(copies)](MTL::CommandBuffer*) mutable {
copies.clear();
});
}
} }
} // namespace mlx::core } // namespace mlx::core

4
mlx/backend/metal/sort.cpp
View File

@ -254,7 +254,9 @@ void multi_block_sort(
// Clear copies // Clear copies
d.get_command_buffer(s.index)->addCompletedHandler( d.get_command_buffer(s.index)->addCompletedHandler(
[copies](MTL::CommandBuffer*) mutable { copies.clear(); }); [copies = std::move(copies)](MTL::CommandBuffer*) mutable {
copies.clear();
});
} }
void gpu_merge_sort( void gpu_merge_sort(