diff --git a/library/portable-simd/crates/core_simd/src/vendor/arm.rs b/library/portable-simd/crates/core_simd/src/vendor/arm.rs
index ff3b69ccf9592..ee5c642137367 100644
--- a/library/portable-simd/crates/core_simd/src/vendor/arm.rs
+++ b/library/portable-simd/crates/core_simd/src/vendor/arm.rs
@@ -7,9 +7,12 @@ use core::arch::arm::*;
 #[cfg(target_arch = "aarch64")]
 use core::arch::aarch64::*;
 
-#[cfg(any(
-    target_arch = "aarch64",
-    all(target_arch = "arm", target_feature = "v7"),
+#[cfg(all(
+    any(
+        target_arch = "aarch64",
+        all(target_arch = "arm", target_feature = "v7"),
+    ),
+    target_endian = "little"
 ))]
 mod neon {
     use super::*;