Windows: Improve the check for endianness when using Visual Studio.

Visual studio does not define __BYTE_ORDER__ so all architectures
were detected as LITTLE_ENDIAN since both __BYTE_ORDER__ and
__ORDER_LITTLE_ENDIAN__ would evaluate to 0 and compare equal. This
updates the checks to use CMakes detection of endianness, with a hard
error, if this also fails.

CMakeLists.txt

@@ -186,6 +186,14 @@ if(MSVC)
 	set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -MP -W4 ${MSVC_DISABLED_WARNINGS_STR}")
 endif()
 
+include(TestBigEndian)
+TEST_BIG_ENDIAN(IS_BIG_ENDIAN)
+if(IS_BIG_ENDIAN)
+	add_definitions(-DHAVE_BIG_ENDIAN)
+else()
+	add_definitions(-DHAVE_LITTLE_ENDIAN)
+endif()
+
 check_function_exists(asprintf HAVE_ASPRINTF)
 if(HAVE_ASPRINTF)
 	add_definitions(-DHAVE_ASPRINTF)

include/compat/endian.h

@@ -13,13 +13,21 @@
 #define PDP_ENDIAN 3412
 
 /*
- * Use GCC and Visual Studio compiler defines to determine endian.
+ * Use compiler and build system defines to determine endianness.
  */
+#if defined(__BYTE_ORDER__)
 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 #define BYTE_ORDER LITTLE_ENDIAN
 #else
 #define BYTE_ORDER BIG_ENDIAN
 #endif
+#elif defined(HAVE_LITTLE_ENDIAN)
+#define BYTE_ORDER LITTLE_ENDIAN
+#elif defined(HAVE_BIG_ENDIAN)
+#define BYTE_ORDER BIG_ENDIAN
+#else
+#error "Could not detect endianness."
+#endif
 
 #elif defined(HAVE_ENDIAN_H)
 #include_next <endian.h>