From 2d19cca87f8fd4980748771eca5719fc78d9a142 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 22 Sep 2024 01:26:20 +0400 Subject: [PATCH 001/219] CLD3: try to build for Windows --- .../Native/toolchain-mingw.cmake | 21 ++++++++++++++++ .../Native/toolchain-osxcross.cmake | 7 ++++++ .../Scripts/run-build.sh | 25 +++++++++++++------ .../Scripts/setup-build.sh | 1 + 4 files changed, 47 insertions(+), 7 deletions(-) create mode 100644 src/LanguageIdentification.CLD3.Native/Native/toolchain-mingw.cmake create mode 100644 src/LanguageIdentification.CLD3.Native/Native/toolchain-osxcross.cmake diff --git a/src/LanguageIdentification.CLD3.Native/Native/toolchain-mingw.cmake b/src/LanguageIdentification.CLD3.Native/Native/toolchain-mingw.cmake new file mode 100644 index 00000000..10245ab8 --- /dev/null +++ b/src/LanguageIdentification.CLD3.Native/Native/toolchain-mingw.cmake @@ -0,0 +1,21 @@ +set(CMAKE_SYSTEM_NAME Windows) +set(TOOLCHAIN_PREFIX x86_64-w64-mingw32) +#set(TOOLCHAIN_PREFIX i686-w64-mingw32) + +# cross compilers to use for C and C++ +#set(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}-gcc) +#set(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}-g++) +set(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}-gcc-posix) +set(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}-g++-posix) +set(CMAKE_RC_COMPILER ${TOOLCHAIN_PREFIX}-windres) + +# target environment on the build host system +set(CMAKE_FIND_ROOT_PATH /usr/${TOOLCHAIN_PREFIX} /usr/lib/gcc/${TOOLCHAIN_PREFIX}/7.3-posix) + + +# modify default behavior of FIND_XXX() commands to +# search for headers/libs in the target environment and +# search for programs in the build host environment +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) \ No newline at end of file diff --git a/src/LanguageIdentification.CLD3.Native/Native/toolchain-osxcross.cmake b/src/LanguageIdentification.CLD3.Native/Native/toolchain-osxcross.cmake new file mode 100644 index 00000000..6e1223e1 --- /dev/null +++ b/src/LanguageIdentification.CLD3.Native/Native/toolchain-osxcross.cmake @@ -0,0 +1,7 @@ +set(CMAKE_SYSTEM_NAME Darwin) +set(CMAKE_SYSTEM_VERSION 1) + +# Path to the osxcross toolchain binaries +set(CMAKE_OSX_SYSROOT /usr/local/osxcross/target) +set(CMAKE_C_COMPILER /usr/local/osxcross/bin/o64-clang) +set(CMAKE_CXX_COMPILER /usr/local/osxcross/bin/o64-clang++) \ No newline at end of file diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh b/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh index facae322..43ddff44 100644 --- a/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh +++ b/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh @@ -17,21 +17,32 @@ zx ../Native/monkey-patch.mjs mkdir build cd build + +# Build for Linux cmake .. make -j $(nproc) # make ./language_identifier_main # run tests ./language_identifier_features_test # run tests -cd .. - -echo $(pwd) -ls -R build -cd .. +ls -R -find "$workspace/build" -name "libcld3.so" -exec cp {} libcld3.so \; -rm -rf "$workspace" ldd libcld3.so +cp libcld3.so ../../libcld3.so + +make clean + +# Build for Windows +cmake .. -DCMAKE_TOOLCHAIN_FILE=./toolchain-mingw.cmake +make -j $(nproc) # make + +ls -R + +cp libcld3.dll ../../libcld3.dll +make clean + +# Clean up +rm -rf "$workspace" echo "Goodbye world"; diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh index f54c7eb6..ad06f438 100644 --- a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh +++ b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh @@ -11,6 +11,7 @@ apt -y update apt -y install cmake apt -y install g++ apt -y install nodejs +apt -y install gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 npm install -g zx From 3996476cf8fb7e259318a66a93a1ea2bcf92b28c Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 22 Sep 2024 01:34:46 +0400 Subject: [PATCH 002/219] Update run-build.sh --- src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh b/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh index 43ddff44..d16640c1 100644 --- a/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh +++ b/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh @@ -30,7 +30,7 @@ ls -R ldd libcld3.so cp libcld3.so ../../libcld3.so -make clean +rm -rf * # Build for Windows cmake .. -DCMAKE_TOOLCHAIN_FILE=./toolchain-mingw.cmake @@ -40,7 +40,7 @@ ls -R cp libcld3.dll ../../libcld3.dll -make clean +rm -rf * # Clean up rm -rf "$workspace" From 2c05af5b107ceb7ef49e709d2d6e0b4ea7605208 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 22 Sep 2024 01:37:29 +0400 Subject: [PATCH 003/219] Update LanguageIdentification.CLD3.Native.csproj --- .../LanguageIdentification.CLD3.Native.csproj | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj b/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj index 71bd96dc..99ff1b81 100644 --- a/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj +++ b/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj @@ -62,6 +62,16 @@ + + + PreserveNewest + true + runtimes/win-x64/native + true + false + + + From 40f501221388fde66e8ca574eae7bc48fccfd950 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 22 Sep 2024 01:59:52 +0400 Subject: [PATCH 004/219] Add Windows in library logic --- .../Properties/launchSettings.json | 3 +++ src/LanguageIdentification.CLD3.Native/.gitignore | 2 ++ src/LanguageIdentification.CLD3.Native/CLD3NativeLibrary.cs | 2 +- src/LanguageIdentification.CLD3/CLD3Detector.cs | 2 +- 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.CLD3.ConsoleTest/Properties/launchSettings.json b/src/LanguageIdentification.CLD3.ConsoleTest/Properties/launchSettings.json index 47d960e0..1f434ec6 100644 --- a/src/LanguageIdentification.CLD3.ConsoleTest/Properties/launchSettings.json +++ b/src/LanguageIdentification.CLD3.ConsoleTest/Properties/launchSettings.json @@ -6,6 +6,9 @@ "WSL": { "commandName": "WSL2", "distributionName": "" + }, + "Project": { + "commandName": "Project" } } } \ No newline at end of file diff --git a/src/LanguageIdentification.CLD3.Native/.gitignore b/src/LanguageIdentification.CLD3.Native/.gitignore index 60230aec..2af070e4 100644 --- a/src/LanguageIdentification.CLD3.Native/.gitignore +++ b/src/LanguageIdentification.CLD3.Native/.gitignore @@ -1,2 +1,4 @@ libcld3.so +libcld3.dll +libcld3.dylib build_temp/** diff --git a/src/LanguageIdentification.CLD3.Native/CLD3NativeLibrary.cs b/src/LanguageIdentification.CLD3.Native/CLD3NativeLibrary.cs index c84346a0..5c5835d8 100644 --- a/src/LanguageIdentification.CLD3.Native/CLD3NativeLibrary.cs +++ b/src/LanguageIdentification.CLD3.Native/CLD3NativeLibrary.cs @@ -5,6 +5,6 @@ public static class CLD3NativeLibrary /// /// Name of native binary /// - public const string Name = "libcld3.so"; + public const string Name = "libcld3"; } } diff --git a/src/LanguageIdentification.CLD3/CLD3Detector.cs b/src/LanguageIdentification.CLD3/CLD3Detector.cs index 5b5a1d9e..b241e388 100644 --- a/src/LanguageIdentification.CLD3/CLD3Detector.cs +++ b/src/LanguageIdentification.CLD3/CLD3Detector.cs @@ -17,7 +17,7 @@ public class CLD3Detector : IDisposable public CLD3Detector(int minNumBytes, int maxNumBytes) { - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Linux) && !RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { throw new NotSupportedException( $"{nameof(CLD3Detector)} is not yet supported on {RuntimeInformation.RuntimeIdentifier}" From 919d76abff7fbeac35bc154b07e61dbacabcafd4 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 22 Sep 2024 12:45:31 +0400 Subject: [PATCH 005/219] Update CMakeLists.txt --- src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt index 6b3d4cde..aaf7325b 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt @@ -6,6 +6,8 @@ set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++") + add_definitions(-fPIC) # Position Independent Code add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0) From 156113b9d42b8baef5c4002cd63bc4e31e9ba967 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 22 Sep 2024 13:09:27 +0400 Subject: [PATCH 006/219] Update CMakeLists.txt --- .../Native/CMakeLists.txt | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt index aaf7325b..a976b313 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt @@ -1,4 +1,4 @@ -project(cld3) +ο»Ώproject(cld3) cmake_minimum_required(VERSION 3.9) @@ -6,7 +6,19 @@ set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++") +if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-exceptions -fno-rtti") +elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") + # nop +endif() + +if(CMAKE_SYSTEM_NAME STREQUAL "Windows") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++ -Wl,--start-group -lgcc -lgcc_eh -Wl,--end-group -lmsvcrt") +elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + # nop +else() + # nop +endif() add_definitions(-fPIC) # Position Independent Code add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0) From 5ec53617c5c3a94a3e631b77df090989a60c3ee2 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 22 Sep 2024 13:17:04 +0400 Subject: [PATCH 007/219] Update CMakeLists.txt --- src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt index a976b313..693b7a5f 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt @@ -13,7 +13,7 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") endif() if(CMAKE_SYSTEM_NAME STREQUAL "Windows") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++ -Wl,--start-group -lgcc -lgcc_eh -Wl,--end-group -lmsvcrt") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++ -Wl,--start-group -lgcc -lgcc_eh -Wl,--end-group -lmsvcrt -lpthread") elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") # nop else() From 83e073a2a99a9e59af22d999651e5308ddcf144d Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 22 Sep 2024 13:26:17 +0400 Subject: [PATCH 008/219] Update CMakeLists.txt --- src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt index 693b7a5f..478d6ecb 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt @@ -13,7 +13,7 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") endif() if(CMAKE_SYSTEM_NAME STREQUAL "Windows") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++ -Wl,--start-group -lgcc -lgcc_eh -Wl,--end-group -lmsvcrt -lpthread") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++ -Wl,--start-group -lgcc -lgcc_eh -Wl,--end-group -lmsvcrt -lpthread -lwinpthread") elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") # nop else() From 03c7f82e675f449d02bc503cf907fb1821c95cda Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 22 Sep 2024 13:31:31 +0400 Subject: [PATCH 009/219] Update CMakeLists.txt --- src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt index 478d6ecb..4fb44600 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt @@ -13,7 +13,7 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") endif() if(CMAKE_SYSTEM_NAME STREQUAL "Windows") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++ -Wl,--start-group -lgcc -lgcc_eh -Wl,--end-group -lmsvcrt -lpthread -lwinpthread") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++ -Wl,--start-group -lwinpthread -Wl,--end-group") elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") # nop else() From 3d2f098f7f32e6ce13b432620ba24c4d6dc726ef Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 22 Sep 2024 13:45:16 +0400 Subject: [PATCH 010/219] Update toolchain-mingw.cmake --- .../Native/toolchain-mingw.cmake | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/LanguageIdentification.CLD3.Native/Native/toolchain-mingw.cmake b/src/LanguageIdentification.CLD3.Native/Native/toolchain-mingw.cmake index 10245ab8..94e97c6b 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/toolchain-mingw.cmake +++ b/src/LanguageIdentification.CLD3.Native/Native/toolchain-mingw.cmake @@ -1,21 +1,16 @@ set(CMAKE_SYSTEM_NAME Windows) set(TOOLCHAIN_PREFIX x86_64-w64-mingw32) -#set(TOOLCHAIN_PREFIX i686-w64-mingw32) -# cross compilers to use for C and C++ -#set(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}-gcc) -#set(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}-g++) -set(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}-gcc-posix) -set(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}-g++-posix) +# cross compilers to use for C, C++ and Fortran +set(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}-gcc) +set(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}-g++) +set(CMAKE_Fortran_COMPILER ${TOOLCHAIN_PREFIX}-gfortran) set(CMAKE_RC_COMPILER ${TOOLCHAIN_PREFIX}-windres) # target environment on the build host system -set(CMAKE_FIND_ROOT_PATH /usr/${TOOLCHAIN_PREFIX} /usr/lib/gcc/${TOOLCHAIN_PREFIX}/7.3-posix) +set(CMAKE_FIND_ROOT_PATH /usr/${TOOLCHAIN_PREFIX}) - -# modify default behavior of FIND_XXX() commands to -# search for headers/libs in the target environment and -# search for programs in the build host environment +# modify default behavior of FIND_XXX() commands set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) \ No newline at end of file From 4f23a6e63b9d2c282da67002c225773da4b34cd2 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 22 Sep 2024 14:57:14 +0400 Subject: [PATCH 011/219] CLD3: native naming changes --- .../Native/src/binding.cc | 10 +++++----- .../Native/src/binding.h | 12 ++++++------ src/LanguageIdentification.CLD3/CLD3Detector.cs | 6 +++--- .../Internal/CLD3DetectorWrapper.cs | 16 ++++++++-------- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc b/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc index 807ca6f9..c887d3ee 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc +++ b/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc @@ -3,15 +3,15 @@ using namespace chrome_lang_id; -void* CreateIdentifier(int minNumBytes, int maxNumBytes) { +void* create_cld3(int minNumBytes, int maxNumBytes) { return new NNetLanguageIdentifier(minNumBytes, maxNumBytes); } -void FreeIdentifier(void* identifier) { +void destroy_cld3(void* identifier) { delete static_cast(identifier); } -PredictionResult FindLanguage(void* identifier, const char* text) { +PredictionResult cld3_find_language(void* identifier, const char* text) { NNetLanguageIdentifier* nativeIdentifier = static_cast(identifier); auto nativeResult = nativeIdentifier->FindLanguage(text); @@ -23,7 +23,7 @@ PredictionResult FindLanguage(void* identifier, const char* text) { return result; } -PredictionResult* FindLanguages(void* identifier, const char* text, int numLangs, int* resultCount) { +PredictionResult* cld3_find_languages(void* identifier, const char* text, int numLangs, int* resultCount) { NNetLanguageIdentifier* nativeIdentifier = static_cast(identifier); auto nativeResults = nativeIdentifier->FindTopNMostFreqLangs(text, numLangs); @@ -38,7 +38,7 @@ PredictionResult* FindLanguages(void* identifier, const char* text, int numLangs return result; } -void FreeResults(PredictionResult* results, int count) { +void cld3_destroy_prediction_result(PredictionResult* results, int count) { for (int i = 0; i < count; ++i) { free((void*)results[i].language); } diff --git a/src/LanguageIdentification.CLD3.Native/Native/src/binding.h b/src/LanguageIdentification.CLD3.Native/Native/src/binding.h index 7e2ba7a6..52acb61e 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/src/binding.h +++ b/src/LanguageIdentification.CLD3.Native/Native/src/binding.h @@ -1,4 +1,4 @@ -#pragma once +ο»Ώ#pragma once #include "base.h" #include "nnet_language_identifier.h" @@ -25,9 +25,9 @@ extern "C" { double proportion; }; - EXPORT void* CreateIdentifier(int minNumBytes, int maxNumBytes); - EXPORT void FreeIdentifier(void* identifier); - EXPORT PredictionResult FindLanguage(void* identifier, const char* text); - EXPORT PredictionResult* FindLanguages(void* identifier, const char* text, int numLangs, int* resultCount); - EXPORT void FreeResults(PredictionResult* results, int count); + EXPORT void* create_cld3(int minNumBytes, int maxNumBytes); + EXPORT void destroy_cld3(void* identifier); + EXPORT PredictionResult cld3_find_language(void* identifier, const char* text); + EXPORT PredictionResult* cld3_find_languages(void* identifier, const char* text, int numLangs, int* resultCount); + EXPORT void cld3_destroy_prediction_result(PredictionResult* results, int count); } \ No newline at end of file diff --git a/src/LanguageIdentification.CLD3/CLD3Detector.cs b/src/LanguageIdentification.CLD3/CLD3Detector.cs index b241e388..3555f95e 100644 --- a/src/LanguageIdentification.CLD3/CLD3Detector.cs +++ b/src/LanguageIdentification.CLD3/CLD3Detector.cs @@ -24,7 +24,7 @@ public CLD3Detector(int minNumBytes, int maxNumBytes) ); } - _identifier = CLD3DetectorWrapper.CreateIdentifier(minNumBytes, maxNumBytes); + _identifier = CLD3DetectorWrapper.CreateCLD3(minNumBytes, maxNumBytes); _semaphore = new(1, 1); } @@ -35,7 +35,7 @@ public void Dispose() try { _semaphore.Wait(); - CLD3DetectorWrapper.FreeIdentifier(_identifier); + CLD3DetectorWrapper.DestroyCLD3(_identifier); } finally { @@ -89,7 +89,7 @@ int count } finally { - CLD3DetectorWrapper.FreeResults(resultPtr, resultCount); + CLD3DetectorWrapper.DestroyPredictionResult(resultPtr, resultCount); } } } diff --git a/src/LanguageIdentification.CLD3/Internal/CLD3DetectorWrapper.cs b/src/LanguageIdentification.CLD3/Internal/CLD3DetectorWrapper.cs index 1d856609..d12f44f6 100644 --- a/src/LanguageIdentification.CLD3/Internal/CLD3DetectorWrapper.cs +++ b/src/LanguageIdentification.CLD3/Internal/CLD3DetectorWrapper.cs @@ -6,20 +6,20 @@ namespace Panlingo.LanguageIdentification.CLD3.Internal { internal static class CLD3DetectorWrapper { - [DllImport(CLD3NativeLibrary.Name, CallingConvention = CallingConvention.Cdecl)] - public static extern IntPtr CreateIdentifier(int minNumBytes, int maxNumBytes); + [DllImport(CLD3NativeLibrary.Name, EntryPoint = "create_cld3", CallingConvention = CallingConvention.Cdecl)] + public static extern IntPtr CreateCLD3(int minNumBytes, int maxNumBytes); - [DllImport(CLD3NativeLibrary.Name, CallingConvention = CallingConvention.Cdecl)] - public static extern void FreeIdentifier(IntPtr identifier); + [DllImport(CLD3NativeLibrary.Name, EntryPoint = "destroy_cld3", CallingConvention = CallingConvention.Cdecl)] + public static extern void DestroyCLD3(IntPtr identifier); - [DllImport(CLD3NativeLibrary.Name, CallingConvention = CallingConvention.Cdecl)] + [DllImport(CLD3NativeLibrary.Name, EntryPoint = "cld3_find_language", CallingConvention = CallingConvention.Cdecl)] public static extern CLD3PredictionResult FindLanguage(IntPtr identifier, string text); - [DllImport(CLD3NativeLibrary.Name, CallingConvention = CallingConvention.Cdecl)] + [DllImport(CLD3NativeLibrary.Name, EntryPoint = "cld3_find_languages", CallingConvention = CallingConvention.Cdecl)] public static extern IntPtr FindLanguages(IntPtr identifier, string text, int numLangs, out int resultCount); - [DllImport(CLD3NativeLibrary.Name, CallingConvention = CallingConvention.Cdecl)] - public static extern void FreeResults(IntPtr results, int count); + [DllImport(CLD3NativeLibrary.Name, EntryPoint = "cld3_destroy_prediction_result", CallingConvention = CallingConvention.Cdecl)] + public static extern void DestroyPredictionResult(IntPtr results, int count); } } From dbfa20e76b048502d97a82d7182fb568eb46f312 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 22 Sep 2024 14:57:28 +0400 Subject: [PATCH 012/219] CLD3: fix EXPORT macros --- .../Native/src/binding.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.CLD3.Native/Native/src/binding.h b/src/LanguageIdentification.CLD3.Native/Native/src/binding.h index 52acb61e..d1f4bd95 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/src/binding.h +++ b/src/LanguageIdentification.CLD3.Native/Native/src/binding.h @@ -9,10 +9,14 @@ using namespace std; # ifdef __linux__ # define EXPORT __attribute__((visibility("default"))) # else -# if defined(_MSC_VER) +# if defined(__GNUC__) // MinGW also uses GCC # define EXPORT __declspec(dllexport) # else -# define EXPORT __attribute__((visibility("default"))) +# if defined(_MSC_VER) +# define EXPORT __declspec(dllexport) +# else +# define EXPORT __attribute__((visibility("default"))) +# endif # endif # endif #endif From 115bb2a8b33d8b12bb2830c4e1742e26074eb47a Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 22 Sep 2024 15:34:18 +0400 Subject: [PATCH 013/219] CLD3: try to fix linking for Windows --- .../Native/CMakeLists.txt | 6 +++--- src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh | 6 ++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt index 4fb44600..63ef11ef 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt @@ -63,10 +63,10 @@ set_target_properties(${PROJECT_NAME} PROPERTIES PUBLIC_HEADER "src/binding.h") # unit tests exec: add_executable(language_identifier_main src/language_identifier_main.cc) -target_link_libraries(language_identifier_main cld3) +target_link_libraries(language_identifier_main PRIVATE cld3) add_executable(getonescriptspan_test src/script_span/getonescriptspan_test.cc) -target_link_libraries(getonescriptspan_test cld3) +target_link_libraries(getonescriptspan_test PRIVATE cld3) add_executable(language_identifier_features_test src/language_identifier_features_test.cc) -target_link_libraries(language_identifier_features_test cld3) +target_link_libraries(language_identifier_features_test PRIVATE cld3) diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh b/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh index d16640c1..513a9e73 100644 --- a/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh +++ b/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh @@ -19,6 +19,7 @@ mkdir build cd build # Build for Linux +rm -rf * cmake .. make -j $(nproc) # make @@ -30,9 +31,8 @@ ls -R ldd libcld3.so cp libcld3.so ../../libcld3.so -rm -rf * - # Build for Windows +rm -rf * cmake .. -DCMAKE_TOOLCHAIN_FILE=./toolchain-mingw.cmake make -j $(nproc) # make @@ -40,8 +40,6 @@ ls -R cp libcld3.dll ../../libcld3.dll -rm -rf * - # Clean up rm -rf "$workspace" echo "Goodbye world"; From a31d83a6fcc5e82adfe72c5002b57d7b148d2150 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 22 Sep 2024 16:04:21 +0400 Subject: [PATCH 014/219] Revert "CLD3: fix EXPORT macros" This reverts commit dbfa20e76b048502d97a82d7182fb568eb46f312. --- .../Native/src/binding.h | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/LanguageIdentification.CLD3.Native/Native/src/binding.h b/src/LanguageIdentification.CLD3.Native/Native/src/binding.h index d1f4bd95..52acb61e 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/src/binding.h +++ b/src/LanguageIdentification.CLD3.Native/Native/src/binding.h @@ -9,14 +9,10 @@ using namespace std; # ifdef __linux__ # define EXPORT __attribute__((visibility("default"))) # else -# if defined(__GNUC__) // MinGW also uses GCC +# if defined(_MSC_VER) # define EXPORT __declspec(dllexport) # else -# if defined(_MSC_VER) -# define EXPORT __declspec(dllexport) -# else -# define EXPORT __attribute__((visibility("default"))) -# endif +# define EXPORT __attribute__((visibility("default"))) # endif # endif #endif From 069230fe05068fd241e3a152173442a6a04cd99e Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 22 Sep 2024 17:19:26 +0400 Subject: [PATCH 015/219] Update binding.cc --- src/LanguageIdentification.CLD3.Native/Native/src/binding.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc b/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc index c887d3ee..322f0486 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc +++ b/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc @@ -16,7 +16,7 @@ PredictionResult cld3_find_language(void* identifier, const char* text) { auto nativeResult = nativeIdentifier->FindLanguage(text); PredictionResult result; - result.language = strdup(nativeResult.language.c_str()); + result.language = nativeResult.language.c_str(); result.probability = nativeResult.probability; result.is_reliable = nativeResult.is_reliable; result.proportion = nativeResult.proportion; From 421caa2935e3ec7d5012abc9e07f56e85cfc4581 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 22 Sep 2024 17:27:52 +0400 Subject: [PATCH 016/219] Revert "Update binding.cc" This reverts commit 069230fe05068fd241e3a152173442a6a04cd99e. --- src/LanguageIdentification.CLD3.Native/Native/src/binding.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc b/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc index 322f0486..c887d3ee 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc +++ b/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc @@ -16,7 +16,7 @@ PredictionResult cld3_find_language(void* identifier, const char* text) { auto nativeResult = nativeIdentifier->FindLanguage(text); PredictionResult result; - result.language = nativeResult.language.c_str(); + result.language = strdup(nativeResult.language.c_str()); result.probability = nativeResult.probability; result.is_reliable = nativeResult.is_reliable; result.proportion = nativeResult.proportion; From cd31cc0f38da1b3f983281873d0db68bad9c3356 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 22 Sep 2024 20:58:07 +0400 Subject: [PATCH 017/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 374a4945..81c1efe8 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -87,7 +87,10 @@ jobs: test-nugets: name: πŸ§ͺ Test ${{matrix.projectName}} - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest] needs: build-nugets permissions: contents: read From 45ff22014995ceac19a9b97d5cfb0519f330d5ed Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 22 Sep 2024 21:10:59 +0400 Subject: [PATCH 018/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 35 +++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 81c1efe8..62f39a4b 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -86,11 +86,13 @@ jobs: compression-level: 0 test-nugets: - name: πŸ§ͺ Test ${{matrix.projectName}} - runs-on: ${{ matrix.os }} + name: πŸ§ͺ Test ${{matrix.os}} + runs-on: ${{matrix.os}} strategy: matrix: - os: [ubuntu-latest, windows-latest] + os: + - ubuntu-latest + - windows-latest needs: build-nugets permissions: contents: read @@ -111,21 +113,28 @@ jobs: merge-multiple: true path: ${{env.ROOT}}/local-nugets + - name: πŸ“‚ Setup .NET Core + uses: actions/setup-dotnet@v4 + with: + dotnet-version: | + 5.0.x + 6.0.x + 7.0.x + 8.0.x + source-url: https://nuget.pkg.github.com/${{github.repository_owner}}/index.json + + - name: πŸ“‚ Use local NuGet + run: dotnet nuget add source ${{env.ROOT}}/local-nugets + - name: πŸ“‚ Files working-directory: ${{env.ROOT}} run: ls -R - - name: πŸ“‚ Build Docker Image - working-directory: ${{env.ROOT}} - run: | - docker build -f ./test.Dockerfile -t langunage-identification-test-runner:latest . - docker build -f ./test-ci.Dockerfile -t langunage-identification-test-runner-ci:latest . - - - name: πŸ§ͺ Run Tests in Docker - working-directory: ${{env.ROOT}} - run: docker run --rm -v $(pwd):/src langunage-identification-test-runner-ci:latest + - name: πŸ§ͺ Run Tests for LanguageIdentification + working-directory: ${{env.ROOT}}/LanguageIdentification.Tests + run: dotnet test -c CI - - name: πŸ§ͺ Run Test for LanguageCode + - name: πŸ§ͺ Run Tests for LanguageCode working-directory: ${{env.ROOT}}/LanguageCode.Tests run: dotnet test -c CI From c27559f2b1134ada92bb246467897790897003dd Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 22 Sep 2024 21:25:02 +0400 Subject: [PATCH 019/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 51 ++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 62f39a4b..89725a68 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -85,14 +85,9 @@ jobs: overwrite: 'true' compression-level: 0 - test-nugets: - name: πŸ§ͺ Test ${{matrix.os}} - runs-on: ${{matrix.os}} - strategy: - matrix: - os: - - ubuntu-latest - - windows-latest + test-nugets-linux: + name: πŸ§ͺ Test on Linux + runs-on: ubuntu-latest needs: build-nugets permissions: contents: read @@ -113,6 +108,29 @@ jobs: merge-multiple: true path: ${{env.ROOT}}/local-nugets + - name: πŸ“‚ Files + working-directory: ${{env.ROOT}} + run: ls -R + + - name: πŸ“‚ Build Docker Image + working-directory: ${{env.ROOT}} + run: | + docker build -f ./test.Dockerfile -t langunage-identification-test-runner:latest . + docker build -f ./test-ci.Dockerfile -t langunage-identification-test-runner-ci:latest . + + test-nugets-windows: + name: πŸ§ͺ Test on Windows + runs-on: windows-latest + needs: build-nugets + permissions: + contents: read + packages: write + env: + ROOT: ./src + NUGET_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}} + steps: + - uses: actions/checkout@v4 + - name: πŸ“‚ Setup .NET Core uses: actions/setup-dotnet@v4 with: @@ -123,13 +141,22 @@ jobs: 8.0.x source-url: https://nuget.pkg.github.com/${{github.repository_owner}}/index.json - - name: πŸ“‚ Use local NuGet - run: dotnet nuget add source ${{env.ROOT}}/local-nugets + - name: πŸ“‚ Files + run: mkdir -p ${{env.ROOT}}/local-nugets + + - uses: actions/download-artifact@v4 + with: + pattern: build-* + merge-multiple: true + path: ${{env.ROOT}}/local-nugets - name: πŸ“‚ Files working-directory: ${{env.ROOT}} run: ls -R + - name: πŸ“‚ Use local NuGet + run: dotnet nuget add source ${{env.ROOT}}/local-nugets + - name: πŸ§ͺ Run Tests for LanguageIdentification working-directory: ${{env.ROOT}}/LanguageIdentification.Tests run: dotnet test -c CI @@ -141,7 +168,9 @@ jobs: deploy-nugets: name: 🚚 Push ${{matrix.projectName}} runs-on: ubuntu-latest - needs: test-nugets + needs: + - test-nugets-linux + - test-nugets-windows if: github.ref == 'refs/heads/master' strategy: fail-fast: false From 0b2961bc9891134a66017d0b6c6fdafec849c566 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 22 Sep 2024 21:54:19 +0400 Subject: [PATCH 020/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 89725a68..b671eeda 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -155,7 +155,7 @@ jobs: run: ls -R - name: πŸ“‚ Use local NuGet - run: dotnet nuget add source ${{env.ROOT}}/local-nugets + run: dotnet nuget add source [System.IO.Path]::GetFullPath(${{env.ROOT}}/local-nugets) - name: πŸ§ͺ Run Tests for LanguageIdentification working-directory: ${{env.ROOT}}/LanguageIdentification.Tests From 8e635d66b646824f79a9549b9d382757b6d921c5 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 22 Sep 2024 22:01:21 +0400 Subject: [PATCH 021/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index b671eeda..1d88fb05 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -155,7 +155,9 @@ jobs: run: ls -R - name: πŸ“‚ Use local NuGet - run: dotnet nuget add source [System.IO.Path]::GetFullPath(${{env.ROOT}}/local-nugets) + run: | + $path = [System.IO.Path]::GetFullPath('${{env.ROOT}}/local-nugets') + dotnet nuget add source $path - name: πŸ§ͺ Run Tests for LanguageIdentification working-directory: ${{env.ROOT}}/LanguageIdentification.Tests From 8884eb1003338accebf5171309eb906f948211dd Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 23 Sep 2024 00:39:24 +0400 Subject: [PATCH 022/219] Cross-platform tests: skip if not supported --- .../CLD2Detector.cs | 7 ++++- .../CLD3Detector.cs | 7 ++++- .../FastTextDetector.cs | 7 ++++- .../LinguaDetector.cs | 7 ++++- .../LinguaDetectorBuilder.cs | 2 +- .../MediaPipeDetector.cs | 7 ++++- src/LanguageIdentification.Tests/CLD2Tests.cs | 4 ++- src/LanguageIdentification.Tests/CLD3Tests.cs | 28 +++++++++++++++++-- .../FastTextTests.cs | 16 ++++++++--- .../LanguageIdentification.Tests.csproj | 15 +++++++--- .../LinguaTests.cs | 16 ++++++++--- src/LanguageIdentification.Tests/MainTests.cs | 4 +-- .../MediaPipeTests.cs | 14 +++++++--- .../WhatlangTests.cs | 8 ++++-- .../WhatlangDetector.cs | 7 ++++- 15 files changed, 118 insertions(+), 31 deletions(-) diff --git a/src/LanguageIdentification.CLD2/CLD2Detector.cs b/src/LanguageIdentification.CLD2/CLD2Detector.cs index 4f257853..66f6ddb4 100644 --- a/src/LanguageIdentification.CLD2/CLD2Detector.cs +++ b/src/LanguageIdentification.CLD2/CLD2Detector.cs @@ -13,7 +13,7 @@ public class CLD2Detector : IDisposable { public CLD2Detector() { - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + if (!IsSupported()) { throw new NotSupportedException( $"{nameof(CLD2Detector)} is not yet supported on {RuntimeInformation.RuntimeIdentifier}" @@ -21,6 +21,11 @@ public CLD2Detector() } } + public static bool IsSupported() + { + return RuntimeInformation.IsOSPlatform(OSPlatform.Linux); + } + /// /// Produces a prediction for 'text' /// diff --git a/src/LanguageIdentification.CLD3/CLD3Detector.cs b/src/LanguageIdentification.CLD3/CLD3Detector.cs index 3555f95e..fd7db99f 100644 --- a/src/LanguageIdentification.CLD3/CLD3Detector.cs +++ b/src/LanguageIdentification.CLD3/CLD3Detector.cs @@ -17,7 +17,7 @@ public class CLD3Detector : IDisposable public CLD3Detector(int minNumBytes, int maxNumBytes) { - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Linux) && !RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + if (!IsSupported()) { throw new NotSupportedException( $"{nameof(CLD3Detector)} is not yet supported on {RuntimeInformation.RuntimeIdentifier}" @@ -28,6 +28,11 @@ public CLD3Detector(int minNumBytes, int maxNumBytes) _semaphore = new(1, 1); } + public static bool IsSupported() + { + return RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + } + public void Dispose() { GC.SuppressFinalize(this); diff --git a/src/LanguageIdentification.FastText/FastTextDetector.cs b/src/LanguageIdentification.FastText/FastTextDetector.cs index abdce527..a9a57665 100644 --- a/src/LanguageIdentification.FastText/FastTextDetector.cs +++ b/src/LanguageIdentification.FastText/FastTextDetector.cs @@ -18,7 +18,7 @@ public class FastTextDetector : IDisposable public FastTextDetector() { - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + if (!IsSupported()) { throw new NotSupportedException( $"{nameof(FastTextDetector)} is not yet supported on {RuntimeInformation.RuntimeIdentifier}" @@ -29,6 +29,11 @@ public FastTextDetector() _semaphore = new SemaphoreSlim(1, 1); } + public static bool IsSupported() + { + return RuntimeInformation.IsOSPlatform(OSPlatform.Linux); + } + public string ModelPath { get; private set; } = string.Empty; /// diff --git a/src/LanguageIdentification.Lingua/LinguaDetector.cs b/src/LanguageIdentification.Lingua/LinguaDetector.cs index 9e39e1f5..d5c459c6 100644 --- a/src/LanguageIdentification.Lingua/LinguaDetector.cs +++ b/src/LanguageIdentification.Lingua/LinguaDetector.cs @@ -16,7 +16,7 @@ public class LinguaDetector : IDisposable internal LinguaDetector(LinguaDetectorBuilder builder) { - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + if (!IsSupported()) { throw new NotSupportedException( $"{nameof(LinguaDetector)} is not yet supported on {RuntimeInformation.RuntimeIdentifier}" @@ -30,6 +30,11 @@ internal LinguaDetector(LinguaDetectorBuilder builder) } } + public static bool IsSupported() + { + return RuntimeInformation.IsOSPlatform(OSPlatform.Linux); + } + /// /// Produces a prediction for 'text' /// diff --git a/src/LanguageIdentification.Lingua/LinguaDetectorBuilder.cs b/src/LanguageIdentification.Lingua/LinguaDetectorBuilder.cs index 2e1624ce..a45363c4 100644 --- a/src/LanguageIdentification.Lingua/LinguaDetectorBuilder.cs +++ b/src/LanguageIdentification.Lingua/LinguaDetectorBuilder.cs @@ -14,7 +14,7 @@ public class LinguaDetectorBuilder : IDisposable public LinguaDetectorBuilder(LinguaLanguage[] languages) { - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + if (!IsSupported()) { throw new NotSupportedException( $"{nameof(LinguaDetector)} is not yet supported on {RuntimeInformation.RuntimeIdentifier}" diff --git a/src/LanguageIdentification.MediaPipe/MediaPipeDetector.cs b/src/LanguageIdentification.MediaPipe/MediaPipeDetector.cs index df87aa36..6618a48c 100644 --- a/src/LanguageIdentification.MediaPipe/MediaPipeDetector.cs +++ b/src/LanguageIdentification.MediaPipe/MediaPipeDetector.cs @@ -25,7 +25,7 @@ public MediaPipeDetector(int resultCount = -1, float scoreThreshold = 0.0f, stri public MediaPipeDetector(MediaPipeOptions options) { - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + if (!IsSupported()) { throw new NotSupportedException( $"{nameof(MediaPipeDetector)} is not yet supported on {RuntimeInformation.RuntimeIdentifier}" @@ -91,6 +91,11 @@ public MediaPipeDetector(MediaPipeOptions options) _semaphore = new SemaphoreSlim(1, 1); } + public static bool IsSupported() + { + return RuntimeInformation.IsOSPlatform(OSPlatform.Linux); + } + public IEnumerable PredictLanguages(string text) { var nativeResult = new LanguageDetectorResult(); diff --git a/src/LanguageIdentification.Tests/CLD2Tests.cs b/src/LanguageIdentification.Tests/CLD2Tests.cs index 870bee44..6a39573d 100644 --- a/src/LanguageIdentification.Tests/CLD2Tests.cs +++ b/src/LanguageIdentification.Tests/CLD2Tests.cs @@ -5,12 +5,14 @@ namespace Panlingo.LanguageIdentification.Tests; public class CLD2Tests { - [Theory] + [SkippableTheory] [InlineData("en", Constants.PHRASE_ENG_1, 0.9999)] [InlineData("uk", Constants.PHRASE_UKR_1, 0.9999)] [InlineData("un", Constants.PHRASE_RUS_1, 0.9999)] public void CLD2SingleLanguage(string languageCode, string text, double score) { + Skip.IfNot(CLD2Detector.IsSupported()); + using var cld2 = new CLD2Detector(); var predictions = cld2.PredictLanguage(text); diff --git a/src/LanguageIdentification.Tests/CLD3Tests.cs b/src/LanguageIdentification.Tests/CLD3Tests.cs index 10c0e662..7f7769c0 100644 --- a/src/LanguageIdentification.Tests/CLD3Tests.cs +++ b/src/LanguageIdentification.Tests/CLD3Tests.cs @@ -5,24 +5,46 @@ namespace Panlingo.LanguageIdentification.Tests; public class CLD3Tests { - [Theory] + [SkippableTheory] [InlineData("en", Constants.PHRASE_ENG_1, 0.9985)] [InlineData("uk", Constants.PHRASE_UKR_1, 0.9992)] [InlineData("ru", Constants.PHRASE_RUS_1, 0.9770)] public void CLD3SingleLanguage(string languageCode, string text, double score) { + Skip.IfNot(CLD3Detector.IsSupported()); + using var cld3 = new CLD3Detector(0, 512); var prediction = cld3.PredictLanguage(text: text); var predictions = cld3.PredictLanguages(text: text, count: 3); - var mainLanguage = predictions.FirstOrDefault(); - if (prediction is null || mainLanguage is null) + if (prediction is null) { throw new NullReferenceException(); } Assert.Equal(languageCode, prediction.Language); + Assert.Equal(score, prediction.Probability, Constants.EPSILON); + } + + [SkippableTheory] + [InlineData("en", Constants.PHRASE_ENG_1, 0.9985)] + [InlineData("uk", Constants.PHRASE_UKR_1, 0.9992)] + [InlineData("ru", Constants.PHRASE_RUS_1, 0.9770)] + public void CLD3MixedLanguage(string languageCode, string text, double score) + { + Skip.IfNot(CLD3Detector.IsSupported()); + + using var cld3 = new CLD3Detector(0, 512); + + var predictions = cld3.PredictLanguages(text: text, count: 3); + var mainLanguage = predictions.FirstOrDefault(); + + if (mainLanguage is null) + { + throw new NullReferenceException(); + } + Assert.Equal(languageCode, mainLanguage.Language); Assert.Equal(score, mainLanguage.Probability, Constants.EPSILON); } diff --git a/src/LanguageIdentification.Tests/FastTextTests.cs b/src/LanguageIdentification.Tests/FastTextTests.cs index f7862221..43e5ff38 100644 --- a/src/LanguageIdentification.Tests/FastTextTests.cs +++ b/src/LanguageIdentification.Tests/FastTextTests.cs @@ -5,12 +5,14 @@ namespace Panlingo.LanguageIdentification.Tests; public class FastTextTests { - [Theory] + [SkippableTheory] [InlineData("__label__en", Constants.PHRASE_ENG_1, 0.9955)] [InlineData("__label__uk", Constants.PHRASE_UKR_1, 0.9900)] [InlineData("__label__ru", Constants.PHRASE_RUS_1, 0.9983)] public void FastTextFileSingleLanguage(string languageCode, string text, double score) { + Skip.IfNot(FastTextDetector.IsSupported()); + using var fastText = new FastTextDetector(); var modelPath = "/models/fasttext176.bin"; @@ -28,12 +30,14 @@ public void FastTextFileSingleLanguage(string languageCode, string text, double Assert.Equal(score, mainLanguage.Probability, Constants.EPSILON); } - [Theory] + [SkippableTheory] [InlineData("__label__en", Constants.PHRASE_ENG_1, 0.9955)] [InlineData("__label__uk", Constants.PHRASE_UKR_1, 0.9900)] [InlineData("__label__ru", Constants.PHRASE_RUS_1, 0.9983)] public void FastTextStreamSingleLanguage(string languageCode, string text, double score) { + Skip.IfNot(FastTextDetector.IsSupported()); + using var fastText = new FastTextDetector(); var modelPath = "/models/fasttext176.bin"; @@ -53,12 +57,14 @@ public void FastTextStreamSingleLanguage(string languageCode, string text, doubl Assert.Equal(score, mainLanguage.Probability, Constants.EPSILON); } - [Theory] + [SkippableTheory] [InlineData("__label__en", Constants.PHRASE_ENG_1, 1.0000)] [InlineData("__label__uk", Constants.PHRASE_UKR_1, 0.8511)] [InlineData("__label__ru", Constants.PHRASE_RUS_1, 0.9693)] public void FastTextContainedSingleLanguage(string languageCode, string text, double score) { + Skip.IfNot(FastTextDetector.IsSupported()); + using var fastText = new FastTextDetector(); fastText.LoadDefaultModel(); @@ -74,9 +80,11 @@ public void FastTextContainedSingleLanguage(string languageCode, string text, do Assert.Equal(score, mainLanguage.Probability, Constants.EPSILON); } - [Fact] + [SkippableFact] public void FastTextLabels() { + Skip.IfNot(FastTextDetector.IsSupported()); + using var fastText = new FastTextDetector(); var modelPath = "/models/fasttext176.bin"; diff --git a/src/LanguageIdentification.Tests/LanguageIdentification.Tests.csproj b/src/LanguageIdentification.Tests/LanguageIdentification.Tests.csproj index 501548f8..bc57a4f9 100644 --- a/src/LanguageIdentification.Tests/LanguageIdentification.Tests.csproj +++ b/src/LanguageIdentification.Tests/LanguageIdentification.Tests.csproj @@ -14,10 +14,17 @@ - - - - + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + diff --git a/src/LanguageIdentification.Tests/LinguaTests.cs b/src/LanguageIdentification.Tests/LinguaTests.cs index 6544a915..8c142e1e 100644 --- a/src/LanguageIdentification.Tests/LinguaTests.cs +++ b/src/LanguageIdentification.Tests/LinguaTests.cs @@ -5,12 +5,14 @@ namespace Panlingo.LanguageIdentification.Tests; public class LinguaTests { - [Theory] + [SkippableTheory] [InlineData(LinguaLanguage.English, Constants.PHRASE_ENG_1, 0.1666)] [InlineData(LinguaLanguage.Ukrainian, Constants.PHRASE_UKR_1, 0.8228)] [InlineData(LinguaLanguage.Russian, Constants.PHRASE_RUS_1, 0.3502)] public void LinguaSingleLanguage(LinguaLanguage languageCode, string text, double score) { + Skip.IfNot(LinguaDetector.IsSupported()); + using var linguaBuilder = new LinguaDetectorBuilder(Enum.GetValues()); using var lingua = linguaBuilder.Build(); @@ -26,12 +28,14 @@ public void LinguaSingleLanguage(LinguaLanguage languageCode, string text, doubl Assert.Equal(score, mainLanguage.Confidence, Constants.EPSILON); } - [Theory] + [SkippableTheory] [InlineData(LinguaLanguage.English, Constants.PHRASE_ENG_1, 0.1666)] [InlineData(LinguaLanguage.Ukrainian, Constants.PHRASE_UKR_1, 0.8228)] [InlineData(LinguaLanguage.Russian, Constants.PHRASE_RUS_1, 0.3502)] public void LinguaMixedLanguage(LinguaLanguage languageCode, string text, double score) { + Skip.IfNot(LinguaDetector.IsSupported()); + using var linguaBuilder = new LinguaDetectorBuilder(Enum.GetValues()); using var lingua = linguaBuilder.Build(); @@ -47,7 +51,7 @@ public void LinguaMixedLanguage(LinguaLanguage languageCode, string text, double Assert.Equal(score, mainLanguage.Confidence, Constants.EPSILON); } - [Theory] + [SkippableTheory] [InlineData(LinguaLanguage.Ukrainian, LinguaLanguageCode.Alpha2, "uk")] [InlineData(LinguaLanguage.Ukrainian, LinguaLanguageCode.Alpha3, "ukr")] [InlineData(LinguaLanguage.Hebrew, LinguaLanguageCode.Alpha2, "he")] @@ -56,6 +60,8 @@ public void LinguaMixedLanguage(LinguaLanguage languageCode, string text, double [InlineData(LinguaLanguage.Serbian, LinguaLanguageCode.Alpha3, "srp")] public void LinguaGetLanguageCode(LinguaLanguage language, LinguaLanguageCode type, string code) { + Skip.IfNot(LinguaDetector.IsSupported()); + using var linguaBuilder = new LinguaDetectorBuilder(Enum.GetValues()); using var lingua = linguaBuilder.Build(); @@ -63,9 +69,11 @@ public void LinguaGetLanguageCode(LinguaLanguage language, LinguaLanguageCode ty Assert.Equal(code, languageCode); } - [Fact] + [SkippableFact] public void LinguaBuilderReuse() { + Skip.IfNot(LinguaDetector.IsSupported()); + using var linguaBuilder = new LinguaDetectorBuilder(Enum.GetValues()); using var lingua1 = linguaBuilder.Build(); using var lingua2 = linguaBuilder.Build(); diff --git a/src/LanguageIdentification.Tests/MainTests.cs b/src/LanguageIdentification.Tests/MainTests.cs index 12eb4f88..82cbd5d1 100644 --- a/src/LanguageIdentification.Tests/MainTests.cs +++ b/src/LanguageIdentification.Tests/MainTests.cs @@ -5,10 +5,10 @@ public class MainTests /// /// Checks the current OS and container environment /// - [Fact] + [SkippableFact] public void CheckPlatform() { - Assert.Equal(PlatformID.Unix, Environment.OSVersion.Platform); + Skip.IfNot(Environment.OSVersion.Platform == PlatformID.Unix); Assert.Equal("true", Environment.GetEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER")); } } diff --git a/src/LanguageIdentification.Tests/MediaPipeTests.cs b/src/LanguageIdentification.Tests/MediaPipeTests.cs index c9f698d5..2ff614a5 100644 --- a/src/LanguageIdentification.Tests/MediaPipeTests.cs +++ b/src/LanguageIdentification.Tests/MediaPipeTests.cs @@ -6,12 +6,14 @@ namespace Panlingo.LanguageIdentification.Tests; public class MediaPipeTests { - [Theory] + [SkippableTheory] [InlineData("en", Constants.PHRASE_ENG_1, 0.9994)] [InlineData("uk", Constants.PHRASE_UKR_1, 0.9999)] [InlineData("ru", Constants.PHRASE_RUS_1, 0.9999)] public void MediaPipeFileSingleLanguage(string languageCode, string text, double score) { + Skip.IfNot(MediaPipeDetector.IsSupported()); + var modelPath = "/models/mediapipe_language_detector.tflite"; using var mediaPipe = new MediaPipeDetector( options: MediaPipeOptions.FromFile(modelPath).WithResultCount(10) @@ -28,13 +30,15 @@ public void MediaPipeFileSingleLanguage(string languageCode, string text, double Assert.Equal(languageCode, mainLanguage.Language); Assert.Equal(score, mainLanguage.Probability, Constants.EPSILON); } - - [Theory] + + [SkippableTheory] [InlineData("en", Constants.PHRASE_ENG_1, 0.9994)] [InlineData("uk", Constants.PHRASE_UKR_1, 0.9999)] [InlineData("ru", Constants.PHRASE_RUS_1, 0.9999)] public void MediaPipeStreamSingleLanguage(string languageCode, string text, double score) { + Skip.IfNot(MediaPipeDetector.IsSupported()); + var modelPath = "/models/mediapipe_language_detector.tflite"; using var stream = File.Open(modelPath, FileMode.Open); @@ -54,12 +58,14 @@ public void MediaPipeStreamSingleLanguage(string languageCode, string text, doub Assert.Equal(score, mainLanguage.Probability, Constants.EPSILON); } - [Theory] + [SkippableTheory] [InlineData("en", Constants.PHRASE_ENG_1, 0.9994)] [InlineData("uk", Constants.PHRASE_UKR_1, 0.9999)] [InlineData("ru", Constants.PHRASE_RUS_1, 0.9999)] public void MediaPipeContainedSingleLanguage(string languageCode, string text, double score) { + Skip.IfNot(MediaPipeDetector.IsSupported()); + var modelPath = "/models/mediapipe_language_detector.tflite"; using var stream = File.Open(modelPath, FileMode.Open); diff --git a/src/LanguageIdentification.Tests/WhatlangTests.cs b/src/LanguageIdentification.Tests/WhatlangTests.cs index 5ea352d6..74207c2b 100644 --- a/src/LanguageIdentification.Tests/WhatlangTests.cs +++ b/src/LanguageIdentification.Tests/WhatlangTests.cs @@ -5,12 +5,14 @@ namespace Panlingo.LanguageIdentification.Tests; public class WhatlangTests { - [Theory] + [SkippableTheory] [InlineData(WhatlangLanguage.Ron, Constants.PHRASE_ENG_1, 0.0274)] [InlineData(WhatlangLanguage.Ukr, Constants.PHRASE_UKR_1, 0.9999)] [InlineData(WhatlangLanguage.Rus, Constants.PHRASE_RUS_1, 0.2308)] public void WhatlangSingleLanguage(WhatlangLanguage languageCode, string text, double score) { + Skip.IfNot(WhatlangDetector.IsSupported()); + using var whatlang = new WhatlangDetector(); var prediction = whatlang.PredictLanguage(text: text); @@ -24,13 +26,15 @@ public void WhatlangSingleLanguage(WhatlangLanguage languageCode, string text, d Assert.Equal(score, prediction.Confidence, Constants.EPSILON); } - [Theory] + [SkippableTheory] [InlineData(WhatlangLanguage.Ukr, "ukr")] [InlineData(WhatlangLanguage.Uzb, "uzb")] [InlineData(WhatlangLanguage.Heb, "heb")] [InlineData(WhatlangLanguage.Srp, "srp")] public void WhatlangGetLanguageCode(WhatlangLanguage language, string code) { + Skip.IfNot(WhatlangDetector.IsSupported()); + using var whatlang = new WhatlangDetector(); var languageCode = whatlang.GetLanguageCode(language); diff --git a/src/LanguageIdentification.Whatlang/WhatlangDetector.cs b/src/LanguageIdentification.Whatlang/WhatlangDetector.cs index fd3076eb..42229fcb 100644 --- a/src/LanguageIdentification.Whatlang/WhatlangDetector.cs +++ b/src/LanguageIdentification.Whatlang/WhatlangDetector.cs @@ -12,7 +12,7 @@ public class WhatlangDetector : IDisposable { public WhatlangDetector() { - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + if (!IsSupported()) { throw new NotSupportedException( $"{nameof(WhatlangDetector)} is not yet supported on {RuntimeInformation.RuntimeIdentifier}" @@ -20,6 +20,11 @@ public WhatlangDetector() } } + public static bool IsSupported() + { + return RuntimeInformation.IsOSPlatform(OSPlatform.Linux); + } + /// /// Produces a prediction for 'text' /// From dc536d24a91c6569b086a0a66de1cbeba66732fb Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 23 Sep 2024 00:44:53 +0400 Subject: [PATCH 023/219] Update LinguaDetectorBuilder.cs --- src/LanguageIdentification.Lingua/LinguaDetectorBuilder.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.Lingua/LinguaDetectorBuilder.cs b/src/LanguageIdentification.Lingua/LinguaDetectorBuilder.cs index a45363c4..70f1221f 100644 --- a/src/LanguageIdentification.Lingua/LinguaDetectorBuilder.cs +++ b/src/LanguageIdentification.Lingua/LinguaDetectorBuilder.cs @@ -14,7 +14,7 @@ public class LinguaDetectorBuilder : IDisposable public LinguaDetectorBuilder(LinguaLanguage[] languages) { - if (!IsSupported()) + if (!LinguaDetector.IsSupported()) { throw new NotSupportedException( $"{nameof(LinguaDetector)} is not yet supported on {RuntimeInformation.RuntimeIdentifier}" From b0d4223d0b80f7f2f9eaa645d927a4c977df5d4d Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 23 Sep 2024 00:46:09 +0400 Subject: [PATCH 024/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 1d88fb05..dd47394a 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -15,13 +15,13 @@ jobs: fail-fast: false matrix: projectName: - - LanguageIdentification.CLD2 + # - LanguageIdentification.CLD2 - LanguageIdentification.CLD3 - - LanguageIdentification.FastText - - LanguageIdentification.Whatlang - - LanguageIdentification.MediaPipe - - LanguageIdentification.Lingua - - LanguageCode + # - LanguageIdentification.FastText + # - LanguageIdentification.Whatlang + # - LanguageIdentification.MediaPipe + # - LanguageIdentification.Lingua + # - LanguageCode permissions: contents: read packages: write From b5bf5e6891a5766dc0da4e35ff0fe4232db8e1c3 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 23 Sep 2024 00:39:24 +0400 Subject: [PATCH 025/219] Cross-platform tests: skip if not supported --- .../CLD2Detector.cs | 7 ++++- .../CLD3Detector.cs | 7 ++++- .../FastTextDetector.cs | 7 ++++- .../LinguaDetector.cs | 7 ++++- .../LinguaDetectorBuilder.cs | 2 +- .../MediaPipeDetector.cs | 7 ++++- src/LanguageIdentification.Tests/CLD2Tests.cs | 4 ++- src/LanguageIdentification.Tests/CLD3Tests.cs | 28 +++++++++++++++++-- .../FastTextTests.cs | 16 ++++++++--- .../LanguageIdentification.Tests.csproj | 15 +++++++--- .../LinguaTests.cs | 16 ++++++++--- src/LanguageIdentification.Tests/MainTests.cs | 4 +-- .../MediaPipeTests.cs | 14 +++++++--- .../WhatlangTests.cs | 8 ++++-- .../WhatlangDetector.cs | 7 ++++- 15 files changed, 118 insertions(+), 31 deletions(-) diff --git a/src/LanguageIdentification.CLD2/CLD2Detector.cs b/src/LanguageIdentification.CLD2/CLD2Detector.cs index 4f257853..66f6ddb4 100644 --- a/src/LanguageIdentification.CLD2/CLD2Detector.cs +++ b/src/LanguageIdentification.CLD2/CLD2Detector.cs @@ -13,7 +13,7 @@ public class CLD2Detector : IDisposable { public CLD2Detector() { - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + if (!IsSupported()) { throw new NotSupportedException( $"{nameof(CLD2Detector)} is not yet supported on {RuntimeInformation.RuntimeIdentifier}" @@ -21,6 +21,11 @@ public CLD2Detector() } } + public static bool IsSupported() + { + return RuntimeInformation.IsOSPlatform(OSPlatform.Linux); + } + /// /// Produces a prediction for 'text' /// diff --git a/src/LanguageIdentification.CLD3/CLD3Detector.cs b/src/LanguageIdentification.CLD3/CLD3Detector.cs index 5b5a1d9e..51c6b1f9 100644 --- a/src/LanguageIdentification.CLD3/CLD3Detector.cs +++ b/src/LanguageIdentification.CLD3/CLD3Detector.cs @@ -17,7 +17,7 @@ public class CLD3Detector : IDisposable public CLD3Detector(int minNumBytes, int maxNumBytes) { - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + if (!IsSupported()) { throw new NotSupportedException( $"{nameof(CLD3Detector)} is not yet supported on {RuntimeInformation.RuntimeIdentifier}" @@ -28,6 +28,11 @@ public CLD3Detector(int minNumBytes, int maxNumBytes) _semaphore = new(1, 1); } + public static bool IsSupported() + { + return RuntimeInformation.IsOSPlatform(OSPlatform.Linux); + } + public void Dispose() { GC.SuppressFinalize(this); diff --git a/src/LanguageIdentification.FastText/FastTextDetector.cs b/src/LanguageIdentification.FastText/FastTextDetector.cs index abdce527..a9a57665 100644 --- a/src/LanguageIdentification.FastText/FastTextDetector.cs +++ b/src/LanguageIdentification.FastText/FastTextDetector.cs @@ -18,7 +18,7 @@ public class FastTextDetector : IDisposable public FastTextDetector() { - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + if (!IsSupported()) { throw new NotSupportedException( $"{nameof(FastTextDetector)} is not yet supported on {RuntimeInformation.RuntimeIdentifier}" @@ -29,6 +29,11 @@ public FastTextDetector() _semaphore = new SemaphoreSlim(1, 1); } + public static bool IsSupported() + { + return RuntimeInformation.IsOSPlatform(OSPlatform.Linux); + } + public string ModelPath { get; private set; } = string.Empty; /// diff --git a/src/LanguageIdentification.Lingua/LinguaDetector.cs b/src/LanguageIdentification.Lingua/LinguaDetector.cs index 9e39e1f5..d5c459c6 100644 --- a/src/LanguageIdentification.Lingua/LinguaDetector.cs +++ b/src/LanguageIdentification.Lingua/LinguaDetector.cs @@ -16,7 +16,7 @@ public class LinguaDetector : IDisposable internal LinguaDetector(LinguaDetectorBuilder builder) { - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + if (!IsSupported()) { throw new NotSupportedException( $"{nameof(LinguaDetector)} is not yet supported on {RuntimeInformation.RuntimeIdentifier}" @@ -30,6 +30,11 @@ internal LinguaDetector(LinguaDetectorBuilder builder) } } + public static bool IsSupported() + { + return RuntimeInformation.IsOSPlatform(OSPlatform.Linux); + } + /// /// Produces a prediction for 'text' /// diff --git a/src/LanguageIdentification.Lingua/LinguaDetectorBuilder.cs b/src/LanguageIdentification.Lingua/LinguaDetectorBuilder.cs index 2e1624ce..a45363c4 100644 --- a/src/LanguageIdentification.Lingua/LinguaDetectorBuilder.cs +++ b/src/LanguageIdentification.Lingua/LinguaDetectorBuilder.cs @@ -14,7 +14,7 @@ public class LinguaDetectorBuilder : IDisposable public LinguaDetectorBuilder(LinguaLanguage[] languages) { - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + if (!IsSupported()) { throw new NotSupportedException( $"{nameof(LinguaDetector)} is not yet supported on {RuntimeInformation.RuntimeIdentifier}" diff --git a/src/LanguageIdentification.MediaPipe/MediaPipeDetector.cs b/src/LanguageIdentification.MediaPipe/MediaPipeDetector.cs index df87aa36..6618a48c 100644 --- a/src/LanguageIdentification.MediaPipe/MediaPipeDetector.cs +++ b/src/LanguageIdentification.MediaPipe/MediaPipeDetector.cs @@ -25,7 +25,7 @@ public MediaPipeDetector(int resultCount = -1, float scoreThreshold = 0.0f, stri public MediaPipeDetector(MediaPipeOptions options) { - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + if (!IsSupported()) { throw new NotSupportedException( $"{nameof(MediaPipeDetector)} is not yet supported on {RuntimeInformation.RuntimeIdentifier}" @@ -91,6 +91,11 @@ public MediaPipeDetector(MediaPipeOptions options) _semaphore = new SemaphoreSlim(1, 1); } + public static bool IsSupported() + { + return RuntimeInformation.IsOSPlatform(OSPlatform.Linux); + } + public IEnumerable PredictLanguages(string text) { var nativeResult = new LanguageDetectorResult(); diff --git a/src/LanguageIdentification.Tests/CLD2Tests.cs b/src/LanguageIdentification.Tests/CLD2Tests.cs index 870bee44..6a39573d 100644 --- a/src/LanguageIdentification.Tests/CLD2Tests.cs +++ b/src/LanguageIdentification.Tests/CLD2Tests.cs @@ -5,12 +5,14 @@ namespace Panlingo.LanguageIdentification.Tests; public class CLD2Tests { - [Theory] + [SkippableTheory] [InlineData("en", Constants.PHRASE_ENG_1, 0.9999)] [InlineData("uk", Constants.PHRASE_UKR_1, 0.9999)] [InlineData("un", Constants.PHRASE_RUS_1, 0.9999)] public void CLD2SingleLanguage(string languageCode, string text, double score) { + Skip.IfNot(CLD2Detector.IsSupported()); + using var cld2 = new CLD2Detector(); var predictions = cld2.PredictLanguage(text); diff --git a/src/LanguageIdentification.Tests/CLD3Tests.cs b/src/LanguageIdentification.Tests/CLD3Tests.cs index 10c0e662..7f7769c0 100644 --- a/src/LanguageIdentification.Tests/CLD3Tests.cs +++ b/src/LanguageIdentification.Tests/CLD3Tests.cs @@ -5,24 +5,46 @@ namespace Panlingo.LanguageIdentification.Tests; public class CLD3Tests { - [Theory] + [SkippableTheory] [InlineData("en", Constants.PHRASE_ENG_1, 0.9985)] [InlineData("uk", Constants.PHRASE_UKR_1, 0.9992)] [InlineData("ru", Constants.PHRASE_RUS_1, 0.9770)] public void CLD3SingleLanguage(string languageCode, string text, double score) { + Skip.IfNot(CLD3Detector.IsSupported()); + using var cld3 = new CLD3Detector(0, 512); var prediction = cld3.PredictLanguage(text: text); var predictions = cld3.PredictLanguages(text: text, count: 3); - var mainLanguage = predictions.FirstOrDefault(); - if (prediction is null || mainLanguage is null) + if (prediction is null) { throw new NullReferenceException(); } Assert.Equal(languageCode, prediction.Language); + Assert.Equal(score, prediction.Probability, Constants.EPSILON); + } + + [SkippableTheory] + [InlineData("en", Constants.PHRASE_ENG_1, 0.9985)] + [InlineData("uk", Constants.PHRASE_UKR_1, 0.9992)] + [InlineData("ru", Constants.PHRASE_RUS_1, 0.9770)] + public void CLD3MixedLanguage(string languageCode, string text, double score) + { + Skip.IfNot(CLD3Detector.IsSupported()); + + using var cld3 = new CLD3Detector(0, 512); + + var predictions = cld3.PredictLanguages(text: text, count: 3); + var mainLanguage = predictions.FirstOrDefault(); + + if (mainLanguage is null) + { + throw new NullReferenceException(); + } + Assert.Equal(languageCode, mainLanguage.Language); Assert.Equal(score, mainLanguage.Probability, Constants.EPSILON); } diff --git a/src/LanguageIdentification.Tests/FastTextTests.cs b/src/LanguageIdentification.Tests/FastTextTests.cs index f7862221..43e5ff38 100644 --- a/src/LanguageIdentification.Tests/FastTextTests.cs +++ b/src/LanguageIdentification.Tests/FastTextTests.cs @@ -5,12 +5,14 @@ namespace Panlingo.LanguageIdentification.Tests; public class FastTextTests { - [Theory] + [SkippableTheory] [InlineData("__label__en", Constants.PHRASE_ENG_1, 0.9955)] [InlineData("__label__uk", Constants.PHRASE_UKR_1, 0.9900)] [InlineData("__label__ru", Constants.PHRASE_RUS_1, 0.9983)] public void FastTextFileSingleLanguage(string languageCode, string text, double score) { + Skip.IfNot(FastTextDetector.IsSupported()); + using var fastText = new FastTextDetector(); var modelPath = "/models/fasttext176.bin"; @@ -28,12 +30,14 @@ public void FastTextFileSingleLanguage(string languageCode, string text, double Assert.Equal(score, mainLanguage.Probability, Constants.EPSILON); } - [Theory] + [SkippableTheory] [InlineData("__label__en", Constants.PHRASE_ENG_1, 0.9955)] [InlineData("__label__uk", Constants.PHRASE_UKR_1, 0.9900)] [InlineData("__label__ru", Constants.PHRASE_RUS_1, 0.9983)] public void FastTextStreamSingleLanguage(string languageCode, string text, double score) { + Skip.IfNot(FastTextDetector.IsSupported()); + using var fastText = new FastTextDetector(); var modelPath = "/models/fasttext176.bin"; @@ -53,12 +57,14 @@ public void FastTextStreamSingleLanguage(string languageCode, string text, doubl Assert.Equal(score, mainLanguage.Probability, Constants.EPSILON); } - [Theory] + [SkippableTheory] [InlineData("__label__en", Constants.PHRASE_ENG_1, 1.0000)] [InlineData("__label__uk", Constants.PHRASE_UKR_1, 0.8511)] [InlineData("__label__ru", Constants.PHRASE_RUS_1, 0.9693)] public void FastTextContainedSingleLanguage(string languageCode, string text, double score) { + Skip.IfNot(FastTextDetector.IsSupported()); + using var fastText = new FastTextDetector(); fastText.LoadDefaultModel(); @@ -74,9 +80,11 @@ public void FastTextContainedSingleLanguage(string languageCode, string text, do Assert.Equal(score, mainLanguage.Probability, Constants.EPSILON); } - [Fact] + [SkippableFact] public void FastTextLabels() { + Skip.IfNot(FastTextDetector.IsSupported()); + using var fastText = new FastTextDetector(); var modelPath = "/models/fasttext176.bin"; diff --git a/src/LanguageIdentification.Tests/LanguageIdentification.Tests.csproj b/src/LanguageIdentification.Tests/LanguageIdentification.Tests.csproj index 501548f8..bc57a4f9 100644 --- a/src/LanguageIdentification.Tests/LanguageIdentification.Tests.csproj +++ b/src/LanguageIdentification.Tests/LanguageIdentification.Tests.csproj @@ -14,10 +14,17 @@ - - - - + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + diff --git a/src/LanguageIdentification.Tests/LinguaTests.cs b/src/LanguageIdentification.Tests/LinguaTests.cs index 6544a915..8c142e1e 100644 --- a/src/LanguageIdentification.Tests/LinguaTests.cs +++ b/src/LanguageIdentification.Tests/LinguaTests.cs @@ -5,12 +5,14 @@ namespace Panlingo.LanguageIdentification.Tests; public class LinguaTests { - [Theory] + [SkippableTheory] [InlineData(LinguaLanguage.English, Constants.PHRASE_ENG_1, 0.1666)] [InlineData(LinguaLanguage.Ukrainian, Constants.PHRASE_UKR_1, 0.8228)] [InlineData(LinguaLanguage.Russian, Constants.PHRASE_RUS_1, 0.3502)] public void LinguaSingleLanguage(LinguaLanguage languageCode, string text, double score) { + Skip.IfNot(LinguaDetector.IsSupported()); + using var linguaBuilder = new LinguaDetectorBuilder(Enum.GetValues()); using var lingua = linguaBuilder.Build(); @@ -26,12 +28,14 @@ public void LinguaSingleLanguage(LinguaLanguage languageCode, string text, doubl Assert.Equal(score, mainLanguage.Confidence, Constants.EPSILON); } - [Theory] + [SkippableTheory] [InlineData(LinguaLanguage.English, Constants.PHRASE_ENG_1, 0.1666)] [InlineData(LinguaLanguage.Ukrainian, Constants.PHRASE_UKR_1, 0.8228)] [InlineData(LinguaLanguage.Russian, Constants.PHRASE_RUS_1, 0.3502)] public void LinguaMixedLanguage(LinguaLanguage languageCode, string text, double score) { + Skip.IfNot(LinguaDetector.IsSupported()); + using var linguaBuilder = new LinguaDetectorBuilder(Enum.GetValues()); using var lingua = linguaBuilder.Build(); @@ -47,7 +51,7 @@ public void LinguaMixedLanguage(LinguaLanguage languageCode, string text, double Assert.Equal(score, mainLanguage.Confidence, Constants.EPSILON); } - [Theory] + [SkippableTheory] [InlineData(LinguaLanguage.Ukrainian, LinguaLanguageCode.Alpha2, "uk")] [InlineData(LinguaLanguage.Ukrainian, LinguaLanguageCode.Alpha3, "ukr")] [InlineData(LinguaLanguage.Hebrew, LinguaLanguageCode.Alpha2, "he")] @@ -56,6 +60,8 @@ public void LinguaMixedLanguage(LinguaLanguage languageCode, string text, double [InlineData(LinguaLanguage.Serbian, LinguaLanguageCode.Alpha3, "srp")] public void LinguaGetLanguageCode(LinguaLanguage language, LinguaLanguageCode type, string code) { + Skip.IfNot(LinguaDetector.IsSupported()); + using var linguaBuilder = new LinguaDetectorBuilder(Enum.GetValues()); using var lingua = linguaBuilder.Build(); @@ -63,9 +69,11 @@ public void LinguaGetLanguageCode(LinguaLanguage language, LinguaLanguageCode ty Assert.Equal(code, languageCode); } - [Fact] + [SkippableFact] public void LinguaBuilderReuse() { + Skip.IfNot(LinguaDetector.IsSupported()); + using var linguaBuilder = new LinguaDetectorBuilder(Enum.GetValues()); using var lingua1 = linguaBuilder.Build(); using var lingua2 = linguaBuilder.Build(); diff --git a/src/LanguageIdentification.Tests/MainTests.cs b/src/LanguageIdentification.Tests/MainTests.cs index 12eb4f88..82cbd5d1 100644 --- a/src/LanguageIdentification.Tests/MainTests.cs +++ b/src/LanguageIdentification.Tests/MainTests.cs @@ -5,10 +5,10 @@ public class MainTests /// /// Checks the current OS and container environment /// - [Fact] + [SkippableFact] public void CheckPlatform() { - Assert.Equal(PlatformID.Unix, Environment.OSVersion.Platform); + Skip.IfNot(Environment.OSVersion.Platform == PlatformID.Unix); Assert.Equal("true", Environment.GetEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER")); } } diff --git a/src/LanguageIdentification.Tests/MediaPipeTests.cs b/src/LanguageIdentification.Tests/MediaPipeTests.cs index c9f698d5..2ff614a5 100644 --- a/src/LanguageIdentification.Tests/MediaPipeTests.cs +++ b/src/LanguageIdentification.Tests/MediaPipeTests.cs @@ -6,12 +6,14 @@ namespace Panlingo.LanguageIdentification.Tests; public class MediaPipeTests { - [Theory] + [SkippableTheory] [InlineData("en", Constants.PHRASE_ENG_1, 0.9994)] [InlineData("uk", Constants.PHRASE_UKR_1, 0.9999)] [InlineData("ru", Constants.PHRASE_RUS_1, 0.9999)] public void MediaPipeFileSingleLanguage(string languageCode, string text, double score) { + Skip.IfNot(MediaPipeDetector.IsSupported()); + var modelPath = "/models/mediapipe_language_detector.tflite"; using var mediaPipe = new MediaPipeDetector( options: MediaPipeOptions.FromFile(modelPath).WithResultCount(10) @@ -28,13 +30,15 @@ public void MediaPipeFileSingleLanguage(string languageCode, string text, double Assert.Equal(languageCode, mainLanguage.Language); Assert.Equal(score, mainLanguage.Probability, Constants.EPSILON); } - - [Theory] + + [SkippableTheory] [InlineData("en", Constants.PHRASE_ENG_1, 0.9994)] [InlineData("uk", Constants.PHRASE_UKR_1, 0.9999)] [InlineData("ru", Constants.PHRASE_RUS_1, 0.9999)] public void MediaPipeStreamSingleLanguage(string languageCode, string text, double score) { + Skip.IfNot(MediaPipeDetector.IsSupported()); + var modelPath = "/models/mediapipe_language_detector.tflite"; using var stream = File.Open(modelPath, FileMode.Open); @@ -54,12 +58,14 @@ public void MediaPipeStreamSingleLanguage(string languageCode, string text, doub Assert.Equal(score, mainLanguage.Probability, Constants.EPSILON); } - [Theory] + [SkippableTheory] [InlineData("en", Constants.PHRASE_ENG_1, 0.9994)] [InlineData("uk", Constants.PHRASE_UKR_1, 0.9999)] [InlineData("ru", Constants.PHRASE_RUS_1, 0.9999)] public void MediaPipeContainedSingleLanguage(string languageCode, string text, double score) { + Skip.IfNot(MediaPipeDetector.IsSupported()); + var modelPath = "/models/mediapipe_language_detector.tflite"; using var stream = File.Open(modelPath, FileMode.Open); diff --git a/src/LanguageIdentification.Tests/WhatlangTests.cs b/src/LanguageIdentification.Tests/WhatlangTests.cs index 5ea352d6..74207c2b 100644 --- a/src/LanguageIdentification.Tests/WhatlangTests.cs +++ b/src/LanguageIdentification.Tests/WhatlangTests.cs @@ -5,12 +5,14 @@ namespace Panlingo.LanguageIdentification.Tests; public class WhatlangTests { - [Theory] + [SkippableTheory] [InlineData(WhatlangLanguage.Ron, Constants.PHRASE_ENG_1, 0.0274)] [InlineData(WhatlangLanguage.Ukr, Constants.PHRASE_UKR_1, 0.9999)] [InlineData(WhatlangLanguage.Rus, Constants.PHRASE_RUS_1, 0.2308)] public void WhatlangSingleLanguage(WhatlangLanguage languageCode, string text, double score) { + Skip.IfNot(WhatlangDetector.IsSupported()); + using var whatlang = new WhatlangDetector(); var prediction = whatlang.PredictLanguage(text: text); @@ -24,13 +26,15 @@ public void WhatlangSingleLanguage(WhatlangLanguage languageCode, string text, d Assert.Equal(score, prediction.Confidence, Constants.EPSILON); } - [Theory] + [SkippableTheory] [InlineData(WhatlangLanguage.Ukr, "ukr")] [InlineData(WhatlangLanguage.Uzb, "uzb")] [InlineData(WhatlangLanguage.Heb, "heb")] [InlineData(WhatlangLanguage.Srp, "srp")] public void WhatlangGetLanguageCode(WhatlangLanguage language, string code) { + Skip.IfNot(WhatlangDetector.IsSupported()); + using var whatlang = new WhatlangDetector(); var languageCode = whatlang.GetLanguageCode(language); diff --git a/src/LanguageIdentification.Whatlang/WhatlangDetector.cs b/src/LanguageIdentification.Whatlang/WhatlangDetector.cs index fd3076eb..42229fcb 100644 --- a/src/LanguageIdentification.Whatlang/WhatlangDetector.cs +++ b/src/LanguageIdentification.Whatlang/WhatlangDetector.cs @@ -12,7 +12,7 @@ public class WhatlangDetector : IDisposable { public WhatlangDetector() { - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + if (!IsSupported()) { throw new NotSupportedException( $"{nameof(WhatlangDetector)} is not yet supported on {RuntimeInformation.RuntimeIdentifier}" @@ -20,6 +20,11 @@ public WhatlangDetector() } } + public static bool IsSupported() + { + return RuntimeInformation.IsOSPlatform(OSPlatform.Linux); + } + /// /// Produces a prediction for 'text' /// From e559565570bf295aef96f30112c5a01af73ca5a8 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 23 Sep 2024 00:44:53 +0400 Subject: [PATCH 026/219] Update LinguaDetectorBuilder.cs --- src/LanguageIdentification.Lingua/LinguaDetectorBuilder.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.Lingua/LinguaDetectorBuilder.cs b/src/LanguageIdentification.Lingua/LinguaDetectorBuilder.cs index a45363c4..70f1221f 100644 --- a/src/LanguageIdentification.Lingua/LinguaDetectorBuilder.cs +++ b/src/LanguageIdentification.Lingua/LinguaDetectorBuilder.cs @@ -14,7 +14,7 @@ public class LinguaDetectorBuilder : IDisposable public LinguaDetectorBuilder(LinguaLanguage[] languages) { - if (!IsSupported()) + if (!LinguaDetector.IsSupported()) { throw new NotSupportedException( $"{nameof(LinguaDetector)} is not yet supported on {RuntimeInformation.RuntimeIdentifier}" From e97666983c3237c43f99f83399c008fff3deaca6 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 23 Sep 2024 00:55:59 +0400 Subject: [PATCH 027/219] 0.0.0.21 -> 0.0.0.22 --- src/LanguageCode/LanguageCode.csproj | 2 +- .../LanguageIdentification.CLD2.Native.csproj | 2 +- .../LanguageIdentification.CLD2.csproj | 2 +- .../LanguageIdentification.CLD3.Native.csproj | 2 +- .../LanguageIdentification.CLD3.csproj | 2 +- .../LanguageIdentification.FastText.Native.csproj | 2 +- .../LanguageIdentification.FastText.csproj | 2 +- .../LanguageIdentification.Lingua.Native.csproj | 2 +- .../LanguageIdentification.Lingua.csproj | 2 +- .../LanguageIdentification.MediaPipe.Native.csproj | 2 +- .../LanguageIdentification.MediaPipe.csproj | 2 +- .../LanguageIdentification.Whatlang.Native.csproj | 2 +- .../LanguageIdentification.Whatlang.csproj | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/LanguageCode/LanguageCode.csproj b/src/LanguageCode/LanguageCode.csproj index 8813ba77..e2f3f039 100644 --- a/src/LanguageCode/LanguageCode.csproj +++ b/src/LanguageCode/LanguageCode.csproj @@ -2,7 +2,7 @@ netstandard2.1;net5.0;net6.0;net7.0;net8.0 - 0.0.0.21 + 0.0.0.22 Panlingo.LanguageCode Panlingo.LanguageCode Panlingo.LanguageCode diff --git a/src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj b/src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj index fdec2604..446587b0 100644 --- a/src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj +++ b/src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.0.0.21 + 0.0.0.22 Panlingo.LanguageIdentification.CLD2.Native Panlingo.LanguageIdentification.CLD2.Native Panlingo.LanguageIdentification.CLD2.Native diff --git a/src/LanguageIdentification.CLD2/LanguageIdentification.CLD2.csproj b/src/LanguageIdentification.CLD2/LanguageIdentification.CLD2.csproj index 71ca49e8..429bd006 100644 --- a/src/LanguageIdentification.CLD2/LanguageIdentification.CLD2.csproj +++ b/src/LanguageIdentification.CLD2/LanguageIdentification.CLD2.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.0.0.21 + 0.0.0.22 Panlingo.LanguageIdentification.CLD2 Panlingo.LanguageIdentification.CLD2 Panlingo.LanguageIdentification.CLD2 diff --git a/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj b/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj index 71bd96dc..50037442 100644 --- a/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj +++ b/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.0.0.21 + 0.0.0.22 Panlingo.LanguageIdentification.CLD3.Native Panlingo.LanguageIdentification.CLD3.Native Panlingo.LanguageIdentification.CLD3.Native diff --git a/src/LanguageIdentification.CLD3/LanguageIdentification.CLD3.csproj b/src/LanguageIdentification.CLD3/LanguageIdentification.CLD3.csproj index 4e6828bc..7e0a156a 100644 --- a/src/LanguageIdentification.CLD3/LanguageIdentification.CLD3.csproj +++ b/src/LanguageIdentification.CLD3/LanguageIdentification.CLD3.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.0.0.21 + 0.0.0.22 Panlingo.LanguageIdentification.CLD3 Panlingo.LanguageIdentification.CLD3 Panlingo.LanguageIdentification.CLD3 diff --git a/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj b/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj index 375f561b..74d9cefb 100644 --- a/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj +++ b/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.0.0.21 + 0.0.0.22 Panlingo.LanguageIdentification.FastText.Native Panlingo.LanguageIdentification.FastText.Native Panlingo.LanguageIdentification.FastText.Native diff --git a/src/LanguageIdentification.FastText/LanguageIdentification.FastText.csproj b/src/LanguageIdentification.FastText/LanguageIdentification.FastText.csproj index 089e5159..5d7d3a7b 100644 --- a/src/LanguageIdentification.FastText/LanguageIdentification.FastText.csproj +++ b/src/LanguageIdentification.FastText/LanguageIdentification.FastText.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.0.0.21 + 0.0.0.22 Panlingo.LanguageIdentification.FastText Panlingo.LanguageIdentification.FastText Panlingo.LanguageIdentification.FastText diff --git a/src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj b/src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj index 317961a2..343f9d67 100644 --- a/src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj +++ b/src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.0.0.21 + 0.0.0.22 Panlingo.LanguageIdentification.Lingua.Native Panlingo.LanguageIdentification.Lingua.Native Panlingo.LanguageIdentification.Lingua.Native diff --git a/src/LanguageIdentification.Lingua/LanguageIdentification.Lingua.csproj b/src/LanguageIdentification.Lingua/LanguageIdentification.Lingua.csproj index 16c4a33b..e54e0244 100644 --- a/src/LanguageIdentification.Lingua/LanguageIdentification.Lingua.csproj +++ b/src/LanguageIdentification.Lingua/LanguageIdentification.Lingua.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.0.0.21 + 0.0.0.22 Panlingo.LanguageIdentification.Lingua Panlingo.LanguageIdentification.Lingua Panlingo.LanguageIdentification.Lingua diff --git a/src/LanguageIdentification.MediaPipe.Native/LanguageIdentification.MediaPipe.Native.csproj b/src/LanguageIdentification.MediaPipe.Native/LanguageIdentification.MediaPipe.Native.csproj index 7cee9456..2b02a786 100644 --- a/src/LanguageIdentification.MediaPipe.Native/LanguageIdentification.MediaPipe.Native.csproj +++ b/src/LanguageIdentification.MediaPipe.Native/LanguageIdentification.MediaPipe.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.0.0.21 + 0.0.0.22 Panlingo.LanguageIdentification.MediaPipe.Native Panlingo.LanguageIdentification.MediaPipe.Native Panlingo.LanguageIdentification.MediaPipe.Native diff --git a/src/LanguageIdentification.MediaPipe/LanguageIdentification.MediaPipe.csproj b/src/LanguageIdentification.MediaPipe/LanguageIdentification.MediaPipe.csproj index e296f9e9..4b7ebff2 100644 --- a/src/LanguageIdentification.MediaPipe/LanguageIdentification.MediaPipe.csproj +++ b/src/LanguageIdentification.MediaPipe/LanguageIdentification.MediaPipe.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.0.0.21 + 0.0.0.22 Panlingo.LanguageIdentification.MediaPipe Panlingo.LanguageIdentification.MediaPipe Panlingo.LanguageIdentification.MediaPipe diff --git a/src/LanguageIdentification.Whatlang.Native/LanguageIdentification.Whatlang.Native.csproj b/src/LanguageIdentification.Whatlang.Native/LanguageIdentification.Whatlang.Native.csproj index 95a71f99..8e9bf555 100644 --- a/src/LanguageIdentification.Whatlang.Native/LanguageIdentification.Whatlang.Native.csproj +++ b/src/LanguageIdentification.Whatlang.Native/LanguageIdentification.Whatlang.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.0.0.21 + 0.0.0.22 Panlingo.LanguageIdentification.Whatlang.Native Panlingo.LanguageIdentification.Whatlang.Native Panlingo.LanguageIdentification.Whatlang.Native diff --git a/src/LanguageIdentification.Whatlang/LanguageIdentification.Whatlang.csproj b/src/LanguageIdentification.Whatlang/LanguageIdentification.Whatlang.csproj index 7f6fef1a..52dbcd78 100644 --- a/src/LanguageIdentification.Whatlang/LanguageIdentification.Whatlang.csproj +++ b/src/LanguageIdentification.Whatlang/LanguageIdentification.Whatlang.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.0.0.21 + 0.0.0.22 Panlingo.LanguageIdentification.Whatlang Panlingo.LanguageIdentification.Whatlang Panlingo.LanguageIdentification.Whatlang From 78f3330f93e9448d14408a69fd5677b9ac1128ff Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 23 Sep 2024 21:56:22 +0400 Subject: [PATCH 028/219] GitHub CI: add tests for MacOS --- .github/workflows/github-ci.yml | 50 +++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index dd47394a..63d4193a 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -167,12 +167,62 @@ jobs: working-directory: ${{env.ROOT}}/LanguageCode.Tests run: dotnet test -c CI + test-nugets-macos: + name: πŸ§ͺ Test on macOS + runs-on: macos-latest + needs: build-nugets + permissions: + contents: read + packages: write + env: + ROOT: ./src + NUGET_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}} + steps: + - uses: actions/checkout@v4 + + - name: πŸ“‚ Setup .NET Core + uses: actions/setup-dotnet@v4 + with: + dotnet-version: | + 5.0.x + 6.0.x + 7.0.x + 8.0.x + source-url: https://nuget.pkg.github.com/${{github.repository_owner}}/index.json + + - name: πŸ“‚ Files + run: mkdir -p ${{env.ROOT}}/local-nugets + + - uses: actions/download-artifact@v4 + with: + pattern: build-* + merge-multiple: true + path: ${{env.ROOT}}/local-nugets + + - name: πŸ“‚ Files + working-directory: ${{env.ROOT}} + run: ls -R + + - name: πŸ“‚ Use local NuGet + run: | + path=$(realpath "${{env.ROOT}}/local-nugets") + dotnet nuget add source $path + + - name: πŸ§ͺ Run Tests for LanguageIdentification + working-directory: ${{env.ROOT}}/LanguageIdentification.Tests + run: dotnet test -c CI + + - name: πŸ§ͺ Run Tests for LanguageCode + working-directory: ${{env.ROOT}}/LanguageCode.Tests + run: dotnet test -c CI + deploy-nugets: name: 🚚 Push ${{matrix.projectName}} runs-on: ubuntu-latest needs: - test-nugets-linux - test-nugets-windows + - test-nugets-macos if: github.ref == 'refs/heads/master' strategy: fail-fast: false From 3e744d563c8939271bfc1434c24c64f0ad70ee2d Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 23 Sep 2024 21:57:54 +0400 Subject: [PATCH 029/219] CLD2: add support for Windows --- .../Properties/launchSettings.json | 7 ++++++ .../.gitignore | 1 + .../CLD2NativeLibrary.cs | 2 +- .../LanguageIdentification.CLD2.Native.csproj | 10 +++++++++ .../Native/CMakeLists.txt | 21 +++++++++++++++--- .../Native/binding.cc | 2 +- .../Native/toolchain-mingw.cmake | 16 ++++++++++++++ .../Native/toolchain-osxcross.cmake | 7 ++++++ .../Scripts/run-build.sh | 22 ++++++++++++++----- .../Scripts/setup-build.sh | 1 + .../CLD2Detector.cs | 2 +- .../LanguageIdentification.CLD3.Native.csproj | 2 +- 12 files changed, 80 insertions(+), 13 deletions(-) create mode 100644 src/LanguageIdentification.CLD2.Native/Native/toolchain-mingw.cmake create mode 100644 src/LanguageIdentification.CLD2.Native/Native/toolchain-osxcross.cmake diff --git a/src/LanguageIdentification.CLD2.ConsoleTest/Properties/launchSettings.json b/src/LanguageIdentification.CLD2.ConsoleTest/Properties/launchSettings.json index 65b8965a..1f434ec6 100644 --- a/src/LanguageIdentification.CLD2.ConsoleTest/Properties/launchSettings.json +++ b/src/LanguageIdentification.CLD2.ConsoleTest/Properties/launchSettings.json @@ -2,6 +2,13 @@ "profiles": { "Docker": { "commandName": "Docker" + }, + "WSL": { + "commandName": "WSL2", + "distributionName": "" + }, + "Project": { + "commandName": "Project" } } } \ No newline at end of file diff --git a/src/LanguageIdentification.CLD2.Native/.gitignore b/src/LanguageIdentification.CLD2.Native/.gitignore index 162500b1..585d5a3f 100644 --- a/src/LanguageIdentification.CLD2.Native/.gitignore +++ b/src/LanguageIdentification.CLD2.Native/.gitignore @@ -1,2 +1,3 @@ libcld2.so +libcld2.dll build_temp/** diff --git a/src/LanguageIdentification.CLD2.Native/CLD2NativeLibrary.cs b/src/LanguageIdentification.CLD2.Native/CLD2NativeLibrary.cs index 5b952e65..7afcdeba 100644 --- a/src/LanguageIdentification.CLD2.Native/CLD2NativeLibrary.cs +++ b/src/LanguageIdentification.CLD2.Native/CLD2NativeLibrary.cs @@ -5,6 +5,6 @@ public static class CLD2NativeLibrary /// /// Name of native binary /// - public const string Name = "libcld2.so"; + public const string Name = "libcld2"; } } diff --git a/src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj b/src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj index 446587b0..08c83a4c 100644 --- a/src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj +++ b/src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj @@ -62,6 +62,16 @@ + + + PreserveNewest + true + runtimes/win-x64/native + true + false + + + diff --git a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt index 4d6a56b7..c147bb1d 100644 --- a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt @@ -1,5 +1,6 @@ +project(cld2) + cmake_minimum_required(VERSION 2.8 FATAL_ERROR) -project(cld2_bridge) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") @@ -12,6 +13,20 @@ elseif(CMAKE_CXX_COMPILER_ID MATCHES "MSVC") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4267") endif() +if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-exceptions -fno-rtti") +elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") + # nop +endif() + +if(CMAKE_SYSTEM_NAME STREQUAL "Windows") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++ -Wl,--start-group -lwinpthread -Wl,--end-group") +elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + # nop +else() + # nop +endif() + set(CMAKE_MACOSX_RPATH 1) set(CMAKE_POSITION_INDEPENDENT_CODE ON) @@ -101,6 +116,6 @@ set(CLD2_SOURCES add_library(objlib OBJECT ${CLD2_SOURCES}) -add_library(cld2 SHARED $) +add_library(${PROJECT_NAME} SHARED $) -set_target_properties(cld2 PROPERTIES PUBLIC_HEADER "${PROJECT_SOURCE_DIR}/cld2/binding.h") \ No newline at end of file +set_target_properties(${PROJECT_NAME} PROPERTIES PUBLIC_HEADER "${PROJECT_SOURCE_DIR}/cld2/binding.h") \ No newline at end of file diff --git a/src/LanguageIdentification.CLD2.Native/Native/binding.cc b/src/LanguageIdentification.CLD2.Native/Native/binding.cc index 43207036..62153eb6 100644 --- a/src/LanguageIdentification.CLD2.Native/Native/binding.cc +++ b/src/LanguageIdentification.CLD2.Native/Native/binding.cc @@ -62,7 +62,7 @@ extern "C" for (int i = 0; i < predictionCount; ++i) { CLD2::Language language = languages[i]; - double probability = scoreTotal > 0 ? scores[i] / (double)scoreTotal : 1.0; + double probability = scoreTotal > 0 ? scores[i] / (double)scoreTotal : 0; double proportion = percents[i] / 100.0; result[i].language = strdup(CLD2::LanguageCode(language)); diff --git a/src/LanguageIdentification.CLD2.Native/Native/toolchain-mingw.cmake b/src/LanguageIdentification.CLD2.Native/Native/toolchain-mingw.cmake new file mode 100644 index 00000000..94e97c6b --- /dev/null +++ b/src/LanguageIdentification.CLD2.Native/Native/toolchain-mingw.cmake @@ -0,0 +1,16 @@ +set(CMAKE_SYSTEM_NAME Windows) +set(TOOLCHAIN_PREFIX x86_64-w64-mingw32) + +# cross compilers to use for C, C++ and Fortran +set(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}-gcc) +set(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}-g++) +set(CMAKE_Fortran_COMPILER ${TOOLCHAIN_PREFIX}-gfortran) +set(CMAKE_RC_COMPILER ${TOOLCHAIN_PREFIX}-windres) + +# target environment on the build host system +set(CMAKE_FIND_ROOT_PATH /usr/${TOOLCHAIN_PREFIX}) + +# modify default behavior of FIND_XXX() commands +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) \ No newline at end of file diff --git a/src/LanguageIdentification.CLD2.Native/Native/toolchain-osxcross.cmake b/src/LanguageIdentification.CLD2.Native/Native/toolchain-osxcross.cmake new file mode 100644 index 00000000..6e1223e1 --- /dev/null +++ b/src/LanguageIdentification.CLD2.Native/Native/toolchain-osxcross.cmake @@ -0,0 +1,7 @@ +set(CMAKE_SYSTEM_NAME Darwin) +set(CMAKE_SYSTEM_VERSION 1) + +# Path to the osxcross toolchain binaries +set(CMAKE_OSX_SYSROOT /usr/local/osxcross/target) +set(CMAKE_C_COMPILER /usr/local/osxcross/bin/o64-clang) +set(CMAKE_CXX_COMPILER /usr/local/osxcross/bin/o64-clang++) \ No newline at end of file diff --git a/src/LanguageIdentification.CLD2.Native/Scripts/run-build.sh b/src/LanguageIdentification.CLD2.Native/Scripts/run-build.sh index f46c6d06..2b9571b6 100644 --- a/src/LanguageIdentification.CLD2.Native/Scripts/run-build.sh +++ b/src/LanguageIdentification.CLD2.Native/Scripts/run-build.sh @@ -15,17 +15,27 @@ cd "$workspace" mkdir build cd build + +# Build for Linux +rm -rf * cmake .. make -j $(nproc) # make -cd .. -echo $(pwd) -ls -R build -cd .. +ls -R -find "$workspace/build" -name "libcld2.so" -exec cp {} libcld2.so \; -rm -rf "$workspace" ldd libcld2.so +cp libcld2.so ../../libcld2.so + +# Build for Windows +rm -rf * +cmake .. -DCMAKE_TOOLCHAIN_FILE=./toolchain-mingw.cmake +make -j $(nproc) # make +ls -R + +cp libcld2.dll ../../libcld2.dll + +# Clean up +rm -rf "$workspace" echo "Goodbye world"; diff --git a/src/LanguageIdentification.CLD2.Native/Scripts/setup-build.sh b/src/LanguageIdentification.CLD2.Native/Scripts/setup-build.sh index 4f5708d0..c0ebe9b5 100644 --- a/src/LanguageIdentification.CLD2.Native/Scripts/setup-build.sh +++ b/src/LanguageIdentification.CLD2.Native/Scripts/setup-build.sh @@ -6,4 +6,5 @@ echo "Installing build packages"; sudo apt -y update | apt -y update sudo apt -y install cmake | apt -y install cmake sudo apt -y install g++ | apt -y install g++ +sudo apt -y install gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 | apt -y install gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 diff --git a/src/LanguageIdentification.CLD2/CLD2Detector.cs b/src/LanguageIdentification.CLD2/CLD2Detector.cs index 66f6ddb4..b1866b14 100644 --- a/src/LanguageIdentification.CLD2/CLD2Detector.cs +++ b/src/LanguageIdentification.CLD2/CLD2Detector.cs @@ -23,7 +23,7 @@ public CLD2Detector() public static bool IsSupported() { - return RuntimeInformation.IsOSPlatform(OSPlatform.Linux); + return RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.Windows); } /// diff --git a/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj b/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj index 82124ca8..f3c2af5b 100644 --- a/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj +++ b/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj @@ -61,7 +61,7 @@ false - + PreserveNewest From dae06fc58ff52b3ae1bcd8f8d21025ebef58657d Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 23 Sep 2024 22:04:37 +0400 Subject: [PATCH 030/219] Update MainTests.cs --- src/LanguageIdentification.Tests/MainTests.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.Tests/MainTests.cs b/src/LanguageIdentification.Tests/MainTests.cs index 82cbd5d1..bf5b2b9a 100644 --- a/src/LanguageIdentification.Tests/MainTests.cs +++ b/src/LanguageIdentification.Tests/MainTests.cs @@ -1,4 +1,6 @@ -ο»Ώnamespace Panlingo.LanguageIdentification.Tests; +ο»Ώusing System.Runtime.InteropServices; + +namespace Panlingo.LanguageIdentification.Tests; public class MainTests { @@ -8,7 +10,7 @@ public class MainTests [SkippableFact] public void CheckPlatform() { - Skip.IfNot(Environment.OSVersion.Platform == PlatformID.Unix); + Skip.IfNot(RuntimeInformation.IsOSPlatform(OSPlatform.Linux)); Assert.Equal("true", Environment.GetEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER")); } } From 67be82f8338c79db6d6516e4a8e9281e83c55a63 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 23 Sep 2024 22:05:25 +0400 Subject: [PATCH 031/219] FastText: try to build for Windows --- .../Properties/launchSettings.json | 7 +++++ .../.gitignore | 1 + .../FastTextNativeLibrary.cs | 2 +- ...guageIdentification.FastText.Native.csproj | 10 +++++++ .../Native/CMakeLists.txt | 30 +++++++++++++++---- .../Native/toolchain-mingw.cmake | 16 ++++++++++ .../Native/toolchain-osxcross.cmake | 7 +++++ .../Scripts/run-build.sh | 23 +++++++++----- .../Scripts/setup-build.sh | 2 +- .../FastTextDetector.cs | 2 +- 10 files changed, 85 insertions(+), 15 deletions(-) create mode 100644 src/LanguageIdentification.FastText.Native/Native/toolchain-mingw.cmake create mode 100644 src/LanguageIdentification.FastText.Native/Native/toolchain-osxcross.cmake diff --git a/src/LanguageIdentification.FastText.ConsoleTest/Properties/launchSettings.json b/src/LanguageIdentification.FastText.ConsoleTest/Properties/launchSettings.json index 65b8965a..1f434ec6 100644 --- a/src/LanguageIdentification.FastText.ConsoleTest/Properties/launchSettings.json +++ b/src/LanguageIdentification.FastText.ConsoleTest/Properties/launchSettings.json @@ -2,6 +2,13 @@ "profiles": { "Docker": { "commandName": "Docker" + }, + "WSL": { + "commandName": "WSL2", + "distributionName": "" + }, + "Project": { + "commandName": "Project" } } } \ No newline at end of file diff --git a/src/LanguageIdentification.FastText.Native/.gitignore b/src/LanguageIdentification.FastText.Native/.gitignore index 01a24561..aba5951e 100644 --- a/src/LanguageIdentification.FastText.Native/.gitignore +++ b/src/LanguageIdentification.FastText.Native/.gitignore @@ -1,2 +1,3 @@ libfasttext.so +libfasttext.dll build_temp/** diff --git a/src/LanguageIdentification.FastText.Native/FastTextNativeLibrary.cs b/src/LanguageIdentification.FastText.Native/FastTextNativeLibrary.cs index a0688250..f7464ba3 100644 --- a/src/LanguageIdentification.FastText.Native/FastTextNativeLibrary.cs +++ b/src/LanguageIdentification.FastText.Native/FastTextNativeLibrary.cs @@ -5,7 +5,7 @@ public class FastTextNativeLibrary /// /// Name of native binary /// - public const string Name = "libfasttext.so"; + public const string Name = "libfasttext"; } } diff --git a/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj b/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj index 74d9cefb..b342a628 100644 --- a/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj +++ b/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj @@ -62,6 +62,16 @@ + + + PreserveNewest + true + runtimes/win-x64/native + true + false + + + diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index 0fe25023..ed49e81c 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -1,5 +1,6 @@ -cmake_minimum_required(VERSION 2.8 FATAL_ERROR) -project(fasttext_bridge) +project(fasttext) + +cmake_minimum_required(VERSION 3.10 FATAL_ERROR) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") @@ -7,6 +8,25 @@ if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3") endif() +if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-rtti -lpthread") +elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") + # nop +endif() + +if(CMAKE_SYSTEM_NAME STREQUAL "Windows") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++ -Wl,--start-group -lpthread -lwinpthread -Wl,--end-group") + add_definitions(-D_USE_ALIGNED_MALLOC) +elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + # nop +else() + # nop +endif() + +if (WIN32) + add_definitions(-D_USE_ALIGNED_MALLOC -D__USE_MINGW_ANSI_STDIO=1) +endif() + set(CMAKE_MACOSX_RPATH 1) set(CMAKE_POSITION_INDEPENDENT_CODE ON) @@ -21,7 +41,7 @@ set(SOURCES ${PROJECT_SOURCE_DIR}/binding.cc) add_library(objlib OBJECT ${SOURCES}) -add_library(fasttext SHARED $) -set_target_properties(fasttext PROPERTIES PUBLIC_HEADER ${PROJECT_SOURCE_DIR}/binding.h) -target_link_libraries(fasttext fasttext-static_pic) +add_library(${PROJECT_NAME} SHARED $) +set_target_properties(${PROJECT_NAME} PROPERTIES PUBLIC_HEADER ${PROJECT_SOURCE_DIR}/binding.h) +target_link_libraries(${PROJECT_NAME} fasttext-static_pic) diff --git a/src/LanguageIdentification.FastText.Native/Native/toolchain-mingw.cmake b/src/LanguageIdentification.FastText.Native/Native/toolchain-mingw.cmake new file mode 100644 index 00000000..94e97c6b --- /dev/null +++ b/src/LanguageIdentification.FastText.Native/Native/toolchain-mingw.cmake @@ -0,0 +1,16 @@ +set(CMAKE_SYSTEM_NAME Windows) +set(TOOLCHAIN_PREFIX x86_64-w64-mingw32) + +# cross compilers to use for C, C++ and Fortran +set(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}-gcc) +set(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}-g++) +set(CMAKE_Fortran_COMPILER ${TOOLCHAIN_PREFIX}-gfortran) +set(CMAKE_RC_COMPILER ${TOOLCHAIN_PREFIX}-windres) + +# target environment on the build host system +set(CMAKE_FIND_ROOT_PATH /usr/${TOOLCHAIN_PREFIX}) + +# modify default behavior of FIND_XXX() commands +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) \ No newline at end of file diff --git a/src/LanguageIdentification.FastText.Native/Native/toolchain-osxcross.cmake b/src/LanguageIdentification.FastText.Native/Native/toolchain-osxcross.cmake new file mode 100644 index 00000000..6e1223e1 --- /dev/null +++ b/src/LanguageIdentification.FastText.Native/Native/toolchain-osxcross.cmake @@ -0,0 +1,7 @@ +set(CMAKE_SYSTEM_NAME Darwin) +set(CMAKE_SYSTEM_VERSION 1) + +# Path to the osxcross toolchain binaries +set(CMAKE_OSX_SYSROOT /usr/local/osxcross/target) +set(CMAKE_C_COMPILER /usr/local/osxcross/bin/o64-clang) +set(CMAKE_CXX_COMPILER /usr/local/osxcross/bin/o64-clang++) \ No newline at end of file diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.sh b/src/LanguageIdentification.FastText.Native/Scripts/run-build.sh index 6e001a81..9e9df94c 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/run-build.sh +++ b/src/LanguageIdentification.FastText.Native/Scripts/run-build.sh @@ -15,17 +15,26 @@ cd "$workspace" mkdir build cd build + +# Build for Linux +rm -rf * cmake .. make -j $(nproc) # make -cd .. -echo $(pwd) -ls -R build -cd .. +ls -R -find "$workspace/build" -name "libfasttext.so" -exec cp {} libfasttext.so \; -rm -rf "$workspace" ldd libfasttext.so +cp libfasttext.so ../../libfasttext.so -echo "Goodbye world"; +# Build for Windows +rm -rf * +cmake .. -DCMAKE_TOOLCHAIN_FILE=./toolchain-mingw.cmake +make -j $(nproc) # make +ls -R + +cp libfasttext.dll ../../libfasttext.dll + +# Clean up +rm -rf "$workspace" +echo "Goodbye world"; diff --git a/src/LanguageIdentification.FastText.Native/Scripts/setup-build.sh b/src/LanguageIdentification.FastText.Native/Scripts/setup-build.sh index 4f5708d0..d18c059a 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/setup-build.sh +++ b/src/LanguageIdentification.FastText.Native/Scripts/setup-build.sh @@ -6,4 +6,4 @@ echo "Installing build packages"; sudo apt -y update | apt -y update sudo apt -y install cmake | apt -y install cmake sudo apt -y install g++ | apt -y install g++ - +sudo apt -y install gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 | apt -y install gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 diff --git a/src/LanguageIdentification.FastText/FastTextDetector.cs b/src/LanguageIdentification.FastText/FastTextDetector.cs index a9a57665..2541e49d 100644 --- a/src/LanguageIdentification.FastText/FastTextDetector.cs +++ b/src/LanguageIdentification.FastText/FastTextDetector.cs @@ -31,7 +31,7 @@ public FastTextDetector() public static bool IsSupported() { - return RuntimeInformation.IsOSPlatform(OSPlatform.Linux); + return RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.Windows); } public string ModelPath { get; private set; } = string.Empty; From ae86f83b910136c72addfc48a5fdc33cd12fc428 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 23 Sep 2024 22:06:24 +0400 Subject: [PATCH 032/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 63d4193a..4e4e22b9 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -15,9 +15,9 @@ jobs: fail-fast: false matrix: projectName: - # - LanguageIdentification.CLD2 + - LanguageIdentification.CLD2 - LanguageIdentification.CLD3 - # - LanguageIdentification.FastText + - LanguageIdentification.FastText # - LanguageIdentification.Whatlang # - LanguageIdentification.MediaPipe # - LanguageIdentification.Lingua From 62d9245fc76cf6cd78626f9a377ad9f64b94a293 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 23 Sep 2024 22:49:16 +0400 Subject: [PATCH 033/219] Update CMakeLists.txt --- .../Native/CMakeLists.txt | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index ed49e81c..cc8f6d29 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -15,18 +15,14 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") endif() if(CMAKE_SYSTEM_NAME STREQUAL "Windows") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++ -Wl,--start-group -lpthread -lwinpthread -Wl,--end-group") - add_definitions(-D_USE_ALIGNED_MALLOC) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++ -Wl,--start-group -fpermissive -lpthread -lwinpthread -Wl,--end-group") + add_definitions(-D_USE_ALIGNED_MALLOC -D__USE_MINGW_ANSI_STDIO=1) elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") # nop else() # nop endif() -if (WIN32) - add_definitions(-D_USE_ALIGNED_MALLOC -D__USE_MINGW_ANSI_STDIO=1) -endif() - set(CMAKE_MACOSX_RPATH 1) set(CMAKE_POSITION_INDEPENDENT_CODE ON) From e98e426093061381398759bb62f27ca911d7d2a9 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 23 Sep 2024 23:01:37 +0400 Subject: [PATCH 034/219] Update CMakeLists.txt --- .../Native/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index cc8f6d29..c98f3564 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -15,7 +15,7 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") endif() if(CMAKE_SYSTEM_NAME STREQUAL "Windows") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++ -Wl,--start-group -fpermissive -lpthread -lwinpthread -Wl,--end-group") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++ -Wl,--start-group -fpermissive -lpthread -Wl,--end-group") add_definitions(-D_USE_ALIGNED_MALLOC -D__USE_MINGW_ANSI_STDIO=1) elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") # nop From 2abc33aa0bc8bebed32509538f5d8aa94b568cd2 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Tue, 24 Sep 2024 00:57:46 +0400 Subject: [PATCH 035/219] Update CMakeLists.txt --- .../Native/CMakeLists.txt | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index c98f3564..3d8ff415 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -2,6 +2,9 @@ project(fasttext) cmake_minimum_required(VERSION 3.10 FATAL_ERROR) +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") @@ -14,15 +17,6 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") # nop endif() -if(CMAKE_SYSTEM_NAME STREQUAL "Windows") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++ -Wl,--start-group -fpermissive -lpthread -Wl,--end-group") - add_definitions(-D_USE_ALIGNED_MALLOC -D__USE_MINGW_ANSI_STDIO=1) -elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") - # nop -else() - # nop -endif() - set(CMAKE_MACOSX_RPATH 1) set(CMAKE_POSITION_INDEPENDENT_CODE ON) From a65c5082ba52cf7ac8dbeb8fad80464e72a4d03d Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 25 Sep 2024 23:31:20 +0400 Subject: [PATCH 036/219] FT: try to use wine --- .../Scripts/run-build.sh | 4 ++-- .../Scripts/setup-build.sh | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.sh b/src/LanguageIdentification.FastText.Native/Scripts/run-build.sh index 9e9df94c..a150c18d 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/run-build.sh +++ b/src/LanguageIdentification.FastText.Native/Scripts/run-build.sh @@ -28,8 +28,8 @@ cp libfasttext.so ../../libfasttext.so # Build for Windows rm -rf * -cmake .. -DCMAKE_TOOLCHAIN_FILE=./toolchain-mingw.cmake -make -j $(nproc) # make +wine cmake .. +wine cmake --build . ls -R diff --git a/src/LanguageIdentification.FastText.Native/Scripts/setup-build.sh b/src/LanguageIdentification.FastText.Native/Scripts/setup-build.sh index d18c059a..658dc46e 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/setup-build.sh +++ b/src/LanguageIdentification.FastText.Native/Scripts/setup-build.sh @@ -7,3 +7,10 @@ sudo apt -y update | apt -y update sudo apt -y install cmake | apt -y install cmake sudo apt -y install g++ | apt -y install g++ sudo apt -y install gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 | apt -y install gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 + +apt -y install wine wine64 | sudo apt -y install wine wine64 + +wget https://github.com/Kitware/CMake/releases/download/v3.30.3/cmake-3.30.3-windows-x86_64.msi +wine msiexec /i cmake-3.30.3-windows-x86_64.msi +wine cmake --version + From d501a0aae2231137a076ddf9f980e758ce26000c Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 25 Sep 2024 23:37:21 +0400 Subject: [PATCH 037/219] Update setup-build.sh --- .../Scripts/setup-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.FastText.Native/Scripts/setup-build.sh b/src/LanguageIdentification.FastText.Native/Scripts/setup-build.sh index 658dc46e..53c76ccf 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/setup-build.sh +++ b/src/LanguageIdentification.FastText.Native/Scripts/setup-build.sh @@ -11,6 +11,6 @@ sudo apt -y install gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 | apt -y install g apt -y install wine wine64 | sudo apt -y install wine wine64 wget https://github.com/Kitware/CMake/releases/download/v3.30.3/cmake-3.30.3-windows-x86_64.msi -wine msiexec /i cmake-3.30.3-windows-x86_64.msi +wine msiexec /i cmake-3.30.3-windows-x86_64.msi /quiet wine cmake --version From 10aba406200c5fde2c071e5ca0bc7b52f2e72597 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 26 Sep 2024 00:02:56 +0400 Subject: [PATCH 038/219] CI: try to build images --- .github/workflows/github-ci.yml | 56 +++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 4e4e22b9..f6487a4b 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -8,6 +8,62 @@ on: workflow_dispatch: jobs: + build-docker-images: + name: πŸš€ Pack ${{matrix.task.projectName}} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + task: + - { projectName: LanguageIdentification.CLD2, imageName: panlingo-cld2-builder } + # - LanguageIdentification.CLD3 + # - LanguageIdentification.FastText + # - LanguageIdentification.Whatlang + # - LanguageIdentification.MediaPipe + # - LanguageIdentification.Lingua + permissions: + contents: read + packages: write + env: + ROOT: ./src + NUGET_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}} + steps: + - uses: actions/checkout@v4 + with: + submodules: 'true' + + - uses: dorny/paths-filter@v3 + id: changes + with: + filters: | + src: + - '${{env.ROOT}}/${{matrix.task.projectName}}/**' + + - name: Docker context + working-directory: . + run: docker context use default + + - name: Docker login + working-directory: . + run: | + echo ${{secrets.GITHUB_TOKEN}} | docker login ghcr.io \ + --username ${{github.repository_owner}} --password-stdin + + - name: Docker build and push + env: + IMAGE: ${{matrix.task.imageName}} + TAG: ${{github.ref_name}} + DOCKERFILE: ./Dockerfile + working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} + run: | + IMAGE_URL="ghcr.io/${{github.repository_owner}}/${{env.IMAGE}}" + docker build --pull --file ${{env.DOCKERFILE}} --tag ${{env.IMAGE}} --no-cache ../.. \ + --build-arg REPO_OWNER=${{github.repository_owner}} \ + --build-arg PACKAGES_WRITE_TOKEN=${{secrets.PACKAGES_WRITE_TOKEN}} + + docker tag ${{env.IMAGE}} $IMAGE_URL:${{env.TAG}} + docker push $IMAGE_URL:${{env.TAG}} + build-nugets: name: πŸš€ Pack ${{matrix.projectName}} runs-on: ubuntu-latest From 633f2c8a0ba006b823c6fd6979e31c8ef3b60b4d Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 26 Sep 2024 00:06:22 +0400 Subject: [PATCH 039/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index f6487a4b..9d2b4608 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -53,11 +53,11 @@ jobs: env: IMAGE: ${{matrix.task.imageName}} TAG: ${{github.ref_name}} - DOCKERFILE: ./Dockerfile - working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} + DOCKERFILE: ${{env.ROOT}}/${{matrix.task.projectName}}/Dockerfile + working-directory: . run: | IMAGE_URL="ghcr.io/${{github.repository_owner}}/${{env.IMAGE}}" - docker build --pull --file ${{env.DOCKERFILE}} --tag ${{env.IMAGE}} --no-cache ../.. \ + docker build --pull --file ${{env.DOCKERFILE}} --tag ${{env.IMAGE}} --no-cache . \ --build-arg REPO_OWNER=${{github.repository_owner}} \ --build-arg PACKAGES_WRITE_TOKEN=${{secrets.PACKAGES_WRITE_TOKEN}} @@ -73,7 +73,7 @@ jobs: projectName: - LanguageIdentification.CLD2 - LanguageIdentification.CLD3 - - LanguageIdentification.FastText + # - LanguageIdentification.FastText # - LanguageIdentification.Whatlang # - LanguageIdentification.MediaPipe # - LanguageIdentification.Lingua From 88354bde26899aff840d11c50289706285520b6f Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 26 Sep 2024 00:11:49 +0400 Subject: [PATCH 040/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 9d2b4608..3e6e3f50 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -15,7 +15,7 @@ jobs: fail-fast: false matrix: task: - - { projectName: LanguageIdentification.CLD2, imageName: panlingo-cld2-builder } + - { projectName: LanguageIdentification.CLD2, imageName: panlingo-builder-base-cld2 } # - LanguageIdentification.CLD3 # - LanguageIdentification.FastText # - LanguageIdentification.Whatlang @@ -53,11 +53,11 @@ jobs: env: IMAGE: ${{matrix.task.imageName}} TAG: ${{github.ref_name}} - DOCKERFILE: ${{env.ROOT}}/${{matrix.task.projectName}}/Dockerfile - working-directory: . + DOCKERFILE: ./Dockerfile + working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} run: | IMAGE_URL="ghcr.io/${{github.repository_owner}}/${{env.IMAGE}}" - docker build --pull --file ${{env.DOCKERFILE}} --tag ${{env.IMAGE}} --no-cache . \ + docker build --pull --file ${{env.DOCKERFILE}} --tag ${{env.IMAGE}} --no-cache ../.. \ --build-arg REPO_OWNER=${{github.repository_owner}} \ --build-arg PACKAGES_WRITE_TOKEN=${{secrets.PACKAGES_WRITE_TOKEN}} From a59d035d7638214634bcfd9ed3bb11147d27520c Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 26 Sep 2024 00:12:32 +0400 Subject: [PATCH 041/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 3e6e3f50..d8a8d31a 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -15,7 +15,7 @@ jobs: fail-fast: false matrix: task: - - { projectName: LanguageIdentification.CLD2, imageName: panlingo-builder-base-cld2 } + - { projectName: LanguageIdentification.CLD2.Native, imageName: panlingo-builder-base-cld2 } # - LanguageIdentification.CLD3 # - LanguageIdentification.FastText # - LanguageIdentification.Whatlang From fde24bcdbd15f3f148aea05a4fa38b9e3d5ddde5 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 26 Sep 2024 00:19:12 +0400 Subject: [PATCH 042/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index d8a8d31a..d3c4a4b5 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -56,17 +56,19 @@ jobs: DOCKERFILE: ./Dockerfile working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} run: | + TAG="${${{env.TAG}}//\//_}" # "16/merge" => "16_merge" IMAGE_URL="ghcr.io/${{github.repository_owner}}/${{env.IMAGE}}" docker build --pull --file ${{env.DOCKERFILE}} --tag ${{env.IMAGE}} --no-cache ../.. \ --build-arg REPO_OWNER=${{github.repository_owner}} \ --build-arg PACKAGES_WRITE_TOKEN=${{secrets.PACKAGES_WRITE_TOKEN}} - docker tag ${{env.IMAGE}} $IMAGE_URL:${{env.TAG}} - docker push $IMAGE_URL:${{env.TAG}} + docker tag ${{env.IMAGE}} $IMAGE_URL:${{TAG}} + docker push $IMAGE_URL:${{TAG}} build-nugets: name: πŸš€ Pack ${{matrix.projectName}} runs-on: ubuntu-latest + needs: build-docker-images strategy: fail-fast: false matrix: From dc1a4a2fd8de3fd7ce0e362a60d9093d2632d9cd Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 26 Sep 2024 00:19:56 +0400 Subject: [PATCH 043/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index d3c4a4b5..5c609b76 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -56,7 +56,8 @@ jobs: DOCKERFILE: ./Dockerfile working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} run: | - TAG="${${{env.TAG}}//\//_}" # "16/merge" => "16_merge" + TAG=${{env.TAG}} + TAG="${TAG//\//_}" # "16/merge" => "16_merge" IMAGE_URL="ghcr.io/${{github.repository_owner}}/${{env.IMAGE}}" docker build --pull --file ${{env.DOCKERFILE}} --tag ${{env.IMAGE}} --no-cache ../.. \ --build-arg REPO_OWNER=${{github.repository_owner}} \ From c8145784c02263318225cc94d64b6a5ada19c80a Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 26 Sep 2024 00:21:14 +0400 Subject: [PATCH 044/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 5c609b76..f43a8310 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -56,9 +56,9 @@ jobs: DOCKERFILE: ./Dockerfile working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} run: | - TAG=${{env.TAG}} - TAG="${TAG//\//_}" # "16/merge" => "16_merge" - IMAGE_URL="ghcr.io/${{github.repository_owner}}/${{env.IMAGE}}" + TAG=${{env.TAG}} \ + TAG="${TAG//\//_}" \ # "16/merge" => "16_merge" + IMAGE_URL="ghcr.io/${{github.repository_owner}}/${{env.IMAGE}}" \ docker build --pull --file ${{env.DOCKERFILE}} --tag ${{env.IMAGE}} --no-cache ../.. \ --build-arg REPO_OWNER=${{github.repository_owner}} \ --build-arg PACKAGES_WRITE_TOKEN=${{secrets.PACKAGES_WRITE_TOKEN}} From 0cd74f44426eb648ee7b717d5fc76ac81fe93837 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 26 Sep 2024 00:22:10 +0400 Subject: [PATCH 045/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index f43a8310..c029383f 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -56,7 +56,7 @@ jobs: DOCKERFILE: ./Dockerfile working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} run: | - TAG=${{env.TAG}} \ + TAG="${{env.TAG}}" \ TAG="${TAG//\//_}" \ # "16/merge" => "16_merge" IMAGE_URL="ghcr.io/${{github.repository_owner}}/${{env.IMAGE}}" \ docker build --pull --file ${{env.DOCKERFILE}} --tag ${{env.IMAGE}} --no-cache ../.. \ From 8125fb511397f63449c2da7bd4b4cbb4f0b5bf21 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 26 Sep 2024 00:23:12 +0400 Subject: [PATCH 046/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index c029383f..ec812e6d 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -57,7 +57,7 @@ jobs: working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} run: | TAG="${{env.TAG}}" \ - TAG="${TAG//\//_}" \ # "16/merge" => "16_merge" + TAG="${TAG//\//_}" \ IMAGE_URL="ghcr.io/${{github.repository_owner}}/${{env.IMAGE}}" \ docker build --pull --file ${{env.DOCKERFILE}} --tag ${{env.IMAGE}} --no-cache ../.. \ --build-arg REPO_OWNER=${{github.repository_owner}} \ From b8fe8b929c1d9992724232b1a338fba5e37b485a Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 26 Sep 2024 00:25:34 +0400 Subject: [PATCH 047/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index ec812e6d..bde8073d 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -63,8 +63,8 @@ jobs: --build-arg REPO_OWNER=${{github.repository_owner}} \ --build-arg PACKAGES_WRITE_TOKEN=${{secrets.PACKAGES_WRITE_TOKEN}} - docker tag ${{env.IMAGE}} $IMAGE_URL:${{TAG}} - docker push $IMAGE_URL:${{TAG}} + docker tag ${{env.IMAGE}} $IMAGE_URL:$TAG + docker push $IMAGE_URL:$TAG build-nugets: name: πŸš€ Pack ${{matrix.projectName}} From ce9f8b2bed58c702543a2c11562722cac2551c85 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 26 Sep 2024 00:31:12 +0400 Subject: [PATCH 048/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index bde8073d..ae3728d8 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -57,7 +57,7 @@ jobs: working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} run: | TAG="${{env.TAG}}" \ - TAG="${TAG//\//_}" \ + TAG="${TAG/\//_}" \ IMAGE_URL="ghcr.io/${{github.repository_owner}}/${{env.IMAGE}}" \ docker build --pull --file ${{env.DOCKERFILE}} --tag ${{env.IMAGE}} --no-cache ../.. \ --build-arg REPO_OWNER=${{github.repository_owner}} \ From faeab9368047d4ffb8fbaf1fb4654f7f8f1a36ea Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 26 Sep 2024 00:33:22 +0400 Subject: [PATCH 049/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index ae3728d8..88fff515 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -58,6 +58,7 @@ jobs: run: | TAG="${{env.TAG}}" \ TAG="${TAG/\//_}" \ + echo $TAG \ IMAGE_URL="ghcr.io/${{github.repository_owner}}/${{env.IMAGE}}" \ docker build --pull --file ${{env.DOCKERFILE}} --tag ${{env.IMAGE}} --no-cache ../.. \ --build-arg REPO_OWNER=${{github.repository_owner}} \ From 6d26274f107610b49045d726e84dea76d47a09be Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 26 Sep 2024 00:39:17 +0400 Subject: [PATCH 050/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 88fff515..bde8073d 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -57,8 +57,7 @@ jobs: working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} run: | TAG="${{env.TAG}}" \ - TAG="${TAG/\//_}" \ - echo $TAG \ + TAG="${TAG//\//_}" \ IMAGE_URL="ghcr.io/${{github.repository_owner}}/${{env.IMAGE}}" \ docker build --pull --file ${{env.DOCKERFILE}} --tag ${{env.IMAGE}} --no-cache ../.. \ --build-arg REPO_OWNER=${{github.repository_owner}} \ From 91a4e151e446b55109bd00ee18684fefc61a8cdf Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 26 Sep 2024 00:42:11 +0400 Subject: [PATCH 051/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index bde8073d..76835964 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -57,7 +57,7 @@ jobs: working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} run: | TAG="${{env.TAG}}" \ - TAG="${TAG//\//_}" \ + TAG=$(echo $TAG | tr '/' '_') \ IMAGE_URL="ghcr.io/${{github.repository_owner}}/${{env.IMAGE}}" \ docker build --pull --file ${{env.DOCKERFILE}} --tag ${{env.IMAGE}} --no-cache ../.. \ --build-arg REPO_OWNER=${{github.repository_owner}} \ From bd303319effbd1a33d241acb712b9e141d0e5c61 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 26 Sep 2024 00:44:52 +0400 Subject: [PATCH 052/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 76835964..924d74e6 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -56,8 +56,8 @@ jobs: DOCKERFILE: ./Dockerfile working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} run: | - TAG="${{env.TAG}}" \ - TAG=$(echo $TAG | tr '/' '_') \ + TAG_RAW="${{env.TAG}}" \ + TAG=$(echo $TAG_RAW | tr '/' '_') \ IMAGE_URL="ghcr.io/${{github.repository_owner}}/${{env.IMAGE}}" \ docker build --pull --file ${{env.DOCKERFILE}} --tag ${{env.IMAGE}} --no-cache ../.. \ --build-arg REPO_OWNER=${{github.repository_owner}} \ From 88a94d719cab3931ca4a6e700eac983b1088a830 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 26 Sep 2024 00:50:00 +0400 Subject: [PATCH 053/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 924d74e6..12d5ec6b 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -52,12 +52,10 @@ jobs: - name: Docker build and push env: IMAGE: ${{matrix.task.imageName}} - TAG: ${{github.ref_name}} DOCKERFILE: ./Dockerfile working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} run: | - TAG_RAW="${{env.TAG}}" \ - TAG=$(echo $TAG_RAW | tr '/' '_') \ + TAG="${GITHUB_HEAD_REF:-$GITHUB_REF_NAME}" \ IMAGE_URL="ghcr.io/${{github.repository_owner}}/${{env.IMAGE}}" \ docker build --pull --file ${{env.DOCKERFILE}} --tag ${{env.IMAGE}} --no-cache ../.. \ --build-arg REPO_OWNER=${{github.repository_owner}} \ From 1a45924b563a8c64dc16cc58de0c512bdc10648a Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 26 Sep 2024 00:56:54 +0400 Subject: [PATCH 054/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 12d5ec6b..81f3a633 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -55,11 +55,10 @@ jobs: DOCKERFILE: ./Dockerfile working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} run: | - TAG="${GITHUB_HEAD_REF:-$GITHUB_REF_NAME}" \ + SHA=$(git rev-parse --short "$GITHUB_SHA") \ + TAG="${GITHUB_REF#refs/heads/}" \ IMAGE_URL="ghcr.io/${{github.repository_owner}}/${{env.IMAGE}}" \ docker build --pull --file ${{env.DOCKERFILE}} --tag ${{env.IMAGE}} --no-cache ../.. \ - --build-arg REPO_OWNER=${{github.repository_owner}} \ - --build-arg PACKAGES_WRITE_TOKEN=${{secrets.PACKAGES_WRITE_TOKEN}} docker tag ${{env.IMAGE}} $IMAGE_URL:$TAG docker push $IMAGE_URL:$TAG From 948e8c8c83274b342d4553c7ce4513bb33b288b6 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 26 Sep 2024 00:59:05 +0400 Subject: [PATCH 055/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 81f3a633..25dc842e 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -60,8 +60,8 @@ jobs: IMAGE_URL="ghcr.io/${{github.repository_owner}}/${{env.IMAGE}}" \ docker build --pull --file ${{env.DOCKERFILE}} --tag ${{env.IMAGE}} --no-cache ../.. \ - docker tag ${{env.IMAGE}} $IMAGE_URL:$TAG - docker push $IMAGE_URL:$TAG + docker tag ${{env.IMAGE}} $IMAGE_URL:$SHA + docker push $IMAGE_URL:$SHA build-nugets: name: πŸš€ Pack ${{matrix.projectName}} From 9bf9a6fa325de632457b96c36484a6c8a24f754f Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sat, 28 Sep 2024 01:35:22 +0400 Subject: [PATCH 056/219] Different platforms for different models --- .github/workflows/github-ci.yml | 45 ++++++++++++++++----------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 25dc842e..d880b6eb 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -52,32 +52,31 @@ jobs: - name: Docker build and push env: IMAGE: ${{matrix.task.imageName}} + TAG: latest # TODO: use branch name DOCKERFILE: ./Dockerfile working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} run: | - SHA=$(git rev-parse --short "$GITHUB_SHA") \ - TAG="${GITHUB_REF#refs/heads/}" \ IMAGE_URL="ghcr.io/${{github.repository_owner}}/${{env.IMAGE}}" \ docker build --pull --file ${{env.DOCKERFILE}} --tag ${{env.IMAGE}} --no-cache ../.. \ - docker tag ${{env.IMAGE}} $IMAGE_URL:$SHA - docker push $IMAGE_URL:$SHA + docker tag ${{env.IMAGE}} $IMAGE_URL:${{env.TAG}} + docker push $IMAGE_URL:${{env.TAG}} build-nugets: name: πŸš€ Pack ${{matrix.projectName}} - runs-on: ubuntu-latest + runs-on: ${{matrix.task.os}} needs: build-docker-images strategy: fail-fast: false matrix: - projectName: - - LanguageIdentification.CLD2 - - LanguageIdentification.CLD3 - # - LanguageIdentification.FastText - # - LanguageIdentification.Whatlang - # - LanguageIdentification.MediaPipe - # - LanguageIdentification.Lingua - # - LanguageCode + task: + - { projectName: LanguageIdentification.CLD2, os: ubuntu-latest } + - { projectName: LanguageIdentification.CLD3, os: ubuntu-latest } + - { projectName: LanguageIdentification.FastText, os: windows-latest } + # - { projectName: LanguageIdentification.Whatlang, os: ubuntu-latest } + # - { projectName: LanguageIdentification.MediaPipe, os: ubuntu-latest } + # - { projectName: LanguageIdentification.Lingua, os: ubuntu-latest } + # - { projectName: LanguageCode, os: ubuntu-latest } permissions: contents: read packages: write @@ -94,14 +93,14 @@ jobs: with: filters: | src: - - '${{env.ROOT}}/${{matrix.projectName}}/**' + - '${{env.ROOT}}/${{matrix.task.projectName}}/**' - uses: dorny/paths-filter@v3 id: changes-native with: filters: | src: - - '${{env.ROOT}}/${{matrix.projectName}}.Native/**' + - '${{env.ROOT}}/${{matrix.task.projectName}}.Native/**' - name: πŸ“‚ Setup .NET Core uses: actions/setup-dotnet@v4 @@ -113,21 +112,21 @@ jobs: 8.0.x source-url: https://nuget.pkg.github.com/${{github.repository_owner}}/index.json - - name: πŸš€ Pack ${{matrix.projectName}}.Native + - name: πŸš€ Pack ${{matrix.task.projectName}}.Native if: steps.changes-native.outputs.src == 'true' - working-directory: ${{env.ROOT}}/${{matrix.projectName}}.Native + working-directory: ${{env.ROOT}}/${{matrix.task.projectName}}.Native run: dotnet pack -c Release -o out - - name: πŸš€ Pack ${{matrix.projectName}} + - name: πŸš€ Pack ${{matrix.task.projectName}} if: steps.changes.outputs.src == 'true' - working-directory: ${{env.ROOT}}/${{matrix.projectName}} + working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} run: dotnet pack -c Release -o out - uses: actions/upload-artifact@v4 if: steps.changes-native.outputs.src == 'true' with: - name: build-${{matrix.projectName}}.Native - path: ${{env.ROOT}}/${{matrix.projectName}}.Native/out + name: build-${{matrix.task.projectName}}.Native + path: ${{env.ROOT}}/${{matrix.task.projectName}}.Native/out retention-days: 1 overwrite: 'true' compression-level: 0 @@ -135,8 +134,8 @@ jobs: - uses: actions/upload-artifact@v4 if: steps.changes.outputs.src == 'true' with: - name: build-${{matrix.projectName}} - path: ${{env.ROOT}}/${{matrix.projectName}}/out + name: build-${{matrix.task.projectName}} + path: ${{env.ROOT}}/${{matrix.task.projectName}}/out retention-days: 1 overwrite: 'true' compression-level: 0 From 8d77bec1dbcb9ac9f9e73725326cf395b58bd117 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sat, 28 Sep 2024 01:38:21 +0400 Subject: [PATCH 057/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index d880b6eb..30d9f6ed 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -65,7 +65,7 @@ jobs: build-nugets: name: πŸš€ Pack ${{matrix.projectName}} runs-on: ${{matrix.task.os}} - needs: build-docker-images + # needs: build-docker-images strategy: fail-fast: false matrix: From b9a1810af324d9f0531a709a6d835a15aca56aa2 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sat, 28 Sep 2024 01:48:36 +0400 Subject: [PATCH 058/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 30d9f6ed..2d8f2e7c 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -63,7 +63,7 @@ jobs: docker push $IMAGE_URL:${{env.TAG}} build-nugets: - name: πŸš€ Pack ${{matrix.projectName}} + name: πŸš€ Pack ${{matrix.task.projectName}} runs-on: ${{matrix.task.os}} # needs: build-docker-images strategy: From a91ace53f3c21b2ed17e8d6952fc7c47b093f2e2 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sat, 28 Sep 2024 01:59:54 +0400 Subject: [PATCH 059/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 2d8f2e7c..e6af82f5 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -88,6 +88,19 @@ jobs: with: submodules: 'true' + - name: Ensure Docker is installed + run: | + echo "Docker version:" + docker --version + + - name: Run a Linux container + run: | + echo "Pulling a Linux container image..." + docker pull ubuntu:latest + + echo "Running a Linux container..." + docker run --rm ubuntu:latest echo "Hello from Ubuntu container!" + - uses: dorny/paths-filter@v3 id: changes with: From 8d98df63b3287e84bc909fdb6b8eea83a40cc8ee Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sat, 28 Sep 2024 22:15:52 +0400 Subject: [PATCH 060/219] FT: fix build for Windows --- .../CLD2NativeLibrary.cs | 2 +- .../CLD3NativeLibrary.cs | 2 +- .../.gitignore | 2 +- .../FastTextNativeLibrary.cs | 2 +- ...guageIdentification.FastText.Native.csproj | 2 +- .../Native/binding.cc | 34 ++++++++++--------- .../Native/binding.h | 2 +- .../LinguaNativeLibrary.cs | 2 +- .../MediaPipeNativeLibrary.cs | 2 +- .../WhatlangNativeLibrary.cs | 2 +- 10 files changed, 27 insertions(+), 25 deletions(-) diff --git a/src/LanguageIdentification.CLD2.Native/CLD2NativeLibrary.cs b/src/LanguageIdentification.CLD2.Native/CLD2NativeLibrary.cs index 7afcdeba..46d99d32 100644 --- a/src/LanguageIdentification.CLD2.Native/CLD2NativeLibrary.cs +++ b/src/LanguageIdentification.CLD2.Native/CLD2NativeLibrary.cs @@ -5,6 +5,6 @@ public static class CLD2NativeLibrary /// /// Name of native binary /// - public const string Name = "libcld2"; + public const string Name = "cld2"; } } diff --git a/src/LanguageIdentification.CLD3.Native/CLD3NativeLibrary.cs b/src/LanguageIdentification.CLD3.Native/CLD3NativeLibrary.cs index 5c5835d8..40b911a5 100644 --- a/src/LanguageIdentification.CLD3.Native/CLD3NativeLibrary.cs +++ b/src/LanguageIdentification.CLD3.Native/CLD3NativeLibrary.cs @@ -5,6 +5,6 @@ public static class CLD3NativeLibrary /// /// Name of native binary /// - public const string Name = "libcld3"; + public const string Name = "cld3"; } } diff --git a/src/LanguageIdentification.FastText.Native/.gitignore b/src/LanguageIdentification.FastText.Native/.gitignore index aba5951e..d7b42a55 100644 --- a/src/LanguageIdentification.FastText.Native/.gitignore +++ b/src/LanguageIdentification.FastText.Native/.gitignore @@ -1,3 +1,3 @@ libfasttext.so -libfasttext.dll +fasttext.dll build_temp/** diff --git a/src/LanguageIdentification.FastText.Native/FastTextNativeLibrary.cs b/src/LanguageIdentification.FastText.Native/FastTextNativeLibrary.cs index f7464ba3..064f5cae 100644 --- a/src/LanguageIdentification.FastText.Native/FastTextNativeLibrary.cs +++ b/src/LanguageIdentification.FastText.Native/FastTextNativeLibrary.cs @@ -5,7 +5,7 @@ public class FastTextNativeLibrary /// /// Name of native binary /// - public const string Name = "libfasttext"; + public const string Name = "fasttext"; } } diff --git a/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj b/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj index b342a628..3f661b5e 100644 --- a/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj +++ b/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj @@ -63,7 +63,7 @@ - + PreserveNewest true runtimes/win-x64/native diff --git a/src/LanguageIdentification.FastText.Native/Native/binding.cc b/src/LanguageIdentification.FastText.Native/Native/binding.cc index c2f269ba..af7f58e6 100644 --- a/src/LanguageIdentification.FastText.Native/Native/binding.cc +++ b/src/LanguageIdentification.FastText.Native/Native/binding.cc @@ -18,30 +18,31 @@ extern "C" { *err_ptr = strdup(e.what()); } - void DestroyString(char* s) { + EXPORT void DestroyString(char* s) { if (s != nullptr) { free(s); } } - fasttext_t* CreateFastText(void) { + EXPORT fasttext_t* CreateFastText(void) { return (fasttext_t*)(new FastTextExtension()); } - void DestroyFastText(fasttext_t* handle) { + EXPORT void DestroyFastText(fasttext_t* handle) { FastTextExtension* x = (FastTextExtension*)handle; delete x; } - void FastTextLoadModel(fasttext_t* handle, const char* filename, char** err_ptr) { + EXPORT void FastTextLoadModel(fasttext_t* handle, const char* filename, char** err_ptr) { try { ((FastTextExtension*)handle)->loadModel(filename); - } catch (const std::invalid_argument& e) { + } + catch (const std::invalid_argument& e) { save_error(err_ptr, e); } } - void FastTextLoadModelData(fasttext_t* handle, const char* buffer, size_t buffer_length, char** err_ptr) { + EXPORT void FastTextLoadModelData(fasttext_t* handle, const char* buffer, size_t buffer_length, char** err_ptr) { try { ((FastTextExtension*)handle)->loadModelData(buffer, buffer_length); } @@ -50,16 +51,17 @@ extern "C" { } } - int FastTextGetModelDimensions(fasttext_t* handle) { + EXPORT int FastTextGetModelDimensions(fasttext_t* handle) { return ((FastTextExtension*)handle)->getDimension(); } - fasttext_predictions_t* FastTextPredict(fasttext_t* handle, const char* text, int32_t k, float threshold, char** err_ptr) { + EXPORT fasttext_predictions_t* FastTextPredict(fasttext_t* handle, const char* text, int32_t k, float threshold, char** err_ptr) { std::vector> predictions; std::stringstream ioss(text); try { ((FastTextExtension*)handle)->predictLine(ioss, predictions, k, threshold); - } catch (const std::invalid_argument& e) { + } + catch (const std::invalid_argument& e) { save_error(err_ptr, e); return nullptr; } @@ -75,7 +77,7 @@ extern "C" { return ret; } - void DestroyPredictions(fasttext_predictions_t* predictions) { + EXPORT void DestroyPredictions(fasttext_predictions_t* predictions) { if (predictions == nullptr) { return; } @@ -87,7 +89,7 @@ extern "C" { free(predictions); } - fasttext_labels_t* FastTextGetLabels(fasttext_t* handle) { + EXPORT fasttext_labels_t* FastTextGetLabels(fasttext_t* handle) { std::shared_ptr d = ((FastTextExtension*)handle)->getDictionary(); std::vector labels_freq = d->getCounts(fasttext::entry_type::label); size_t len = labels_freq.size(); @@ -106,7 +108,7 @@ extern "C" { return ret; } - void DestroyLabels(fasttext_labels_t* labels) { + EXPORT void DestroyLabels(fasttext_labels_t* labels) { if (labels == nullptr) { return; } @@ -118,18 +120,18 @@ extern "C" { free(labels); } - void FastTextAbort(fasttext_t* handle) { + EXPORT void FastTextAbort(fasttext_t* handle) { ((FastTextExtension*)handle)->abort(); } - fasttext_tokens_t* FastTextTokenize(fasttext_t* handle, const char* text) { + EXPORT fasttext_tokens_t* FastTextTokenize(fasttext_t* handle, const char* text) { std::vector text_split; std::shared_ptr d = ((FastTextExtension*)handle)->getDictionary(); std::stringstream ioss(text); std::string token; while (!ioss.eof()) { while (d->readWord(ioss, token)) { - text_split.push_back(token); + text_split.push_back(token); } } size_t len = text_split.size(); @@ -143,7 +145,7 @@ extern "C" { return ret; } - void DestroyTokens(fasttext_tokens_t* tokens) { + EXPORT void DestroyTokens(fasttext_tokens_t* tokens) { for (size_t i = 0; i < tokens->length; i++) { free(tokens->tokens[i]); } diff --git a/src/LanguageIdentification.FastText.Native/Native/binding.h b/src/LanguageIdentification.FastText.Native/Native/binding.h index ad4909f6..77826f36 100644 --- a/src/LanguageIdentification.FastText.Native/Native/binding.h +++ b/src/LanguageIdentification.FastText.Native/Native/binding.h @@ -6,7 +6,7 @@ #ifndef EXPORT # if defined(_WIN32) || defined(_WIN64) -# define EXPORT __declspec(dllimport) +# define EXPORT __declspec(dllexport) # else # define EXPORT extern # endif diff --git a/src/LanguageIdentification.Lingua.Native/LinguaNativeLibrary.cs b/src/LanguageIdentification.Lingua.Native/LinguaNativeLibrary.cs index 4019366a..7c6326bb 100644 --- a/src/LanguageIdentification.Lingua.Native/LinguaNativeLibrary.cs +++ b/src/LanguageIdentification.Lingua.Native/LinguaNativeLibrary.cs @@ -5,6 +5,6 @@ public static class LinguaNativeLibrary /// /// Name of native binary /// - public const string Name = "liblingua.so"; + public const string Name = "lingua"; } } diff --git a/src/LanguageIdentification.MediaPipe.Native/MediaPipeNativeLibrary.cs b/src/LanguageIdentification.MediaPipe.Native/MediaPipeNativeLibrary.cs index 44f627e6..74c0cf9a 100644 --- a/src/LanguageIdentification.MediaPipe.Native/MediaPipeNativeLibrary.cs +++ b/src/LanguageIdentification.MediaPipe.Native/MediaPipeNativeLibrary.cs @@ -5,7 +5,7 @@ public class MediaPipeNativeLibrary /// /// Name of native binary /// - public const string Name = "liblanguage_detector.so"; + public const string Name = "language_detector"; /// /// Name of model /// diff --git a/src/LanguageIdentification.Whatlang.Native/WhatlangNativeLibrary.cs b/src/LanguageIdentification.Whatlang.Native/WhatlangNativeLibrary.cs index 2ec7e231..836f41f6 100644 --- a/src/LanguageIdentification.Whatlang.Native/WhatlangNativeLibrary.cs +++ b/src/LanguageIdentification.Whatlang.Native/WhatlangNativeLibrary.cs @@ -5,6 +5,6 @@ public static class WhatlangNativeLibrary /// /// Name of native binary /// - public const string Name = "libwhatlang.so"; + public const string Name = "whatlang"; } } From 250c9c20c95a626266e259337be0fa5c191849fe Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 30 Sep 2024 22:41:52 +0400 Subject: [PATCH 061/219] CLD2, CLD3, FT: enforce using UTF8 --- .../Program.cs | 2 +- .../CLD2NativeLibrary.cs | 2 +- .../Internal/CLD2DetectorWrapper.cs | 5 ++++- .../Program.cs | 2 +- .../CLD3NativeLibrary.cs | 2 +- .../Native/src/binding.cc | 4 +++- .../Internal/CLD3DetectorWrapper.cs | 12 ++++++++++-- .../Program.cs | 2 +- .../FastTextDetector.cs | 7 ++++--- .../Internal/FastTextDetectorWrapper.cs | 12 +++++++++--- 10 files changed, 35 insertions(+), 15 deletions(-) diff --git a/src/LanguageIdentification.CLD2.ConsoleTest/Program.cs b/src/LanguageIdentification.CLD2.ConsoleTest/Program.cs index b94ab372..cb3672c6 100644 --- a/src/LanguageIdentification.CLD2.ConsoleTest/Program.cs +++ b/src/LanguageIdentification.CLD2.ConsoleTest/Program.cs @@ -6,7 +6,7 @@ static void Main(string[] args) { using var cld2 = new CLD2Detector(); - var text = "Hello, how are you? ΠŸΡ€ΠΈΠ²Ρ–Ρ‚, як справи? ΠŸΡ€ΠΈΠ²Π΅Ρ‚, ΠΊΠ°ΠΊ Π΄Π΅Π»Π°?"; + var text = "ΠŸΡ€ΠΈΠ²Ρ–Ρ‚, як справи?"; var predictions = cld2.PredictLanguage(text); diff --git a/src/LanguageIdentification.CLD2.Native/CLD2NativeLibrary.cs b/src/LanguageIdentification.CLD2.Native/CLD2NativeLibrary.cs index 46d99d32..7afcdeba 100644 --- a/src/LanguageIdentification.CLD2.Native/CLD2NativeLibrary.cs +++ b/src/LanguageIdentification.CLD2.Native/CLD2NativeLibrary.cs @@ -5,6 +5,6 @@ public static class CLD2NativeLibrary /// /// Name of native binary /// - public const string Name = "cld2"; + public const string Name = "libcld2"; } } diff --git a/src/LanguageIdentification.CLD2/Internal/CLD2DetectorWrapper.cs b/src/LanguageIdentification.CLD2/Internal/CLD2DetectorWrapper.cs index 3ef06497..21897b92 100644 --- a/src/LanguageIdentification.CLD2/Internal/CLD2DetectorWrapper.cs +++ b/src/LanguageIdentification.CLD2/Internal/CLD2DetectorWrapper.cs @@ -7,7 +7,10 @@ namespace Panlingo.LanguageIdentification.CLD2.Internal internal static class CLD2DetectorWrapper { [DllImport(CLD2NativeLibrary.Name, CallingConvention = CallingConvention.Cdecl)] - public static extern IntPtr PredictLanguage(string text, out int resultCount); + public static extern IntPtr PredictLanguage( + [MarshalAs(UnmanagedType.LPUTF8Str)] string text, + out int resultCount + ); [DllImport(CLD2NativeLibrary.Name, CallingConvention = CallingConvention.Cdecl)] public static extern void FreeResults(IntPtr results, int count); diff --git a/src/LanguageIdentification.CLD3.ConsoleTest/Program.cs b/src/LanguageIdentification.CLD3.ConsoleTest/Program.cs index 63710c75..ec55d826 100644 --- a/src/LanguageIdentification.CLD3.ConsoleTest/Program.cs +++ b/src/LanguageIdentification.CLD3.ConsoleTest/Program.cs @@ -6,7 +6,7 @@ static void Main(string[] args) { using var cld3 = new CLD3Detector(minNumBytes: 0, maxNumBytes: 512); - var text = "Hello, how are you? ΠŸΡ€ΠΈΠ²Ρ–Ρ‚, як справи? ΠŸΡ€ΠΈΠ²Π΅Ρ‚, ΠΊΠ°ΠΊ Π΄Π΅Π»Π°?"; + var text = "ΠŸΡ€ΠΈΠ²Ρ–Ρ‚, як справи?"; var singlePrediction = cld3.PredictLanguage(text); diff --git a/src/LanguageIdentification.CLD3.Native/CLD3NativeLibrary.cs b/src/LanguageIdentification.CLD3.Native/CLD3NativeLibrary.cs index 40b911a5..5c5835d8 100644 --- a/src/LanguageIdentification.CLD3.Native/CLD3NativeLibrary.cs +++ b/src/LanguageIdentification.CLD3.Native/CLD3NativeLibrary.cs @@ -5,6 +5,6 @@ public static class CLD3NativeLibrary /// /// Name of native binary /// - public const string Name = "cld3"; + public const string Name = "libcld3"; } } diff --git a/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc b/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc index c887d3ee..3198cbab 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc +++ b/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc @@ -15,7 +15,9 @@ PredictionResult cld3_find_language(void* identifier, const char* text) { NNetLanguageIdentifier* nativeIdentifier = static_cast(identifier); auto nativeResult = nativeIdentifier->FindLanguage(text); - PredictionResult result; + + PredictionResult x = {}; + PredictionResult* result = static_cast(); result.language = strdup(nativeResult.language.c_str()); result.probability = nativeResult.probability; result.is_reliable = nativeResult.is_reliable; diff --git a/src/LanguageIdentification.CLD3/Internal/CLD3DetectorWrapper.cs b/src/LanguageIdentification.CLD3/Internal/CLD3DetectorWrapper.cs index d12f44f6..009a2360 100644 --- a/src/LanguageIdentification.CLD3/Internal/CLD3DetectorWrapper.cs +++ b/src/LanguageIdentification.CLD3/Internal/CLD3DetectorWrapper.cs @@ -13,10 +13,18 @@ internal static class CLD3DetectorWrapper public static extern void DestroyCLD3(IntPtr identifier); [DllImport(CLD3NativeLibrary.Name, EntryPoint = "cld3_find_language", CallingConvention = CallingConvention.Cdecl)] - public static extern CLD3PredictionResult FindLanguage(IntPtr identifier, string text); + public static extern CLD3PredictionResult FindLanguage( + IntPtr identifier, + [MarshalAs(UnmanagedType.LPUTF8Str)] string text + ); [DllImport(CLD3NativeLibrary.Name, EntryPoint = "cld3_find_languages", CallingConvention = CallingConvention.Cdecl)] - public static extern IntPtr FindLanguages(IntPtr identifier, string text, int numLangs, out int resultCount); + public static extern IntPtr FindLanguages( + IntPtr identifier, + [MarshalAs(UnmanagedType.LPUTF8Str)] string text, + int numLangs, + out int resultCount + ); [DllImport(CLD3NativeLibrary.Name, EntryPoint = "cld3_destroy_prediction_result", CallingConvention = CallingConvention.Cdecl)] public static extern void DestroyPredictionResult(IntPtr results, int count); diff --git a/src/LanguageIdentification.FastText.ConsoleTest/Program.cs b/src/LanguageIdentification.FastText.ConsoleTest/Program.cs index fc018ec3..2e294170 100644 --- a/src/LanguageIdentification.FastText.ConsoleTest/Program.cs +++ b/src/LanguageIdentification.FastText.ConsoleTest/Program.cs @@ -4,7 +4,7 @@ internal class Program { static void Main(string[] args) { - var text = "Hello, how are you? ΠŸΡ€ΠΈΠ²Ρ–Ρ‚, як справи? ΠŸΡ€ΠΈΠ²Π΅Ρ‚, ΠΊΠ°ΠΊ Π΄Π΅Π»Π°?"; + var text = "ΠŸΡ€ΠΈΠ²Ρ–Ρ‚, як справи?"; using var fastText = new FastTextDetector(); diff --git a/src/LanguageIdentification.FastText/FastTextDetector.cs b/src/LanguageIdentification.FastText/FastTextDetector.cs index 2541e49d..e17c2a28 100644 --- a/src/LanguageIdentification.FastText/FastTextDetector.cs +++ b/src/LanguageIdentification.FastText/FastTextDetector.cs @@ -158,10 +158,11 @@ ref errPtr var predictions = Marshal.PtrToStructure(predictionPtr); var result = new List(); - for (ulong i = 0; i < predictions.Length; i++) + var structSize = Marshal.SizeOf(); + + for (var i = 0; i < (int)predictions.Length; i++) { - IntPtr elementPtr = new IntPtr(predictions.Predictions.ToInt64() + (long)(i * (uint)Marshal.SizeOf())); - var prediction = Marshal.PtrToStructure(elementPtr); + var prediction = Marshal.PtrToStructure(predictions.Predictions + i * structSize); var label = DecodeString(prediction.Label); result.Add(new FastTextPrediction( diff --git a/src/LanguageIdentification.FastText/Internal/FastTextDetectorWrapper.cs b/src/LanguageIdentification.FastText/Internal/FastTextDetectorWrapper.cs index f8815d77..3f4756b1 100644 --- a/src/LanguageIdentification.FastText/Internal/FastTextDetectorWrapper.cs +++ b/src/LanguageIdentification.FastText/Internal/FastTextDetectorWrapper.cs @@ -14,16 +14,22 @@ internal static class FastTextDetectorWrapper public static extern void DestroyFastText(IntPtr handle); [DllImport(FastTextNativeLibrary.Name, CallingConvention = CallingConvention.Cdecl)] - public static extern void FastTextLoadModel(IntPtr handle, string filename, ref IntPtr errptr); + public static extern void FastTextLoadModel(IntPtr handle, string filename, ref IntPtr errPtr); [DllImport(FastTextNativeLibrary.Name, CallingConvention = CallingConvention.Cdecl)] - public static extern void FastTextLoadModelData(IntPtr handle, IntPtr buffer, uint bufferLength, ref IntPtr errptr); + public static extern void FastTextLoadModelData(IntPtr handle, IntPtr buffer, uint bufferLength, ref IntPtr errPtr); [DllImport(FastTextNativeLibrary.Name, CallingConvention = CallingConvention.Cdecl)] public static extern int FastTextGetModelDimensions(IntPtr handle); [DllImport(FastTextNativeLibrary.Name, CallingConvention = CallingConvention.Cdecl)] - public static extern IntPtr FastTextPredict(IntPtr handle, string text, int k, float threshold, ref IntPtr errptr); + public static extern IntPtr FastTextPredict( + IntPtr handle, + [MarshalAs(UnmanagedType.LPUTF8Str)] string text, + int k, + float threshold, + ref IntPtr errPtr + ); [DllImport(FastTextNativeLibrary.Name, CallingConvention = CallingConvention.Cdecl)] public static extern void DestroyPredictions(IntPtr predictions); From 0fad2fbe06fdac9bf883283380d61160e7abd7a8 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 30 Sep 2024 22:42:14 +0400 Subject: [PATCH 062/219] Update CMakeLists.txt --- .../Native/CMakeLists.txt | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index 3d8ff415..ad915958 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -4,17 +4,16 @@ cmake_minimum_required(VERSION 3.10 FATAL_ERROR) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) - -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") -if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3") -endif() +set(CMAKE_C_STANDARD 99) +set(CMAKE_C_STANDARD_REQUIRED ON) if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3 -march=native") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3 -march=native") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-rtti -lpthread") elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") - # nop + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /W3 /O2 /fp:precise /arch:AVX2") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W3 /O2 /fp:precise /arch:AVX2") endif() set(CMAKE_MACOSX_RPATH 1) From e9a0e03e489765c2c35e91553dd355f7b47bc69e Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 30 Sep 2024 22:57:08 +0400 Subject: [PATCH 063/219] CI: remove "build-docker-images" stage --- .github/workflows/github-ci.yml | 55 --------------------------------- 1 file changed, 55 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index e6af82f5..ca37b27c 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -8,64 +8,9 @@ on: workflow_dispatch: jobs: - build-docker-images: - name: πŸš€ Pack ${{matrix.task.projectName}} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - task: - - { projectName: LanguageIdentification.CLD2.Native, imageName: panlingo-builder-base-cld2 } - # - LanguageIdentification.CLD3 - # - LanguageIdentification.FastText - # - LanguageIdentification.Whatlang - # - LanguageIdentification.MediaPipe - # - LanguageIdentification.Lingua - permissions: - contents: read - packages: write - env: - ROOT: ./src - NUGET_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}} - steps: - - uses: actions/checkout@v4 - with: - submodules: 'true' - - - uses: dorny/paths-filter@v3 - id: changes - with: - filters: | - src: - - '${{env.ROOT}}/${{matrix.task.projectName}}/**' - - - name: Docker context - working-directory: . - run: docker context use default - - - name: Docker login - working-directory: . - run: | - echo ${{secrets.GITHUB_TOKEN}} | docker login ghcr.io \ - --username ${{github.repository_owner}} --password-stdin - - - name: Docker build and push - env: - IMAGE: ${{matrix.task.imageName}} - TAG: latest # TODO: use branch name - DOCKERFILE: ./Dockerfile - working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} - run: | - IMAGE_URL="ghcr.io/${{github.repository_owner}}/${{env.IMAGE}}" \ - docker build --pull --file ${{env.DOCKERFILE}} --tag ${{env.IMAGE}} --no-cache ../.. \ - - docker tag ${{env.IMAGE}} $IMAGE_URL:${{env.TAG}} - docker push $IMAGE_URL:${{env.TAG}} - build-nugets: name: πŸš€ Pack ${{matrix.task.projectName}} runs-on: ${{matrix.task.os}} - # needs: build-docker-images strategy: fail-fast: false matrix: From 2f2b6e07c291a2b701a10390c993b6889276ae1b Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 30 Sep 2024 22:57:28 +0400 Subject: [PATCH 064/219] CI: remove custom code for FT --- .github/workflows/github-ci.yml | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index ca37b27c..b4b71e23 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -17,7 +17,7 @@ jobs: task: - { projectName: LanguageIdentification.CLD2, os: ubuntu-latest } - { projectName: LanguageIdentification.CLD3, os: ubuntu-latest } - - { projectName: LanguageIdentification.FastText, os: windows-latest } + - { projectName: LanguageIdentification.FastText, os: ubuntu-latest } # - { projectName: LanguageIdentification.Whatlang, os: ubuntu-latest } # - { projectName: LanguageIdentification.MediaPipe, os: ubuntu-latest } # - { projectName: LanguageIdentification.Lingua, os: ubuntu-latest } @@ -33,19 +33,6 @@ jobs: with: submodules: 'true' - - name: Ensure Docker is installed - run: | - echo "Docker version:" - docker --version - - - name: Run a Linux container - run: | - echo "Pulling a Linux container image..." - docker pull ubuntu:latest - - echo "Running a Linux container..." - docker run --rm ubuntu:latest echo "Hello from Ubuntu container!" - - uses: dorny/paths-filter@v3 id: changes with: From a579a1ae92342b7baa5f3733cb5dd5c429c8db1f Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 30 Sep 2024 22:58:01 +0400 Subject: [PATCH 065/219] Update binding.cc --- src/LanguageIdentification.CLD3.Native/Native/src/binding.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc b/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc index 3198cbab..5104c3a1 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc +++ b/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc @@ -17,7 +17,7 @@ PredictionResult cld3_find_language(void* identifier, const char* text) { PredictionResult x = {}; - PredictionResult* result = static_cast(); + PredictionResult* result = static_cast(x); result.language = strdup(nativeResult.language.c_str()); result.probability = nativeResult.probability; result.is_reliable = nativeResult.is_reliable; From e61a0776266a469267450294d0be44d1bfa260bb Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 30 Sep 2024 23:04:25 +0400 Subject: [PATCH 066/219] Update binding.cc --- .../Native/src/binding.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc b/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc index 5104c3a1..1b8e2333 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc +++ b/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc @@ -1,5 +1,6 @@ #include "binding.h" #include +#include using namespace chrome_lang_id; @@ -16,12 +17,11 @@ PredictionResult cld3_find_language(void* identifier, const char* text) { auto nativeResult = nativeIdentifier->FindLanguage(text); - PredictionResult x = {}; - PredictionResult* result = static_cast(x); - result.language = strdup(nativeResult.language.c_str()); - result.probability = nativeResult.probability; - result.is_reliable = nativeResult.is_reliable; - result.proportion = nativeResult.proportion; + PredictionResult* result = static_cast(std::malloc(sizeof(PredictionResult))); + result->language = _strdup(nativeResult.language.c_str()); + result->probability = nativeResult.probability; + result->is_reliable = nativeResult.is_reliable; + result->proportion = nativeResult.proportion; return result; } @@ -32,7 +32,7 @@ PredictionResult* cld3_find_languages(void* identifier, const char* text, int nu *resultCount = static_cast(nativeResults.size()); PredictionResult* result = new PredictionResult[*resultCount]; for (int i = 0; i < *resultCount; ++i) { - result[i].language = strdup(nativeResults[i].language.c_str()); + result[i].language = _strdup(nativeResults[i].language.c_str()); result[i].probability = nativeResults[i].probability; result[i].is_reliable = nativeResults[i].is_reliable; result[i].proportion = nativeResults[i].proportion; From 5f892fe18b326e9cd904d2e65ae0e312f74142e2 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 30 Sep 2024 23:08:22 +0400 Subject: [PATCH 067/219] Update binding.cc --- src/LanguageIdentification.CLD3.Native/Native/src/binding.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc b/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc index 1b8e2333..f2e27258 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc +++ b/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc @@ -18,7 +18,7 @@ PredictionResult cld3_find_language(void* identifier, const char* text) { PredictionResult* result = static_cast(std::malloc(sizeof(PredictionResult))); - result->language = _strdup(nativeResult.language.c_str()); + result->language = strdup(nativeResult.language.c_str()); result->probability = nativeResult.probability; result->is_reliable = nativeResult.is_reliable; result->proportion = nativeResult.proportion; @@ -32,7 +32,7 @@ PredictionResult* cld3_find_languages(void* identifier, const char* text, int nu *resultCount = static_cast(nativeResults.size()); PredictionResult* result = new PredictionResult[*resultCount]; for (int i = 0; i < *resultCount; ++i) { - result[i].language = _strdup(nativeResults[i].language.c_str()); + result[i].language = strdup(nativeResults[i].language.c_str()); result[i].probability = nativeResults[i].probability; result[i].is_reliable = nativeResults[i].is_reliable; result[i].proportion = nativeResults[i].proportion; From 8d2f0283fac512852ef7f5c038ac9e7127f2fc10 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 30 Sep 2024 23:08:54 +0400 Subject: [PATCH 068/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index b4b71e23..88513ca5 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -17,7 +17,7 @@ jobs: task: - { projectName: LanguageIdentification.CLD2, os: ubuntu-latest } - { projectName: LanguageIdentification.CLD3, os: ubuntu-latest } - - { projectName: LanguageIdentification.FastText, os: ubuntu-latest } + # - { projectName: LanguageIdentification.FastText, os: ubuntu-latest } # - { projectName: LanguageIdentification.Whatlang, os: ubuntu-latest } # - { projectName: LanguageIdentification.MediaPipe, os: ubuntu-latest } # - { projectName: LanguageIdentification.Lingua, os: ubuntu-latest } From c1ca6fddede16fb75c9e18207dbd5848791ea7f2 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 30 Sep 2024 23:12:19 +0400 Subject: [PATCH 069/219] Update binding.cc --- src/LanguageIdentification.CLD3.Native/Native/src/binding.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc b/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc index f2e27258..59e14038 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc +++ b/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc @@ -12,7 +12,7 @@ void destroy_cld3(void* identifier) { delete static_cast(identifier); } -PredictionResult cld3_find_language(void* identifier, const char* text) { +PredictionResult* cld3_find_language(void* identifier, const char* text) { NNetLanguageIdentifier* nativeIdentifier = static_cast(identifier); auto nativeResult = nativeIdentifier->FindLanguage(text); From ebb55c353b05f9abef6e37e3e9e56477e9314c22 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 30 Sep 2024 23:21:15 +0400 Subject: [PATCH 070/219] CLD3: change cld3_find_language return type --- .../Native/src/binding.cc | 15 ++++---- .../Native/src/binding.h | 2 +- .../CLD3Detector.cs | 34 +++++++++++++++---- .../Internal/CLD3DetectorWrapper.cs | 7 ++-- 4 files changed, 40 insertions(+), 18 deletions(-) diff --git a/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc b/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc index 59e14038..ef91a907 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc +++ b/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc @@ -1,6 +1,5 @@ #include "binding.h" #include -#include using namespace chrome_lang_id; @@ -12,16 +11,16 @@ void destroy_cld3(void* identifier) { delete static_cast(identifier); } -PredictionResult* cld3_find_language(void* identifier, const char* text) { +PredictionResult* cld3_find_language(void* identifier, const char* text, int* resultCount) { NNetLanguageIdentifier* nativeIdentifier = static_cast(identifier); auto nativeResult = nativeIdentifier->FindLanguage(text); - - PredictionResult* result = static_cast(std::malloc(sizeof(PredictionResult))); - result->language = strdup(nativeResult.language.c_str()); - result->probability = nativeResult.probability; - result->is_reliable = nativeResult.is_reliable; - result->proportion = nativeResult.proportion; + *resultCount = 1; + PredictionResult* result = new PredictionResult[*resultCount]; + result[0].language = strdup(nativeResult.language.c_str()); + result[0].probability = nativeResult.probability; + result[0].is_reliable = nativeResult.is_reliable; + result[0].proportion = nativeResult.proportion; return result; } diff --git a/src/LanguageIdentification.CLD3.Native/Native/src/binding.h b/src/LanguageIdentification.CLD3.Native/Native/src/binding.h index 52acb61e..ca7d6186 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/src/binding.h +++ b/src/LanguageIdentification.CLD3.Native/Native/src/binding.h @@ -27,7 +27,7 @@ extern "C" { EXPORT void* create_cld3(int minNumBytes, int maxNumBytes); EXPORT void destroy_cld3(void* identifier); - EXPORT PredictionResult cld3_find_language(void* identifier, const char* text); + EXPORT PredictionResult* cld3_find_language(void* identifier, const char* text, int* resultCount); EXPORT PredictionResult* cld3_find_languages(void* identifier, const char* text, int numLangs, int* resultCount); EXPORT void cld3_destroy_prediction_result(PredictionResult* results, int count); } \ No newline at end of file diff --git a/src/LanguageIdentification.CLD3/CLD3Detector.cs b/src/LanguageIdentification.CLD3/CLD3Detector.cs index fd7db99f..d860f8ad 100644 --- a/src/LanguageIdentification.CLD3/CLD3Detector.cs +++ b/src/LanguageIdentification.CLD3/CLD3Detector.cs @@ -1,5 +1,6 @@ ο»Ώusing System; using System.Collections.Generic; +using System.Drawing; using System.Linq; using System.Runtime.InteropServices; using System.Threading; @@ -55,8 +56,29 @@ public void Dispose() /// List of language predictions public CLD3Prediction PredictLanguage(string text) { - var result = CLD3DetectorWrapper.FindLanguage(_identifier, text); - return new CLD3Prediction(result); + var resultPtr = CLD3DetectorWrapper.PredictLanguage( + identifier: _identifier, + text: text, + resultCount: out var resultCount + ); + + try + { + var nativeResult = new CLD3PredictionResult[resultCount]; + var structSize = Marshal.SizeOf(typeof(CLD3PredictionResult)); + + for (var i = 0; i < resultCount; i++) + { + nativeResult[i] = Marshal.PtrToStructure(resultPtr + i * structSize); + } + + var firstItem = nativeResult.First(); + return new CLD3Prediction(firstItem); + } + finally + { + CLD3DetectorWrapper.DestroyPredictionResult(resultPtr, resultCount); + } } /// @@ -70,7 +92,7 @@ public IEnumerable PredictLanguages( int count ) { - var resultPtr = CLD3DetectorWrapper.FindLanguages( + var resultPtr = CLD3DetectorWrapper.PredictLanguages( identifier: _identifier, text: text, numLangs: count, @@ -79,15 +101,15 @@ int count try { - var result = new CLD3PredictionResult[resultCount]; + var nativeResult = new CLD3PredictionResult[resultCount]; var structSize = Marshal.SizeOf(typeof(CLD3PredictionResult)); for (var i = 0; i < resultCount; i++) { - result[i] = Marshal.PtrToStructure(resultPtr + i * structSize); + nativeResult[i] = Marshal.PtrToStructure(resultPtr + i * structSize); } - return result + return nativeResult .OrderByDescending(x => x.Probability) .Select(x => new CLD3Prediction(x)) .ToArray(); diff --git a/src/LanguageIdentification.CLD3/Internal/CLD3DetectorWrapper.cs b/src/LanguageIdentification.CLD3/Internal/CLD3DetectorWrapper.cs index 009a2360..9fc9e0de 100644 --- a/src/LanguageIdentification.CLD3/Internal/CLD3DetectorWrapper.cs +++ b/src/LanguageIdentification.CLD3/Internal/CLD3DetectorWrapper.cs @@ -13,13 +13,14 @@ internal static class CLD3DetectorWrapper public static extern void DestroyCLD3(IntPtr identifier); [DllImport(CLD3NativeLibrary.Name, EntryPoint = "cld3_find_language", CallingConvention = CallingConvention.Cdecl)] - public static extern CLD3PredictionResult FindLanguage( + public static extern IntPtr PredictLanguage( IntPtr identifier, - [MarshalAs(UnmanagedType.LPUTF8Str)] string text + [MarshalAs(UnmanagedType.LPUTF8Str)] string text, + out int resultCount ); [DllImport(CLD3NativeLibrary.Name, EntryPoint = "cld3_find_languages", CallingConvention = CallingConvention.Cdecl)] - public static extern IntPtr FindLanguages( + public static extern IntPtr PredictLanguages( IntPtr identifier, [MarshalAs(UnmanagedType.LPUTF8Str)] string text, int numLangs, From 4f5d1b8e6e756ecd36dba355784cb3433533598a Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 30 Sep 2024 23:34:25 +0400 Subject: [PATCH 071/219] CLD2: fix test --- src/LanguageIdentification.Tests/CLD2Tests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.Tests/CLD2Tests.cs b/src/LanguageIdentification.Tests/CLD2Tests.cs index 6a39573d..220b30ab 100644 --- a/src/LanguageIdentification.Tests/CLD2Tests.cs +++ b/src/LanguageIdentification.Tests/CLD2Tests.cs @@ -8,7 +8,7 @@ public class CLD2Tests [SkippableTheory] [InlineData("en", Constants.PHRASE_ENG_1, 0.9999)] [InlineData("uk", Constants.PHRASE_UKR_1, 0.9999)] - [InlineData("un", Constants.PHRASE_RUS_1, 0.9999)] + [InlineData("un", Constants.PHRASE_RUS_1, 0)] public void CLD2SingleLanguage(string languageCode, string text, double score) { Skip.IfNot(CLD2Detector.IsSupported()); From f2d2d98bcc47ccc64d59101a53587ce18b8e72ea Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 30 Sep 2024 23:50:31 +0400 Subject: [PATCH 072/219] FT: Try to build in Windows environment --- .github/workflows/github-ci.yml | 40 +++++++++++++++++- .../Native/toolchain-mingw.cmake | 16 ------- .../Native/toolchain-osxcross.cmake | 7 ---- .../Scripts/run-build.ps1 | 42 +++++++++++++++++++ .../Scripts/run-build.sh | 9 ---- .../Scripts/setup-build.sh | 7 ---- 6 files changed, 81 insertions(+), 40 deletions(-) delete mode 100644 src/LanguageIdentification.FastText.Native/Native/toolchain-mingw.cmake delete mode 100644 src/LanguageIdentification.FastText.Native/Native/toolchain-osxcross.cmake create mode 100644 src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 88513ca5..68703a0b 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -8,6 +8,44 @@ on: workflow_dispatch: jobs: + build-native: + name: πŸš€ Build ${{matrix.task.projectName}} for (${{matrix.task.os}}) + runs-on: ${{matrix.task.os}} + strategy: + fail-fast: false + matrix: + task: + - { projectName: LanguageIdentification.FastText.Native, os: windows-latest, script: ./Scripts/run-build.ps1 } + permissions: + contents: read + packages: write + env: + ROOT: ./src + NUGET_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}} + steps: + - uses: actions/checkout@v4 + with: + submodules: 'true' + + - uses: dorny/paths-filter@v3 + id: changes + with: + filters: | + src: + - '${{env.ROOT}}/${{matrix.task.projectName}}/**' + + - uses: dorny/paths-filter@v3 + id: changes-native + with: + filters: | + src: + - '${{env.ROOT}}/${{matrix.task.projectName}}.Native/**' + + - name: πŸš€ Build ${{matrix.task.projectName}} + if: steps.changes.outputs.src == 'true' + working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} + run: ${{matrix.task.script}} + build-nugets: name: πŸš€ Pack ${{matrix.task.projectName}} runs-on: ${{matrix.task.os}} @@ -17,7 +55,7 @@ jobs: task: - { projectName: LanguageIdentification.CLD2, os: ubuntu-latest } - { projectName: LanguageIdentification.CLD3, os: ubuntu-latest } - # - { projectName: LanguageIdentification.FastText, os: ubuntu-latest } + - { projectName: LanguageIdentification.FastText, os: ubuntu-latest } # - { projectName: LanguageIdentification.Whatlang, os: ubuntu-latest } # - { projectName: LanguageIdentification.MediaPipe, os: ubuntu-latest } # - { projectName: LanguageIdentification.Lingua, os: ubuntu-latest } diff --git a/src/LanguageIdentification.FastText.Native/Native/toolchain-mingw.cmake b/src/LanguageIdentification.FastText.Native/Native/toolchain-mingw.cmake deleted file mode 100644 index 94e97c6b..00000000 --- a/src/LanguageIdentification.FastText.Native/Native/toolchain-mingw.cmake +++ /dev/null @@ -1,16 +0,0 @@ -set(CMAKE_SYSTEM_NAME Windows) -set(TOOLCHAIN_PREFIX x86_64-w64-mingw32) - -# cross compilers to use for C, C++ and Fortran -set(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}-gcc) -set(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}-g++) -set(CMAKE_Fortran_COMPILER ${TOOLCHAIN_PREFIX}-gfortran) -set(CMAKE_RC_COMPILER ${TOOLCHAIN_PREFIX}-windres) - -# target environment on the build host system -set(CMAKE_FIND_ROOT_PATH /usr/${TOOLCHAIN_PREFIX}) - -# modify default behavior of FIND_XXX() commands -set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) -set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) \ No newline at end of file diff --git a/src/LanguageIdentification.FastText.Native/Native/toolchain-osxcross.cmake b/src/LanguageIdentification.FastText.Native/Native/toolchain-osxcross.cmake deleted file mode 100644 index 6e1223e1..00000000 --- a/src/LanguageIdentification.FastText.Native/Native/toolchain-osxcross.cmake +++ /dev/null @@ -1,7 +0,0 @@ -set(CMAKE_SYSTEM_NAME Darwin) -set(CMAKE_SYSTEM_VERSION 1) - -# Path to the osxcross toolchain binaries -set(CMAKE_OSX_SYSROOT /usr/local/osxcross/target) -set(CMAKE_C_COMPILER /usr/local/osxcross/bin/o64-clang) -set(CMAKE_CXX_COMPILER /usr/local/osxcross/bin/o64-clang++) \ No newline at end of file diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 b/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 new file mode 100644 index 00000000..7b78b152 --- /dev/null +++ b/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 @@ -0,0 +1,42 @@ +# PowerShell equivalent script +# Ensure script stops if any command fails +$ErrorActionPreference = "Stop" + +Write-Output "Hello world" + +$workspace = "build_temp" + +# Create directory if it doesn't exist +if (-Not (Test-Path $workspace)) { + New-Item -Path $workspace -ItemType Directory +} + +# Copy directories +Copy-Item -Path "../../third_party/fastText" -Destination "$workspace/fasttext" -Recurse -Force +Copy-Item -Path "Native" -Destination $workspace -Recurse -Force + +# List directory contents recursively +Get-ChildItem -Recurse -Path . + +Set-Location $workspace + +# Create and enter build directory +if (-Not (Test-Path "build")) { + New-Item -Path "build" -ItemType Directory +} +Set-Location "build" + +# Build for Windows +Remove-Item -Recurse -Force * +cmake .. +cmake --build . + +# List directory contents recursively +Get-ChildItem -Recurse -Path . + +# Display shared library dependencies +Copy-Item -Path "fasttext.so" -Destination "../../fasttext.so" + +# Clean up +Remove-Item -Recurse -Force ("../" + $workspace) +Write-Output "Goodbye world" \ No newline at end of file diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.sh b/src/LanguageIdentification.FastText.Native/Scripts/run-build.sh index a150c18d..e069b375 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/run-build.sh +++ b/src/LanguageIdentification.FastText.Native/Scripts/run-build.sh @@ -26,15 +26,6 @@ ls -R ldd libfasttext.so cp libfasttext.so ../../libfasttext.so -# Build for Windows -rm -rf * -wine cmake .. -wine cmake --build . - -ls -R - -cp libfasttext.dll ../../libfasttext.dll - # Clean up rm -rf "$workspace" echo "Goodbye world"; diff --git a/src/LanguageIdentification.FastText.Native/Scripts/setup-build.sh b/src/LanguageIdentification.FastText.Native/Scripts/setup-build.sh index 53c76ccf..4f5708d0 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/setup-build.sh +++ b/src/LanguageIdentification.FastText.Native/Scripts/setup-build.sh @@ -6,11 +6,4 @@ echo "Installing build packages"; sudo apt -y update | apt -y update sudo apt -y install cmake | apt -y install cmake sudo apt -y install g++ | apt -y install g++ -sudo apt -y install gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 | apt -y install gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 - -apt -y install wine wine64 | sudo apt -y install wine wine64 - -wget https://github.com/Kitware/CMake/releases/download/v3.30.3/cmake-3.30.3-windows-x86_64.msi -wine msiexec /i cmake-3.30.3-windows-x86_64.msi /quiet -wine cmake --version From 716852526e441f37bf62b764588153f460389073 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 30 Sep 2024 23:56:01 +0400 Subject: [PATCH 073/219] Update run-build.ps1 --- .../Scripts/run-build.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 b/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 index 7b78b152..137ad164 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 +++ b/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 @@ -35,7 +35,10 @@ cmake --build . Get-ChildItem -Recurse -Path . # Display shared library dependencies -Copy-Item -Path "fasttext.so" -Destination "../../fasttext.so" +Copy-Item -Path "fasttext.dll" -Destination "../../fasttext.dll" + +# List directory contents recursively +Get-ChildItem -Recurse -Path . # Clean up Remove-Item -Recurse -Force ("../" + $workspace) From 2127bca85ba0b1941daa531d4b91b9a918a25366 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Tue, 1 Oct 2024 00:03:31 +0400 Subject: [PATCH 074/219] Update run-build.ps1 --- .../Scripts/run-build.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 b/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 index 137ad164..20bcd1c2 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 +++ b/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 @@ -35,7 +35,7 @@ cmake --build . Get-ChildItem -Recurse -Path . # Display shared library dependencies -Copy-Item -Path "fasttext.dll" -Destination "../../fasttext.dll" +Copy-Item -Path ".\Debug\fasttext.dll" -Destination "../../fasttext.dll" # List directory contents recursively Get-ChildItem -Recurse -Path . From 4a0db847c10442c73ecca99e5f6ea96df200558f Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Tue, 1 Oct 2024 00:08:08 +0400 Subject: [PATCH 075/219] Update run-build.ps1 --- src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 b/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 index 20bcd1c2..3683ccf9 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 +++ b/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 @@ -27,7 +27,6 @@ if (-Not (Test-Path "build")) { Set-Location "build" # Build for Windows -Remove-Item -Recurse -Force * cmake .. cmake --build . From b1f135ab0fb25d95998f7833bbebf9f50e4ef8fa Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Tue, 1 Oct 2024 00:23:43 +0400 Subject: [PATCH 076/219] Update run-build.ps1 --- .../Scripts/run-build.ps1 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 b/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 index 3683ccf9..3e0987d6 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 +++ b/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 @@ -12,8 +12,8 @@ if (-Not (Test-Path $workspace)) { } # Copy directories -Copy-Item -Path "../../third_party/fastText" -Destination "$workspace/fasttext" -Recurse -Force -Copy-Item -Path "Native" -Destination $workspace -Recurse -Force +Copy-Item -Path "../../third_party/fastText/*" -Destination "$workspace/fasttext" -Recurse -Force +Copy-Item -Path "Native/*" -Destination $workspace -Recurse -Force # List directory contents recursively Get-ChildItem -Recurse -Path . @@ -40,5 +40,4 @@ Copy-Item -Path ".\Debug\fasttext.dll" -Destination "../../fasttext.dll" Get-ChildItem -Recurse -Path . # Clean up -Remove-Item -Recurse -Force ("../" + $workspace) Write-Output "Goodbye world" \ No newline at end of file From 4701beff367fb97cd9668ba2011f960ff9158471 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Tue, 1 Oct 2024 01:42:30 +0400 Subject: [PATCH 077/219] FT: WIndows build fixes + MacOS support for fun --- .github/workflows/github-ci.yml | 2 ++ .../Scripts/run-build.ps1 | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 68703a0b..e184c77a 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -16,6 +16,7 @@ jobs: matrix: task: - { projectName: LanguageIdentification.FastText.Native, os: windows-latest, script: ./Scripts/run-build.ps1 } + - { projectName: LanguageIdentification.FastText.Native, os: macos-latest, script: ./Scripts/run-build.sh } permissions: contents: read packages: write @@ -49,6 +50,7 @@ jobs: build-nugets: name: πŸš€ Pack ${{matrix.task.projectName}} runs-on: ${{matrix.task.os}} + needs: build-native strategy: fail-fast: false matrix: diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 b/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 index 3e0987d6..c34c510c 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 +++ b/src/LanguageIdentification.FastText.Native/Scripts/run-build.ps1 @@ -9,6 +9,7 @@ $workspace = "build_temp" # Create directory if it doesn't exist if (-Not (Test-Path $workspace)) { New-Item -Path $workspace -ItemType Directory + New-Item -Path "$workspace/fasttext" -ItemType Directory } # Copy directories @@ -40,4 +41,5 @@ Copy-Item -Path ".\Debug\fasttext.dll" -Destination "../../fasttext.dll" Get-ChildItem -Recurse -Path . # Clean up +cd ../.. Write-Output "Goodbye world" \ No newline at end of file From ec296f00d7e5f8ec0cefe0f59e4ba646777ebcae Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Tue, 1 Oct 2024 22:05:58 +0400 Subject: [PATCH 078/219] FT: try to fix build-native stage + artifact upload --- .github/workflows/github-ci.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index e184c77a..636708ee 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -9,14 +9,14 @@ on: jobs: build-native: - name: πŸš€ Build ${{matrix.task.projectName}} for (${{matrix.task.os}}) + name: πŸš€ Build ${{matrix.task.projectName}} for ${{matrix.task.os}} runs-on: ${{matrix.task.os}} strategy: fail-fast: false matrix: task: - - { projectName: LanguageIdentification.FastText.Native, os: windows-latest, script: ./Scripts/run-build.ps1 } - - { projectName: LanguageIdentification.FastText.Native, os: macos-latest, script: ./Scripts/run-build.sh } + - { projectName: LanguageIdentification.FastText.Native, os: windows-latest, script: ./Scripts/run-build.ps1, artifact: fasttext.dll } + - { projectName: LanguageIdentification.FastText.Native, os: macos-latest, script: ./Scripts/run-build.sh, artifact: libfasttext.dylib } permissions: contents: read packages: write @@ -45,7 +45,19 @@ jobs: - name: πŸš€ Build ${{matrix.task.projectName}} if: steps.changes.outputs.src == 'true' working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} - run: ${{matrix.task.script}} + run: | + git update-index --chmod=+x ${{matrix.task.script}} + ${{matrix.task.script}} + + - uses: actions/upload-artifact@v4 + if: steps.changes-native.outputs.src == 'true' + with: + name: build-${{matrix.task.projectName}}:${{matrix.task.os}} + path: ${{env.ROOT}}/${{matrix.task.projectName}}/${{matrix.task.artifact}} + retention-days: 1 + overwrite: 'true' + compression-level: 0 + if-no-files-found: 'error' build-nugets: name: πŸš€ Pack ${{matrix.task.projectName}} From 9831062b78d42d24989af0f94b40a2eb49248c12 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Tue, 1 Oct 2024 22:08:38 +0400 Subject: [PATCH 079/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 636708ee..ad967ce0 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -16,7 +16,7 @@ jobs: matrix: task: - { projectName: LanguageIdentification.FastText.Native, os: windows-latest, script: ./Scripts/run-build.ps1, artifact: fasttext.dll } - - { projectName: LanguageIdentification.FastText.Native, os: macos-latest, script: ./Scripts/run-build.sh, artifact: libfasttext.dylib } + # - { projectName: LanguageIdentification.FastText.Native, os: macos-latest, script: ./Scripts/run-build.sh, artifact: libfasttext.dylib } permissions: contents: read packages: write @@ -50,7 +50,7 @@ jobs: ${{matrix.task.script}} - uses: actions/upload-artifact@v4 - if: steps.changes-native.outputs.src == 'true' + if: steps.changes.outputs.src == 'true' with: name: build-${{matrix.task.projectName}}:${{matrix.task.os}} path: ${{env.ROOT}}/${{matrix.task.projectName}}/${{matrix.task.artifact}} From 6f2ed01b1d984d7b24ffd6cab5684ccd076b936f Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Tue, 1 Oct 2024 22:11:54 +0400 Subject: [PATCH 080/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index ad967ce0..af99e068 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -52,7 +52,7 @@ jobs: - uses: actions/upload-artifact@v4 if: steps.changes.outputs.src == 'true' with: - name: build-${{matrix.task.projectName}}:${{matrix.task.os}} + name: build-${{matrix.task.projectName}}__${{matrix.task.os}} path: ${{env.ROOT}}/${{matrix.task.projectName}}/${{matrix.task.artifact}} retention-days: 1 overwrite: 'true' From e03748b12fe355a853367b00e7a69ce39eec440e Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Tue, 1 Oct 2024 22:27:27 +0400 Subject: [PATCH 081/219] TF: Download artifact in "build-nugets" stage --- .github/workflows/github-ci.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index af99e068..7f65f0b6 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -52,7 +52,7 @@ jobs: - uses: actions/upload-artifact@v4 if: steps.changes.outputs.src == 'true' with: - name: build-${{matrix.task.projectName}}__${{matrix.task.os}} + name: build-${{matrix.task.projectName}}-${{matrix.task.os}} path: ${{env.ROOT}}/${{matrix.task.projectName}}/${{matrix.task.artifact}} retention-days: 1 overwrite: 'true' @@ -85,6 +85,12 @@ jobs: with: submodules: 'true' + - uses: actions/download-artifact@v4 + with: + pattern: build-${{matrix.task.projectName}}-* + merge-multiple: true + path: ${{env.ROOT}}/${{matrix.task.projectName}}/ + - uses: dorny/paths-filter@v3 id: changes with: From addb3a2ea1668a39fe8989281c468d389c12fd30 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Tue, 1 Oct 2024 22:31:30 +0400 Subject: [PATCH 082/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 7f65f0b6..c614ad7d 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -89,7 +89,7 @@ jobs: with: pattern: build-${{matrix.task.projectName}}-* merge-multiple: true - path: ${{env.ROOT}}/${{matrix.task.projectName}}/ + path: ${{env.ROOT}}/${{matrix.task.projectName}}.Native/ - uses: dorny/paths-filter@v3 id: changes From 1be36b88319fc7cdd23541fecbc09c4a1ba71726 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Tue, 1 Oct 2024 22:37:00 +0400 Subject: [PATCH 083/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index c614ad7d..88ece7aa 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -87,7 +87,7 @@ jobs: - uses: actions/download-artifact@v4 with: - pattern: build-${{matrix.task.projectName}}-* + pattern: build-${{matrix.task.projectName}}.Native-* merge-multiple: true path: ${{env.ROOT}}/${{matrix.task.projectName}}.Native/ From f4d7d672533ecd690b13ca40c8273ee4048a8781 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Tue, 1 Oct 2024 22:57:57 +0400 Subject: [PATCH 084/219] FT: download pretrained models for win & mac tests --- .github/workflows/github-ci.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 88ece7aa..097d2012 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -214,8 +214,14 @@ jobs: - name: πŸ“‚ Use local NuGet run: | - $path = [System.IO.Path]::GetFullPath('${{env.ROOT}}/local-nugets') - dotnet nuget add source $path + $path = [System.IO.Path]::GetFullPath('${{env.ROOT}}/local-nugets') + dotnet nuget add source $path + + - name: πŸ“‚ Download pretrained models + run: | + mkdir /models + Invoke-WebRequest -OutFile /models/fasttext176.bin https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin + Invoke-WebRequest -OutFile /models/mediapipe_language_detector.tflite https://storage.googleapis.com/mediapipe-models/language_detector/language_detector/float32/1/language_detector.tflite - name: πŸ§ͺ Run Tests for LanguageIdentification working-directory: ${{env.ROOT}}/LanguageIdentification.Tests @@ -263,8 +269,14 @@ jobs: - name: πŸ“‚ Use local NuGet run: | - path=$(realpath "${{env.ROOT}}/local-nugets") - dotnet nuget add source $path + path=$(realpath "${{env.ROOT}}/local-nugets") + dotnet nuget add source $path + + - name: πŸ“‚ Download pretrained models + run: | + mkdir /models -p + curl --location -o /models/fasttext176.bin https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin + curl --location -o /models/mediapipe_language_detector.tflite https://storage.googleapis.com/mediapipe-models/language_detector/language_detector/float32/1/language_detector.tflite - name: πŸ§ͺ Run Tests for LanguageIdentification working-directory: ${{env.ROOT}}/LanguageIdentification.Tests From 4daa82a2e8bde2e64cd556878679f22fcabcbff9 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Tue, 1 Oct 2024 23:16:43 +0400 Subject: [PATCH 085/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 097d2012..ec880672 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -274,9 +274,9 @@ jobs: - name: πŸ“‚ Download pretrained models run: | - mkdir /models -p - curl --location -o /models/fasttext176.bin https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin - curl --location -o /models/mediapipe_language_detector.tflite https://storage.googleapis.com/mediapipe-models/language_detector/language_detector/float32/1/language_detector.tflite + mkdir ${{runner.temp}}/models -p + curl --location -o ${{runner.temp}}/models/fasttext176.bin https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin + curl --location -o ${{runner.temp}}/models/mediapipe_language_detector.tflite https://storage.googleapis.com/mediapipe-models/language_detector/language_detector/float32/1/language_detector.tflite - name: πŸ§ͺ Run Tests for LanguageIdentification working-directory: ${{env.ROOT}}/LanguageIdentification.Tests From f34a9767524fbd475db238afaced1004b8af450c Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Tue, 1 Oct 2024 23:17:20 +0400 Subject: [PATCH 086/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index ec880672..808d3c7d 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -274,9 +274,9 @@ jobs: - name: πŸ“‚ Download pretrained models run: | - mkdir ${{runner.temp}}/models -p - curl --location -o ${{runner.temp}}/models/fasttext176.bin https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin - curl --location -o ${{runner.temp}}/models/mediapipe_language_detector.tflite https://storage.googleapis.com/mediapipe-models/language_detector/language_detector/float32/1/language_detector.tflite + mkdir ~/models -p + curl --location -o ~/models/fasttext176.bin https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin + curl --location -o ~/models/mediapipe_language_detector.tflite https://storage.googleapis.com/mediapipe-models/language_detector/language_detector/float32/1/language_detector.tflite - name: πŸ§ͺ Run Tests for LanguageIdentification working-directory: ${{env.ROOT}}/LanguageIdentification.Tests From 1cdba8d07d5c380ce9eb1733b833a8796b22295f Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Tue, 1 Oct 2024 23:30:36 +0400 Subject: [PATCH 087/219] FT: try to fix build for MacOS --- .github/workflows/github-ci.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 808d3c7d..26f59ff7 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -9,19 +9,19 @@ on: jobs: build-native: - name: πŸš€ Build ${{matrix.task.projectName}} for ${{matrix.task.os}} + name: πŸ’Ύ Build ${{matrix.task.title}} for ${{matrix.task.os}} runs-on: ${{matrix.task.os}} strategy: fail-fast: false matrix: task: - - { projectName: LanguageIdentification.FastText.Native, os: windows-latest, script: ./Scripts/run-build.ps1, artifact: fasttext.dll } - # - { projectName: LanguageIdentification.FastText.Native, os: macos-latest, script: ./Scripts/run-build.sh, artifact: libfasttext.dylib } + - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: windows-latest, script: ./Scripts/run-build.ps1, artifact: fasttext.dll } + - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: macos-latest, script: ./Scripts/run-build.sh, artifact: libfasttext.dylib } permissions: contents: read packages: write env: - ROOT: ./src + ROOT: ${{runner.temp}}/src NUGET_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}} steps: - uses: actions/checkout@v4 @@ -60,20 +60,20 @@ jobs: if-no-files-found: 'error' build-nugets: - name: πŸš€ Pack ${{matrix.task.projectName}} + name: πŸš€ Pack ${{matrix.task.title}} runs-on: ${{matrix.task.os}} needs: build-native strategy: fail-fast: false matrix: task: - - { projectName: LanguageIdentification.CLD2, os: ubuntu-latest } - - { projectName: LanguageIdentification.CLD3, os: ubuntu-latest } - - { projectName: LanguageIdentification.FastText, os: ubuntu-latest } - # - { projectName: LanguageIdentification.Whatlang, os: ubuntu-latest } - # - { projectName: LanguageIdentification.MediaPipe, os: ubuntu-latest } - # - { projectName: LanguageIdentification.Lingua, os: ubuntu-latest } - # - { projectName: LanguageCode, os: ubuntu-latest } + - { title: CLD2, projectName: LanguageIdentification.CLD2, os: ubuntu-latest } + - { title: CLD3, projectName: LanguageIdentification.CLD3, os: ubuntu-latest } + - { title: FastText, projectName: LanguageIdentification.FastText, os: ubuntu-latest } + # - { title: Whatlang, projectName: LanguageIdentification.Whatlang, os: ubuntu-latest } + # - { title: MediaPipe, projectName: LanguageIdentification.MediaPipe, os: ubuntu-latest } + # - { title: Lingua, projectName: LanguageIdentification.Lingua, os: ubuntu-latest } + # - { title: LanguageCode, projectName: LanguageCode, os: ubuntu-latest } permissions: contents: read packages: write From 21b29faa6a69ebc71deb3003c73fbea6ff0b6747 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Tue, 1 Oct 2024 23:39:16 +0400 Subject: [PATCH 088/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 26f59ff7..c579e2e0 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -21,7 +21,7 @@ jobs: contents: read packages: write env: - ROOT: ${{runner.temp}}/src + ROOT: $RUNNER_TEMP/src NUGET_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}} steps: - uses: actions/checkout@v4 From 3c1e1010603fbdf7a16ed840544c6e7c23cc5c18 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Tue, 1 Oct 2024 23:44:08 +0400 Subject: [PATCH 089/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index c579e2e0..9b44bf87 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -21,7 +21,7 @@ jobs: contents: read packages: write env: - ROOT: $RUNNER_TEMP/src + ROOT: ~/src NUGET_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}} steps: - uses: actions/checkout@v4 From a091713e0ee6e8dd64eb65bf1f3e78c8169faa7f Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Tue, 1 Oct 2024 23:48:33 +0400 Subject: [PATCH 090/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 9b44bf87..34f12f7e 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -21,7 +21,7 @@ jobs: contents: read packages: write env: - ROOT: ~/src + ROOT: $GITHUB_WORKSPACE/src NUGET_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}} steps: - uses: actions/checkout@v4 From 8ce9640ecb68c44fbf37111c4b24594be5fdda26 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Tue, 1 Oct 2024 23:50:08 +0400 Subject: [PATCH 091/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 34f12f7e..11acdbff 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -21,7 +21,7 @@ jobs: contents: read packages: write env: - ROOT: $GITHUB_WORKSPACE/src + ROOT: ./src NUGET_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}} steps: - uses: actions/checkout@v4 From 9faf7dafdb27d6442bd1d1a9261e0a8cee12a49f Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 00:04:15 +0400 Subject: [PATCH 092/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 11acdbff..e7deb9a0 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -46,6 +46,7 @@ jobs: if: steps.changes.outputs.src == 'true' working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} run: | + echo $(pwd) git update-index --chmod=+x ${{matrix.task.script}} ${{matrix.task.script}} From 0e59a6a51c61897995e93259ebba4e81bc5ceea8 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 00:20:21 +0400 Subject: [PATCH 093/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index e7deb9a0..0315bc05 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -48,6 +48,7 @@ jobs: run: | echo $(pwd) git update-index --chmod=+x ${{matrix.task.script}} + chmod +x ${{matrix.task.script}} ${{matrix.task.script}} - uses: actions/upload-artifact@v4 From 609865f4132eba8712d0489f0beea713e358f023 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 00:52:17 +0400 Subject: [PATCH 094/219] FT: fix MacOS build (-static-libgcc remove) --- .github/workflows/github-ci.yml | 2 +- .../Native/CMakeLists.txt | 6 ++++ .../Native/CMakeLists.txt | 6 ++++ .../Native/CMakeLists.txt | 13 ++++++-- .../Scripts/run-build.macos.sh | 31 +++++++++++++++++++ 5 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 0315bc05..fbea9600 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -16,7 +16,7 @@ jobs: matrix: task: - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: windows-latest, script: ./Scripts/run-build.ps1, artifact: fasttext.dll } - - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: macos-latest, script: ./Scripts/run-build.sh, artifact: libfasttext.dylib } + - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: macos-latest, script: ./Scripts/run-build.macos.sh, artifact: libfasttext.dylib } permissions: contents: read packages: write diff --git a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt index c147bb1d..a6ab3a15 100644 --- a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt @@ -31,6 +31,12 @@ endif() set(CMAKE_MACOSX_RPATH 1) set(CMAKE_POSITION_INDEPENDENT_CODE ON) +message(STATUS "CMake version: ${CMAKE_VERSION}") +message(STATUS "C++ compiler: ${CMAKE_CXX_COMPILER_ID}") +message(STATUS "C compiler: ${CMAKE_C_COMPILER_ID}") +message(STATUS "CXX flags: ${CMAKE_CXX_FLAGS}") +message(STATUS "C flags: ${CMAKE_C_FLAGS}") + include_directories( ${PROJECT_SOURCE_DIR}/cld2/internal ${PROJECT_SOURCE_DIR}/cld2/public diff --git a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt index 63ef11ef..ade1fecf 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt @@ -23,6 +23,12 @@ endif() add_definitions(-fPIC) # Position Independent Code add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0) +message(STATUS "CMake version: ${CMAKE_VERSION}") +message(STATUS "C++ compiler: ${CMAKE_CXX_COMPILER_ID}") +message(STATUS "C compiler: ${CMAKE_C_COMPILER_ID}") +message(STATUS "CXX flags: ${CMAKE_CXX_FLAGS}") +message(STATUS "C flags: ${CMAKE_C_FLAGS}") + add_library(${PROJECT_NAME} SHARED src/base.cc src/embedding_feature_extractor.cc diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index ad915958..f9078958 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -1,4 +1,4 @@ -project(fasttext) +ο»Ώproject(fasttext) cmake_minimum_required(VERSION 3.10 FATAL_ERROR) @@ -7,10 +7,13 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_C_STANDARD 99) set(CMAKE_C_STANDARD_REQUIRED ON) -if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") +if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3 -march=native") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3 -march=native") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-rtti -lpthread") +elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3 -march=native") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3 -march=native") elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /W3 /O2 /fp:precise /arch:AVX2") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W3 /O2 /fp:precise /arch:AVX2") @@ -19,6 +22,12 @@ endif() set(CMAKE_MACOSX_RPATH 1) set(CMAKE_POSITION_INDEPENDENT_CODE ON) +message(STATUS "CMake version: ${CMAKE_VERSION}") +message(STATUS "C++ compiler: ${CMAKE_CXX_COMPILER_ID}") +message(STATUS "C compiler: ${CMAKE_C_COMPILER_ID}") +message(STATUS "CXX flags: ${CMAKE_CXX_FLAGS}") +message(STATUS "C flags: ${CMAKE_C_FLAGS}") + add_subdirectory(fasttext EXCLUDE_FROM_ALL) include_directories( diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh new file mode 100644 index 00000000..9230ba61 --- /dev/null +++ b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -e + +echo "Hello world"; + +workspace="build_temp" + +mkdir "$workspace" -p +cp -a ../../third_party/fastText/. $workspace/fasttext +cp -a Native/. $workspace + +ls -R . + +cd "$workspace" + +mkdir build +cd build + +# Build for MacOS +rm -rf * +cmake .. +make -j $(nproc) # make + +ls -R + +ldd libfasttext.dylib +cp libfasttext.dylib ../../libfasttext.dylib + +# Clean up +rm -rf "$workspace" +echo "Goodbye world"; From 7e91c4fe719790a2e2e89c13ffd9a3988df48e37 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 00:58:54 +0400 Subject: [PATCH 095/219] FT: add MacOS support --- .../LanguageIdentification.FastText.Native.csproj | 12 +++++++++++- .../Native/CMakeLists.txt | 1 + .../Scripts/run-build.macos.sh | 2 +- .../FastTextDetector.cs | 2 +- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj b/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj index 3f661b5e..b2f1927c 100644 --- a/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj +++ b/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj @@ -71,7 +71,17 @@ false - + + + + PreserveNewest + true + runtimes/osx-x64/native + true + false + + + diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index f9078958..cd0a0b14 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -14,6 +14,7 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3 -march=native") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3 -march=native") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xlinker -lgcc") elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /W3 /O2 /fp:precise /arch:AVX2") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W3 /O2 /fp:precise /arch:AVX2") diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh index 9230ba61..03f8d798 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh +++ b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh @@ -23,7 +23,7 @@ make -j $(nproc) # make ls -R -ldd libfasttext.dylib +otool -L libfasttext.dylib cp libfasttext.dylib ../../libfasttext.dylib # Clean up diff --git a/src/LanguageIdentification.FastText/FastTextDetector.cs b/src/LanguageIdentification.FastText/FastTextDetector.cs index e17c2a28..75cf6008 100644 --- a/src/LanguageIdentification.FastText/FastTextDetector.cs +++ b/src/LanguageIdentification.FastText/FastTextDetector.cs @@ -31,7 +31,7 @@ public FastTextDetector() public static bool IsSupported() { - return RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + return RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.Windows) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX); } public string ModelPath { get; private set; } = string.Empty; From 451acecb8f16ad8e04bb413392a552c703d44a4e Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 01:02:00 +0400 Subject: [PATCH 096/219] Update CMakeLists.txt --- src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index cd0a0b14..f9078958 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -14,7 +14,6 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3 -march=native") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3 -march=native") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xlinker -lgcc") elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /W3 /O2 /fp:precise /arch:AVX2") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W3 /O2 /fp:precise /arch:AVX2") From 7cfb73aaed9b8a890d6183b2202a2d1de1bce516 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 01:24:05 +0400 Subject: [PATCH 097/219] Update CMakeLists.txt --- .../Native/CMakeLists.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index f9078958..796035e5 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -10,10 +10,12 @@ set(CMAKE_C_STANDARD_REQUIRED ON) if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3 -march=native") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3 -march=native") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-rtti -lpthread") elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3 -march=native") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3 -march=native") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++") elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /W3 /O2 /fp:precise /arch:AVX2") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W3 /O2 /fp:precise /arch:AVX2") @@ -34,7 +36,6 @@ include_directories( ${PROJECT_SOURCE_DIR}/fasttext/src ${PROJECT_SOURCE_DIR}/include) - set(SOURCES ${PROJECT_SOURCE_DIR}/binding.cc) add_library(objlib OBJECT ${SOURCES}) @@ -43,3 +44,7 @@ add_library(${PROJECT_NAME} SHARED $) set_target_properties(${PROJECT_NAME} PROPERTIES PUBLIC_HEADER ${PROJECT_SOURCE_DIR}/binding.h) target_link_libraries(${PROJECT_NAME} fasttext-static_pic) +# Ensure that libraries are linked statically wherever possible +if(CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") + target_link_libraries(${PROJECT_NAME} -Wl,-Bstatic -lstdc++ -lpthread -Wl,-Bdynamic) +endif() \ No newline at end of file From e1a18c6f2dd7c5d11c9840fcdba71ea5047476eb Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 02:19:24 +0400 Subject: [PATCH 098/219] Update CMakeLists.txt --- .../Native/CMakeLists.txt | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index 796035e5..a4004602 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -10,12 +10,10 @@ set(CMAKE_C_STANDARD_REQUIRED ON) if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3 -march=native") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3 -march=native") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-rtti -lpthread") elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3 -march=native") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3 -march=native") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++") elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /W3 /O2 /fp:precise /arch:AVX2") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W3 /O2 /fp:precise /arch:AVX2") @@ -36,15 +34,23 @@ include_directories( ${PROJECT_SOURCE_DIR}/fasttext/src ${PROJECT_SOURCE_DIR}/include) + set(SOURCES ${PROJECT_SOURCE_DIR}/binding.cc) add_library(objlib OBJECT ${SOURCES}) add_library(${PROJECT_NAME} SHARED $) set_target_properties(${PROJECT_NAME} PROPERTIES PUBLIC_HEADER ${PROJECT_SOURCE_DIR}/binding.h) -target_link_libraries(${PROJECT_NAME} fasttext-static_pic) -# Ensure that libraries are linked statically wherever possible -if(CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") - target_link_libraries(${PROJECT_NAME} -Wl,-Bstatic -lstdc++ -lpthread -Wl,-Bdynamic) +if (APPLE) + set(CLANG_LIB_DIR /opt/homebrew/opt/llvm/lib/c++) + + target_link_libraries(${PROJECT_NAME} fasttext-static_pic + -nostdlib++ + -Wl,${CLANG_LIB_DIR}/libc++.a + -Wl,${CLANG_LIB_DIR}/libc++abi.a + -Wl,-framework -Wl,Foundation + -Wl,-framework -Wl,Security) +else() + target_link_libraries(${PROJECT_NAME} fasttext-static_pic -static-libgcc -static-libstdc++) endif() \ No newline at end of file From b88b85777de0eebaabcad578fc7b19fecc68efad Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 02:49:59 +0400 Subject: [PATCH 099/219] FT: /usr/lib/libc++.a --- .../Native/CMakeLists.txt | 6 ++---- .../Scripts/run-build.macos.sh | 2 ++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index a4004602..abaf1a5d 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -43,12 +43,10 @@ add_library(${PROJECT_NAME} SHARED $) set_target_properties(${PROJECT_NAME} PROPERTIES PUBLIC_HEADER ${PROJECT_SOURCE_DIR}/binding.h) if (APPLE) - set(CLANG_LIB_DIR /opt/homebrew/opt/llvm/lib/c++) - target_link_libraries(${PROJECT_NAME} fasttext-static_pic -nostdlib++ - -Wl,${CLANG_LIB_DIR}/libc++.a - -Wl,${CLANG_LIB_DIR}/libc++abi.a + -Wl,/usr/lib/libc++.a + -Wl,/usr/lib/libc++abi.a -Wl,-framework -Wl,Foundation -Wl,-framework -Wl,Security) else() diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh index 03f8d798..8606039d 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh +++ b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh @@ -11,6 +11,8 @@ cp -a Native/. $workspace ls -R . +ls /usr/lib + cd "$workspace" mkdir build From 64e75ae4516d9ad654f7f9c2e2ca531fd64ad44c Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 03:04:52 +0400 Subject: [PATCH 100/219] Update CMakeLists.txt --- .../Native/CMakeLists.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index abaf1a5d..a4fa01a5 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -14,6 +14,7 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3 -march=native") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3 -march=native") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -nostdlib++ -Wl,/usr/lib/libc++.a -Wl,/usr/lib/libc++abi.a") elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /W3 /O2 /fp:precise /arch:AVX2") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W3 /O2 /fp:precise /arch:AVX2") @@ -46,9 +47,7 @@ if (APPLE) target_link_libraries(${PROJECT_NAME} fasttext-static_pic -nostdlib++ -Wl,/usr/lib/libc++.a - -Wl,/usr/lib/libc++abi.a - -Wl,-framework -Wl,Foundation - -Wl,-framework -Wl,Security) + -Wl,/usr/lib/libc++abi.a) else() target_link_libraries(${PROJECT_NAME} fasttext-static_pic -static-libgcc -static-libstdc++) endif() \ No newline at end of file From 81ee3b381880f87e615e0a69e779f1511f9922c6 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 03:07:28 +0400 Subject: [PATCH 101/219] Update CMakeLists.txt --- .../Native/CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index a4fa01a5..022f3a84 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -44,10 +44,12 @@ add_library(${PROJECT_NAME} SHARED $) set_target_properties(${PROJECT_NAME} PROPERTIES PUBLIC_HEADER ${PROJECT_SOURCE_DIR}/binding.h) if (APPLE) + set(CLANG_LIB_DIR /usr/lib) + target_link_libraries(${PROJECT_NAME} fasttext-static_pic -nostdlib++ - -Wl,/usr/lib/libc++.a - -Wl,/usr/lib/libc++abi.a) + -Wl,${CLANG_LIB_DIR}/libc++.a + -Wl,${CLANG_LIB_DIR}/libc++abi.a) else() target_link_libraries(${PROJECT_NAME} fasttext-static_pic -static-libgcc -static-libstdc++) endif() \ No newline at end of file From 8142a671a482174c60d18b6478542eb794a6e0c0 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 03:40:11 +0400 Subject: [PATCH 102/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index fbea9600..349df78a 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -42,6 +42,20 @@ jobs: src: - '${{env.ROOT}}/${{matrix.task.projectName}}.Native/**' + - name: Test + if: steps.changes.outputs.src == 'true' + working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} + run: | + echo "1" + ls /opt/homebrew/opt/llvm/lib -r | grep libc++.a + ls /opt/homebrew/opt/llvm/lib -r | grep libc++abi.a + echo "2" + ls /usr/bin -r | grep libc++.a + ls /usr/bin -r | grep libc++abi.a + echo "3" + ls /usr/lib -r | grep libc++.a + ls /usr/lib -r | grep libc++abi.a + - name: πŸš€ Build ${{matrix.task.projectName}} if: steps.changes.outputs.src == 'true' working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} From 35187de12d61fe1d474d7a48e5ababc31f3ff1a2 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 03:40:13 +0400 Subject: [PATCH 103/219] Update CMakeLists.txt --- src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index 022f3a84..3d5dc528 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -14,7 +14,6 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3 -march=native") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3 -march=native") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -nostdlib++ -Wl,/usr/lib/libc++.a -Wl,/usr/lib/libc++abi.a") elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /W3 /O2 /fp:precise /arch:AVX2") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W3 /O2 /fp:precise /arch:AVX2") From d6e5e0feead895e0e2943482f386404a269a0bcc Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 04:16:04 +0400 Subject: [PATCH 104/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 349df78a..4c123873 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -46,6 +46,9 @@ jobs: if: steps.changes.outputs.src == 'true' working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} run: | + brew install llvm + brew install libc++ --with-static + brew install libc++abi --with-static echo "1" ls /opt/homebrew/opt/llvm/lib -r | grep libc++.a ls /opt/homebrew/opt/llvm/lib -r | grep libc++abi.a From 1e09a1be9312d8cf28fabbbd5bfa10817b0101bb Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 04:18:02 +0400 Subject: [PATCH 105/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 4c123873..e2ec9886 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -47,8 +47,8 @@ jobs: working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} run: | brew install llvm - brew install libc++ --with-static - brew install libc++abi --with-static + brew install libc++ + brew install libc++abi echo "1" ls /opt/homebrew/opt/llvm/lib -r | grep libc++.a ls /opt/homebrew/opt/llvm/lib -r | grep libc++abi.a From 4274abb24a31ccaaee850dff4b646b8a71a8b59e Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 04:21:57 +0400 Subject: [PATCH 106/219] FT: brew install llvm --- .github/workflows/github-ci.yml | 2 -- .../Native/CMakeLists.txt | 2 +- .../Scripts/run-build.macos.sh | 2 -- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index e2ec9886..955702aa 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -47,8 +47,6 @@ jobs: working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} run: | brew install llvm - brew install libc++ - brew install libc++abi echo "1" ls /opt/homebrew/opt/llvm/lib -r | grep libc++.a ls /opt/homebrew/opt/llvm/lib -r | grep libc++abi.a diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index 3d5dc528..598cfb04 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -43,7 +43,7 @@ add_library(${PROJECT_NAME} SHARED $) set_target_properties(${PROJECT_NAME} PROPERTIES PUBLIC_HEADER ${PROJECT_SOURCE_DIR}/binding.h) if (APPLE) - set(CLANG_LIB_DIR /usr/lib) + set(CLANG_LIB_DIR /opt/homebrew/opt/llvm/lib) target_link_libraries(${PROJECT_NAME} fasttext-static_pic -nostdlib++ diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh index 8606039d..03f8d798 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh +++ b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh @@ -11,8 +11,6 @@ cp -a Native/. $workspace ls -R . -ls /usr/lib - cd "$workspace" mkdir build From 10f68ac708904a2e8bef56a8d7850239397d3617 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 04:24:13 +0400 Subject: [PATCH 107/219] FT: /opt/homebrew/opt/llvm/bin --- .github/workflows/github-ci.yml | 4 ++-- .../Native/CMakeLists.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 955702aa..7a0510f5 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -48,8 +48,8 @@ jobs: run: | brew install llvm echo "1" - ls /opt/homebrew/opt/llvm/lib -r | grep libc++.a - ls /opt/homebrew/opt/llvm/lib -r | grep libc++abi.a + ls /opt/homebrew/opt/llvm/bin -r | grep libc++.a + ls /opt/homebrew/opt/llvm/bin -r | grep libc++abi.a echo "2" ls /usr/bin -r | grep libc++.a ls /usr/bin -r | grep libc++abi.a diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index 598cfb04..21637081 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -43,7 +43,7 @@ add_library(${PROJECT_NAME} SHARED $) set_target_properties(${PROJECT_NAME} PROPERTIES PUBLIC_HEADER ${PROJECT_SOURCE_DIR}/binding.h) if (APPLE) - set(CLANG_LIB_DIR /opt/homebrew/opt/llvm/lib) + set(CLANG_LIB_DIR /opt/homebrew/opt/llvm/bin) target_link_libraries(${PROJECT_NAME} fasttext-static_pic -nostdlib++ From 6943ec32eda604ae58051c8a7ae863ba8ca1fa1d Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 04:25:01 +0400 Subject: [PATCH 108/219] FT: /opt/homebrew/opt/llvm/lib --- .github/workflows/github-ci.yml | 10 ++-------- .../Native/CMakeLists.txt | 2 +- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 7a0510f5..4371ee2f 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -48,14 +48,8 @@ jobs: run: | brew install llvm echo "1" - ls /opt/homebrew/opt/llvm/bin -r | grep libc++.a - ls /opt/homebrew/opt/llvm/bin -r | grep libc++abi.a - echo "2" - ls /usr/bin -r | grep libc++.a - ls /usr/bin -r | grep libc++abi.a - echo "3" - ls /usr/lib -r | grep libc++.a - ls /usr/lib -r | grep libc++abi.a + ls /opt/homebrew/opt/llvm/lib | grep libc++.a + ls /opt/homebrew/opt/llvm/lib | grep libc++abi.a - name: πŸš€ Build ${{matrix.task.projectName}} if: steps.changes.outputs.src == 'true' diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index 21637081..598cfb04 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -43,7 +43,7 @@ add_library(${PROJECT_NAME} SHARED $) set_target_properties(${PROJECT_NAME} PROPERTIES PUBLIC_HEADER ${PROJECT_SOURCE_DIR}/binding.h) if (APPLE) - set(CLANG_LIB_DIR /opt/homebrew/opt/llvm/bin) + set(CLANG_LIB_DIR /opt/homebrew/opt/llvm/lib) target_link_libraries(${PROJECT_NAME} fasttext-static_pic -nostdlib++ From 590812a8e90126315d86122ab18af473a5d8ca11 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 04:27:01 +0400 Subject: [PATCH 109/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 4371ee2f..c1486d05 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -47,9 +47,6 @@ jobs: working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} run: | brew install llvm - echo "1" - ls /opt/homebrew/opt/llvm/lib | grep libc++.a - ls /opt/homebrew/opt/llvm/lib | grep libc++abi.a - name: πŸš€ Build ${{matrix.task.projectName}} if: steps.changes.outputs.src == 'true' From acb10d058d679af777d12ae34e92cc65babc897f Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 04:29:15 +0400 Subject: [PATCH 110/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index c1486d05..ea499665 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -47,6 +47,7 @@ jobs: working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} run: | brew install llvm + ls /opt/homebrew/opt/llvm/lib - name: πŸš€ Build ${{matrix.task.projectName}} if: steps.changes.outputs.src == 'true' From 47f42fcbe74e43670434d5a6414f9ad1f00a2b3a Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 04:32:18 +0400 Subject: [PATCH 111/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index ea499665..a87cb18e 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -47,7 +47,8 @@ jobs: working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} run: | brew install llvm - ls /opt/homebrew/opt/llvm/lib + find /opt/homebrew/opt/llvm/lib -type f -name '*.a' + find /opt/homebrew/opt/llvm/lib -type f -name '*.dylib' - name: πŸš€ Build ${{matrix.task.projectName}} if: steps.changes.outputs.src == 'true' From 84aedac820629800ef6eb1594f3322095ad7fa86 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 04:35:32 +0400 Subject: [PATCH 112/219] Update CMakeLists.txt --- .../Native/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index 598cfb04..bc356621 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -43,7 +43,7 @@ add_library(${PROJECT_NAME} SHARED $) set_target_properties(${PROJECT_NAME} PROPERTIES PUBLIC_HEADER ${PROJECT_SOURCE_DIR}/binding.h) if (APPLE) - set(CLANG_LIB_DIR /opt/homebrew/opt/llvm/lib) + set(CLANG_LIB_DIR /opt/homebrew/opt/llvm/lib/c++) target_link_libraries(${PROJECT_NAME} fasttext-static_pic -nostdlib++ From 8632261bb4a778f022849e27d4016443005dd227 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 04:39:04 +0400 Subject: [PATCH 113/219] FT: brew install llvm --- .github/workflows/github-ci.yml | 8 -------- .../Native/CMakeLists.txt | 1 + .../Scripts/run-build.macos.sh | 4 ++++ 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index a87cb18e..fbea9600 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -42,14 +42,6 @@ jobs: src: - '${{env.ROOT}}/${{matrix.task.projectName}}.Native/**' - - name: Test - if: steps.changes.outputs.src == 'true' - working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} - run: | - brew install llvm - find /opt/homebrew/opt/llvm/lib -type f -name '*.a' - find /opt/homebrew/opt/llvm/lib -type f -name '*.dylib' - - name: πŸš€ Build ${{matrix.task.projectName}} if: steps.changes.outputs.src == 'true' working-directory: ${{env.ROOT}}/${{matrix.task.projectName}} diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index bc356621..d26d1a0d 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -47,6 +47,7 @@ if (APPLE) target_link_libraries(${PROJECT_NAME} fasttext-static_pic -nostdlib++ + -nodefaultlibs -Wl,${CLANG_LIB_DIR}/libc++.a -Wl,${CLANG_LIB_DIR}/libc++abi.a) else() diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh index 03f8d798..752a47b9 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh +++ b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh @@ -3,6 +3,10 @@ set -e echo "Hello world"; +brew install llvm +find /opt/homebrew/opt/llvm/lib -type f -name '*.a' +find /opt/homebrew/opt/llvm/lib -type f -name '*.dylib' + workspace="build_temp" mkdir "$workspace" -p From fbcef309cc84c6d62ebcc8f5f7240c9cefc9a440 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 04:41:18 +0400 Subject: [PATCH 114/219] Update CMakeLists.txt --- src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index d26d1a0d..bc356621 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -47,7 +47,6 @@ if (APPLE) target_link_libraries(${PROJECT_NAME} fasttext-static_pic -nostdlib++ - -nodefaultlibs -Wl,${CLANG_LIB_DIR}/libc++.a -Wl,${CLANG_LIB_DIR}/libc++abi.a) else() From 5226ff3cc676e4b24ef424b36e40746123e1ed4c Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 04:56:10 +0400 Subject: [PATCH 115/219] Update CMakeLists.txt --- .../Native/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index bc356621..239b8e62 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -19,7 +19,7 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W3 /O2 /fp:precise /arch:AVX2") endif() -set(CMAKE_MACOSX_RPATH 1) +set(CMAKE_MACOSX_RPATH OFF) set(CMAKE_POSITION_INDEPENDENT_CODE ON) message(STATUS "CMake version: ${CMAKE_VERSION}") From a5864750ee95e85e9cd3433b8e02cb6a83c0344d Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 05:05:16 +0400 Subject: [PATCH 116/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index fbea9600..03dfd441 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -282,7 +282,11 @@ jobs: - name: πŸ§ͺ Run Tests for LanguageIdentification working-directory: ${{env.ROOT}}/LanguageIdentification.Tests - run: dotnet test -c CI + run: + export DYLD_PRINT_LIBRARIES=1 + dotnet build -c CI + find . -type f -name '*.dylib' + dotnet test -c CI - name: πŸ§ͺ Run Tests for LanguageCode working-directory: ${{env.ROOT}}/LanguageCode.Tests From fff2194f12c629663b027e4d94db3e1f8fda9cd9 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 05:06:49 +0400 Subject: [PATCH 117/219] Revert "Update CMakeLists.txt" This reverts commit 5226ff3cc676e4b24ef424b36e40746123e1ed4c. --- .../Native/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index 239b8e62..bc356621 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -19,7 +19,7 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W3 /O2 /fp:precise /arch:AVX2") endif() -set(CMAKE_MACOSX_RPATH OFF) +set(CMAKE_MACOSX_RPATH 1) set(CMAKE_POSITION_INDEPENDENT_CODE ON) message(STATUS "CMake version: ${CMAKE_VERSION}") From fcdc3373bd62e928d44505b561cad953d35a55b0 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 05:10:19 +0400 Subject: [PATCH 118/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 03dfd441..1a710368 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -282,7 +282,7 @@ jobs: - name: πŸ§ͺ Run Tests for LanguageIdentification working-directory: ${{env.ROOT}}/LanguageIdentification.Tests - run: + run: | export DYLD_PRINT_LIBRARIES=1 dotnet build -c CI find . -type f -name '*.dylib' From ad43ccc437260a8e4c3a72825ac4217725aeb10e Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 05:20:19 +0400 Subject: [PATCH 119/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 1a710368..0a10724e 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -284,6 +284,7 @@ jobs: working-directory: ${{env.ROOT}}/LanguageIdentification.Tests run: | export DYLD_PRINT_LIBRARIES=1 + export DYLD_LIBRARY_PATH=./bin/CI/net8.0/runtimes/osx-x64/native dotnet build -c CI find . -type f -name '*.dylib' dotnet test -c CI From 527358e547ba411f853820963992735a3e0e54ce Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 20:17:58 +0400 Subject: [PATCH 120/219] MacOS: amd vs x86 --- .github/workflows/github-ci.yml | 11 ++++------- .../CLD2Detector.cs | 7 ++++++- .../CLD3Detector.cs | 7 ++++++- ...guageIdentification.FastText.Native.csproj | 12 +++++++++++- .../Scripts/run-build.macos.sh | 19 ++++++++++++++++--- .../FastTextDetector.cs | 9 ++++++++- 6 files changed, 51 insertions(+), 14 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 0a10724e..5f5426de 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -235,7 +235,9 @@ jobs: test-nugets-macos: name: πŸ§ͺ Test on macOS - runs-on: macos-latest + runs-on: | + - macos-latest + - macos-12 needs: build-nugets permissions: contents: read @@ -282,12 +284,7 @@ jobs: - name: πŸ§ͺ Run Tests for LanguageIdentification working-directory: ${{env.ROOT}}/LanguageIdentification.Tests - run: | - export DYLD_PRINT_LIBRARIES=1 - export DYLD_LIBRARY_PATH=./bin/CI/net8.0/runtimes/osx-x64/native - dotnet build -c CI - find . -type f -name '*.dylib' - dotnet test -c CI + run: dotnet test -c CI - name: πŸ§ͺ Run Tests for LanguageCode working-directory: ${{env.ROOT}}/LanguageCode.Tests diff --git a/src/LanguageIdentification.CLD2/CLD2Detector.cs b/src/LanguageIdentification.CLD2/CLD2Detector.cs index b1866b14..1b3b5011 100644 --- a/src/LanguageIdentification.CLD2/CLD2Detector.cs +++ b/src/LanguageIdentification.CLD2/CLD2Detector.cs @@ -23,7 +23,12 @@ public CLD2Detector() public static bool IsSupported() { - return RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + return RuntimeInformation.OSArchitecture switch + { + Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => true, + Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => true, + _ => false, + }; } /// diff --git a/src/LanguageIdentification.CLD3/CLD3Detector.cs b/src/LanguageIdentification.CLD3/CLD3Detector.cs index d860f8ad..b8984117 100644 --- a/src/LanguageIdentification.CLD3/CLD3Detector.cs +++ b/src/LanguageIdentification.CLD3/CLD3Detector.cs @@ -31,7 +31,12 @@ public CLD3Detector(int minNumBytes, int maxNumBytes) public static bool IsSupported() { - return RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.Windows); + return RuntimeInformation.OSArchitecture switch + { + Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => true, + Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => true, + _ => false, + }; } public void Dispose() diff --git a/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj b/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj index b2f1927c..7ed69fca 100644 --- a/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj +++ b/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj @@ -81,7 +81,17 @@ false - + + + + PreserveNewest + true + runtimes/osx-arm64/native + true + false + + + diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh index 752a47b9..8b9d1011 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh +++ b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh @@ -20,15 +20,28 @@ cd "$workspace" mkdir build cd build -# Build for MacOS +echo "Build for MacOS on x86"; rm -rf * -cmake .. +cmake -DCMAKE_OSX_ARCHITECTURES=x86_64 .. make -j $(nproc) # make ls -R otool -L libfasttext.dylib -cp libfasttext.dylib ../../libfasttext.dylib +cp libfasttext.dylib ../../libfasttext.x86_64.dylib + +echo "Build for MacOS on M1"; +rm -rf * +cmake -DCMAKE_OSX_ARCHITECTURES=arm64 .. +make -j $(nproc) # make + +ls -R + +otool -L libfasttext.dylib +cp libfasttext.dylib ../../libfasttext.arm64.dylib + +echo "Make universal binary"; +lipo -create ../../libfasttext.x86_64.dylib ../../libfasttext.arm64.dylib -output ../../libfasttext.dylib # Clean up rm -rf "$workspace" diff --git a/src/LanguageIdentification.FastText/FastTextDetector.cs b/src/LanguageIdentification.FastText/FastTextDetector.cs index 75cf6008..5c7f4181 100644 --- a/src/LanguageIdentification.FastText/FastTextDetector.cs +++ b/src/LanguageIdentification.FastText/FastTextDetector.cs @@ -31,7 +31,14 @@ public FastTextDetector() public static bool IsSupported() { - return RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.Windows) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX); + return RuntimeInformation.OSArchitecture switch + { + Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => true, + Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => true, + Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + _ => false, + }; } public string ModelPath { get; private set; } = string.Empty; From 3788d0f179a59f99426035b1ea8cf2656a159d25 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 20:27:17 +0400 Subject: [PATCH 121/219] Update CMakeLists.txt --- .../Native/CMakeLists.txt | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index bc356621..431f87cb 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -8,12 +8,20 @@ set(CMAKE_C_STANDARD 99) set(CMAKE_C_STANDARD_REQUIRED ON) if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3 -march=native") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3 -march=native") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-rtti -lpthread") elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3 -march=native") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3 -march=native") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3") + + if (CMAKE_OSX_ARCHITECTURES MATCHES "arm64") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mcpu=apple-m1") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=apple-m1") + elseif (CMAKE_OSX_ARCHITECTURES MATCHES "x86_64") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=x86-64") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=x86-64") + endif() elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /W3 /O2 /fp:precise /arch:AVX2") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W3 /O2 /fp:precise /arch:AVX2") From 931acada5862db8ffdc8676c4598a217c3e2c0db Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 20:34:40 +0400 Subject: [PATCH 122/219] Update CMakeLists.txt --- .../Native/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index 431f87cb..39b733fc 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -16,11 +16,11 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3") if (CMAKE_OSX_ARCHITECTURES MATCHES "arm64") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mcpu=apple-m1") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=apple-m1") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch arm64") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch arm64") elseif (CMAKE_OSX_ARCHITECTURES MATCHES "x86_64") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=x86-64") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=x86-64") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch x86_64") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch x86_64") endif() elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /W3 /O2 /fp:precise /arch:AVX2") From 19029210b0b87465ffa3178d2490673f26484475 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 20:39:01 +0400 Subject: [PATCH 123/219] Update run-build.macos.sh --- .../Scripts/run-build.macos.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh index 8b9d1011..38e24094 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh +++ b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh @@ -20,25 +20,25 @@ cd "$workspace" mkdir build cd build -echo "Build for MacOS on x86"; +echo "Build for MacOS on M1"; rm -rf * -cmake -DCMAKE_OSX_ARCHITECTURES=x86_64 .. +cmake -DCMAKE_OSX_ARCHITECTURES=arm64 .. make -j $(nproc) # make ls -R otool -L libfasttext.dylib -cp libfasttext.dylib ../../libfasttext.x86_64.dylib +cp libfasttext.dylib ../../libfasttext.arm64.dylib -echo "Build for MacOS on M1"; +echo "Build for MacOS on x86"; rm -rf * -cmake -DCMAKE_OSX_ARCHITECTURES=arm64 .. +cmake -DCMAKE_OSX_ARCHITECTURES=x86_64 .. make -j $(nproc) # make ls -R otool -L libfasttext.dylib -cp libfasttext.dylib ../../libfasttext.arm64.dylib +cp libfasttext.dylib ../../libfasttext.x86_64.dylib echo "Make universal binary"; lipo -create ../../libfasttext.x86_64.dylib ../../libfasttext.arm64.dylib -output ../../libfasttext.dylib From 585cf5f3f9e35254aef52fc7da4c08ef6bf4a03c Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 20:56:54 +0400 Subject: [PATCH 124/219] MacOS: more flags --- .../Native/CMakeLists.txt | 4 ++-- .../Scripts/run-build.macos.sh | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index 39b733fc..16d9bac1 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -19,8 +19,8 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch arm64") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch arm64") elseif (CMAKE_OSX_ARCHITECTURES MATCHES "x86_64") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch x86_64") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch x86_64") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch x86_64 -march=native") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch x86_64 -march=native") endif() elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /W3 /O2 /fp:precise /arch:AVX2") diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh index 38e24094..733845d2 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh +++ b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh @@ -22,8 +22,8 @@ cd build echo "Build for MacOS on M1"; rm -rf * -cmake -DCMAKE_OSX_ARCHITECTURES=arm64 .. -make -j $(nproc) # make +cmake -D CMAKE_BUILD_TYPE=Release -DCMAKE_OSX_ARCHITECTURES=arm64 .. +make -j $(sysctl -n hw.logicalcpu) ls -R @@ -32,8 +32,8 @@ cp libfasttext.dylib ../../libfasttext.arm64.dylib echo "Build for MacOS on x86"; rm -rf * -cmake -DCMAKE_OSX_ARCHITECTURES=x86_64 .. -make -j $(nproc) # make +cmake -D CMAKE_BUILD_TYPE=Release -DCMAKE_OSX_ARCHITECTURES=x86_64 .. +make -j $(sysctl -n hw.logicalcpu) ls -R From 0615edfbbcebe453c734c361214d73bc441ca749 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 21:18:49 +0400 Subject: [PATCH 125/219] Update CMakeLists.txt --- .../Native/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index 16d9bac1..39b733fc 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -19,8 +19,8 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch arm64") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch arm64") elseif (CMAKE_OSX_ARCHITECTURES MATCHES "x86_64") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch x86_64 -march=native") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch x86_64 -march=native") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch x86_64") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch x86_64") endif() elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /W3 /O2 /fp:precise /arch:AVX2") From 658c46b31b821446465235fc8b6fdb6334d84760 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 21:24:04 +0400 Subject: [PATCH 126/219] Update CMakeLists.txt --- .../Native/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index 39b733fc..c4278677 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -19,8 +19,8 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch arm64") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch arm64") elseif (CMAKE_OSX_ARCHITECTURES MATCHES "x86_64") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch x86_64") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch x86_64") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch x86_64 -target x86_64-apple-macos10.15") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch x86_64 -target x86_64-apple-macos10.15") endif() elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /W3 /O2 /fp:precise /arch:AVX2") From 035540bf913489d572414b6233fafcbcbaa5d911 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 21:30:49 +0400 Subject: [PATCH 127/219] Update CMakeLists.txt --- .../Native/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index c4278677..7a5d4918 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -19,8 +19,8 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch arm64") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch arm64") elseif (CMAKE_OSX_ARCHITECTURES MATCHES "x86_64") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch x86_64 -target x86_64-apple-macos10.15") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch x86_64 -target x86_64-apple-macos10.15") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -target x86_64-apple-macos10.15") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -target x86_64-apple-macos10.15") endif() elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /W3 /O2 /fp:precise /arch:AVX2") From d8ef35bfb9efb7ec466d2a86cfb22434c15e9bcf Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 23:33:21 +0400 Subject: [PATCH 128/219] Update run-build.macos.sh --- .../Scripts/run-build.macos.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh index 733845d2..35d409df 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh +++ b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh @@ -32,8 +32,8 @@ cp libfasttext.dylib ../../libfasttext.arm64.dylib echo "Build for MacOS on x86"; rm -rf * -cmake -D CMAKE_BUILD_TYPE=Release -DCMAKE_OSX_ARCHITECTURES=x86_64 .. -make -j $(sysctl -n hw.logicalcpu) +arch -x86_64 /bin/bash -c "cmake -DCMAKE_OSX_ARCHITECTURES=x86_64 .." +arch -x86_64 /bin/bash -c "make -j $(sysctl -n hw.logicalcpu)" ls -R From f246ba1e93132560ed29a15e586c0d557205e6df Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Wed, 2 Oct 2024 23:40:15 +0400 Subject: [PATCH 129/219] Update CMakeLists.txt --- .../Native/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index 7a5d4918..1a2c713c 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -16,11 +16,11 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3") if (CMAKE_OSX_ARCHITECTURES MATCHES "arm64") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch arm64") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch arm64") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch arm64 -target arm64-apple-macos11") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch arm64 -target arm64-apple-macos11") elseif (CMAKE_OSX_ARCHITECTURES MATCHES "x86_64") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -target x86_64-apple-macos10.15") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -target x86_64-apple-macos10.15") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch x86_64 -target x86_64-apple-macos10.15") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch x86_64 -target x86_64-apple-macos10.15") endif() elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /W3 /O2 /fp:precise /arch:AVX2") From 3c57402feaf4e9106147726727f9afb051be3a1e Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 3 Oct 2024 00:00:19 +0400 Subject: [PATCH 130/219] Update run-build.macos.sh --- .../Scripts/run-build.macos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh index 35d409df..35eb5778 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh +++ b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh @@ -32,7 +32,7 @@ cp libfasttext.dylib ../../libfasttext.arm64.dylib echo "Build for MacOS on x86"; rm -rf * -arch -x86_64 /bin/bash -c "cmake -DCMAKE_OSX_ARCHITECTURES=x86_64 .." +arch -x86_64 /bin/bash -c "cmake -DCMAKE_OSX_ARCHITECTURES=x86_64 -DCMAKE_OSX_DEPLOYMENT_TARGET=10.15 .." arch -x86_64 /bin/bash -c "make -j $(sysctl -n hw.logicalcpu)" ls -R From 665b724164f3959fcd093eb093e07de3d5b1aa6b Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 3 Oct 2024 00:17:44 +0400 Subject: [PATCH 131/219] Update run-build.macos.sh --- .../Scripts/run-build.macos.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh index 35eb5778..a47dde9d 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh +++ b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh @@ -20,25 +20,25 @@ cd "$workspace" mkdir build cd build -echo "Build for MacOS on M1"; +echo "Build for MacOS on x86"; rm -rf * -cmake -D CMAKE_BUILD_TYPE=Release -DCMAKE_OSX_ARCHITECTURES=arm64 .. -make -j $(sysctl -n hw.logicalcpu) +arch -x86_64 /bin/bash -c "cmake -DCMAKE_OSX_ARCHITECTURES=x86_64 -DCMAKE_OSX_DEPLOYMENT_TARGET=10.15 .." +arch -x86_64 /bin/bash -c "make -j $(sysctl -n hw.logicalcpu)" ls -R otool -L libfasttext.dylib -cp libfasttext.dylib ../../libfasttext.arm64.dylib +cp libfasttext.dylib ../../libfasttext.x86_64.dylib -echo "Build for MacOS on x86"; +echo "Build for MacOS on M1"; rm -rf * -arch -x86_64 /bin/bash -c "cmake -DCMAKE_OSX_ARCHITECTURES=x86_64 -DCMAKE_OSX_DEPLOYMENT_TARGET=10.15 .." -arch -x86_64 /bin/bash -c "make -j $(sysctl -n hw.logicalcpu)" +cmake -D CMAKE_BUILD_TYPE=Release -DCMAKE_OSX_ARCHITECTURES=arm64 .. +make -j $(sysctl -n hw.logicalcpu) ls -R otool -L libfasttext.dylib -cp libfasttext.dylib ../../libfasttext.x86_64.dylib +cp libfasttext.dylib ../../libfasttext.arm64.dylib echo "Make universal binary"; lipo -create ../../libfasttext.x86_64.dylib ../../libfasttext.arm64.dylib -output ../../libfasttext.dylib From adca9525ae4b128361b60597fd0b2892453d7a56 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 3 Oct 2024 00:44:22 +0400 Subject: [PATCH 132/219] Update run-build.macos.sh --- .../Scripts/run-build.macos.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh index a47dde9d..911ca265 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh +++ b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh @@ -1,4 +1,4 @@ -#!/bin/bash +ο»Ώ#!/bin/bash set -e echo "Hello world"; @@ -7,6 +7,10 @@ brew install llvm find /opt/homebrew/opt/llvm/lib -type f -name '*.a' find /opt/homebrew/opt/llvm/lib -type f -name '*.dylib' +arch -x86_64 /usr/local/bin/brew install llvm +find /usr/local/opt/llvm/lib -type f -name '*.a' +find /usr/local/opt/llvm/lib -type f -name '*.dylib' + workspace="build_temp" mkdir "$workspace" -p @@ -32,7 +36,7 @@ cp libfasttext.dylib ../../libfasttext.x86_64.dylib echo "Build for MacOS on M1"; rm -rf * -cmake -D CMAKE_BUILD_TYPE=Release -DCMAKE_OSX_ARCHITECTURES=arm64 .. +cmake -DCMAKE_OSX_ARCHITECTURES=arm64 .. make -j $(sysctl -n hw.logicalcpu) ls -R From 751883c2749bc9464cdb2a01e04f0c0bcd09f57e Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 3 Oct 2024 00:46:53 +0400 Subject: [PATCH 133/219] Update run-build.macos.sh --- .../Scripts/run-build.macos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh index 911ca265..d5d48f36 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh +++ b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh @@ -7,7 +7,7 @@ brew install llvm find /opt/homebrew/opt/llvm/lib -type f -name '*.a' find /opt/homebrew/opt/llvm/lib -type f -name '*.dylib' -arch -x86_64 /usr/local/bin/brew install llvm +arch -x86_64 /bin/bash -c "brew install llvm" find /usr/local/opt/llvm/lib -type f -name '*.a' find /usr/local/opt/llvm/lib -type f -name '*.dylib' From 00a683281e710b529f5fe089e76fb7ba942bd78b Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 3 Oct 2024 01:14:24 +0400 Subject: [PATCH 134/219] MacOS: build split for each arch --- .github/workflows/github-ci.yml | 3 +- .../Scripts/run-build.macos.sh | 33 +++++++------------ 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 5f5426de..9e5d2e4e 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -16,7 +16,8 @@ jobs: matrix: task: - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: windows-latest, script: ./Scripts/run-build.ps1, artifact: fasttext.dll } - - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: macos-latest, script: ./Scripts/run-build.macos.sh, artifact: libfasttext.dylib } + - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: macos-14, script: ./Scripts/run-build.macos.sh arm64, artifact: libfasttext.arm64.dylib } + - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: macos-12, script: ./Scripts/run-build.macos.sh x86_64, artifact: libfasttext.x86_64.dylib } permissions: contents: read packages: write diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh index d5d48f36..58fba32b 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh +++ b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh @@ -1,16 +1,20 @@ ο»Ώ#!/bin/bash set -e -echo "Hello world"; +if [ -z "$1" ]; then + echo "Error: No architecture specified." + echo "Usage: $0 " + exit 1 +fi + +ARCH=$1 + +echo "Hello world $ARCH"; brew install llvm find /opt/homebrew/opt/llvm/lib -type f -name '*.a' find /opt/homebrew/opt/llvm/lib -type f -name '*.dylib' -arch -x86_64 /bin/bash -c "brew install llvm" -find /usr/local/opt/llvm/lib -type f -name '*.a' -find /usr/local/opt/llvm/lib -type f -name '*.dylib' - workspace="build_temp" mkdir "$workspace" -p @@ -24,28 +28,15 @@ cd "$workspace" mkdir build cd build -echo "Build for MacOS on x86"; -rm -rf * -arch -x86_64 /bin/bash -c "cmake -DCMAKE_OSX_ARCHITECTURES=x86_64 -DCMAKE_OSX_DEPLOYMENT_TARGET=10.15 .." -arch -x86_64 /bin/bash -c "make -j $(sysctl -n hw.logicalcpu)" - -ls -R - -otool -L libfasttext.dylib -cp libfasttext.dylib ../../libfasttext.x86_64.dylib - -echo "Build for MacOS on M1"; +echo "Build for MacOS on $ARCH"; rm -rf * -cmake -DCMAKE_OSX_ARCHITECTURES=arm64 .. +cmake -DCMAKE_OSX_ARCHITECTURES=$ARCH .. make -j $(sysctl -n hw.logicalcpu) ls -R otool -L libfasttext.dylib -cp libfasttext.dylib ../../libfasttext.arm64.dylib - -echo "Make universal binary"; -lipo -create ../../libfasttext.x86_64.dylib ../../libfasttext.arm64.dylib -output ../../libfasttext.dylib +cp libfasttext.dylib ../../libfasttext.$ARCH.dylib # Clean up rm -rf "$workspace" From 8e999a117ac3d7a6d976a2c848fb3ea1b38b5e23 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 3 Oct 2024 01:29:20 +0400 Subject: [PATCH 135/219] MacOS: arch as argument --- .github/workflows/github-ci.yml | 8 ++++---- .../Scripts/run-build.macos.sh | 5 +++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 9e5d2e4e..16ff1107 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -15,9 +15,9 @@ jobs: fail-fast: false matrix: task: - - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: windows-latest, script: ./Scripts/run-build.ps1, artifact: fasttext.dll } - - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: macos-14, script: ./Scripts/run-build.macos.sh arm64, artifact: libfasttext.arm64.dylib } - - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: macos-12, script: ./Scripts/run-build.macos.sh x86_64, artifact: libfasttext.x86_64.dylib } + - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: windows-latest, arch: x86_64, script: ./Scripts/run-build.ps1, artifact: fasttext.dll } + - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: libfasttext.arm64.dylib } + - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: macos-12, arch: x86_64, script: ./Scripts/run-build.macos.sh, artifact: libfasttext.x86_64.dylib } permissions: contents: read packages: write @@ -50,7 +50,7 @@ jobs: echo $(pwd) git update-index --chmod=+x ${{matrix.task.script}} chmod +x ${{matrix.task.script}} - ${{matrix.task.script}} + ${{matrix.task.script}} ${{matrix.task.arch}} - uses: actions/upload-artifact@v4 if: steps.changes.outputs.src == 'true' diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh index 58fba32b..f6d9c7ed 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh +++ b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh @@ -9,6 +9,11 @@ fi ARCH=$1 +if [[ "$ARCH" != "x86_64" && "$ARCH" != "arm64" ]]; then + echo "Error: Invalid architecture specified. Use 'x86_64' or 'arm64'." + exit 1 +fi + echo "Hello world $ARCH"; brew install llvm From b51853ebd9b55dc7d36aab7b92f3cb9bbb00eba0 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 3 Oct 2024 01:49:28 +0400 Subject: [PATCH 136/219] Update LanguageIdentification.FastText.Native.csproj --- .../LanguageIdentification.FastText.Native.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj b/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj index 7ed69fca..fc1aa054 100644 --- a/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj +++ b/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj @@ -73,7 +73,7 @@ - + PreserveNewest true runtimes/osx-x64/native @@ -83,7 +83,7 @@ - + PreserveNewest true runtimes/osx-arm64/native From fac901a9ef0007b127ccdca09ce4cc6e2fd417fa Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 3 Oct 2024 01:54:32 +0400 Subject: [PATCH 137/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 16ff1107..1717d7d3 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -17,7 +17,7 @@ jobs: task: - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: windows-latest, arch: x86_64, script: ./Scripts/run-build.ps1, artifact: fasttext.dll } - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: libfasttext.arm64.dylib } - - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: macos-12, arch: x86_64, script: ./Scripts/run-build.macos.sh, artifact: libfasttext.x86_64.dylib } + - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: macos-13, arch: x86_64, script: ./Scripts/run-build.macos.sh, artifact: libfasttext.x86_64.dylib } permissions: contents: read packages: write From deaead1308811b0c69542cf8605e419ecb6a5dca Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 3 Oct 2024 02:03:30 +0400 Subject: [PATCH 138/219] Update CMakeLists.txt --- .../Native/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index 1a2c713c..abae04ba 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -51,7 +51,11 @@ add_library(${PROJECT_NAME} SHARED $) set_target_properties(${PROJECT_NAME} PROPERTIES PUBLIC_HEADER ${PROJECT_SOURCE_DIR}/binding.h) if (APPLE) - set(CLANG_LIB_DIR /opt/homebrew/opt/llvm/lib/c++) + if (CMAKE_OSX_ARCHITECTURES MATCHES "arm64") + set(CLANG_LIB_DIR /opt/homebrew/opt/llvm/lib/c++) + elseif (CMAKE_OSX_ARCHITECTURES MATCHES "x86_64") + set(CLANG_LIB_DIR /usr/local/opt/llvm/lib/c++) + endif() target_link_libraries(${PROJECT_NAME} fasttext-static_pic -nostdlib++ From 3bc8f3e9af3e470332aab5d9d3ae1fbe80e409ca Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 3 Oct 2024 02:05:51 +0400 Subject: [PATCH 139/219] Update run-build.macos.sh --- .../Scripts/run-build.macos.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh index f6d9c7ed..dcf0d7b0 100644 --- a/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh +++ b/src/LanguageIdentification.FastText.Native/Scripts/run-build.macos.sh @@ -17,8 +17,6 @@ fi echo "Hello world $ARCH"; brew install llvm -find /opt/homebrew/opt/llvm/lib -type f -name '*.a' -find /opt/homebrew/opt/llvm/lib -type f -name '*.dylib' workspace="build_temp" From 47d100914f9bc62e2169c91488c2a11292a510f2 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 3 Oct 2024 02:17:00 +0400 Subject: [PATCH 140/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 1717d7d3..012e9f1e 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -9,7 +9,7 @@ on: jobs: build-native: - name: πŸ’Ύ Build ${{matrix.task.title}} for ${{matrix.task.os}} + name: πŸ’Ύ Build ${{matrix.task.title}} for ${{matrix.task.os}} (${{matrix.task.arch}}) runs-on: ${{matrix.task.os}} strategy: fail-fast: false @@ -235,10 +235,14 @@ jobs: run: dotnet test -c CI test-nugets-macos: - name: πŸ§ͺ Test on macOS - runs-on: | - - macos-latest - - macos-12 + name: πŸ§ͺ Test on macOS (${{matrix.task.os}}) + runs-on: ${{matrix.task.os}} + strategy: + fail-fast: false + matrix: + task: + - macos-14 + - macos-13 needs: build-nugets permissions: contents: read From 9342f4040b1fc905e1d38c60a96a8daa434197bd Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 3 Oct 2024 02:25:00 +0400 Subject: [PATCH 141/219] MasOS: pipeline fixes + IsSupported() unification --- .github/workflows/github-ci.yml | 6 +++--- src/LanguageIdentification.Lingua/LinguaDetector.cs | 6 +++++- src/LanguageIdentification.MediaPipe/MediaPipeDetector.cs | 6 +++++- src/LanguageIdentification.Whatlang/WhatlangDetector.cs | 6 +++++- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 012e9f1e..f44b2799 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -235,12 +235,12 @@ jobs: run: dotnet test -c CI test-nugets-macos: - name: πŸ§ͺ Test on macOS (${{matrix.task.os}}) - runs-on: ${{matrix.task.os}} + name: πŸ§ͺ Test on macOS (${{matrix.os}}) + runs-on: ${{matrix.os}} strategy: fail-fast: false matrix: - task: + os: - macos-14 - macos-13 needs: build-nugets diff --git a/src/LanguageIdentification.Lingua/LinguaDetector.cs b/src/LanguageIdentification.Lingua/LinguaDetector.cs index d5c459c6..3a1808e4 100644 --- a/src/LanguageIdentification.Lingua/LinguaDetector.cs +++ b/src/LanguageIdentification.Lingua/LinguaDetector.cs @@ -32,7 +32,11 @@ internal LinguaDetector(LinguaDetectorBuilder builder) public static bool IsSupported() { - return RuntimeInformation.IsOSPlatform(OSPlatform.Linux); + return RuntimeInformation.OSArchitecture switch + { + Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => true, + _ => false, + }; } /// diff --git a/src/LanguageIdentification.MediaPipe/MediaPipeDetector.cs b/src/LanguageIdentification.MediaPipe/MediaPipeDetector.cs index 6618a48c..7a1a8610 100644 --- a/src/LanguageIdentification.MediaPipe/MediaPipeDetector.cs +++ b/src/LanguageIdentification.MediaPipe/MediaPipeDetector.cs @@ -93,7 +93,11 @@ public MediaPipeDetector(MediaPipeOptions options) public static bool IsSupported() { - return RuntimeInformation.IsOSPlatform(OSPlatform.Linux); + return RuntimeInformation.OSArchitecture switch + { + Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => true, + _ => false, + }; } public IEnumerable PredictLanguages(string text) diff --git a/src/LanguageIdentification.Whatlang/WhatlangDetector.cs b/src/LanguageIdentification.Whatlang/WhatlangDetector.cs index 42229fcb..00b9c8b0 100644 --- a/src/LanguageIdentification.Whatlang/WhatlangDetector.cs +++ b/src/LanguageIdentification.Whatlang/WhatlangDetector.cs @@ -22,7 +22,11 @@ public WhatlangDetector() public static bool IsSupported() { - return RuntimeInformation.IsOSPlatform(OSPlatform.Linux); + return RuntimeInformation.OSArchitecture switch + { + Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => true, + _ => false, + }; } /// From 494bb9e100eed5daa00cb7b8d04941d42b5ad591 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 3 Oct 2024 02:48:19 +0400 Subject: [PATCH 142/219] MacOS: cross-arch naming of libfasttext.dylib --- src/LanguageIdentification.FastText.Native/.gitignore | 2 ++ .../LanguageIdentification.FastText.Native.csproj | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/.gitignore b/src/LanguageIdentification.FastText.Native/.gitignore index d7b42a55..d15bc74b 100644 --- a/src/LanguageIdentification.FastText.Native/.gitignore +++ b/src/LanguageIdentification.FastText.Native/.gitignore @@ -1,3 +1,5 @@ libfasttext.so fasttext.dll +libfasttext.arm64.dylib +libfasttext.x86_64.dylib build_temp/** diff --git a/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj b/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj index fc1aa054..64511c5c 100644 --- a/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj +++ b/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj @@ -76,7 +76,7 @@ PreserveNewest true - runtimes/osx-x64/native + runtimes/osx-x64/native/libfasttext.dylib true false @@ -86,7 +86,7 @@ PreserveNewest true - runtimes/osx-arm64/native + runtimes/osx-arm64/native/libfasttext.dylib true false From 0bdb64b8371c1a6d3a8465485f475b1d41a9ffaa Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 3 Oct 2024 02:57:29 +0400 Subject: [PATCH 143/219] MasOS: specific model path --- .../FastTextTests.cs | 19 +++++++++++++++---- .../MediaPipeTests.cs | 17 ++++++++++++++--- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/LanguageIdentification.Tests/FastTextTests.cs b/src/LanguageIdentification.Tests/FastTextTests.cs index 43e5ff38..ca1a582f 100644 --- a/src/LanguageIdentification.Tests/FastTextTests.cs +++ b/src/LanguageIdentification.Tests/FastTextTests.cs @@ -1,10 +1,21 @@ -ο»Ώusing Panlingo.LanguageIdentification.FastText; +ο»Ώusing System.Runtime.InteropServices; +using Panlingo.LanguageIdentification.FastText; using Panlingo.LanguageIdentification.Tests.Helpers; namespace Panlingo.LanguageIdentification.Tests; public class FastTextTests { + private static string GetModelPath() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + return "~/models/fasttext176.bin"; + } + + return "/models/fasttext176.bin"; + } + [SkippableTheory] [InlineData("__label__en", Constants.PHRASE_ENG_1, 0.9955)] [InlineData("__label__uk", Constants.PHRASE_UKR_1, 0.9900)] @@ -15,7 +26,7 @@ public void FastTextFileSingleLanguage(string languageCode, string text, double using var fastText = new FastTextDetector(); - var modelPath = "/models/fasttext176.bin"; + var modelPath = GetModelPath(); fastText.LoadModel(modelPath); var predictions = fastText.Predict(text: text, count: 10); @@ -40,7 +51,7 @@ public void FastTextStreamSingleLanguage(string languageCode, string text, doubl using var fastText = new FastTextDetector(); - var modelPath = "/models/fasttext176.bin"; + var modelPath = GetModelPath(); using var stream = File.Open(modelPath, FileMode.Open); fastText.LoadModel(stream); @@ -87,7 +98,7 @@ public void FastTextLabels() using var fastText = new FastTextDetector(); - var modelPath = "/models/fasttext176.bin"; + var modelPath = GetModelPath(); fastText.LoadModel(modelPath); var labels = fastText.GetLabels(); diff --git a/src/LanguageIdentification.Tests/MediaPipeTests.cs b/src/LanguageIdentification.Tests/MediaPipeTests.cs index 2ff614a5..a35604cf 100644 --- a/src/LanguageIdentification.Tests/MediaPipeTests.cs +++ b/src/LanguageIdentification.Tests/MediaPipeTests.cs @@ -1,4 +1,5 @@ ο»Ώusing System.IO; +using System.Runtime.InteropServices; using Panlingo.LanguageIdentification.MediaPipe; using Panlingo.LanguageIdentification.Tests.Helpers; @@ -6,6 +7,16 @@ namespace Panlingo.LanguageIdentification.Tests; public class MediaPipeTests { + private static string GetModelPath() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + return "~/models/mediapipe_language_detector.tflite"; + } + + return "/models/mediapipe_language_detector.tflite"; + } + [SkippableTheory] [InlineData("en", Constants.PHRASE_ENG_1, 0.9994)] [InlineData("uk", Constants.PHRASE_UKR_1, 0.9999)] @@ -14,7 +25,7 @@ public void MediaPipeFileSingleLanguage(string languageCode, string text, double { Skip.IfNot(MediaPipeDetector.IsSupported()); - var modelPath = "/models/mediapipe_language_detector.tflite"; + var modelPath = GetModelPath(); using var mediaPipe = new MediaPipeDetector( options: MediaPipeOptions.FromFile(modelPath).WithResultCount(10) ); @@ -39,7 +50,7 @@ public void MediaPipeStreamSingleLanguage(string languageCode, string text, doub { Skip.IfNot(MediaPipeDetector.IsSupported()); - var modelPath = "/models/mediapipe_language_detector.tflite"; + var modelPath = GetModelPath(); using var stream = File.Open(modelPath, FileMode.Open); using var mediaPipe = new MediaPipeDetector( @@ -66,7 +77,7 @@ public void MediaPipeContainedSingleLanguage(string languageCode, string text, d { Skip.IfNot(MediaPipeDetector.IsSupported()); - var modelPath = "/models/mediapipe_language_detector.tflite"; + var modelPath = GetModelPath(); using var stream = File.Open(modelPath, FileMode.Open); using var mediaPipe = new MediaPipeDetector( From 7585c56ab25b1b4baaf4be6e2edfb7ea0c2e9f11 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 3 Oct 2024 03:06:51 +0400 Subject: [PATCH 144/219] MacOS: another try to fix paths to models --- src/LanguageIdentification.Tests/FastTextTests.cs | 6 ++++-- src/LanguageIdentification.Tests/MediaPipeTests.cs | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/LanguageIdentification.Tests/FastTextTests.cs b/src/LanguageIdentification.Tests/FastTextTests.cs index ca1a582f..6d9cbedd 100644 --- a/src/LanguageIdentification.Tests/FastTextTests.cs +++ b/src/LanguageIdentification.Tests/FastTextTests.cs @@ -8,12 +8,14 @@ public class FastTextTests { private static string GetModelPath() { + var root = "/"; + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { - return "~/models/fasttext176.bin"; + root = Environment.GetEnvironmentVariable("HOME") ?? "/"; } - return "/models/fasttext176.bin"; + return Path.Combine(root, "/models/fasttext176.bin"); } [SkippableTheory] diff --git a/src/LanguageIdentification.Tests/MediaPipeTests.cs b/src/LanguageIdentification.Tests/MediaPipeTests.cs index a35604cf..cb5a151c 100644 --- a/src/LanguageIdentification.Tests/MediaPipeTests.cs +++ b/src/LanguageIdentification.Tests/MediaPipeTests.cs @@ -9,12 +9,14 @@ public class MediaPipeTests { private static string GetModelPath() { + var root = "/"; + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { - return "~/models/mediapipe_language_detector.tflite"; + root = Environment.GetEnvironmentVariable("HOME") ?? "/"; } - return "/models/mediapipe_language_detector.tflite"; + return Path.Combine(root, "/models/mediapipe_language_detector.tflite"); } [SkippableTheory] From 31501552d3dc57f8cda53a1039ff2f93ef1354a4 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 3 Oct 2024 03:21:03 +0400 Subject: [PATCH 145/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index f44b2799..bc380cb9 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -289,7 +289,9 @@ jobs: - name: πŸ§ͺ Run Tests for LanguageIdentification working-directory: ${{env.ROOT}}/LanguageIdentification.Tests - run: dotnet test -c CI + run: | + export HOME=${{runner.temp}} + dotnet test -c CI - name: πŸ§ͺ Run Tests for LanguageCode working-directory: ${{env.ROOT}}/LanguageCode.Tests From 028915f0232ecc8a9a5508a5029fa0b190c69b70 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 3 Oct 2024 03:32:32 +0400 Subject: [PATCH 146/219] Update FastTextDetector.cs --- src/LanguageIdentification.FastText/FastTextDetector.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/LanguageIdentification.FastText/FastTextDetector.cs b/src/LanguageIdentification.FastText/FastTextDetector.cs index 5c7f4181..02f999da 100644 --- a/src/LanguageIdentification.FastText/FastTextDetector.cs +++ b/src/LanguageIdentification.FastText/FastTextDetector.cs @@ -31,6 +31,8 @@ public FastTextDetector() public static bool IsSupported() { + throw new Exception(RuntimeInformation.OSArchitecture.ToString()); + return RuntimeInformation.OSArchitecture switch { Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => true, From d25378b3fd8b202349a596d12ac2c3e312dc57b0 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 3 Oct 2024 04:10:20 +0400 Subject: [PATCH 147/219] Update FastTextDetector.cs --- src/LanguageIdentification.FastText/FastTextDetector.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/LanguageIdentification.FastText/FastTextDetector.cs b/src/LanguageIdentification.FastText/FastTextDetector.cs index 02f999da..5c7f4181 100644 --- a/src/LanguageIdentification.FastText/FastTextDetector.cs +++ b/src/LanguageIdentification.FastText/FastTextDetector.cs @@ -31,8 +31,6 @@ public FastTextDetector() public static bool IsSupported() { - throw new Exception(RuntimeInformation.OSArchitecture.ToString()); - return RuntimeInformation.OSArchitecture switch { Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => true, From 5395b6d65e2732d1973fa66a714856a4b44d09c6 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 3 Oct 2024 04:12:53 +0400 Subject: [PATCH 148/219] 0.0.0.22 -> 0.0.1 --- src/LanguageCode/LanguageCode.csproj | 2 +- .../LanguageIdentification.CLD2.Native.csproj | 2 +- .../LanguageIdentification.CLD2.csproj | 2 +- .../LanguageIdentification.CLD3.Native.csproj | 2 +- .../LanguageIdentification.CLD3.csproj | 2 +- .../LanguageIdentification.FastText.Native.csproj | 2 +- .../LanguageIdentification.FastText.csproj | 2 +- .../LanguageIdentification.Lingua.Native.csproj | 2 +- .../LanguageIdentification.Lingua.csproj | 2 +- .../LanguageIdentification.MediaPipe.Native.csproj | 2 +- .../LanguageIdentification.MediaPipe.csproj | 2 +- .../LanguageIdentification.Whatlang.Native.csproj | 2 +- .../LanguageIdentification.Whatlang.csproj | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/LanguageCode/LanguageCode.csproj b/src/LanguageCode/LanguageCode.csproj index e2f3f039..eb9153c9 100644 --- a/src/LanguageCode/LanguageCode.csproj +++ b/src/LanguageCode/LanguageCode.csproj @@ -2,7 +2,7 @@ netstandard2.1;net5.0;net6.0;net7.0;net8.0 - 0.0.0.22 + 0.0.1 Panlingo.LanguageCode Panlingo.LanguageCode Panlingo.LanguageCode diff --git a/src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj b/src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj index 08c83a4c..611ece97 100644 --- a/src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj +++ b/src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.0.0.22 + 0.0.1 Panlingo.LanguageIdentification.CLD2.Native Panlingo.LanguageIdentification.CLD2.Native Panlingo.LanguageIdentification.CLD2.Native diff --git a/src/LanguageIdentification.CLD2/LanguageIdentification.CLD2.csproj b/src/LanguageIdentification.CLD2/LanguageIdentification.CLD2.csproj index 429bd006..f8ba0d0f 100644 --- a/src/LanguageIdentification.CLD2/LanguageIdentification.CLD2.csproj +++ b/src/LanguageIdentification.CLD2/LanguageIdentification.CLD2.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.0.0.22 + 0.0.1 Panlingo.LanguageIdentification.CLD2 Panlingo.LanguageIdentification.CLD2 Panlingo.LanguageIdentification.CLD2 diff --git a/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj b/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj index f3c2af5b..f3cf0428 100644 --- a/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj +++ b/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.0.0.22 + 0.0.1 Panlingo.LanguageIdentification.CLD3.Native Panlingo.LanguageIdentification.CLD3.Native Panlingo.LanguageIdentification.CLD3.Native diff --git a/src/LanguageIdentification.CLD3/LanguageIdentification.CLD3.csproj b/src/LanguageIdentification.CLD3/LanguageIdentification.CLD3.csproj index 7e0a156a..51ab902c 100644 --- a/src/LanguageIdentification.CLD3/LanguageIdentification.CLD3.csproj +++ b/src/LanguageIdentification.CLD3/LanguageIdentification.CLD3.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.0.0.22 + 0.0.1 Panlingo.LanguageIdentification.CLD3 Panlingo.LanguageIdentification.CLD3 Panlingo.LanguageIdentification.CLD3 diff --git a/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj b/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj index 64511c5c..f2691d86 100644 --- a/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj +++ b/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.0.0.22 + 0.0.1 Panlingo.LanguageIdentification.FastText.Native Panlingo.LanguageIdentification.FastText.Native Panlingo.LanguageIdentification.FastText.Native diff --git a/src/LanguageIdentification.FastText/LanguageIdentification.FastText.csproj b/src/LanguageIdentification.FastText/LanguageIdentification.FastText.csproj index 5d7d3a7b..eff7a554 100644 --- a/src/LanguageIdentification.FastText/LanguageIdentification.FastText.csproj +++ b/src/LanguageIdentification.FastText/LanguageIdentification.FastText.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.0.0.22 + 0.0.1 Panlingo.LanguageIdentification.FastText Panlingo.LanguageIdentification.FastText Panlingo.LanguageIdentification.FastText diff --git a/src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj b/src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj index 343f9d67..0a6895a9 100644 --- a/src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj +++ b/src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.0.0.22 + 0.0.1 Panlingo.LanguageIdentification.Lingua.Native Panlingo.LanguageIdentification.Lingua.Native Panlingo.LanguageIdentification.Lingua.Native diff --git a/src/LanguageIdentification.Lingua/LanguageIdentification.Lingua.csproj b/src/LanguageIdentification.Lingua/LanguageIdentification.Lingua.csproj index e54e0244..046d3f1e 100644 --- a/src/LanguageIdentification.Lingua/LanguageIdentification.Lingua.csproj +++ b/src/LanguageIdentification.Lingua/LanguageIdentification.Lingua.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.0.0.22 + 0.0.1 Panlingo.LanguageIdentification.Lingua Panlingo.LanguageIdentification.Lingua Panlingo.LanguageIdentification.Lingua diff --git a/src/LanguageIdentification.MediaPipe.Native/LanguageIdentification.MediaPipe.Native.csproj b/src/LanguageIdentification.MediaPipe.Native/LanguageIdentification.MediaPipe.Native.csproj index 2b02a786..3c60d96e 100644 --- a/src/LanguageIdentification.MediaPipe.Native/LanguageIdentification.MediaPipe.Native.csproj +++ b/src/LanguageIdentification.MediaPipe.Native/LanguageIdentification.MediaPipe.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.0.0.22 + 0.0.1 Panlingo.LanguageIdentification.MediaPipe.Native Panlingo.LanguageIdentification.MediaPipe.Native Panlingo.LanguageIdentification.MediaPipe.Native diff --git a/src/LanguageIdentification.MediaPipe/LanguageIdentification.MediaPipe.csproj b/src/LanguageIdentification.MediaPipe/LanguageIdentification.MediaPipe.csproj index 4b7ebff2..0147cdaa 100644 --- a/src/LanguageIdentification.MediaPipe/LanguageIdentification.MediaPipe.csproj +++ b/src/LanguageIdentification.MediaPipe/LanguageIdentification.MediaPipe.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.0.0.22 + 0.0.1 Panlingo.LanguageIdentification.MediaPipe Panlingo.LanguageIdentification.MediaPipe Panlingo.LanguageIdentification.MediaPipe diff --git a/src/LanguageIdentification.Whatlang.Native/LanguageIdentification.Whatlang.Native.csproj b/src/LanguageIdentification.Whatlang.Native/LanguageIdentification.Whatlang.Native.csproj index 8e9bf555..c9652ed7 100644 --- a/src/LanguageIdentification.Whatlang.Native/LanguageIdentification.Whatlang.Native.csproj +++ b/src/LanguageIdentification.Whatlang.Native/LanguageIdentification.Whatlang.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.0.0.22 + 0.0.1 Panlingo.LanguageIdentification.Whatlang.Native Panlingo.LanguageIdentification.Whatlang.Native Panlingo.LanguageIdentification.Whatlang.Native diff --git a/src/LanguageIdentification.Whatlang/LanguageIdentification.Whatlang.csproj b/src/LanguageIdentification.Whatlang/LanguageIdentification.Whatlang.csproj index 52dbcd78..b943a784 100644 --- a/src/LanguageIdentification.Whatlang/LanguageIdentification.Whatlang.csproj +++ b/src/LanguageIdentification.Whatlang/LanguageIdentification.Whatlang.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.0.0.22 + 0.0.1 Panlingo.LanguageIdentification.Whatlang Panlingo.LanguageIdentification.Whatlang Panlingo.LanguageIdentification.Whatlang From ad63ed52f142833ba2fa0747ab1ae58056ade2da Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 3 Oct 2024 21:22:55 +0400 Subject: [PATCH 149/219] Running tests outside of Docker --- .github/workflows/github-ci.yml | 67 ++++++++++++------- .../FastTextTests.cs | 8 +-- .../MediaPipeTests.cs | 8 +-- 3 files changed, 45 insertions(+), 38 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index bc380cb9..5957326b 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -62,7 +62,7 @@ jobs: compression-level: 0 if-no-files-found: 'error' - build-nugets: + build-package: name: πŸš€ Pack ${{matrix.task.title}} runs-on: ${{matrix.task.os}} needs: build-native @@ -146,10 +146,10 @@ jobs: overwrite: 'true' compression-level: 0 - test-nugets-linux: + test-package-linux: name: πŸ§ͺ Test on Linux runs-on: ubuntu-latest - needs: build-nugets + needs: build-package permissions: contents: read packages: write @@ -161,28 +161,45 @@ jobs: - name: πŸ“‚ Files working-directory: ${{env.ROOT}} - run: mkdir -p local-nugets + run: mkdir -p local-packages - uses: actions/download-artifact@v4 with: pattern: build-* merge-multiple: true - path: ${{env.ROOT}}/local-nugets + path: ${{env.ROOT}}/local-packages - name: πŸ“‚ Files working-directory: ${{env.ROOT}} run: ls -R - - name: πŸ“‚ Build Docker Image - working-directory: ${{env.ROOT}} + - name: πŸ“‚ Use local NuGet + run: | + path=$(realpath "${{env.ROOT}}/local-packages") + dotnet nuget add source $path + + - name: πŸ“‚ Download pretrained models run: | - docker build -f ./test.Dockerfile -t langunage-identification-test-runner:latest . - docker build -f ./test-ci.Dockerfile -t langunage-identification-test-runner-ci:latest . + mkdir ~/models -p + curl --location -o ~/models/fasttext176.bin https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin + curl --location -o ~/models/mediapipe_language_detector.tflite https://storage.googleapis.com/mediapipe-models/language_detector/language_detector/float32/1/language_detector.tflite - test-nugets-windows: + - name: πŸ§ͺ Run Tests for LanguageIdentification + working-directory: ${{env.ROOT}}/LanguageIdentification.Tests + run: | + export HOME=${{runner.temp}} + dotnet test -c CI + + - name: πŸ§ͺ Run Tests for LanguageCode + working-directory: ${{env.ROOT}}/LanguageCode.Tests + run: + export HOME=${{runner.temp}} + dotnet test -c CI + + test-package-windows: name: πŸ§ͺ Test on Windows runs-on: windows-latest - needs: build-nugets + needs: build-package permissions: contents: read packages: write @@ -203,13 +220,13 @@ jobs: source-url: https://nuget.pkg.github.com/${{github.repository_owner}}/index.json - name: πŸ“‚ Files - run: mkdir -p ${{env.ROOT}}/local-nugets + run: mkdir -p ${{env.ROOT}}/local-packages - uses: actions/download-artifact@v4 with: pattern: build-* merge-multiple: true - path: ${{env.ROOT}}/local-nugets + path: ${{env.ROOT}}/local-packages - name: πŸ“‚ Files working-directory: ${{env.ROOT}} @@ -217,7 +234,7 @@ jobs: - name: πŸ“‚ Use local NuGet run: | - $path = [System.IO.Path]::GetFullPath('${{env.ROOT}}/local-nugets') + $path = [System.IO.Path]::GetFullPath('${{env.ROOT}}/local-packages') dotnet nuget add source $path - name: πŸ“‚ Download pretrained models @@ -234,7 +251,7 @@ jobs: working-directory: ${{env.ROOT}}/LanguageCode.Tests run: dotnet test -c CI - test-nugets-macos: + test-package-macos: name: πŸ§ͺ Test on macOS (${{matrix.os}}) runs-on: ${{matrix.os}} strategy: @@ -243,7 +260,7 @@ jobs: os: - macos-14 - macos-13 - needs: build-nugets + needs: build-package permissions: contents: read packages: write @@ -264,13 +281,13 @@ jobs: source-url: https://nuget.pkg.github.com/${{github.repository_owner}}/index.json - name: πŸ“‚ Files - run: mkdir -p ${{env.ROOT}}/local-nugets + run: mkdir -p ${{env.ROOT}}/local-packages - uses: actions/download-artifact@v4 with: pattern: build-* merge-multiple: true - path: ${{env.ROOT}}/local-nugets + path: ${{env.ROOT}}/local-packages - name: πŸ“‚ Files working-directory: ${{env.ROOT}} @@ -278,7 +295,7 @@ jobs: - name: πŸ“‚ Use local NuGet run: | - path=$(realpath "${{env.ROOT}}/local-nugets") + path=$(realpath "${{env.ROOT}}/local-packages") dotnet nuget add source $path - name: πŸ“‚ Download pretrained models @@ -295,15 +312,17 @@ jobs: - name: πŸ§ͺ Run Tests for LanguageCode working-directory: ${{env.ROOT}}/LanguageCode.Tests - run: dotnet test -c CI + run: + export HOME=${{runner.temp}} + dotnet test -c CI - deploy-nugets: + deploy-package: name: 🚚 Push ${{matrix.projectName}} runs-on: ubuntu-latest needs: - - test-nugets-linux - - test-nugets-windows - - test-nugets-macos + - test-package-linux + - test-package-windows + - test-package-macos if: github.ref == 'refs/heads/master' strategy: fail-fast: false diff --git a/src/LanguageIdentification.Tests/FastTextTests.cs b/src/LanguageIdentification.Tests/FastTextTests.cs index 6d9cbedd..f909a4ee 100644 --- a/src/LanguageIdentification.Tests/FastTextTests.cs +++ b/src/LanguageIdentification.Tests/FastTextTests.cs @@ -8,13 +8,7 @@ public class FastTextTests { private static string GetModelPath() { - var root = "/"; - - if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) - { - root = Environment.GetEnvironmentVariable("HOME") ?? "/"; - } - + var root = Environment.GetEnvironmentVariable("HOME") ?? "/"; return Path.Combine(root, "/models/fasttext176.bin"); } diff --git a/src/LanguageIdentification.Tests/MediaPipeTests.cs b/src/LanguageIdentification.Tests/MediaPipeTests.cs index cb5a151c..d5191c5f 100644 --- a/src/LanguageIdentification.Tests/MediaPipeTests.cs +++ b/src/LanguageIdentification.Tests/MediaPipeTests.cs @@ -9,13 +9,7 @@ public class MediaPipeTests { private static string GetModelPath() { - var root = "/"; - - if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) - { - root = Environment.GetEnvironmentVariable("HOME") ?? "/"; - } - + var root = Environment.GetEnvironmentVariable("HOME") ?? "/"; return Path.Combine(root, "/models/mediapipe_language_detector.tflite"); } From 58128bfd372029f857e70e3a171242706e149cad Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 3 Oct 2024 22:17:55 +0400 Subject: [PATCH 150/219] Update MainTests.cs --- src/LanguageIdentification.Tests/MainTests.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.Tests/MainTests.cs b/src/LanguageIdentification.Tests/MainTests.cs index bf5b2b9a..18ee4f47 100644 --- a/src/LanguageIdentification.Tests/MainTests.cs +++ b/src/LanguageIdentification.Tests/MainTests.cs @@ -10,7 +10,8 @@ public class MainTests [SkippableFact] public void CheckPlatform() { - Skip.IfNot(RuntimeInformation.IsOSPlatform(OSPlatform.Linux)); - Assert.Equal("true", Environment.GetEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER")); + Assert.True(true); + //Skip.IfNot(RuntimeInformation.IsOSPlatform(OSPlatform.Linux)); + //Assert.Equal("true", Environment.GetEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER")); } } From c8bc8002f8ba97e4fe81b3f55a90454dbc368594 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 3 Oct 2024 22:19:13 +0400 Subject: [PATCH 151/219] CI fixes --- src/LanguageCode.Tests/LanguageCode.Tests.csproj | 4 ++++ src/LanguageCode.Tests/LanguageCodeTests.cs | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/LanguageCode.Tests/LanguageCode.Tests.csproj b/src/LanguageCode.Tests/LanguageCode.Tests.csproj index 205df9fa..310f39d6 100644 --- a/src/LanguageCode.Tests/LanguageCode.Tests.csproj +++ b/src/LanguageCode.Tests/LanguageCode.Tests.csproj @@ -7,6 +7,10 @@ false true + + Panlingo.LanguageCode.Tests + + Debug;Release;CI diff --git a/src/LanguageCode.Tests/LanguageCodeTests.cs b/src/LanguageCode.Tests/LanguageCodeTests.cs index 1e0a88a9..efcd3605 100644 --- a/src/LanguageCode.Tests/LanguageCodeTests.cs +++ b/src/LanguageCode.Tests/LanguageCodeTests.cs @@ -1,7 +1,7 @@ ο»Ώusing Panlingo.LanguageCode; using Panlingo.LanguageCode.Models; -namespace LanguageCode.Tests +namespace Panlingo.LanguageCode.Tests { public class LanguageCodeTests { From a400307788d34d36095be1e83779d0d31894aee0 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 3 Oct 2024 23:46:58 +0400 Subject: [PATCH 152/219] FT + MediaPipe: preload models from code --- .github/workflows/github-ci.yml | 18 --------- .../FastTextTests.cs | 36 +++++++++++------ .../Helpers/FileHelper.cs | 19 +++++++++ .../LanguageIdentification.Tests.csproj | 1 + src/LanguageIdentification.Tests/MainTests.cs | 6 ++- .../MediaPipeTests.cs | 40 ++++++++++++------- 6 files changed, 74 insertions(+), 46 deletions(-) create mode 100644 src/LanguageIdentification.Tests/Helpers/FileHelper.cs diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 5957326b..db28de2c 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -178,12 +178,6 @@ jobs: path=$(realpath "${{env.ROOT}}/local-packages") dotnet nuget add source $path - - name: πŸ“‚ Download pretrained models - run: | - mkdir ~/models -p - curl --location -o ~/models/fasttext176.bin https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin - curl --location -o ~/models/mediapipe_language_detector.tflite https://storage.googleapis.com/mediapipe-models/language_detector/language_detector/float32/1/language_detector.tflite - - name: πŸ§ͺ Run Tests for LanguageIdentification working-directory: ${{env.ROOT}}/LanguageIdentification.Tests run: | @@ -237,12 +231,6 @@ jobs: $path = [System.IO.Path]::GetFullPath('${{env.ROOT}}/local-packages') dotnet nuget add source $path - - name: πŸ“‚ Download pretrained models - run: | - mkdir /models - Invoke-WebRequest -OutFile /models/fasttext176.bin https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin - Invoke-WebRequest -OutFile /models/mediapipe_language_detector.tflite https://storage.googleapis.com/mediapipe-models/language_detector/language_detector/float32/1/language_detector.tflite - - name: πŸ§ͺ Run Tests for LanguageIdentification working-directory: ${{env.ROOT}}/LanguageIdentification.Tests run: dotnet test -c CI @@ -298,12 +286,6 @@ jobs: path=$(realpath "${{env.ROOT}}/local-packages") dotnet nuget add source $path - - name: πŸ“‚ Download pretrained models - run: | - mkdir ~/models -p - curl --location -o ~/models/fasttext176.bin https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin - curl --location -o ~/models/mediapipe_language_detector.tflite https://storage.googleapis.com/mediapipe-models/language_detector/language_detector/float32/1/language_detector.tflite - - name: πŸ§ͺ Run Tests for LanguageIdentification working-directory: ${{env.ROOT}}/LanguageIdentification.Tests run: | diff --git a/src/LanguageIdentification.Tests/FastTextTests.cs b/src/LanguageIdentification.Tests/FastTextTests.cs index f909a4ee..c4113ea7 100644 --- a/src/LanguageIdentification.Tests/FastTextTests.cs +++ b/src/LanguageIdentification.Tests/FastTextTests.cs @@ -4,13 +4,9 @@ namespace Panlingo.LanguageIdentification.Tests; -public class FastTextTests +public class FastTextTests : IAsyncLifetime { - private static string GetModelPath() - { - var root = Environment.GetEnvironmentVariable("HOME") ?? "/"; - return Path.Combine(root, "/models/fasttext176.bin"); - } + private readonly string _modelPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "/models/fasttext176.bin"); [SkippableTheory] [InlineData("__label__en", Constants.PHRASE_ENG_1, 0.9955)] @@ -22,8 +18,7 @@ public void FastTextFileSingleLanguage(string languageCode, string text, double using var fastText = new FastTextDetector(); - var modelPath = GetModelPath(); - fastText.LoadModel(modelPath); + fastText.LoadModel(_modelPath); var predictions = fastText.Predict(text: text, count: 10); var mainLanguage = predictions.FirstOrDefault(); @@ -47,8 +42,7 @@ public void FastTextStreamSingleLanguage(string languageCode, string text, doubl using var fastText = new FastTextDetector(); - var modelPath = GetModelPath(); - using var stream = File.Open(modelPath, FileMode.Open); + using var stream = File.Open(_modelPath, FileMode.Open); fastText.LoadModel(stream); @@ -94,8 +88,7 @@ public void FastTextLabels() using var fastText = new FastTextDetector(); - var modelPath = GetModelPath(); - fastText.LoadModel(modelPath); + fastText.LoadModel(_modelPath); var labels = fastText.GetLabels(); @@ -103,4 +96,23 @@ public void FastTextLabels() Assert.Contains(labels, x => x.Label == "__label__uk"); Assert.Contains(labels, x => x.Label == "__label__ru"); } + + public async Task InitializeAsync() + { + var url = "https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin"; + await FileHelper.DownloadAsync( + path: _modelPath, + url: url + ); + } + + public async Task DisposeAsync() + { + if (File.Exists(_modelPath)) + { + File.Delete(_modelPath); + } + + await Task.CompletedTask; + } } diff --git a/src/LanguageIdentification.Tests/Helpers/FileHelper.cs b/src/LanguageIdentification.Tests/Helpers/FileHelper.cs new file mode 100644 index 00000000..8d53e83d --- /dev/null +++ b/src/LanguageIdentification.Tests/Helpers/FileHelper.cs @@ -0,0 +1,19 @@ +ο»Ώnamespace Panlingo.LanguageIdentification.Tests.Helpers; + +public class FileHelper +{ + public static async Task DownloadAsync(string path, string url) + { + using var client = new HttpClient(); + using var stream = await client.GetStreamAsync(url); + + var directory = Path.GetDirectoryName(path) ?? throw new Exception("No directory"); + if (!Directory.Exists(directory)) + { + Directory.CreateDirectory(directory); + } + + using var file = new FileStream(path, FileMode.OpenOrCreate); + await stream.CopyToAsync(file); + } +} diff --git a/src/LanguageIdentification.Tests/LanguageIdentification.Tests.csproj b/src/LanguageIdentification.Tests/LanguageIdentification.Tests.csproj index bc57a4f9..8ce25e43 100644 --- a/src/LanguageIdentification.Tests/LanguageIdentification.Tests.csproj +++ b/src/LanguageIdentification.Tests/LanguageIdentification.Tests.csproj @@ -18,6 +18,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/src/LanguageIdentification.Tests/MainTests.cs b/src/LanguageIdentification.Tests/MainTests.cs index 18ee4f47..14c6a2a1 100644 --- a/src/LanguageIdentification.Tests/MainTests.cs +++ b/src/LanguageIdentification.Tests/MainTests.cs @@ -1,4 +1,4 @@ -ο»Ώusing System.Runtime.InteropServices; +ο»Ώusing Microsoft.Build.Construction; namespace Panlingo.LanguageIdentification.Tests; @@ -13,5 +13,9 @@ public void CheckPlatform() Assert.True(true); //Skip.IfNot(RuntimeInformation.IsOSPlatform(OSPlatform.Linux)); //Assert.Equal("true", Environment.GetEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER")); + + /*var projectRootElement = ProjectRootElement.Open(""); + projectRootElement.AddProperty("Version", "3.4.5"); + projectRootElement.Save();*/ } } diff --git a/src/LanguageIdentification.Tests/MediaPipeTests.cs b/src/LanguageIdentification.Tests/MediaPipeTests.cs index d5191c5f..a2bf9087 100644 --- a/src/LanguageIdentification.Tests/MediaPipeTests.cs +++ b/src/LanguageIdentification.Tests/MediaPipeTests.cs @@ -1,17 +1,11 @@ -ο»Ώusing System.IO; -using System.Runtime.InteropServices; -using Panlingo.LanguageIdentification.MediaPipe; +ο»Ώusing Panlingo.LanguageIdentification.MediaPipe; using Panlingo.LanguageIdentification.Tests.Helpers; namespace Panlingo.LanguageIdentification.Tests; -public class MediaPipeTests +public class MediaPipeTests : IAsyncLifetime { - private static string GetModelPath() - { - var root = Environment.GetEnvironmentVariable("HOME") ?? "/"; - return Path.Combine(root, "/models/mediapipe_language_detector.tflite"); - } + private readonly string _modelPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "/models/mediapipe_language_detector.tflite"); [SkippableTheory] [InlineData("en", Constants.PHRASE_ENG_1, 0.9994)] @@ -21,9 +15,8 @@ public void MediaPipeFileSingleLanguage(string languageCode, string text, double { Skip.IfNot(MediaPipeDetector.IsSupported()); - var modelPath = GetModelPath(); using var mediaPipe = new MediaPipeDetector( - options: MediaPipeOptions.FromFile(modelPath).WithResultCount(10) + options: MediaPipeOptions.FromFile(_modelPath).WithResultCount(10) ); var predictions = mediaPipe.PredictLanguages(text: text); @@ -46,8 +39,7 @@ public void MediaPipeStreamSingleLanguage(string languageCode, string text, doub { Skip.IfNot(MediaPipeDetector.IsSupported()); - var modelPath = GetModelPath(); - using var stream = File.Open(modelPath, FileMode.Open); + using var stream = File.Open(_modelPath, FileMode.Open); using var mediaPipe = new MediaPipeDetector( options: MediaPipeOptions.FromStream(stream).WithResultCount(10) @@ -73,8 +65,7 @@ public void MediaPipeContainedSingleLanguage(string languageCode, string text, d { Skip.IfNot(MediaPipeDetector.IsSupported()); - var modelPath = GetModelPath(); - using var stream = File.Open(modelPath, FileMode.Open); + using var stream = File.Open(_modelPath, FileMode.Open); using var mediaPipe = new MediaPipeDetector( options: MediaPipeOptions.FromDefault().WithResultCount(10) @@ -91,4 +82,23 @@ public void MediaPipeContainedSingleLanguage(string languageCode, string text, d Assert.Equal(languageCode, mainLanguage.Language); Assert.Equal(score, mainLanguage.Probability, Constants.EPSILON); } + + public async Task InitializeAsync() + { + var url = "https://storage.googleapis.com/mediapipe-models/language_detector/language_detector/float32/1/language_detector.tflite"; + await FileHelper.DownloadAsync( + path: _modelPath, + url: url + ); + } + + public async Task DisposeAsync() + { + if (File.Exists(_modelPath)) + { + File.Delete(_modelPath); + } + + await Task.CompletedTask; + } } From 4f008631bdcccddc38029a53c2ea1d16908d32c1 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Thu, 3 Oct 2024 23:59:12 +0400 Subject: [PATCH 153/219] Update Dockerfiles --- src/test-ci.Dockerfile | 2 +- src/test.Dockerfile | 15 --------------- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/src/test-ci.Dockerfile b/src/test-ci.Dockerfile index 27d7a2ab..bc46a1d6 100644 --- a/src/test-ci.Dockerfile +++ b/src/test-ci.Dockerfile @@ -3,7 +3,7 @@ WORKDIR /src COPY . . -RUN dotnet nuget add source /src/local-nugets +RUN dotnet nuget add source /src/local-packages RUN ls -R diff --git a/src/test.Dockerfile b/src/test.Dockerfile index 8fafba69..9e6935da 100644 --- a/src/test.Dockerfile +++ b/src/test.Dockerfile @@ -3,18 +3,3 @@ RUN wget https://aka.ms/getvsdbgsh && \ sh getvsdbgsh -v latest -l /vsdbg -### FastText -RUN apt -y update -RUN apt -y install curl -RUN mkdir /models -p -RUN curl --location -o /models/fasttext176.bin https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin -# RUN curl --location -o /models/fasttext217.bin https://huggingface.co/facebook/fasttext-language-identification/resolve/main/model.bin?download=true -### - -### MediaPipe -RUN apt -y update -RUN apt -y install curl -RUN curl --location -o /models/mediapipe_language_detector.tflite https://storage.googleapis.com/mediapipe-models/language_detector/language_detector/float32/1/language_detector.tflite -### - - From e949514a695f7556025585cc7cba051693947667 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 4 Oct 2024 00:16:17 +0400 Subject: [PATCH 154/219] Test --- src/LanguageIdentification.Tests/FastTextTests.cs | 2 ++ src/LanguageIdentification.Tests/Helpers/FileHelper.cs | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/src/LanguageIdentification.Tests/FastTextTests.cs b/src/LanguageIdentification.Tests/FastTextTests.cs index c4113ea7..a2ed3f95 100644 --- a/src/LanguageIdentification.Tests/FastTextTests.cs +++ b/src/LanguageIdentification.Tests/FastTextTests.cs @@ -99,6 +99,8 @@ public void FastTextLabels() public async Task InitializeAsync() { + throw new Exception($"Root: '{AppDomain.CurrentDomain.BaseDirectory}'"); + var url = "https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin"; await FileHelper.DownloadAsync( path: _modelPath, diff --git a/src/LanguageIdentification.Tests/Helpers/FileHelper.cs b/src/LanguageIdentification.Tests/Helpers/FileHelper.cs index 8d53e83d..5c20991c 100644 --- a/src/LanguageIdentification.Tests/Helpers/FileHelper.cs +++ b/src/LanguageIdentification.Tests/Helpers/FileHelper.cs @@ -4,6 +4,11 @@ public class FileHelper { public static async Task DownloadAsync(string path, string url) { + if (File.Exists(path)) + { + return; + } + using var client = new HttpClient(); using var stream = await client.GetStreamAsync(url); From 721c82884d30935e03c09113e38d73fb72e41bd2 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 4 Oct 2024 00:25:12 +0400 Subject: [PATCH 155/219] Update FastTextTests.cs --- src/LanguageIdentification.Tests/FastTextTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.Tests/FastTextTests.cs b/src/LanguageIdentification.Tests/FastTextTests.cs index a2ed3f95..b0d1935d 100644 --- a/src/LanguageIdentification.Tests/FastTextTests.cs +++ b/src/LanguageIdentification.Tests/FastTextTests.cs @@ -99,7 +99,7 @@ public void FastTextLabels() public async Task InitializeAsync() { - throw new Exception($"Root: '{AppDomain.CurrentDomain.BaseDirectory}'"); + throw new Exception($"Root: '{_modelPath}'"); var url = "https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin"; await FileHelper.DownloadAsync( From ec6880014e4079ec32d1b47757e3eb1e5ce528dd Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 4 Oct 2024 00:35:38 +0400 Subject: [PATCH 156/219] Fixes --- src/LanguageIdentification.Tests/FastTextTests.cs | 4 +--- src/LanguageIdentification.Tests/MediaPipeTests.cs | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/LanguageIdentification.Tests/FastTextTests.cs b/src/LanguageIdentification.Tests/FastTextTests.cs index b0d1935d..1d6a6f78 100644 --- a/src/LanguageIdentification.Tests/FastTextTests.cs +++ b/src/LanguageIdentification.Tests/FastTextTests.cs @@ -6,7 +6,7 @@ namespace Panlingo.LanguageIdentification.Tests; public class FastTextTests : IAsyncLifetime { - private readonly string _modelPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "/models/fasttext176.bin"); + private readonly string _modelPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "models/fasttext176.bin"); [SkippableTheory] [InlineData("__label__en", Constants.PHRASE_ENG_1, 0.9955)] @@ -99,8 +99,6 @@ public void FastTextLabels() public async Task InitializeAsync() { - throw new Exception($"Root: '{_modelPath}'"); - var url = "https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin"; await FileHelper.DownloadAsync( path: _modelPath, diff --git a/src/LanguageIdentification.Tests/MediaPipeTests.cs b/src/LanguageIdentification.Tests/MediaPipeTests.cs index a2bf9087..0aaf4a1d 100644 --- a/src/LanguageIdentification.Tests/MediaPipeTests.cs +++ b/src/LanguageIdentification.Tests/MediaPipeTests.cs @@ -5,7 +5,7 @@ namespace Panlingo.LanguageIdentification.Tests; public class MediaPipeTests : IAsyncLifetime { - private readonly string _modelPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "/models/mediapipe_language_detector.tflite"); + private readonly string _modelPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "models/mediapipe_language_detector.tflite"); [SkippableTheory] [InlineData("en", Constants.PHRASE_ENG_1, 0.9994)] From c7f527a3bbca3ac5072ccd1d9bd1959279bbeec0 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 4 Oct 2024 01:00:55 +0400 Subject: [PATCH 157/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index db28de2c..c533c4b1 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -188,7 +188,7 @@ jobs: working-directory: ${{env.ROOT}}/LanguageCode.Tests run: export HOME=${{runner.temp}} - dotnet test -c CI + dotnet test test-package-windows: name: πŸ§ͺ Test on Windows @@ -237,7 +237,7 @@ jobs: - name: πŸ§ͺ Run Tests for LanguageCode working-directory: ${{env.ROOT}}/LanguageCode.Tests - run: dotnet test -c CI + run: dotnet test test-package-macos: name: πŸ§ͺ Test on macOS (${{matrix.os}}) @@ -296,7 +296,7 @@ jobs: working-directory: ${{env.ROOT}}/LanguageCode.Tests run: export HOME=${{runner.temp}} - dotnet test -c CI + dotnet test deploy-package: name: 🚚 Push ${{matrix.projectName}} From c86a59a48adc3c4a58b34f9752dcecb0d3b1784f Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 4 Oct 2024 01:24:48 +0400 Subject: [PATCH 158/219] New test stage --- .github/workflows/github-ci.yml | 136 +++----------------- src/LanguageCode.Tests/LanguageCodeTests.cs | 3 +- 2 files changed, 22 insertions(+), 117 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index c533c4b1..a13fcc04 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -146,10 +146,18 @@ jobs: overwrite: 'true' compression-level: 0 - test-package-linux: - name: πŸ§ͺ Test on Linux - runs-on: ubuntu-latest + test-package: + name: πŸ§ͺ Test on ${{matrix.task.os}} + runs-on: ${{matrix.task.os}} needs: build-package + strategy: + fail-fast: false + matrix: + task: + - { title: CLD2, projectName: LanguageIdentification.CLD2, os: ubuntu-latest } + - { title: CLD3, projectName: LanguageIdentification.CLD3, os: windows-latest } + - { title: FastText, projectName: LanguageIdentification.FastText, os: macos-13 } + - { title: FastText, projectName: LanguageIdentification.FastText, os: macos-14 } permissions: contents: read packages: write @@ -170,141 +178,39 @@ jobs: path: ${{env.ROOT}}/local-packages - name: πŸ“‚ Files - working-directory: ${{env.ROOT}} + working-directory: ${{env.ROOT}}/local-packages run: ls -R - - name: πŸ“‚ Use local NuGet + - name: πŸ“‚ Use local NuGet (Linux) + if: matrix.task.os == 'ubuntu-latest' run: | path=$(realpath "${{env.ROOT}}/local-packages") dotnet nuget add source $path - - name: πŸ§ͺ Run Tests for LanguageIdentification - working-directory: ${{env.ROOT}}/LanguageIdentification.Tests - run: | - export HOME=${{runner.temp}} - dotnet test -c CI - - - name: πŸ§ͺ Run Tests for LanguageCode - working-directory: ${{env.ROOT}}/LanguageCode.Tests - run: - export HOME=${{runner.temp}} - dotnet test - - test-package-windows: - name: πŸ§ͺ Test on Windows - runs-on: windows-latest - needs: build-package - permissions: - contents: read - packages: write - env: - ROOT: ./src - NUGET_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}} - steps: - - uses: actions/checkout@v4 - - - name: πŸ“‚ Setup .NET Core - uses: actions/setup-dotnet@v4 - with: - dotnet-version: | - 5.0.x - 6.0.x - 7.0.x - 8.0.x - source-url: https://nuget.pkg.github.com/${{github.repository_owner}}/index.json - - - name: πŸ“‚ Files - run: mkdir -p ${{env.ROOT}}/local-packages - - - uses: actions/download-artifact@v4 - with: - pattern: build-* - merge-multiple: true - path: ${{env.ROOT}}/local-packages - - - name: πŸ“‚ Files - working-directory: ${{env.ROOT}} - run: ls -R - - - name: πŸ“‚ Use local NuGet + - name: πŸ“‚ Use local NuGet (Windows) + if: matrix.task.os == 'windows-latest' run: | $path = [System.IO.Path]::GetFullPath('${{env.ROOT}}/local-packages') dotnet nuget add source $path - - name: πŸ§ͺ Run Tests for LanguageIdentification - working-directory: ${{env.ROOT}}/LanguageIdentification.Tests - run: dotnet test -c CI - - - name: πŸ§ͺ Run Tests for LanguageCode - working-directory: ${{env.ROOT}}/LanguageCode.Tests - run: dotnet test - - test-package-macos: - name: πŸ§ͺ Test on macOS (${{matrix.os}}) - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - os: - - macos-14 - - macos-13 - needs: build-package - permissions: - contents: read - packages: write - env: - ROOT: ./src - NUGET_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}} - steps: - - uses: actions/checkout@v4 - - - name: πŸ“‚ Setup .NET Core - uses: actions/setup-dotnet@v4 - with: - dotnet-version: | - 5.0.x - 6.0.x - 7.0.x - 8.0.x - source-url: https://nuget.pkg.github.com/${{github.repository_owner}}/index.json - - - name: πŸ“‚ Files - run: mkdir -p ${{env.ROOT}}/local-packages - - - uses: actions/download-artifact@v4 - with: - pattern: build-* - merge-multiple: true - path: ${{env.ROOT}}/local-packages - - - name: πŸ“‚ Files - working-directory: ${{env.ROOT}} - run: ls -R - - - name: πŸ“‚ Use local NuGet + - name: πŸ“‚ Use local NuGet (OSX) + if: matrix.task.os == 'macos-13' || matrix.task.os == 'macos-14' run: | path=$(realpath "${{env.ROOT}}/local-packages") dotnet nuget add source $path - name: πŸ§ͺ Run Tests for LanguageIdentification working-directory: ${{env.ROOT}}/LanguageIdentification.Tests - run: | - export HOME=${{runner.temp}} - dotnet test -c CI + run: dotnet test -c CI - name: πŸ§ͺ Run Tests for LanguageCode working-directory: ${{env.ROOT}}/LanguageCode.Tests - run: - export HOME=${{runner.temp}} - dotnet test + run: dotnet test deploy-package: name: 🚚 Push ${{matrix.projectName}} runs-on: ubuntu-latest - needs: - - test-package-linux - - test-package-windows - - test-package-macos + needs: test-package if: github.ref == 'refs/heads/master' strategy: fail-fast: false diff --git a/src/LanguageCode.Tests/LanguageCodeTests.cs b/src/LanguageCode.Tests/LanguageCodeTests.cs index efcd3605..b03f19c1 100644 --- a/src/LanguageCode.Tests/LanguageCodeTests.cs +++ b/src/LanguageCode.Tests/LanguageCodeTests.cs @@ -1,5 +1,4 @@ -ο»Ώusing Panlingo.LanguageCode; -using Panlingo.LanguageCode.Models; +ο»Ώusing Panlingo.LanguageCode.Models; namespace Panlingo.LanguageCode.Tests { From fbcb1eaeb8f9a432ef38cb1b4e3d7de47e3161a9 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 4 Oct 2024 02:58:16 +0400 Subject: [PATCH 159/219] Tests: add CheckPackageVersion --- .github/workflows/github-ci.yml | 4 +- src/LanguageIdentification.Tests/MainTests.cs | 92 ++++++++++++++++--- 2 files changed, 83 insertions(+), 13 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index a13fcc04..a5b4a352 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -55,7 +55,7 @@ jobs: - uses: actions/upload-artifact@v4 if: steps.changes.outputs.src == 'true' with: - name: build-${{matrix.task.projectName}}-${{matrix.task.os}} + name: native-${{matrix.task.projectName}}-${{matrix.task.os}} path: ${{env.ROOT}}/${{matrix.task.projectName}}/${{matrix.task.artifact}} retention-days: 1 overwrite: 'true' @@ -90,7 +90,7 @@ jobs: - uses: actions/download-artifact@v4 with: - pattern: build-${{matrix.task.projectName}}.Native-* + pattern: native-${{matrix.task.projectName}}.Native-* merge-multiple: true path: ${{env.ROOT}}/${{matrix.task.projectName}}.Native/ diff --git a/src/LanguageIdentification.Tests/MainTests.cs b/src/LanguageIdentification.Tests/MainTests.cs index 14c6a2a1..28dd57dd 100644 --- a/src/LanguageIdentification.Tests/MainTests.cs +++ b/src/LanguageIdentification.Tests/MainTests.cs @@ -4,18 +4,88 @@ namespace Panlingo.LanguageIdentification.Tests; public class MainTests { - /// - /// Checks the current OS and container environment - /// - [SkippableFact] - public void CheckPlatform() + [Fact] + public void CheckPackageVersion() { - Assert.True(true); - //Skip.IfNot(RuntimeInformation.IsOSPlatform(OSPlatform.Linux)); - //Assert.Equal("true", Environment.GetEnvironmentVariable("DOTNET_RUNNING_IN_CONTAINER")); + Type[] types = [ + typeof(Panlingo.LanguageIdentification.CLD2.CLD2Detector), + typeof(Panlingo.LanguageIdentification.CLD3.CLD3Detector), + typeof(Panlingo.LanguageIdentification.FastText.FastTextDetector), + typeof(Panlingo.LanguageIdentification.Lingua.LinguaDetector), + typeof(Panlingo.LanguageIdentification.MediaPipe.MediaPipeDetector), + typeof(Panlingo.LanguageIdentification.Whatlang.WhatlangDetector), + ]; - /*var projectRootElement = ProjectRootElement.Open(""); - projectRootElement.AddProperty("Version", "3.4.5"); - projectRootElement.Save();*/ + var root = AppDomain.CurrentDomain.BaseDirectory; + + var src = root; + while (src != "/") + { + if (Path.GetFileName(src) == "src") + { + break; + } + + src = Path.GetDirectoryName(src) ?? "/"; + } + + var projectFiles = Directory.GetFiles(src, "*.csproj", SearchOption.AllDirectories); + + var packageProjects = new Dictionary(); + + foreach (var projectFile in projectFiles) + { + var projectRootElement = ProjectRootElement.Open(projectFile); + var assemblyName = projectRootElement.Properties.FirstOrDefault(x => x.Name == "AssemblyName"); + var version = projectRootElement.Properties.FirstOrDefault(x => x.Name == "Version"); + + if (assemblyName is null || version is null) + { + continue; + } + + if (string.IsNullOrEmpty(assemblyName.Value) || string.IsNullOrEmpty(version.Value)) + { + continue; + } + + packageProjects[assemblyName.Value] = version.Value; + } + + if (packageProjects.Count == 0) + { + throw new Exception("Projects are not found"); + } + + var assemblies = AppDomain.CurrentDomain.GetAssemblies() + .Where(x => + { + foreach (var type in x.GetTypes()) + { + if (types.Contains(type)) + { + return true; + } + } + + return false; + }) + .ToArray(); + + var assemblyNames = new List(); + var packageNames = new List(); + + foreach (var assembly in assemblies) + { + var assemblyName = assembly.GetName(); + + assemblyNames.Add($"{assemblyName.Name} {assemblyName.Version}"); + if (assemblyName.Name != null && packageProjects.TryGetValue(assemblyName.Name, out var packageVersion)) + { + packageNames.Add($"{assemblyName.Name} {packageVersion}"); + } + } + + Assert.Equal(packageNames, assemblyNames); } } From 4e68dcd73b3e77ff446928da1101ee9be1245481 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 4 Oct 2024 03:07:51 +0400 Subject: [PATCH 160/219] Fix CheckPackageVersion --- .../LanguageIdentification.Lingua.Native.csproj | 2 +- .../LanguageIdentification.Lingua.csproj | 2 +- .../LanguageIdentification.MediaPipe.Native.csproj | 2 +- .../LanguageIdentification.MediaPipe.csproj | 2 +- src/LanguageIdentification.Tests/MainTests.cs | 6 ++++++ .../LanguageIdentification.Whatlang.Native.csproj | 2 +- .../LanguageIdentification.Whatlang.csproj | 2 +- 7 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj b/src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj index 0a6895a9..343f9d67 100644 --- a/src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj +++ b/src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.0.1 + 0.0.0.22 Panlingo.LanguageIdentification.Lingua.Native Panlingo.LanguageIdentification.Lingua.Native Panlingo.LanguageIdentification.Lingua.Native diff --git a/src/LanguageIdentification.Lingua/LanguageIdentification.Lingua.csproj b/src/LanguageIdentification.Lingua/LanguageIdentification.Lingua.csproj index 046d3f1e..e54e0244 100644 --- a/src/LanguageIdentification.Lingua/LanguageIdentification.Lingua.csproj +++ b/src/LanguageIdentification.Lingua/LanguageIdentification.Lingua.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.0.1 + 0.0.0.22 Panlingo.LanguageIdentification.Lingua Panlingo.LanguageIdentification.Lingua Panlingo.LanguageIdentification.Lingua diff --git a/src/LanguageIdentification.MediaPipe.Native/LanguageIdentification.MediaPipe.Native.csproj b/src/LanguageIdentification.MediaPipe.Native/LanguageIdentification.MediaPipe.Native.csproj index 3c60d96e..2b02a786 100644 --- a/src/LanguageIdentification.MediaPipe.Native/LanguageIdentification.MediaPipe.Native.csproj +++ b/src/LanguageIdentification.MediaPipe.Native/LanguageIdentification.MediaPipe.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.0.1 + 0.0.0.22 Panlingo.LanguageIdentification.MediaPipe.Native Panlingo.LanguageIdentification.MediaPipe.Native Panlingo.LanguageIdentification.MediaPipe.Native diff --git a/src/LanguageIdentification.MediaPipe/LanguageIdentification.MediaPipe.csproj b/src/LanguageIdentification.MediaPipe/LanguageIdentification.MediaPipe.csproj index 0147cdaa..4b7ebff2 100644 --- a/src/LanguageIdentification.MediaPipe/LanguageIdentification.MediaPipe.csproj +++ b/src/LanguageIdentification.MediaPipe/LanguageIdentification.MediaPipe.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.0.1 + 0.0.0.22 Panlingo.LanguageIdentification.MediaPipe Panlingo.LanguageIdentification.MediaPipe Panlingo.LanguageIdentification.MediaPipe diff --git a/src/LanguageIdentification.Tests/MainTests.cs b/src/LanguageIdentification.Tests/MainTests.cs index 28dd57dd..968218c5 100644 --- a/src/LanguageIdentification.Tests/MainTests.cs +++ b/src/LanguageIdentification.Tests/MainTests.cs @@ -14,6 +14,12 @@ public void CheckPackageVersion() typeof(Panlingo.LanguageIdentification.Lingua.LinguaDetector), typeof(Panlingo.LanguageIdentification.MediaPipe.MediaPipeDetector), typeof(Panlingo.LanguageIdentification.Whatlang.WhatlangDetector), + typeof(Panlingo.LanguageIdentification.CLD2.Native.CLD2NativeLibrary), + typeof(Panlingo.LanguageIdentification.CLD3.Native.CLD3NativeLibrary), + typeof(Panlingo.LanguageIdentification.FastText.Native.FastTextNativeLibrary), + typeof(Panlingo.LanguageIdentification.Lingua.Native.LinguaNativeLibrary), + typeof(Panlingo.LanguageIdentification.MediaPipe.Native.MediaPipeNativeLibrary), + typeof(Panlingo.LanguageIdentification.Whatlang.Native.WhatlangNativeLibrary), ]; var root = AppDomain.CurrentDomain.BaseDirectory; diff --git a/src/LanguageIdentification.Whatlang.Native/LanguageIdentification.Whatlang.Native.csproj b/src/LanguageIdentification.Whatlang.Native/LanguageIdentification.Whatlang.Native.csproj index c9652ed7..8e9bf555 100644 --- a/src/LanguageIdentification.Whatlang.Native/LanguageIdentification.Whatlang.Native.csproj +++ b/src/LanguageIdentification.Whatlang.Native/LanguageIdentification.Whatlang.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.0.1 + 0.0.0.22 Panlingo.LanguageIdentification.Whatlang.Native Panlingo.LanguageIdentification.Whatlang.Native Panlingo.LanguageIdentification.Whatlang.Native diff --git a/src/LanguageIdentification.Whatlang/LanguageIdentification.Whatlang.csproj b/src/LanguageIdentification.Whatlang/LanguageIdentification.Whatlang.csproj index b943a784..52dbcd78 100644 --- a/src/LanguageIdentification.Whatlang/LanguageIdentification.Whatlang.csproj +++ b/src/LanguageIdentification.Whatlang/LanguageIdentification.Whatlang.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.0.1 + 0.0.0.22 Panlingo.LanguageIdentification.Whatlang Panlingo.LanguageIdentification.Whatlang Panlingo.LanguageIdentification.Whatlang From 490bf154db9c4db1ec4f1f77f61928e343f1beba Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 4 Oct 2024 03:23:13 +0400 Subject: [PATCH 161/219] Fix version parsing --- src/LanguageIdentification.Tests/MainTests.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/LanguageIdentification.Tests/MainTests.cs b/src/LanguageIdentification.Tests/MainTests.cs index 968218c5..ea1cdb08 100644 --- a/src/LanguageIdentification.Tests/MainTests.cs +++ b/src/LanguageIdentification.Tests/MainTests.cs @@ -88,7 +88,15 @@ public void CheckPackageVersion() assemblyNames.Add($"{assemblyName.Name} {assemblyName.Version}"); if (assemblyName.Name != null && packageProjects.TryGetValue(assemblyName.Name, out var packageVersion)) { - packageNames.Add($"{assemblyName.Name} {packageVersion}"); + var a = Version.Parse(packageVersion); + var b = new Version( + major: a.Major != -1 ? a.Major : 0, + minor: a.Minor != -1 ? a.Minor : 0, + build: a.Build != -1 ? a.Build : 0, + revision: a.Revision != -1 ? a.Revision : 0 + ); + + packageNames.Add($"{assemblyName.Name} {b}"); } } From 10868fb9d3e6b4b55b12009037ef8e35f56ff9d0 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 4 Oct 2024 22:37:39 +0400 Subject: [PATCH 162/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index a5b4a352..5cce35cf 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -152,12 +152,17 @@ jobs: needs: build-package strategy: fail-fast: false + # https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#standard-github-hosted-runners-for--private-repositories matrix: task: - - { title: CLD2, projectName: LanguageIdentification.CLD2, os: ubuntu-latest } - - { title: CLD3, projectName: LanguageIdentification.CLD3, os: windows-latest } - - { title: FastText, projectName: LanguageIdentification.FastText, os: macos-13 } - - { title: FastText, projectName: LanguageIdentification.FastText, os: macos-14 } + - { os: ubuntu-24.04 } + - { os: ubuntu-22.04 } + - { os: ubuntu-20.04 } + - { os: windows-2022 } + - { os: windows-2019 } + - { os: macos-13 } + - { os: macos-14 } + - { os: macos-15 } permissions: contents: read packages: write @@ -182,19 +187,19 @@ jobs: run: ls -R - name: πŸ“‚ Use local NuGet (Linux) - if: matrix.task.os == 'ubuntu-latest' + if: ${{ startsWith(matrix.task.os, 'ubuntu') }} run: | path=$(realpath "${{env.ROOT}}/local-packages") dotnet nuget add source $path - name: πŸ“‚ Use local NuGet (Windows) - if: matrix.task.os == 'windows-latest' + if: ${{ startsWith(matrix.task.os, 'windows') }} run: | $path = [System.IO.Path]::GetFullPath('${{env.ROOT}}/local-packages') dotnet nuget add source $path - name: πŸ“‚ Use local NuGet (OSX) - if: matrix.task.os == 'macos-13' || matrix.task.os == 'macos-14' + if: ${{ startsWith(matrix.task.os, 'macos') }} run: | path=$(realpath "${{env.ROOT}}/local-packages") dotnet nuget add source $path From b6b26cbab42b4bf56342000905494eaa7874b29a Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 4 Oct 2024 22:50:41 +0400 Subject: [PATCH 163/219] Update CMakeLists.txt --- src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt index a6ab3a15..44d42966 100644 --- a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt @@ -15,6 +15,8 @@ endif() if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-exceptions -fno-rtti") + set(BUILD_SHARED_LIBS OFF) + set(CMAKE_EXE_LINKER_FLAGS "-static") elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") # nop endif() From 11ccaf0efb949c34b5e3e64f6bec12b39917ccb0 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Fri, 4 Oct 2024 23:20:11 +0400 Subject: [PATCH 164/219] CLD2 + CLD3 + FT: unification of CMakeLists.txt --- .../Native/CMakeLists.txt | 43 +++++++++++-------- .../Native/CMakeLists.txt | 7 +-- .../Native/CMakeLists.txt | 4 +- 3 files changed, 30 insertions(+), 24 deletions(-) diff --git a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt index 44d42966..176bb129 100644 --- a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt @@ -1,6 +1,11 @@ +cmake_minimum_required(VERSION 3.10 FATAL_ERROR) + project(cld2) -cmake_minimum_required(VERSION 2.8 FATAL_ERROR) +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_C_STANDARD 99) +set(CMAKE_C_STANDARD_REQUIRED ON) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") @@ -14,9 +19,9 @@ elseif(CMAKE_CXX_COMPILER_ID MATCHES "MSVC") endif() if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-exceptions -fno-rtti") - set(BUILD_SHARED_LIBS OFF) - set(CMAKE_EXE_LINKER_FLAGS "-static") + # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-exceptions -fno-rtti") + # set(BUILD_SHARED_LIBS OFF) + # set(CMAKE_EXE_LINKER_FLAGS "-static") elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") # nop endif() @@ -62,28 +67,28 @@ set(CLD2_SOURCES ${PROJECT_SOURCE_DIR}/cld2/internal/tote.cc ${PROJECT_SOURCE_DIR}/cld2/internal/utf8statetable.cc ${PROJECT_SOURCE_DIR}/cld2/internal/generated_distinct_bi_0.cc - - ### Chrome (less perfect predictions) - # ${PROJECT_SOURCE_DIR}/cld2/internal/cld_generated_cjk_uni_prop_80.cc - # ${PROJECT_SOURCE_DIR}/cld2/internal/cld2_generated_cjk_compatible.cc + + ### Chrome (less perfect predictions) + # ${PROJECT_SOURCE_DIR}/cld2/internal/cld_generated_cjk_uni_prop_80.cc + # ${PROJECT_SOURCE_DIR}/cld2/internal/cld2_generated_cjk_compatible.cc # ${PROJECT_SOURCE_DIR}/cld2/internal/cld_generated_cjk_delta_bi_4.cc - # ${PROJECT_SOURCE_DIR}/cld2/internal/generated_distinct_bi_0.cc + # ${PROJECT_SOURCE_DIR}/cld2/internal/generated_distinct_bi_0.cc # ${PROJECT_SOURCE_DIR}/cld2/internal/cld2_generated_quadchrome_2.cc - # ${PROJECT_SOURCE_DIR}/cld2/internal/cld2_generated_deltaoctachrome.cc + # ${PROJECT_SOURCE_DIR}/cld2/internal/cld2_generated_deltaoctachrome.cc # ${PROJECT_SOURCE_DIR}/cld2/internal/cld2_generated_distinctoctachrome.cc - # ${PROJECT_SOURCE_DIR}/cld2/internal/cld_generated_score_quad_octa_2.cc - ### + # ${PROJECT_SOURCE_DIR}/cld2/internal/cld_generated_score_quad_octa_2.cc + ### - ### Full - ${PROJECT_SOURCE_DIR}/cld2/internal/cld_generated_cjk_uni_prop_80.cc - ${PROJECT_SOURCE_DIR}/cld2/internal/cld2_generated_cjk_compatible.cc + ### Full + ${PROJECT_SOURCE_DIR}/cld2/internal/cld_generated_cjk_uni_prop_80.cc + ${PROJECT_SOURCE_DIR}/cld2/internal/cld2_generated_cjk_compatible.cc ${PROJECT_SOURCE_DIR}/cld2/internal/cld_generated_cjk_delta_bi_32.cc - ${PROJECT_SOURCE_DIR}/cld2/internal/generated_distinct_bi_0.cc + ${PROJECT_SOURCE_DIR}/cld2/internal/generated_distinct_bi_0.cc ${PROJECT_SOURCE_DIR}/cld2/internal/cld2_generated_quad0122.cc - ${PROJECT_SOURCE_DIR}/cld2/internal/cld2_generated_deltaocta0122.cc + ${PROJECT_SOURCE_DIR}/cld2/internal/cld2_generated_deltaocta0122.cc ${PROJECT_SOURCE_DIR}/cld2/internal/cld2_generated_distinctocta0122.cc - ${PROJECT_SOURCE_DIR}/cld2/internal/cld_generated_score_quad_octa_0122.cc - ### + ${PROJECT_SOURCE_DIR}/cld2/internal/cld_generated_score_quad_octa_0122.cc + ### ${PROJECT_SOURCE_DIR}/cld2/internal/cld2_dynamic_compat.h ${PROJECT_SOURCE_DIR}/cld2/internal/cld2_dynamic_data_extractor.h diff --git a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt index ade1fecf..0d6bdaff 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt @@ -1,10 +1,11 @@ -ο»Ώproject(cld3) +ο»Ώcmake_minimum_required(VERSION 3.10 FATAL_ERROR) -cmake_minimum_required(VERSION 3.9) +project(cld3) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_EXTENSIONS OFF) +set(CMAKE_C_STANDARD 99) +set(CMAKE_C_STANDARD_REQUIRED ON) if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-exceptions -fno-rtti") diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index abae04ba..c0d803a0 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -1,6 +1,6 @@ -ο»Ώproject(fasttext) +ο»Ώcmake_minimum_required(VERSION 3.10 FATAL_ERROR) -cmake_minimum_required(VERSION 3.10 FATAL_ERROR) +project(fasttext) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) From a8a823f7daffcf4eb148f0a27d500b5c6f582c2d Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sat, 5 Oct 2024 02:03:22 +0400 Subject: [PATCH 165/219] CLD2 + FT: no static linking on Linux --- .../Native/CMakeLists.txt | 1 - .../Native/CMakeLists.txt | 66 +++++++++---------- .../Native/CMakeLists.txt | 5 +- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt index 176bb129..d759bdab 100644 --- a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt @@ -34,7 +34,6 @@ else() # nop endif() - set(CMAKE_MACOSX_RPATH 1) set(CMAKE_POSITION_INDEPENDENT_CODE ON) diff --git a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt index 0d6bdaff..5dd92877 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt @@ -8,7 +8,7 @@ set(CMAKE_C_STANDARD 99) set(CMAKE_C_STANDARD_REQUIRED ON) if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-exceptions -fno-rtti") + # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-exceptions -fno-rtti") elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") # nop endif() @@ -21,8 +21,8 @@ else() # nop endif() -add_definitions(-fPIC) # Position Independent Code -add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0) +set(CMAKE_MACOSX_RPATH 1) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) message(STATUS "CMake version: ${CMAKE_VERSION}") message(STATUS "C++ compiler: ${CMAKE_CXX_COMPILER_ID}") @@ -31,36 +31,36 @@ message(STATUS "CXX flags: ${CMAKE_CXX_FLAGS}") message(STATUS "C flags: ${CMAKE_C_FLAGS}") add_library(${PROJECT_NAME} SHARED - src/base.cc - src/embedding_feature_extractor.cc - src/embedding_network.cc - src/feature_extractor.cc - src/feature_extractor.h - src/feature_types.cc - src/fml_parser.cc - src/language_identifier_features.cc - src/lang_id_nn_params.cc - src/nnet_language_identifier.cc - src/registry.cc - src/relevant_script_feature.cc - src/sentence_features.cc - src/task_context.cc - src/task_context_params.cc - src/unicodetext.cc - src/utils.cc - src/workspace.cc - - src/script_span/generated_entities.cc - src/script_span/getonescriptspan.cc - src/script_span/getonescriptspan.h - src/script_span/getonescriptspan_test.cc - src/script_span/utf8statetable.cc - src/script_span/offsetmap.cc - src/script_span/text_processing.cc - src/script_span/text_processing.h - src/script_span/fixunicodevalue.cc - - # bindings + src/base.cc + src/embedding_feature_extractor.cc + src/embedding_network.cc + src/feature_extractor.cc + src/feature_extractor.h + src/feature_types.cc + src/fml_parser.cc + src/language_identifier_features.cc + src/lang_id_nn_params.cc + src/nnet_language_identifier.cc + src/registry.cc + src/relevant_script_feature.cc + src/sentence_features.cc + src/task_context.cc + src/task_context_params.cc + src/unicodetext.cc + src/utils.cc + src/workspace.cc + + src/script_span/generated_entities.cc + src/script_span/getonescriptspan.cc + src/script_span/getonescriptspan.h + src/script_span/getonescriptspan_test.cc + src/script_span/utf8statetable.cc + src/script_span/offsetmap.cc + src/script_span/text_processing.cc + src/script_span/text_processing.h + src/script_span/fixunicodevalue.cc + + # bindings src/binding.cc src/binding.h src/fake_protobuf.h diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index c0d803a0..6a8288cd 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -10,7 +10,7 @@ set(CMAKE_C_STANDARD_REQUIRED ON) if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-rtti -lpthread") + # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-rtti -lpthread") elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3") @@ -62,5 +62,6 @@ if (APPLE) -Wl,${CLANG_LIB_DIR}/libc++.a -Wl,${CLANG_LIB_DIR}/libc++abi.a) else() - target_link_libraries(${PROJECT_NAME} fasttext-static_pic -static-libgcc -static-libstdc++) + #target_link_libraries(${PROJECT_NAME} fasttext-static_pic -static-libgcc -static-libstdc++) + target_link_libraries(${PROJECT_NAME} fasttext-static_pic) endif() \ No newline at end of file From a99bc86b8a5876df180b917d99d2b7c059cbe7ae Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sat, 5 Oct 2024 02:24:18 +0400 Subject: [PATCH 166/219] CLD2 + CLD3 --- .../Native/CMakeLists.txt | 6 ++---- .../Native/CMakeLists.txt | 3 +++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt index d759bdab..76170722 100644 --- a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt @@ -7,10 +7,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_C_STANDARD 99) set(CMAKE_C_STANDARD_REQUIRED ON) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") -if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3") -endif() +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3") if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing") @@ -37,6 +34,7 @@ endif() set(CMAKE_MACOSX_RPATH 1) set(CMAKE_POSITION_INDEPENDENT_CODE ON) +message(STATUS "System name: ${CMAKE_SYSTEM_NAME}") message(STATUS "CMake version: ${CMAKE_VERSION}") message(STATUS "C++ compiler: ${CMAKE_CXX_COMPILER_ID}") message(STATUS "C compiler: ${CMAKE_C_COMPILER_ID}") diff --git a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt index 5dd92877..e5556f4b 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt @@ -7,6 +7,8 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_C_STANDARD 99) set(CMAKE_C_STANDARD_REQUIRED ON) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3") + if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-exceptions -fno-rtti") elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") @@ -24,6 +26,7 @@ endif() set(CMAKE_MACOSX_RPATH 1) set(CMAKE_POSITION_INDEPENDENT_CODE ON) +message(STATUS "System name: ${CMAKE_SYSTEM_NAME}") message(STATUS "CMake version: ${CMAKE_VERSION}") message(STATUS "C++ compiler: ${CMAKE_CXX_COMPILER_ID}") message(STATUS "C compiler: ${CMAKE_C_COMPILER_ID}") From e0658493f200b13d239c139eee11cb245e1ebc56 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sat, 5 Oct 2024 22:06:50 +0400 Subject: [PATCH 167/219] Update README.md --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index c9cb57d3..ae7a8177 100644 --- a/README.md +++ b/README.md @@ -89,14 +89,14 @@ functionality into their applications. ## Platform support -| Model | Linux | Windows | macOS | Blazor WASM | -| :-------- | :----------------: | :--------------: | :----: | :------------: | -| CLD2 | :white_check_mark: | :construction: | :x: | :x: | -| CLD3 | :white_check_mark: | :construction: | :x: | :x: | -| FastText | :white_check_mark: | :construction: | :x: | :x: | -| Whatlang | :white_check_mark: | :construction: | :x: | :x: | -| MediaPipe | :white_check_mark: | :construction: | :x: | :x: | -| Lingua | :white_check_mark: | :construction: | :x: | :x: | +| Model | Linux | Windows | macOS | Blazor WASM | +| :-------- | :----------------: | :----------------: | :----------------: | :------------: | +| CLD2 | :white_check_mark: | :white_check_mark: | :construction: | :x: | +| CLD3 | :white_check_mark: | :white_check_mark: | :construction: | :x: | +| FastText | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | +| Whatlang | :white_check_mark: | :construction: | :construction: | :x: | +| MediaPipe | :white_check_mark: | :construction: | :construction: | :x: | +| Lingua | :white_check_mark: | :construction: | :construction: | :x: | :white_check_mark: β€” Full support | :x: β€” No support | From 34a54381c27d4457c82e9a47efaec77d0adadfe5 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sat, 5 Oct 2024 22:08:06 +0400 Subject: [PATCH 168/219] Try to use static linking for CLD2 + CLD3 --- .../Native/CMakeLists.txt | 59 +++++++++++-------- .../Native/CMakeLists.txt | 47 +++++++++++---- .../Native/CMakeLists.txt | 5 +- 3 files changed, 72 insertions(+), 39 deletions(-) diff --git a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt index 76170722..1003bba7 100644 --- a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt @@ -7,28 +7,26 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_C_STANDARD 99) set(CMAKE_C_STANDARD_REQUIRED ON) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3") - -if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing") -elseif(CMAKE_CXX_COMPILER_ID MATCHES "MSVC") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4267") -endif() - -if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") - # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-exceptions -fno-rtti") - # set(BUILD_SHARED_LIBS OFF) - # set(CMAKE_EXE_LINKER_FLAGS "-static") -elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") - # nop -endif() - -if(CMAKE_SYSTEM_NAME STREQUAL "Windows") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++ -Wl,--start-group -lwinpthread -Wl,--end-group") -elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") - # nop -else() - # nop +if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") + if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-rtti -lpthread") + elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++ -Wl,--start-group -lwinpthread -Wl,--end-group") + endif() +elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3") + + if (CMAKE_OSX_ARCHITECTURES MATCHES "arm64") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch arm64 -target arm64-apple-macos11") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch arm64 -target arm64-apple-macos11") + elseif (CMAKE_OSX_ARCHITECTURES MATCHES "x86_64") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch x86_64 -target x86_64-apple-macos10.15") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch x86_64 -target x86_64-apple-macos10.15") + endif() endif() set(CMAKE_MACOSX_RPATH 1) @@ -128,4 +126,19 @@ add_library(objlib OBJECT ${CLD2_SOURCES}) add_library(${PROJECT_NAME} SHARED $) -set_target_properties(${PROJECT_NAME} PROPERTIES PUBLIC_HEADER "${PROJECT_SOURCE_DIR}/cld2/binding.h") \ No newline at end of file +set_target_properties(${PROJECT_NAME} PROPERTIES PUBLIC_HEADER "${PROJECT_SOURCE_DIR}/cld2/binding.h") + +if (APPLE) + if (CMAKE_OSX_ARCHITECTURES MATCHES "arm64") + set(CLANG_LIB_DIR /opt/homebrew/opt/llvm/lib/c++) + elseif (CMAKE_OSX_ARCHITECTURES MATCHES "x86_64") + set(CLANG_LIB_DIR /usr/local/opt/llvm/lib/c++) + endif() + + target_link_libraries(${PROJECT_NAME} + -nostdlib++ + -Wl,${CLANG_LIB_DIR}/libc++.a + -Wl,${CLANG_LIB_DIR}/libc++abi.a) +else() + target_link_libraries(${PROJECT_NAME} -static-libgcc -static-libstdc++) +endif() \ No newline at end of file diff --git a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt index e5556f4b..5af042c4 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt @@ -7,20 +7,26 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_C_STANDARD 99) set(CMAKE_C_STANDARD_REQUIRED ON) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3") +if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") + if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-rtti -lpthread") + elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++ -Wl,--start-group -lwinpthread -Wl,--end-group") + endif() +elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3") -if (CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") - # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-exceptions -fno-rtti") -elseif (CMAKE_CXX_COMPILER_ID MATCHES "MSVC") - # nop -endif() - -if(CMAKE_SYSTEM_NAME STREQUAL "Windows") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++ -Wl,--start-group -lwinpthread -Wl,--end-group") -elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin") - # nop -else() - # nop + if (CMAKE_OSX_ARCHITECTURES MATCHES "arm64") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch arm64 -target arm64-apple-macos11") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch arm64 -target arm64-apple-macos11") + elseif (CMAKE_OSX_ARCHITECTURES MATCHES "x86_64") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch x86_64 -target x86_64-apple-macos10.15") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -arch x86_64 -target x86_64-apple-macos10.15") + endif() endif() set(CMAKE_MACOSX_RPATH 1) @@ -71,6 +77,21 @@ add_library(${PROJECT_NAME} SHARED set_target_properties(${PROJECT_NAME} PROPERTIES PUBLIC_HEADER "src/binding.h") +if (APPLE) + if (CMAKE_OSX_ARCHITECTURES MATCHES "arm64") + set(CLANG_LIB_DIR /opt/homebrew/opt/llvm/lib/c++) + elseif (CMAKE_OSX_ARCHITECTURES MATCHES "x86_64") + set(CLANG_LIB_DIR /usr/local/opt/llvm/lib/c++) + endif() + + target_link_libraries(${PROJECT_NAME} + -nostdlib++ + -Wl,${CLANG_LIB_DIR}/libc++.a + -Wl,${CLANG_LIB_DIR}/libc++abi.a) +else() + target_link_libraries(${PROJECT_NAME} -static-libgcc -static-libstdc++) +endif() + # unit tests exec: add_executable(language_identifier_main src/language_identifier_main.cc) target_link_libraries(language_identifier_main PRIVATE cld3) diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index 6a8288cd..c0d803a0 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -10,7 +10,7 @@ set(CMAKE_C_STANDARD_REQUIRED ON) if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3") - # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-rtti -lpthread") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-rtti -lpthread") elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3") @@ -62,6 +62,5 @@ if (APPLE) -Wl,${CLANG_LIB_DIR}/libc++.a -Wl,${CLANG_LIB_DIR}/libc++abi.a) else() - #target_link_libraries(${PROJECT_NAME} fasttext-static_pic -static-libgcc -static-libstdc++) - target_link_libraries(${PROJECT_NAME} fasttext-static_pic) + target_link_libraries(${PROJECT_NAME} fasttext-static_pic -static-libgcc -static-libstdc++) endif() \ No newline at end of file From 993b6f04134eecf698a05d57acda4e08173e7f1d Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sat, 5 Oct 2024 22:19:05 +0400 Subject: [PATCH 169/219] CLD2 + CLD3: Build fixes --- src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt | 1 + src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt index 1003bba7..4a420c69 100644 --- a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt @@ -11,6 +11,7 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") if(CMAKE_SYSTEM_NAME STREQUAL "Linux") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-rtti -lpthread") elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3") diff --git a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt index 5af042c4..1383012f 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt @@ -14,7 +14,9 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-rtti -lpthread") elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++ -Wl,--start-group -lwinpthread -Wl,--end-group") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++ + -Wl,--start-group -lwinpthread -Wl,--end-group + -Wl,-allow-multiple-definition") endif() elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") From 87892228662c43f33ac0bd52578a7848379c4f0c Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sat, 5 Oct 2024 22:29:38 +0400 Subject: [PATCH 170/219] CLD2 + CLD3: fixes --- .../Native/CMakeLists.txt | 7 +++++-- .../Native/CMakeLists.txt | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt index 4a420c69..6989d790 100644 --- a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt @@ -11,11 +11,14 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") if(CMAKE_SYSTEM_NAME STREQUAL "Linux") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-rtti -lpthread") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing") elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++ -Wl,--start-group -lwinpthread -Wl,--end-group") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ + -Wl,--start-group + -lwinpthread + -Wl,--end-group") endif() elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") diff --git a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt index 1383012f..9d197643 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt @@ -12,10 +12,13 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-rtti -lpthread") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing") elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++ - -Wl,--start-group -lwinpthread -Wl,--end-group + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ + -Wl,--start-group + -lwinpthread + -Wl,--end-group -Wl,-allow-multiple-definition") endif() elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") From 71c720200696001a31f89fee98f1e6395247b710 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sat, 5 Oct 2024 22:35:47 +0400 Subject: [PATCH 171/219] CLD2 + CLD3: yet another build fixes --- .../Native/CMakeLists.txt | 6 ++---- .../Native/CMakeLists.txt | 8 +++----- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt index 6989d790..a3183c32 100644 --- a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt @@ -15,10 +15,8 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing") elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ - -Wl,--start-group - -lwinpthread - -Wl,--end-group") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--start-group -lwinpthread -Wl,--end-group") endif() elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") diff --git a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt index 9d197643..893bda4e 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt @@ -15,11 +15,9 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing") elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ - -Wl,--start-group - -lwinpthread - -Wl,--end-group - -Wl,-allow-multiple-definition") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--start-group -lwinpthread -Wl,--end-group") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-allow-multiple-definition") endif() elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") From c75a2373d8ac3ecfbea9c123c3bf6b8c7887a1ce Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sat, 5 Oct 2024 22:44:31 +0400 Subject: [PATCH 172/219] CLD2 + CLD3: -Wno-narrowing everywhere --- src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt | 1 + src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt index a3183c32..34128f9d 100644 --- a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt @@ -17,6 +17,7 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--start-group -lwinpthread -Wl,--end-group") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing") endif() elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") diff --git a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt index 893bda4e..6b6dace1 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt @@ -18,6 +18,7 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--start-group -lwinpthread -Wl,--end-group") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-allow-multiple-definition") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing") endif() elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") From 4e5965a5ce3dc0ddcbd5e52adfc0ee4c11d1af79 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sat, 5 Oct 2024 23:01:49 +0400 Subject: [PATCH 173/219] CLD2 + CLD3: try to build for MacOS --- .github/workflows/github-ci.yml | 4 ++ .../Scripts/run-build.macos.sh | 48 +++++++++++++++++++ .../Scripts/run-build.macos.sh | 48 +++++++++++++++++++ 3 files changed, 100 insertions(+) create mode 100644 src/LanguageIdentification.CLD2.Native/Scripts/run-build.macos.sh create mode 100644 src/LanguageIdentification.CLD3.Native/Scripts/run-build.macos.sh diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 5cce35cf..bfd9e3c6 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -18,6 +18,10 @@ jobs: - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: windows-latest, arch: x86_64, script: ./Scripts/run-build.ps1, artifact: fasttext.dll } - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: libfasttext.arm64.dylib } - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: macos-13, arch: x86_64, script: ./Scripts/run-build.macos.sh, artifact: libfasttext.x86_64.dylib } + - { title: CLD2, projectName: LanguageIdentification.CLD2.Native, os: macos-13, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: libcld2.arm64.dylib } + - { title: CLD2, projectName: LanguageIdentification.CLD2.Native, os: macos-14, arch: x86_64, script: ./Scripts/run-build.macos.sh, artifact: libcld2.x86_64.dylib } + - { title: CLD3, projectName: LanguageIdentification.CLD3.Native, os: macos-13, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: libcld3.arm64.dylib } + - { title: CLD3, projectName: LanguageIdentification.CLD3.Native, os: macos-14, arch: x86_64, script: ./Scripts/run-build.macos.sh, artifact: libcld3.x86_64.dylib } permissions: contents: read packages: write diff --git a/src/LanguageIdentification.CLD2.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.CLD2.Native/Scripts/run-build.macos.sh new file mode 100644 index 00000000..d8c39161 --- /dev/null +++ b/src/LanguageIdentification.CLD2.Native/Scripts/run-build.macos.sh @@ -0,0 +1,48 @@ +ο»Ώ#!/bin/bash +set -e + +if [ -z "$1" ]; then + echo "Error: No architecture specified." + echo "Usage: $0 " + exit 1 +fi + +ARCH=$1 + +if [[ "$ARCH" != "x86_64" && "$ARCH" != "arm64" ]]; then + echo "Error: Invalid architecture specified. Use 'x86_64' or 'arm64'." + exit 1 +fi + +echo "Hello world $ARCH"; + +brew install llvm + +workspace="build_temp" + +mkdir "$workspace" -p +cp -a ../../third_party/cld3/. $workspace/. +cp -a Native/. $workspace + +ls -R . + +cd "$workspace" + +zx ../Native/monkey-patch.mjs + +mkdir build +cd build + +echo "Build for MacOS on $ARCH"; +rm -rf * +cmake -DCMAKE_OSX_ARCHITECTURES=$ARCH .. +make -j $(sysctl -n hw.logicalcpu) + +ls -R + +otool -L libcld2.dylib +cp libcld2.dylib ../../libcld2.$ARCH.dylib + +# Clean up +rm -rf "$workspace" +echo "Goodbye world"; diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.CLD3.Native/Scripts/run-build.macos.sh new file mode 100644 index 00000000..0639b342 --- /dev/null +++ b/src/LanguageIdentification.CLD3.Native/Scripts/run-build.macos.sh @@ -0,0 +1,48 @@ +ο»Ώ#!/bin/bash +set -e + +if [ -z "$1" ]; then + echo "Error: No architecture specified." + echo "Usage: $0 " + exit 1 +fi + +ARCH=$1 + +if [[ "$ARCH" != "x86_64" && "$ARCH" != "arm64" ]]; then + echo "Error: Invalid architecture specified. Use 'x86_64' or 'arm64'." + exit 1 +fi + +echo "Hello world $ARCH"; + +brew install llvm + +workspace="build_temp" + +mkdir "$workspace" -p +cp -a ../../third_party/cld3/. $workspace/. +cp -a Native/. $workspace + +ls -R . + +cd "$workspace" + +zx ../Native/monkey-patch.mjs + +mkdir build +cd build + +echo "Build for MacOS on $ARCH"; +rm -rf * +cmake -DCMAKE_OSX_ARCHITECTURES=$ARCH .. +make -j $(sysctl -n hw.logicalcpu) + +ls -R + +otool -L libcld3.dylib +cp libcld3.dylib ../../libcld3.$ARCH.dylib + +# Clean up +rm -rf "$workspace" +echo "Goodbye world"; From 83ed53ab83f830f8e3227db59c88a192e88f31b2 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sat, 5 Oct 2024 23:33:45 +0400 Subject: [PATCH 174/219] CLD2 + CLD3: fixe MacOS build --- .../Scripts/run-build.macos.sh | 6 +----- .../Scripts/run-build.macos.sh | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/LanguageIdentification.CLD2.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.CLD2.Native/Scripts/run-build.macos.sh index d8c39161..20ea9ce3 100644 --- a/src/LanguageIdentification.CLD2.Native/Scripts/run-build.macos.sh +++ b/src/LanguageIdentification.CLD2.Native/Scripts/run-build.macos.sh @@ -16,20 +16,16 @@ fi echo "Hello world $ARCH"; -brew install llvm - workspace="build_temp" mkdir "$workspace" -p -cp -a ../../third_party/cld3/. $workspace/. +cp -a ../../third_party/cld2/. $workspace/cld2 cp -a Native/. $workspace ls -R . cd "$workspace" -zx ../Native/monkey-patch.mjs - mkdir build cd build diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.CLD3.Native/Scripts/run-build.macos.sh index 0639b342..fab87418 100644 --- a/src/LanguageIdentification.CLD3.Native/Scripts/run-build.macos.sh +++ b/src/LanguageIdentification.CLD3.Native/Scripts/run-build.macos.sh @@ -16,7 +16,7 @@ fi echo "Hello world $ARCH"; -brew install llvm +npm install -g zx workspace="build_temp" From 5b318799e491889a68f47c2b6bf29e9819964958 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sat, 5 Oct 2024 23:46:25 +0400 Subject: [PATCH 175/219] CLD2 + CLD3: fix stupid mistakes --- .github/workflows/github-ci.yml | 8 ++++---- .../Scripts/run-build.macos.sh | 2 ++ .../Scripts/run-build.macos.sh | 1 + 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index bfd9e3c6..9d0ffa13 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -18,10 +18,10 @@ jobs: - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: windows-latest, arch: x86_64, script: ./Scripts/run-build.ps1, artifact: fasttext.dll } - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: libfasttext.arm64.dylib } - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: macos-13, arch: x86_64, script: ./Scripts/run-build.macos.sh, artifact: libfasttext.x86_64.dylib } - - { title: CLD2, projectName: LanguageIdentification.CLD2.Native, os: macos-13, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: libcld2.arm64.dylib } - - { title: CLD2, projectName: LanguageIdentification.CLD2.Native, os: macos-14, arch: x86_64, script: ./Scripts/run-build.macos.sh, artifact: libcld2.x86_64.dylib } - - { title: CLD3, projectName: LanguageIdentification.CLD3.Native, os: macos-13, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: libcld3.arm64.dylib } - - { title: CLD3, projectName: LanguageIdentification.CLD3.Native, os: macos-14, arch: x86_64, script: ./Scripts/run-build.macos.sh, artifact: libcld3.x86_64.dylib } + - { title: CLD2, projectName: LanguageIdentification.CLD2.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: libcld2.arm64.dylib } + - { title: CLD2, projectName: LanguageIdentification.CLD2.Native, os: macos-13, arch: x86_64, script: ./Scripts/run-build.macos.sh, artifact: libcld2.x86_64.dylib } + - { title: CLD3, projectName: LanguageIdentification.CLD3.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: libcld3.arm64.dylib } + - { title: CLD3, projectName: LanguageIdentification.CLD3.Native, os: macos-13, arch: x86_64, script: ./Scripts/run-build.macos.sh, artifact: libcld3.x86_64.dylib } permissions: contents: read packages: write diff --git a/src/LanguageIdentification.CLD2.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.CLD2.Native/Scripts/run-build.macos.sh index 20ea9ce3..dedf7bcb 100644 --- a/src/LanguageIdentification.CLD2.Native/Scripts/run-build.macos.sh +++ b/src/LanguageIdentification.CLD2.Native/Scripts/run-build.macos.sh @@ -16,6 +16,8 @@ fi echo "Hello world $ARCH"; +brew install llvm + workspace="build_temp" mkdir "$workspace" -p diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/run-build.macos.sh b/src/LanguageIdentification.CLD3.Native/Scripts/run-build.macos.sh index fab87418..cd169515 100644 --- a/src/LanguageIdentification.CLD3.Native/Scripts/run-build.macos.sh +++ b/src/LanguageIdentification.CLD3.Native/Scripts/run-build.macos.sh @@ -16,6 +16,7 @@ fi echo "Hello world $ARCH"; +brew install llvm npm install -g zx workspace="build_temp" From ecacdcf9f4761a4e24f5d9d8ff035997829e493c Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sat, 5 Oct 2024 23:56:30 +0400 Subject: [PATCH 176/219] Update CMakeLists.txt --- src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt index 34128f9d..8b3acfd2 100644 --- a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt @@ -22,6 +22,7 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing") if (CMAKE_OSX_ARCHITECTURES MATCHES "arm64") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch arm64 -target arm64-apple-macos11") From af25730cc0d9cb6d67f8ecd337a38891e213c6b0 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 00:15:00 +0400 Subject: [PATCH 177/219] CLD2 + CLD3 + FT: try to use '-static' --- .../Native/CMakeLists.txt | 5 +++-- .../Native/CMakeLists.txt | 6 ++++-- .../Native/CMakeLists.txt | 5 +++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt index 8b3acfd2..25de70bb 100644 --- a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt @@ -11,11 +11,12 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") if(CMAKE_SYSTEM_NAME STREQUAL "Linux") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-rtti -lpthread") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -static") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti -lpthread") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing") elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -static") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--start-group -lwinpthread -Wl,--end-group") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing") endif() diff --git a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt index 6b6dace1..bc0d9698 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt @@ -11,11 +11,12 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") if(CMAKE_SYSTEM_NAME STREQUAL "Linux") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-rtti -lpthread") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -static") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti -lpthread") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing") elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -static") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--start-group -lwinpthread -Wl,--end-group") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-allow-multiple-definition") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing") @@ -23,6 +24,7 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing") if (CMAKE_OSX_ARCHITECTURES MATCHES "arm64") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -arch arm64 -target arm64-apple-macos11") diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index c0d803a0..55e6e239 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -10,7 +10,8 @@ set(CMAKE_C_STANDARD_REQUIRED ON) if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -fno-rtti -lpthread") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -static") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti -lpthread") elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3") @@ -62,5 +63,5 @@ if (APPLE) -Wl,${CLANG_LIB_DIR}/libc++.a -Wl,${CLANG_LIB_DIR}/libc++abi.a) else() - target_link_libraries(${PROJECT_NAME} fasttext-static_pic -static-libgcc -static-libstdc++) + target_link_libraries(${PROJECT_NAME} fasttext-static_pic -static-libgcc -static-libstdc++ -static) endif() \ No newline at end of file From a6504cee5935000aa0d8ddc9cd3704d8a83ad58b Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 00:40:10 +0400 Subject: [PATCH 178/219] Using debian:bullseye-slim --- src/LanguageIdentification.CLD2.Native/Dockerfile | 2 +- src/LanguageIdentification.CLD3.Native/Dockerfile | 2 +- src/LanguageIdentification.FastText.Native/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/LanguageIdentification.CLD2.Native/Dockerfile b/src/LanguageIdentification.CLD2.Native/Dockerfile index 3e7e769a..ec95e24f 100644 --- a/src/LanguageIdentification.CLD2.Native/Dockerfile +++ b/src/LanguageIdentification.CLD2.Native/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build +FROM debian:bullseye-slim AS build ARG BUILD_CONFIGURATION=Release WORKDIR /repo diff --git a/src/LanguageIdentification.CLD3.Native/Dockerfile b/src/LanguageIdentification.CLD3.Native/Dockerfile index a575ae8f..fb9b67f7 100644 --- a/src/LanguageIdentification.CLD3.Native/Dockerfile +++ b/src/LanguageIdentification.CLD3.Native/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build +FROM debian:bullseye-slim AS build ARG BUILD_CONFIGURATION=Release WORKDIR /repo diff --git a/src/LanguageIdentification.FastText.Native/Dockerfile b/src/LanguageIdentification.FastText.Native/Dockerfile index 2948215c..1e7b2899 100644 --- a/src/LanguageIdentification.FastText.Native/Dockerfile +++ b/src/LanguageIdentification.FastText.Native/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build +FROM debian:bullseye-slim AS build ARG BUILD_CONFIGURATION=Release WORKDIR /repo From 041a3a3502e38c6e7301215f886c6502eb8ebabb Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 00:53:30 +0400 Subject: [PATCH 179/219] CLD2 + CLD3 + FT: no static libs for Linux --- .../Native/CMakeLists.txt | 9 ++++++--- .../Native/CMakeLists.txt | 9 ++++++--- .../Native/CMakeLists.txt | 8 ++++++-- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt index 25de70bb..d416d77b 100644 --- a/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD2.Native/Native/CMakeLists.txt @@ -11,12 +11,11 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") if(CMAKE_SYSTEM_NAME STREQUAL "Linux") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -static") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti -lpthread") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing") elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -static") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--start-group -lwinpthread -Wl,--end-group") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing") endif() @@ -145,5 +144,9 @@ if (APPLE) -Wl,${CLANG_LIB_DIR}/libc++.a -Wl,${CLANG_LIB_DIR}/libc++abi.a) else() - target_link_libraries(${PROJECT_NAME} -static-libgcc -static-libstdc++) + if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + # nop + elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") + target_link_libraries(${PROJECT_NAME} -static-libgcc -static-libstdc++) + endif() endif() \ No newline at end of file diff --git a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt index bc0d9698..fd1152e6 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt @@ -11,12 +11,11 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") if(CMAKE_SYSTEM_NAME STREQUAL "Linux") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -static") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti -lpthread") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing") elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -static") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--start-group -lwinpthread -Wl,--end-group") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-allow-multiple-definition") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing") @@ -95,7 +94,11 @@ if (APPLE) -Wl,${CLANG_LIB_DIR}/libc++.a -Wl,${CLANG_LIB_DIR}/libc++abi.a) else() - target_link_libraries(${PROJECT_NAME} -static-libgcc -static-libstdc++) + if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + # nop + elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") + target_link_libraries(${PROJECT_NAME} -static-libgcc -static-libstdc++) + endif() endif() # unit tests exec: diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index 55e6e239..d468c8d7 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -10,7 +10,6 @@ set(CMAKE_C_STANDARD_REQUIRED ON) if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -pthread -funroll-loops -O3") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libgcc -static-libstdc++ -static") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti -lpthread") elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99 -pthread -funroll-loops -O3") @@ -63,5 +62,10 @@ if (APPLE) -Wl,${CLANG_LIB_DIR}/libc++.a -Wl,${CLANG_LIB_DIR}/libc++abi.a) else() - target_link_libraries(${PROJECT_NAME} fasttext-static_pic -static-libgcc -static-libstdc++ -static) + if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_link_libraries(${PROJECT_NAME} fasttext-static_pic) + elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") + target_link_libraries(${PROJECT_NAME} fasttext-static_pic -static-libgcc -static-libstdc++) + endif() + endif() \ No newline at end of file From 289c45e81f2e8ad54f190b91c272d5653b064206 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 01:07:58 +0400 Subject: [PATCH 180/219] Update setup-build.sh --- src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh index ad06f438..06c97e7c 100644 --- a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh +++ b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh @@ -10,8 +10,9 @@ apt-get update && apt-get install -y ca-certificates curl gnupg && \ apt -y update apt -y install cmake apt -y install g++ -apt -y install nodejs apt -y install gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 +apt -y install nodejs +apt -y install npm npm install -g zx From 22937037da367abf6140c64722cef5dc114d0d7a Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 01:17:11 +0400 Subject: [PATCH 181/219] debian:bullseye-slim -> debian:bookworm-slim --- src/LanguageIdentification.CLD2.Native/Dockerfile | 2 +- src/LanguageIdentification.CLD3.Native/Dockerfile | 2 +- src/LanguageIdentification.FastText.Native/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/LanguageIdentification.CLD2.Native/Dockerfile b/src/LanguageIdentification.CLD2.Native/Dockerfile index ec95e24f..bfd27d34 100644 --- a/src/LanguageIdentification.CLD2.Native/Dockerfile +++ b/src/LanguageIdentification.CLD2.Native/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:bullseye-slim AS build +FROM debian:bookworm-slim AS build ARG BUILD_CONFIGURATION=Release WORKDIR /repo diff --git a/src/LanguageIdentification.CLD3.Native/Dockerfile b/src/LanguageIdentification.CLD3.Native/Dockerfile index fb9b67f7..bed88e04 100644 --- a/src/LanguageIdentification.CLD3.Native/Dockerfile +++ b/src/LanguageIdentification.CLD3.Native/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:bullseye-slim AS build +FROM debian:bookworm-slim AS build ARG BUILD_CONFIGURATION=Release WORKDIR /repo diff --git a/src/LanguageIdentification.FastText.Native/Dockerfile b/src/LanguageIdentification.FastText.Native/Dockerfile index 1e7b2899..7ca1c476 100644 --- a/src/LanguageIdentification.FastText.Native/Dockerfile +++ b/src/LanguageIdentification.FastText.Native/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:bullseye-slim AS build +FROM debian:bookworm-slim AS build ARG BUILD_CONFIGURATION=Release WORKDIR /repo From b59fbca3cf7e877e738620c050b5855a7017ea3e Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 01:22:51 +0400 Subject: [PATCH 182/219] Update setup-build.sh --- .../Scripts/setup-build.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh index 06c97e7c..5d6972fc 100644 --- a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh +++ b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh @@ -3,16 +3,19 @@ set -e echo "Installing build packages"; -apt-get update && apt-get install -y ca-certificates curl gnupg && \ - curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ - echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list + apt -y update apt -y install cmake apt -y install g++ apt -y install gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 + +apt -y install curl +curl -sL https://deb.nodesource.com/setup_22.x | sudo -E bash - apt -y install nodejs apt -y install npm +node --version + npm install -g zx From bc8ce6020ba850c4184a30e733bc07b460fae825 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 01:28:07 +0400 Subject: [PATCH 183/219] Update setup-build.sh --- src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh index 5d6972fc..4e1939cc 100644 --- a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh +++ b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh @@ -11,7 +11,7 @@ apt -y install g++ apt -y install gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 apt -y install curl -curl -sL https://deb.nodesource.com/setup_22.x | sudo -E bash - +curl -sL https://deb.nodesource.com/setup_22.x | bash - apt -y install nodejs apt -y install npm From 9c034fbf6724e9a49cd73558a649c037176c1ec5 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 01:42:35 +0400 Subject: [PATCH 184/219] Update setup-build.sh --- src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh index 4e1939cc..22080d45 100644 --- a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh +++ b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh @@ -11,7 +11,7 @@ apt -y install g++ apt -y install gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 apt -y install curl -curl -sL https://deb.nodesource.com/setup_22.x | bash - +curl -sL https://deb.nodesource.com/setup_20.x | bash - apt -y install nodejs apt -y install npm From 62395945ca37ed0cda39df8436793af00f13348f Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 01:47:21 +0400 Subject: [PATCH 185/219] CLD2 + CLD3: turn on MacOS support in packages --- .../LanguageIdentification.CLD2.Native.csproj | 20 +++++++++++++++++++ .../CLD2Detector.cs | 2 ++ .../LanguageIdentification.CLD3.Native.csproj | 20 +++++++++++++++++++ .../CLD3Detector.cs | 2 ++ 4 files changed, 44 insertions(+) diff --git a/src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj b/src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj index 611ece97..00801dc0 100644 --- a/src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj +++ b/src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj @@ -72,6 +72,26 @@ + + + PreserveNewest + true + runtimes/osx-x64/native/libcld2.dylib + true + false + + + + + + PreserveNewest + true + runtimes/osx-arm64/native/libcld2.dylib + true + false + + + diff --git a/src/LanguageIdentification.CLD2/CLD2Detector.cs b/src/LanguageIdentification.CLD2/CLD2Detector.cs index 1b3b5011..881451a3 100644 --- a/src/LanguageIdentification.CLD2/CLD2Detector.cs +++ b/src/LanguageIdentification.CLD2/CLD2Detector.cs @@ -27,6 +27,8 @@ public static bool IsSupported() { Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => true, Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => true, + Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, _ => false, }; } diff --git a/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj b/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj index f3cf0428..65e129d3 100644 --- a/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj +++ b/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj @@ -72,6 +72,26 @@ + + + PreserveNewest + true + runtimes/osx-x64/native/libcld3.dylib + true + false + + + + + + PreserveNewest + true + runtimes/osx-arm64/native/libcld3.dylib + true + false + + + diff --git a/src/LanguageIdentification.CLD3/CLD3Detector.cs b/src/LanguageIdentification.CLD3/CLD3Detector.cs index b8984117..a22dc05f 100644 --- a/src/LanguageIdentification.CLD3/CLD3Detector.cs +++ b/src/LanguageIdentification.CLD3/CLD3Detector.cs @@ -35,6 +35,8 @@ public static bool IsSupported() { Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => true, Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => true, + Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, _ => false, }; } From 98330a34478385d4125274fe2bae96532728f55a Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 01:54:31 +0400 Subject: [PATCH 186/219] Update setup-build.sh --- .../Scripts/setup-build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh index 22080d45..f6f4188d 100644 --- a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh +++ b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh @@ -3,15 +3,15 @@ set -e echo "Installing build packages"; - +apt -y update +apt -y install curl +curl -sL https://deb.nodesource.com/setup_20.x | bash - apt -y update apt -y install cmake apt -y install g++ apt -y install gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 -apt -y install curl -curl -sL https://deb.nodesource.com/setup_20.x | bash - apt -y install nodejs apt -y install npm From c7fb8f197627d18998f05ebbb07f7e9df0a9d30a Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 01:59:55 +0400 Subject: [PATCH 187/219] Update setup-build.sh --- src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh index f6f4188d..eaf96fc8 100644 --- a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh +++ b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh @@ -4,7 +4,7 @@ set -e echo "Installing build packages"; apt -y update -apt -y install curl +apt -y install curl dirmngr apt-transport-https lsb-release ca-certificates curl -sL https://deb.nodesource.com/setup_20.x | bash - apt -y update From 073a4984ce75a5026362171f6a7d7e374f012590 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 02:01:20 +0400 Subject: [PATCH 188/219] Update setup-build.sh --- src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh index eaf96fc8..645d83ab 100644 --- a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh +++ b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh @@ -5,7 +5,7 @@ echo "Installing build packages"; apt -y update apt -y install curl dirmngr apt-transport-https lsb-release ca-certificates -curl -sL https://deb.nodesource.com/setup_20.x | bash - +curl -sL https://deb.nodesource.com/setup_22.x | bash - apt -y update apt -y install cmake From 12a8cfff1bb990c1f848c4e81c18696a15c1181a Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 02:06:00 +0400 Subject: [PATCH 189/219] Update setup-build.sh --- src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh index 645d83ab..e0be0a7b 100644 --- a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh +++ b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh @@ -5,7 +5,7 @@ echo "Installing build packages"; apt -y update apt -y install curl dirmngr apt-transport-https lsb-release ca-certificates -curl -sL https://deb.nodesource.com/setup_22.x | bash - +curl -sL https://deb.nodesource.com/setup_18.x | bash - apt -y update apt -y install cmake From 2500b7a44416e4e0e5738330b48a962c10fbdc31 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 02:17:06 +0400 Subject: [PATCH 190/219] Update setup-build.sh --- .../Scripts/setup-build.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh index e0be0a7b..3f2589bd 100644 --- a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh +++ b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh @@ -5,17 +5,17 @@ echo "Installing build packages"; apt -y update apt -y install curl dirmngr apt-transport-https lsb-release ca-certificates -curl -sL https://deb.nodesource.com/setup_18.x | bash - +curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash +source ~/.bashrc +nvm install 22 apt -y update apt -y install cmake apt -y install g++ apt -y install gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 -apt -y install nodejs -apt -y install npm - node --version +npm --version npm install -g zx From e28d3115be9cd8adabbeb4a614d9017694f3f5a9 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 02:28:34 +0400 Subject: [PATCH 191/219] Update run-build.sh --- src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh b/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh index 513a9e73..3dc8b33a 100644 --- a/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh +++ b/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh @@ -13,7 +13,7 @@ ls -R . cd "$workspace" -zx ../Native/monkey-patch.mjs +npx zx ../Native/monkey-patch.mjs mkdir build cd build From 43b78d893f7ef8e266a591686054b92330a4d3ea Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 22:17:43 +0400 Subject: [PATCH 192/219] Update setup-build.sh --- .../Scripts/setup-build.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh index 3f2589bd..969c2b99 100644 --- a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh +++ b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh @@ -8,14 +8,13 @@ apt -y install curl dirmngr apt-transport-https lsb-release ca-certificates curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash source ~/.bashrc nvm install 22 +npm install -g zx +source ~/.bashrc apt -y update apt -y install cmake apt -y install g++ apt -y install gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 -node --version -npm --version -npm install -g zx From f7750bd7334908bb07d12bdc140bac80f2efd221 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 22:39:01 +0400 Subject: [PATCH 193/219] Move zx to run-build --- src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh | 1 + src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh b/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh index 3dc8b33a..a70ee71d 100644 --- a/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh +++ b/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh @@ -13,6 +13,7 @@ ls -R . cd "$workspace" +npm install -g zx npx zx ../Native/monkey-patch.mjs mkdir build diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh index 969c2b99..7f749119 100644 --- a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh +++ b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh @@ -8,7 +8,6 @@ apt -y install curl dirmngr apt-transport-https lsb-release ca-certificates curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash source ~/.bashrc nvm install 22 -npm install -g zx source ~/.bashrc apt -y update From 0c5af620325d7e49698d44afc744deaa26aab45c Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 22:43:57 +0400 Subject: [PATCH 194/219] Update Dockerfile --- src/LanguageIdentification.CLD3.Native/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/LanguageIdentification.CLD3.Native/Dockerfile b/src/LanguageIdentification.CLD3.Native/Dockerfile index bed88e04..b4655fdc 100644 --- a/src/LanguageIdentification.CLD3.Native/Dockerfile +++ b/src/LanguageIdentification.CLD3.Native/Dockerfile @@ -11,6 +11,8 @@ WORKDIR /repo/src/LanguageIdentification.CLD3.Native RUN bash ./Scripts/setup-runtime.sh RUN bash ./Scripts/setup-build.sh +ENV PATH="/root/.nvm/versions/node/v22/bin:${PATH}" + COPY . . ENTRYPOINT ["bash", "./Scripts/run-build.sh"] From 9eacefeafb88b8d8be51ce232cea8612c6c720bf Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 23:01:59 +0400 Subject: [PATCH 195/219] Fixes --- src/LanguageIdentification.CLD2.Native/Dockerfile | 1 - src/LanguageIdentification.CLD3.Native/Dockerfile | 1 - src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh | 3 +-- .../Scripts/setup-build.sh | 5 +++++ src/LanguageIdentification.FastText.Native/Dockerfile | 1 - 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/LanguageIdentification.CLD2.Native/Dockerfile b/src/LanguageIdentification.CLD2.Native/Dockerfile index bfd27d34..18c40659 100644 --- a/src/LanguageIdentification.CLD2.Native/Dockerfile +++ b/src/LanguageIdentification.CLD2.Native/Dockerfile @@ -1,5 +1,4 @@ FROM debian:bookworm-slim AS build -ARG BUILD_CONFIGURATION=Release WORKDIR /repo COPY ["src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj", "src/LanguageIdentification.CLD2.Native/"] diff --git a/src/LanguageIdentification.CLD3.Native/Dockerfile b/src/LanguageIdentification.CLD3.Native/Dockerfile index b4655fdc..08cbff32 100644 --- a/src/LanguageIdentification.CLD3.Native/Dockerfile +++ b/src/LanguageIdentification.CLD3.Native/Dockerfile @@ -1,5 +1,4 @@ FROM debian:bookworm-slim AS build -ARG BUILD_CONFIGURATION=Release WORKDIR /repo COPY ["src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj", "src/LanguageIdentification.CLD3.Native/"] diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh b/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh index a70ee71d..513a9e73 100644 --- a/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh +++ b/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh @@ -13,8 +13,7 @@ ls -R . cd "$workspace" -npm install -g zx -npx zx ../Native/monkey-patch.mjs +zx ../Native/monkey-patch.mjs mkdir build cd build diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh index 7f749119..824aa97e 100644 --- a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh +++ b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh @@ -8,6 +8,11 @@ apt -y install curl dirmngr apt-transport-https lsb-release ca-certificates curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash source ~/.bashrc nvm install 22 +nvm use 22 +source ~/.bashrc +npm install -g zx +node --version +npx zx --version source ~/.bashrc apt -y update diff --git a/src/LanguageIdentification.FastText.Native/Dockerfile b/src/LanguageIdentification.FastText.Native/Dockerfile index 7ca1c476..3de53667 100644 --- a/src/LanguageIdentification.FastText.Native/Dockerfile +++ b/src/LanguageIdentification.FastText.Native/Dockerfile @@ -1,5 +1,4 @@ FROM debian:bookworm-slim AS build -ARG BUILD_CONFIGURATION=Release WORKDIR /repo COPY ["src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj", "src/LanguageIdentification.FastText.Native/"] From 20288be0bafdf0e91748d78d42b981d1f70880e5 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 23:06:50 +0400 Subject: [PATCH 196/219] Install NPM in run-build.sh --- .../Scripts/run-build.sh | 5 +++++ .../Scripts/setup-build.sh | 11 ----------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh b/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh index 513a9e73..ce38acfd 100644 --- a/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh +++ b/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh @@ -13,6 +13,11 @@ ls -R . cd "$workspace" +curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash +nvm install 22 +nvm use 22 +npm install -g zx + zx ../Native/monkey-patch.mjs mkdir build diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh index 824aa97e..324f3894 100644 --- a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh +++ b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh @@ -5,17 +5,6 @@ echo "Installing build packages"; apt -y update apt -y install curl dirmngr apt-transport-https lsb-release ca-certificates -curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash -source ~/.bashrc -nvm install 22 -nvm use 22 -source ~/.bashrc -npm install -g zx -node --version -npx zx --version -source ~/.bashrc - -apt -y update apt -y install cmake apt -y install g++ apt -y install gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 From fe7e9acfed86dc0ab24089b20f480c057dbb2b23 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 23:13:48 +0400 Subject: [PATCH 197/219] Update Dockerfile --- src/LanguageIdentification.CLD3.Native/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/LanguageIdentification.CLD3.Native/Dockerfile b/src/LanguageIdentification.CLD3.Native/Dockerfile index 08cbff32..f1313241 100644 --- a/src/LanguageIdentification.CLD3.Native/Dockerfile +++ b/src/LanguageIdentification.CLD3.Native/Dockerfile @@ -10,8 +10,6 @@ WORKDIR /repo/src/LanguageIdentification.CLD3.Native RUN bash ./Scripts/setup-runtime.sh RUN bash ./Scripts/setup-build.sh -ENV PATH="/root/.nvm/versions/node/v22/bin:${PATH}" - COPY . . ENTRYPOINT ["bash", "./Scripts/run-build.sh"] From 8348b293d19644015c10feafb3cc16c0719d0386 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 23:24:57 +0400 Subject: [PATCH 198/219] Update run-build.sh --- src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh b/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh index ce38acfd..ef0e887b 100644 --- a/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh +++ b/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh @@ -14,6 +14,8 @@ ls -R . cd "$workspace" curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash +export NVM_DIR="$HOME/.nvm" +[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" nvm install 22 nvm use 22 npm install -g zx From 2b640801c67741c21174090955fccd8164bb829d Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 23:39:09 +0400 Subject: [PATCH 199/219] Try to fix build for MacOS --- .github/workflows/github-ci.yml | 6 +- .../Native/binding.cc | 4 +- .../Native/src/binding.cc | 75 ++++++++++--------- 3 files changed, 44 insertions(+), 41 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 9d0ffa13..897ea836 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -16,12 +16,12 @@ jobs: matrix: task: - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: windows-latest, arch: x86_64, script: ./Scripts/run-build.ps1, artifact: fasttext.dll } - - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: libfasttext.arm64.dylib } - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: macos-13, arch: x86_64, script: ./Scripts/run-build.macos.sh, artifact: libfasttext.x86_64.dylib } - - { title: CLD2, projectName: LanguageIdentification.CLD2.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: libcld2.arm64.dylib } + - { title: FastText, projectName: LanguageIdentification.FastText.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: libfasttext.arm64.dylib } - { title: CLD2, projectName: LanguageIdentification.CLD2.Native, os: macos-13, arch: x86_64, script: ./Scripts/run-build.macos.sh, artifact: libcld2.x86_64.dylib } - - { title: CLD3, projectName: LanguageIdentification.CLD3.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: libcld3.arm64.dylib } + - { title: CLD2, projectName: LanguageIdentification.CLD2.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: libcld2.arm64.dylib } - { title: CLD3, projectName: LanguageIdentification.CLD3.Native, os: macos-13, arch: x86_64, script: ./Scripts/run-build.macos.sh, artifact: libcld3.x86_64.dylib } + - { title: CLD3, projectName: LanguageIdentification.CLD3.Native, os: macos-14, arch: arm64, script: ./Scripts/run-build.macos.sh, artifact: libcld3.arm64.dylib } permissions: contents: read packages: write diff --git a/src/LanguageIdentification.CLD2.Native/Native/binding.cc b/src/LanguageIdentification.CLD2.Native/Native/binding.cc index 62153eb6..ee3bafe5 100644 --- a/src/LanguageIdentification.CLD2.Native/Native/binding.cc +++ b/src/LanguageIdentification.CLD2.Native/Native/binding.cc @@ -9,7 +9,7 @@ extern "C" { - PredictionResult* PredictLanguage(char *text, int* resultCount) + EXPORT PredictionResult* PredictLanguage(char *text, int* resultCount) { int textLength = strlen(text); @@ -75,7 +75,7 @@ extern "C" return result; } - void FreeResults(PredictionResult* results, int count) + EXPORT void FreeResults(PredictionResult* results, int count) { for (int i = 0; i < count; ++i) { free((void*)results[i].language); diff --git a/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc b/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc index ef91a907..860d1ec7 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc +++ b/src/LanguageIdentification.CLD3.Native/Native/src/binding.cc @@ -3,45 +3,48 @@ using namespace chrome_lang_id; -void* create_cld3(int minNumBytes, int maxNumBytes) { - return new NNetLanguageIdentifier(minNumBytes, maxNumBytes); -} +extern "C" { + EXPORT void* create_cld3(int minNumBytes, int maxNumBytes) { + return new NNetLanguageIdentifier(minNumBytes, maxNumBytes); + } -void destroy_cld3(void* identifier) { - delete static_cast(identifier); -} + EXPORT void destroy_cld3(void* identifier) { + delete static_cast(identifier); + } -PredictionResult* cld3_find_language(void* identifier, const char* text, int* resultCount) { - NNetLanguageIdentifier* nativeIdentifier = static_cast(identifier); - auto nativeResult = nativeIdentifier->FindLanguage(text); - - *resultCount = 1; - PredictionResult* result = new PredictionResult[*resultCount]; - result[0].language = strdup(nativeResult.language.c_str()); - result[0].probability = nativeResult.probability; - result[0].is_reliable = nativeResult.is_reliable; - result[0].proportion = nativeResult.proportion; - return result; -} + EXPORT PredictionResult* cld3_find_language(void* identifier, const char* text, int* resultCount) { + NNetLanguageIdentifier* nativeIdentifier = static_cast(identifier); + auto nativeResult = nativeIdentifier->FindLanguage(text); + + *resultCount = 1; + PredictionResult* result = new PredictionResult[*resultCount]; + result[0].language = strdup(nativeResult.language.c_str()); + result[0].probability = nativeResult.probability; + result[0].is_reliable = nativeResult.is_reliable; + result[0].proportion = nativeResult.proportion; + return result; + } -PredictionResult* cld3_find_languages(void* identifier, const char* text, int numLangs, int* resultCount) { - NNetLanguageIdentifier* nativeIdentifier = static_cast(identifier); - auto nativeResults = nativeIdentifier->FindTopNMostFreqLangs(text, numLangs); - - *resultCount = static_cast(nativeResults.size()); - PredictionResult* result = new PredictionResult[*resultCount]; - for (int i = 0; i < *resultCount; ++i) { - result[i].language = strdup(nativeResults[i].language.c_str()); - result[i].probability = nativeResults[i].probability; - result[i].is_reliable = nativeResults[i].is_reliable; - result[i].proportion = nativeResults[i].proportion; + EXPORT PredictionResult* cld3_find_languages(void* identifier, const char* text, int numLangs, int* resultCount) { + NNetLanguageIdentifier* nativeIdentifier = static_cast(identifier); + auto nativeResults = nativeIdentifier->FindTopNMostFreqLangs(text, numLangs); + + *resultCount = static_cast(nativeResults.size()); + PredictionResult* result = new PredictionResult[*resultCount]; + for (int i = 0; i < *resultCount; ++i) { + result[i].language = strdup(nativeResults[i].language.c_str()); + result[i].probability = nativeResults[i].probability; + result[i].is_reliable = nativeResults[i].is_reliable; + result[i].proportion = nativeResults[i].proportion; + } + return result; } - return result; -} -void cld3_destroy_prediction_result(PredictionResult* results, int count) { - for (int i = 0; i < count; ++i) { - free((void*)results[i].language); + EXPORT void cld3_destroy_prediction_result(PredictionResult* results, int count) { + for (int i = 0; i < count; ++i) { + free((void*)results[i].language); + } + delete[] results; } - delete[] results; -} \ No newline at end of file +} + From c9d9248a141c172b710d58d16d1861350ec0b61d Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 23:42:50 +0400 Subject: [PATCH 200/219] Update CLD2Detector.cs --- src/LanguageIdentification.CLD2/CLD2Detector.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.CLD2/CLD2Detector.cs b/src/LanguageIdentification.CLD2/CLD2Detector.cs index 881451a3..d5dc3910 100644 --- a/src/LanguageIdentification.CLD2/CLD2Detector.cs +++ b/src/LanguageIdentification.CLD2/CLD2Detector.cs @@ -27,8 +27,8 @@ public static bool IsSupported() { Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => true, Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => true, - Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, - Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + //Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + //Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, _ => false, }; } From b69cb9f0696dca34113fddf0fb576506c312d547 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 23:43:26 +0400 Subject: [PATCH 201/219] test --- src/LanguageIdentification.CLD2/CLD2Detector.cs | 4 ++-- src/LanguageIdentification.CLD3/CLD3Detector.cs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/LanguageIdentification.CLD2/CLD2Detector.cs b/src/LanguageIdentification.CLD2/CLD2Detector.cs index d5dc3910..881451a3 100644 --- a/src/LanguageIdentification.CLD2/CLD2Detector.cs +++ b/src/LanguageIdentification.CLD2/CLD2Detector.cs @@ -27,8 +27,8 @@ public static bool IsSupported() { Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => true, Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => true, - //Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, - //Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, _ => false, }; } diff --git a/src/LanguageIdentification.CLD3/CLD3Detector.cs b/src/LanguageIdentification.CLD3/CLD3Detector.cs index a22dc05f..195c80d6 100644 --- a/src/LanguageIdentification.CLD3/CLD3Detector.cs +++ b/src/LanguageIdentification.CLD3/CLD3Detector.cs @@ -35,8 +35,8 @@ public static bool IsSupported() { Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => true, Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => true, - Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, - Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + //Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + //Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, _ => false, }; } From 70d380eafa4ab414c14ae0b51d7e169846aa3087 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 23:43:38 +0400 Subject: [PATCH 202/219] Update CLD2Detector.cs --- src/LanguageIdentification.CLD2/CLD2Detector.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.CLD2/CLD2Detector.cs b/src/LanguageIdentification.CLD2/CLD2Detector.cs index 881451a3..d5dc3910 100644 --- a/src/LanguageIdentification.CLD2/CLD2Detector.cs +++ b/src/LanguageIdentification.CLD2/CLD2Detector.cs @@ -27,8 +27,8 @@ public static bool IsSupported() { Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => true, Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => true, - Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, - Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + //Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + //Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, _ => false, }; } From 50c3950c9ff83905c1d61b85d535bcd3080e23aa Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 23:50:12 +0400 Subject: [PATCH 203/219] CLD2 + CLD3: using DLLEXPORT --- .../Native/binding.h | 10 +++++----- .../Native/src/binding.h | 10 +++------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/LanguageIdentification.CLD2.Native/Native/binding.h b/src/LanguageIdentification.CLD2.Native/Native/binding.h index f221d840..54b4df01 100644 --- a/src/LanguageIdentification.CLD2.Native/Native/binding.h +++ b/src/LanguageIdentification.CLD2.Native/Native/binding.h @@ -5,11 +5,11 @@ #include "./cld2/public/compact_lang_det.h" #ifndef EXPORT -#if defined(_WIN32) || defined(_WIN64) -#define EXPORT __declspec(dllimport) -#else -#define EXPORT extern -#endif +# if defined(_WIN32) || defined(_WIN64) +# define EXPORT __declspec(dllexport) +# else +# define EXPORT extern +# endif #endif extern "C" diff --git a/src/LanguageIdentification.CLD3.Native/Native/src/binding.h b/src/LanguageIdentification.CLD3.Native/Native/src/binding.h index ca7d6186..4ce96492 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/src/binding.h +++ b/src/LanguageIdentification.CLD3.Native/Native/src/binding.h @@ -6,14 +6,10 @@ using namespace std; #ifndef EXPORT -# ifdef __linux__ -# define EXPORT __attribute__((visibility("default"))) +# if defined(_WIN32) || defined(_WIN64) +# define EXPORT __declspec(dllexport) # else -# if defined(_MSC_VER) -# define EXPORT __declspec(dllexport) -# else -# define EXPORT __attribute__((visibility("default"))) -# endif +# define EXPORT extern # endif #endif From 677b0696687c197d9bbc9718a530dabbb8ad650d Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 23:51:19 +0400 Subject: [PATCH 204/219] CLD3 for Mac is on --- src/LanguageIdentification.CLD3/CLD3Detector.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.CLD3/CLD3Detector.cs b/src/LanguageIdentification.CLD3/CLD3Detector.cs index 195c80d6..a22dc05f 100644 --- a/src/LanguageIdentification.CLD3/CLD3Detector.cs +++ b/src/LanguageIdentification.CLD3/CLD3Detector.cs @@ -35,8 +35,8 @@ public static bool IsSupported() { Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => true, Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => true, - //Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, - //Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, _ => false, }; } From 9647196d4d94c4efb6d9b7d70146eafa54fbe408 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Sun, 6 Oct 2024 23:51:36 +0400 Subject: [PATCH 205/219] CLD2 for Mac is on --- src/LanguageIdentification.CLD2/CLD2Detector.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.CLD2/CLD2Detector.cs b/src/LanguageIdentification.CLD2/CLD2Detector.cs index d5dc3910..881451a3 100644 --- a/src/LanguageIdentification.CLD2/CLD2Detector.cs +++ b/src/LanguageIdentification.CLD2/CLD2Detector.cs @@ -27,8 +27,8 @@ public static bool IsSupported() { Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => true, Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => true, - //Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, - //Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, _ => false, }; } From 44340233f17c58f3559e8959562c86c8c0b0c9af Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 7 Oct 2024 00:09:42 +0400 Subject: [PATCH 206/219] FT: __attribute__((visibility("default"))) --- .../Native/binding.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/Native/binding.h b/src/LanguageIdentification.FastText.Native/Native/binding.h index 77826f36..a9170377 100644 --- a/src/LanguageIdentification.FastText.Native/Native/binding.h +++ b/src/LanguageIdentification.FastText.Native/Native/binding.h @@ -7,13 +7,15 @@ #ifndef EXPORT # if defined(_WIN32) || defined(_WIN64) # define EXPORT __declspec(dllexport) +# elif defined(__GNUC__) || defined(__clang__) +# define EXPORT __attribute__((visibility("default"))) # else -# define EXPORT extern +# define EXPORT # endif #endif -extern "C" { - +extern "C" +{ typedef struct fasttext_t fasttext_t; typedef struct { From 817f0c83747244e2fc1db2c2b78a67b0bf130409 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 7 Oct 2024 00:10:00 +0400 Subject: [PATCH 207/219] CLD3: __attribute__((visibility("default"))) --- .../Native/src/binding.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.CLD3.Native/Native/src/binding.h b/src/LanguageIdentification.CLD3.Native/Native/src/binding.h index 4ce96492..0abea8b8 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/src/binding.h +++ b/src/LanguageIdentification.CLD3.Native/Native/src/binding.h @@ -8,12 +8,15 @@ using namespace std; #ifndef EXPORT # if defined(_WIN32) || defined(_WIN64) # define EXPORT __declspec(dllexport) +# elif defined(__GNUC__) || defined(__clang__) +# define EXPORT __attribute__((visibility("default"))) # else -# define EXPORT extern +# define EXPORT # endif #endif -extern "C" { +extern "C" +{ struct PredictionResult { const char* language; double probability; From 84c92196f510ea498b95ef6d8c36804f8cd3a85e Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 7 Oct 2024 00:10:08 +0400 Subject: [PATCH 208/219] CLD2: __attribute__((visibility("default"))) --- src/LanguageIdentification.CLD2.Native/Native/binding.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/LanguageIdentification.CLD2.Native/Native/binding.h b/src/LanguageIdentification.CLD2.Native/Native/binding.h index 54b4df01..7feba150 100644 --- a/src/LanguageIdentification.CLD2.Native/Native/binding.h +++ b/src/LanguageIdentification.CLD2.Native/Native/binding.h @@ -7,8 +7,10 @@ #ifndef EXPORT # if defined(_WIN32) || defined(_WIN64) # define EXPORT __declspec(dllexport) +# elif defined(__GNUC__) || defined(__clang__) +# define EXPORT __attribute__((visibility("default"))) # else -# define EXPORT extern +# define EXPORT # endif #endif From 1adb77b8ab3b74e4317f7933913e2ebbfd202331 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 7 Oct 2024 00:24:44 +0400 Subject: [PATCH 209/219] Update CMakeLists.txt --- .../Native/CMakeLists.txt | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt index fd1152e6..82c9ea70 100644 --- a/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.CLD3.Native/Native/CMakeLists.txt @@ -101,12 +101,16 @@ else() endif() endif() -# unit tests exec: -add_executable(language_identifier_main src/language_identifier_main.cc) -target_link_libraries(language_identifier_main PRIVATE cld3) +# Build unit tests +if (CMAKE_CXX_COMPILER_ID MATCHES "GNU") + if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + add_executable(language_identifier_main src/language_identifier_main.cc) + target_link_libraries(language_identifier_main PRIVATE cld3) -add_executable(getonescriptspan_test src/script_span/getonescriptspan_test.cc) -target_link_libraries(getonescriptspan_test PRIVATE cld3) + add_executable(getonescriptspan_test src/script_span/getonescriptspan_test.cc) + target_link_libraries(getonescriptspan_test PRIVATE cld3) -add_executable(language_identifier_features_test src/language_identifier_features_test.cc) -target_link_libraries(language_identifier_features_test PRIVATE cld3) + add_executable(language_identifier_features_test src/language_identifier_features_test.cc) + target_link_libraries(language_identifier_features_test PRIVATE cld3) + endif() +endif() \ No newline at end of file From f0bed9a64fea12394809fb8b45fafaf25921ff13 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 7 Oct 2024 00:52:22 +0400 Subject: [PATCH 210/219] Update CMakeLists.txt --- .../Native/CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt index d468c8d7..9dfc83c8 100644 --- a/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt +++ b/src/LanguageIdentification.FastText.Native/Native/CMakeLists.txt @@ -67,5 +67,4 @@ else() elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows") target_link_libraries(${PROJECT_NAME} fasttext-static_pic -static-libgcc -static-libstdc++) endif() - -endif() \ No newline at end of file +endif() From 305d48ad9d81aa8813923d38f347aa7fe4b35681 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 7 Oct 2024 00:52:25 +0400 Subject: [PATCH 211/219] Update CLD2Detector.cs --- src/LanguageIdentification.CLD2/CLD2Detector.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.CLD2/CLD2Detector.cs b/src/LanguageIdentification.CLD2/CLD2Detector.cs index 881451a3..d5dc3910 100644 --- a/src/LanguageIdentification.CLD2/CLD2Detector.cs +++ b/src/LanguageIdentification.CLD2/CLD2Detector.cs @@ -27,8 +27,8 @@ public static bool IsSupported() { Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => true, Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => true, - Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, - Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + //Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + //Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, _ => false, }; } From 2b10621b6d8542888557157ffe9c2468d9872646 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 7 Oct 2024 00:58:33 +0400 Subject: [PATCH 212/219] Update CLD3Detector.cs --- src/LanguageIdentification.CLD3/CLD3Detector.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.CLD3/CLD3Detector.cs b/src/LanguageIdentification.CLD3/CLD3Detector.cs index a22dc05f..195c80d6 100644 --- a/src/LanguageIdentification.CLD3/CLD3Detector.cs +++ b/src/LanguageIdentification.CLD3/CLD3Detector.cs @@ -35,8 +35,8 @@ public static bool IsSupported() { Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => true, Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => true, - Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, - Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + //Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + //Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, _ => false, }; } From 829d7aa3664f7974849ba31f9c8ca9f6d84458cd Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 7 Oct 2024 01:08:00 +0400 Subject: [PATCH 213/219] Update CLD2Detector.cs --- src/LanguageIdentification.CLD2/CLD2Detector.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LanguageIdentification.CLD2/CLD2Detector.cs b/src/LanguageIdentification.CLD2/CLD2Detector.cs index d5dc3910..881451a3 100644 --- a/src/LanguageIdentification.CLD2/CLD2Detector.cs +++ b/src/LanguageIdentification.CLD2/CLD2Detector.cs @@ -27,8 +27,8 @@ public static bool IsSupported() { Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => true, Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => true, - //Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, - //Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + Architecture.X64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, + Architecture.Arm64 when RuntimeInformation.IsOSPlatform(OSPlatform.OSX) => true, _ => false, }; } From 37968120d120237db14ad7584344a0271fadd87c Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 7 Oct 2024 01:21:52 +0400 Subject: [PATCH 214/219] debian:bookworm-slim -> debian:bullseye-slim --- src/LanguageIdentification.CLD2.Native/Dockerfile | 2 +- src/LanguageIdentification.CLD3.Native/Dockerfile | 2 +- src/LanguageIdentification.FastText.Native/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/LanguageIdentification.CLD2.Native/Dockerfile b/src/LanguageIdentification.CLD2.Native/Dockerfile index 18c40659..e0d63a0e 100644 --- a/src/LanguageIdentification.CLD2.Native/Dockerfile +++ b/src/LanguageIdentification.CLD2.Native/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:bookworm-slim AS build +FROM debian:bullseye-slim AS build WORKDIR /repo COPY ["src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj", "src/LanguageIdentification.CLD2.Native/"] diff --git a/src/LanguageIdentification.CLD3.Native/Dockerfile b/src/LanguageIdentification.CLD3.Native/Dockerfile index f1313241..d249b1b3 100644 --- a/src/LanguageIdentification.CLD3.Native/Dockerfile +++ b/src/LanguageIdentification.CLD3.Native/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:bookworm-slim AS build +FROM debian:bullseye-slim AS build WORKDIR /repo COPY ["src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj", "src/LanguageIdentification.CLD3.Native/"] diff --git a/src/LanguageIdentification.FastText.Native/Dockerfile b/src/LanguageIdentification.FastText.Native/Dockerfile index 3de53667..2a0e5655 100644 --- a/src/LanguageIdentification.FastText.Native/Dockerfile +++ b/src/LanguageIdentification.FastText.Native/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:bookworm-slim AS build +FROM debian:bullseye-slim AS build WORKDIR /repo COPY ["src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj", "src/LanguageIdentification.FastText.Native/"] From e97ef6aed3e49ec74c3a3d619157f6e0549f207b Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 7 Oct 2024 01:43:19 +0400 Subject: [PATCH 215/219] Fix --- README.md | 2 +- src/LanguageIdentification.Lingua.Native/Dockerfile | 3 +-- src/LanguageIdentification.MediaPipe.Native/Dockerfile | 3 +-- src/LanguageIdentification.Whatlang.Native/Dockerfile | 3 +-- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index ae7a8177..73e52774 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ functionality into their applications. | Model | Linux | Windows | macOS | Blazor WASM | | :-------- | :----------------: | :----------------: | :----------------: | :------------: | -| CLD2 | :white_check_mark: | :white_check_mark: | :construction: | :x: | +| CLD2 | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | | CLD3 | :white_check_mark: | :white_check_mark: | :construction: | :x: | | FastText | :white_check_mark: | :white_check_mark: | :white_check_mark: | :x: | | Whatlang | :white_check_mark: | :construction: | :construction: | :x: | diff --git a/src/LanguageIdentification.Lingua.Native/Dockerfile b/src/LanguageIdentification.Lingua.Native/Dockerfile index f60462e0..c133a988 100644 --- a/src/LanguageIdentification.Lingua.Native/Dockerfile +++ b/src/LanguageIdentification.Lingua.Native/Dockerfile @@ -1,5 +1,4 @@ -FROM ubuntu:22.04 AS build -ARG BUILD_CONFIGURATION=Release +FROM debian:bullseye-slim AS build WORKDIR /repo COPY ["src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj", "src/LanguageIdentification.Lingua.Native/"] diff --git a/src/LanguageIdentification.MediaPipe.Native/Dockerfile b/src/LanguageIdentification.MediaPipe.Native/Dockerfile index 078d583f..de8f6344 100644 --- a/src/LanguageIdentification.MediaPipe.Native/Dockerfile +++ b/src/LanguageIdentification.MediaPipe.Native/Dockerfile @@ -1,5 +1,4 @@ -FROM ubuntu:22.04 AS build -ARG BUILD_CONFIGURATION=Release +FROM ubuntu:20.04 AS build WORKDIR /repo COPY ["src/LanguageIdentification.MediaPipe.Native/LanguageIdentification.MediaPipe.Native.csproj", "src/LanguageIdentification.MediaPipe.Native/"] diff --git a/src/LanguageIdentification.Whatlang.Native/Dockerfile b/src/LanguageIdentification.Whatlang.Native/Dockerfile index 9c1f75c8..853100b4 100644 --- a/src/LanguageIdentification.Whatlang.Native/Dockerfile +++ b/src/LanguageIdentification.Whatlang.Native/Dockerfile @@ -1,5 +1,4 @@ -FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build -ARG BUILD_CONFIGURATION=Release +FROM debian:bullseye-slim AS build WORKDIR /repo COPY ["src/LanguageIdentification.Whatlang.Native/LanguageIdentification.Whatlang.Native.csproj", "src/LanguageIdentification.Whatlang.Native/"] From 882477bb21c040c22d281e9879be227a245e061b Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 7 Oct 2024 01:45:34 +0400 Subject: [PATCH 216/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 897ea836..5bf25d03 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -77,10 +77,10 @@ jobs: - { title: CLD2, projectName: LanguageIdentification.CLD2, os: ubuntu-latest } - { title: CLD3, projectName: LanguageIdentification.CLD3, os: ubuntu-latest } - { title: FastText, projectName: LanguageIdentification.FastText, os: ubuntu-latest } - # - { title: Whatlang, projectName: LanguageIdentification.Whatlang, os: ubuntu-latest } - # - { title: MediaPipe, projectName: LanguageIdentification.MediaPipe, os: ubuntu-latest } - # - { title: Lingua, projectName: LanguageIdentification.Lingua, os: ubuntu-latest } - # - { title: LanguageCode, projectName: LanguageCode, os: ubuntu-latest } + - { title: Whatlang, projectName: LanguageIdentification.Whatlang, os: ubuntu-latest } + - { title: MediaPipe, projectName: LanguageIdentification.MediaPipe, os: ubuntu-latest } + - { title: Lingua, projectName: LanguageIdentification.Lingua, os: ubuntu-latest } + - { title: LanguageCode, projectName: LanguageCode, os: ubuntu-latest } permissions: contents: read packages: write From cc341a745b1ea5f44861d458915b24a77455de3a Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 7 Oct 2024 01:53:11 +0400 Subject: [PATCH 217/219] ubuntu:20.04 -> ubuntu:22.04 --- src/LanguageIdentification.Lingua.Native/Dockerfile | 2 +- src/LanguageIdentification.MediaPipe.Native/Dockerfile | 2 +- src/LanguageIdentification.Whatlang.Native/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/LanguageIdentification.Lingua.Native/Dockerfile b/src/LanguageIdentification.Lingua.Native/Dockerfile index c133a988..b0bbaabc 100644 --- a/src/LanguageIdentification.Lingua.Native/Dockerfile +++ b/src/LanguageIdentification.Lingua.Native/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:bullseye-slim AS build +FROM ubuntu:22.04 AS build WORKDIR /repo COPY ["src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj", "src/LanguageIdentification.Lingua.Native/"] diff --git a/src/LanguageIdentification.MediaPipe.Native/Dockerfile b/src/LanguageIdentification.MediaPipe.Native/Dockerfile index de8f6344..5fa55755 100644 --- a/src/LanguageIdentification.MediaPipe.Native/Dockerfile +++ b/src/LanguageIdentification.MediaPipe.Native/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 AS build +FROM ubuntu:22.04 AS build WORKDIR /repo COPY ["src/LanguageIdentification.MediaPipe.Native/LanguageIdentification.MediaPipe.Native.csproj", "src/LanguageIdentification.MediaPipe.Native/"] diff --git a/src/LanguageIdentification.Whatlang.Native/Dockerfile b/src/LanguageIdentification.Whatlang.Native/Dockerfile index 853100b4..80c722c2 100644 --- a/src/LanguageIdentification.Whatlang.Native/Dockerfile +++ b/src/LanguageIdentification.Whatlang.Native/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:bullseye-slim AS build +FROM ubuntu:22.04 AS build WORKDIR /repo COPY ["src/LanguageIdentification.Whatlang.Native/LanguageIdentification.Whatlang.Native.csproj", "src/LanguageIdentification.Whatlang.Native/"] From cf1003df0a42f23c5084e4f17746e18ddb5e3cbd Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 7 Oct 2024 01:53:31 +0400 Subject: [PATCH 218/219] Update github-ci.yml --- .github/workflows/github-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml index 5bf25d03..c8624c30 100644 --- a/.github/workflows/github-ci.yml +++ b/.github/workflows/github-ci.yml @@ -161,7 +161,7 @@ jobs: task: - { os: ubuntu-24.04 } - { os: ubuntu-22.04 } - - { os: ubuntu-20.04 } + # - { os: ubuntu-20.04 } - { os: windows-2022 } - { os: windows-2019 } - { os: macos-13 } From 8dd5464fa3130b4b31b140aa7a3b6263ecfc7460 Mon Sep 17 00:00:00 2001 From: Alexander Gluschenko Date: Mon, 7 Oct 2024 23:16:20 +0400 Subject: [PATCH 219/219] Bump versions and update readme --- README_CLD2.md | 4 ++-- README_CLD3.md | 2 +- README_FASTTEXT.md | 4 ++-- src/LanguageCode/LanguageCode.csproj | 2 +- .../LanguageIdentification.CLD2.Native.csproj | 2 +- .../LanguageIdentification.CLD2.csproj | 8 ++++++-- .../LanguageIdentification.CLD3.Native.csproj | 2 +- .../LanguageIdentification.CLD3.csproj | 5 ++++- .../LanguageIdentification.FastText.Native.csproj | 2 +- .../LanguageIdentification.FastText.csproj | 5 ++++- .../LanguageIdentification.Lingua.Native.csproj | 2 +- .../LanguageIdentification.Lingua.csproj | 2 +- .../LanguageIdentification.MediaPipe.Native.csproj | 2 +- .../LanguageIdentification.MediaPipe.csproj | 2 +- .../LanguageIdentification.Whatlang.Native.csproj | 2 +- .../LanguageIdentification.Whatlang.csproj | 2 +- 16 files changed, 29 insertions(+), 19 deletions(-) diff --git a/README_CLD2.md b/README_CLD2.md index 3a1b62df..785c614d 100644 --- a/README_CLD2.md +++ b/README_CLD2.md @@ -5,8 +5,8 @@ Welcome to **Panlingo.LanguageIdentification.CLD2**, a .NET wrapper for the Chro ## Requirements - Runtime: **.NET >= 5.0** -- OS: **Linux** -- Arch: **AMD64** +- OS: **Linux (Ubuntu, Debian)**, **Windows 10+** or **Windows Server 2019+**, **macOS** +- Arch: **AMD64** (or **ARM** for macOS) ## Installation diff --git a/README_CLD3.md b/README_CLD3.md index ed827ed8..ddfd5085 100644 --- a/README_CLD3.md +++ b/README_CLD3.md @@ -5,7 +5,7 @@ Welcome to **Panlingo.LanguageIdentification.CLD3**, a .NET wrapper for the Chro ## Requirements - Runtime: **.NET >= 5.0** -- OS: **Linux** +- OS: **Linux (Ubuntu, Debian)**, **Windows 10+** or **Windows Server 2019+** - Arch: **AMD64** ## Installation diff --git a/README_FASTTEXT.md b/README_FASTTEXT.md index bb37021c..2df9b77a 100644 --- a/README_FASTTEXT.md +++ b/README_FASTTEXT.md @@ -5,8 +5,8 @@ Welcome to **Panlingo.LanguageIdentification.FastText**, a .NET wrapper for the ## Requirements - Runtime: **.NET >= 5.0** -- OS: **Linux** -- Arch: **AMD64** +- OS: **Linux (Ubuntu, Debian)**, **Windows 10+** or **Windows Server 2019+**, **macOS** +- Arch: **AMD64** (or **ARM** for macOS) ## Installation diff --git a/src/LanguageCode/LanguageCode.csproj b/src/LanguageCode/LanguageCode.csproj index eb9153c9..15927afe 100644 --- a/src/LanguageCode/LanguageCode.csproj +++ b/src/LanguageCode/LanguageCode.csproj @@ -2,7 +2,7 @@ netstandard2.1;net5.0;net6.0;net7.0;net8.0 - 0.0.1 + 0.0.0.23 Panlingo.LanguageCode Panlingo.LanguageCode Panlingo.LanguageCode diff --git a/src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj b/src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj index 00801dc0..1bdd177a 100644 --- a/src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj +++ b/src/LanguageIdentification.CLD2.Native/LanguageIdentification.CLD2.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.0.1 + 0.1.0 Panlingo.LanguageIdentification.CLD2.Native Panlingo.LanguageIdentification.CLD2.Native Panlingo.LanguageIdentification.CLD2.Native diff --git a/src/LanguageIdentification.CLD2/LanguageIdentification.CLD2.csproj b/src/LanguageIdentification.CLD2/LanguageIdentification.CLD2.csproj index f8ba0d0f..189b9e60 100644 --- a/src/LanguageIdentification.CLD2/LanguageIdentification.CLD2.csproj +++ b/src/LanguageIdentification.CLD2/LanguageIdentification.CLD2.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.0.1 + 0.1.0 Panlingo.LanguageIdentification.CLD2 Panlingo.LanguageIdentification.CLD2 Panlingo.LanguageIdentification.CLD2 @@ -15,7 +15,11 @@ nlp lid language-identification language-detection cld2 README_CLD2.md - - Initial release +0.1.0.0 +- Windows and MacOS support + +0.0.0.1 +- Initial release This is a .NET wrapper for the Chrome Language Detection (CLD2) library by Google Inc. diff --git a/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj b/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj index 65e129d3..f70f6e96 100644 --- a/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj +++ b/src/LanguageIdentification.CLD3.Native/LanguageIdentification.CLD3.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.0.1 + 0.1.0 Panlingo.LanguageIdentification.CLD3.Native Panlingo.LanguageIdentification.CLD3.Native Panlingo.LanguageIdentification.CLD3.Native diff --git a/src/LanguageIdentification.CLD3/LanguageIdentification.CLD3.csproj b/src/LanguageIdentification.CLD3/LanguageIdentification.CLD3.csproj index 51ab902c..2f715fe8 100644 --- a/src/LanguageIdentification.CLD3/LanguageIdentification.CLD3.csproj +++ b/src/LanguageIdentification.CLD3/LanguageIdentification.CLD3.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.0.1 + 0.1.0 Panlingo.LanguageIdentification.CLD3 Panlingo.LanguageIdentification.CLD3 Panlingo.LanguageIdentification.CLD3 @@ -15,6 +15,9 @@ nlp lid language-identification language-detection cld3 README_CLD3.md +0.1.0.0 +- Windows support + 0.0.0.20: - Protobuf is not required now! diff --git a/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj b/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj index f2691d86..197a8b72 100644 --- a/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj +++ b/src/LanguageIdentification.FastText.Native/LanguageIdentification.FastText.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.0.1 + 0.1.0 Panlingo.LanguageIdentification.FastText.Native Panlingo.LanguageIdentification.FastText.Native Panlingo.LanguageIdentification.FastText.Native diff --git a/src/LanguageIdentification.FastText/LanguageIdentification.FastText.csproj b/src/LanguageIdentification.FastText/LanguageIdentification.FastText.csproj index eff7a554..486577bb 100644 --- a/src/LanguageIdentification.FastText/LanguageIdentification.FastText.csproj +++ b/src/LanguageIdentification.FastText/LanguageIdentification.FastText.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.0.1 + 0.1.0 Panlingo.LanguageIdentification.FastText Panlingo.LanguageIdentification.FastText Panlingo.LanguageIdentification.FastText @@ -15,6 +15,9 @@ nlp lid language-identification language-detection fasttext README_FASTTEXT.md +0.1.0.0 +- Windows and MacOS support + 0.0.0.21: - Default FastText model is included in NuGet package diff --git a/src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj b/src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj index 343f9d67..a2fa47e8 100644 --- a/src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj +++ b/src/LanguageIdentification.Lingua.Native/LanguageIdentification.Lingua.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.0.0.22 + 0.0.0.23 Panlingo.LanguageIdentification.Lingua.Native Panlingo.LanguageIdentification.Lingua.Native Panlingo.LanguageIdentification.Lingua.Native diff --git a/src/LanguageIdentification.Lingua/LanguageIdentification.Lingua.csproj b/src/LanguageIdentification.Lingua/LanguageIdentification.Lingua.csproj index e54e0244..60341491 100644 --- a/src/LanguageIdentification.Lingua/LanguageIdentification.Lingua.csproj +++ b/src/LanguageIdentification.Lingua/LanguageIdentification.Lingua.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.0.0.22 + 0.0.0.23 Panlingo.LanguageIdentification.Lingua Panlingo.LanguageIdentification.Lingua Panlingo.LanguageIdentification.Lingua diff --git a/src/LanguageIdentification.MediaPipe.Native/LanguageIdentification.MediaPipe.Native.csproj b/src/LanguageIdentification.MediaPipe.Native/LanguageIdentification.MediaPipe.Native.csproj index 2b02a786..1c747b5b 100644 --- a/src/LanguageIdentification.MediaPipe.Native/LanguageIdentification.MediaPipe.Native.csproj +++ b/src/LanguageIdentification.MediaPipe.Native/LanguageIdentification.MediaPipe.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.0.0.22 + 0.0.0.23 Panlingo.LanguageIdentification.MediaPipe.Native Panlingo.LanguageIdentification.MediaPipe.Native Panlingo.LanguageIdentification.MediaPipe.Native diff --git a/src/LanguageIdentification.MediaPipe/LanguageIdentification.MediaPipe.csproj b/src/LanguageIdentification.MediaPipe/LanguageIdentification.MediaPipe.csproj index 4b7ebff2..f4d48028 100644 --- a/src/LanguageIdentification.MediaPipe/LanguageIdentification.MediaPipe.csproj +++ b/src/LanguageIdentification.MediaPipe/LanguageIdentification.MediaPipe.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.0.0.22 + 0.0.0.23 Panlingo.LanguageIdentification.MediaPipe Panlingo.LanguageIdentification.MediaPipe Panlingo.LanguageIdentification.MediaPipe diff --git a/src/LanguageIdentification.Whatlang.Native/LanguageIdentification.Whatlang.Native.csproj b/src/LanguageIdentification.Whatlang.Native/LanguageIdentification.Whatlang.Native.csproj index 8e9bf555..ab0966ce 100644 --- a/src/LanguageIdentification.Whatlang.Native/LanguageIdentification.Whatlang.Native.csproj +++ b/src/LanguageIdentification.Whatlang.Native/LanguageIdentification.Whatlang.Native.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 0.0.0.22 + 0.0.0.23 Panlingo.LanguageIdentification.Whatlang.Native Panlingo.LanguageIdentification.Whatlang.Native Panlingo.LanguageIdentification.Whatlang.Native diff --git a/src/LanguageIdentification.Whatlang/LanguageIdentification.Whatlang.csproj b/src/LanguageIdentification.Whatlang/LanguageIdentification.Whatlang.csproj index 52dbcd78..13fc8084 100644 --- a/src/LanguageIdentification.Whatlang/LanguageIdentification.Whatlang.csproj +++ b/src/LanguageIdentification.Whatlang/LanguageIdentification.Whatlang.csproj @@ -2,7 +2,7 @@ net5.0;net6.0;net7.0;net8.0 - 0.0.0.22 + 0.0.0.23 Panlingo.LanguageIdentification.Whatlang Panlingo.LanguageIdentification.Whatlang Panlingo.LanguageIdentification.Whatlang