Pre-release (#4)

* CLD3: FindTopNMostFreqLangs -> FindLanguages * Try to convert 'CRLF' to 'LF' * Run tests in CI/CD * Update github-ci.yml * Update github-ci.yml * Update test.Dockerfile * Update github-ci.yml * Update github-ci.yml * Update github-ci.yml * Update github-ci.yml * Keep it simple * Update test.Dockerfile * Update test.Dockerfile * Fixes * Add test-ci.Dockerfile * Update test-ci.Dockerfile * Update github-ci.yml * Update test-ci.Dockerfile * Update test-ci.Dockerfile * Update github-ci.yml * Update github-ci.yml * Update github-ci.yml * Save artifacts to /src/local-nugets * Final fixes * Update test-ci.Dockerfile
gluschenko · Jul 14, 2024 · 9643e87 · 9643e87
1 parent 8df2dc7
commit 9643e87
Show file tree

Hide file tree

Showing 21 changed files with 116 additions and 20 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -1,2 +1,3 @@
 # Auto detect text files and perform LF normalization
 * text=auto
+*.sh text eol=lf
diff --git a/.github/workflows/github-ci.yml b/.github/workflows/github-ci.yml
@@ -8,8 +8,7 @@ on:
   workflow_dispatch:
 
 jobs:
-  # DEPLOY NUGETS
-  build_nugets:
+  build-nugets:
     name: 🚀 Pack ${{matrix.projectName}}
     runs-on: ubuntu-latest
     strategy:
@@ -49,17 +48,55 @@ jobs:
       working-directory: ${{env.ROOT}}/${{matrix.projectName}}
       run: dotnet pack -c Release -o out 
 
-    - uses: actions/upload-artifact@v2
-      if: github.ref == 'refs/heads/master'
+    - uses: actions/upload-artifact@v4
       with:
-        name: build_nugets
-        path: ${{env.ROOT}}/**/out
+        name: build-${{matrix.projectName}}
+        path: ${{env.ROOT}}/${{matrix.projectName}}/out
         retention-days: 1
+        overwrite: 'true'
+        compression-level: 0
 
-  deploy_nugets:
+  test-nugets:
+    name: 🧪 Test ${{matrix.projectName}}
+    runs-on: ubuntu-latest
+    needs: build-nugets
+    permissions:
+      contents: read
+      packages: write
+    env:
+      ROOT: ./src
+      NUGET_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}} 
+    steps:
+    - uses: actions/checkout@v4
+      with: 
+        submodules: 'true'
+
+    - uses: actions/download-artifact@v4
+      with:
+        pattern: build-*
+        merge-multiple: true
+        path: ${{env.ROOT}}/local-nugets
+
+    - name: 📂 Files
+      working-directory: ${{env.ROOT}}
+      run: ls -R
+
+    - name: 📂 Build Docker Image \#1
+      working-directory: ${{env.ROOT}}
+      run: docker build -f ./test.Dockerfile -t local-dotnet-test-sdk:latest .
+
+    - name: 📂 Build Docker Image \#2
+      working-directory: ${{env.ROOT}}
+      run: docker build -f ./test-ci.Dockerfile -t local-dotnet-test-ci-sdk:latest .
+
+    - name: 🧪 Test
+      working-directory: ${{env.ROOT}}
+      run: docker run --rm -v $(pwd):/src local-dotnet-test-ci-sdk:latest
+
+  deploy-nugets:
     name: 🚚 Push ${{matrix.projectName}}
     runs-on: ubuntu-latest
-    needs: build_nugets
+    needs: test-nugets
     if: github.ref == 'refs/heads/master'
     strategy:
       fail-fast: false
@@ -81,11 +118,13 @@ jobs:
       NUGET_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 
     steps:
     - uses: actions/checkout@v4
+      with: 
+        submodules: 'true'
 
-    - uses: actions/download-artifact@v2
+    - uses: actions/download-artifact@v4
       with:
-        name: build_nugets
-        path: ${{env.ROOT}}
+        pattern: build-${{matrix.projectName}}
+        path: ${{env.ROOT}}/${{matrix.projectName}}/out
 
     - name: 📂 Setup .NET Core
       uses: actions/setup-dotnet@v4
@@ -101,6 +140,6 @@ jobs:
       working-directory: ${{env.ROOT}}/${{matrix.projectName}}
       run: dotnet nuget push ./out/*.nupkg --skip-duplicate -k ${{env.NUGET_AUTH_TOKEN}}
 
-    - uses: geekyeggo/delete-artifact@v1
+    - uses: geekyeggo/delete-artifact@v5
       with:
-        name: build_nugets
+        name: build-${{matrix.projectName}}
diff --git a/src/LanguageIdentification.CLD2.Native/Scripts/run-build.sh b/src/LanguageIdentification.CLD2.Native/Scripts/run-build.sh
@@ -28,3 +28,4 @@ rm -rf "$workspace"
 ldd libcld2.so
 
 echo "Goodbye world";
+
diff --git a/src/LanguageIdentification.CLD2.Native/Scripts/setup-build.sh b/src/LanguageIdentification.CLD2.Native/Scripts/setup-build.sh
@@ -6,3 +6,4 @@ echo "Installing build packages";
 sudo apt -y update | apt -y update
 sudo apt -y install cmake | apt -y install cmake
 sudo apt -y install g++ | apt -y install g++
+
diff --git a/src/LanguageIdentification.CLD2.Native/Scripts/setup-runtime.sh b/src/LanguageIdentification.CLD2.Native/Scripts/setup-runtime.sh
@@ -3,3 +3,4 @@ set -e
 
 echo "Installing runtime packages";
 
+
diff --git a/src/LanguageIdentification.CLD3.Native/Native/binding.cc b/src/LanguageIdentification.CLD3.Native/Native/binding.cc
@@ -23,7 +23,7 @@ PredictionResult FindLanguage(void* identifier, const char* text) {
     return result;
 }
 
-PredictionResult* FindTopNMostFreqLangs(void* identifier, const char* text, int numLangs, int* resultCount) {
+PredictionResult* FindLanguages(void* identifier, const char* text, int numLangs, int* resultCount) {
     NNetLanguageIdentifier* nativeIdentifier = static_cast<NNetLanguageIdentifier*>(identifier);
     auto nativeResults = nativeIdentifier->FindTopNMostFreqLangs(text, numLangs);
 

diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh b/src/LanguageIdentification.CLD3.Native/Scripts/run-build.sh
@@ -28,3 +28,4 @@ rm -rf "$workspace"
 ldd libcld3.so
 
 echo "Goodbye world";
+
diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh b/src/LanguageIdentification.CLD3.Native/Scripts/setup-build.sh
@@ -8,3 +8,4 @@ sudo apt -y install g++ | apt -y install g++
 sudo apt -y install python3 | apt -y install python3
 sudo apt -y install python3-pip | apt -y install python3-pip
 sudo apt -y install protobuf-compiler | apt -y install protobuf-compiler
+
diff --git a/src/LanguageIdentification.CLD3.Native/Scripts/setup-runtime.sh b/src/LanguageIdentification.CLD3.Native/Scripts/setup-runtime.sh
@@ -5,3 +5,4 @@ echo "Installing runtime packages";
 
 sudo apt -y update | apt -y update
 sudo apt -y install protobuf-compiler libprotobuf-dev | apt -y install protobuf-compiler libprotobuf-dev
+
diff --git a/src/LanguageIdentification.CLD3/CLD3Detector.cs b/src/LanguageIdentification.CLD3/CLD3Detector.cs
@@ -37,7 +37,7 @@ public CLD3PredictionResult[] FindLanguageNMostFreqLangs(
         int numLangs
     )
     {
-        var resultPtr = CLD3DetectorWrapper.FindTopNMostFreqLangs(
+        var resultPtr = CLD3DetectorWrapper.FindLanguages(
             identifier: _identifier,
             text: text,
             numLangs: numLangs,

diff --git a/src/LanguageIdentification.CLD3/CLD3DetectorWrapper.cs b/src/LanguageIdentification.CLD3/CLD3DetectorWrapper.cs
@@ -15,7 +15,7 @@ internal static class CLD3DetectorWrapper
     public static extern CLD3PredictionResult FindLanguage(nint identifier, string text);
 
     [DllImport(CLD3NativeLibrary.Name, CallingConvention = CallingConvention.Cdecl)]
-    public static extern nint FindTopNMostFreqLangs(nint identifier, string text, int numLangs, out int resultCount);
+    public static extern nint FindLanguages(nint identifier, string text, int numLangs, out int resultCount);
 
     [DllImport(CLD3NativeLibrary.Name, CallingConvention = CallingConvention.Cdecl)]
     public static extern void FreeResults(nint results, int count);

diff --git a/src/LanguageIdentification.FastText.Native/Scripts/run-build.sh b/src/LanguageIdentification.FastText.Native/Scripts/run-build.sh
@@ -28,3 +28,4 @@ rm -rf "$workspace"
 ldd libfasttext.so
 
 echo "Goodbye world";
+
diff --git a/src/LanguageIdentification.FastText.Native/Scripts/setup-build.sh b/src/LanguageIdentification.FastText.Native/Scripts/setup-build.sh
@@ -6,3 +6,4 @@ echo "Installing build packages";
 sudo apt -y update | apt -y update
 sudo apt -y install cmake | apt -y install cmake
 sudo apt -y install g++ | apt -y install g++
+
diff --git a/src/LanguageIdentification.FastText.Native/Scripts/setup-runtime.sh b/src/LanguageIdentification.FastText.Native/Scripts/setup-runtime.sh
@@ -3,3 +3,4 @@ set -e
 
 echo "Installing runtime packages";
 
+
diff --git a/src/LanguageIdentification.Tests/LanguageIdentification.Tests.csproj b/src/LanguageIdentification.Tests/LanguageIdentification.Tests.csproj
@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>net8.0</TargetFramework>
@@ -16,6 +16,20 @@
     <PackageReference Include="xunit.runner.visualstudio" Version="2.5.3" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(Configuration)' == 'Debug'">
+    <ProjectReference Include="..\LanguageIdentification.CLD2\LanguageIdentification.CLD2.csproj" />
+    <ProjectReference Include="..\LanguageIdentification.CLD3\LanguageIdentification.CLD3.csproj" />
+    <ProjectReference Include="..\LanguageIdentification.FastText\LanguageIdentification.FastText.csproj" />
+    <ProjectReference Include="..\LanguageIdentification.Whatlang\LanguageIdentification.Whatlang.csproj" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(Configuration)' == 'Release'">
+    <PackageReference Include="LanguageIdentification.CLD2" Version="*" />
+    <PackageReference Include="LanguageIdentification.CLD3" Version="*" />
+    <PackageReference Include="LanguageIdentification.FastText" Version="*" />
+    <PackageReference Include="LanguageIdentification.Whatlang" Version="*" />
+  </ItemGroup>
+
   <ItemGroup>
     <Using Include="Xunit" />
   </ItemGroup>

diff --git a/src/LanguageIdentification.Whatlang.Native/Scripts/run-build.sh b/src/LanguageIdentification.Whatlang.Native/Scripts/run-build.sh
@@ -24,3 +24,4 @@ rm -rf "$workspace"
 ldd libwhatlang.so
 
 echo "Goodbye world";
+
diff --git a/src/LanguageIdentification.Whatlang.Native/Scripts/setup-build.sh b/src/LanguageIdentification.Whatlang.Native/Scripts/setup-build.sh
@@ -5,3 +5,4 @@ echo "Installing build packages";
 
 sudo apt -y update | apt -y update
 sudo apt -y install cargo | apt -y install cargo
+
diff --git a/src/LanguageIdentification.Whatlang.Native/Scripts/setup-runtime.sh b/src/LanguageIdentification.Whatlang.Native/Scripts/setup-runtime.sh
@@ -3,3 +3,4 @@ set -e
 
 echo "Installing runtime packages";
 
+
diff --git a/src/LanguageIdentification.sln b/src/LanguageIdentification.sln
@@ -6,6 +6,7 @@ MinimumVisualStudioVersion = 10.0.40219.1
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".misc", ".misc", "{F741FF11-5F70-4E3B-BC72-9DA3090B90E9}"
 	ProjectSection(SolutionItems) = preProject
 		.editorconfig = .editorconfig
+		test-ci.Dockerfile = test-ci.Dockerfile
 		test.Dockerfile = test.Dockerfile
 		testEnvironments.json = testEnvironments.json
 	EndProjectSection
@@ -48,11 +49,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LanguageIdentification.CLD2
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "whatlang", "whatlang", "{C930A06F-0951-4774-97CC-001A477C1092}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LanguageIdentification.Whatlang.ConsoleTest", "LanguageIdentification.Whatlang.ConsoleTest\LanguageIdentification.Whatlang.ConsoleTest.csproj", "{B76A01EA-4FA5-414F-B7FD-0F7741F7F29D}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LanguageIdentification.Whatlang.ConsoleTest", "LanguageIdentification.Whatlang.ConsoleTest\LanguageIdentification.Whatlang.ConsoleTest.csproj", "{B76A01EA-4FA5-414F-B7FD-0F7741F7F29D}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LanguageIdentification.Whatlang.Native", "LanguageIdentification.Whatlang.Native\LanguageIdentification.Whatlang.Native.csproj", "{047FA65C-E597-4C7A-9AC9-C689ACB85C60}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LanguageIdentification.Whatlang.Native", "LanguageIdentification.Whatlang.Native\LanguageIdentification.Whatlang.Native.csproj", "{047FA65C-E597-4C7A-9AC9-C689ACB85C60}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LanguageIdentification.Whatlang", "LanguageIdentification.Whatlang\LanguageIdentification.Whatlang.csproj", "{E12065C1-B27B-49F6-A401-0F468B63F0F4}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LanguageIdentification.Whatlang", "LanguageIdentification.Whatlang\LanguageIdentification.Whatlang.csproj", "{E12065C1-B27B-49F6-A401-0F468B63F0F4}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution

diff --git a/src/test-ci.Dockerfile b/src/test-ci.Dockerfile
@@ -0,0 +1,15 @@
+FROM local-dotnet-test-sdk:latest AS build
+
+WORKDIR /src
+COPY . .
+
+RUN dotnet nuget add source /src/local-nugets
+
+ENTRYPOINT [
+    "dotnet", 
+    "test", 
+    "-c", "Release", 
+    "/src/LanguageIdentification.Tests/LanguageIdentification.Tests.csproj"
+]
+
+
diff --git a/src/test.Dockerfile b/src/test.Dockerfile
@@ -3,3 +3,17 @@
 RUN wget https://aka.ms/getvsdbgsh && \
     sh getvsdbgsh -v latest  -l /vsdbg
 
+### CLD3
+RUN apt -y update
+RUN apt -y install protobuf-compiler libprotobuf-dev
+###
+
+### FastText
+RUN apt -y update
+RUN apt -y install curl
+RUN mkdir /models -p
+RUN curl --location -o /models/fasttext176.bin https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin
+# RUN curl --location -o /models/fasttext217.bin https://huggingface.co/facebook/fasttext-language-identification/resolve/main/model.bin?download=true
+###
+
+