initial: 🎉 first commit

.devcontainer/devcontainer.json

@@ -0,0 +1,20 @@
+{
+  "name": "webnavix",
+  "workspaceFolder": "/workspaces/webnavix/",
+  "dockerComposeFile": ["../docker/docker-compose.development.yaml"],
+  "service": "app",
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "adam-bender.commit-message-editor",
+        "charliermarsh.ruff",
+        "eamodio.gitlens",
+        "EditorConfig.EditorConfig",
+        "esbenp.prettier-vscode",
+        "ms-python.python",
+        "tamasfe.even-better-toml",
+        "VisualStudioExptTeam.vscodeintellicode"
+      ]
+    }
+  }
+}

.dockerignore

@@ -0,0 +1,27 @@
+# cache
+**/__pycache__/
+**/.mypy_cache/
+**/.ruff_cache/
+**/*.egg-info/
+
+# dataset
+**/wl_data/
+
+# debug
+**/*log*
+
+# deliverable
+**/build/
+**/dist/
+**/out/
+
+# dependency
+**/.venv/
+
+# env file
+**/.env*
+!**/.env.example
+
+# misc
+**/.DS_Store
+**/*.pem

.github/workflows/app-test.yaml

@@ -0,0 +1,19 @@
+name: app test
+
+on: push
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+
+      - name: setup rye
+        uses: eifinger/setup-rye@v4
+
+      - name: install dependencies
+        run: rye sync
+
+      - name: check
+        run: rye check

.gitignore

@@ -0,0 +1,30 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# cache
+__pycache__/
+.mypy_cache/
+.ruff_cache/
+*.egg-info/
+
+# dataset
+wl_data/
+
+# debug
+*log*
+
+# deliverable
+build/
+checkpoints/
+dist/
+out/
+
+# dependency
+.venv/
+
+# env file
+.env*
+!.env.example
+
+# misc
+.DS_Store
+*.pem

.python-version

@@ -0,0 +1 @@
+3.12.3

.vscode/settings.json

@@ -0,0 +1,159 @@
+{
+  "commit-message-editor.tokens": [
+    {
+      "label": "Type",
+      "name": "type",
+      "type": "enum",
+      "description": "Type of changes.",
+      "combobox": true,
+      "options": [
+        {
+          "label": "feat: ✨",
+          "value": "feat: ✨",
+          "description": "Implementation of new features."
+        },
+        {
+          "label": "feat: 🎈",
+          "value": "feat: 🎈",
+          "description": "Repair of existing features."
+        },
+        {
+          "label": "feat: ⚰️",
+          "value": "feat: ⚰️",
+          "description": "Deletion of features."
+        },
+        {
+          "label": "fix: 🐛",
+          "value": "fix: 🐛",
+          "description": "Bug fixes."
+        },
+        {
+          "label": "fix: 🚑️",
+          "value": "fix: 🚑️",
+          "description": "Critical bug fixes or major changes."
+        },
+        {
+          "label": "doc: 📝",
+          "value": "doc: 📝",
+          "description": "Documentation changes."
+        },
+        {
+          "label": "typo: 🖋️",
+          "value": "typo: 🖋️",
+          "description": "Typography changes."
+        },
+        {
+          "label": "style: 💄",
+          "value": "style: 💄",
+          "description": "Style changes."
+        },
+        {
+          "label": "refactor: ♻️",
+          "value": "refactor: ♻️",
+          "description": "Code formatting or refactoring."
+        },
+        {
+          "label": "test: 🧪",
+          "value": "test: 🧪",
+          "description": "Test cases changes."
+        },
+        {
+          "label": "ci: 🦺",
+          "value": "ci: 🦺",
+          "description": "CI changes."
+        },
+        {
+          "label": "build: 📦️",
+          "value": "build: 📦️",
+          "description": "Build system or dependency changes."
+        },
+        {
+          "label": "container: 🐳",
+          "value": "container: 🐳",
+          "description": "The Dockerfile changes."
+        },
+        {
+          "label": "container: 🐙",
+          "value": "container: 🐙",
+          "description": "The docker-compose changes."
+        },
+        {
+          "label": "chore: 🔧",
+          "value": "chore: 🔧",
+          "description": "Configuration changes."
+        },
+        {
+          "label": "chore: 🔨",
+          "value": "chore: 🔨",
+          "description": "Development script changes."
+        },
+        {
+          "label": "chore: 🍱",
+          "value": "chore: 🍱",
+          "description": "Assets changes."
+        },
+        {
+          "label": "revert: ⏪️",
+          "value": "revert: ⏪️",
+          "description": "Reversion of changes."
+        },
+        {
+          "label": "wip: 🚧",
+          "value": "wip: 🚧",
+          "description": "Changes that will be squashed."
+        },
+        {
+          "label": "initial: 🎉",
+          "value": "initial: 🎉",
+          "description": "The first commit."
+        }
+      ]
+    },
+    {
+      "label": "Scope",
+      "name": "scope",
+      "type": "text",
+      "description": "Scope of changes.",
+      "prefix": " (",
+      "suffix": ")"
+    },
+    {
+      "label": "Short Description",
+      "name": "description",
+      "type": "text",
+      "description": "Commit summary.",
+      "prefix": " "
+    },
+    {
+      "label": "Body",
+      "name": "body",
+      "type": "text",
+      "description": "Detailed description of commit.",
+      "maxLines": 10,
+      "multiline": true,
+      "lines": 5
+    },
+    {
+      "label": "Footer",
+      "name": "footer",
+      "description": "Description of disruptive changes or signature.",
+      "type": "text",
+      "multiline": true
+    }
+  ],
+  "commit-message-editor.dynamicTemplate": ["{type}{scope}{description}", "", "{body}", "", "{footer}"],
+  "commit-message-editor.staticTemplate": ["label: emoji (scope) short-description", "", "body", "", "footer"],
+  "commit-message-editor.view.defaultView": "form",
+  "editor.defaultFormatter": "esbenp.prettier-vscode",
+  "files.encoding": "utf8",
+  "files.eol": "\n",
+  "python.analysis.typeCheckingMode": "basic",
+  "python.defaultInterpreterPath": "/opt/rye/shims/python",
+  "[python]": {
+    "editor.codeActionsOnSave": {
+      "source.fixAll.ruff": "explicit",
+      "source.organizeImports.ruff": "explicit"
+    },
+    "editor.defaultFormatter": "charliermarsh.ruff"
+  }
+}

README.md

@@ -0,0 +1 @@
+# WebNavix

docker/Dockerfile.development

@@ -0,0 +1,27 @@
+FROM python:3.12
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+# hadolint ignore=DL3008
+RUN apt-get update \
+  && apt-get --no-install-recommends -y install git gnupg2 ca-certificates curl pipx \
+  && pipx ensurepath \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists
+
+RUN curl -sSf https://rye.astral.sh/get | RYE_INSTALL_OPTION="--yes" bash \
+  && echo "source '$HOME/.rye/env'" >> ~/.bashrc \
+  && /root/.rye/shims/rye config --set-bool behavior.global-python=true \
+  && /root/.rye/shims/rye config --set-bool behavior.use-uv=true
+
+ENV RYE_HOME="/opt/rye"
+ENV PATH="$RYE_HOME/shims:$PATH"
+
+WORKDIR /workspaces/webnavix/
+
+COPY ./.python-version ./pyproject.toml ./requirements* ./
+RUN /root/.rye/shims/rye pin "$(cat ./.python-version)" && /root/.rye/shims/rye sync
+
+COPY ./ ./
+
+RUN if [ ! -d ./wl_data/ ]; then /root/.rye/shims/rye run python ./src/dataset.py; fi

docker/docker-compose.development.yaml

@@ -0,0 +1,13 @@
+services:
+  app:
+    container_name: webnavix
+    build:
+      context: ../
+      dockerfile: ./docker/Dockerfile.development
+    volumes:
+      - type: bind
+        source: ../
+        target: /workspaces/webnavix/
+    environment:
+      WEBLINX_PROJECT_DIR: /workspaces/webnavix/
+    tty: true

lefthook.yaml

@@ -0,0 +1,9 @@
+pre-commit:
+  parallel: true
+  commands:
+    check-py:
+      glob: "*.*{py}*"
+      run: ruff check --fix {staged_files}
+    format-py:
+      glob: "*.*{py}*"
+      run: ruff format --fix {staged_files}

pyproject.toml

@@ -0,0 +1,48 @@
+[project]
+name = "webnavix"
+version = "0.1.0"
+description = "Add your description here"
+authors = [
+    { name = "shio", email = "85730998+dino3616@users.noreply.github.com" },
+]
+dependencies = [
+    "accelerate>=0.32.1",
+    "backoff>=2.2.1",
+    "bert-score>=0.3.13",
+    "bitsandbytes>=0.42.0",
+    "coloredlogs>=15.0.1",
+    "datasets>=2.20.0",
+    "huggingface-hub>=0.24.5",
+    "hydra-core>=1.3.2",
+    "lxml>=5.2.2",
+    "ninja>=1.11.1.1",
+    "numpy>=1.26.4",
+    "openai>=1.35.15",
+    "optimum>=1.21.2",
+    "packaging>=24.1",
+    "pandas>=2.2.2",
+    "peft>=0.11.1",
+    "pillow>=10.4.0",
+    "sacrebleu>=2.4.2",
+    "sentence-transformers>=3.0.1",
+    "setuptools>=71.1.0",
+    "tensorboardx>=2.6.2.2",
+    "tiktoken>=0.7.0",
+    "torch>=2.3.1",
+    "tqdm>=4.66.4",
+    "transformers>=4.42.4",
+    "trl>=0.9.6",
+    "weblinx>=0.3.0",
+]
+readme = "README.md"
+requires-python = ">= 3.12"
+
+[tool.rye]
+managed = true
+dev-dependencies = ["ruff>=0.5.3", "lefthook>=0.1.2"]
+
+[tool.rye.scripts]
+check = { chain = ["lint", "lint:type", "fmt"] }
+"lint" = "ruff check ./ --diff"
+"lint:type" = "mypy ./ --explicit-package-bases"
+"fmt" = "ruff fmt ./"