docker-compose.yaml

version: '3.3'
services:
  triton:
    build:
      context: .
      dockerfile: triton.Dockerfile
    command: bash -c "CUDA_VISIBLE_DEVICES=${GPUS} mpirun -n 1 --allow-run-as-root /opt/tritonserver/bin/tritonserver --model-repository=/model"
    shm_size: '2gb'
    volumes:
      - ${MODEL_DIR}:/model
      - ${HF_CACHE_DIR}:/root/.cache/huggingface
    ports:
      - "8000:8000"
      - "${TRITON_PORT}:8001"
      - "8002:8002"
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
  copilot_proxy:
    # For dockerhub version
    # image: moyix/copilot_proxy:latest
    # For local build
    build:
      context: .
      dockerfile: proxy.Dockerfile
    command: uvicorn app:app --host 0.0.0.0 --port 5000
    env_file:
      # Automatically created via ./setup.sh
      - .env
    ports:
      - "${API_EXTERNAL_PORT}:5000"