Skip to content

Commit

Permalink
Merge pull request #225 from backend-developers-ltd/multiple-executor…
Browse files Browse the repository at this point in the history
…-classes

Multiple executor classes support
  • Loading branch information
mpnowacki-reef authored Sep 3, 2024
2 parents 0e28455 + 656b2bf commit 7f010b5
Show file tree
Hide file tree
Showing 6 changed files with 18 additions and 13 deletions.
10 changes: 8 additions & 2 deletions install_miner.sh
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
#!/bin/bash
set -euxo pipefail

if [ $# -lt 3 ] || [ $# -gt 4 ];
if [ $# -lt 3 ] || [ $# -gt 5 ];
then
>&2 echo "USAGE: ./install_miner.sh MODE[production|local] SSH_DESTINATION HOTKEY_PATH|VALIDATOR_PUBLIC_KEY MINER_PORT:8000"
>&2 echo "USAGE: ./install_miner.sh MODE[production|local] SSH_DESTINATION HOTKEY_PATH|VALIDATOR_PUBLIC_KEY MINER_PORT:8000(optional) DEFAULT_EXECUTOR_CLASS(optional, local mode only)"
exit 1
fi

Expand Down Expand Up @@ -36,13 +36,16 @@ if [ "$MODE" == "production" ]; then
REMOTE_WALLET_NAME="$(basename "$(dirname "$REMOTE_HOTKEY_DIR")")"

VALIDATOR_PUBLIC_KEY=""
DEFAULT_EXECUTOR_CLASS=

elif [ "$MODE" == "local" ]; then
VALIDATOR_PUBLIC_KEY="$3"

REMOTE_HOTKEY_DIR=.bittensor/wallets/dummy/hotkeys
REMOTE_HOTKEY_NAME=dummy
REMOTE_WALLET_NAME=dummy

DEFAULT_EXECUTOR_CLASS="${5:-}"
else
>&2 echo "Invalid mode $MODE. Must be 'production' or 'local'"
exit 1
Expand All @@ -63,6 +66,7 @@ DEFAULT_ADMIN_PASSWORD="$DEFAULT_ADMIN_PASSWORD"
MIGRATING=$MIGRATING
MINER_PORT=$MINER_PORT
VALIDATOR_PUBLIC_KEY=$VALIDATOR_PUBLIC_KEY
DEFAULT_EXECUTOR_CLASS=$DEFAULT_EXECUTOR_CLASS
ENDCAT
ENDSSH

Expand Down Expand Up @@ -169,6 +173,8 @@ BITTENSOR_MINER_PORT=$(. ~/tmpvars && echo "$MINER_PORT")
BITTENSOR_MINER_ADDRESS=auto
COMPOSE_PROJECT_NAME=compute_horde_miner
DEFAULT_EXECUTOR_CLASS=$(. ~/tmpvars && echo "$DEFAULT_EXECUTOR_CLASS")
# make sure to unblock access to that port in your firewall
PORT_FOR_EXECUTORS=$(. ~/tmpvars && echo "$MINER_PORT")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@
import time

from compute_horde.executor_class import (
DEFAULT_EXECUTOR_CLASS,
DEFAULT_EXECUTOR_TIMEOUT,
MAX_EXECUTOR_TIMEOUT,
ExecutorClass,
)
Expand Down Expand Up @@ -132,10 +130,6 @@ async def get_executor_class_pool(self, executor_class):
pool.set_count(executor_count)
return self._executor_class_pools[executor_class]

async def reserve_executor(self, token):
# TODO: deprecated - new code should use `reserve_executor_class`
await self.reserve_executor_class(token, DEFAULT_EXECUTOR_CLASS, DEFAULT_EXECUTOR_TIMEOUT)

async def reserve_executor_class(self, token, executor_class, timeout):
pool = await self.get_executor_class_pool(executor_class)
await pool.reserve_executor(token, timeout)
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import subprocess
import sys

from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS
from django.conf import settings

from compute_horde_miner.miner.executor_manager._internal.base import BaseExecutorManager
Expand Down Expand Up @@ -37,4 +36,4 @@ async def wait_for_executor(self, executor, timeout):
pass

async def get_manifest(self):
return {DEFAULT_EXECUTOR_CLASS: 1}
return {settings.DEFAULT_EXECUTOR_CLASS: 1}
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import os
import subprocess

from compute_horde.executor_class import DEFAULT_EXECUTOR_CLASS
from django.conf import settings

from compute_horde_miner.miner.executor_manager._internal.base import (
Expand Down Expand Up @@ -114,4 +113,4 @@ async def wait_for_executor(self, executor, timeout):
pass

async def get_manifest(self):
return {DEFAULT_EXECUTOR_CLASS: 1}
return {settings.DEFAULT_EXECUTOR_CLASS: 1}
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,9 @@ async def handle(self, msg: BaseValidatorRequest):
self.pending_jobs[msg.job_uuid] = job

try:
await current.executor_manager.reserve_executor(token)
await current.executor_manager.reserve_executor_class(
token, msg.executor_class, msg.timeout_seconds
)
except ExecutorUnavailable:
await self.send(
miner_requests.V0DeclineJobRequest(job_uuid=msg.job_uuid).model_dump_json()
Expand Down
5 changes: 5 additions & 0 deletions miner/app/src/compute_horde_miner/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"""

from compute_horde import base # noqa
from compute_horde import executor_class

import inspect
import ipaddress
Expand Down Expand Up @@ -354,6 +355,10 @@ def wrapped(*args, **kwargs):
EXECUTOR_IMAGE = env.str(
"EXECUTOR_IMAGE", default="backenddevelopersltd/compute-horde-executor:v0-latest"
)
DEFAULT_EXECUTOR_CLASS = (
env.str("DEFAULT_EXECUTOR_CLASS", None) or executor_class.DEFAULT_EXECUTOR_CLASS
)

DEBUG_SKIP_PULLING_EXECUTOR_IMAGE = env.bool("DEBUG_SKIP_PULLING_EXECUTOR_IMAGE", default=False)
ADDRESS_FOR_EXECUTORS = env.str("ADDRESS_FOR_EXECUTORS", default="")
PORT_FOR_EXECUTORS = env.int("PORT_FOR_EXECUTORS")
Expand Down

0 comments on commit 7f010b5

Please sign in to comment.