Merge pull request #6 from cybercongress/updates

Fix running validator in background thread

neurons/miner.py

@@ -136,7 +136,7 @@ async def priority(self, synapse: Dummy) -> float:
         Returns:
             float: A priority score derived from the stake of the calling entity.
 
-        Miners may recieve messages from multiple entities at once. This function determines which request should be
+        Miners may receive messages from multiple entities at once. This function determines which request should be
         processed first. Higher values indicate that the request should be processed first. Lower values indicate
         that the request should be processed later.
 
@@ -147,18 +147,18 @@ async def priority(self, synapse: Dummy) -> float:
         caller_uid = self.metagraph.hotkeys.index(
             synapse.dendrite.hotkey
         )  # Get the caller index.
-        prirority = float(
+        priority = float(
             self.metagraph.S[caller_uid]
         )  # Return the stake as the priority.
         ct.logging.trace(
-            f"Prioritizing {synapse.dendrite.hotkey} with value: ", prirority
+            f"Prioritizing {synapse.dendrite.hotkey} with value: ", priority
         )
-        return prirority
+        return priority
 
 
 # This is the main function, which runs the miner.
 if __name__ == "__main__":
     with Miner() as miner:
         while True:
             ct.logging.info("Miner running...", time.time())
-            time.sleep(5)
+            time.sleep(15)

neurons/validator.py

@@ -45,8 +45,6 @@ class Validator(BaseValidatorNeuron):
     def __init__(self, config: Optional[ct.Config] = None):
         super(Validator, self).__init__(config=config)
 
-        self.load_state()
-
         # TODO(developer): Anything specific to your use case you can do here
 
     async def forward(self):
@@ -66,5 +64,20 @@ async def forward(self):
 if __name__ == "__main__":
     with Validator() as validator:
         while True:
-            ct.logging.info("Validator running...", time.time())
-            time.sleep(5)
+            try:
+                validator.metagraph.sync(cwtensor=validator.cwtensor)
+                ct.logging.info(
+                    f"Validator {'is up and running' if validator.thread and validator.thread.is_alive() else 'is running and not working'}\t"
+                    f"step {validator.step if validator.step else '-'}\t"
+                    f"block {validator.block if validator.block else None:>,}\t\t"
+                    f"blocks until sync {validator.config.neuron.epoch_length - validator.block + validator.metagraph.last_update[validator.uid]}"
+                )
+                if validator.thread is None or not validator.thread.is_alive():
+                    ct.logging.debug("Stopped")
+                    validator.is_running = False
+                    time.sleep(60)
+                    validator.run_in_background_thread()
+            except Exception as e:
+                ct.logging.error(f"[red]Error:[/red] {e}\t[red]trace:[/red] {traceback.format_exc()}")
+
+            time.sleep(15)

template/__init__.py

@@ -20,7 +20,7 @@
 
 # TODO(developer): Change this value when updating your code base.
 # Define the version of the template module.
-__version__ = "0.1.1"
+__version__ = "0.1.2"
 version_split = __version__.split(".")
 __spec_version__ = (
     (1000 * int(version_split[0]))

template/base/miner.py

@@ -136,6 +136,7 @@ def run(self):
                 # Sync metagraph and potentially set weights.
                 self.sync()
                 self.step += 1
+                time.sleep(5)
 
         # If someone intentionally stops the miner, it'll safely terminate operations.
         except KeyboardInterrupt:
@@ -196,7 +197,7 @@ def __exit__(self, exc_type, exc_value, traceback):
 
     def resync_metagraph(self):
         """Resyncs the metagraph and updates the hotkeys and moving averages based on the new metagraph."""
-        ct.logging.info("resync_metagraph()")
+        ct.logging.trace("resync_metagraph()")
 
         # Sync the metagraph.
         self.metagraph.sync(cwtensor=self.cwtensor)

template/base/neuron.py

@@ -128,12 +128,13 @@ def sync(self):
         """
         # Ensure miner or validator hotkey is still registered on the network.
         self.check_registered()
+        self.metagraph.sync(cwtensor=self.cwtensor)
 
         if self.should_sync_metagraph():
             self.resync_metagraph()
 
-        if self.should_set_weights():
-            self.set_weights()
+            if self.should_set_weights():
+                self.set_weights()
 
         # Always save state.
         self.save_state()

template/base/validator.py

@@ -20,6 +20,8 @@
 
 
 import copy
+import time
+
 import torch
 import asyncio
 import argparse
@@ -48,6 +50,7 @@ def add_args(cls, parser: argparse.ArgumentParser):
 
     def __init__(self, config=None):
         super().__init__(config=config)
+        self.load_state()
 
         # Save a copy of the hotkeys to local memory.
         self.hotkeys = copy.deepcopy(self.metagraph.hotkeys)
@@ -148,19 +151,21 @@ def run(self):
         # This loop maintains the validator's operations until intentionally stopped.
         try:
             while True:
-                ct.logging.info(f"Starting validator forward function at step({self.step}) block({self.block})")
+                if self.should_sync_metagraph():
+                    ct.logging.info(f"Starting validator forward function at step {self.step}\tblock {self.block:>,}")
 
-                # Run multiple forwards concurrently.
-                self.loop.run_until_complete(self.concurrent_forward())
+                    # Run multiple forwards concurrently.
+                    self.loop.run_until_complete(self.concurrent_forward())
 
-                # Check if we should exit.
-                if self.should_exit:
-                    break
+                    # Check if we should exit.
+                    if self.should_exit:
+                        break
 
-                # Sync metagraph and potentially set weights.
-                self.sync()
+                    # Sync metagraph and potentially set weights.
+                    self.sync()
 
-                self.step += 1
+                    self.step += 1
+                time.sleep(5)
 
         # If someone intentionally stops the validator, it'll safely terminate operations.
         except KeyboardInterrupt:
@@ -272,7 +277,7 @@ def set_weights(self):
             netuid=self.config.netuid,
             uids=uint_uids,
             weights=uint_weights,
-            wait_for_finalization=False,
+            wait_for_finalization=True,
             version_key=self.spec_version,
         )
         if result is True:
@@ -282,7 +287,7 @@ def set_weights(self):
 
     def resync_metagraph(self):
         """Resyncs the metagraph and updates the hotkeys and moving averages based on the new metagraph."""
-        ct.logging.info("resync_metagraph()")
+        ct.logging.trace("resync_metagraph()")
 
         # Copies state of metagraph before syncing.
         previous_metagraph = copy.deepcopy(self.metagraph)
@@ -371,3 +376,4 @@ def load_state(self):
         self.step = state["step"]
         self.scores = state["scores"]
         self.hotkeys = state["hotkeys"]
+        ct.logging.debug(f"Loaded validator state\t step: {self.step}\t scores: {self.scores}")