diff --git a/torchtitan/utils.py b/torchtitan/utils.py
index ab7dc1e9..277f12e3 100644
--- a/torchtitan/utils.py
+++ b/torchtitan/utils.py
@@ -106,7 +106,8 @@ def set_determinism(
 
     # The native RNGs and python RNG may not be important, except for the 1-D PP case, but we seed them for consistency.
     torch.manual_seed(seed)
-    os.environ["PYTHONHASHSEED"] = str(seed)
+    # PYTHONHASHSEED can be a decimal number in the range [0,2**32 - 1]
+    os.environ["PYTHONHASHSEED"] = str(seed % (2**32 - 1))
 
     # As long as we are not in the 1-D (PP-only) case, we will have a seed to use for all ranks of the SPMD mesh.
     # IF PP is also used, this seed is unique per PP rank.