Initial commit.

LICENSE.md

@@ -0,0 +1,20 @@
+MIT License
+
+Copyright 2022 National Technology & Engineering Solutions of Sandia, LLC (NTESS). Under the terms
+of Contract DE-NA0003525 with NTESS, the U.S. Government retains certain rights in this software.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+associated documentation files (the "Software"), to deal in the Software without restriction,
+including without limitation the rights to use, copy, modify, merge, publish, distribute,
+sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial
+portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES
+OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

README.md

@@ -0,0 +1,29 @@
+hod-carrier Overview
+====================
+**What is "hod-carrier"?**
+
+The hod-carrier library is designed to move data between a host and a BlueField DPU using Infiniband
+RDMA.  The basic design is that the DPU runs a server and the host runs a client.  The API is
+structured so that a program executing on the host can issue a request to the server running on the
+DPU to transfer data from host memory to BlueField2 DPU memory.
+
+This repository contains the code necessary to build four components:
+- *libhodclient* : a software library containing the code necessary to run an instance of a
+  hod-carrier client
+
+- *libhodserver* : a software library containing the code necessary to run an instance of a
+  hod-carrier server
+
+- *hod_client_app* : a simple example of an executable that launches a hod-carrier client and uses
+  it request the server running on a BlueField DPU to transfer data from host memory to DPU memory.
+
+- *hod_server_app* : a simple example of an executable that launches a hod-carrier server and
+  services requests from a client instance running on the host processor.
+
+**Why "hod-carrier"?**
+
+A hod carrier is a person who delivers construction supplies (e.g., mortar, brick, stone) to
+bricklayers and stone masons, commonly by employing a brick hod.  A brick hod is an exceedingly
+simple device to simplify the transport of these materials.  Similarly, this software library is a
+very simple device for moving data between a host and a BlueField DPU.  As use cases emerge for
+this service, we expect that the features and sophistication of this library will grow.

data/bluefield_test.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+# Copyright 2022 National Technology & Engineering Solutions of Sandia, LLC 
+# (NTESS).  Under the terms of Contract DE-NA0003525 with NTESS, the U.S. 
+# Government retains certain rights in this software.
+
+PWD=`pwd`
+HOSTLIST=`./process_nodelist_hosts.py`
+CLIENTEXE=${PWD}/../src/build-host/hod_client_app
+SERVEREXE=${PWD}/../src/build-bf/hod_server_app
+
+CLIENT_PORT=65432
+BF_LIBRARY_PATH=${LD_LIBRARY_PATH}:${PWD}/../src/build-bf/
+ENV_VARS="export LD_LIBRARY_PATH=$BF_LIBRARY_PATH; "
+ENV_VARS+="export IBVERBS_CLIENT_SOCKET_PORT=${CLIENT_PORT}; "
+
+echo "SLURM_JOB_NODELIST = $SLURM_JOB_NODELIST" 
+
+for HOST in $HOSTLIST; do
+  BF="${HOST}-bf"
+  CLIENTADDR=`./lookup_host_ip.py $HOST`
+  ENV_VARS+="export IBVERBS_CLIENT_IP_ADDR=$CLIENTADDR;"
+  ENV_VARS+="export IBVERBS_SERVER_DEVICE=mlx5_0;"
+  echo "CLIENTADDR = ${CLIENTADDR}"
+  echo "ENV_VARS = ${ENV_VARS}"
+  ssh -o "StrictHostKeyChecking no" $BF ${ENV_VARS} $SERVEREXE &
+done
+
+HOST_LIBRARY_PATH=${LD_LIBRARY_PATH}:${PWD}/../src/build-host/
+echo "HOST_LIBRARY_PATH=${HOST_LIBRARY_PATH}"
+export IBVERBS_CLIENT_SOCKET_PORT=${CLIENT_PORT}
+export IBVERBS_CLIENT_SOCKET_IF=eth1
+export LD_LIBRARY_PATH=$HOST_LIBRARY_PATH
+srun -n 1 -N 1 $CLIENTEXE &
+
+wait

data/lookup_host_ip.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+
+# Copyright 2022 National Technology & Engineering Solutions of Sandia, LLC 
+# (NTESS).  Under the terms of Contract DE-NA0003525 with NTESS, the U.S. 
+# Government retains certain rights in this software.
+
+import socket
+import sys
+
+def lookup_host_ip(hostname):
+  ipaddr = socket.gethostbyname(hostname)
+  return ipaddr
+
+
+if __name__ == "__main__":
+  ipaddr = lookup_host_ip(sys.argv[1])
+  print(ipaddr)

data/process_nodelist_hosts.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+
+# Copyright 2022 National Technology & Engineering Solutions of Sandia, LLC 
+# (NTESS).  Under the terms of Contract DE-NA0003525 with NTESS, the U.S. 
+# Government retains certain rights in this software.
+
+import re
+import sys
+import os
+
+# [sllevy@klogin2 ~]$ srun -N 1 -n 1 echo $SLURM_JOB_NODELIST
+# gn[3-4]
+
+nodelist_re = re.compile("(?P<partition>[a-z-]+(?:-bf)?)\[?(?P<nodenumbers>[0-9,-]+)\]?")
+noderange_re = re.compile("(?P<start>\d+)-(?P<end>\d+)")
+
+# STRING for aggregating results
+result = ""
+
+"""
+if len(sys.argv) == 2:
+  slurm_nodelist = sys.argv[1]
+  ppn = int(sys.argv[2])
+else:
+  print("USAGE: %s <nodelist> <ppn>" % (sys.argv[0]))
+  exit(-1)
+"""
+
+slurm_nodelist = os.environ['SLURM_JOB_NODELIST']
+nodelist_strings = re.findall("[a-z-]+(?:-bf)?\[?[0-9,-]+\]?", slurm_nodelist)
+
+## There should be BlueField nodes
+# !!!! DEEbug !!!!
+#assert any(map(lambda x: "-bf" in x, nodelist_strings))
+
+for nodelist_string in nodelist_strings:
+  if "-bf" in nodelist_string:
+    continue
+  nodelist_match = nodelist_re.match(nodelist_string)
+  if nodelist_match != None:
+    nodenumbers = nodelist_match.group("nodenumbers")
+    partition = nodelist_match.group("partition")
+
+    for nodenumber in nodenumbers.split(","):
+      noderange_match = noderange_re.match(nodenumber)
+      if noderange_match == None:
+        start = int(nodenumber)
+        end = int(nodenumber)
+      else:
+        start = int(noderange_match.group("start"))
+        end = int(noderange_match.group("end"))
+
+      for n in range(start,end+1):
+        if len(result) > 0:
+          #result += ","
+          result += " "
+        #result += ("%s%d:%d" % (partition,n,ppn))
+        result += ("%s%d" % (partition,n))
+  else:
+    print("WHOOPS: no match")
+    exit(-1)
+
+print(result)

data/run-bf-test.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# Copyright 2022 National Technology & Engineering Solutions of Sandia, LLC 
+# (NTESS).  Under the terms of Contract DE-NA0003525 with NTESS, the U.S. 
+# Government retains certain rights in this software.
+
+# ALL command line arguments to this script will be passed to the salloc command
+# For example, on kahuna at SNL calling this script with "-p glinda" will run it on the partition
+# that includes BF2 devices.
+salloc -N 1 $@ ./bluefield_test.sh