Jai/hyp 2345 add local model endpoint flow invocations

CHANGELOG.md

@@ -7,6 +7,7 @@
 - Import Manifest code [#416](https://github.com/hypermodeinc/modus/pull/416)
 - Update the runtime's manifest usage [#417](https://github.com/hypermodeinc/modus/pull/417)
 - Add Modus Go SDK [#418](https://github.com/hypermodeinc/modus/pull/418)
+- Add Local Model Invocation Support [#421](https://github.com/hypermodeinc/modus/pull/421)
 
 ## 2024-10-02 - Version 0.12.7
 

runtime/main.go

@@ -43,6 +43,12 @@ func main() {
 	if err != nil && !os.IsNotExist(err) {
 		log.Warn().Err(err).Msg("Error reading .env file.  Ignoring.")
 	}
+	if config.IsDevEnvironment() {
+		err = godotenv.Load(filepath.Join(config.StoragePath, ".env.local"))
+		if err != nil && !os.IsNotExist(err) {
+			log.Warn().Err(err).Msg("Error reading .env.local file.  Ignoring.")
+		}
+	}
 
 	// Initialize Sentry
 	rootSourcePath := getRootSourcePath()

runtime/models/models.go

@@ -24,6 +24,8 @@ import (
 	"github.com/hypermodeinc/modus/runtime/utils"
 )
 
+var localHypermodeModels = map[string]bool{"meta-llama/meta-llama-3.1-8b-instruct": true, "sentence-transformers/all-minilm-l6-v2": true, "antoinemc/distilbart-mnli-github-issues": true, "distilbert/distilbert-base-uncased-finetuned-sst-2-english": true}
+
 func GetModel(modelName string) (*manifest.ModelInfo, error) {
 	model, ok := manifestdata.GetManifest().Models[modelName]
 	if !ok {
@@ -76,7 +78,9 @@ func PostToModelEndpoint[TResult any](ctx context.Context, model *manifest.Model
 		if host.Name != hosts.HypermodeHost {
 			return secrets.ApplyHostSecretsToHttpRequest(ctx, host, req)
 		}
-
+		if config.IsDevEnvironment() {
+			return secrets.ApplyAuthToLocalModelRequest(ctx, host, req)
+		}
 		return nil
 	}
 
@@ -99,6 +103,13 @@ func getModelEndpointAndHost(model *manifest.ModelInfo) (string, *manifest.HTTPH
 	}
 
 	if host.Name == hosts.HypermodeHost {
+		if config.IsDevEnvironment() {
+			if !isValidLocalHypermodeModel(model.SourceModel) {
+				return "", host, fmt.Errorf("model %s is not available in the local Hypermode environment", model.SourceModel)
+			}
+			endpoint := fmt.Sprintf("https://models.hypermode.host/%s", strings.ToLower(model.SourceModel))
+			return endpoint, host, nil
+		}
 		// Compose the Hypermode hosted model endpoint URL.
 		// Example: http://modelname.bckid.svc.cluster.local/v1/models/modelname:predict
 		endpoint := fmt.Sprintf("http://%s.%s/%[1]s:predict", strings.ToLower(model.Name), config.ModelHost)
@@ -123,3 +134,8 @@ func getModelEndpointAndHost(model *manifest.ModelInfo) (string, *manifest.HTTPH
 
 	return host.Endpoint, host, nil
 }
+
+func isValidLocalHypermodeModel(modelName string) bool {
+	_, ok := localHypermodeModels[strings.ToLower(modelName)]
+	return ok
+}

runtime/secrets/secrets.go

@@ -14,6 +14,7 @@ import (
 	"encoding/base64"
 	"fmt"
 	"net/http"
+	"os"
 	"regexp"
 
 	"github.com/hypermodeinc/modus/pkg/manifest"
@@ -93,6 +94,21 @@ func ApplyHostSecretsToHttpRequest(ctx context.Context, host *manifest.HTTPHostI
 	return nil
 }
 
+func ApplyAuthToLocalModelRequest(ctx context.Context, host manifest.HostInfo, req *http.Request) error {
+
+	jwt := os.Getenv("HYP_JWT")
+	orgId := os.Getenv("HYP_ORG_ID")
+
+	if jwt == "" || orgId == "" {
+		return fmt.Errorf("missing HYP_JWT or HYP_ORG_ID environment variables, login to Hypermode using 'hyp login'")
+	}
+
+	req.Header.Set("Authorization", "Bearer "+jwt)
+	req.Header.Set("HYP-ORG-ID", orgId)
+
+	return nil
+}
+
 // ApplyHostSecretsToString evaluates the given string and replaces any placeholders
 // present in the string with their secret values for the given host.
 func ApplyHostSecretsToString(ctx context.Context, host manifest.HostInfo, str string) (string, error) {