diff --git a/dgraph/dgraph.go b/dgraph/dgraph.go index 9c926ace..24cf9d7f 100644 --- a/dgraph/dgraph.go +++ b/dgraph/dgraph.go @@ -7,6 +7,7 @@ package dgraph import ( "context" "fmt" + "hmruntime/config" "hmruntime/utils" ) diff --git a/functions/ashelpers_test.go b/functions/ashelpers_test.go index 69aa3c2c..b240be3f 100644 --- a/functions/ashelpers_test.go +++ b/functions/ashelpers_test.go @@ -6,8 +6,9 @@ package functions import ( "bytes" - "hmruntime/testutils" "testing" + + "hmruntime/testutils" ) // "Hello World" in Japanese diff --git a/functions/env_test.go b/functions/env_test.go index a1d806d9..71965f99 100644 --- a/functions/env_test.go +++ b/functions/env_test.go @@ -5,9 +5,10 @@ package functions import ( - "hmruntime/testutils" "testing" "time" + + "hmruntime/testutils" ) func Test_DateNow(t *testing.T) { diff --git a/functions/fncall.go b/functions/fncall.go index d1499aee..fd9b4e0d 100644 --- a/functions/fncall.go +++ b/functions/fncall.go @@ -8,9 +8,10 @@ import ( "context" "encoding/json" "fmt" + "time" + "hmruntime/logger" "hmruntime/schema" - "time" "github.com/dgraph-io/gqlparser/ast" wasm "github.com/tetratelabs/wazero/api" diff --git a/functions/registration.go b/functions/registration.go index 2c454b4e..6092547c 100644 --- a/functions/registration.go +++ b/functions/registration.go @@ -6,11 +6,12 @@ package functions import ( "context" + "reflect" + "strings" + "hmruntime/host" "hmruntime/logger" "hmruntime/schema" - "reflect" - "strings" ) // map that holds the function info for each resolver @@ -46,9 +47,10 @@ func registerFunctions(ctx context.Context, gqlSchema string) error { // Build a map of resolvers to function info, including the plugin name. // If there are function name conflicts between plugins, the last plugin loaded wins. - for pluginName, cm := range host.CompiledModules { + for pluginName, plugin := range host.Plugins { for _, scma := range funcSchemas { - for _, fn := range cm.ExportedFunctions() { + module := *plugin.Module + for _, fn := range module.ExportedFunctions() { fnName := fn.ExportNames()[0] if strings.EqualFold(fnName, scma.FunctionName()) { info := schema.FunctionInfo{PluginName: pluginName, Schema: scma} @@ -78,8 +80,8 @@ func registerFunctions(ctx context.Context, gqlSchema string) error { break } } - _, foundModule := host.CompiledModules[info.PluginName] - if !foundSchema || !foundModule { + _, foundPlugin := host.Plugins[info.PluginName] + if !foundSchema || !foundPlugin { delete(FunctionsMap, resolver) logger.Info(ctx). Str("resolver", resolver). diff --git a/functions/schemaWatcher.go b/functions/schemaWatcher.go index 95b096a3..998aeab6 100644 --- a/functions/schemaWatcher.go +++ b/functions/schemaWatcher.go @@ -7,12 +7,13 @@ package functions import ( "context" "errors" + "net/url" + "time" + "hmruntime/config" "hmruntime/dgraph" "hmruntime/host" "hmruntime/logger" - "net/url" - "time" ) // Holds the current GraphQL schema diff --git a/host/host.go b/host/host.go index 9795f4d7..41b13339 100644 --- a/host/host.go +++ b/host/host.go @@ -9,8 +9,8 @@ import "github.com/tetratelabs/wazero" // runtime instance for the WASM modules var WasmRuntime wazero.Runtime -// map that holds the compiled modules for each plugin -var CompiledModules = make(map[string]wazero.CompiledModule) +// map that holds all of the loaded plugins, indexed by their name +var Plugins = make(map[string]Plugin) // Channel used to signal that registration is needed var RegistrationRequest chan bool = make(chan bool) diff --git a/plugins/loader.go b/host/loader.go similarity index 95% rename from plugins/loader.go rename to host/loader.go index f87fb131..6a55d589 100644 --- a/plugins/loader.go +++ b/host/loader.go @@ -2,21 +2,21 @@ * Copyright 2023 Hypermode, Inc. */ -package plugins +package host import ( "context" "fmt" - "hmruntime/aws" - "hmruntime/config" - "hmruntime/host" - "hmruntime/logger" "os" "path" "regexp" "strings" "time" + "hmruntime/aws" + "hmruntime/config" + "hmruntime/logger" + "github.com/radovskyb/watcher" ) @@ -45,9 +45,9 @@ func ReloadPlugins(ctx context.Context) error { } // Unload any plugins that are no longer present - for name := range host.CompiledModules { + for name := range Plugins { if !loaded[name] { - err := unloadPluginModule(ctx, name) + err := unloadPlugin(ctx, name) if err != nil { return fmt.Errorf("failed to unload plugin '%s': %w", name, err) } @@ -124,7 +124,7 @@ func loadPlugins(ctx context.Context) (map[string]bool, error) { } for plugin := range plugins { - err := loadPluginModule(ctx, plugin) + err := loadPlugin(ctx, plugin) if err != nil { logger.Err(ctx, err). Str("plugin", plugin). @@ -143,7 +143,7 @@ func loadPlugins(ctx context.Context) (map[string]bool, error) { // If the plugins path is a single plugin's base directory, load the single plugin. if _, err := os.Stat(config.PluginsPath + "/build/debug.wasm"); err == nil { pluginName := path.Base(config.PluginsPath) - err := loadPluginModule(ctx, pluginName) + err := loadPlugin(ctx, pluginName) if err != nil { logger.Err(ctx, err). Str("plugin", pluginName). @@ -177,7 +177,7 @@ func loadPlugins(ctx context.Context) (map[string]bool, error) { } // Load the plugin - err := loadPluginModule(ctx, pluginName) + err := loadPlugin(ctx, pluginName) if err != nil { logger.Err(ctx, err). Str("plugin", pluginName). @@ -201,7 +201,7 @@ func WatchForJsonChanges(ctx context.Context) error { func WatchForPluginChanges(ctx context.Context) error { if config.NoReload { - logger.Warn(ctx).Msg("Automatic plugin reloading is disabled. Restart the server to load new or modified plugins.") + logger.Warn(ctx).Msg("Automatic plugin reloading is disabled. Restart the server to load new or modified host.") return nil } @@ -275,14 +275,14 @@ func watchDirectoryForPluginChanges(ctx context.Context) error { switch evt.Op { case watcher.Create, watcher.Write: - err = loadPluginModule(ctx, pluginName) + err = loadPlugin(ctx, pluginName) if err != nil { logger.Err(ctx, err). Str("plugin", pluginName). Msg("Failed to load plugin.") } case watcher.Remove: - err = unloadPluginModule(ctx, pluginName) + err = unloadPlugin(ctx, pluginName) if err != nil { logger.Err(ctx, err). Str("plugin", pluginName). @@ -291,7 +291,7 @@ func watchDirectoryForPluginChanges(ctx context.Context) error { } // Signal that we need to register functions - host.RegistrationRequest <- true + RegistrationRequest <- true case err := <-w.Error: logger.Err(ctx, err).Msg("Failure while watching plugin directory.") @@ -440,7 +440,7 @@ func watchStorageForPluginChanges(ctx context.Context) error { // Load/reload any new or modified plugins for name, etag := range plugins { if awsPlugins[name] != etag { - err := loadPluginModule(ctx, name) + err := loadPlugin(ctx, name) if err != nil { logger.Err(ctx, err). Str("plugin", name). @@ -454,7 +454,7 @@ func watchStorageForPluginChanges(ctx context.Context) error { // Unload any plugins that are no longer present for name := range awsPlugins { if _, found := plugins[name]; !found { - err := unloadPluginModule(ctx, name) + err := unloadPlugin(ctx, name) if err != nil { logger.Err(ctx, err). Str("plugin", name). @@ -467,7 +467,7 @@ func watchStorageForPluginChanges(ctx context.Context) error { // If anything changed, signal that we need to register functions if changed { - host.RegistrationRequest <- true + RegistrationRequest <- true } select { diff --git a/plugins/management.go b/host/management.go similarity index 74% rename from plugins/management.go rename to host/management.go index a8ac6d86..e8f04200 100644 --- a/plugins/management.go +++ b/host/management.go @@ -2,23 +2,22 @@ * Copyright 2023 Hypermode, Inc. */ -package plugins +package host import ( "context" "encoding/json" "fmt" - "hmruntime/aws" - "hmruntime/config" - "hmruntime/functions" - "hmruntime/host" - "hmruntime/logger" "io" "os" "reflect" "runtime" "strings" + "hmruntime/aws" + "hmruntime/config" + "hmruntime/logger" + "github.com/google/uuid" "github.com/rs/zerolog" "github.com/tetratelabs/wazero" @@ -54,12 +53,6 @@ func InitWasmRuntime(ctx context.Context) (wazero.Runtime, error) { return nil, err } - // Connect Hypermode host functions - err = functions.InstantiateHostFunctions(ctx, runtime) - if err != nil { - return nil, err - } - return runtime, nil } @@ -125,37 +118,68 @@ func getJsonBytes(ctx context.Context, name string) ([]byte, error) { return bytes, nil } -func loadPluginModule(ctx context.Context, name string) error { - _, reloading := host.CompiledModules[name] +func loadPlugin(ctx context.Context, name string) error { + _, reloading := Plugins[name] logger.Info(ctx). - Str("plugin", name). + Str("filename", name+".wasm"). Bool("reloading", reloading). Msg("Loading plugin.") + // TODO: Separate plugin name from file name throughout the codebase. + // The plugin name should always come from the metadata, not the file name. + // This requires significant changes, so it's not done yet. + // Load the binary content of the plugin. - plugin, err := getPluginBytes(ctx, name) + bytes, err := getPluginBytes(ctx, name) if err != nil { return err } // Compile the plugin into a module. - cm, err := host.WasmRuntime.CompileModule(ctx, plugin) + cm, err := WasmRuntime.CompileModule(ctx, bytes) if err != nil { return fmt.Errorf("failed to compile the plugin: %w", err) } - // Store the compiled module for later retrieval. - host.CompiledModules[name] = cm + // Get the metadata for the plugin. + metadata := getPluginMetadata(&cm) + if metadata == (PluginMetadata{}) { + logger.Warn(ctx). + Str("filename", name+".wasm"). + Msg("No metadata found in plugin. Please recompile your plugin using the latest version of the Hypermode Functions library.") + } - // TODO: We should close the old module, but that leaves the _new_ module in an invalid state, - // giving an error when querying: "source module must be compiled before instantiation" - // if reloading { - // cmOld.Close(ctx) - // } + // Finally, store the plugin to complete the loading process. + Plugins[name] = Plugin{&cm, metadata} + logPluginLoaded(ctx, metadata) return nil } +func logPluginLoaded(ctx context.Context, metadata PluginMetadata) { + evt := logger.Info(ctx) + + if metadata != (PluginMetadata{}) { + evt.Str("plugin", metadata.Name) + evt.Str("version", metadata.Version) + evt.Str("language", metadata.Language.String()) + evt.Str("build_id", metadata.BuildId) + evt.Time("build_time", metadata.BuildTime) + evt.Str("hypermode_library", metadata.LibraryName) + evt.Str("hypermode_library_version", metadata.LibraryVersion) + + if metadata.GitRepo != "" { + evt.Str("git_repo", metadata.GitRepo) + } + + if metadata.GitCommit != "" { + evt.Str("git_commit", metadata.GitCommit) + } + } + + evt.Msg("Loaded plugin.") +} + func getPluginBytes(ctx context.Context, name string) ([]byte, error) { if aws.UseAwsForPluginStorage() { @@ -180,8 +204,8 @@ func getPluginBytes(ctx context.Context, name string) ([]byte, error) { return bytes, nil } -func unloadPluginModule(ctx context.Context, name string) error { - cmOld, found := host.CompiledModules[name] +func unloadPlugin(ctx context.Context, name string) error { + plugin, found := Plugins[name] if !found { return fmt.Errorf("plugin not found '%s'", name) } @@ -190,8 +214,9 @@ func unloadPluginModule(ctx context.Context, name string) error { Str("plugin", name). Msg("Unloading plugin.") - delete(host.CompiledModules, name) - cmOld.Close(ctx) + mod := *plugin.Module + delete(Plugins, name) + mod.Close(ctx) return nil } @@ -209,10 +234,10 @@ func GetModuleInstance(ctx context.Context, pluginName string) (wasm.Module, buf wOut := io.MultiWriter(buf.Stdout, wInfoLog) wErr := io.MultiWriter(buf.Stderr, wErrorLog) - // Get the compiled module. - compiled, ok := host.CompiledModules[pluginName] + // Get the plugin. + plugin, ok := Plugins[pluginName] if !ok { - return nil, buf, fmt.Errorf("no compiled module found for plugin '%s'", pluginName) + return nil, buf, fmt.Errorf("plugin not found with name '%s'", pluginName) } // Configure the module instance. @@ -224,7 +249,7 @@ func GetModuleInstance(ctx context.Context, pluginName string) (wasm.Module, buf // Instantiate the plugin as a module. // NOTE: This will also invoke the plugin's `_start` function, // which will call any top-level code in the plugin. - mod, err := host.WasmRuntime.InstantiateModule(ctx, compiled, cfg) + mod, err := WasmRuntime.InstantiateModule(ctx, *plugin.Module, cfg) if err != nil { return nil, buf, fmt.Errorf("failed to instantiate the plugin module: %w", err) } diff --git a/host/plugins.go b/host/plugins.go new file mode 100644 index 00000000..35c65665 --- /dev/null +++ b/host/plugins.go @@ -0,0 +1,94 @@ +/* + * Copyright 2024 Hypermode, Inc. + */ + +package host + +import ( + "strings" + "time" + + "github.com/tetratelabs/wazero" +) + +type Plugin struct { + Module *wazero.CompiledModule + Metadata PluginMetadata +} + +type PluginMetadata struct { + Name string + Version string + Language PluginLanguage + LibraryName string + LibraryVersion string + BuildId string + BuildTime time.Time + GitRepo string + GitCommit string +} + +type PluginLanguage int + +const ( + UnknownLanguage PluginLanguage = iota + AssemblyScript + GoLang +) + +func (lang PluginLanguage) String() string { + switch lang { + case AssemblyScript: + return "AssemblyScript" + case GoLang: + return "Go" + default: + return "Unknown" + } +} + +func getPluginLanguage(libraryName string) PluginLanguage { + switch libraryName { + case "@hypermode/functions-as": + return AssemblyScript + case "gohypermode/functions-go": + return GoLang + default: + return UnknownLanguage + } +} + +func parseNameAndVersion(s string) (name string, version string) { + i := strings.LastIndex(s, "@") + if i == -1 { + return s, "" + } + return s[:i], s[i+1:] +} + +func getPluginMetadata(cm *wazero.CompiledModule) PluginMetadata { + var metadata = PluginMetadata{} + + for _, sec := range (*cm).CustomSections() { + name := sec.Name() + data := sec.Data() + + switch name { + case "build_id": + metadata.BuildId = string(data) + case "build_ts": + metadata.BuildTime, _ = time.Parse(time.RFC3339, string(data)) + case "hypermode_library": + metadata.LibraryName, metadata.LibraryVersion = parseNameAndVersion(string(data)) + metadata.Language = getPluginLanguage(metadata.LibraryName) + case "hypermode_plugin": + metadata.Name, metadata.Version = parseNameAndVersion(string(data)) + case "git_repo": + metadata.GitRepo = string(data) + case "git_commit": + metadata.GitCommit = string(data) + } + } + + return metadata +} diff --git a/logger/logger.go b/logger/logger.go index 5786a7b5..19ff5d6a 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -6,10 +6,11 @@ package logger import ( "context" - "hmruntime/config" "os" "time" + "hmruntime/config" + "github.com/rs/zerolog" "github.com/rs/zerolog/log" ) diff --git a/main.go b/main.go index 49087d86..9eb97afd 100644 --- a/main.go +++ b/main.go @@ -14,7 +14,6 @@ import ( "hmruntime/functions" "hmruntime/host" "hmruntime/logger" - "hmruntime/plugins" "github.com/joho/godotenv" ) @@ -53,12 +52,18 @@ func main() { } // Initialize the WebAssembly runtime - host.WasmRuntime, err = plugins.InitWasmRuntime(ctx) + host.WasmRuntime, err = host.InitWasmRuntime(ctx) if err != nil { log.Fatal().Err(err).Msg("Failed to initialize the WebAssembly runtime. Exiting.") } defer host.WasmRuntime.Close(ctx) + // Connect Hypermode host functions + err = functions.InstantiateHostFunctions(ctx, host.WasmRuntime) + if err != nil { + log.Fatal().Err(err).Msg("Failed to instantiate host functions. Exiting.") + } + // Load environment variables from plugins path // note: .env file is optional, so don't log if it's not found err = godotenv.Load(path.Join(config.PluginsPath, ".env")) @@ -67,13 +72,13 @@ func main() { } // Load json - err = plugins.LoadJsons(ctx) + err = host.LoadJsons(ctx) if err != nil { log.Fatal().Err(err).Msg("Failed to load hypermode.json. Exiting.") } // Load plugins - err = plugins.LoadPlugins(ctx) + err = host.LoadPlugins(ctx) if err != nil { log.Fatal().Err(err).Msg("Failed to load plugins. Exiting.") } @@ -85,13 +90,13 @@ func main() { functions.MonitorGqlSchema(ctx) // Watch for plugin changes - err = plugins.WatchForPluginChanges(ctx) + err = host.WatchForPluginChanges(ctx) if err != nil { log.Fatal().Err(err).Msg("Failed to watch for plugin changes. Exiting.") } // Watch for hypermode.json changes - err = plugins.WatchForJsonChanges(ctx) + err = host.WatchForJsonChanges(ctx) if err != nil { log.Fatal().Err(err).Msg("Failed to watch for hypermode.json changes. Exiting.") } diff --git a/server.go b/server.go index 48892a9f..d8768302 100644 --- a/server.go +++ b/server.go @@ -8,12 +8,13 @@ import ( "context" "encoding/json" "fmt" + "net/http" + "strings" + "hmruntime/config" "hmruntime/functions" + "hmruntime/host" "hmruntime/logger" - "hmruntime/plugins" - "net/http" - "strings" "github.com/rs/xid" ) @@ -71,7 +72,7 @@ func handleRequest(w http.ResponseWriter, r *http.Request) { // Each request will get its own instance of the plugin module, // so that we can run multiple requests in parallel without risk // of corrupting the module's memory. - mod, buf, err := plugins.GetModuleInstance(ctx, info.PluginName) + mod, buf, err := host.GetModuleInstance(ctx, info.PluginName) if err != nil { logger.Err(ctx, err). Str("plugin", info.PluginName). @@ -239,7 +240,7 @@ func handleAdminRequest(w http.ResponseWriter, r *http.Request) { // Perform the requested action switch req.Action { case "reload": - err = plugins.ReloadPlugins(r.Context()) + err = host.ReloadPlugins(r.Context()) default: err = fmt.Errorf("unknown action: %s", req.Action) }