diff --git a/CHANGELOG.md b/CHANGELOG.md index af3ef12561..d48cc5df9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,8 @@ Main (unreleased) - SNMP exporter now supports labels in both `target` and `targets` parameters. (@mattdurham) +- Add support for relative paths to `import.file`. This new functionality allows users to use `import.file` blocks in modules + imported via `import.git` and other `import.file`. (@wildum) ### Bugfixes diff --git a/docs/sources/reference/config-blocks/import.file.md b/docs/sources/reference/config-blocks/import.file.md index 09046a0a43..79cd2a98e1 100644 --- a/docs/sources/reference/config-blocks/import.file.md +++ b/docs/sources/reference/config-blocks/import.file.md @@ -17,6 +17,9 @@ Imported directories are treated as single modules to support composability. That means that you can define a custom component in one file and use it in another custom component in another file in the same directory. +You can use the keyword `module_path` in combination with the `stdlib` function [file.path_join][] to import a module relative to the current module's path. +The `module_path` keyword works for modules that are imported via `import.file`, `import.git`, and `import.string`. + ## Usage ```alloy @@ -37,12 +40,25 @@ The following arguments are supported: {{< docs/shared lookup="reference/components/local-file-arguments-text.md" source="alloy" version="" >}} -## Example +## Examples + +### Import a module from a local file This example imports a module from a file and instantiates a custom component from the import that adds two numbers: -{{< collapse title="module.alloy" >}} +main.alloy +```alloy +import.file "math" { + filename = "module.alloy" +} +math.add "default" { + a = 15 + b = 45 +} +``` + +module.alloy ```alloy declare "add" { argument "a" {} @@ -54,13 +70,67 @@ declare "add" { } ``` -{{< /collapse >}} +### Import a module in a module imported via import.git + +This example imports a module from a file inside of a module that is imported via [import.git][]: + +main.alloy +```alloy +import.git "math" { + repository = "https://github.com/wildum/module.git" + path = "relative_math.alloy" + revision = "master" +} + +math.add "default" { + a = 15 + b = 45 +} +``` + + +relative_math.alloy +```alloy +import.file "lib" { + filename = file.path_join(module_path, "lib.alloy") +} + +declare "add" { + argument "a" {} + argument "b" {} + + lib.plus "default" { + a = argument.a.value + b = argument.b.value + } + + export "output" { + value = lib.plus.default.sum + } +} +``` + +lib.alloy +```alloy +declare "plus" { + argument "a" {} + argument "b" {} + + export "sum" { + value = argument.a.value + argument.b.value + } +} +``` + +### Import a module in a module imported via import.file -{{< collapse title="importer.alloy" >}} +This example imports a module from a file inside of a module that is imported via another `import.file`: + +main.alloy ```alloy import.file "math" { - filename = "module.alloy" + filename = "path/to/module/relative_math.alloy" } math.add "default" { @@ -69,4 +139,40 @@ math.add "default" { } ``` -{{< /collapse >}} +relative_math.alloy +```alloy +import.file "lib" { + filename = file.path_join(module_path, "lib.alloy") +} + +declare "add" { + argument "a" {} + argument "b" {} + + lib.plus "default" { + a = argument.a.value + b = argument.b.value + } + + export "output" { + value = lib.plus.default.sum + } +} +``` + +lib.alloy +```alloy +declare "plus" { + argument "a" {} + argument "b" {} + + export "sum" { + value = argument.a.value + argument.b.value + } +} +``` + + + +[file.path_join]: ../../stdlib/file/ +[import.git]: ../import.git/ \ No newline at end of file diff --git a/docs/sources/reference/config-blocks/import.git.md b/docs/sources/reference/config-blocks/import.git.md index 81ba649469..6aad5cd069 100644 --- a/docs/sources/reference/config-blocks/import.git.md +++ b/docs/sources/reference/config-blocks/import.git.md @@ -9,6 +9,9 @@ title: import.git The `import.git` block imports custom components from a Git repository and exposes them to the importer. `import.git` blocks must be given a label that determines the namespace where custom components are exposed. +The entire repository is cloned, and the module path is accessible via the `module_path` keyword. +This enables, for example, your module to import other modules within the repository by setting relative paths in the [import.file][] blocks. + ## Usage ```alloy @@ -101,5 +104,6 @@ math.add "default" { } ``` +[import.file]: ../import.file/ [basic_auth]: #basic_auth-block [ssh_key]: #ssh_key-block diff --git a/docs/sources/reference/config-blocks/import.http.md b/docs/sources/reference/config-blocks/import.http.md index 77f791424d..552581851b 100644 --- a/docs/sources/reference/config-blocks/import.http.md +++ b/docs/sources/reference/config-blocks/import.http.md @@ -78,7 +78,7 @@ The `tls_config` block configures TLS settings for connecting to HTTPS servers. This example imports custom components from an HTTP response and instantiates a custom component for adding two numbers: -{{< collapse title="HTTP response" >}} +module.alloy ```alloy declare "add" { argument "a" {} @@ -89,9 +89,8 @@ declare "add" { } } ``` -{{< /collapse >}} -{{< collapse title="importer.alloy" >}} +main.alloy ```alloy import.http "math" { url = SERVER_URL @@ -102,7 +101,7 @@ math.add "default" { b = 45 } ``` -{{< /collapse >}} + [client]: #client-block [basic_auth]: #basic_auth-block diff --git a/internal/alloycli/cmd_run.go b/internal/alloycli/cmd_run.go index 9cbbbb27a7..89357f2323 100644 --- a/internal/alloycli/cmd_run.go +++ b/internal/alloycli/cmd_run.go @@ -292,6 +292,7 @@ func (fr *alloyRun) Run(configPath string) error { remoteCfgService, err := remotecfgservice.New(remotecfgservice.Options{ Logger: log.With(l, "service", "remotecfg"), + ConfigPath: configPath, StoragePath: fr.storagePath, Metrics: reg, }) @@ -340,7 +341,7 @@ func (fr *alloyRun) Run(configPath string) error { if err != nil { return nil, fmt.Errorf("reading config path %q: %w", configPath, err) } - if err := f.LoadSource(alloySource, nil); err != nil { + if err := f.LoadSource(alloySource, nil, configPath); err != nil { return alloySource, fmt.Errorf("error during the initial load: %w", err) } diff --git a/internal/converter/internal/test_common/testing.go b/internal/converter/internal/test_common/testing.go index 98b5377a45..a315f0362c 100644 --- a/internal/converter/internal/test_common/testing.go +++ b/internal/converter/internal/test_common/testing.go @@ -217,7 +217,7 @@ func attemptLoadingAlloyConfig(t *testing.T, bb []byte) { }, EnableCommunityComps: true, }) - err = f.LoadSource(cfg, nil) + err = f.LoadSource(cfg, nil, "") // Many components will fail to build as e.g. the cert files are missing, so we ignore these errors. // This is not ideal, but we still validate for other potential issues. diff --git a/internal/runtime/alloy.go b/internal/runtime/alloy.go index 4479ae13c3..613fc204da 100644 --- a/internal/runtime/alloy.go +++ b/internal/runtime/alloy.go @@ -56,11 +56,14 @@ import ( "github.com/grafana/alloy/internal/featuregate" "github.com/grafana/alloy/internal/runtime/internal/controller" + "github.com/grafana/alloy/internal/runtime/internal/importsource" "github.com/grafana/alloy/internal/runtime/internal/worker" "github.com/grafana/alloy/internal/runtime/logging" "github.com/grafana/alloy/internal/runtime/logging/level" "github.com/grafana/alloy/internal/runtime/tracing" "github.com/grafana/alloy/internal/service" + "github.com/grafana/alloy/internal/util" + "github.com/grafana/alloy/syntax/vm" ) // Options holds static options for an Alloy controller. @@ -296,22 +299,37 @@ func (f *Runtime) Run(ctx context.Context) { // The controller will only start running components after Load is called once // without any configuration errors. // LoadSource uses default loader configuration. -func (f *Runtime) LoadSource(source *Source, args map[string]any) error { - return f.loadSource(source, args, nil) +func (f *Runtime) LoadSource(source *Source, args map[string]any, configPath string) error { + modulePath, err := util.ExtractDirPath(configPath) + if err != nil { + level.Warn(f.log).Log("msg", "failed to extract directory path from configPath", "configPath", configPath, "err", err) + } + return f.applyLoaderConfig(controller.ApplyOptions{ + Args: args, + ComponentBlocks: source.components, + ConfigBlocks: source.configBlocks, + DeclareBlocks: source.declareBlocks, + ArgScope: vm.NewScope(map[string]interface{}{ + importsource.ModulePath: modulePath, + }), + }) } // Same as above but with a customComponentRegistry that provides custom component definitions. func (f *Runtime) loadSource(source *Source, args map[string]any, customComponentRegistry *controller.CustomComponentRegistry) error { - f.loadMut.Lock() - defer f.loadMut.Unlock() - - applyOptions := controller.ApplyOptions{ + return f.applyLoaderConfig(controller.ApplyOptions{ Args: args, ComponentBlocks: source.components, ConfigBlocks: source.configBlocks, DeclareBlocks: source.declareBlocks, CustomComponentRegistry: customComponentRegistry, - } + ArgScope: customComponentRegistry.Scope(), + }) +} + +func (f *Runtime) applyLoaderConfig(applyOptions controller.ApplyOptions) error { + f.loadMut.Lock() + defer f.loadMut.Unlock() diags := f.loader.Apply(applyOptions) if !f.loadedOnce.Load() && diags.HasErrors() { diff --git a/internal/runtime/alloy_services.go b/internal/runtime/alloy_services.go index ae22785eef..6c6171a7a6 100644 --- a/internal/runtime/alloy_services.go +++ b/internal/runtime/alloy_services.go @@ -93,12 +93,12 @@ type ServiceController struct { } func (sc ServiceController) Run(ctx context.Context) { sc.f.Run(ctx) } -func (sc ServiceController) LoadSource(b []byte, args map[string]any) error { +func (sc ServiceController) LoadSource(b []byte, args map[string]any, configPath string) error { source, err := ParseSource("", b) if err != nil { return err } - return sc.f.LoadSource(source, args) + return sc.f.LoadSource(source, args, configPath) } func (sc ServiceController) Ready() bool { return sc.f.Ready() } diff --git a/internal/runtime/alloy_services_test.go b/internal/runtime/alloy_services_test.go index 0e35261e09..14bc228afa 100644 --- a/internal/runtime/alloy_services_test.go +++ b/internal/runtime/alloy_services_test.go @@ -38,7 +38,7 @@ func TestServices(t *testing.T) { opts.Services = append(opts.Services, svc) ctrl := New(opts) - require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil)) + require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil, "")) // Start the controller. This should cause our service to run. go ctrl.Run(ctx) @@ -90,7 +90,7 @@ func TestServices_Configurable(t *testing.T) { ctrl := New(opts) - require.NoError(t, ctrl.LoadSource(f, nil)) + require.NoError(t, ctrl.LoadSource(f, nil, "")) // Start the controller. This should cause our service to run. go ctrl.Run(ctx) @@ -137,7 +137,7 @@ func TestServices_Configurable_Optional(t *testing.T) { ctrl := New(opts) - require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil)) + require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil, "")) // Start the controller. This should cause our service to run. go ctrl.Run(ctx) @@ -171,7 +171,7 @@ func TestAlloy_GetServiceConsumers(t *testing.T) { ctrl := New(opts) defer cleanUpController(ctrl) - require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil)) + require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil, "")) expectConsumers := []service.Consumer{{ Type: service.ConsumerTypeService, @@ -253,7 +253,7 @@ func TestComponents_Using_Services(t *testing.T) { ComponentRegistry: registry, ModuleRegistry: newModuleRegistry(), }) - require.NoError(t, ctrl.LoadSource(f, nil)) + require.NoError(t, ctrl.LoadSource(f, nil, "")) go ctrl.Run(ctx) require.NoError(t, componentBuilt.Wait(5*time.Second), "Component should have been built") @@ -332,7 +332,7 @@ func TestComponents_Using_Services_In_Modules(t *testing.T) { ComponentRegistry: registry, ModuleRegistry: newModuleRegistry(), }) - require.NoError(t, ctrl.LoadSource(f, nil)) + require.NoError(t, ctrl.LoadSource(f, nil, "")) go ctrl.Run(ctx) require.NoError(t, componentBuilt.Wait(5*time.Second), "Component should have been built") @@ -360,7 +360,7 @@ func TestNewControllerNoLeak(t *testing.T) { opts.Services = append(opts.Services, svc) ctrl := New(opts) - require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil)) + require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil, "")) // Start the controller. This should cause our service to run. go ctrl.Run(ctx) diff --git a/internal/runtime/alloy_test.go b/internal/runtime/alloy_test.go index a9c7efcf01..ebce0b656c 100644 --- a/internal/runtime/alloy_test.go +++ b/internal/runtime/alloy_test.go @@ -43,7 +43,7 @@ func TestController_LoadSource_Evaluation(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) require.Len(t, ctrl.loader.Components(), 4) @@ -54,6 +54,73 @@ func TestController_LoadSource_Evaluation(t *testing.T) { require.Equal(t, "hello, world!", out.(testcomponents.PassthroughExports).Output) } +var modulePathTestFile = ` + testcomponents.tick "ticker" { + frequency = "1s" + } + testcomponents.passthrough "static" { + input = module_path + } + testcomponents.passthrough "ticker" { + input = testcomponents.tick.ticker.tick_time + } + testcomponents.passthrough "forwarded" { + input = testcomponents.passthrough.ticker.output + } +` + +func TestController_LoadSource_WithModulePath_Evaluation(t *testing.T) { + defer verifyNoGoroutineLeaks(t) + ctrl := New(testOptions(t)) + defer cleanUpController(ctrl) + + f, err := ParseSource(t.Name(), []byte(modulePathTestFile)) + require.NoError(t, err) + require.NotNil(t, f) + + filePath := "tmp_modulePath_test/test/main.alloy" + require.NoError(t, os.Mkdir("tmp_modulePath_test", 0700)) + require.NoError(t, os.Mkdir("tmp_modulePath_test/test", 0700)) + defer os.RemoveAll("tmp_modulePath_test") + require.NoError(t, os.WriteFile(filePath, []byte(""), 0664)) + + err = ctrl.LoadSource(f, nil, filePath) + require.NoError(t, err) + require.Len(t, ctrl.loader.Components(), 4) + + // Check the inputs and outputs of things that should be immediately resolved + // without having to run the components. + in, out := getFields(t, ctrl.loader.Graph(), "testcomponents.passthrough.static") + require.Equal(t, "tmp_modulePath_test/test", in.(testcomponents.PassthroughConfig).Input) + require.Equal(t, "tmp_modulePath_test/test", out.(testcomponents.PassthroughExports).Output) +} + +func TestController_LoadSource_WithModulePathWithoutFileExtension_Evaluation(t *testing.T) { + defer verifyNoGoroutineLeaks(t) + ctrl := New(testOptions(t)) + defer cleanUpController(ctrl) + + f, err := ParseSource(t.Name(), []byte(modulePathTestFile)) + require.NoError(t, err) + require.NotNil(t, f) + + filePath := "tmp_modulePath_test/test/main" + require.NoError(t, os.Mkdir("tmp_modulePath_test", 0700)) + require.NoError(t, os.Mkdir("tmp_modulePath_test/test", 0700)) + defer os.RemoveAll("tmp_modulePath_test") + require.NoError(t, os.WriteFile(filePath, []byte(""), 0664)) + + err = ctrl.LoadSource(f, nil, filePath) + require.NoError(t, err) + require.Len(t, ctrl.loader.Components(), 4) + + // Check the inputs and outputs of things that should be immediately resolved + // without having to run the components. + in, out := getFields(t, ctrl.loader.Graph(), "testcomponents.passthrough.static") + require.Equal(t, "tmp_modulePath_test/test", in.(testcomponents.PassthroughConfig).Input) + require.Equal(t, "tmp_modulePath_test/test", out.(testcomponents.PassthroughExports).Output) +} + func getFields(t *testing.T, g *dag.Graph, nodeID string) (component.Arguments, component.Exports) { t.Helper() diff --git a/internal/runtime/alloy_updates_test.go b/internal/runtime/alloy_updates_test.go index 3bc4a631d3..cff70d898c 100644 --- a/internal/runtime/alloy_updates_test.go +++ b/internal/runtime/alloy_updates_test.go @@ -42,7 +42,7 @@ func TestController_Updates(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -122,7 +122,7 @@ func TestController_Updates_WithQueueFull(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -195,7 +195,7 @@ func TestController_Updates_WithLag(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -269,7 +269,7 @@ func TestController_Updates_WithOtherLaggingPipeline(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -338,7 +338,7 @@ func TestController_Updates_WithLaggingComponent(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) diff --git a/internal/runtime/declare_test.go b/internal/runtime/declare_test.go index d3c727a9eb..d5bb56d74f 100644 --- a/internal/runtime/declare_test.go +++ b/internal/runtime/declare_test.go @@ -336,7 +336,7 @@ func TestDeclare(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -358,6 +358,44 @@ func TestDeclare(t *testing.T) { } } +func TestDeclareModulePath(t *testing.T) { + defer verifyNoGoroutineLeaks(t) + config := ` + declare "mod" { + export "output" { + value = module_path + } + } + + mod "myModule" {} + + testcomponents.passthrough "pass" { + input = mod.myModule.output + } + ` + ctrl := runtime.New(testOptions(t)) + f, err := runtime.ParseSource(t.Name(), []byte(config)) + require.NoError(t, err) + require.NotNil(t, f) + + err = ctrl.LoadSource(f, nil, "") + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + ctrl.Run(ctx) + close(done) + }() + defer func() { + cancel() + <-done + }() + time.Sleep(30 * time.Millisecond) + passthrough := getExport[testcomponents.PassthroughExports](t, ctrl, "", "testcomponents.passthrough.pass") + require.Equal(t, passthrough.Output, "") +} + type errorTestCase struct { name string config string @@ -461,7 +499,7 @@ func TestDeclareError(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") if err == nil { t.Errorf("Expected error to match regex %q, but got: nil", tc.expectedError) } else if !tc.expectedError.MatchString(err.Error()) { @@ -545,7 +583,7 @@ func TestDeclareUpdateConfig(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -569,7 +607,7 @@ func TestDeclareUpdateConfig(t *testing.T) { require.NotNil(t, f) // Reload the controller with the new config. - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) require.Eventually(t, func() bool { diff --git a/internal/runtime/import_git_test.go b/internal/runtime/import_git_test.go index 6f1a922c6b..393e75faa1 100644 --- a/internal/runtime/import_git_test.go +++ b/internal/runtime/import_git_test.go @@ -56,7 +56,7 @@ testImport.add "cc" { defer verifyNoGoroutineLeaks(t) ctrl, f := setup(t, main) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -123,7 +123,7 @@ testImport.add "cc" { defer verifyNoGoroutineLeaks(t) ctrl, f := setup(t, main) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -206,7 +206,7 @@ testImport.add "cc" { defer verifyNoGoroutineLeaks(t) ctrl, f := setup(t, main) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -270,7 +270,7 @@ testImport.add "cc" { defer verifyNoGoroutineLeaks(t) ctrl, f := setup(t, main) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -355,7 +355,7 @@ testImport.add "cc" { defer verifyNoGoroutineLeaks(t) ctrl, f := setup(t, main) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") expectedErr := vcs.InvalidRevisionError{ Revision: "nonexistent", } diff --git a/internal/runtime/import_test.go b/internal/runtime/import_test.go index dab9400d31..f6a1ab499b 100644 --- a/internal/runtime/import_test.go +++ b/internal/runtime/import_test.go @@ -27,13 +27,15 @@ const mainFile = "main.alloy" // The tests are using the .txtar files stored in the testdata folder. type testImportFile struct { - description string // description at the top of the txtar file - main string // root config that the controller should load - module string // module imported by the root config - nestedModule string // nested module that can be imported by the module - reloadConfig string // root config that the controller should apply on reload - otherNestedModule string // another nested module - update *updateFile // update can be used to update the content of a file at runtime + description string // description at the top of the txtar file + main string // root config that the controller should load + module string // module imported by the root config + nestedModule string // nested module that can be imported by the module + reloadConfig string // root config that the controller should apply on reload + otherNestedModule string // another nested module + nestedPathModule string // a module in a subdirectory + deeplyNestedPathModule string // a module in a sub-subdirectory + update *updateFile // update can be used to update the content of a file at runtime } type updateFile struct { @@ -70,6 +72,10 @@ func buildTestImportFile(t *testing.T, filename string) testImportFile { tc.reloadConfig = string(alloyConfig.Data) case "other_nested_module.alloy": tc.otherNestedModule = string(alloyConfig.Data) + case "nested_test/module.alloy": + tc.nestedPathModule = string(alloyConfig.Data) + case "nested_test/utils/module.alloy": + tc.deeplyNestedPathModule = string(alloyConfig.Data) } } return tc @@ -91,6 +97,18 @@ func TestImportFile(t *testing.T) { require.NoError(t, os.WriteFile("other_nested_module.alloy", []byte(tc.otherNestedModule), 0664)) } + if tc.nestedPathModule != "" || tc.deeplyNestedPathModule != "" { + require.NoError(t, os.Mkdir("nested_test", 0700)) + defer os.RemoveAll("nested_test") + if tc.nestedPathModule != "" { + require.NoError(t, os.WriteFile("nested_test/module.alloy", []byte(tc.nestedPathModule), 0664)) + } + if tc.deeplyNestedPathModule != "" { + require.NoError(t, os.Mkdir("nested_test/utils", 0700)) + require.NoError(t, os.WriteFile("nested_test/utils/module.alloy", []byte(tc.deeplyNestedPathModule), 0664)) + } + } + if tc.update != nil { testConfig(t, tc.main, tc.reloadConfig, func() { require.NoError(t, os.WriteFile(tc.update.name, []byte(tc.update.updateConfig), 0664)) @@ -117,6 +135,8 @@ func TestImportGit(t *testing.T) { // Extract repo.git.tar so tests can make use of it. // Make repo.git.tar with: // tar -C repo.git -cvf repo.git.tar . + // NOTE: when modifying the files in the repo, make sure to commit the files else + // the changes will not be taken into account. require.NoError(t, util.Untar("./testdata/repo.git.tar", "./testdata/repo.git")) require.NoError(t, util.Untar("./testdata/repo2.git.tar", "./testdata/repo2.git")) t.Cleanup(func() { @@ -146,13 +166,14 @@ func TestImportHTTP(t *testing.T) { } type testImportFileFolder struct { - description string // description at the top of the txtar file - main string // root config that the controller should load - module1 string // module imported by the root config - module2 string // another module imported by the root config - removed string // module will be removed in the dir on update - added string // module which will be added in the dir on update - update *updateFile // update can be used to update the content of a file at runtime + description string // description at the top of the txtar file + main string // root config that the controller should load + module1 string // module imported by the root config + module2 string // another module imported by the root config + utilsModule2 string // another module in a nested subdirectory + removed string // module will be removed in the dir on update + added string // module which will be added in the dir on update + update *updateFile // update can be used to update the content of a file at runtime } func buildTestImportFileFolder(t *testing.T, filename string) testImportFileFolder { @@ -168,6 +189,8 @@ func buildTestImportFileFolder(t *testing.T, filename string) testImportFileFold tc.module1 = string(alloyConfig.Data) case "module2.alloy": tc.module2 = string(alloyConfig.Data) + case "utils/module2.alloy": + tc.utilsModule2 = string(alloyConfig.Data) case "added.alloy": tc.added = string(alloyConfig.Data) case "removed.alloy": @@ -184,6 +207,12 @@ func buildTestImportFileFolder(t *testing.T, filename string) testImportFileFold name: "module2.alloy", updateConfig: string(alloyConfig.Data), } + case "utils/update_module2.alloy": + require.Nil(t, tc.update) + tc.update = &updateFile{ + name: "utils/module2.alloy", + updateConfig: string(alloyConfig.Data), + } } } return tc @@ -210,6 +239,12 @@ func TestImportFileFolder(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(dir, "removed.alloy"), []byte(tc.removed), 0700)) } + if tc.utilsModule2 != "" { + nestedDir := filepath.Join(dir, "utils") + require.NoError(t, os.Mkdir(nestedDir, 0700)) + require.NoError(t, os.WriteFile(filepath.Join(nestedDir, "module2.alloy"), []byte(tc.utilsModule2), 0700)) + } + // TODO: ideally we would like to check the health of the node but that's not yet possible for import nodes. // We should expect that adding or removing files in the dir is gracefully handled and the node should be // healthy once it polls the content of the dir again. @@ -265,7 +300,7 @@ func testConfig(t *testing.T, config string, reloadConfig string, update func()) defer verifyNoGoroutineLeaks(t) ctrl, f := setup(t, config) - err := ctrl.LoadSource(f, nil) + err := ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -303,7 +338,7 @@ func testConfig(t *testing.T, config string, reloadConfig string, update func()) require.NotNil(t, f) // Reload the controller with the new config. - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) // Export should be -10 after update @@ -317,7 +352,7 @@ func testConfig(t *testing.T, config string, reloadConfig string, update func()) func testConfigError(t *testing.T, config string, expectedError string) { defer verifyNoGoroutineLeaks(t) ctrl, f := setup(t, config) - err := ctrl.LoadSource(f, nil) + err := ctrl.LoadSource(f, nil, "") require.ErrorContains(t, err, expectedError) ctx, cancel := context.WithCancel(context.Background()) var wg sync.WaitGroup diff --git a/internal/runtime/internal/controller/component_references.go b/internal/runtime/internal/controller/component_references.go index 8aee9a4915..cc5205dfdc 100644 --- a/internal/runtime/internal/controller/component_references.go +++ b/internal/runtime/internal/controller/component_references.go @@ -29,7 +29,7 @@ type Reference struct { // ComponentReferences returns the list of references a component is making to // other components. -func ComponentReferences(cn dag.Node, g *dag.Graph, l log.Logger) ([]Reference, diag.Diagnostics) { +func ComponentReferences(cn dag.Node, g *dag.Graph, l log.Logger, scope *vm.Scope) ([]Reference, diag.Diagnostics) { var ( traversals []Traversal @@ -48,25 +48,20 @@ func ComponentReferences(cn dag.Node, g *dag.Graph, l log.Logger) ([]Reference, ref, resolveDiags := resolveTraversal(t, g) componentRefMatch := !resolveDiags.HasErrors() - // We use an empty scope to determine if a reference refers to something in - // the stdlib, since vm.Scope.Lookup will search the scope tree + the - // stdlib. - // - // Any call to an stdlib function is ignored. - var emptyScope vm.Scope - _, stdlibMatch := emptyScope.Lookup(t[0].Name) + // we look for a match in the provided scope and the stdlib + _, scopeMatch := scope.Lookup(t[0].Name) - if !componentRefMatch && !stdlibMatch { + if !componentRefMatch && !scopeMatch { diags = append(diags, resolveDiags...) continue } if componentRefMatch { - if stdlibMatch { + if scope.IsStdlibIdentifiers(t[0].Name) { level.Warn(l).Log("msg", "a component is shadowing an existing stdlib name", "component", strings.Join(ref.Target.Block().Name, "."), "stdlib name", t[0].Name) } refs = append(refs, ref) - } else if stdlibMatch && emptyScope.IsDeprecated(t[0].Name) { + } else if scope.IsStdlibDeprecated(t[0].Name) { level.Warn(l).Log("msg", "this stdlib function is deprecated; please refer to the documentation for updated usage and alternatives", "function", t[0].Name) } } diff --git a/internal/runtime/internal/controller/custom_component_registry.go b/internal/runtime/internal/controller/custom_component_registry.go index 63f6e83557..2f64423a3e 100644 --- a/internal/runtime/internal/controller/custom_component_registry.go +++ b/internal/runtime/internal/controller/custom_component_registry.go @@ -5,6 +5,7 @@ import ( "sync" "github.com/grafana/alloy/syntax/ast" + "github.com/grafana/alloy/syntax/vm" ) // CustomComponentRegistry holds custom component definitions that are available in the context. @@ -14,15 +15,17 @@ type CustomComponentRegistry struct { parent *CustomComponentRegistry // nil if root config mut sync.RWMutex + scope *vm.Scope imports map[string]*CustomComponentRegistry // importNamespace: importScope declares map[string]ast.Body // customComponentName: template } // NewCustomComponentRegistry creates a new CustomComponentRegistry with a parent. // parent can be nil. -func NewCustomComponentRegistry(parent *CustomComponentRegistry) *CustomComponentRegistry { +func NewCustomComponentRegistry(parent *CustomComponentRegistry, scope *vm.Scope) *CustomComponentRegistry { return &CustomComponentRegistry{ parent: parent, + scope: scope, declares: make(map[string]ast.Body), imports: make(map[string]*CustomComponentRegistry), } @@ -42,6 +45,12 @@ func (s *CustomComponentRegistry) getImport(name string) (*CustomComponentRegist return im, ok } +func (s *CustomComponentRegistry) Scope() *vm.Scope { + s.mut.RLock() + defer s.mut.RUnlock() + return s.scope +} + // registerDeclare stores a local declare block. func (s *CustomComponentRegistry) registerDeclare(declare *ast.BlockStmt) { s.mut.Lock() @@ -69,7 +78,7 @@ func (s *CustomComponentRegistry) updateImportContent(importNode *ImportConfigNo if _, exist := s.imports[importNode.label]; !exist { panic(fmt.Errorf("import %q was not registered", importNode.label)) } - importScope := NewCustomComponentRegistry(nil) + importScope := NewCustomComponentRegistry(nil, importNode.Scope()) importScope.declares = importNode.ImportedDeclares() importScope.updateImportContentChildren(importNode) s.imports[importNode.label] = importScope @@ -79,7 +88,7 @@ func (s *CustomComponentRegistry) updateImportContent(importNode *ImportConfigNo // and update their scope with the imported declare blocks. func (s *CustomComponentRegistry) updateImportContentChildren(importNode *ImportConfigNode) { for _, child := range importNode.ImportConfigNodesChildren() { - childScope := NewCustomComponentRegistry(nil) + childScope := NewCustomComponentRegistry(nil, child.Scope()) childScope.declares = child.ImportedDeclares() childScope.updateImportContentChildren(child) s.imports[child.label] = childScope diff --git a/internal/runtime/internal/controller/loader.go b/internal/runtime/internal/controller/loader.go index d10a8dd0a7..8cbe0061fe 100644 --- a/internal/runtime/internal/controller/loader.go +++ b/internal/runtime/internal/controller/loader.go @@ -18,6 +18,7 @@ import ( "github.com/grafana/alloy/internal/service" "github.com/grafana/alloy/syntax/ast" "github.com/grafana/alloy/syntax/diag" + "github.com/grafana/alloy/syntax/vm" "github.com/grafana/dskit/backoff" "github.com/hashicorp/go-multierror" "go.opentelemetry.io/otel/attribute" @@ -124,6 +125,9 @@ type ApplyOptions struct { // The definition of a custom component instantiated inside of the loaded config // should be passed via this field if it's not declared or imported in the config. CustomComponentRegistry *CustomComponentRegistry + + // ArgScope contains additional variables that can be used in the current module. + ArgScope *vm.Scope } // Apply loads a new set of components into the Loader. Apply will drop any @@ -145,6 +149,8 @@ func (l *Loader) Apply(options ApplyOptions) diag.Diagnostics { l.cm.controllerEvaluation.Set(1) defer l.cm.controllerEvaluation.Set(0) + l.cache.SetScope(options.ArgScope) + for key, value := range options.Args { l.cache.CacheModuleArgument(key, value) } @@ -152,7 +158,7 @@ func (l *Loader) Apply(options ApplyOptions) diag.Diagnostics { // Create a new CustomComponentRegistry based on the provided one. // The provided one should be nil for the root config. - l.componentNodeManager.setCustomComponentRegistry(NewCustomComponentRegistry(options.CustomComponentRegistry)) + l.componentNodeManager.setCustomComponentRegistry(NewCustomComponentRegistry(options.CustomComponentRegistry, options.ArgScope)) newGraph, diags := l.loadNewGraph(options.Args, options.ComponentBlocks, options.ConfigBlocks, options.DeclareBlocks) if diags.HasErrors() { return diags @@ -608,7 +614,9 @@ func (l *Loader) wireGraphEdges(g *dag.Graph) diag.Diagnostics { } // Finally, wire component references. - refs, nodeDiags := ComponentReferences(n, g, l.log) + l.cache.mut.RLock() + refs, nodeDiags := ComponentReferences(n, g, l.log, l.cache.scope) + l.cache.mut.RUnlock() for _, ref := range refs { g.AddEdge(dag.Edge{From: n, To: ref.Target}) } diff --git a/internal/runtime/internal/controller/node_config_import.go b/internal/runtime/internal/controller/node_config_import.go index f01795405d..5d6e5a200a 100644 --- a/internal/runtime/internal/controller/node_config_import.go +++ b/internal/runtime/internal/controller/node_config_import.go @@ -289,6 +289,11 @@ func (cn *ImportConfigNode) processImportBlock(stmt *ast.BlockStmt, fullName str childGlobals.OnBlockNodeUpdate = cn.onChildrenContentUpdate // Children data paths are nested inside their parents to avoid collisions. childGlobals.DataPath = filepath.Join(childGlobals.DataPath, cn.globalID) + + if importsource.GetSourceType(cn.block.GetBlockName()) == importsource.HTTP && sourceType == importsource.File { + return fmt.Errorf("importing a module via import.http (nodeID: %s) that contains an import.file block is not supported", cn.nodeID) + } + cn.importConfigNodesChildren[stmt.Label] = NewImportConfigNode(stmt, childGlobals, sourceType) return nil } @@ -296,10 +301,9 @@ func (cn *ImportConfigNode) processImportBlock(stmt *ast.BlockStmt, fullName str // evaluateChildren evaluates the import nodes managed by this import node. func (cn *ImportConfigNode) evaluateChildren() error { for _, child := range cn.importConfigNodesChildren { - err := child.Evaluate(&vm.Scope{ - Parent: nil, - Variables: make(map[string]interface{}), - }) + err := child.Evaluate(vm.NewScope(map[string]interface{}{ + importsource.ModulePath: cn.source.ModulePath(), + })) if err != nil { return fmt.Errorf("imported node %s failed to evaluate, %v", child.label, err) } @@ -424,6 +428,13 @@ func (cn *ImportConfigNode) ImportedDeclares() map[string]ast.Body { return cn.importedDeclares } +// Scope returns the scope associated with the import source. +func (cn *ImportConfigNode) Scope() *vm.Scope { + return vm.NewScope(map[string]interface{}{ + importsource.ModulePath: cn.source.ModulePath(), + }) +} + // ImportConfigNodesChildren returns the ImportConfigNodesChildren of this ImportConfigNode. func (cn *ImportConfigNode) ImportConfigNodesChildren() map[string]*ImportConfigNode { cn.mut.Lock() diff --git a/internal/runtime/internal/controller/value_cache.go b/internal/runtime/internal/controller/value_cache.go index fa2761ba81..6aae014ba9 100644 --- a/internal/runtime/internal/controller/value_cache.go +++ b/internal/runtime/internal/controller/value_cache.go @@ -21,6 +21,7 @@ type valueCache struct { moduleArguments map[string]any // key -> module arguments value moduleExports map[string]any // name -> value for the value of module exports moduleChangedIndex int // Everytime a change occurs this is incremented + scope *vm.Scope // scope provides additional context for the nodes in the module } // newValueCache creates a new ValueCache. @@ -34,6 +35,12 @@ func newValueCache() *valueCache { } } +func (vc *valueCache) SetScope(scope *vm.Scope) { + vc.mut.Lock() + defer vc.mut.Unlock() + vc.scope = scope +} + // CacheArguments will cache the provided arguments by the given id. args may // be nil to store an empty object. func (vc *valueCache) CacheArguments(id ComponentID, args component.Arguments) { @@ -164,10 +171,7 @@ func (vc *valueCache) BuildContext() *vm.Scope { vc.mut.RLock() defer vc.mut.RUnlock() - scope := &vm.Scope{ - Parent: nil, - Variables: make(map[string]interface{}), - } + scope := vm.NewScopeWithParent(vc.scope, make(map[string]interface{})) // First, partition components by Alloy block name. var componentsByBlockName = make(map[string][]ComponentID) diff --git a/internal/runtime/internal/importsource/import_file.go b/internal/runtime/internal/importsource/import_file.go index 811047bb48..e4691d9ed5 100644 --- a/internal/runtime/internal/importsource/import_file.go +++ b/internal/runtime/internal/importsource/import_file.go @@ -16,6 +16,7 @@ import ( "github.com/grafana/alloy/internal/component" filedetector "github.com/grafana/alloy/internal/filedetector" "github.com/grafana/alloy/internal/runtime/logging/level" + "github.com/grafana/alloy/internal/util" "github.com/grafana/alloy/syntax/vm" ) @@ -254,3 +255,12 @@ func collectFilesFromDir(path string) ([]string, error) { func (im *ImportFile) SetEval(eval *vm.Evaluator) { im.eval = eval } + +func (im *ImportFile) ModulePath() string { + path, err := util.ExtractDirPath(im.args.Filename) + + if err != nil { + level.Error(im.managedOpts.Logger).Log("msg", "failed to extract module path", "module path", im.args.Filename, "err", err) + } + return path +} diff --git a/internal/runtime/internal/importsource/import_git.go b/internal/runtime/internal/importsource/import_git.go index f0b77965fe..02ee1ed675 100644 --- a/internal/runtime/internal/importsource/import_git.go +++ b/internal/runtime/internal/importsource/import_git.go @@ -29,6 +29,7 @@ type ImportGit struct { repo *vcs.GitRepo repoOpts vcs.GitRepoOptions args GitArguments + repoPath string onContentChange func(map[string]string) argsChanged chan struct{} @@ -197,7 +198,7 @@ func (im *ImportGit) Update(args component.Arguments) (err error) { // TODO(rfratto): store in a repo-specific directory so changing repositories // doesn't risk break the module loader if there's a SHA collision between // the two different repositories. - repoPath := filepath.Join(im.opts.DataPath, "repo") + im.repoPath = filepath.Join(im.opts.DataPath, "repo") repoOpts := vcs.GitRepoOptions{ Repository: newArgs.Repository, @@ -208,7 +209,7 @@ func (im *ImportGit) Update(args component.Arguments) (err error) { // Create or update the repo field. // Failure to update repository makes the module loader temporarily use cached contents on disk if im.repo == nil || !reflect.DeepEqual(repoOpts, im.repoOpts) { - r, err := vcs.NewGitRepo(context.Background(), repoPath, repoOpts) + r, err := vcs.NewGitRepo(context.Background(), im.repoPath, repoOpts) if err != nil { if errors.As(err, &vcs.UpdateFailedError{}) { level.Error(im.log).Log("msg", "failed to update repository", "err", err) @@ -303,3 +304,7 @@ func (im *ImportGit) CurrentHealth() component.Health { func (im *ImportGit) SetEval(eval *vm.Evaluator) { im.eval = eval } + +func (im *ImportGit) ModulePath() string { + return im.repoPath +} diff --git a/internal/runtime/internal/importsource/import_http.go b/internal/runtime/internal/importsource/import_http.go index 23bb896a1d..f8cbfd469b 100644 --- a/internal/runtime/internal/importsource/import_http.go +++ b/internal/runtime/internal/importsource/import_http.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "path" "reflect" "time" @@ -16,7 +17,7 @@ import ( // ImportHTTP imports a module from a HTTP server via the remote.http component. type ImportHTTP struct { managedRemoteHTTP *remote_http.Component - arguments component.Arguments + arguments HTTPArguments managedOpts component.Options eval *vm.Evaluator } @@ -106,3 +107,8 @@ func (im *ImportHTTP) CurrentHealth() component.Health { func (im *ImportHTTP) SetEval(eval *vm.Evaluator) { im.eval = eval } + +func (im *ImportHTTP) ModulePath() string { + dir, _ := path.Split(im.arguments.URL) + return dir +} diff --git a/internal/runtime/internal/importsource/import_source.go b/internal/runtime/internal/importsource/import_source.go index 79686d6735..ce3a369b98 100644 --- a/internal/runtime/internal/importsource/import_source.go +++ b/internal/runtime/internal/importsource/import_source.go @@ -24,6 +24,8 @@ const ( BlockImportGit = "import.git" ) +const ModulePath = "module_path" + // ImportSource retrieves a module from a source. type ImportSource interface { // Evaluate updates the arguments provided via the Alloy block. @@ -34,6 +36,8 @@ type ImportSource interface { CurrentHealth() component.Health // Update evaluator SetEval(eval *vm.Evaluator) + // ModulePath is the path where the module is stored locally. + ModulePath() string } // NewImportSource creates a new ImportSource depending on the type. diff --git a/internal/runtime/internal/importsource/import_string.go b/internal/runtime/internal/importsource/import_string.go index 91057f9994..a8a1249fc4 100644 --- a/internal/runtime/internal/importsource/import_string.go +++ b/internal/runtime/internal/importsource/import_string.go @@ -15,6 +15,7 @@ type ImportString struct { arguments component.Arguments eval *vm.Evaluator onContentChange func(map[string]string) + modulePath string } var _ ImportSource = (*ImportString)(nil) @@ -41,6 +42,8 @@ func (im *ImportString) Evaluate(scope *vm.Scope) error { } im.arguments = arguments + im.modulePath, _ = scope.Variables[ModulePath].(string) + // notifies that the content has changed im.onContentChange(map[string]string{"import_string": arguments.Content.Value}) @@ -63,3 +66,7 @@ func (im *ImportString) CurrentHealth() component.Health { func (im *ImportString) SetEval(eval *vm.Evaluator) { im.eval = eval } + +func (im *ImportString) ModulePath() string { + return im.modulePath +} diff --git a/internal/runtime/module.go b/internal/runtime/module.go index 2f2955d63b..3bce44e99d 100644 --- a/internal/runtime/module.go +++ b/internal/runtime/module.go @@ -160,7 +160,7 @@ func (c *module) LoadConfig(config []byte, args map[string]any) error { if err != nil { return err } - return c.f.LoadSource(ff, args) + return c.f.LoadSource(ff, args, "") } // LoadBody loads a pre-parsed Alloy config. diff --git a/internal/runtime/module_eval_test.go b/internal/runtime/module_eval_test.go index a75b18d174..3075f23aae 100644 --- a/internal/runtime/module_eval_test.go +++ b/internal/runtime/module_eval_test.go @@ -62,7 +62,7 @@ func TestUpdates_EmptyModule(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -123,7 +123,7 @@ func TestUpdates_ThroughModule(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -185,7 +185,7 @@ func TestUpdates_TwoModules_SameCompNames(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -252,7 +252,7 @@ func TestUpdates_ReloadConfig(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -307,7 +307,7 @@ func TestUpdates_ReloadConfig(t *testing.T) { require.NoError(t, err) require.NotNil(t, f) - err = ctrl.LoadSource(f, nil) + err = ctrl.LoadSource(f, nil, "") require.NoError(t, err) require.Eventually(t, func() bool { diff --git a/internal/runtime/module_fail_test.go b/internal/runtime/module_fail_test.go index 8b792b26b0..042c674612 100644 --- a/internal/runtime/module_fail_test.go +++ b/internal/runtime/module_fail_test.go @@ -16,7 +16,7 @@ func TestIDRemovalIfFailedToLoad(t *testing.T) { fullContent := "test.fail.module \"t1\" { content = \"\" }" fl, err := ParseSource("test", []byte(fullContent)) require.NoError(t, err) - err = f.LoadSource(fl, nil) + err = f.LoadSource(fl, nil, "") require.NoError(t, err) ctx := context.Background() ctx, cnc := context.WithTimeout(ctx, 600*time.Second) diff --git a/internal/runtime/module_test.go b/internal/runtime/module_test.go index a869ca6189..a93edc95c6 100644 --- a/internal/runtime/module_test.go +++ b/internal/runtime/module_test.go @@ -156,7 +156,7 @@ func TestArgsNotInModules(t *testing.T) { defer cleanUpController(f) fl, err := ParseSource("test", []byte("argument \"arg\"{}")) require.NoError(t, err) - err = f.LoadSource(fl, nil) + err = f.LoadSource(fl, nil, "") require.ErrorContains(t, err, "argument blocks only allowed inside a module") } @@ -166,7 +166,7 @@ func TestExportsNotInModules(t *testing.T) { defer cleanUpController(f) fl, err := ParseSource("test", []byte("export \"arg\"{ value = 1}")) require.NoError(t, err) - err = f.LoadSource(fl, nil) + err = f.LoadSource(fl, nil, "") require.ErrorContains(t, err, "export blocks only allowed inside a module") } @@ -177,7 +177,7 @@ func TestExportsWhenNotUsed(t *testing.T) { fullContent := "test.module \"t1\" { content = \"" + content + "\" }" fl, err := ParseSource("test", []byte(fullContent)) require.NoError(t, err) - err = f.LoadSource(fl, nil) + err = f.LoadSource(fl, nil, "") require.NoError(t, err) ctx := context.Background() ctx, cnc := context.WithTimeout(ctx, 1*time.Second) diff --git a/internal/runtime/source_test.go b/internal/runtime/source_test.go index 0128b342fc..8d1e9a92bd 100644 --- a/internal/runtime/source_test.go +++ b/internal/runtime/source_test.go @@ -89,7 +89,7 @@ func TestParseSources_DuplicateComponent(t *testing.T) { require.NoError(t, err) ctrl := New(testOptions(t)) defer cleanUpController(ctrl) - err = ctrl.LoadSource(s, nil) + err = ctrl.LoadSource(s, nil, "") diagErrs, ok := err.(diag.Diagnostics) require.True(t, ok) require.Len(t, diagErrs, 2) @@ -120,7 +120,7 @@ func TestParseSources_UniqueComponent(t *testing.T) { require.NoError(t, err) ctrl := New(testOptions(t)) defer cleanUpController(ctrl) - err = ctrl.LoadSource(s, nil) + err = ctrl.LoadSource(s, nil, "") require.NoError(t, err) } diff --git a/internal/runtime/testdata/import_file/import_file_18.txtar b/internal/runtime/testdata/import_file/import_file_18.txtar new file mode 100644 index 0000000000..36584f2f22 --- /dev/null +++ b/internal/runtime/testdata/import_file/import_file_18.txtar @@ -0,0 +1,50 @@ +Import nested passthrough module with relative import path. + +-- main.alloy -- +testcomponents.count "inc" { + frequency = "10ms" + max = 10 +} + +import.file "testImport" { + filename = "nested_test/module.alloy" +} + +testImport.a "cc" { + input = testcomponents.count.inc.count +} + +testcomponents.summation "sum" { + input = testImport.a.cc.output +} + +-- nested_test/module.alloy -- +import.file "testImport" { + filename = file.path_join(module_path, "utils/module.alloy") +} + +declare "a" { + argument "input" {} + + testImport.a "cc" { + input = argument.input.value + } + + export "output" { + value = testImport.a.cc.output + } +} + +-- nested_test/utils/module.alloy -- +declare "a" { + argument "input" {} + + testcomponents.passthrough "pt" { + input = argument.input.value + lag = "1ms" + } + + export "output" { + value = testcomponents.passthrough.pt.output + } +} diff --git a/internal/runtime/testdata/import_file/import_file_19.txtar b/internal/runtime/testdata/import_file/import_file_19.txtar new file mode 100644 index 0000000000..728855b84a --- /dev/null +++ b/internal/runtime/testdata/import_file/import_file_19.txtar @@ -0,0 +1,49 @@ +Import string with import file with relative import path. + +-- main.alloy -- +testcomponents.count "inc" { + frequency = "10ms" + max = 10 +} + +import.string "testImport" { + content = ` + import.file "testImport" { + filename = file.path_join(module_path, "nested_test/module.alloy") + } + + declare "a" { + argument "input" {} + + testImport.a "cc" { + input = argument.input.value + } + + export "output" { + value = testImport.a.cc.output + } + } + ` +} + +testImport.a "cc" { + input = testcomponents.count.inc.count +} + +testcomponents.summation "sum" { + input = testImport.a.cc.output +} + +-- nested_test/module.alloy -- +declare "a" { + argument "input" {} + + testcomponents.passthrough "pt" { + input = argument.input.value + lag = "1ms" + } + + export "output" { + value = testcomponents.passthrough.pt.output + } +} diff --git a/internal/runtime/testdata/import_file/import_file_20.txtar b/internal/runtime/testdata/import_file/import_file_20.txtar new file mode 100644 index 0000000000..7fd7f3b697 --- /dev/null +++ b/internal/runtime/testdata/import_file/import_file_20.txtar @@ -0,0 +1,50 @@ +Import nested passthrough module with relative import path in a declare. + +-- main.alloy -- +testcomponents.count "inc" { + frequency = "10ms" + max = 10 +} + +import.file "testImport" { + filename = "nested_test/module.alloy" +} + +testImport.a "cc" { + input = testcomponents.count.inc.count +} + +testcomponents.summation "sum" { + input = testImport.a.cc.output +} + +-- nested_test/module.alloy -- +declare "a" { + argument "input" {} + + import.file "testImport" { + filename = file.path_join(module_path, "utils/module.alloy") + } + + testImport.a "cc" { + input = argument.input.value + } + + export "output" { + value = testImport.a.cc.output + } +} + +-- nested_test/utils/module.alloy -- +declare "a" { + argument "input" {} + + testcomponents.passthrough "pt" { + input = argument.input.value + lag = "1ms" + } + + export "output" { + value = testcomponents.passthrough.pt.output + } +} diff --git a/internal/runtime/testdata/import_file_folder/import_file_folder_7.txtar b/internal/runtime/testdata/import_file_folder/import_file_folder_7.txtar new file mode 100644 index 0000000000..8c2ae7866b --- /dev/null +++ b/internal/runtime/testdata/import_file_folder/import_file_folder_7.txtar @@ -0,0 +1,58 @@ +Import nested folder with relative path. + +-- main.alloy -- +testcomponents.count "inc" { + frequency = "10ms" + max = 10 +} + +import.file "testImport" { + filename = "tmpTest" +} + +testImport.a "cc" { + input = testcomponents.count.inc.count +} + +testcomponents.summation "sum" { + input = testImport.a.cc.output +} + +-- module1.alloy -- +import.file "testImport" { + filename = file.path_join(module_path, "utils") +} +declare "a" { + argument "input" {} + + testImport.b "cc" { + input = argument.input.value + } + + export "output" { + value = testImport.b.cc.output + } +} + +-- utils/module2.alloy -- +declare "b" { + argument "input" {} + + testcomponents.passthrough "pt" { + input = argument.input.value + lag = "1ms" + } + + export "output" { + value = testcomponents.passthrough.pt.output + } +} + +-- utils/update_module2.alloy -- +declare "b" { + argument "input" {} + + export "output" { + value = -argument.input.value + } +} diff --git a/internal/runtime/testdata/import_git/import_git_4.txtar b/internal/runtime/testdata/import_git/import_git_4.txtar new file mode 100644 index 0000000000..f4f8feef9c --- /dev/null +++ b/internal/runtime/testdata/import_git/import_git_4.txtar @@ -0,0 +1,21 @@ +Import a module that contains an import.file with a relative import path. + +-- main.alloy -- +testcomponents.count "inc" { + frequency = "10ms" + max = 10 +} + +import.git "testImport" { + // Requires repo.git.tar to be extracted + repository = "./testdata/repo.git" + path = "module_import_file.alloy" +} + +testImport.a "cc" { + input = testcomponents.count.inc.count +} + +testcomponents.summation "sum" { + input = testImport.a.cc.output +} diff --git a/internal/runtime/testdata/import_git/import_git_5.txtar b/internal/runtime/testdata/import_git/import_git_5.txtar new file mode 100644 index 0000000000..c5cefee502 --- /dev/null +++ b/internal/runtime/testdata/import_git/import_git_5.txtar @@ -0,0 +1,21 @@ +Import a module that contains an import.file with a relative import path inside of a declare. + +-- main.alloy -- +testcomponents.count "inc" { + frequency = "10ms" + max = 10 +} + +import.git "testImport" { + // Requires repo.git.tar to be extracted + repository = "./testdata/repo.git" + path = "module_import_file_in_declare.alloy" +} + +testImport.a "cc" { + input = testcomponents.count.inc.count +} + +testcomponents.summation "sum" { + input = testImport.a.cc.output +} diff --git a/internal/runtime/testdata/repo.git.tar b/internal/runtime/testdata/repo.git.tar index 53147ced4e..156b4abb22 100644 Binary files a/internal/runtime/testdata/repo.git.tar and b/internal/runtime/testdata/repo.git.tar differ diff --git a/internal/service/remotecfg/remotecfg.go b/internal/service/remotecfg/remotecfg.go index cbe5dcedbd..50ef83aee0 100644 --- a/internal/service/remotecfg/remotecfg.go +++ b/internal/service/remotecfg/remotecfg.go @@ -90,6 +90,7 @@ const namespaceDelimiter = "." type Options struct { Logger log.Logger // Where to send logs. StoragePath string // Where to cache configuration on-disk. + ConfigPath string // Where the root config file is. Metrics prometheus.Registerer // Where to send metrics to. } @@ -467,7 +468,7 @@ func (s *Service) parseAndLoad(b []byte) error { return nil } - err := ctrl.LoadSource(b, nil) + err := ctrl.LoadSource(b, nil, s.opts.ConfigPath) if err != nil { return err } diff --git a/internal/service/remotecfg/remotecfg_test.go b/internal/service/remotecfg/remotecfg_test.go index 171046c08c..5fe015fbb8 100644 --- a/internal/service/remotecfg/remotecfg_test.go +++ b/internal/service/remotecfg/remotecfg_test.go @@ -290,11 +290,11 @@ type serviceController struct { } func (sc serviceController) Run(ctx context.Context) { sc.f.Run(ctx) } -func (sc serviceController) LoadSource(b []byte, args map[string]any) error { +func (sc serviceController) LoadSource(b []byte, args map[string]any, configPath string) error { source, err := alloy_runtime.ParseSource("", b) if err != nil { return err } - return sc.f.LoadSource(source, args) + return sc.f.LoadSource(source, args, configPath) } func (sc serviceController) Ready() bool { return sc.f.Ready() } diff --git a/internal/service/service.go b/internal/service/service.go index c01d8562ec..b6fc24675f 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -75,7 +75,7 @@ type Host interface { // Controller is implemented by alloy.Alloy. type Controller interface { Run(ctx context.Context) - LoadSource(source []byte, args map[string]any) error + LoadSource(source []byte, args map[string]any, configPath string) error Ready() bool } diff --git a/internal/util/filepath.go b/internal/util/filepath.go new file mode 100644 index 0000000000..b158e2cebc --- /dev/null +++ b/internal/util/filepath.go @@ -0,0 +1,21 @@ +package util + +import ( + "os" + "path/filepath" +) + +// ExtractDirPath removes the file part of a path if it exists. +func ExtractDirPath(p string) (string, error) { + info, err := os.Stat(p) + + if err != nil { + return "", err + } + + if !info.IsDir() { + return filepath.Dir(p), nil + } + + return p, nil +} diff --git a/syntax/alloytypes/secret_test.go b/syntax/alloytypes/secret_test.go index 69b770a615..eedda3b48c 100644 --- a/syntax/alloytypes/secret_test.go +++ b/syntax/alloytypes/secret_test.go @@ -39,9 +39,7 @@ func decodeTo(t *testing.T, input interface{}, target interface{}) error { require.NoError(t, err) eval := vm.New(expr) - return eval.Evaluate(&vm.Scope{ - Variables: map[string]interface{}{ - "val": input, - }, - }, target) + return eval.Evaluate(vm.NewScope(map[string]interface{}{ + "val": input, + }), target) } diff --git a/syntax/vm/op_binary_test.go b/syntax/vm/op_binary_test.go index 11803c2283..015711323d 100644 --- a/syntax/vm/op_binary_test.go +++ b/syntax/vm/op_binary_test.go @@ -11,13 +11,11 @@ import ( ) func TestVM_OptionalSecret_Conversion(t *testing.T) { - scope := &vm.Scope{ - Variables: map[string]any{ - "string_val": "hello", - "non_secret_val": alloytypes.OptionalSecret{IsSecret: false, Value: "world"}, - "secret_val": alloytypes.OptionalSecret{IsSecret: true, Value: "secret"}, - }, - } + scope := vm.NewScope(map[string]any{ + "string_val": "hello", + "non_secret_val": alloytypes.OptionalSecret{IsSecret: false, Value: "world"}, + "secret_val": alloytypes.OptionalSecret{IsSecret: true, Value: "secret"}, + }) tt := []struct { name string diff --git a/syntax/vm/vm.go b/syntax/vm/vm.go index 2df052b92b..71b2893ccc 100644 --- a/syntax/vm/vm.go +++ b/syntax/vm/vm.go @@ -469,6 +469,19 @@ type Scope struct { Variables map[string]interface{} } +func NewScope(variables map[string]interface{}) *Scope { + return &Scope{ + Variables: variables, + } +} + +func NewScopeWithParent(parent *Scope, variables map[string]interface{}) *Scope { + return &Scope{ + Parent: parent, + Variables: variables, + } +} + // Lookup looks up a named identifier from the scope, all of the scope's // parents, and the stdlib. func (s *Scope) Lookup(name string) (interface{}, bool) { @@ -485,8 +498,14 @@ func (s *Scope) Lookup(name string) (interface{}, bool) { return nil, false } -// IsDeprecated returns true if the identifier exists and is deprecated. -func (s *Scope) IsDeprecated(name string) bool { +// IsStdlibIdentifiers returns true if the identifier exists. +func (s *Scope) IsStdlibIdentifiers(name string) bool { + _, exist := stdlib.Identifiers[name] + return exist +} + +// IsStdlibDeprecated returns true if the identifier exists and is deprecated. +func (s *Scope) IsStdlibDeprecated(name string) bool { _, exist := stdlib.DeprecatedIdentifiers[name] return exist } diff --git a/syntax/vm/vm_benchmarks_test.go b/syntax/vm/vm_benchmarks_test.go index 0d1e37335d..d5bfce790b 100644 --- a/syntax/vm/vm_benchmarks_test.go +++ b/syntax/vm/vm_benchmarks_test.go @@ -13,11 +13,9 @@ import ( func BenchmarkExprs(b *testing.B) { // Shared scope across all tests below - scope := &vm.Scope{ - Variables: map[string]interface{}{ - "foobar": int(42), - }, - } + scope := vm.NewScope(map[string]interface{}{ + "foobar": int(42), + }) tt := []struct { name string diff --git a/syntax/vm/vm_errors_test.go b/syntax/vm/vm_errors_test.go index 219d6d47c7..d2f2502eae 100644 --- a/syntax/vm/vm_errors_test.go +++ b/syntax/vm/vm_errors_test.go @@ -46,15 +46,13 @@ func TestVM_ExprErrors(t *testing.T) { name: "deeply nested indirect", input: `key = key_value`, into: &Target{}, - scope: &vm.Scope{ - Variables: map[string]interface{}{ - "key_value": map[string]interface{}{ - "object": map[string]interface{}{ - "field1": []interface{}{15, 30, "Hello, world!"}, - }, + scope: vm.NewScope(map[string]interface{}{ + "key_value": map[string]interface{}{ + "object": map[string]interface{}{ + "field1": []interface{}{15, 30, "Hello, world!"}, }, }, - }, + }), expect: `test:1:7: key_value.object.field1[2] should be number, got string`, }, { diff --git a/syntax/vm/vm_stdlib_test.go b/syntax/vm/vm_stdlib_test.go index 4454a31d89..e8009ae157 100644 --- a/syntax/vm/vm_stdlib_test.go +++ b/syntax/vm/vm_stdlib_test.go @@ -119,12 +119,10 @@ func TestStdlibJsonPath(t *testing.T) { } func TestStdlib_Nonsensitive(t *testing.T) { - scope := &vm.Scope{ - Variables: map[string]any{ - "secret": alloytypes.Secret("foo"), - "optionalSecret": alloytypes.OptionalSecret{Value: "bar"}, - }, - } + scope := vm.NewScope(map[string]any{ + "secret": alloytypes.Secret("foo"), + "optionalSecret": alloytypes.OptionalSecret{Value: "bar"}, + }) tt := []struct { name string @@ -152,9 +150,7 @@ func TestStdlib_Nonsensitive(t *testing.T) { } } func TestStdlib_StringFunc(t *testing.T) { - scope := &vm.Scope{ - Variables: map[string]any{}, - } + scope := vm.NewScope(make(map[string]interface{})) tt := []struct { name string @@ -276,11 +272,9 @@ func BenchmarkConcat(b *testing.B) { Attrs: data, }) } - scope := &vm.Scope{ - Variables: map[string]interface{}{ - "values_ref": valuesRef, - }, - } + scope := vm.NewScope(map[string]interface{}{ + "values_ref": valuesRef, + }) // Reset timer before running the actual test b.ResetTimer() diff --git a/syntax/vm/vm_test.go b/syntax/vm/vm_test.go index 877ac879b3..7fc8fc577d 100644 --- a/syntax/vm/vm_test.go +++ b/syntax/vm/vm_test.go @@ -63,11 +63,9 @@ func TestVM_Evaluate_Literals(t *testing.T) { func TestVM_Evaluate(t *testing.T) { // Shared scope across all tests below - scope := &vm.Scope{ - Variables: map[string]interface{}{ - "foobar": int(42), - }, - } + scope := vm.NewScope(map[string]interface{}{ + "foobar": int(42), + }) tt := []struct { input string @@ -176,11 +174,9 @@ func TestVM_Evaluate_Null(t *testing.T) { func TestVM_Evaluate_IdentifierExpr(t *testing.T) { t.Run("Valid lookup", func(t *testing.T) { - scope := &vm.Scope{ - Variables: map[string]interface{}{ - "foobar": 15, - }, - } + scope := vm.NewScope(map[string]interface{}{ + "foobar": 15, + }) expr, err := parser.ParseExpression(`foobar`) require.NoError(t, err) @@ -210,11 +206,9 @@ func TestVM_Evaluate_AccessExpr(t *testing.T) { Name string `alloy:"name,attr,optional"` } - scope := &vm.Scope{ - Variables: map[string]interface{}{ - "person": Person{}, - }, - } + scope := vm.NewScope(map[string]interface{}{ + "person": Person{}, + }) expr, err := parser.ParseExpression(`person.name`) require.NoError(t, err)