Modules support relative path 2 (#1726)

* Pass module_path via vm.Scope to enable the use of relative path

* update LoadSource refs

* add tests

* add changelog

* add docs

* Update docs/sources/reference/config-blocks/import.file.md

Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com>

* add module_path const

* improve import git doc

* use os.Stat to extract the dir path

* fix relative path with declare

* add support for remotecfg

* use mutex to avoid data race

* add vm.scope constructors

* update import git doc

* update import examples

* Update docs/sources/reference/config-blocks/import.file.md

Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com>

* add missing revision in example

---------

Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com>

CHANGELOG.md

@@ -39,6 +39,8 @@ Main (unreleased)
 
 - SNMP exporter now supports labels in both `target` and `targets` parameters. (@mattdurham)
 
+- Add support for relative paths to `import.file`. This new functionality allows users to use `import.file` blocks in modules
+  imported via `import.git` and other `import.file`. (@wildum)
 
 ### Bugfixes
 

docs/sources/reference/config-blocks/import.file.md

@@ -17,6 +17,9 @@ Imported directories are treated as single modules to support composability.
 That means that you can define a custom component in one file and use it in another custom component in another file
 in the same directory.
 
+You can use the keyword `module_path` in combination with the `stdlib` function [file.path_join][] to import a module relative to the current module's path.
+The `module_path` keyword works for modules that are imported via `import.file`, `import.git`, and `import.string`.
+
 ## Usage
 
 ```alloy
@@ -37,12 +40,25 @@ The following arguments are supported:
 
 {{< docs/shared lookup="reference/components/local-file-arguments-text.md" source="alloy" version="<ALLOY_VERSION>" >}}
 
-## Example
+## Examples
+
+### Import a module from a local file
 
 This example imports a module from a file and instantiates a custom component from the import that adds two numbers:
 
-{{< collapse title="module.alloy" >}}
+main.alloy
+```alloy
+import.file "math" {
+  filename = "module.alloy"
+}
 
+math.add "default" {
+  a = 15
+  b = 45
+}
+```
+
+module.alloy
 ```alloy
 declare "add" {
   argument "a" {}
@@ -54,13 +70,67 @@ declare "add" {
 }
 ```
 
-{{< /collapse >}}
+### Import a module in a module imported via import.git 
+
+This example imports a module from a file inside of a module that is imported via [import.git][]:
+
+main.alloy
+```alloy
+import.git "math" {
+  repository = "https://github.com/wildum/module.git"
+  path       = "relative_math.alloy"
+  revision   = "master"
+}
+
+math.add "default" {
+  a = 15
+  b = 45
+}
+```
+
+
+relative_math.alloy
+```alloy
+import.file "lib" {
+  filename = file.path_join(module_path, "lib.alloy")
+}
+
+declare "add" {
+  argument "a" {}
+  argument "b" {}
+
+  lib.plus "default" {
+    a = argument.a.value
+    b = argument.b.value
+  }
+
+  export "output" {
+    value = lib.plus.default.sum
+  }
+}
+```
+
+lib.alloy
+```alloy
+declare "plus" {
+  argument "a" {}
+  argument "b" {}
+
+  export "sum" {
+    value = argument.a.value + argument.b.value
+  }
+}
+```
+
+### Import a module in a module imported via import.file 
 
-{{< collapse title="importer.alloy" >}}
+This example imports a module from a file inside of a module that is imported via another `import.file`:
+
+main.alloy
 
 ```alloy
 import.file "math" {
-  filename = "module.alloy"
+  filename = "path/to/module/relative_math.alloy"
 }
 
 math.add "default" {
@@ -69,4 +139,40 @@ math.add "default" {
 }
 ```
 
-{{< /collapse >}}
+relative_math.alloy
+```alloy
+import.file "lib" {
+  filename = file.path_join(module_path, "lib.alloy")
+}
+
+declare "add" {
+  argument "a" {}
+  argument "b" {}
+
+  lib.plus "default" {
+    a = argument.a.value
+    b = argument.b.value
+  }
+
+  export "output" {
+    value = lib.plus.default.sum
+  }
+}
+```
+
+lib.alloy
+```alloy
+declare "plus" {
+  argument "a" {}
+  argument "b" {}
+
+  export "sum" {
+    value = argument.a.value + argument.b.value
+  }
+}
+```
+
+
+
+[file.path_join]: ../../stdlib/file/
+[import.git]: ../import.git/

docs/sources/reference/config-blocks/import.git.md

@@ -9,6 +9,9 @@ title: import.git
 The `import.git` block imports custom components from a Git repository and exposes them to the importer.
 `import.git` blocks must be given a label that determines the namespace where custom components are exposed.
 
+The entire repository is cloned, and the module path is accessible via the `module_path` keyword.
+This enables, for example, your module to import other modules within the repository by setting relative paths in the [import.file][] blocks.
+
 ## Usage
 
 ```alloy
@@ -101,5 +104,6 @@ math.add "default" {
 }
 ```
 
+[import.file]: ../import.file/
 [basic_auth]: #basic_auth-block
 [ssh_key]: #ssh_key-block

docs/sources/reference/config-blocks/import.http.md

@@ -78,7 +78,7 @@ The `tls_config` block configures TLS settings for connecting to HTTPS servers.
 
 This example imports custom components from an HTTP response and instantiates a custom component for adding two numbers:
 
-{{< collapse title="HTTP response" >}}
+module.alloy
 ```alloy
 declare "add" {
   argument "a" {}
@@ -89,9 +89,8 @@ declare "add" {
   }
 }
 ```
-{{< /collapse >}}
 
-{{< collapse title="importer.alloy" >}}
+main.alloy
 ```alloy
 import.http "math" {
   url = SERVER_URL
@@ -102,7 +101,7 @@ math.add "default" {
   b = 45
 }
 ```
-{{< /collapse >}}
+
 
 [client]: #client-block
 [basic_auth]: #basic_auth-block

internal/alloycli/cmd_run.go

@@ -292,6 +292,7 @@ func (fr *alloyRun) Run(configPath string) error {
 
 	remoteCfgService, err := remotecfgservice.New(remotecfgservice.Options{
 		Logger:      log.With(l, "service", "remotecfg"),
+		ConfigPath:  configPath,
 		StoragePath: fr.storagePath,
 		Metrics:     reg,
 	})
@@ -340,7 +341,7 @@ func (fr *alloyRun) Run(configPath string) error {
 		if err != nil {
 			return nil, fmt.Errorf("reading config path %q: %w", configPath, err)
 		}
-		if err := f.LoadSource(alloySource, nil); err != nil {
+		if err := f.LoadSource(alloySource, nil, configPath); err != nil {
 			return alloySource, fmt.Errorf("error during the initial load: %w", err)
 		}
 

internal/converter/internal/test_common/testing.go

@@ -217,7 +217,7 @@ func attemptLoadingAlloyConfig(t *testing.T, bb []byte) {
 		},
 		EnableCommunityComps: true,
 	})
-	err = f.LoadSource(cfg, nil)
+	err = f.LoadSource(cfg, nil, "")
 
 	// Many components will fail to build as e.g. the cert files are missing, so we ignore these errors.
 	// This is not ideal, but we still validate for other potential issues.

internal/runtime/alloy.go

@@ -56,11 +56,14 @@ import (
 
 	"github.com/grafana/alloy/internal/featuregate"
 	"github.com/grafana/alloy/internal/runtime/internal/controller"
+	"github.com/grafana/alloy/internal/runtime/internal/importsource"
 	"github.com/grafana/alloy/internal/runtime/internal/worker"
 	"github.com/grafana/alloy/internal/runtime/logging"
 	"github.com/grafana/alloy/internal/runtime/logging/level"
 	"github.com/grafana/alloy/internal/runtime/tracing"
 	"github.com/grafana/alloy/internal/service"
+	"github.com/grafana/alloy/internal/util"
+	"github.com/grafana/alloy/syntax/vm"
 )
 
 // Options holds static options for an Alloy controller.
@@ -296,22 +299,37 @@ func (f *Runtime) Run(ctx context.Context) {
 // The controller will only start running components after Load is called once
 // without any configuration errors.
 // LoadSource uses default loader configuration.
-func (f *Runtime) LoadSource(source *Source, args map[string]any) error {
-	return f.loadSource(source, args, nil)
+func (f *Runtime) LoadSource(source *Source, args map[string]any, configPath string) error {
+	modulePath, err := util.ExtractDirPath(configPath)
+	if err != nil {
+		level.Warn(f.log).Log("msg", "failed to extract directory path from configPath", "configPath", configPath, "err", err)
+	}
+	return f.applyLoaderConfig(controller.ApplyOptions{
+		Args:            args,
+		ComponentBlocks: source.components,
+		ConfigBlocks:    source.configBlocks,
+		DeclareBlocks:   source.declareBlocks,
+		ArgScope: vm.NewScope(map[string]interface{}{
+			importsource.ModulePath: modulePath,
+		}),
+	})
 }
 
 // Same as above but with a customComponentRegistry that provides custom component definitions.
 func (f *Runtime) loadSource(source *Source, args map[string]any, customComponentRegistry *controller.CustomComponentRegistry) error {
-	f.loadMut.Lock()
-	defer f.loadMut.Unlock()
-
-	applyOptions := controller.ApplyOptions{
+	return f.applyLoaderConfig(controller.ApplyOptions{
 		Args:                    args,
 		ComponentBlocks:         source.components,
 		ConfigBlocks:            source.configBlocks,
 		DeclareBlocks:           source.declareBlocks,
 		CustomComponentRegistry: customComponentRegistry,
-	}
+		ArgScope:                customComponentRegistry.Scope(),
+	})
+}
+
+func (f *Runtime) applyLoaderConfig(applyOptions controller.ApplyOptions) error {
+	f.loadMut.Lock()
+	defer f.loadMut.Unlock()
 
 	diags := f.loader.Apply(applyOptions)
 	if !f.loadedOnce.Load() && diags.HasErrors() {

internal/runtime/alloy_services.go

@@ -93,12 +93,12 @@ type ServiceController struct {
 }
 
 func (sc ServiceController) Run(ctx context.Context) { sc.f.Run(ctx) }
-func (sc ServiceController) LoadSource(b []byte, args map[string]any) error {
+func (sc ServiceController) LoadSource(b []byte, args map[string]any, configPath string) error {
 	source, err := ParseSource("", b)
 	if err != nil {
 		return err
 	}
-	return sc.f.LoadSource(source, args)
+	return sc.f.LoadSource(source, args, configPath)
 }
 func (sc ServiceController) Ready() bool { return sc.f.Ready() }
 

internal/runtime/alloy_services_test.go

@@ -38,7 +38,7 @@ func TestServices(t *testing.T) {
 	opts.Services = append(opts.Services, svc)
 
 	ctrl := New(opts)
-	require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil))
+	require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil, ""))
 
 	// Start the controller. This should cause our service to run.
 	go ctrl.Run(ctx)
@@ -90,7 +90,7 @@ func TestServices_Configurable(t *testing.T) {
 
 	ctrl := New(opts)
 
-	require.NoError(t, ctrl.LoadSource(f, nil))
+	require.NoError(t, ctrl.LoadSource(f, nil, ""))
 
 	// Start the controller. This should cause our service to run.
 	go ctrl.Run(ctx)
@@ -137,7 +137,7 @@ func TestServices_Configurable_Optional(t *testing.T) {
 
 	ctrl := New(opts)
 
-	require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil))
+	require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil, ""))
 
 	// Start the controller. This should cause our service to run.
 	go ctrl.Run(ctx)
@@ -171,7 +171,7 @@ func TestAlloy_GetServiceConsumers(t *testing.T) {
 
 	ctrl := New(opts)
 	defer cleanUpController(ctrl)
-	require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil))
+	require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil, ""))
 
 	expectConsumers := []service.Consumer{{
 		Type:  service.ConsumerTypeService,
@@ -253,7 +253,7 @@ func TestComponents_Using_Services(t *testing.T) {
 		ComponentRegistry: registry,
 		ModuleRegistry:    newModuleRegistry(),
 	})
-	require.NoError(t, ctrl.LoadSource(f, nil))
+	require.NoError(t, ctrl.LoadSource(f, nil, ""))
 	go ctrl.Run(ctx)
 
 	require.NoError(t, componentBuilt.Wait(5*time.Second), "Component should have been built")
@@ -332,7 +332,7 @@ func TestComponents_Using_Services_In_Modules(t *testing.T) {
 		ComponentRegistry: registry,
 		ModuleRegistry:    newModuleRegistry(),
 	})
-	require.NoError(t, ctrl.LoadSource(f, nil))
+	require.NoError(t, ctrl.LoadSource(f, nil, ""))
 	go ctrl.Run(ctx)
 
 	require.NoError(t, componentBuilt.Wait(5*time.Second), "Component should have been built")
@@ -360,7 +360,7 @@ func TestNewControllerNoLeak(t *testing.T) {
 	opts.Services = append(opts.Services, svc)
 
 	ctrl := New(opts)
-	require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil))
+	require.NoError(t, ctrl.LoadSource(makeEmptyFile(t), nil, ""))
 
 	// Start the controller. This should cause our service to run.
 	go ctrl.Run(ctx)