From 148f6a31525c21e1be69ed6687dec7bcc76db395 Mon Sep 17 00:00:00 2001 From: mattdurham Date: Thu, 25 Apr 2024 15:40:23 -0400 Subject: [PATCH] Add details for setting memory limit automatically. (#655) * Add details for setting memory limit automatically. * fix typo * Update docs/sources/reference/cli/run.md Co-authored-by: Mischa Thompson * Update internal/alloycli/cmd_run.go Co-authored-by: Mischa Thompson * add logging * Mark as public preview. * Mark as public preview. * pr feedback * use default logger --------- Co-authored-by: Mischa Thompson --- CHANGELOG.md | 4 ++ docs/sources/reference/cli/run.md | 1 + go.mod | 3 ++ go.sum | 6 +++ internal/alloycli/cmd_run.go | 8 +++ internal/featuregate/featuregate.go | 5 ++ internal/featuregate/featuregate_test.go | 67 ++++++++++++++++++++++++ 7 files changed, 94 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45a96f00b6..8a11a425ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ internal API changes are not present. Main (unreleased) ----------------- +### Features + +- (_Public preview_) Add support for setting GOMEMLIMIT based on cgroup setting. (@mattdurham) + ### Enhancements - Update `prometheus.exporter.kafka` with the following functionalities (@wildum): diff --git a/docs/sources/reference/cli/run.md b/docs/sources/reference/cli/run.md index 3c09772c6d..9b004768d9 100644 --- a/docs/sources/reference/cli/run.md +++ b/docs/sources/reference/cli/run.md @@ -164,6 +164,7 @@ original configuration. Include `--config.extra-args` to pass additional command line flags from the original format to the converter. Refer to [alloy convert][] for more details on how `extra-args` work. + [alloy convert]: ../convert/ [clustering]: ../../../concepts/clustering/ [go-discover]: https://github.com/hashicorp/go-discover diff --git a/go.mod b/go.mod index 763ea37cd7..542a9a1e4d 100644 --- a/go.mod +++ b/go.mod @@ -258,6 +258,7 @@ require ( ) require ( + github.com/KimMachineGun/automemlimit v0.6.0 github.com/Shopify/sarama v1.38.1 github.com/grafana/kafka_exporter v0.0.0-20240409084445-5e3488ad9f9a ) @@ -355,6 +356,7 @@ require ( github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa // indirect github.com/containerd/cgroups v1.1.0 // indirect + github.com/containerd/cgroups/v3 v3.0.2 // indirect github.com/containerd/console v1.0.3 // indirect github.com/containerd/containerd v1.7.11 // indirect github.com/containerd/continuity v0.4.2 // indirect @@ -579,6 +581,7 @@ require ( github.com/ovh/go-ovh v1.4.3 // indirect github.com/packethost/packngo v0.1.1-0.20180711074735-b9cb5096f54c // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect + github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect diff --git a/go.sum b/go.sum index ed5236a759..537290e3fa 100644 --- a/go.sum +++ b/go.sum @@ -193,6 +193,8 @@ github.com/IBM/sarama v1.43.0/go.mod h1:zlE6HEbC/SMQ9mhEYaF7nNLYOUyrs0obySKCckWP github.com/Jeffail/gabs v1.1.0/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= +github.com/KimMachineGun/automemlimit v0.6.0 h1:p/BXkH+K40Hax+PuWWPQ478hPjsp9h1CPDhLlA3Z37E= +github.com/KimMachineGun/automemlimit v0.6.0/go.mod h1:T7xYht7B8r6AG/AqFcUdc7fzd2bIdBKmepfP2S1svPY= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Lusitaniae/apache_exporter v0.11.1-0.20220518131644-f9522724dab4 h1:UEm6tMvLH2t2honcWG0q+h0qr/60++9cuh/fy7hLyMc= github.com/Lusitaniae/apache_exporter v0.11.1-0.20220518131644-f9522724dab4/go.mod h1:IfUbHkoXypdKnVGHWQ3DLRRRZzIU/JIExQAd9xgxRfw= @@ -474,6 +476,8 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= +github.com/containerd/cgroups/v3 v3.0.2 h1:f5WFqIVSgo5IZmtTT3qVBo6TzI1ON6sycSBKkymb9L0= +github.com/containerd/cgroups/v3 v3.0.2/go.mod h1:JUgITrzdFqp42uI2ryGA+ge0ap/nxzYgkGmIcetmErE= github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= @@ -1855,6 +1859,8 @@ github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144T github.com/patrickmn/go-cache v0.0.0-20180527043350-9f6ff22cfff8/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pborman/getopt v0.0.0-20180811024354-2b5b3bfb099b/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= diff --git a/internal/alloycli/cmd_run.go b/internal/alloycli/cmd_run.go index 0d4d3f2c03..bc81018e35 100644 --- a/internal/alloycli/cmd_run.go +++ b/internal/alloycli/cmd_run.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io/fs" + "log/slog" "os" "os/signal" "path/filepath" @@ -15,6 +16,7 @@ import ( "syscall" "time" + "github.com/KimMachineGun/automemlimit/memlimit" "github.com/fatih/color" "github.com/go-kit/log" "github.com/grafana/alloy/internal/alloy" @@ -192,6 +194,12 @@ func (fr *alloyRun) Run(configPath string) error { level.Info(l).Log("boringcrypto enabled", boringcrypto.Enabled) + // Set the memory limit, this will honor GOMEMLIMIT if set + // If there is a cgroup will follow that + if fr.minStability.Permits(featuregate.StabilityPublicPreview) { + memlimit.SetGoMemLimitWithOpts(memlimit.WithLogger(slog.Default())) + } + // Enable the profiling. setMutexBlockProfiling(l) diff --git a/internal/featuregate/featuregate.go b/internal/featuregate/featuregate.go index 5193792f65..2a5f0c3256 100644 --- a/internal/featuregate/featuregate.go +++ b/internal/featuregate/featuregate.go @@ -97,3 +97,8 @@ func (s *Stability) Set(str string) error { func (s Stability) Type() string { return "" } + +// Permits will check if the stability level is allowed. For instance experimental would allow public preview, experimental and ga. Whereas ga would only allow ga. +func (s Stability) Permits(stability Stability) bool { + return s <= stability +} diff --git a/internal/featuregate/featuregate_test.go b/internal/featuregate/featuregate_test.go index 50f676b51f..d6220c26e3 100644 --- a/internal/featuregate/featuregate_test.go +++ b/internal/featuregate/featuregate_test.go @@ -63,3 +63,70 @@ func TestCheckAllowed(t *testing.T) { }) } } + +func TestAllow(t *testing.T) { + type args struct { + current Stability + required Stability + result bool + } + tests := []struct { + name string + args args + }{ + { + name: "Experimental to Experimental", + args: args{ + current: StabilityExperimental, + required: StabilityExperimental, + result: true, + }, + }, + { + name: "Experimental to Public Preview", + args: args{ + current: StabilityExperimental, + required: StabilityPublicPreview, + result: true, + }, + }, + { + name: "Experimental to GA", + args: args{ + current: StabilityExperimental, + required: StabilityPublicPreview, + result: true, + }, + }, + { + name: "GA to GA", + args: args{ + current: StabilityGenerallyAvailable, + required: StabilityGenerallyAvailable, + result: true, + }, + }, + { + name: "GA to Experimental", + args: args{ + current: StabilityGenerallyAvailable, + required: StabilityExperimental, + result: false, + }, + }, + { + name: "GA to Public Preview", + args: args{ + current: StabilityGenerallyAvailable, + required: StabilityPublicPreview, + result: false, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.args.current.Permits(tt.args.required) + require.True(t, result == tt.args.result) + }) + } +}