From 38fe6d9a4377c17f7cd4ce349973337723aa458a Mon Sep 17 00:00:00 2001 From: Martin Angers Date: Wed, 2 Oct 2024 16:18:37 -0400 Subject: [PATCH] Fix windows installer stuck in pending state forever (#22592) --- orbit/changes/22558-fix-stuck-windows-installers | 1 + orbit/pkg/installer/installer.go | 14 +++++++++++--- pkg/scripts/scripts.go | 8 ++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 orbit/changes/22558-fix-stuck-windows-installers diff --git a/orbit/changes/22558-fix-stuck-windows-installers b/orbit/changes/22558-fix-stuck-windows-installers new file mode 100644 index 000000000000..029ec41c32d6 --- /dev/null +++ b/orbit/changes/22558-fix-stuck-windows-installers @@ -0,0 +1 @@ +* Added a timeout to all script executions during software installs to prevent having install requests stuck in pending state forever. diff --git a/orbit/pkg/installer/installer.go b/orbit/pkg/installer/installer.go index 4059416f31ca..e0faf63c0fdf 100644 --- a/orbit/pkg/installer/installer.go +++ b/orbit/pkg/installer/installer.go @@ -15,6 +15,7 @@ import ( "github.com/fleetdm/fleet/v4/orbit/pkg/constant" "github.com/fleetdm/fleet/v4/orbit/pkg/scripts" "github.com/fleetdm/fleet/v4/pkg/file" + pkgscripts "github.com/fleetdm/fleet/v4/pkg/scripts" "github.com/fleetdm/fleet/v4/server/fleet" "github.com/fleetdm/fleet/v4/server/ptr" "github.com/osquery/osquery-go" @@ -43,6 +44,9 @@ type Runner struct { OsqueryClient QueryClient OrbitClient Client + // limit execution time of the various scripts run during software installation + installerExecutionTimeout time.Duration + // osquerySocketPath is used to establish the osquery connection // if it's ever lost or disconnected osquerySocketPath string @@ -70,9 +74,10 @@ type Runner struct { func NewRunner(client Client, socketPath string, scriptsEnabled func() bool) *Runner { r := &Runner{ - OrbitClient: client, - osquerySocketPath: socketPath, - scriptsEnabled: scriptsEnabled, + OrbitClient: client, + osquerySocketPath: socketPath, + scriptsEnabled: scriptsEnabled, + installerExecutionTimeout: pkgscripts.MaxHostSoftwareInstallExecutionTime, } return r @@ -277,6 +282,9 @@ func (r *Runner) runInstallerScript(ctx context.Context, scriptContents string, return "", -1, fmt.Errorf("writing script: %w", err) } + ctx, cancel := context.WithTimeout(context.Background(), r.installerExecutionTimeout) + defer cancel() + execFn := r.execCmdFn if execFn == nil { execFn = scripts.ExecCmd diff --git a/pkg/scripts/scripts.go b/pkg/scripts/scripts.go index 7a424b109c98..75f406be3ae7 100644 --- a/pkg/scripts/scripts.go +++ b/pkg/scripts/scripts.go @@ -13,4 +13,12 @@ const ( // to account for the notification system used to deliver scripts to the // host. MaxServerWaitTime = MaxHostExecutionTime + 1*time.Minute + + // MaxHostSoftwareInstallExecutionTime is the maximum time allowed for a + // software installer script to run on a host before is terminated. That same + // timeout is used for all steps of the install process - install, + // post-install, "implicit" uninstall after a failure of the post-install + // script, and "explicit" uninstall request. This does NOT include the + // download time. + MaxHostSoftwareInstallExecutionTime = 1 * time.Hour )