From 9d421b9fae8105aebb9abc3616178a59aa668d5e Mon Sep 17 00:00:00 2001 From: Sarah Gillespie <73313222+gillespi314@users.noreply.github.com> Date: Tue, 24 Sep 2024 14:47:46 -0500 Subject: [PATCH 1/4] Improve MySQL queries that aggregate MDM profile statuses for Apple hosts (#22252) --- changes/22122-mdm-apple-status-queries | 1 + server/datastore/mysql/apple_mdm.go | 467 +++++++------------------ server/datastore/mysql/hosts.go | 32 +- server/datastore/mysql/labels.go | 6 + 4 files changed, 144 insertions(+), 362 deletions(-) create mode 100644 changes/22122-mdm-apple-status-queries diff --git a/changes/22122-mdm-apple-status-queries b/changes/22122-mdm-apple-status-queries new file mode 100644 index 000000000000..2ea893d31ff5 --- /dev/null +++ b/changes/22122-mdm-apple-status-queries @@ -0,0 +1 @@ +- Improved performance of SQL queries used to determine MDM profile status for Apple hosts. \ No newline at end of file diff --git a/server/datastore/mysql/apple_mdm.go b/server/datastore/mysql/apple_mdm.go index c545c898cc74..2e52f7f7eca9 100644 --- a/server/datastore/mysql/apple_mdm.go +++ b/server/datastore/mysql/apple_mdm.go @@ -2535,377 +2535,150 @@ func (ds *Datastore) UpdateOrDeleteHostMDMAppleProfile(ctx context.Context, prof return err } -const ( - appleMDMFailedProfilesStmt = ` - h.uuid = hmap.host_uuid AND - hmap.status = :failed` - - appleMDMPendingProfilesStmt = ` - h.uuid = hmap.host_uuid AND - ( - hmap.status IS NULL OR - hmap.status = :pending OR +// sqlCaseMDMAppleStatus returns a SQL snippet that can be used to determine the status of a host +// based on the status of its profiles and declarations and filevault status. It should be used in +// conjunction with sqlJoinMDMAppleProfilesStatus and sqlJoinMDMAppleDeclarationsStatus. It assumes the +// hosts table to be aliased as 'h' and the host_disk_encryption_keys table to be aliased as 'hdek'. +func sqlCaseMDMAppleStatus() string { + // NOTE: To make this snippet reusable, we're not using sqlx.Named here because it would + // complicate usage in other queries (e.g., list hosts). + var ( + failed = fmt.Sprintf("'%s'", string(fleet.MDMDeliveryFailed)) + pending = fmt.Sprintf("'%s'", string(fleet.MDMDeliveryPending)) + verifying = fmt.Sprintf("'%s'", string(fleet.MDMDeliveryVerifying)) + verified = fmt.Sprintf("'%s'", string(fleet.MDMDeliveryVerified)) + ) + return ` + CASE WHEN (prof_failed + OR decl_failed + OR fv_failed) THEN + ` + failed + ` + WHEN (prof_pending + OR decl_pending -- special case for filevault, it's pending if the profile is -- pending OR the profile is verified or verifying but we still -- don't have an encryption key. - ( - hmap.profile_identifier = :filevault AND - hmap.status IN (:verifying, :verified) AND - hmap.operation_type = :install AND - NOT EXISTS ( - SELECT 1 - FROM host_disk_encryption_keys hdek - WHERE h.id = hdek.host_id AND - (hdek.decryptable = 1 OR hdek.decryptable IS NULL) - ) - ) - )` - - appleMDMVerifyingProfilesStmt = ` - h.uuid = hmap.host_uuid AND - hmap.operation_type = :install AND - ( - -- all profiles except filevault that are 'verifying' - ( - hmap.profile_identifier != :filevault AND - hmap.status = :verifying - ) - OR - -- special cases for filevault - ( - hmap.profile_identifier = :filevault AND - ( - -- filevault profile is verified, but we didn't verify the encryption key - ( - hmap.status = :verified AND - EXISTS ( - SELECT 1 - FROM host_disk_encryption_keys AS hdek - WHERE h.id = hdek.host_id AND - hdek.decryptable IS NULL - ) - ) - OR - -- filevault profile is verifying, and we already have an encryption key, in any state - ( - hmap.status = :verifying AND - EXISTS ( - SELECT 1 - FROM host_disk_encryption_keys AS hdek - WHERE h.id = hdek.host_id AND - hdek.decryptable = 1 OR hdek.decryptable IS NULL - ) - ) - ) - ) - )` - - appleVerifiedProfilesStmt = ` - h.uuid = hmap.host_uuid AND - hmap.operation_type = :install AND - hmap.status = :verified AND - ( - hmap.profile_identifier != :filevault OR - EXISTS ( - SELECT 1 - FROM host_disk_encryption_keys hdek - WHERE h.id = hdek.host_id AND - hdek.decryptable = 1 - ) - )` -) - -// subqueryAppleProfileStatus builds the right subquery that can be used to -// filter hosts based on their profile status. -// -// The subquery mechanism works by finding profiles for hosts that: -// - match with the provided status -// - match any status that supercedes the provided status (eg: failed supercedes verifying) -// -// Hosts will be considered to be in the given status only if the profiles -// match the given status and zero profiles match any superceding status. -func subqueryAppleProfileStatus(status fleet.MDMDeliveryStatus) (string, []any, error) { - var condition string - var excludeConditions string - switch status { - case fleet.MDMDeliveryFailed: - condition = appleMDMFailedProfilesStmt - excludeConditions = "FALSE" - case fleet.MDMDeliveryPending: - condition = appleMDMPendingProfilesStmt - excludeConditions = appleMDMFailedProfilesStmt - case fleet.MDMDeliveryVerifying: - condition = appleMDMVerifyingProfilesStmt - excludeConditions = fmt.Sprintf("(%s) OR (%s)", appleMDMPendingProfilesStmt, appleMDMFailedProfilesStmt) - case fleet.MDMDeliveryVerified: - condition = appleVerifiedProfilesStmt - excludeConditions = fmt.Sprintf("(%s) OR (%s) OR (%s)", appleMDMPendingProfilesStmt, appleMDMFailedProfilesStmt, appleMDMVerifyingProfilesStmt) - default: - return "", nil, fmt.Errorf("invalid status: %s", status) - } - - sql := fmt.Sprintf(` - SELECT 1 - FROM host_mdm_apple_profiles hmap - WHERE %s AND - NOT EXISTS ( - SELECT 1 - FROM host_mdm_apple_profiles hmap - WHERE %s - )`, condition, excludeConditions) - - arg := map[string]any{ - "install": fleet.MDMOperationTypeInstall, - "verifying": fleet.MDMDeliveryVerifying, - "failed": fleet.MDMDeliveryFailed, - "verified": fleet.MDMDeliveryVerified, - "pending": fleet.MDMDeliveryPending, - "filevault": mobileconfig.FleetFileVaultPayloadIdentifier, - } - query, args, err := sqlx.Named(sql, arg) - if err != nil { - return "", nil, fmt.Errorf("subqueryAppleProfileStatus %s: %w", status, err) - } - - return query, args, nil + OR(fv_pending + OR((fv_verifying + OR fv_verified) + AND (hdek.base64_encrypted IS NULL OR (hdek.decryptable IS NOT NULL AND hdek.decryptable != 1))))) THEN + ` + pending + ` + WHEN (prof_verifying + OR decl_verifying + -- special case when fv profile is verifying, and we already have an encryption key, in any state, we treat as verifying + OR(fv_verifying + AND hdek.base64_encrypted IS NOT NULL AND (hdek.decryptable IS NULL OR hdek.decryptable = 1)) + -- special case when fv profile is verified, but we didn't verify the encryption key, we treat as verifying + OR(fv_verified + AND hdek.base64_encrypted IS NOT NULL AND hdek.decryptable IS NULL)) THEN + ` + verifying + ` + WHEN (prof_verified + OR decl_verified + OR(fv_verified + AND hdek.base64_encrypted IS NOT NULL AND hdek.decryptable = 1)) THEN + ` + verified + ` + END +` } -// subqueryAppleDeclarationStatus builds out the subquery for declaration status -func subqueryAppleDeclarationStatus() (string, []any, error) { - const declNamedStmt = ` - CASE WHEN EXISTS ( - SELECT - 1 - FROM - host_mdm_apple_declarations d1 - WHERE - h.uuid = d1.host_uuid - AND d1.operation_type = :install - AND d1.status = :failed - AND d1.declaration_name NOT IN (:reserved_names)) THEN - 'declarations_failed' - WHEN EXISTS ( - SELECT - 1 - FROM - host_mdm_apple_declarations d2 - WHERE - h.uuid = d2.host_uuid - AND d2.operation_type = :install - AND(d2.status IS NULL - OR d2.status = :pending) - AND d2.declaration_name NOT IN (:reserved_names) - AND NOT EXISTS ( - SELECT - 1 - FROM - host_mdm_apple_declarations d3 - WHERE - h.uuid = d3.host_uuid - AND d3.operation_type = :install - AND d3.status = :failed - AND d3.declaration_name NOT IN (:reserved_names))) THEN - 'declarations_pending' - WHEN EXISTS ( - SELECT - 1 - FROM - host_mdm_apple_declarations d4 - WHERE - h.uuid = d4.host_uuid - AND d4.operation_type = :install - AND d4.status = :verifying - AND d4.declaration_name NOT IN (:reserved_names) - AND NOT EXISTS ( - SELECT - 1 - FROM - host_mdm_apple_declarations d5 - WHERE (h.uuid = d5.host_uuid - AND d5.operation_type = :install - AND d5.declaration_name NOT IN (:reserved_names) - AND(d5.status IS NULL - OR d5.status IN(:pending, :failed))))) THEN - 'declarations_verifying' - WHEN EXISTS ( - SELECT - 1 - FROM - host_mdm_apple_declarations d6 - WHERE - h.uuid = d6.host_uuid - AND d6.operation_type = :install - AND d6.status = :verified - AND d6.declaration_name NOT IN (:reserved_names) - AND NOT EXISTS ( - SELECT - 1 - FROM - host_mdm_apple_declarations d7 - WHERE (h.uuid = d7.host_uuid - AND d7.operation_type = :install - AND d7.declaration_name NOT IN (:reserved_names) - AND(d7.status IS NULL - OR d7.status IN(:pending, :failed, :verifying))))) THEN - 'declarations_verified' - ELSE - '' - END` - - arg := map[string]any{ - "install": fleet.MDMOperationTypeInstall, - "verifying": fleet.MDMDeliveryVerifying, - "failed": fleet.MDMDeliveryFailed, - "verified": fleet.MDMDeliveryVerified, - "pending": fleet.MDMDeliveryPending, - "reserved_names": fleetmdm.ListFleetReservedMacOSDeclarationNames(), - } - query, args, err := sqlx.Named(declNamedStmt, arg) - if err != nil { - return "", nil, fmt.Errorf("subqueryAppleDeclarationStatus: %w", err) - } - query, args, err = sqlx.In(query, args...) - if err != nil { - return "", nil, fmt.Errorf("subqueryAppleDeclarationStatus resolve IN: %w", err) - } - - return query, args, nil +// sqlJoinMDMAppleProfilesStatus returns a SQL snippet that can be used to join a table derived from +// host_mdm_apple_profiles (grouped by host_uuid and status) and the hosts table. For each host_uuid, +// it derives a boolean value for each status category. The value will be 1 if the host has any +// profile in the given status category. Separate columns are used for status of the filevault profile +// vs. all other profiles. The snippet assumes the hosts table to be aliased as 'h'. +func sqlJoinMDMAppleProfilesStatus() string { + // NOTE: To make this snippet reusable, we're not using sqlx.Named here because it would + // complicate usage in other queries (e.g., list hosts). + var ( + failed = fmt.Sprintf("'%s'", string(fleet.MDMDeliveryFailed)) + pending = fmt.Sprintf("'%s'", string(fleet.MDMDeliveryPending)) + verifying = fmt.Sprintf("'%s'", string(fleet.MDMDeliveryVerifying)) + verified = fmt.Sprintf("'%s'", string(fleet.MDMDeliveryVerified)) + install = fmt.Sprintf("'%s'", string(fleet.MDMOperationTypeInstall)) + filevault = fmt.Sprintf("'%s'", mobileconfig.FleetFileVaultPayloadIdentifier) + ) + return ` + LEFT JOIN ( + -- profile statuses grouped by host uuid, boolean value will be 1 if host has any profile with the given status + -- filevault profiles are treated separately + SELECT + host_uuid, + MAX( IF((status IS NULL OR status = ` + pending + `) AND profile_identifier != ` + filevault + `, 1, 0)) AS prof_pending, + MAX( IF(status = ` + failed + ` AND profile_identifier != ` + filevault + `, 1, 0)) AS prof_failed, + MAX( IF(status = ` + verifying + ` AND profile_identifier != ` + filevault + ` AND operation_type = ` + install + `, 1, 0)) AS prof_verifying, + MAX( IF(status = ` + verified + ` AND profile_identifier != ` + filevault + ` AND operation_type = ` + install + `, 1, 0)) AS prof_verified, + MAX( IF((status IS NULL OR status = ` + pending + `) AND profile_identifier = ` + filevault + `, 1, 0)) AS fv_pending, + MAX( IF(status = ` + failed + ` AND profile_identifier = ` + filevault + `, 1, 0)) AS fv_failed, + MAX( IF(status = ` + verifying + ` AND profile_identifier = ` + filevault + ` AND operation_type = ` + install + `, 1, 0)) AS fv_verifying, + MAX( IF(status = ` + verified + ` AND profile_identifier = ` + filevault + ` AND operation_type = ` + install + `, 1, 0)) AS fv_verified + FROM + host_mdm_apple_profiles + GROUP BY + host_uuid) hmap ON h.uuid = hmap.host_uuid +` } -func subqueryOSSettingsStatusMac() (string, []any, error) { - var profArgs []any - profFailed, profFailedArgs, err := subqueryAppleProfileStatus(fleet.MDMDeliveryFailed) - if err != nil { - return "", nil, err - } - profArgs = append(profArgs, profFailedArgs...) - - profPending, profPendingArgs, err := subqueryAppleProfileStatus(fleet.MDMDeliveryPending) - if err != nil { - return "", nil, err - } - profArgs = append(profArgs, profPendingArgs...) - - profVerifying, profVerifyingArgs, err := subqueryAppleProfileStatus(fleet.MDMDeliveryVerifying) - if err != nil { - return "", nil, err - } - profArgs = append(profArgs, profVerifyingArgs...) - - profVerified, profVerifiedArgs, err := subqueryAppleProfileStatus(fleet.MDMDeliveryVerified) - if err != nil { - return "", nil, err - } - profArgs = append(profArgs, profVerifiedArgs...) - - profStmt := fmt.Sprintf(` - CASE WHEN EXISTS (%s) THEN - 'profiles_failed' - WHEN EXISTS (%s) THEN - 'profiles_pending' - WHEN EXISTS (%s) THEN - 'profiles_verifying' - WHEN EXISTS (%s) THEN - 'profiles_verified' - ELSE - '' - END`, - profFailed, - profPending, - profVerifying, - profVerified, +// sqlJoinMDMAppleDeclarationsStatus returns a SQL snippet that can be used to join a table derived from +// host_mdm_apple_declarations (grouped by host_uuid and status) and the hosts table. For each host_uuid, +// it derives a boolean value for each status category. The value will be 1 if the host has any +// declaration in the given status category. The snippet assumes the hosts table to be aliased as 'h'. +func sqlJoinMDMAppleDeclarationsStatus() string { + // NOTE: To make this snippet reusable, we're not using sqlx.Named here because it would + // complicate usage in other queries (e.g., list hosts). + var ( + failed = fmt.Sprintf("'%s'", string(fleet.MDMDeliveryFailed)) + pending = fmt.Sprintf("'%s'", string(fleet.MDMDeliveryPending)) + verifying = fmt.Sprintf("'%s'", string(fleet.MDMDeliveryVerifying)) + verified = fmt.Sprintf("'%s'", string(fleet.MDMDeliveryVerified)) + install = fmt.Sprintf("'%s'", string(fleet.MDMOperationTypeInstall)) + reservedDeclNames = fmt.Sprintf("'%s', '%s', '%s'", fleetmdm.FleetMacOSUpdatesProfileName, fleetmdm.FleetIOSUpdatesProfileName, fleetmdm.FleetIPadOSUpdatesProfileName) ) - - declStmt, declArgs, err := subqueryAppleDeclarationStatus() - if err != nil { - return "", nil, err - } - - stmt := fmt.Sprintf(` - CASE (%s) - WHEN 'profiles_failed' THEN - 'failed' - WHEN 'profiles_pending' THEN ( - CASE (%s) - WHEN 'declarations_failed' THEN - 'failed' - ELSE - 'pending' - END) - WHEN 'profiles_verifying' THEN ( - CASE (%s) - WHEN 'declarations_failed' THEN - 'failed' - WHEN 'declarations_pending' THEN - 'pending' - ELSE - 'verifying' - END) - WHEN 'profiles_verified' THEN ( - CASE (%s) - WHEN 'declarations_failed' THEN - 'failed' - WHEN 'declarations_pending' THEN - 'pending' - WHEN 'declarations_verifying' THEN - 'verifying' - ELSE - 'verified' - END) - ELSE - REPLACE((%s), 'declarations_', '') - END`, profStmt, declStmt, declStmt, declStmt, declStmt) - - args := append(profArgs, declArgs...) - args = append(args, declArgs...) - args = append(args, declArgs...) - args = append(args, declArgs...) - - // FIXME(roberto): we found issues in MySQL 5.7.17 (only that version, - // which we must support for now) with prepared statements on this - // query. The results returned by the DB were always different what - // expected unless the arguments are inlined in the query. - // - // We decided to do this given: - // - // - The time constraints we were given to develop DDM - // - The fact that all the variables in this query are really strings managed by us - // - The imminent deprecation of MySQL 5.7 - return fmt.Sprintf(strings.Replace(stmt, "?", "'%s'", -1), args...), []any{}, nil + return ` + LEFT JOIN ( + -- declaration statuses grouped by host uuid, boolean value will be 1 if host has any declaration with the given status + SELECT + host_uuid, + MAX( IF((status IS NULL OR status = ` + pending + `), 1, 0)) AS decl_pending, + MAX( IF(status = ` + failed + `, 1, 0)) AS decl_failed, + MAX( IF(status = ` + verifying + ` , 1, 0)) AS decl_verifying, + MAX( IF(status = ` + verified + ` , 1, 0)) AS decl_verified + FROM + host_mdm_apple_declarations + WHERE + operation_type = ` + install + ` AND declaration_name NOT IN(` + reservedDeclNames + `) + GROUP BY + host_uuid) hmad ON h.uuid = hmad.host_uuid +` } func (ds *Datastore) GetMDMAppleProfilesSummary(ctx context.Context, teamID *uint) (*fleet.MDMProfilesSummary, error) { - subquery, args, err := subqueryOSSettingsStatusMac() - if err != nil { - return nil, ctxerr.Wrap(ctx, err, "building os settings subquery") - } - - sqlFmt := ` + stmt := ` SELECT - %s as status, - COUNT(id) as count + COUNT(id) AS count, + %s AS status FROM - hosts h -WHERE platform = 'darwin' OR platform = 'ios' OR platform = 'ipados' -GROUP BY status, team_id HAVING status IN (?, ?, ?, ?) AND %s` - - args = append(args, fleet.MDMDeliveryFailed, fleet.MDMDeliveryPending, fleet.MDMDeliveryVerifying, fleet.MDMDeliveryVerified) + hosts h + %s + %s + LEFT JOIN host_disk_encryption_keys hdek ON h.id = hdek.host_id +WHERE + platform IN('darwin', 'ios', 'ipad_os') AND %s +GROUP BY + status HAVING status IS NOT NULL` teamFilter := "team_id IS NULL" if teamID != nil && *teamID > 0 { - teamFilter = "team_id = ?" - args = append(args, *teamID) + teamFilter = fmt.Sprintf("team_id = %d", *teamID) } - stmt := fmt.Sprintf(sqlFmt, subquery, teamFilter) + stmt = fmt.Sprintf(stmt, sqlCaseMDMAppleStatus(), sqlJoinMDMAppleProfilesStatus(), sqlJoinMDMAppleDeclarationsStatus(), teamFilter) var dest []struct { Count uint `db:"count"` Status string `db:"status"` } - err = sqlx.SelectContext(ctx, ds.reader(ctx), &dest, stmt, args...) - if err != nil { + if err := sqlx.SelectContext(ctx, ds.reader(ctx), &dest, stmt); err != nil { return nil, err } diff --git a/server/datastore/mysql/hosts.go b/server/datastore/mysql/hosts.go index 0b3a0e498362..ee23749de80a 100644 --- a/server/datastore/mysql/hosts.go +++ b/server/datastore/mysql/hosts.go @@ -1114,6 +1114,14 @@ func (ds *Datastore) applyHostFilters( whereParams = append(whereParams, microsoft_mdm.MDMDeviceStateEnrolled) } + mdmAppleProfilesStatusJoin := "" + mdmAppleDeclarationsStatusJoin := "" + if opt.OSSettingsFilter.IsValid() || + opt.MacOSSettingsFilter.IsValid() { + mdmAppleProfilesStatusJoin = sqlJoinMDMAppleProfilesStatus() + mdmAppleDeclarationsStatusJoin = sqlJoinMDMAppleDeclarationsStatus() + } + sqlStmt += fmt.Sprintf( `FROM hosts h LEFT JOIN host_seen_times hst ON (h.id = hst.host_id) @@ -1128,6 +1136,8 @@ func (ds *Datastore) applyHostFilters( %s %s %s + %s + %s %s WHERE TRUE AND %s AND %s AND %s AND %s `, @@ -1142,6 +1152,8 @@ func (ds *Datastore) applyHostFilters( munkiJoin, displayNameJoin, connectedToFleetJoin, + mdmAppleProfilesStatusJoin, + mdmAppleDeclarationsStatusJoin, // Conditions ds.whereFilterHostsByTeams(filter, "h"), @@ -1304,15 +1316,9 @@ func filterHostsByMacOSSettingsStatus(sql string, opt fleet.HostListOptions, par whereStatus += ` AND h.team_id IS NULL` } - subqueryStatus, paramsStatus, err := subqueryOSSettingsStatusMac() - if err != nil { - return "", nil, err - } - - whereStatus += fmt.Sprintf(` AND %s = ?`, subqueryStatus) - paramsStatus = append(paramsStatus, opt.MacOSSettingsFilter) + whereStatus += fmt.Sprintf(` AND %s = ?`, sqlCaseMDMAppleStatus()) - return sql + whereStatus, append(params, paramsStatus...), nil + return sql + whereStatus, append(params, opt.MacOSSettingsFilter), nil } func filterHostsByMacOSDiskEncryptionStatus(sql string, opt fleet.HostListOptions, params []interface{}) (string, []interface{}) { @@ -1364,13 +1370,9 @@ func (ds *Datastore) filterHostsByOSSettingsStatus(sql string, opt fleet.HostLis AND ((h.platform = 'windows' AND (%s)) OR ((h.platform = 'darwin' OR h.platform = 'ios' OR h.platform = 'ipados') AND (%s)))` - whereMacOS, paramsMacOS, err := subqueryOSSettingsStatusMac() - if err != nil { - return "", nil, err - } - whereMacOS += ` = ?` - // ensure the host has MDM turned on - paramsMacOS = append(paramsMacOS, opt.OSSettingsFilter) + // construct the WHERE for macOS + whereMacOS = fmt.Sprintf(`(%s) = ?`, sqlCaseMDMAppleStatus()) + paramsMacOS := []any{opt.OSSettingsFilter} // construct the WHERE for windows whereWindows = `hmdm.is_server = 0` diff --git a/server/datastore/mysql/labels.go b/server/datastore/mysql/labels.go index d604b286a4aa..5ac777be3939 100644 --- a/server/datastore/mysql/labels.go +++ b/server/datastore/mysql/labels.go @@ -638,6 +638,12 @@ func (ds *Datastore) applyHostLabelFilters(ctx context.Context, filter fleet.Tea joinParams = append(joinParams, microsoft_mdm.MDMDeviceStateEnrolled) } + if opt.OSSettingsFilter.IsValid() || + opt.MacOSSettingsFilter.IsValid() { + query += sqlJoinMDMAppleProfilesStatus() + query += sqlJoinMDMAppleDeclarationsStatus() + } + query += fmt.Sprintf(` WHERE lm.label_id = ? AND %s `, ds.whereFilterHostsByTeams(filter, "h")) whereParams = append(whereParams, lid) From d0007766309db05c8efddc1c791095406b96c343 Mon Sep 17 00:00:00 2001 From: Ian Littman Date: Thu, 26 Sep 2024 13:23:50 -0500 Subject: [PATCH 2/4] Fix software-with-bundle-ID add when the same title with different/no bundle ID, add missing request timeout special case for edit package endpoint (#22413) Same as #22412, for #21370, but against `main` rather than 4.57.0. If some of the following don't apply, delete the relevant line. - [x] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/Committing-Changes.md#changes-files) for more information. - [x] Added/updated tests - [x] Manual QA for all new/changed functionality --------- Co-authored-by: Roberto Dip --- changes/21370-bundle-id-quickfix | 1 + changes/software-edit-request-deadline | 1 + cmd/fleet/serve.go | 3 +- server/datastore/mysql/software_installers.go | 5 +- .../mysql/software_installers_test.go | 84 ++++++++++++++++ server/datastore/mysql/vpp.go | 25 +++-- server/datastore/mysql/vpp_test.go | 96 +++++++++++++++++++ server/service/integration_enterprise_test.go | 38 ++++++++ 8 files changed, 242 insertions(+), 11 deletions(-) create mode 100644 changes/21370-bundle-id-quickfix create mode 100644 changes/software-edit-request-deadline diff --git a/changes/21370-bundle-id-quickfix b/changes/21370-bundle-id-quickfix new file mode 100644 index 000000000000..1f27bedc40af --- /dev/null +++ b/changes/21370-bundle-id-quickfix @@ -0,0 +1 @@ +- Fix "no rows" error when adding a software installer that matches an existing title's name and source but not its bundle ID diff --git a/changes/software-edit-request-deadline b/changes/software-edit-request-deadline new file mode 100644 index 000000000000..f3576256f5ed --- /dev/null +++ b/changes/software-edit-request-deadline @@ -0,0 +1 @@ +* Ensure request timeouts for software installer edits are just as high as for initial software installer uploads diff --git a/cmd/fleet/serve.go b/cmd/fleet/serve.go index e330cedcb34b..fff5cf13536d 100644 --- a/cmd/fleet/serve.go +++ b/cmd/fleet/serve.go @@ -1145,7 +1145,8 @@ the way that the Fleet server works. } } - if req.Method == http.MethodPost && strings.HasSuffix(req.URL.Path, "/fleet/software/package") { + if (req.Method == http.MethodPost && strings.HasSuffix(req.URL.Path, "/fleet/software/package")) || + (req.Method == http.MethodPatch && strings.HasSuffix(req.URL.Path, "/package") && strings.Contains(req.URL.Path, "/fleet/software/titles/")) { // when uploading a software installer, the file might be large so // the read timeout (to read the full request body) must be extended. rc := http.NewResponseController(rw) diff --git a/server/datastore/mysql/software_installers.go b/server/datastore/mysql/software_installers.go index 7d7f0169e3ad..da9f5f7ff395 100644 --- a/server/datastore/mysql/software_installers.go +++ b/server/datastore/mysql/software_installers.go @@ -174,8 +174,9 @@ func (ds *Datastore) getOrGenerateSoftwareInstallerTitleID(ctx context.Context, insertArgs := []any{payload.Title, payload.Source} if payload.BundleIdentifier != "" { - selectStmt = `SELECT id FROM software_titles WHERE bundle_identifier = ?` - selectArgs = []any{payload.BundleIdentifier} + // match by bundle identifier first, or standard matching if we don't have a bundle identifier match + selectStmt = `SELECT id FROM software_titles WHERE bundle_identifier = ? OR (name = ? AND source = ? AND browser = '') ORDER BY bundle_identifier = ? DESC LIMIT 1` + selectArgs = []any{payload.BundleIdentifier, payload.Title, payload.Source, payload.BundleIdentifier} insertStmt = `INSERT INTO software_titles (name, source, bundle_identifier, browser) VALUES (?, ?, ?, '')` insertArgs = append(insertArgs, payload.BundleIdentifier) } diff --git a/server/datastore/mysql/software_installers_test.go b/server/datastore/mysql/software_installers_test.go index 178b85807148..49be5c29c220 100644 --- a/server/datastore/mysql/software_installers_test.go +++ b/server/datastore/mysql/software_installers_test.go @@ -34,6 +34,7 @@ func TestSoftwareInstallers(t *testing.T) { {"HasSelfServiceSoftwareInstallers", testHasSelfServiceSoftwareInstallers}, {"DeleteSoftwareInstallersAssignedToPolicy", testDeleteSoftwareInstallersAssignedToPolicy}, {"GetHostLastInstallData", testGetHostLastInstallData}, + {"GetOrGenerateSoftwareInstallerTitleID", testGetOrGenerateSoftwareInstallerTitleID}, } for _, c := range cases { @@ -1098,3 +1099,86 @@ func testGetHostLastInstallData(t *testing.T, ds *Datastore) { require.NoError(t, err) require.Nil(t, host2LastInstall) } + +func testGetOrGenerateSoftwareInstallerTitleID(t *testing.T, ds *Datastore) { + ctx := context.Background() + + host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) + host2 := test.NewHost(t, ds, "host2", "", "host2key", "host2uuid", time.Now()) + + software1 := []fleet.Software{ + {Name: "Existing Title", Version: "0.0.1", Source: "apps", BundleIdentifier: "existing.title"}, + } + software2 := []fleet.Software{ + {Name: "Existing Title", Version: "v0.0.2", Source: "apps", BundleIdentifier: "existing.title"}, + {Name: "Existing Title", Version: "0.0.3", Source: "apps", BundleIdentifier: "existing.title"}, + {Name: "Existing Title Without Bundle", Version: "0.0.3", Source: "apps"}, + } + + _, err := ds.UpdateHostSoftware(ctx, host1.ID, software1) + require.NoError(t, err) + _, err = ds.UpdateHostSoftware(ctx, host2.ID, software2) + require.NoError(t, err) + require.NoError(t, ds.SyncHostsSoftware(ctx, time.Now())) + require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) + require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now())) + + tests := []struct { + name string + payload *fleet.UploadSoftwareInstallerPayload + }{ + { + name: "title that already exists, no bundle identifier in payload", + payload: &fleet.UploadSoftwareInstallerPayload{ + Title: "Existing Title", + Source: "apps", + }, + }, + { + name: "title that already exists, mismatched bundle identifier in payload", + payload: &fleet.UploadSoftwareInstallerPayload{ + Title: "Existing Title", + Source: "apps", + BundleIdentifier: "com.existing.bundle", + }, + }, + { + name: "title that already exists but doesn't have a bundle identifier", + payload: &fleet.UploadSoftwareInstallerPayload{ + Title: "Existing Title Without Bundle", + Source: "apps", + }, + }, + { + name: "title that already exists, no bundle identifier in DB, bundle identifier in payload", + payload: &fleet.UploadSoftwareInstallerPayload{ + Title: "Existing Title Without Bundle", + Source: "apps", + BundleIdentifier: "com.new.bundleid", + }, + }, + { + name: "title that doesn't exist, no bundle identifier in payload", + payload: &fleet.UploadSoftwareInstallerPayload{ + Title: "New Title", + Source: "some_source", + }, + }, + { + name: "title that doesn't exist, with bundle identifier in payload", + payload: &fleet.UploadSoftwareInstallerPayload{ + Title: "New Title With Bundle", + Source: "some_source", + BundleIdentifier: "com.new.bundle", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + id, err := ds.getOrGenerateSoftwareInstallerTitleID(ctx, tt.payload) + require.NoError(t, err) + require.NotEmpty(t, id) + }) + } +} diff --git a/server/datastore/mysql/vpp.go b/server/datastore/mysql/vpp.go index 1127497f888e..cecb68ab3820 100644 --- a/server/datastore/mysql/vpp.go +++ b/server/datastore/mysql/vpp.go @@ -371,17 +371,26 @@ func (ds *Datastore) getOrInsertSoftwareTitleForVPPApp(ctx context.Context, tx s insertArgs := []any{app.Name, source} if app.BundleIdentifier != "" { - // NOTE: The index `idx_sw_titles` doesn't include the bundle - // identifier. It's possible for the select to return nothing - // but for the insert to fail if an app with the same name but - // no bundle identifier exists in the DB. + // match by bundle identifier first, or standard matching if we + // don't have a bundle identifier match switch source { case "ios_apps", "ipados_apps": - selectStmt = `SELECT id FROM software_titles WHERE bundle_identifier = ? AND source = ?` - selectArgs = []any{app.BundleIdentifier, source} + selectStmt = ` + SELECT id + FROM software_titles + WHERE (bundle_identifier = ? AND source = ?) OR (name = ? AND source = ? AND browser = '') + ORDER BY bundle_identifier = ? DESC + LIMIT 1` + selectArgs = []any{app.BundleIdentifier, source, app.Name, source, app.BundleIdentifier} default: - selectStmt = `SELECT id FROM software_titles WHERE bundle_identifier = ? AND source NOT IN ('ios_apps', 'ipados_apps')` - selectArgs = []any{app.BundleIdentifier} + selectStmt = ` + SELECT id + FROM software_titles + WHERE (bundle_identifier = ? OR (name = ? AND browser = '')) + AND source NOT IN ('ios_apps', 'ipados_apps') + ORDER BY bundle_identifier = ? DESC + LIMIT 1` + selectArgs = []any{app.BundleIdentifier, app.Name, app.BundleIdentifier} } insertStmt = `INSERT INTO software_titles (name, source, bundle_identifier, browser) VALUES (?, ?, ?, '')` insertArgs = append(insertArgs, app.BundleIdentifier) diff --git a/server/datastore/mysql/vpp_test.go b/server/datastore/mysql/vpp_test.go index 9a6c6796ceee..78cab8acba22 100644 --- a/server/datastore/mysql/vpp_test.go +++ b/server/datastore/mysql/vpp_test.go @@ -2,6 +2,7 @@ package mysql import ( "context" + "fmt" "testing" "time" @@ -30,6 +31,7 @@ func TestVPP(t *testing.T) { {"GetVPPAppByTeamAndTitleID", testGetVPPAppByTeamAndTitleID}, {"VPPTokensCRUD", testVPPTokensCRUD}, {"VPPTokenAppTeamAssociations", testVPPTokenAppTeamAssociations}, + {"GetOrInsertSoftwareTitleForVPPApp", testGetOrInsertSoftwareTitleForVPPApp}, } for _, c := range cases { @@ -1201,3 +1203,97 @@ func testVPPTokenAppTeamAssociations(t *testing.T, ds *Datastore) { _, err = ds.InsertVPPAppWithTeam(ctx, app1, &team2.ID) assert.Error(t, err) } + +func testGetOrInsertSoftwareTitleForVPPApp(t *testing.T, ds *Datastore) { + ctx := context.Background() + + host1 := test.NewHost(t, ds, "host1", "", "host1key", "host1uuid", time.Now()) + host2 := test.NewHost(t, ds, "host2", "", "host2key", "host2uuid", time.Now()) + + software1 := []fleet.Software{ + {Name: "Existing Title", Version: "0.0.1", Source: "apps", BundleIdentifier: "existing.title"}, + } + software2 := []fleet.Software{ + {Name: "Existing Title", Version: "v0.0.2", Source: "apps", BundleIdentifier: "existing.title"}, + {Name: "Existing Title", Version: "0.0.3", Source: "apps", BundleIdentifier: "existing.title"}, + {Name: "Existing Title Without Bundle", Version: "0.0.3", Source: "apps"}, + } + + _, err := ds.UpdateHostSoftware(ctx, host1.ID, software1) + require.NoError(t, err) + _, err = ds.UpdateHostSoftware(ctx, host2.ID, software2) + require.NoError(t, err) + require.NoError(t, ds.SyncHostsSoftware(ctx, time.Now())) + require.NoError(t, ds.ReconcileSoftwareTitles(ctx)) + require.NoError(t, ds.SyncHostsSoftwareTitles(ctx, time.Now())) + + tests := []struct { + name string + app *fleet.VPPApp + }{ + { + name: "title that already exists, no bundle identifier in payload", + app: &fleet.VPPApp{ + Name: "Existing Title", + LatestVersion: "0.0.1", + BundleIdentifier: "", + }, + }, + { + name: "title that already exists, bundle identifier in payload", + app: &fleet.VPPApp{ + Name: "Existing Title", + LatestVersion: "0.0.2", + BundleIdentifier: "existing.title", + }, + }, + { + name: "title that already exists but doesn't have a bundle identifier", + app: &fleet.VPPApp{ + Name: "Existing Title Without Bundle", + LatestVersion: "0.0.3", + BundleIdentifier: "", + }, + }, + { + name: "title that already exists, no bundle identifier in DB, bundle identifier in payload", + app: &fleet.VPPApp{ + Name: "Existing Title Without Bundle", + LatestVersion: "0.0.3", + BundleIdentifier: "new.bundle.id", + }, + }, + { + name: "title that doesn't exist, no bundle identifier in payload", + app: &fleet.VPPApp{ + Name: "New Title", + LatestVersion: "0.1.0", + BundleIdentifier: "", + }, + }, + { + name: "title that doesn't exist, with bundle identifier in payload", + app: &fleet.VPPApp{ + Name: "New Title", + LatestVersion: "0.1.0", + BundleIdentifier: "new.title.bundle", + }, + }, + } + + for _, platform := range fleet.VPPAppsPlatforms { + for _, tt := range tests { + t.Run(fmt.Sprintf("%s_%v", tt.name, platform), func(t *testing.T) { + tt.app.Platform = platform + var id uint + err := ds.withTx(ctx, func(tx sqlx.ExtContext) error { + var err error + id, err = ds.getOrInsertSoftwareTitleForVPPApp(ctx, tx, tt.app) + return err + }) + require.NoError(t, err) + require.NotEmpty(t, id) + }) + } + } +} diff --git a/server/service/integration_enterprise_test.go b/server/service/integration_enterprise_test.go index cb2a97966846..3d6673afd3b6 100644 --- a/server/service/integration_enterprise_test.go +++ b/server/service/integration_enterprise_test.go @@ -14366,3 +14366,41 @@ func (s *integrationEnterpriseTestSuite) TestPolicyAutomationsSoftwareInstallers require.NoError(t, err) require.Nil(t, hostVanillaOsquery5Team1LastInstall) } + +func (s *integrationEnterpriseTestSuite) TestSoftwareInstallersWithoutBundleIdentifier() { + t := s.T() + ctx := context.Background() + + // Create a host without a team + host, err := s.ds.NewHost(context.Background(), &fleet.Host{ + DetailUpdatedAt: time.Now(), + LabelUpdatedAt: time.Now(), + PolicyUpdatedAt: time.Now(), + SeenTime: time.Now().Add(-1 * time.Minute), + OsqueryHostID: ptr.String(t.Name()), + NodeKey: ptr.String(t.Name()), + UUID: uuid.New().String(), + Hostname: fmt.Sprintf("%sfoo.local", t.Name()), + Platform: "darwin", + }) + require.NoError(t, err) + + software := []fleet.Software{ + {Name: "DummyApp.app", Version: "0.0.2", Source: "apps"}, + } + // we must ingest the title with an empty bundle identifier for this + // test to be valid + require.Empty(t, software[0].BundleIdentifier) + _, err = s.ds.UpdateHostSoftware(ctx, host.ID, software) + require.NoError(t, err) + require.NoError(t, s.ds.SyncHostsSoftware(ctx, time.Now())) + require.NoError(t, s.ds.ReconcileSoftwareTitles(ctx)) + require.NoError(t, s.ds.SyncHostsSoftwareTitles(ctx, time.Now())) + + payload := &fleet.UploadSoftwareInstallerPayload{ + InstallScript: "install", + Filename: "dummy_installer.pkg", + Version: "0.0.2", + } + s.uploadSoftwareInstaller(payload, http.StatusOK, "") +} From 69d07b3cfdfb85f83c4d5fadbabdd715c8f9a1ca Mon Sep 17 00:00:00 2001 From: Roberto Dip Date: Fri, 27 Sep 2024 14:13:27 -0300 Subject: [PATCH 3/4] fix VPP migration edge case (#22460) https://github.com/fleetdm/fleet/issues/22415 # Checklist for submitter If some of the following don't apply, delete the relevant line. - [x] Changes file added for user-visible changes in `changes/`, `orbit/changes/` or `ee/fleetd-chrome/changes`. See [Changes files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/Committing-Changes.md#changes-files) for more information. - [x] Input data is properly validated, `SELECT *` is avoided, SQL injection is prevented (using placeholders for values in statements) - [x] Added/updated tests - [x] If database migrations are included, checked table schema to confirm autoupdate - For database migrations: - [x] Checked schema for all modified table for columns that will auto-update timestamps during migration. - [x] Confirmed that updating the timestamps is acceptable, and will not cause unwanted side effects. - [x] Ensured the correct collation is explicitly set for character columns (`COLLATE utf8mb4_unicode_ci`). - [x] Manual QA for all new/changed functionality --- changes/22415-fix-vpp-migration | 1 + ...40829170033_AddVPPTokenIDToVppAppsTeams.go | 26 ++++++- ...170033_AddVPPTokenIDToVppAppsTeams_test.go | 70 +++++++++++++++++++ 3 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 changes/22415-fix-vpp-migration create mode 100644 server/datastore/mysql/migrations/tables/20240829170033_AddVPPTokenIDToVppAppsTeams_test.go diff --git a/changes/22415-fix-vpp-migration b/changes/22415-fix-vpp-migration new file mode 100644 index 000000000000..eed21b93dc12 --- /dev/null +++ b/changes/22415-fix-vpp-migration @@ -0,0 +1 @@ +* Fixed an issue with the migration adding support for multiple VPP tokens that would happen if a token is removed prior to upgrading Fleet. diff --git a/server/datastore/mysql/migrations/tables/20240829170033_AddVPPTokenIDToVppAppsTeams.go b/server/datastore/mysql/migrations/tables/20240829170033_AddVPPTokenIDToVppAppsTeams.go index ea367f59ea69..b8ef59e25ffd 100644 --- a/server/datastore/mysql/migrations/tables/20240829170033_AddVPPTokenIDToVppAppsTeams.go +++ b/server/datastore/mysql/migrations/tables/20240829170033_AddVPPTokenIDToVppAppsTeams.go @@ -3,6 +3,8 @@ package tables import ( "database/sql" "fmt" + + "github.com/pkg/errors" ) func init() { @@ -14,7 +16,11 @@ func Up_20240829170033(tx *sql.Tx) error { ALTER TABLE vpp_apps_teams ADD COLUMN vpp_token_id int(10) UNSIGNED NOT NULL` - stmtAssociate := `UPDATE vpp_apps_teams SET vpp_token_id = (SELECT id FROM vpp_tokens LIMIT 1)` + stmtFindToken := `SELECT id FROM vpp_tokens LIMIT 1` //nolint:gosec + + stmtCleanAssociations := `DELETE FROM vpp_apps_teams` + + stmtAssociate := `UPDATE vpp_apps_teams SET vpp_token_id = ?` stmtAddConstraint := ` ALTER TABLE vpp_apps_teams @@ -27,8 +33,22 @@ ALTER TABLE vpp_apps_teams // Associate apps with the first token available. If we're // migrating from single-token VPP this should be correct. - if _, err := tx.Exec(stmtAssociate); err != nil { - return fmt.Errorf("failed to associate vpp apps with first token: %w", err) + var vppTokenID uint + err := tx.QueryRow(stmtFindToken).Scan(&vppTokenID) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("get existing VPP token ID: %w", err) + } + } + + if vppTokenID > 0 { + if _, err := tx.Exec(stmtAssociate, vppTokenID); err != nil { + return fmt.Errorf("failed to associate vpp apps with first token: %w", err) + } + } else { + if _, err := tx.Exec(stmtCleanAssociations); err != nil { + return fmt.Errorf("failed clean orphaned VPP team associations: %w", err) + } } if _, err := tx.Exec(stmtAddConstraint); err != nil { diff --git a/server/datastore/mysql/migrations/tables/20240829170033_AddVPPTokenIDToVppAppsTeams_test.go b/server/datastore/mysql/migrations/tables/20240829170033_AddVPPTokenIDToVppAppsTeams_test.go new file mode 100644 index 000000000000..7329d628de93 --- /dev/null +++ b/server/datastore/mysql/migrations/tables/20240829170033_AddVPPTokenIDToVppAppsTeams_test.go @@ -0,0 +1,70 @@ +package tables + +import ( + "testing" + "time" + + "github.com/jmoiron/sqlx" + "github.com/stretchr/testify/require" +) + +func TestUp_20240829170033_Existing(t *testing.T) { + db := applyUpToPrev(t) + + // insert a vpp token + vppTokenID := execNoErrLastID(t, db, "INSERT INTO vpp_tokens (organization_name, location, renew_at, token) VALUES (?, ?, ?, ?)", "org", "location", time.Now(), "token") + + // create a couple teams + tm1 := execNoErrLastID(t, db, "INSERT INTO teams (name) VALUES ('team1')") + tm2 := execNoErrLastID(t, db, "INSERT INTO teams (name) VALUES ('team2')") + + // create a couple of vpp apps + adamID1 := "123" + execNoErr(t, db, `INSERT INTO vpp_apps (adam_id, platform) VALUES (?, "iOS")`, adamID1) + adamID2 := "456" + execNoErr(t, db, `INSERT INTO vpp_apps (adam_id, platform) VALUES (?, "iOS")`, adamID2) + + // insert some teams with vpp apps + execNoErr(t, db, `INSERT INTO vpp_apps_teams (adam_id, team_id, global_or_team_id, platform, self_service) VALUES (?, ?, ?, ?, ?)`, adamID1, tm1, 0, "iOS", 0) + execNoErr(t, db, `INSERT INTO vpp_apps_teams (adam_id, team_id, global_or_team_id, platform, self_service) VALUES (?, ?, ?, ?, ?)`, adamID2, tm2, 0, "iOS", 0) + + // apply current migration + applyNext(t, db) + + // ensure vpp_token_id is set for all teams + var vppTokenIDs []int + err := sqlx.Select(db, &vppTokenIDs, `SELECT vpp_token_id FROM vpp_apps_teams`) + require.NoError(t, err) + require.Len(t, vppTokenIDs, 2) + for _, tokenID := range vppTokenIDs { + require.Equal(t, int(vppTokenID), tokenID) + } +} + +func TestUp_20240829170033_NoTokens(t *testing.T) { + db := applyUpToPrev(t) + + // create a couple teams + tm1 := execNoErrLastID(t, db, "INSERT INTO teams (name) VALUES ('team1')") + tm2 := execNoErrLastID(t, db, "INSERT INTO teams (name) VALUES ('team2')") + + // create a couple of vpp apps + adamID1 := "123" + execNoErr(t, db, `INSERT INTO vpp_apps (adam_id, platform) VALUES (?, "iOS")`, adamID1) + adamID2 := "456" + execNoErr(t, db, `INSERT INTO vpp_apps (adam_id, platform) VALUES (?, "iOS")`, adamID2) + + // insert some teams with vpp apps + execNoErr(t, db, `INSERT INTO vpp_apps_teams (adam_id, team_id, global_or_team_id, platform, self_service) VALUES (?, ?, ?, ?, ?)`, adamID1, tm1, 0, "iOS", 0) + execNoErr(t, db, `INSERT INTO vpp_apps_teams (adam_id, team_id, global_or_team_id, platform, self_service) VALUES (?, ?, ?, ?, ?)`, adamID2, tm2, 0, "iOS", 0) + + // apply current migration + applyNext(t, db) + + // ensure no rows are left in vpp_apps_teams (since there are no tokens) + var count int + err := sqlx.Get(db, &count, `SELECT COUNT(*) FROM vpp_apps_teams`) + require.NoError(t, err) + require.Zero(t, count) + +} From dae57bf1dc90a2322ed26d59584f2c1b94503512 Mon Sep 17 00:00:00 2001 From: George Karr Date: Tue, 1 Oct 2024 08:30:11 -0500 Subject: [PATCH 4/4] Adding changes for Fleet v4.57.1 --- CHANGELOG.md | 14 ++++++++++++++ changes/21370-bundle-id-quickfix | 1 - changes/22122-mdm-apple-status-queries | 1 - changes/22415-fix-vpp-migration | 1 - changes/software-edit-request-deadline | 1 - charts/fleet/Chart.yaml | 2 +- charts/fleet/values.yaml | 2 +- infrastructure/dogfood/terraform/aws/variables.tf | 2 +- infrastructure/dogfood/terraform/gcp/variables.tf | 2 +- terraform/addons/vuln-processing/variables.tf | 4 ++-- terraform/byo-vpc/byo-db/byo-ecs/variables.tf | 4 ++-- terraform/byo-vpc/byo-db/variables.tf | 4 ++-- terraform/byo-vpc/example/main.tf | 2 +- terraform/byo-vpc/variables.tf | 4 ++-- terraform/example/main.tf | 4 ++-- terraform/variables.tf | 4 ++-- tools/fleetctl-npm/package.json | 2 +- 17 files changed, 32 insertions(+), 22 deletions(-) delete mode 100644 changes/21370-bundle-id-quickfix delete mode 100644 changes/22122-mdm-apple-status-queries delete mode 100644 changes/22415-fix-vpp-migration delete mode 100644 changes/software-edit-request-deadline diff --git a/CHANGELOG.md b/CHANGELOG.md index f1f4dd2cee3e..03086714c556 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +## Fleet 4.57.1 (Oct 01, 2024) + +### Bug fixes + +```markdown +* Improved performance of SQL queries used to determine MDM profile status for Apple hosts. + +* Ensured request timeouts for software installer edits were just as high as for initial software installer uploads. + +* Fixed an issue with the migration adding support for multiple VPP tokens that would happen if a token was removed prior to upgrading Fleet. + +* Fixed a "no rows" error when adding a software installer that matched an existing title's name and source but not its bundle ID. +``` + ## Fleet 4.57.0 (Sep 23, 2024) **Endpoint Operations** diff --git a/changes/21370-bundle-id-quickfix b/changes/21370-bundle-id-quickfix deleted file mode 100644 index 1f27bedc40af..000000000000 --- a/changes/21370-bundle-id-quickfix +++ /dev/null @@ -1 +0,0 @@ -- Fix "no rows" error when adding a software installer that matches an existing title's name and source but not its bundle ID diff --git a/changes/22122-mdm-apple-status-queries b/changes/22122-mdm-apple-status-queries deleted file mode 100644 index 2ea893d31ff5..000000000000 --- a/changes/22122-mdm-apple-status-queries +++ /dev/null @@ -1 +0,0 @@ -- Improved performance of SQL queries used to determine MDM profile status for Apple hosts. \ No newline at end of file diff --git a/changes/22415-fix-vpp-migration b/changes/22415-fix-vpp-migration deleted file mode 100644 index eed21b93dc12..000000000000 --- a/changes/22415-fix-vpp-migration +++ /dev/null @@ -1 +0,0 @@ -* Fixed an issue with the migration adding support for multiple VPP tokens that would happen if a token is removed prior to upgrading Fleet. diff --git a/changes/software-edit-request-deadline b/changes/software-edit-request-deadline deleted file mode 100644 index f3576256f5ed..000000000000 --- a/changes/software-edit-request-deadline +++ /dev/null @@ -1 +0,0 @@ -* Ensure request timeouts for software installer edits are just as high as for initial software installer uploads diff --git a/charts/fleet/Chart.yaml b/charts/fleet/Chart.yaml index c23438bf22aa..f9abb8e29901 100644 --- a/charts/fleet/Chart.yaml +++ b/charts/fleet/Chart.yaml @@ -8,7 +8,7 @@ version: v6.2.0 home: https://github.com/fleetdm/fleet sources: - https://github.com/fleetdm/fleet.git -appVersion: v4.57.0 +appVersion: v4.57.1 dependencies: - name: mysql condition: mysql.enabled diff --git a/charts/fleet/values.yaml b/charts/fleet/values.yaml index 03539df9da98..15eda9b9e8ce 100644 --- a/charts/fleet/values.yaml +++ b/charts/fleet/values.yaml @@ -3,7 +3,7 @@ hostName: fleet.localhost replicas: 3 # The number of Fleet instances to deploy imageRepository: fleetdm/fleet -imageTag: v4.57.0 # Version of Fleet to deploy +imageTag: v4.57.1 # Version of Fleet to deploy podAnnotations: {} # Additional annotations to add to the Fleet pod serviceAccountAnnotations: {} # Additional annotations to add to the Fleet service account resources: diff --git a/infrastructure/dogfood/terraform/aws/variables.tf b/infrastructure/dogfood/terraform/aws/variables.tf index 2020de2f8306..d61f8819ec4b 100644 --- a/infrastructure/dogfood/terraform/aws/variables.tf +++ b/infrastructure/dogfood/terraform/aws/variables.tf @@ -56,7 +56,7 @@ variable "database_name" { variable "fleet_image" { description = "the name of the container image to run" - default = "fleetdm/fleet:v4.57.0" + default = "fleetdm/fleet:v4.57.1" } variable "software_inventory" { diff --git a/infrastructure/dogfood/terraform/gcp/variables.tf b/infrastructure/dogfood/terraform/gcp/variables.tf index 906a58c153f8..a8877ab5e00f 100644 --- a/infrastructure/dogfood/terraform/gcp/variables.tf +++ b/infrastructure/dogfood/terraform/gcp/variables.tf @@ -68,7 +68,7 @@ variable "redis_mem" { } variable "image" { - default = "fleetdm/fleet:v4.57.0" + default = "fleetdm/fleet:v4.57.1" } variable "software_installers_bucket_name" { diff --git a/terraform/addons/vuln-processing/variables.tf b/terraform/addons/vuln-processing/variables.tf index 8d296903fdc4..11c5bb870715 100644 --- a/terraform/addons/vuln-processing/variables.tf +++ b/terraform/addons/vuln-processing/variables.tf @@ -24,7 +24,7 @@ variable "fleet_config" { vuln_processing_cpu = optional(number, 2048) vuln_data_stream_mem = optional(number, 1024) vuln_data_stream_cpu = optional(number, 512) - image = optional(string, "fleetdm/fleet:v4.57.0") + image = optional(string, "fleetdm/fleet:v4.57.1") family = optional(string, "fleet-vuln-processing") sidecars = optional(list(any), []) extra_environment_variables = optional(map(string), {}) @@ -82,7 +82,7 @@ variable "fleet_config" { vuln_processing_cpu = 2048 vuln_data_stream_mem = 1024 vuln_data_stream_cpu = 512 - image = "fleetdm/fleet:v4.57.0" + image = "fleetdm/fleet:v4.57.1" family = "fleet-vuln-processing" sidecars = [] extra_environment_variables = {} diff --git a/terraform/byo-vpc/byo-db/byo-ecs/variables.tf b/terraform/byo-vpc/byo-db/byo-ecs/variables.tf index 27565cb90fa8..51dc799bc5b7 100644 --- a/terraform/byo-vpc/byo-db/byo-ecs/variables.tf +++ b/terraform/byo-vpc/byo-db/byo-ecs/variables.tf @@ -16,7 +16,7 @@ variable "fleet_config" { mem = optional(number, 4096) cpu = optional(number, 512) pid_mode = optional(string, null) - image = optional(string, "fleetdm/fleet:v4.57.0") + image = optional(string, "fleetdm/fleet:v4.57.1") family = optional(string, "fleet") sidecars = optional(list(any), []) depends_on = optional(list(any), []) @@ -119,7 +119,7 @@ variable "fleet_config" { mem = 512 cpu = 256 pid_mode = null - image = "fleetdm/fleet:v4.57.0" + image = "fleetdm/fleet:v4.57.1" family = "fleet" sidecars = [] depends_on = [] diff --git a/terraform/byo-vpc/byo-db/variables.tf b/terraform/byo-vpc/byo-db/variables.tf index 041ff9d0f861..b19d74931ec5 100644 --- a/terraform/byo-vpc/byo-db/variables.tf +++ b/terraform/byo-vpc/byo-db/variables.tf @@ -77,7 +77,7 @@ variable "fleet_config" { mem = optional(number, 4096) cpu = optional(number, 512) pid_mode = optional(string, null) - image = optional(string, "fleetdm/fleet:v4.57.0") + image = optional(string, "fleetdm/fleet:v4.57.1") family = optional(string, "fleet") sidecars = optional(list(any), []) depends_on = optional(list(any), []) @@ -205,7 +205,7 @@ variable "fleet_config" { mem = 512 cpu = 256 pid_mode = null - image = "fleetdm/fleet:v4.57.0" + image = "fleetdm/fleet:v4.57.1" family = "fleet" sidecars = [] depends_on = [] diff --git a/terraform/byo-vpc/example/main.tf b/terraform/byo-vpc/example/main.tf index 3176d07def1f..7f75b975ad42 100644 --- a/terraform/byo-vpc/example/main.tf +++ b/terraform/byo-vpc/example/main.tf @@ -17,7 +17,7 @@ provider "aws" { } locals { - fleet_image = "fleetdm/fleet:v4.57.0" + fleet_image = "fleetdm/fleet:v4.57.1" domain_name = "example.com" } diff --git a/terraform/byo-vpc/variables.tf b/terraform/byo-vpc/variables.tf index ce2a81f88c41..2c8fdb1d1742 100644 --- a/terraform/byo-vpc/variables.tf +++ b/terraform/byo-vpc/variables.tf @@ -170,7 +170,7 @@ variable "fleet_config" { mem = optional(number, 4096) cpu = optional(number, 512) pid_mode = optional(string, null) - image = optional(string, "fleetdm/fleet:v4.57.0") + image = optional(string, "fleetdm/fleet:v4.57.1") family = optional(string, "fleet") sidecars = optional(list(any), []) depends_on = optional(list(any), []) @@ -298,7 +298,7 @@ variable "fleet_config" { mem = 512 cpu = 256 pid_mode = null - image = "fleetdm/fleet:v4.57.0" + image = "fleetdm/fleet:v4.57.1" family = "fleet" sidecars = [] depends_on = [] diff --git a/terraform/example/main.tf b/terraform/example/main.tf index 2b2112517925..bf8f569a36b8 100644 --- a/terraform/example/main.tf +++ b/terraform/example/main.tf @@ -63,8 +63,8 @@ module "fleet" { fleet_config = { # To avoid pull-rate limiting from dockerhub, consider using our quay.io mirror - # for the Fleet image. e.g. "quay.io/fleetdm/fleet:v4.57.0" - image = "fleetdm/fleet:v4.57.0" # override default to deploy the image you desire + # for the Fleet image. e.g. "quay.io/fleetdm/fleet:v4.57.1" + image = "fleetdm/fleet:v4.57.1" # override default to deploy the image you desire # See https://fleetdm.com/docs/deploy/reference-architectures#aws for appropriate scaling # memory and cpu. autoscaling = { diff --git a/terraform/variables.tf b/terraform/variables.tf index 7dc798cf63d8..de37b0020f25 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -218,7 +218,7 @@ variable "fleet_config" { mem = optional(number, 4096) cpu = optional(number, 512) pid_mode = optional(string, null) - image = optional(string, "fleetdm/fleet:v4.57.0") + image = optional(string, "fleetdm/fleet:v4.57.1") family = optional(string, "fleet") sidecars = optional(list(any), []) depends_on = optional(list(any), []) @@ -346,7 +346,7 @@ variable "fleet_config" { mem = 512 cpu = 256 pid_mode = null - image = "fleetdm/fleet:v4.57.0" + image = "fleetdm/fleet:v4.57.1" family = "fleet" sidecars = [] depends_on = [] diff --git a/tools/fleetctl-npm/package.json b/tools/fleetctl-npm/package.json index 96a4dcd08170..13ac3172e426 100644 --- a/tools/fleetctl-npm/package.json +++ b/tools/fleetctl-npm/package.json @@ -1,6 +1,6 @@ { "name": "fleetctl", - "version": "v4.57.0", + "version": "v4.57.1", "description": "Installer for the fleetctl CLI tool", "bin": { "fleetctl": "./run.js"