QE: Improve how we wait for each product to be synchronized

testsuite/features/reposync/srv_sync_products.feature

@@ -61,10 +61,12 @@ Feature: Synchronize products in the products page of the Setup Wizard
     When I click the Add Product button
     And I wait until I see "SUSE Linux Enterprise Server 15 SP4 x86_64" product has been added
     Then the SLE15 SP4 product should be added
+    When I wait until all synchronized channels for "sles15-sp4" have finished
 
 @uyuni
   Scenario: Add openSUSE Leap 15.5 product, including Uyuni Client Tools
     When I use spacewalk-common-channel to add channel "opensuse_leap15_5 opensuse_leap15_5-non-oss opensuse_leap15_5-non-oss-updates opensuse_leap15_5-updates opensuse_leap15_5-backports-updates opensuse_leap15_5-sle-updates uyuni-proxy-devel-leap opensuse_leap15_5-uyuni-client" with arch "x86_64"
+    And I kill running spacewalk-repo-sync for "leap15.5-x86_64"
 
 @proxy
 @susemanager
@@ -77,6 +79,7 @@ Feature: Synchronize products in the products page of the Setup Wizard
     When I click the Add Product button
     And I wait until I see "Selected channels/products were scheduled successfully for syncing." text
     And I wait until I see "SUSE Manager Proxy 4.3 x86_64" product has been added
+    And I wait until all synchronized channels for "suma-proxy-43" have finished
 
 @proxy
 @susemanager
@@ -89,6 +92,7 @@ Feature: Synchronize products in the products page of the Setup Wizard
     When I click the Add Product button
     And I wait until I see "Selected channels/products were scheduled successfully for syncing." text
     And I wait until I see "SUSE Manager Retail Branch Server 4.3 x86_64" product has been added
+    And I wait until all synchronized channels for "suma-retail-branch-server-43" have finished
 
 @scc_credentials
 @susemanager

testsuite/features/reposync/srv_wait_for_reposync.feature

@@ -1,7 +1,7 @@
 # Copyright (c) 2019-2023 SUSE LLC
 # Licensed under the terms of the MIT license.
 
-Feature: Wait for reposync activity to finish in CI context
+Feature: Delete the scheduled task for mgr-sync-refresh
 
   Scenario: Log in as admin user
     Given I am authorized for the "Admin" section
@@ -12,9 +12,3 @@ Feature: Wait for reposync activity to finish in CI context
     And I choose "disabled"
     And I click on "Update Schedule"
     And I click on "Delete Schedule"
-
-  Scenario: Kill running reposyncs or wait for them to finish
-    When I kill all running spacewalk-repo-sync, excepted the ones needed to bootstrap
-
-  Scenario: Wait until all synchronized channels have finished
-    When I wait until all synchronized channels have finished

testsuite/features/step_definitions/command_steps.rb

@@ -139,8 +139,7 @@
 end
 
 When(/^I use spacewalk\-repo\-sync to sync channel "([^"]*)"$/) do |channel|
-  command = "spacewalk-repo-sync -c #{channel}"
-  $command_output, _code = get_target('server').run(command)
+  $command_output, _code = get_target('server').run_until_ok("spacewalk-repo-sync -c #{channel}")
 end
 
 Then(/^I should get "([^"]*)"$/) do |value|
@@ -344,41 +343,21 @@
   end
 end
 
-# This function kills all spacewalk-repo-sync processes, excepted the ones in a whitelist.
+# This function kills spacewalk-repo-sync processes for a particular OS product version.
 # It waits for all the reposyncs in the whitelist to complete, and kills all others.
-#
-# This function is written as a state machine. It bails out if no process is seen during
-# 60 seconds in a row, or if the whitelisted reposyncs last more than 7200 seconds in a row.
-When(/^I kill all running spacewalk\-repo\-sync, excepted the ones needed to bootstrap$/) do
-  do_not_kill = compute_channels_to_leave_running
-  reposync_not_running_streak = 0
-  reposync_left_running_streak = 0
-  while reposync_not_running_streak <= 60
+When(/^I kill running spacewalk\-repo\-sync for "([^"]*)"$/) do |os_product_version|
+  next if CHANNEL_TO_SYNCH_BY_OS_PRODUCT_VERSION[os_product_version].nil?
+  channels_to_kill = CHANNEL_TO_SYNCH_BY_OS_PRODUCT_VERSION[os_product_version]
+  repeat_until_timeout(timeout: DEFAULT_TIMEOUT, message: 'Some reposync processes were not killed properly') do
     command_output, _code = get_target('server').run('ps axo pid,cmd | grep spacewalk-repo-sync | grep -v grep', check_errors: false)
-    if command_output.empty?
-      reposync_not_running_streak += 1
-      reposync_left_running_streak = 0
-      sleep 1
-      next
-    end
-    reposync_not_running_streak = 0
-
     process = command_output.split("\n")[0]
     channel = process.split(' ')[5]
-    if do_not_kill.include? channel
-      $channels_synchronized.add(channel)
-      log "Reposync of channel #{channel} left running" if (reposync_left_running_streak % 60).zero?
-      reposync_left_running_streak += 1
-
-      raise 'We have a reposync process that still running after 2 hours' if reposync_left_running_streak > 7200
-      sleep 1
-      next
-    end
-    reposync_left_running_streak = 0
-
+    next unless CHANNEL_TO_SYNCH_BY_OS_PRODUCT_VERSION[os_product_version].include? channel
+    channels_to_kill.delete(channel)
     pid = process.split(' ')[0]
     get_target('server').run("kill #{pid}", check_errors: false)
     log "Reposync of channel #{channel} killed"
+    break if channels_to_kill.empty?
   end
 end
 
@@ -448,6 +427,13 @@
   end
 end
 
+When(/I wait until all synchronized channels for "([^"]*)" have finished$/) do |product_os_version|
+  CHANNEL_TO_SYNCH_BY_OS_PRODUCT_VERSION[product_os_version].each do |channel|
+    log "I wait until '#{channel}' synchronized channel has finished"
+    step %(I wait until the channel "#{channel}" has been synced)
+  end
+end
+
 When(/^I execute mgr\-bootstrap "([^"]*)"$/) do |arg1|
   $command_output, _code = get_target('server').run("mgr-bootstrap #{arg1}")
 end
@@ -770,15 +756,15 @@
 end
 
 When(/^I call spacewalk\-repo\-sync for channel "(.*?)" with a custom url "(.*?)"$/) do |arg1, arg2|
-  @command_output, _code = get_target('server').run("spacewalk-repo-sync -c #{arg1} -u #{arg2}", check_errors: false)
+  @command_output, _code = get_target('server').run_until_ok("spacewalk-repo-sync -c #{arg1} -u #{arg2}")
 end
 
 When(/^I call spacewalk\-repo\-sync to sync the channel "(.*?)"$/) do |channel|
-  @command_output, _code = get_target('server').run("spacewalk-repo-sync -c #{channel}", check_errors: false)
+  @command_output, _code = get_target('server').run_until_ok("spacewalk-repo-sync -c #{channel}")
 end
 
 When(/^I call spacewalk\-repo\-sync to sync the parent channel "(.*?)"$/) do |channel|
-  @command_output, _code = get_target('server').run("spacewalk-repo-sync -p #{channel}", check_errors: false)
+  @command_output, _code = get_target('server').run_until_ok("spacewalk-repo-sync -p #{channel}")
 end
 
 When(/^I get "(.*?)" file details for channel "(.*?)" via spacecmd$/) do |arg1, arg2|

testsuite/features/step_definitions/navigation_steps.rb

@@ -90,10 +90,10 @@
 
 When(/^I wait at most (\d+) seconds until the event is completed, refreshing the page$/) do |timeout|
   last = Time.now
-  next if has_content?("This action's status is: Completed.", wait: 3)
+  next if has_content?('This action\'s status is: Completed.', wait: 3)
   repeat_until_timeout(timeout: timeout.to_i, message: 'Event not yet completed') do
-    break if has_content?("This action's status is: Completed.", wait: 3)
-    raise 'Event failed' if has_content?("This action's status is: Failed.", wait: 3)
+    break if has_content?('This action\'s status is: Completed.', wait: 3)
+    raise 'Event failed' if has_content?('This action\'s status is: Failed.', wait: 3)
     current = Time.now
     if current - last > 150
       log "#{current} Still waiting for action to complete..."
@@ -225,13 +225,13 @@
 
 When(/^I (include|exclude) the recommended child channels$/) do |action|
   toggle = "//span[@class='pointer']"
-  step %(I wait at most 10 seconds until I see "include recommended" text)
+  step 'I wait at most 10 seconds until I see "include recommended" text'
   raise 'The toggle is not present' unless page.has_xpath?(toggle, wait: 5)
   if action == 'include'
-    toggle_off = "//i[contains(@class, 'fa-toggle-off')]"
+    toggle_off = '//i[contains(@class, \'fa-toggle-off\')]'
     find(:xpath, toggle).click if page.has_xpath?(toggle_off, wait: 5)
   else
-    toggle_on = "//i[contains(@class, 'fa-toggle-on')]"
+    toggle_on = '//i[contains(@class, \'fa-toggle-on\')]'
     find(:xpath, toggle).click if page.has_xpath?(toggle_on, wait: 5)
   end
 end
@@ -295,13 +295,11 @@
 #
 # Click on a button and confirm in alert box
 When(/^I click on "([^"]*)" and confirm$/) do |text|
-  begin
-    accept_alert do
-      step %(I click on "#{text}")
-    end
-  rescue Capybara::ModalNotFound
-    # ignored
+  accept_alert do
+    step %(I click on "#{text}")
   end
+rescue Capybara::ModalNotFound
+  warn 'Modal not found'
 end
 
 #
@@ -365,8 +363,8 @@
   menu_levels = menu_path.split('>').map(&:strip)
 
   # define reusable patterns
-  prefix_path = "//aside/div[@id='nav']/nav"
-  link_path = "/ul/li/div/a[contains(.,'%s')]"
+  prefix_path = '//aside/div[@id=\'nav\']/nav'
+  link_path = '/ul/li/div/a[contains(.,\'%s\')]'
   parent_wrapper_path = '/parent::div'
   parent_level_path = '/parent::li'
 
@@ -379,8 +377,14 @@
     # if this is the last element of the path
     break if index == (menu_levels.count - 1)
     # open the submenu if needed
-    unless find(:xpath, target_link_path + parent_wrapper_path + parent_level_path)[:class].include?('open')
-      find(:xpath, target_link_path + parent_wrapper_path).click
+    begin
+      unless find(:xpath, target_link_path + parent_wrapper_path + parent_level_path)[:class].include?('open')
+        find(:xpath, target_link_path + parent_wrapper_path).click
+      end
+    rescue NoMethodError
+      warn 'The browser session seems broken. See debug details below:'
+      warn "target_link_path=#{target_link_path}, parent_wrapper_path=#{parent_wrapper_path}, parent_level_path=#{parent_level_path}"
+      visit Capybara.app_host
     end
     # point the target to the current menu level
     target_link_path += parent_wrapper_path + parent_level_path
@@ -952,11 +956,11 @@
 #
 # Test if a checkbox is disabled
 #
-Then(/^the "([^\"]*)" checkbox should be disabled$/) do |arg1|
+Then(/^the "([^"]*)" checkbox should be disabled$/) do |arg1|
   has_css?("##{arg1}[disabled]")
 end
 
-Then(/^the "([^\"]*)" field should be disabled$/) do |arg1|
+Then(/^the "([^"]*)" field should be disabled$/) do |arg1|
   has_css?("##{arg1}[disabled]")
 end
 
@@ -1006,12 +1010,10 @@
   # We wait until the element is not shown, because
   # the fade out animation might still be in progress
   repeat_until_timeout(message: "The #{title} modal dialog is still present") do
-    begin
-      break if has_no_xpath?(path, wait: 1)
-    rescue Selenium::WebDriver::Error::StaleElementReferenceError
-      # We need to consider the case that after obtaining the element it is detached from the page document
-      break
-    end
+    break if has_no_xpath?(path, wait: 1)
+  rescue Selenium::WebDriver::Error::StaleElementReferenceError
+    # We need to consider the case that after obtaining the element it is detached from the page document
+    break
   end
 end
 
@@ -1078,13 +1080,11 @@
 end
 
 When(/^I refresh the page$/) do
-  begin
-    accept_prompt do
-      execute_script 'window.location.reload()'
-    end
-  rescue Capybara::ModalNotFound
-    # ignored
+  accept_prompt do
+    execute_script 'window.location.reload()'
   end
+rescue Capybara::ModalNotFound
+  # ignored
 end
 
 When(/^I make a list of the existing systems$/) do

testsuite/features/support/commonlib.rb

@@ -19,30 +19,6 @@ def generate_temp_file(name, content)
   end
 end
 
-# If we for example
-#  - start a reposync in reposync/srv_sync_channels.feature
-#  - then kill it in reposync/srv_wait_for_reposync.feature
-#  - then restart it later on in init_clients/sle_minion.feature
-# then the channel will be in an inconsistent state.
-#
-# This function computes a list of reposyncs to avoid killing, because they might be involved in bootstrapping.
-#
-# This is a safety net only, the best thing to do is to not start the reposync at all.
-def compute_channels_to_leave_running
-  # keep the repos needed for the auto-installation tests
-  do_not_kill = CHANNEL_TO_SYNCH_BY_OS_VERSION['default']
-  [get_target('sle_minion'), get_target('build_host'), get_target('ssh_minion'), get_target('rhlike_minion')].each do |node|
-    next unless node
-    os_version = node.os_version
-    os_family = node.os_family
-    next unless ['sles', 'rocky'].include?(os_family)
-    os_version = os_version.split('.')[0] if os_family == 'rocky'
-    log 'Can\'t build list of reposyncs to leave running' unless %w[15-SP3 15-SP4 8].include? os_version
-    do_not_kill += CHANNEL_TO_SYNCH_BY_OS_VERSION[os_version]
-  end
-  do_not_kill.uniq
-end
-
 def count_table_items
   # count table items using the table counter component
   items_label_xpath = '//span[contains(text(), \'Items \')]'