From 8fb08f0d81370a85b9f1d65479392b331fadf92d Mon Sep 17 00:00:00 2001 From: MelsHyrule Date: Wed, 8 Feb 2023 14:08:54 -0500 Subject: [PATCH 01/11] host params for update schema fields --- .../providers/vmware/infra_manager/host.rb | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/app/models/manageiq/providers/vmware/infra_manager/host.rb b/app/models/manageiq/providers/vmware/infra_manager/host.rb index b282d478c..756ee905c 100644 --- a/app/models/manageiq/providers/vmware/infra_manager/host.rb +++ b/app/models/manageiq/providers/vmware/infra_manager/host.rb @@ -3,6 +3,7 @@ class ManageIQ::Providers::Vmware::InfraManager::Host < ::Host include ManageIQ::Providers::Vmware::InfraManager::EmsRefObjMixin supports :capture + supports :update # overrides base start to support "standby" powerstate supports :start do @@ -100,4 +101,93 @@ def detect_discovered_hypervisor(_ost, ipaddr) def self.display_name(number = 1) n_('Host (Vmware)', 'Hosts (Vmware)', number) end + + def params_for_update + { + :fields => [ + { + :component => 'sub-form', + :id => 'endpoints-subform', + :name => 'endpoints-subform', + :title => _("Endpoints"), + :fields => [ + :component => 'tabs', + :name => 'tabs', + :fields => [ + { + :component => 'tab-item', + :id => 'ws-tab', + :name => 'ws-tab', + :title => _('Web Service'), + :fields => [ + { + :component => 'validate-host-credentials', + :id => 'endpoints.ws.valid', + :name => 'endpoints.ws.valid', + :skipSubmit => true, + :isRequired => true, + :fields => [ + { + :component => "text-field", + :id => "authentications.ws.userid", + :name => "authentications.ws.userid", + :label => _("Username"), + :isRequired => true, + :validate => [{:type => "required"}], + }, + { + :component => "password-field", + :id => "authentications.ws.password", + :name => "authentications.ws.password", + :label => _("Password"), + :type => "password", + :isRequired => true, + :validate => [{:type => "required"}], + :helperText => _('Used for access to Web Services.') + }, + ], + }, + ], + }, + { + :component => 'tab-item', + :id => 'remote-tab', + :name => 'remote-tab', + :title => _('Remote'), + :fields => [ + { + :component => 'validate-remote-credentials', + :id => 'endpoints.remote.valid', + :name => 'endpoints.remote.valid', + :skipSubmit => true, + :isRequired => true, + :fields => [ + { + :component => "text-field", + :id => "authentications.remote.userid", + :name => "authentications.remote.userid", + :label => _("Username"), + :isRequired => true, + :validate => [{:type => "required"}], + }, + { + :component => "password-field", + :id => "authentications.remote.password", + :name => "authentications.remote.password", + :label => _("Password"), + :type => "password", + :isRequired => true, + :validate => [{:type => "required"}], + :helperText => _('Used for SSH login.') + }, + ], + }, + ], + }, + ] + ] + }, + ] + } + end end From 9cbb5b20ea8677cba6f37600d7982c20da68948b Mon Sep 17 00:00:00 2001 From: Keenan Brock Date: Thu, 16 Feb 2023 16:13:22 -0500 Subject: [PATCH 02/11] removing host discovery --- .../providers/vmware/infra_manager/host.rb | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/app/models/manageiq/providers/vmware/infra_manager/host.rb b/app/models/manageiq/providers/vmware/infra_manager/host.rb index b282d478c..bed7bb2ab 100644 --- a/app/models/manageiq/providers/vmware/infra_manager/host.rb +++ b/app/models/manageiq/providers/vmware/infra_manager/host.rb @@ -70,31 +70,6 @@ def reserve_next_available_vnc_port end end - def detect_discovered_hypervisor(_ost, ipaddr) - find_method = :find_by_ipaddress - - self.name = "VMware ESX Server (#{ipaddr})" - self.ipaddress = ipaddr - self.vmm_vendor = "vmware" - self.vmm_product = "Esx" - if has_credentials?(:ws) - begin - with_provider_connection(:ip => ipaddr) do |vim| - _log.info("VIM Information for ESX Host with IP Address: [#{ipaddr}], Information: #{vim.about.inspect}") - self.vmm_product = vim.about['name'].dup.split(' ').last - self.vmm_version = vim.about['version'] - self.vmm_buildnumber = vim.about['build'] - self.name = "#{vim.about['name']} (#{ipaddr})" - end - rescue => err - _log.warn("Cannot connect to ESX Host with IP Address: [#{ipaddr}], Username: [#{authentication_userid(:ws)}] because #{err.message}") - end - end - self.type = %w(esx esxi).include?(vmm_product.to_s.downcase) ? "ManageIQ::Providers::Vmware::InfraManager::HostEsx" : "ManageIQ::Providers::Vmware::InfraManager::Host" - - find_method - end - supports :quick_stats def self.display_name(number = 1) From 7006e8bbfe66eb5fd8f59b73c03299af7c3dd657 Mon Sep 17 00:00:00 2001 From: Keenan Brock Date: Tue, 28 Feb 2023 20:46:06 -0500 Subject: [PATCH 03/11] make default credentials verify ws Host used to default credential verification to ws. HostEsx is the only host that implements ws. So now, the default verification is going over ssh. This change keeps the default verification method as ws for Esx Hosts --- .../manageiq/providers/vmware/infra_manager/host_esx.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/models/manageiq/providers/vmware/infra_manager/host_esx.rb b/app/models/manageiq/providers/vmware/infra_manager/host_esx.rb index 276491b9e..384602734 100644 --- a/app/models/manageiq/providers/vmware/infra_manager/host_esx.rb +++ b/app/models/manageiq/providers/vmware/infra_manager/host_esx.rb @@ -164,6 +164,10 @@ def verify_credentials_with_ws(auth_type = nil) end end + def verify_credentials_default(auth_type, _options) + verify_credentials_with_ws(auth_type) + end + def refresh_logs if self.missing_credentials? _log.warn "No credentials defined for Host [#{name}]" From 0c607c6bc6b9a0097f367bc9490517565fdaf71c Mon Sep 17 00:00:00 2001 From: Adam Grare Date: Wed, 1 Mar 2023 16:29:54 -0500 Subject: [PATCH 04/11] Set the VMwareWebService.logger --- lib/manageiq/providers/vmware/engine.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/manageiq/providers/vmware/engine.rb b/lib/manageiq/providers/vmware/engine.rb index e4f335e6a..5f735ebc7 100644 --- a/lib/manageiq/providers/vmware/engine.rb +++ b/lib/manageiq/providers/vmware/engine.rb @@ -29,6 +29,9 @@ def self.init_loggers require "rbvmomi" RbVmomi.logger = $vim_log + + require "vmware_web_service" + VMwareWebService.logger = $vim_log end def self.apply_logger_config(config) From 5f9cd33af12b8fd55cead91249aad7d91cb41f78 Mon Sep 17 00:00:00 2001 From: Adam Grare Date: Fri, 3 Mar 2023 14:51:05 -0500 Subject: [PATCH 05/11] Fix wrong number of arguments delete_snapshot_by_description Fix `wrong number of arguments (given 3, expected 0; required keywords: server, username, password)` --- .../manageiq/providers/vmware/infra_manager/scanning/job.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/manageiq/providers/vmware/infra_manager/scanning/job.rb b/app/models/manageiq/providers/vmware/infra_manager/scanning/job.rb index ab7301fc6..b7fa3e8a0 100644 --- a/app/models/manageiq/providers/vmware/infra_manager/scanning/job.rb +++ b/app/models/manageiq/providers/vmware/infra_manager/scanning/job.rb @@ -191,7 +191,7 @@ def delete_snapshot_by_description(mor) begin password_decrypt = ManageIQ::Password.decrypt(miqVimHost[:password]) require 'VMwareWebService/MiqVim' - miqVim = MiqVim.new(server, miqVimHost[:username], password_decrypt) + miqVim = MiqVim.new(:server => server, :username => miqVimHost[:username], :password => password_decrypt) vimVm = miqVim.getVimVm(vm.path) vimVm.removeSnapshotByDescription(mor, true) unless vimVm.nil? From 54a20ea76c49979e48140dc6666644c762bd9acb Mon Sep 17 00:00:00 2001 From: Adam Grare Date: Sat, 4 Mar 2023 14:34:58 -0500 Subject: [PATCH 06/11] Pull in vmware_web_service v3.2.0 --- manageiq-providers-vmware.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manageiq-providers-vmware.gemspec b/manageiq-providers-vmware.gemspec index e3c0ec573..c352dfe0e 100644 --- a/manageiq-providers-vmware.gemspec +++ b/manageiq-providers-vmware.gemspec @@ -21,7 +21,7 @@ Gem::Specification.new do |spec| spec.add_dependency "fog-vcloud-director", "~> 0.3.0" spec.add_dependency "ffi-vix_disk_lib", "~>1.1" spec.add_dependency "rbvmomi2", "~>3.5" - spec.add_dependency "vmware_web_service", "~>3.0" + spec.add_dependency "vmware_web_service", "~>3.2" spec.add_dependency "vsphere-automation-sdk", "~>0.4.7" spec.add_development_dependency "manageiq-style" From e5c0bd9d08585f5fd8771ac451f2a3141191207a Mon Sep 17 00:00:00 2001 From: MelsHyrule Date: Thu, 23 Mar 2023 12:29:47 -0400 Subject: [PATCH 07/11] Enabled / Disabled field --- .../providers/vmware/infra_manager/host.rb | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/app/models/manageiq/providers/vmware/infra_manager/host.rb b/app/models/manageiq/providers/vmware/infra_manager/host.rb index 99b81e0ab..9b31e3a18 100644 --- a/app/models/manageiq/providers/vmware/infra_manager/host.rb +++ b/app/models/manageiq/providers/vmware/infra_manager/host.rb @@ -131,11 +131,32 @@ def params_for_update :title => _('Remote'), :fields => [ { - :component => 'validate-remote-credentials', + :component => 'protocol-selector', + :id => 'remoteEnabled', + :name => 'remoteEnabled', + :skipSubmit => true, + :initialValue => 'disabled', + :label => _('Enabled'), + :options => [ + { + :label => _('Disabled'), + :value => 'disabled' + }, + { + :label => _('Enabled'), + :value => 'enabled', + }, + ], + }, + { + :component => 'validate-host-credentials', :id => 'endpoints.remote.valid', :name => 'endpoints.remote.valid', :skipSubmit => true, - :isRequired => true, + :condition => { + :when => 'remoteEnabled', + :is => 'enabled', + }, :fields => [ { :component => "text-field", From 7cd5f8dd4aa79c271d60798ed4898f611ac68e0e Mon Sep 17 00:00:00 2001 From: Adam Grare Date: Tue, 25 Apr 2023 14:54:35 -0400 Subject: [PATCH 08/11] Prevent `""` domain_name causing trailing `.` If a domain_name is an empty string and not just blank we will build a hostname with a trailing `.` --- .../vmware/infra_manager/inventory/parser/host_system.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/manageiq/providers/vmware/infra_manager/inventory/parser/host_system.rb b/app/models/manageiq/providers/vmware/infra_manager/inventory/parser/host_system.rb index c77460f41..f7902ed96 100644 --- a/app/models/manageiq/providers/vmware/infra_manager/inventory/parser/host_system.rb +++ b/app/models/manageiq/providers/vmware/infra_manager/inventory/parser/host_system.rb @@ -46,7 +46,7 @@ def parse_host_system_network(host_hash, props) hostname = dns_config[:hostName] domain_name = dns_config[:domainName] - hostname = "#{hostname}.#{domain_name}" if domain_name + hostname = "#{hostname}.#{domain_name}" if domain_name.present? host_hash[:name] = hostname host_hash[:hostname] = hostname From 9ed9aaa7ae701174fe75fea27c80d3c19ec85079 Mon Sep 17 00:00:00 2001 From: Joe Rafaniello Date: Wed, 26 Apr 2023 16:35:44 -0400 Subject: [PATCH 09/11] Drop lib/gem-name.rb in favor of lib/gem/name.rb Adopt convention already supported by bundler and required by zeitwerk so we don't need to tell zeitwerk to ignore this lib/gem-name.rb file. --- lib/manageiq-providers-vmware.rb | 4 ---- lib/manageiq/providers/vmware.rb | 3 +++ spec/spec_helper.rb | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) delete mode 100644 lib/manageiq-providers-vmware.rb diff --git a/lib/manageiq-providers-vmware.rb b/lib/manageiq-providers-vmware.rb deleted file mode 100644 index 0a2b54bfe..000000000 --- a/lib/manageiq-providers-vmware.rb +++ /dev/null @@ -1,4 +0,0 @@ -require "manageiq/providers/vmware/engine" -require "manageiq/providers/vmware/version" - -require "manageiq/providers/vmware" diff --git a/lib/manageiq/providers/vmware.rb b/lib/manageiq/providers/vmware.rb index bf4e72496..426f17660 100644 --- a/lib/manageiq/providers/vmware.rb +++ b/lib/manageiq/providers/vmware.rb @@ -1,3 +1,6 @@ +require "manageiq/providers/vmware/engine" +require "manageiq/providers/vmware/version" + module ManageIQ module Providers module Vmware diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 140159c6b..00068351d 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -6,7 +6,7 @@ Dir[Rails.root.join("spec/shared/**/*.rb")].each { |f| require f } Dir[File.join(__dir__, "support/**/*.rb")].each { |f| require f } -require "manageiq-providers-vmware" +require "manageiq/providers/vmware" VCR.configure do |config| config.ignore_hosts 'codeclimate.com' if ENV['CI'] From cca8a80d93f42f21560187faaf237bdd45bb97bb Mon Sep 17 00:00:00 2001 From: Keenan Brock Date: Sun, 14 May 2023 22:35:00 -0400 Subject: [PATCH 10/11] Converge batch_size and concurrent_requests Before - concurrent_requests, thought it existed in core, was vmware only. It was split up into historical, hourly, and realtime. - batch_size was per provider (but only implemented by vmware) It just has a single option. After - concurrent_requests defined in the provider (but only implemented by vmare) It is split up into historical, hourly, and realtime. Reason for change While batch_size may be 250, concurrent_requests was then applied and we were sending only 20 requests for realtime and 1 for historical. So in the end, we only are sending 1 request to the provider. The optimization has basically been disabled. This gives better values for concurrent requests, and matches the requests sent to the collector over the queue and from the collector to the provider This requires changes in both vmware and core. --- .../providers/vmware/infra_manager/metrics_capture.rb | 9 ++++++++- config/settings.yml | 5 ++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/app/models/manageiq/providers/vmware/infra_manager/metrics_capture.rb b/app/models/manageiq/providers/vmware/infra_manager/metrics_capture.rb index 5d2ebc96f..1c2baf288 100644 --- a/app/models/manageiq/providers/vmware/infra_manager/metrics_capture.rb +++ b/app/models/manageiq/providers/vmware/infra_manager/metrics_capture.rb @@ -371,7 +371,7 @@ def perf_query(params, counter_info, interval_name) Benchmark.current_realtime[:num_vim_queries] = params.length _log.debug { "Total item(s) to be requested: [#{params.length}], #{params.inspect}" } - query_size = Metric::Capture.concurrent_requests(interval_name) + query_size = concurrent_requests(interval_name) vim_trips = 0 params.each_slice(query_size) do |query| vim_trips += 1 @@ -389,6 +389,13 @@ def perf_query(params, counter_info, interval_name) return counters_by_mor, counter_values_by_mor_and_ts end + def concurrent_requests(interval_name) + query_size = super + # even when batching is turned off, still want to send multiple realtime records + query_size = 20 if query_size < 20 && interval_name == "realtime" + query_size + end + class << self private diff --git a/config/settings.yml b/config/settings.yml index 597bde1a2..e23636814 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -8,7 +8,10 @@ :critical: :power: :critical: - :capture_batch_size: 250 + :concurrent_requests: + :historical: 100 + :hourly: 250 + :realtime: 100 :ems_refresh: :vmware_tanzu: :refresh_interval: 24.hours From fadcca00fb084ddb707086cda60eb46fb1686e34 Mon Sep 17 00:00:00 2001 From: Keenan Brock Date: Thu, 25 May 2023 13:01:02 -0400 Subject: [PATCH 11/11] Defer metrics collection batching to queue Before: Core partitions the metrics collection work and putting them onto the queue. Vmware then double checks the work, adding partitioning. This is unnecessary and in the process, overriding the concurrent_requests value confuses core when it slices up the metrics collector requests. After: Vmware is now expecting the queue messages to be pre-partitioned. So no extra work is done here. --- .../vmware/infra_manager/metrics_capture.rb | 29 +++++-------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/app/models/manageiq/providers/vmware/infra_manager/metrics_capture.rb b/app/models/manageiq/providers/vmware/infra_manager/metrics_capture.rb index 1c2baf288..0654ce852 100644 --- a/app/models/manageiq/providers/vmware/infra_manager/metrics_capture.rb +++ b/app/models/manageiq/providers/vmware/infra_manager/metrics_capture.rb @@ -281,7 +281,7 @@ def perf_collect_metrics(interval_name, start_time = nil, end_time = nil) counter_info, = Benchmark.realtime_block(:counter_info) { self.class.counter_info_by_counter_id(ems, @perf_vim_hist) } interval_by_mor, = Benchmark.realtime_block(:capture_intervals) { perf_capture_intervals(targets_by_mor.keys, interval_name) } query_params, = Benchmark.realtime_block(:build_query_params) { perf_build_query_params(interval_by_mor, counter_info, start_time, end_time) } - counters_by_mor, counter_values_by_mor_and_ts = perf_query(query_params, counter_info, interval_name) + counters_by_mor, counter_values_by_mor_and_ts = perf_query(query_params, counter_info) return counters_by_mor, counter_values_by_mor_and_ts rescue HTTPClient::ReceiveTimeoutError => err @@ -363,39 +363,24 @@ def perf_build_query_params(interval_by_mor, counter_info, start_time, end_time) params end - def perf_query(params, counter_info, interval_name) + def perf_query(params, counter_info) counters_by_mor = {} counter_values_by_mor_and_ts = {} return counter_values_by_mor_and_ts if params.blank? Benchmark.current_realtime[:num_vim_queries] = params.length - _log.debug { "Total item(s) to be requested: [#{params.length}], #{params.inspect}" } - query_size = concurrent_requests(interval_name) - vim_trips = 0 - params.each_slice(query_size) do |query| - vim_trips += 1 + _log.debug { "Starting request for [#{params.length}] item(s), #{params.inspect}" } + data, = Benchmark.realtime_block(:vim_execute_time) { @perf_vim_hist.queryPerfMulti(params) } + _log.debug { "Finished request for [#{params.length}] item(s)" } - _log.debug { "Starting request for [#{query.length}] item(s), #{query.inspect}" } - data, = Benchmark.realtime_block(:vim_execute_time) { @perf_vim_hist.queryPerfMulti(query) } - _log.debug { "Finished request for [#{query.length}] item(s)" } - - Benchmark.realtime_block(:perf_processing) do - self.class.preprocess_data(data, counter_info, counters_by_mor, counter_values_by_mor_and_ts) - end + Benchmark.realtime_block(:perf_processing) do + self.class.preprocess_data(data, counter_info, counters_by_mor, counter_values_by_mor_and_ts) end - Benchmark.current_realtime[:num_vim_trips] = vim_trips return counters_by_mor, counter_values_by_mor_and_ts end - def concurrent_requests(interval_name) - query_size = super - # even when batching is turned off, still want to send multiple realtime records - query_size = 20 if query_size < 20 && interval_name == "realtime" - query_size - end - class << self private