From b8c670294288b56b7f0aa88bed5020c0dbf7329d Mon Sep 17 00:00:00 2001 From: Jeff Keen Date: Mon, 11 Jul 2022 07:39:38 -0700 Subject: [PATCH] Improve how LMS zips are downloaded/stored/cached --- Gemfile.lock | 2 + fcc.gemspec | 1 + lib/fcc.rb | 12 ++++- lib/fcc/station/cache.rb | 23 +++++++-- lib/fcc/station/extended_info.rb | 10 ++-- lib/fcc/station/lms_data.rb | 85 ++++++++++++++------------------ lib/fcc/station/result.rb | 16 +++--- 7 files changed, 81 insertions(+), 68 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index ddedf0e..dba9ea3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -5,6 +5,7 @@ PATH activesupport (>= 6.1) httparty (~> 0.18) lightly (~> 0.3.3) + logger rubyzip GEM @@ -23,6 +24,7 @@ GEM i18n (1.10.0) concurrent-ruby (~> 1.0) lightly (0.3.3) + logger (1.5.0) mime-types (3.4.1) mime-types-data (~> 3.2015) mime-types-data (3.2022.0105) diff --git a/fcc.gemspec b/fcc.gemspec index d97b9de..a8068d9 100644 --- a/fcc.gemspec +++ b/fcc.gemspec @@ -21,6 +21,7 @@ Gem::Specification.new do |spec| spec.add_dependency "httparty", "~> 0.18" spec.add_dependency "lightly", "~> 0.3.3" spec.add_dependency "rubyzip" + spec.add_dependency "logger" spec.add_development_dependency "bundler", "~> 2.1" spec.add_development_dependency "rake", "~> 12.3.3" spec.add_development_dependency 'rspec', '~> 3.9.0' diff --git a/lib/fcc.rb b/lib/fcc.rb index 2759513..e567e8e 100644 --- a/lib/fcc.rb +++ b/lib/fcc.rb @@ -6,8 +6,6 @@ require_relative './fcc/station/record_delegate' module FCC - TMP_DIR = File.expand_path(File.join(File.dirname(__FILE__), '../tmp')) - FM_FULL_SERVICE = 'FM' FM_LOW_POWER = 'FL' FM_BOOSTER = 'FB' @@ -20,4 +18,14 @@ def self.cache def self.cache=(cache_service) @cache = cache_service end + + def self.log(message) + @logger ||= Logger.new($stdout) + @logger.info(message) + end + + def self.error(message) + @error_logger ||= Logger.new($stderr) + @error_logger.error(message) + end end \ No newline at end of file diff --git a/lib/fcc/station/cache.rb b/lib/fcc/station/cache.rb index aeaaec9..5772d31 100644 --- a/lib/fcc/station/cache.rb +++ b/lib/fcc/station/cache.rb @@ -1,6 +1,7 @@ require 'httparty' require_relative './parsers/extended_info' require 'lightly' +require 'logger' module FCC module Station @@ -8,13 +9,25 @@ class Cache attr_reader :store def initialize - @lightly = Lightly.new dir: FCC::TMP_DIR, life: '3d', hash: true + @lightly = Lightly.new(life: '3d', hash: true).tap do |cache| + cache.prune + end + end + + def flush + @lightly.flush end - def fetch key - @lightly.get key.to_s do - puts "Loading up the cache with results from query: #{key}. This might take a while, but after that you're good for a while" - yield + def fetch(key) + FCC.log("Retreiving #{key} from cache") + @lightly.get(key.to_s) do + FCC.log "Loading up the cache with results for key: #{key}. This might take a minute…" + value = yield + if value && value.present? + value + else + nil + end end end end diff --git a/lib/fcc/station/extended_info.rb b/lib/fcc/station/extended_info.rb index 3c7cc59..21fecb0 100644 --- a/lib/fcc/station/extended_info.rb +++ b/lib/fcc/station/extended_info.rb @@ -43,16 +43,17 @@ def initialize(service) end def all_results + begin cache_key = "#{self.class.instance_variable_get('@default_options')[:base_uri]}/#{@service.to_s.downcase}q" FCC.cache.fetch cache_key do response = self.class.get("/#{service.to_s.downcase}q", @options.merge(query: @query)) - puts response.request.uri.to_s.gsub('&list=4', '&list=0') + FCC.log(response.request.uri.to_s.gsub('&list=4', '&list=0')) response.parsed_response end rescue StandardError => e - puts e.message - puts e.backtrace + FCC.error(e.message) + FCC.error(e.backtrace) end end @@ -68,8 +69,9 @@ def find(id_or_call_sign) end def find_directly(options) + response = self.class.get("/#{service.to_s.downcase}q", @options.merge(query: @query.merge(options))) - puts response.request.uri.to_s.gsub('&list=4', '&list=0') + FCC.log response.request.uri.to_s.gsub('&list=4', '&list=0') response.parsed_response end diff --git a/lib/fcc/station/lms_data.rb b/lib/fcc/station/lms_data.rb index 95f9b2e..0549af5 100644 --- a/lib/fcc/station/lms_data.rb +++ b/lib/fcc/station/lms_data.rb @@ -57,11 +57,11 @@ def find_related_stations(call_sign: ) end def facilities - @facilities ||= CSV.parse(File.read(lms_file(:facility)), col_sep: '|', headers: true) + @facilities ||= read(:facility) end def common_station_data - @common_station_data ||= CSV.parse(File.read(lms_file(:common_station)), col_sep: '|', headers: true) + @common_station_data ||= read(:common_station) end def find_facilities(facility_ids:, call_signs: []) @@ -84,64 +84,51 @@ def find_call_signs(call_signs:) end end - protected - - def call_signs_match?(ours, theirs) - theirs.to_s.upcase.to_s == ours.to_s.upcase.to_s || theirs.to_s.upcase =~ Regexp.new("^#{ours.to_s.upcase}[-—–][A-Z0-9]+$") - end - - def lms_file(file_name) + def read(file_name) + key = "#{lms_date}-#{file_name}" remote_url = URI("#{BASE_URI}/#{lms_date}/#{file_name}.zip") - # FCC.cache.fetch "#{lms_date}-#{file_name}-cache" do - - base_file_name = File.join(FCC::TMP_DIR, "#{lms_date}-#{file_name}") - zip_file = "#{base_file_name}.zip" - dat_file = "#{base_file_name}.dat" - - unless File.exist?(zip_file) - response = nil - http_download_uri(remote_url, zip_file) - end - - unless File.exist?(dat_file) - paths = [] - Zip::File.open(zip_file) do |zip_file| - zip_file.each do |f| - FileUtils.mkdir_p(File.dirname(dat_file)) - zip_file.extract(f, dat_file) - + FCC.log remote_url + contents = FCC.cache.fetch key do + begin + temp_file = http_download_uri(remote_url) + break if temp_file.empty? + + contents = "" + Zip::File.open_buffer(temp_file) do |zf| + contents = zf.read(zf.entries.first) break end + + value = contents + rescue Exception => e + FCC.error(e.message) + value = nil + ensure + value end end - dat_file - # end + if contents + CSV.parse(contents, col_sep: '|', headers: true) + end end - def http_download_uri(uri, filename) - puts 'Downloading ' + uri.to_s - http_object = Net::HTTP.new(uri.host, uri.port) - http_object.use_ssl = true if uri.scheme == 'https' + protected + + def call_signs_match?(ours, theirs) + theirs.to_s.upcase.to_s == ours.to_s.upcase.to_s || theirs.to_s.upcase =~ Regexp.new("^#{ours.to_s.upcase}[-—–][A-Z0-9]+$") + end + + def http_download_uri(uri) + FCC.log 'Downloading ' + uri.to_s begin - http_object.start do |http| - request = Net::HTTP::Get.new uri.request_uri - http.read_timeout = 500 - http.request request do |response| - open(filename, 'w') do |io| - response.read_body do |chunk| - io.write chunk - end - end - end - end + Tempfile.create { HTTParty.get(uri)&.body } rescue Exception => e - puts "=> Exception: '#{e}'. Skipping download." - return - end - puts 'Stored download as ' + filename + '.' + FCC.error "=> Exception: '#{e}'. Skipping download." - filename + raise e + return false + end end def lms_date diff --git a/lib/fcc/station/result.rb b/lib/fcc/station/result.rb index 1003a42..00576ad 100644 --- a/lib/fcc/station/result.rb +++ b/lib/fcc/station/result.rb @@ -100,24 +100,24 @@ def related end def print_broadcast_summary - puts "[primary]" + FCC.log "[primary]" transition_records.each do |record| - puts "[#{record.band}] #{record.frequency} #{record.call_sign} — #{record.community.city} #{record.community.state}" + FCC.log "[#{record.band}] #{record.frequency} #{record.call_sign} — #{record.community.city} #{record.community.state}" end - puts "[translators]" + FCC.log "[translators]" related_translators.each do |record| - puts "[#{record.band}] #{record.frequency} #{record.call_sign} — #{record.community.city} #{record.community.state}" + FCC.log "[#{record.band}] #{record.frequency} #{record.call_sign} — #{record.community.city} #{record.community.state}" end - puts "[related stations]" + FCC.log "[related stations]" related_stations.each do |record| - puts "[#{record.band}] #{record.frequency} #{record.call_sign} — #{record.community.city} #{record.community.state}" + FCC.log "[#{record.band}] #{record.frequency} #{record.call_sign} — #{record.community.city} #{record.community.state}" end - puts "[all related]" + FCC.log "[all related]" related.each do |record| - puts "[#{record.band}] #{record.frequency} #{record.call_sign} — #{record.community.city} #{record.community.state}" + FCC.log "[#{record.band}] #{record.frequency} #{record.call_sign} — #{record.community.city} #{record.community.state}" end nil