Improve how LMS zips are downloaded/stored/cached

Gemfile.lock

@@ -5,6 +5,7 @@ PATH
       activesupport (>= 6.1)
       httparty (~> 0.18)
       lightly (~> 0.3.3)
+      logger
       rubyzip
 
 GEM
@@ -23,6 +24,7 @@ GEM
     i18n (1.10.0)
       concurrent-ruby (~> 1.0)
     lightly (0.3.3)
+    logger (1.5.0)
     mime-types (3.4.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2022.0105)

fcc.gemspec

@@ -21,6 +21,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "httparty", "~> 0.18"
   spec.add_dependency "lightly", "~> 0.3.3"
   spec.add_dependency "rubyzip"
+  spec.add_dependency "logger"
   spec.add_development_dependency "bundler", "~> 2.1"
   spec.add_development_dependency "rake", "~> 12.3.3"
   spec.add_development_dependency 'rspec', '~> 3.9.0'

lib/fcc.rb

@@ -6,8 +6,6 @@
 require_relative './fcc/station/record_delegate'
 
 module FCC
-  TMP_DIR = File.expand_path(File.join(File.dirname(__FILE__), '../tmp'))
-
   FM_FULL_SERVICE = 'FM'
   FM_LOW_POWER = 'FL'
   FM_BOOSTER = 'FB'
@@ -20,4 +18,14 @@ def self.cache
   def self.cache=(cache_service)
     @cache = cache_service
   end
+
+  def self.log(message)
+    @logger ||= Logger.new($stdout)
+    @logger.info(message)
+  end
+
+  def self.error(message)
+    @error_logger ||= Logger.new($stderr)
+    @error_logger.error(message)
+  end
 end

lib/fcc/station/cache.rb

@@ -1,20 +1,33 @@
 require 'httparty'
 require_relative './parsers/extended_info'
 require 'lightly'
+require 'logger'
 
 module FCC
   module Station
     class Cache
       attr_reader :store
 
       def initialize
-        @lightly = Lightly.new dir: FCC::TMP_DIR, life: '3d', hash: true
+        @lightly = Lightly.new(life: '3d', hash: true).tap do |cache|
+          cache.prune
+        end
+      end
+
+      def flush
+        @lightly.flush
       end
 
-      def fetch key
-        @lightly.get key.to_s do
-          puts "Loading up the cache with results from query: #{key}. This might take a while, but after that you're good for a while"
-          yield
+      def fetch(key)
+        FCC.log("Retreiving #{key} from cache")
+        @lightly.get(key.to_s) do
+          FCC.log "Loading up the cache with results for key: #{key}. This might take a minute…"
+          value = yield
+          if value && value.present?
+            value
+          else
+            nil
+          end
         end
       end
     end

lib/fcc/station/extended_info.rb

@@ -43,16 +43,17 @@ def initialize(service)
       end
 
       def all_results
+
         begin
           cache_key = "#{self.class.instance_variable_get('@default_options')[:base_uri]}/#{@service.to_s.downcase}q"
           FCC.cache.fetch cache_key do
             response = self.class.get("/#{service.to_s.downcase}q", @options.merge(query: @query))
-            puts response.request.uri.to_s.gsub('&list=4', '&list=0')
+            FCC.log(response.request.uri.to_s.gsub('&list=4', '&list=0'))
             response.parsed_response
           end
         rescue StandardError => e
-          puts e.message
-          puts e.backtrace
+          FCC.error(e.message)
+          FCC.error(e.backtrace)
         end
       end
 
@@ -68,8 +69,9 @@ def find(id_or_call_sign)
       end
 
       def find_directly(options)
+
         response = self.class.get("/#{service.to_s.downcase}q", @options.merge(query: @query.merge(options)))
-        puts response.request.uri.to_s.gsub('&list=4', '&list=0')
+        FCC.log response.request.uri.to_s.gsub('&list=4', '&list=0')
         response.parsed_response
       end
 

lib/fcc/station/lms_data.rb

@@ -57,11 +57,11 @@ def find_related_stations(call_sign: )
       end
 
       def facilities
-        @facilities ||= CSV.parse(File.read(lms_file(:facility)), col_sep: '|', headers: true)
+        @facilities ||= read(:facility)
       end
 
       def common_station_data
-        @common_station_data ||= CSV.parse(File.read(lms_file(:common_station)), col_sep: '|', headers: true)
+        @common_station_data ||= read(:common_station)
       end
 
       def find_facilities(facility_ids:, call_signs: [])
@@ -84,64 +84,51 @@ def find_call_signs(call_signs:)
         end
       end
 
-      protected
-
-      def call_signs_match?(ours, theirs)
-        theirs.to_s.upcase.to_s == ours.to_s.upcase.to_s || theirs.to_s.upcase =~ Regexp.new("^#{ours.to_s.upcase}[-—–][A-Z0-9]+$")
-      end
-
-      def lms_file(file_name)
+      def read(file_name)
+        key = "#{lms_date}-#{file_name}"
         remote_url = URI("#{BASE_URI}/#{lms_date}/#{file_name}.zip")
-        # FCC.cache.fetch "#{lms_date}-#{file_name}-cache" do
-
-        base_file_name = File.join(FCC::TMP_DIR, "#{lms_date}-#{file_name}")
-        zip_file = "#{base_file_name}.zip"
-        dat_file = "#{base_file_name}.dat"
-
-        unless File.exist?(zip_file)
-          response = nil
-          http_download_uri(remote_url, zip_file)
-        end
-
-        unless File.exist?(dat_file)
-          paths = []
-          Zip::File.open(zip_file) do |zip_file|
-            zip_file.each do |f|
-              FileUtils.mkdir_p(File.dirname(dat_file))
-              zip_file.extract(f, dat_file)
-
+        FCC.log remote_url
+        contents = FCC.cache.fetch key do
+          begin
+            temp_file = http_download_uri(remote_url)
+            break if temp_file.empty?
+
+            contents = ""
+            Zip::File.open_buffer(temp_file) do |zf| 
+              contents = zf.read(zf.entries.first)
               break
             end
+
+            value = contents
+          rescue Exception => e
+            FCC.error(e.message)
+            value = nil
+          ensure
+            value
           end
         end
 
-        dat_file
-        # end
+        if contents
+          CSV.parse(contents, col_sep: '|', headers: true)
+        end
       end
 
-      def http_download_uri(uri, filename)
-        puts 'Downloading ' + uri.to_s
-        http_object = Net::HTTP.new(uri.host, uri.port)
-        http_object.use_ssl = true if uri.scheme == 'https'
+      protected
+
+      def call_signs_match?(ours, theirs)
+        theirs.to_s.upcase.to_s == ours.to_s.upcase.to_s || theirs.to_s.upcase =~ Regexp.new("^#{ours.to_s.upcase}[-—–][A-Z0-9]+$")
+      end
+
+      def http_download_uri(uri)
+        FCC.log 'Downloading ' + uri.to_s
         begin
-          http_object.start do |http|
-            request = Net::HTTP::Get.new uri.request_uri
-            http.read_timeout = 500
-            http.request request do |response|
-              open(filename, 'w') do |io|
-                response.read_body do |chunk|
-                  io.write chunk
-                end
-              end
-            end
-          end
+          Tempfile.create { HTTParty.get(uri)&.body }
         rescue Exception => e
-          puts "=> Exception: '#{e}'. Skipping download."
-          return
-        end
-        puts 'Stored download as ' + filename + '.'
+          FCC.error "=> Exception: '#{e}'. Skipping download."
 
-        filename
+          raise e
+          return false
+        end
       end
 
       def lms_date

lib/fcc/station/result.rb

@@ -100,24 +100,24 @@ def related
       end
 
       def print_broadcast_summary
-        puts "[primary]"
+        FCC.log "[primary]"
         transition_records.each do |record|
-          puts "[#{record.band}] #{record.frequency} #{record.call_sign} — #{record.community.city} #{record.community.state}"
+          FCC.log "[#{record.band}] #{record.frequency} #{record.call_sign} — #{record.community.city} #{record.community.state}"
         end
 
-        puts "[translators]"
+        FCC.log "[translators]"
         related_translators.each do |record|
-          puts "[#{record.band}] #{record.frequency} #{record.call_sign} — #{record.community.city} #{record.community.state}"
+          FCC.log "[#{record.band}] #{record.frequency} #{record.call_sign} — #{record.community.city} #{record.community.state}"
         end
 
-        puts "[related stations]"
+        FCC.log "[related stations]"
         related_stations.each do |record|
-          puts "[#{record.band}] #{record.frequency} #{record.call_sign} — #{record.community.city} #{record.community.state}"
+          FCC.log "[#{record.band}] #{record.frequency} #{record.call_sign} — #{record.community.city} #{record.community.state}"
         end
 
-        puts "[all related]"
+        FCC.log "[all related]"
         related.each do |record|
-          puts "[#{record.band}] #{record.frequency} #{record.call_sign} — #{record.community.city} #{record.community.state}"
+          FCC.log "[#{record.band}] #{record.frequency} #{record.call_sign} — #{record.community.city} #{record.community.state}"
         end
 
         nil