Merge pull request #426 from themarshallproject/develop

v0.5.0

.circleci/config.yml

@@ -7,7 +7,7 @@ jobs:
     # Includes ruby, node, and chrome/phantomjs
     # TODO: Go back to the non-legacy version of this image, when our test
     # infrastructure no longer depends on phantomjs
-    - image: circleci/ruby:2.5.7 # ...with this image as the primary container; this is where all `steps` will run
+    - image: cimg/ruby:2.7.2 # ...with this image as the primary container; this is where all `steps` will run
       environment: # environment variables for primary container
         BUNDLE_JOBS: 3
         BUNDLE_RETRY: 3

.ruby-version

@@ -1 +1 @@
-2.5.7
+2.7.2

Dockerfile

@@ -1,4 +1,4 @@
-FROM ruby:2.5.7
+FROM ruby:2.7.2
 
 # throw errors if Gemfile has been modified since Gemfile.lock
 RUN bundle config --global frozen 1

Gemfile

@@ -1,11 +1,11 @@
 source 'https://rubygems.org'
-ruby '2.5.7'
+ruby '2.7.2'
 
 gem 'rails', '~> 6.0'
 gem 'pg', '~> 0.21'
 gem 'sass-rails', '~> 5.0'
 gem 'uglifier', '>= 1.3.0'
-gem 'therubyracer'
+gem 'mini_racer', '~> 0.3'
 
 gem 'dotenv'
 gem 'jquery-rails'
@@ -28,7 +28,7 @@ gem 'httparty'
 gem 'diffy'
 gem 'kramdown'
 
-gem 'aws-sdk-sqs', '~> 1.30'
+gem 'aws-sdk-sqs', '~> 1.35'
 
 group :development, :test do
   gem 'byebug'

Gemfile.lock

@@ -59,20 +59,20 @@ GEM
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
     aws-eventstream (1.1.0)
-    aws-partitions (1.368.0)
-    aws-sdk-core (3.105.0)
+    aws-partitions (1.409.0)
+    aws-sdk-core (3.110.0)
       aws-eventstream (~> 1, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-sqs (1.32.0)
-      aws-sdk-core (~> 3, >= 3.99.0)
+    aws-sdk-sqs (1.35.0)
+      aws-sdk-core (~> 3, >= 3.109.0)
       aws-sigv4 (~> 1.1)
     aws-sigv4 (1.2.2)
       aws-eventstream (~> 1, >= 1.0.2)
     bcrypt (3.1.16)
     bindex (0.8.1)
-    bootsnap (1.4.8)
+    bootsnap (1.5.1)
       msgpack (~> 1.0)
     builder (3.2.4)
     byebug (11.1.3)
@@ -87,8 +87,8 @@ GEM
       execjs
     coffee-script-source (1.12.2)
     concurrent-ruby (1.1.7)
-    crack (0.4.3)
-      safe_yaml (~> 1.0.0)
+    crack (0.4.5)
+      rexml
     crass (1.0.6)
     css_parser (1.7.1)
       addressable
@@ -97,7 +97,7 @@ GEM
     diffy (3.4.0)
     docile (1.1.5)
     dotenv (2.7.6)
-    erubi (1.9.0)
+    erubi (1.10.0)
     execjs (2.7.0)
     factory_bot (6.1.0)
       activesupport (>= 5.0.0)
@@ -127,7 +127,7 @@ GEM
     httparty (0.18.1)
       mime-types (~> 3.0)
       multi_xml (>= 0.5.2)
-    i18n (1.8.5)
+    i18n (1.8.7)
       concurrent-ruby (~> 1.0)
     jmespath (1.4.0)
     jquery-rails (4.4.0)
@@ -142,11 +142,11 @@ GEM
       addressable (~> 2.7)
     letter_opener (1.7.0)
       launchy (~> 2.2)
-    libv8 (3.16.14.19)
+    libv8 (8.4.255.0)
     listen (3.2.1)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
-    loofah (2.7.0)
+    loofah (2.9.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     lumberjack (1.2.8)
@@ -160,16 +160,19 @@ GEM
     mime-types-data (3.2020.0512)
     mimemagic (0.3.5)
     mini_mime (1.0.2)
-    mini_portile2 (2.4.0)
-    minitest (5.14.2)
+    mini_portile2 (2.5.0)
+    mini_racer (0.3.1)
+      libv8 (~> 8.4.255)
+    minitest (5.14.3)
     msgpack (1.3.3)
     multi_xml (0.6.0)
     mustermann (1.1.1)
       ruby2_keywords (~> 0.0.1)
     nenv (0.3.0)
     nio4r (2.5.3)
-    nokogiri (1.10.10)
-      mini_portile2 (~> 2.4.0)
+    nokogiri (1.11.1)
+      mini_portile2 (~> 2.5.0)
+      racc (~> 1.4)
     notiffany (0.1.3)
       nenv (~> 0.1)
       shellany (~> 0.0)
@@ -187,8 +190,9 @@ GEM
     public_suffix (4.0.6)
     puma (4.3.6)
       nio4r (~> 2.0)
+    racc (1.5.2)
     rack (2.2.3)
-    rack-cache (1.12.0)
+    rack-cache (1.12.1)
       rack (>= 0.4)
     rack-protection (2.1.0)
       rack
@@ -234,7 +238,6 @@ GEM
     rb-inotify (0.10.1)
       ffi (~> 1.0)
     rdoc (6.2.1)
-    ref (2.0.0)
     rexml (3.2.4)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
@@ -260,7 +263,6 @@ GEM
     rspec_junit_formatter (0.4.1)
       rspec-core (>= 2, < 4, != 2.12.0)
     ruby2_keywords (0.0.2)
-    safe_yaml (1.0.5)
     sass (3.7.4)
       sass-listen (~> 4.0.0)
     sass-listen (4.0.0)
@@ -275,7 +277,7 @@ GEM
     sdoc (1.0.0)
       rdoc (>= 5.0)
     shellany (0.0.1)
-    simple_form (5.0.2)
+    simple_form (5.0.3)
       actionpack (>= 5.0)
       activemodel (>= 5.0)
     simplecov (0.13.0)
@@ -296,16 +298,13 @@ GEM
       actionpack (>= 4.0)
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
-    therubyracer (0.12.3)
-      libv8 (~> 3.16.14.15)
-      ref
     thor (1.0.1)
     thread_safe (0.3.6)
     tilt (2.0.10)
     turbolinks (5.2.1)
       turbolinks-source (~> 5.2)
     turbolinks-source (5.2.0)
-    tzinfo (1.2.7)
+    tzinfo (1.2.9)
       thread_safe (~> 0.1)
     uglifier (4.2.0)
       execjs (>= 0.3.0, < 3)
@@ -314,20 +313,20 @@ GEM
       activemodel (>= 5.0)
       bindex (>= 0.4.0)
       railties (>= 5.0)
-    webmock (3.8.3)
+    webmock (3.11.1)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
     websocket-driver (0.7.3)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
-    zeitwerk (2.4.0)
+    zeitwerk (2.4.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  aws-sdk-sqs (~> 1.30)
+  aws-sdk-sqs (~> 1.35)
   bcrypt (~> 3.1.12)
   bootsnap
   byebug
@@ -343,6 +342,7 @@ DEPENDENCIES
   jwt
   kramdown
   letter_opener
+  mini_racer (~> 0.3)
   pg (~> 0.21)
   premailer-rails
   puma (~> 4.3)
@@ -357,14 +357,13 @@ DEPENDENCIES
   simple_form (~> 5.0)
   sinatra
   spring
-  therubyracer
   turbolinks
   uglifier (>= 1.3.0)
   web-console (~> 3.7)
   webmock
 
 RUBY VERSION
-   ruby 2.5.7p206
+   ruby 2.7.2p137
 
 BUNDLED WITH
-   1.17.3
+   2.2.5

README.md

@@ -70,6 +70,22 @@ Click on the button that says “Manage App”. This takes you behind the scenes
 
 Unfortunately our email provider Sendgrid now requires an additional step to confirm that you are not a spammer. Your new Sendgrid account is now in a "suspended" state, and to get it unsuspended you have to contact [Sendgrid support](https://support.sendgrid.com/hc/en-us/requests/new#login-issue). You can do this by clicking the Sendgrid logo on the Resources tab. If clicking on the logo takes you to an error page, do not worry. This has been known to happen as Sendgrid's system has undergone redesigns. Instead, go to [Sendgrid's page to ask for support](https://support.sendgrid.com/hc/en-us/requests/new#login-issue). Be sure to use the same email address associated with your Heroku account and provide the url of your Klaxon instance. When they ask for "Business impact," choose "P3 General - You have a question about Sendgrid or how to use its products". This step is a nuisance, but important. **You will not be able to get an email log in to Klaxon until you are cleared by Sendgrid.** This usually happens pretty quickly (hours not days).
 
+Unfortunately you are not yet done configuring Sendgrid. There are more steps to set up your account.
+
+1. From the Heroku application page, click on the "Resources" tab, and click the link to the Sendgrid plugin, this will take you to the Sendgrid website.
+2. In the left hand column choose Settings -> API Keys. Click the blue "Create an API Key" button in the top right corner.
+3. In the form that appears fill in the API key name (doesn't matter what you name it), make sure the "Full Access" option is selected, and click "Create & View".
+4. Click the API key to copy it to your clipboard. Then navigate back to Heroku.
+5. In the "Settings" tab of your Heroku app click the "Reveal Config Vars" button.
+6. Change the `SENDGRID_PASSWORD` variable to the API Key by clicking the pencil icon next to it, pasting it in, and saving it.
+7. Change the `SENDGRID_USERNAME` variable to the string "apikey" in the same manner.
+
+Now you'll need to set up a "Verified Sender" account in Sendgrid using an email address that you have access to. See https://github.com/themarshallproject/klaxon/issues/404 for some more context.
+
+When you've completed this process in Sendgrid, you'll need to set the `MAILER_FROM_ADDRESS` variable as you did above to your verified sender email address.
+
+Finally, now, you should be done setting up your Sendgrid account.
+
 At the top of the scheduler page, click the link that is the name of your app (“sl-klaxon”). This will take you to back to Klaxon's dashboard. Then click the button in the upper right that says "Open app," and this should take you to your Klaxon's login screen on the web.
 
 ![](docs/login_screen.png)

SENDGRID.md

@@ -1,11 +1,11 @@
-Last week Sendgrid identified that some accounts had their credentials posted online. They accordingly reset the passwords of those accounts. If your account (like ours) was affected you would have received an email about it. Unfortunately this means that the connection between your Klaxon instance and Sendgrid has been broken. If your emails are not sending properly (which you can test simply by trying to log in to Klaxon), you may need to follow these steps.
+Sendgrid requires that you authenticate using an API key, instead of the user name and password that they automatically provide through the plugin. Unfortunately this means that some manual steps are required to properly configure your Klaxon instance. You will create an API key for Sendgrid, and set it as the Sendgrid password.
 
-1. From the Heroku application page, click on the "Resources" tab, and click the link to the Sendgrid plugin, this will take you to the Sendgrid website.
+1. From the Heroku application page, click on the "Resources" tab, and click the link to the Sendgrid plugin, this will take you to the Sendgrid website, which will ask you to update your email. Be sure to put your real email address you use for Klaxon.
 2. In the left hand column choose Settings -> API Keys. Click the blue "Create an API Key" button in the top right corner.
 3. In the form that appears fill in the API key name (doesn't matter what you name it), make sure the "Full Access" option is selected, and click "Create & View".
 4. Click the API key to copy it to your clipboard. Then navigate back to Heroku.
 5. In the "Settings" tab of your Heroku app click the "Reveal Config Vars" button.
 6. Change the `SENDGRID_PASSWORD` variable to the API Key by clicking the pencil icon next to it, pasting it in, and saving it.
 7. Change the `SENDGRID_USERNAME` variable to the string "apikey" in the same manner.
 
-You should be all set! Confirm that your changes worked as expected by trying to log in to Klaxon again, and ensuring that the login email arrives as expected.
+You should be all set! Confirm that your changes worked as expected by trying to log in to Klaxon, and ensuring that the login email arrives as expected.

app.json

@@ -5,6 +5,7 @@
   "website": "https://github.com/themarshallproject/klaxon",
   "repository": "https://github.com/themarshallproject/klaxon",
   "success_url": "/",
+  "stack": "heroku-20",
   "scripts": {
     "postdeploy": "bash ./postdeploy.sh"
   },

app/lib/safe_string.rb

@@ -2,6 +2,6 @@ class SafeString
   # this should always produce a valid UTF-8 string
   # diffing/etc will fail if it's nil, non-utf8, etc
   def self.coerce(dirty)
-    dirty.to_s.force_encoding("UTF-8").encode("UTF-8", invalid: :replace, replace: "")
+    (+dirty.to_s).force_encoding("UTF-8").encode("UTF-8", invalid: :replace, replace: "")
   end
 end

app/models/page.rb

@@ -37,11 +37,11 @@ def document
   end
 
   def match_text
-    @match = document.css(self.css_selector)
+    @match = document.css(self.css_selector.strip)
 
     if self.exclude_selector.present?
       # Set the content of the exclude selector to the empty string
-      @match.css(self.exclude_selector).each do |node|
+      @match.css(self.exclude_selector.strip).each do |node|
         node.content = ""
       end
     end
@@ -50,7 +50,7 @@ def match_text
   end
 
   def match_html
-    document.css(self.css_selector).to_html
+    document.css(self.css_selector.strip).to_html
   end
 
   def sha2_hash
@@ -59,6 +59,8 @@ def sha2_hash
 
   def sanitize
     self.url = url.strip
+    self.css_selector = css_selector.strip unless self.css_selector.nil?
+    self.exclude_selector = exclude_selector.strip unless self.exclude_selector.nil?
   end
 
   def update_subscriptions

app/models/page_snapshot.rb

@@ -12,11 +12,11 @@ def document
   end
 
   def match_text
-    @match = document.css(self.page.css_selector)
+    @match = document.css(self.page.css_selector.strip)
 
     if self.page.exclude_selector.present?
       # Set the content of the exclude selector to the empty string
-      @match.css(self.page.exclude_selector).each do |node|
+      @match.css(self.page.exclude_selector.strip).each do |node|
         node.content = ""
       end
     end

app/models/poll_page.rb

@@ -21,9 +21,9 @@ def self.perform_all
     Page.all.shuffle.each do |page|
       begin
         self.perform(page: page)
-      rescue
+      rescue StandardError => e
         # TODO: send notifications about failed updates?
-        puts "Failed to update page #{page.id}"
+        puts "Failed to update page #{page.id}", e
       end
     end
   end

config/environments/development.rb

@@ -45,4 +45,6 @@
 
   # Set host to localhost specifically to override AppSetting
   Rails.application.routes.default_url_options = { host: ENV['HOST'] }
+
+  config.hosts << "klaxon.test"
 end

rootfs/etc/cont-init.d/10-config.sh

@@ -2,7 +2,8 @@
 
 # run klaxon startup commands
 cd /usr/src/app
-bundle exec rake db:create db:migrate || true
+bundle exec rake db:create || true
+bundle exec rake db:migrate || true
 bundle exec rake users:create_admin || true
 
 
@@ -30,4 +31,4 @@ if [[ ! -f /config/klaxon-crontab ]]; then
 echo "*/15 * * * * root /config/klaxon-cron > /proc/1/fd/1 2>&1" > /etc/cron.d/klaxon-crontab
 ln -s /config/klaxon-crontab /etc/cron.d/klaxon-crontab
 
-fi
+fi

spec/models/page_spec.rb

@@ -70,6 +70,11 @@
     expect(page.url).to eq url
   end
 
+  it "strips whitespace from the selectors" do
+    selector = ".test"
+    page = create(:page, css_selector: " #{selector}    ", exclude_selector: " #{selector} ")
+  end
+
   it "gracefully handles parsing an invalid uri" do
     weird_url = " bad:///site.com"
     page = build(:page, url: weird_url) # our sanitizer is on before_save, which does not run here