logstash.conf

input {
  tcp {
    tags => ["input_heroku_syslog"]
    port => 5228
    codec => json_lines
  }
}

filter {
  if ("" in [search_submission]) {
    mutate {
      add_field => { "search_dashified" => "%{search_submission}" }
    }
  }
  mutate {
    gsub => [
      "search_dashified", " ", "-"
    ]
  }
  kv {
    field_split => "&?"
    source => "path"
  }
}

output {
 if [user_agent] in ["(compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)","AdsBot-Google (+http://www.google.com/adsbot.html)","AdsBot-Google-Mobile-Apps","APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)","Googlebot-Image/1.0","Googlebot-News","Googlebot-Video/1.0","Googlebot/2.1 (+http://www.google.com/bot.html)","Mediapartners-Google","Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)","Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1 (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)","Mozilla/5.0 (Linux; Android 5.0; SM-G920A) AppleWebKit (KHTML, like Gecko) Chrome Mobile Safari (compatible; AdsBot-Google-Mobile; +http://www.google.com/mobile/adsbot.html)","Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"] {
    elasticsearch {
      hosts => ["${ES_URL}"]
      codec => json_lines
      index => "test_crawlers_2018-08-15"
    }
  }
  else {
    elasticsearch {
      hosts => ["${ES_URL}"]
      codec => json_lines
      index => "test_requests_2018-08-15"
    }
  }
}