diff --git a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/selenium/RemoteDriverProtocol.java b/core/src/main/java/com/digitalpebble/stormcrawler/protocol/selenium/RemoteDriverProtocol.java index 25bf0880e..4eab4d44f 100644 --- a/core/src/main/java/com/digitalpebble/stormcrawler/protocol/selenium/RemoteDriverProtocol.java +++ b/core/src/main/java/com/digitalpebble/stormcrawler/protocol/selenium/RemoteDriverProtocol.java @@ -18,6 +18,7 @@ import com.digitalpebble.stormcrawler.util.ConfUtils; import java.net.URL; import java.time.Duration; +import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -33,30 +34,48 @@ */ public class RemoteDriverProtocol extends SeleniumProtocol { + private void substituteUserAgent(Map keyvals, final String userAgentString) { + if (keyvals == null) return; + + Iterator> iter = keyvals.entrySet().iterator(); + while (iter.hasNext()) { + Entry entry = iter.next(); + Object val = entry.getValue(); + // substitute variable $useragent for the real value + if (val instanceof String && val.toString().contains("$useragent")) { + String newval = ((String) val).replaceAll("\\$useragent", userAgentString); + entry.setValue(newval); + } else if (val instanceof Map) { + substituteUserAgent((Map) val, userAgentString); + } else if (val instanceof List) { + List newList = new ArrayList(); + ((List) val) + .forEach( + s -> { + String newval = s.replaceAll("\\$useragent", userAgentString); + newList.add(newval); + }); + entry.setValue(newList); + } + } + } + @Override public void configure(Config conf) { super.configure(conf); - // see https://github.com/SeleniumHQ/selenium/wiki/DesiredCapabilities - DesiredCapabilities capabilities = new DesiredCapabilities(); - - String userAgentString = getAgentString(conf); + final String userAgentString = getAgentString(conf); // custom capabilities - Map confCapabilities = + final Map confCapabilities = (Map) conf.get("selenium.capabilities"); - if (confCapabilities != null) { - Iterator> iter = confCapabilities.entrySet().iterator(); - while (iter.hasNext()) { - Entry entry = iter.next(); - Object val = entry.getValue(); - // substitute variable $useragent for the real value - if (val instanceof String && "$useragent".equalsIgnoreCase(val.toString())) { - val = userAgentString; - } - capabilities.setCapability(entry.getKey(), val); - } - } + + substituteUserAgent(confCapabilities, userAgentString); + + // see https://github.com/SeleniumHQ/selenium/wiki/DesiredCapabilities + final DesiredCapabilities capabilities = new DesiredCapabilities(); + + confCapabilities.forEach((k, v) -> capabilities.setCapability(k, v)); LOG.info("Configuring Selenium with {}", capabilities);