feat: support robots

Cargo.toml

@@ -41,3 +41,4 @@ log = "0.4.20"
 toml = "0.8.8"
 rand = "0.8.5"
 once_cell = "1.19.0"
+reqwest = { version = "0.11.23", features = ["blocking", "json"] }

README.md

@@ -4,7 +4,7 @@ Daemon is a browser crawler that does URL harvesting in chrome headless mode
 
 ## Roadmap
 
-- [ ] Robots
+- [x] Robots
 - [ ] Javascript Content
 - [x] Custom Headers
 - [x] Proxy

src/cli/cmd.rs

@@ -52,7 +52,6 @@ pub fn cli() -> Result<(), Box<dyn std::error::Error>> {
             let proxy = Some(c.proxy.as_deref().unwrap_or_default());
             let launch_options = LaunchOptions::default_builder()
                 .path(chromium_path)
-                .proxy_server(None)
                 .headless(c.headless)
                 .sandbox(c.sandbox)
                 .proxy_server(proxy)

src/handler/crawler.rs

@@ -20,13 +20,12 @@ pub fn browse_wikipedia(
                 .collect(),
         )
         .unwrap();
-        let h1 = tab.wait_for_xpath("/html/body/div/h1")?;
-        assert_eq!(h1.get_inner_text().unwrap().as_str(), "Example Domain");
         let ug = tab
             .evaluate("window.navigator.userAgent", false)?
             .value
             .unwrap();
         assert_eq!(random_ug, ug);
+        _ = tab.close(true);
     }
 
     Ok(())

src/handler/mod.rs

@@ -1 +1,2 @@
 pub mod crawler;
+pub mod robots;

src/handler/robots.rs

@@ -0,0 +1,35 @@
+use std::collections::HashSet;
+use std::time;
+
+use reqwest::header::{HeaderMap, USER_AGENT};
+
+use crate::common;
+
+pub fn parse_robots(site: String) -> Result<HashSet<String>, Box<dyn std::error::Error>> {
+    let site = site + "/robots.txt";
+
+    let mut headers = HeaderMap::new();
+    headers.insert(
+        USER_AGENT,
+        common::user_agent::random_user_agent().parse().unwrap(),
+    );
+
+    let rsp = reqwest::blocking::Client::new()
+        .get(site)
+        .timeout(time::Duration::from_secs(5))
+        .headers(headers)
+        .send();
+    assert_eq!(rsp.as_ref().unwrap().status(), 200);
+    let txt = rsp?.text()?;
+    let allow_values: HashSet<String> = txt
+        .lines()
+        .flat_map(|line| {
+            let parts: Vec<&str> = line.split_whitespace().collect();
+            match parts.len() {
+                2 if parts[0] == "Allow:" || parts[0] == "Disallow:" => Some(parts[1].to_string()),
+                _ => None,
+            }
+        })
+        .collect();
+    Ok(allow_values)
+}

tests/integration_test.rs

@@ -1,5 +1,23 @@
 #[cfg(test)]
 mod tests {
+    use demon::handler;
+    use handler::robots::parse_robots;
+
+    fn load() {
+        let buf = std::env::current_dir()
+            .unwrap()
+            .join("files/user_agent.toml");
+        std::env::set_var("user_agent", buf);
+    }
+
     #[test]
-    fn cli_test() {}
+    fn parse_robots_test() {
+        load();
+        assert_ne!(
+            parse_robots("https://www.dvwa.co.uk".to_string())
+                .unwrap()
+                .len(),
+            0
+        )
+    }
 }