-
Notifications
You must be signed in to change notification settings - Fork 2
/
http.rs
116 lines (100 loc) · 3.07 KB
/
http.rs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
use reqwest::blocking::{Client, Response};
use reqwest::header::HeaderMap;
use reqwest::header::{
ACCEPT, ACCEPT_ENCODING, CONNECTION, CONTENT_ENCODING, LOCATION, USER_AGENT,
};
use serde::de::DeserializeOwned;
use std::fs::File;
use std::path::Path;
use std::time::Duration;
use crate::lib::{Error, Result};
use crate::C_USER_AGENT;
/// Download into file
pub fn http_download<P: AsRef<Path>>(url: &str, target: P) -> Result<()> {
let mut response = get_raw(url, HeaderType::Html)?;
let mut file = File::create(target)?;
response.copy_to(&mut file)?;
Ok(())
}
/// Header type for get requests
pub enum HeaderType {
/// Html browser request
Html,
/// Ajax js request
Ajax,
}
/// Does a raw get request under the provided url & header
fn get_raw(url: &str, htype: HeaderType) -> Result<Response> {
trace!("Starting request {}", url);
let client = Client::builder()
.gzip(true)
.timeout(Duration::from_secs(10))
.build()?;
let builder = client.get(url);
let res = builder.headers(header(htype)).send()?;
debug!("Response header: {:?}", res.headers());
debug!("Response status: {:?}", res.status());
debug!("Final URL: {:?}", res.headers().get(LOCATION));
trace!("DEV header: {:?}", res.headers().get(CONTENT_ENCODING));
Ok(res)
}
/// Do an http(s) get request, returning text
pub fn http_text_get(url: &str) -> Result<String> {
trace!("Starting request {}", url);
let response = get_raw(url, HeaderType::Ajax)?;
Ok(response.text()?)
}
/// Do an http(s) get request, returning JSON
pub fn http_json_get<T>(url: &str) -> Result<T>
where
T: DeserializeOwned,
{
trace!("Starting request {}", url);
let response = get_raw(url, HeaderType::Ajax)?;
let body = response.text()?;
match serde_json::from_str(&body) {
Err(e) => Err(Error::InternalError(format!(
"Parsing error {} for {}",
e, body
))),
Ok(v) => Ok(v),
}
}
/// Construct a header
/// This function does not check for errors and is
/// verified by the tests
fn header(htype: HeaderType) -> HeaderMap {
let mut headers = HeaderMap::new();
headers.insert(ACCEPT_ENCODING, "gzip, deflate, br".parse().unwrap());
match htype {
HeaderType::Html => {
headers.insert(
ACCEPT,
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
.parse()
.unwrap(),
);
}
HeaderType::Ajax => {
headers.insert(
ACCEPT,
"application/json, text/javascript, */*; q=0.01"
.parse()
.unwrap(),
);
}
}
headers.insert(CONNECTION, "close".parse().unwrap());
headers.insert(USER_AGENT, C_USER_AGENT.parse().unwrap());
trace!("Generated headers: {:?}", headers);
headers
}
#[cfg(test)]
mod test {
use serde_json::Value;
use super::*;
#[test]
fn get_ajax() {
assert!(http_json_get::<Value>("https://httpbin.org/user-agent").is_ok());
}
}