From b02f3f28e3e0a7d8585c8a34e7fd5bf2025ceeea Mon Sep 17 00:00:00 2001 From: Andy Wortman Date: Tue, 18 Jul 2017 21:44:05 -0700 Subject: get non-truncated tweets, non-broken tweet/user lookups --- main.rs | 131 +++++++++++++++++++++++++++++++++++++--------------------------- 1 file changed, 76 insertions(+), 55 deletions(-) (limited to 'main.rs') diff --git a/main.rs b/main.rs index 7b16588..2391d40 100644 --- a/main.rs +++ b/main.rs @@ -28,7 +28,7 @@ static consumer_secret: &str = "1fxEzRhQtQSWKus4oqDwdg5DALIjGpINg0PGjkYVwKT8EEMF static token: &str = "629126745-VePBD9ciKwpuVuIeEcNnxwxQFNWDXEy8KL3dGRRg"; static token_secret: &str = "uAAruZzJu03NvMlH6cTeGku7NqVPro1ddKN4BxORy5hWG"; -static streamurl: &str = "https://userstream.twitter.com/1.1/user.json"; +static streamurl: &str = "https://userstream.twitter.com/1.1/user.json?tweet_mode=extended"; static tweet_lookup_url: &str = "https://api.twitter.com/1.1/statuses/show.json"; static user_lookup_url: &str = "https://api.twitter.com/1.1/users/lookup.json"; @@ -36,17 +36,18 @@ header! { (Authorization, "Authorization") => [String] } header! { (Accept, "Accept") => [String] } header! { (ContentType, "Content-Type") => [String] } -fn render_tweet(structure: serde_json::Map) { +fn render_tweet(structure: serde_json::Map, client: &hyper::client::Client>, c2: &mut Core) { if structure.contains_key("event") { match &structure["event"].as_str().unwrap() { - &"follow" => println!("followed! by {} (@{})", structure["source"]["name"], structure["source"]["screen_name"]), - &"favorite" => println!("fav"), + &"follow" => println!("followed! by {} (@{})", structure["source"]["name"].as_str().unwrap(), structure["source"]["screen_name"].as_str().unwrap()), + &"favorite" => println!("{} fav'd", structure["source"]["screen_name"].as_str().unwrap()), + &"favorited_retweet" => println!("{} fav'd rt", structure["source"]["screen_name"].as_str().unwrap()), e => println!("unrecognized event: {}", e) } } else if structure.contains_key("delete") { println!("delete..."); let deleted_user_id = structure["delete"]["status"]["user_id_str"].as_str().unwrap(); - let userjson = look_up_user(deleted_user_id); + let userjson = look_up_user(deleted_user_id, client, c2); let screen_name = match userjson { Some(ref json) => { json[0]["screen_name"].as_str().clone().unwrap() @@ -66,31 +67,56 @@ fn render_tweet(structure: serde_json::Map) { let author_name = twete["user"]["name"].as_str().unwrap(); let author_screen_name = twete["user"]["screen_name"].as_str().unwrap(); - println!("{} (@{}) via {} (@{}) RT:", author_name, author_screen_name, source_name, source_screen_name); + println!("{} (@{}) via {} (@{}) RT: https://twitter.com/i/web/status/{}", author_name, author_screen_name, source_name, source_screen_name, twete["id_str"].as_str().unwrap()); } else { - println!("{} (@{})", source_name, source_screen_name); + println!("{} (@{}) https://twitter.com/i/web/status/{}", source_name, source_screen_name, twete["id_str"].as_str().unwrap()); } - let mut twete_text = (if twete["truncated"].as_bool().unwrap() { - // get full text here! - println!(" ... :/"); - "asdf" - } else { - twete["text"].as_str().unwrap() - }) - .replace("&", "&") - .replace(">", ">") - .replace("<", "<"); - for url in twete["entries"]["urls"].as_array().unwrap() { - twete_text.replace(url["url"].as_str().unwrap(), url["expanded_url"].as_str().unwrap()); - } - println!("{}", twete_text); + + let twete_text = full_twete_text(twete); + + println!("{}", twete_text); // to remove the link for quoted twetes, gotta look for: "is the last url the same as the url for the quoted id?" and if so s/// + if twete.contains_key("quoted_status") { - println!(" and it's a quote "); + let quoted = twete["quoted_status"].as_object().unwrap(); + println!( + " {} (@{}) https://twitter.com/i/web/status/{}\n {}", + quoted["user"]["name"].as_str().unwrap(), + quoted["user"]["screen_name"].as_str().unwrap(), + quoted["id_str"].as_str().unwrap(), + full_twete_text(quoted).split("\n").collect::>().join("\n ") + ); +// println!(" and it's a quote "); } } + println!("-------------------------------------------------------"); println!(""); } +fn full_twete_text(twete: &serde_json::map::Map) -> String { +// let looked_up; + let mut twete_text: String; + twete_text = if twete["truncated"].as_bool().unwrap() { + twete["extended_tweet"]["full_text"].as_str().unwrap().to_string() +// looked_up = look_up_tweet(twete["id_str"].as_str().unwrap()).unwrap(); +// looked_up["text"].as_str().unwrap().to_string() + } else { + twete["text"].as_str().unwrap().to_string() + }; + + twete_text = twete_text + .replace("&", "&") + .replace(">", ">") + .replace("<", "<"); + + for url in twete["entities"]["urls"].as_array().unwrap() { + if url["expanded_url"].as_str().unwrap().len() < 200 { + twete_text = twete_text.replace(url["url"].as_str().unwrap(), url["expanded_url"].as_str().unwrap()); + } + } + + twete_text +} + fn signed_get(url: &str) -> hyper::client::Request { // let params: Vec<(String, String)> = vec![("track".to_string(), "london".to_string())]; let params: Vec<(String, String)> = vec![]; @@ -126,22 +152,15 @@ fn signed_get(url: &str) -> hyper::client::Request { req } -fn look_up_user(id: &str) -> Option { - let mut core = Core::new().unwrap(); - let connector = HttpsConnector::new(4, &core.handle()).unwrap(); - - let client = Client::configure() - .connector(connector) - .build(&core.handle()); - +fn look_up_user(id: &str, client: &hyper::client::Client>, core: &mut Core) -> Option { let lookup = client.request(signed_get(&format!("{}?user_id={}", user_lookup_url, id))); + let resp: hyper::Response = core.run(lookup).unwrap(); -// println!("user lookup request out.."); - let w = resp.body() - .map(|chunk| futures::stream::iter(chunk.into_iter().map(|b| -> Result { Ok(b) }))) - .flatten() - .wait(); - let resp_body = w.map(|r| { r.unwrap() }).collect::>(); + + let chunks: Vec = core.run(resp.body().collect()).unwrap(); + + let resp_body: Vec = chunks.into_iter().flat_map(|chunk| chunk.into_iter()).collect(); + match serde_json::from_slice(&resp_body) { Ok(value) => Some(value), Err(e) => { @@ -151,21 +170,15 @@ fn look_up_user(id: &str) -> Option { } } -fn look_up_tweet(id: &str) -> Option { - let mut core = Core::new().unwrap(); - let connector = HttpsConnector::new(4, &core.handle()).unwrap(); - - let client = Client::configure() - .connector(connector) - .build(&core.handle()); - +fn look_up_tweet(id: &str, client: &hyper::client::Client>, core: &mut Core) -> Option { let lookup = client.request(signed_get(&format!("{}?id={}", tweet_lookup_url, id))); + let resp: hyper::Response = core.run(lookup).unwrap(); - let w = resp.body() - .map(|chunk| futures::stream::iter(chunk.into_iter().map(|b| -> Result { Ok(b) }))) - .flatten() - .wait(); - let resp_body = w.map(|r| { r.unwrap() }).collect::>(); + + let chunks: Vec = core.run(resp.body().collect()).unwrap(); + + let resp_body: Vec = chunks.into_iter().flat_map(|chunk| chunk.into_iter()).collect(); + match serde_json::from_slice(&resp_body) { Ok(value) => Some(value), Err(e) => { @@ -185,7 +198,7 @@ fn main() { let mut core = Core::new().unwrap(); - let connector = HttpsConnector::new(4, &core.handle()).unwrap(); + let connector = HttpsConnector::new(1, &core.handle()).unwrap(); let client = Client::configure() .keep_alive(true) @@ -195,7 +208,6 @@ fn main() { let req = signed_get(streamurl); println!("starting!"); -// println!("{}", look_up_user("12503922").unwrap()[0]["screen_name"].as_str().unwrap()); // println!("lookup'd"); // println!("requesting..."); @@ -211,7 +223,16 @@ fn main() { let (tx, rx) = std::sync::mpsc::channel::>(); + let remote = core.remote(); + std::thread::spawn(move || { + let mut c2 = Core::new().unwrap(); // i swear this is not where the botnet lives + let handle = &c2.handle(); + let secondaryConnector = HttpsConnector::new(4, handle).unwrap(); + + let secondaryClient = Client::configure() + .connector(secondaryConnector) + .build(handle); loop { match rx.recv() { Ok(line) => { @@ -219,7 +240,7 @@ fn main() { //println!("{}", jsonstr); let json: serde_json::Value = serde_json::from_str(&jsonstr).unwrap(); match json { - serde_json::Value::Object(objmap) => render_tweet(objmap), + serde_json::Value::Object(objmap) => render_tweet(objmap, &secondaryClient, &mut c2), f => println!("Unexpected object: {}", f) } } @@ -228,20 +249,20 @@ fn main() { } }); + println!("Before?"); let work = client.request(req).and_then(|res| { LineStream::new(res.body() - .map(|chunk| futures::stream::iter(chunk.into_iter().map(|b| -> Result { Ok(b) }))) + .map(|chunk| futures::stream::iter(chunk.into_iter().map(|b| Ok(b)))) .flatten()) .for_each(|s| { if s.len() != 1 { - println!("Send!: {}", std::str::from_utf8(&s).unwrap()); +// println!("Send!: {}", std::str::from_utf8(&s).unwrap()); tx.send(s); }; Ok(()) }) }); - println!("Before?"); let resp = core.run(work).unwrap(); println!("After?"); -- cgit v1.1