From 000002400452336744e36bfb1bbb85c855002a33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Jane=C5=BEi=C4=8D?= Date: Sat, 31 Dec 2022 19:29:50 +0100 Subject: [PATCH] feat: download_all and async downloading with tokio --- .cargo/config | 1 + Cargo.lock | 84 +++++++++++++++++++++++++++++++++++------ Cargo.toml | 12 +++--- src/bin/download.rs | 8 ++-- src/bin/download_all.rs | 57 ++++++++++++++++++++++++++++ 5 files changed, 142 insertions(+), 20 deletions(-) create mode 100644 src/bin/download_all.rs diff --git a/.cargo/config b/.cargo/config index 794eb6d..e963c04 100644 --- a/.cargo/config +++ b/.cargo/config @@ -1,5 +1,6 @@ [alias] scaffold = "run --bin scaffold --quiet --release --" download = "run --bin download --quiet --release --" +download_all = "run --bin download_all --quiet --release --" solve = "run --bin" diff --git a/Cargo.lock b/Cargo.lock index 780ba43..f97cf41 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,7 +24,7 @@ dependencies = [ [[package]] name = "aoc" -version = "21.0.0" +version = "24.0.0" dependencies = [ "dotenv", "elves", @@ -35,6 +35,7 @@ dependencies = [ "pico-args", "regex", "reqwest", + "tokio", ] [[package]] @@ -181,12 +182,6 @@ version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" -[[package]] -name = "futures-io" -version = "0.3.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" - [[package]] name = "futures-sink" version = "0.3.25" @@ -206,12 +201,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" dependencies = [ "futures-core", - "futures-io", "futures-task", - "memchr", "pin-project-lite", "pin-utils", - "slab", ] [[package]] @@ -405,6 +397,16 @@ version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" +[[package]] +name = "lock_api" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.17" @@ -517,6 +519,29 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ff9f3fef3968a3ec5945535ed654cb38ff72d7495a25619e2247fb15a2ed9ba" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys 0.42.0", +] + [[package]] name = "percent-encoding" version = "2.2.0" @@ -653,6 +678,12 @@ dependencies = [ "windows-sys 0.36.1", ] +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + [[package]] name = "security-framework" version = "2.7.0" @@ -678,9 +709,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.151" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fed41fc1a24994d044e6db6935e69511a1153b52c15eb42493b26fa87feba0" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" [[package]] name = "serde_json" @@ -705,6 +736,15 @@ dependencies = [ "serde", ] +[[package]] +name = "signal-hook-registry" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +dependencies = [ + "libc", +] + [[package]] name = "slab" version = "0.4.7" @@ -714,6 +754,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + [[package]] name = "socket2" version = "0.4.7" @@ -776,11 +822,25 @@ dependencies = [ "memchr", "mio", "num_cpus", + "parking_lot", "pin-project-lite", + "signal-hook-registry", "socket2", + "tokio-macros", "windows-sys 0.42.0", ] +[[package]] +name = "tokio-macros" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tokio-native-tls" version = "0.3.0" diff --git a/Cargo.toml b/Cargo.toml index d889bb1..6d71be8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,14 +1,15 @@ [package] name = "aoc" -version = "23.0.0" +description = "template for advent of code" +readme = "README.md" +version = "24.0.0" edition = "2021" +license = "MIT" authors = ["Matej Janežič "] +repository = "https://github.com/janezicmatej/aoc-template.git" default-run = "aoc" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] -# framework dotenv = "0.15.0" elves = "0.2.0" hashbrown = "0.13.1" @@ -17,4 +18,5 @@ itertools = "0.10.5" lazy_static = "1.4.0" pico-args = "0.5.0" regex = "1.7.0" -reqwest = { version = "0.11.13", features = ["blocking"] } +reqwest = "0.11.13" +tokio = { version = "1.23.0", features = ["full"] } diff --git a/src/bin/download.rs b/src/bin/download.rs index 54632e7..dba71ee 100644 --- a/src/bin/download.rs +++ b/src/bin/download.rs @@ -1,9 +1,9 @@ use dotenv::dotenv; -use reqwest::blocking::Client; -use reqwest::header; +use reqwest::{header, Client}; use std::{env, fs::OpenOptions, io::Write, process}; -fn main() { +#[tokio::main] +async fn main() { let day: u8 = match aoc::parse_args() { Ok(day) => day, Err(_) => { @@ -30,8 +30,10 @@ fn main() { let res = client .get(format!("https://adventofcode.com/{year}/day/{day}/input")) .send() + .await .unwrap() .text() + .await .unwrap(); let input_path = format!("src/inputs/{day_padded}.txt"); diff --git a/src/bin/download_all.rs b/src/bin/download_all.rs new file mode 100644 index 000000000..3d0bb37 --- /dev/null +++ b/src/bin/download_all.rs @@ -0,0 +1,57 @@ +use dotenv::dotenv; +use itertools::Itertools; +use reqwest::{header, Client}; +use std::{env, fs::OpenOptions, io::Write, process}; + +#[tokio::main] +async fn main() { + dotenv().ok(); + let token = env::var("TOKEN").expect("$TOKEN is not set"); + let year = env::var("YEAR") + .expect("$YEAR is not set") + .parse::() + .expect("$YEAR must be a number"); + + let mut headers = header::HeaderMap::new(); + let mut session_header = header::HeaderValue::from_str(format!("session={token}").as_str()) + .expect("Error building cookie header"); + session_header.set_sensitive(true); + headers.insert(header::COOKIE, session_header); + + let client = Client::builder().default_headers(headers).build().unwrap(); + let responses = (1..=25) + .map(|d| { + let endpoint = format!("https://adventofcode.com/{year}/day/{d}/input"); + println!("{endpoint}"); + client.get(endpoint).send() + }) + .collect_vec() + .into_iter() + .map(|x| async { x.await.unwrap().text().await.unwrap() }) + .collect_vec(); + + for (day, res) in (1..=25).zip(responses) { + let input_path = format!("src/inputs/{day:02}.txt"); + let mut file = match OpenOptions::new() + .write(true) + .create(true) + .open(&input_path) + { + Ok(file) => file, + Err(e) => { + eprintln!("Failed to create module file: {e}"); + process::exit(1); + } + }; + + match file.write_all(res.await.as_bytes()) { + Ok(_) => { + println!("Downloaded input file \"{}\"", &input_path); + } + Err(e) => { + eprintln!("Failed to write module contents: {e}"); + process::exit(1); + } + } + } +}