Skip to main content
L

LeifMessinger

User since Oct 28, 2022

63 Posts

Recent Posts

use indicatif::{ProgressBar, ProgressStyle};
use reqwest::blocking::Client;
use std::sync::{Arc, Mutex};
use std::thread;
use std::time::Duration;

const ACCESS_TOKEN: &str = "YOUR_CANVAS_API_TOKEN";
const BASE_URL: &str = "https://unt.instructure.com/api/v1/files";
const CENTER_ID: u64 = 37826490;
const RADIUS: u64 = 200000;
const NUM_WORKERS: usize = 20;

/// Pause all workers when quota drops to or below this value.
const RATE_LIMIT_PAUSE_THRESHOLD: f64 = 50.0;
/// Resume once quota recovers above this value (Canvas resets the window automatically).
const RATE_LIMIT_RESUME_THRESHOLD: f64 = 200.0;
/// How long each paused worker sleeps before re-checking.
const RATE_LIMIT_SLEEP_MS: u64 = 2000;

fn parse_rate_limit(resp: &reqwest::blocking::Response) -> Option<f64> {
    resp.headers()
        .get("x-rate-limit-remaining")?
        .to_str()
        .ok()?
        .parse::<f64>()
        .ok()
}

fn main() {
    let start = CENTER_ID - RADIUS;
    let end = CENTER_ID + RADIUS;
    let ids: Vec<u64> = (start..=end).collect();
    let total = ids.len() as u64;

    let pb = ProgressBar::new(total);
    pb.set_style(
        ProgressStyle::with_template(
            "[{percent:>3}%] {spinner:.cyan} | {pos}/{len} IDs | elapsed {elapsed_precise} | ETA {eta_precise} | RL: {msg} {bar:25.green/white}",
        )
        .unwrap()
        .progress_chars("█▉▊▋▌▍▎▏  ")
        .tick_chars("|/-\\"),
    );
    pb.set_message("---");
    pb.enable_steady_tick(Duration::from_millis(100));

    let ids = Arc::new(Mutex::new(ids.into_iter()));
    let pb = Arc::new(pb);
    let found: Arc<Mutex<Vec<(u64, String)>>> = Arc::new(Mutex::new(Vec::new()));

    // Shared rate-limit state across all workers.
    let rate_limit_remaining: Arc<Mutex<f64>> = Arc::new(Mutex::new(f64::MAX));

    let mut handles = vec![];

    for _ in 0..NUM_WORKERS {
        let ids = Arc::clone(&ids);
        let pb = Arc::clone(&pb);
        let found = Arc::clone(&found);
        let rl = Arc::clone(&rate_limit_remaining);

        let handle = thread::spawn(move || {
            let client = Client::builder()
                .timeout(Duration::from_secs(15))
                .build()
                .expect("Failed to build HTTP client");

            loop {
                // --- Rate-limit gate: pause until quota recovers ---
                loop {
                    let remaining = *rl.lock().unwrap();
                    if remaining > RATE_LIMIT_PAUSE_THRESHOLD {
                        break;
                    }
                    pb.set_message(format!("PAUSED ({:.0} left)", remaining));
                    thread::sleep(Duration::from_millis(RATE_LIMIT_SLEEP_MS));
                    // Optimistically let one thread probe the server to check
                    // if the Canvas window has reset.
                    let mut r = rl.lock().unwrap();
                    if *r <= RATE_LIMIT_PAUSE_THRESHOLD {
                        *r = RATE_LIMIT_RESUME_THRESHOLD;
                    }
                }

                // --- Pull next ID ---
                let id = {
                    let mut iter = ids.lock().unwrap();
                    iter.next()
                };
                let id = match id {
                    Some(id) => id,
                    None => break,
                };

                let url = format!("{}/{}?access_token={}", BASE_URL, id, ACCESS_TOKEN);

                // HEAD request to check existence + read rate-limit header
                let head_resp = client.head(&url).send();

                if let Ok(resp) = head_resp {
                    // Update shared quota from every response
                    if let Some(remaining) = parse_rate_limit(&resp) {
                        let mut r = rl.lock().unwrap();
                        *r = remaining;
                        pb.set_message(format!("{:.0}", remaining));
                    }

                    if resp.status().is_success() {
                        // GET to retrieve display_name
                        if let Ok(get_resp) = client.get(&url).send() {
                            if let Some(remaining) = parse_rate_limit(&get_resp) {
                                let mut r = rl.lock().unwrap();
                                *r = remaining;
                                pb.set_message(format!("{:.0}", remaining));
                            }
                            if let Ok(json) = get_resp.json::<serde_json::Value>() {
                                let name = json["display_name"]
                                    .as_str()
                                    .unwrap_or("<unknown>")
                                    .to_string();
                                pb.println(format!("FOUND: {}. Name: {}", id, name));
                                found.lock().unwrap().push((id, name));
                            }
                        }
                    }
                }

                pb.inc(1);
            }
        });

        handles.push(handle);
    }

    for h in handles {
        h.join().expect("Worker thread panicked");
    }

    pb.finish_with_message("done");

    let found = found.lock().unwrap();
    println!("\n=== Results ({} found) ===", found.len());
    for (id, name) in found.iter() {
        println!("  ID {}: {}", id, name);
    }
}
//===============Header File==================
#include <iostream>
#include <sstream> //stringbuf
#include <utility> //exchange

//Couple rules:
//Characters given through the getter functions have to be removed from the buffer.
    //This is so that bufferEmpty() == buffer.in_avail() > 0 basically always.
    //skipWhitespace doesn't remove the text from the buffer, but it does return the number of characters.
//nextWord will trim whitespace before the word
//nextInt will trim non-numbers before the number
//hasNextInt and hasNextWord will trim the whitespace. If you think you need it, you should get nextWhitespace before doing any of those.
//Whitespace after a word or an int is left on the buffer.
//nextWhitespace will (get and) remove whitespace until the end of the line, including the newline character, but stops before the next line.
    //nextWhitespace won't read the next line when called before the end of the line, and it won't prompt the user for the next line if interactive.
        //If nextWhitespace is called after reading the end of the line, then it will read a new line into the buffer, which will prompt the user.
    //It acts like nextLine, but if there's something non-whitespace on the current line it stops there.
class Scanner {
    public:
        std::stringbuf buffer;
        std::istream& input;
        
        Scanner(std::istream& in = std::cin) : buffer(), input(in) {}
        
    //Buffer debugging
        bool fillBuffer();
    
        bool bufferEmpty();
        void printBufferEmpty();
        std::string getBuffer();
        size_t bufferLength();
        void printBufferStats();
    
    //Int
        bool hasNextInt();
        int nextInt();
    
    //Word
        bool hasNextWord();
        std::string nextWord();
    
    //Line
        bool hasNextLine();
    
    //Whitespace
        size_t skipWhitespace();    //Prob should be private, but I don't believe in that private shit.
        bool hasNextWhitespace();
        std::string nextWhitespace();
        std::string nextWhitespaceAll();
        std::string nextLine();
};

//===============Source File==================
bool Scanner::fillBuffer() {    //Returns if it had to get the next line from the input.
    const bool badInput = input.eof() || input.bad();
    const bool shouldFillBuffer = bufferEmpty() && !badInput;
    if (shouldFillBuffer) {
        std::string line;
        if (std::getline(input, line)) {
            buffer.str(buffer.str() + line + "\n");
        }
    }
    return shouldFillBuffer;
}

bool Scanner::bufferEmpty(){
    return buffer.str() == "";
}
void Scanner::printBufferEmpty(){
    std::cout << "The buffer is " << (bufferEmpty()? "" : "not") << " empty." << std::endl;
}

std::string Scanner::getBuffer(){
    return buffer.str();
}
size_t Scanner::bufferLength(){
    return buffer.str().length();
}
void Scanner::printBufferStats(){
    if(bufferEmpty()){
        std::cout << "The buffer is \"\"" << std::endl;
        return;
    }
    std::cout << "The length of the buffer is " << bufferLength() << std::endl;
    if(buffer.sgetc() == '\r'){
        std::cout << "The buffer is \\r\\n" << std::endl;
    }else if(buffer.sgetc() == '\n'){
        std::cout << "The buffer is \\n" << std::endl;
    }
}

bool Scanner::hasNextInt() {
    return hasNextWord() && (std::isdigit(buffer.sgetc()) || buffer.sgetc() == '-');
}

int Scanner::nextInt() {
    if (!hasNextInt()) {   //Will fill the buffer if not filled. Will also trim whitespace.
        return 0;
    }
    
    std::string num;
    size_t charactersRead = 0;
    while (buffer.in_avail() > 0 && (std::isdigit(buffer.sgetc()) || buffer.sgetc() == '-')) {
        num += buffer.sbumpc();
        ++charactersRead;
    }
    buffer.str(buffer.str().erase(0, charactersRead));
    return std::stoi(num);
}

bool Scanner::hasNextWord() {
    nextWhitespaceAll();
    return buffer.in_avail() > 0;
}

std::string Scanner::nextWord() {
    if (!hasNextWord()) {   //Will fill the buffer if not filled. Will also trim whitespace.
        return "";
    }
    
    std::string word;
    size_t charactersRead = 0;
    while (buffer.in_avail() > 0 && !std::isspace(buffer.sgetc())) {
        word += buffer.sbumpc();
        ++charactersRead;
    }
    buffer.str(buffer.str().erase(0, charactersRead));
    return word;
}

bool Scanner::hasNextLine() {
    return (!bufferEmpty()) || fillBuffer();
}

size_t Scanner::skipWhitespace() {   //Returns characters read
    size_t charactersRead = 0;
    while (buffer.in_avail() > 0 && std::isspace(buffer.sgetc())) {
        buffer.sbumpc();
        ++charactersRead;
    }
    return charactersRead;
}
bool Scanner::hasNextWhitespace(){
    fillBuffer();
    return buffer.in_avail() > 0 && std::isspace(buffer.sgetc());
}
std::string Scanner::nextWhitespace() {
    if (!hasNextWhitespace()) {   //Will fill the buffer if not filled
        return "";
    }
    const size_t charactersRead = skipWhitespace();
    std::string whitespace = buffer.str().substr(charactersRead);
    buffer.str(buffer.str().erase(0, charactersRead));
    return whitespace;
}
std::string Scanner::nextWhitespaceAll(){
    std::string whitespace;
    while(hasNextWhitespace()){
        std::string gottenWhiteSpace = nextWhitespace();
        whitespace += gottenWhiteSpace;
    }
    return whitespace;
}
std::string Scanner::nextLine(){
    if (!hasNextLine()) {
        return "";
    }
    
    fillBuffer();
    //Swap out the old buffer with an empty buffer, and get the old buffer as a variable.
    std::string line = std::exchange(buffer, std::stringbuf()).str();
    
    //Remove the newline.
    if(line[line.length() - 1] == '\n' || line[line.length() - 1] == '\r' ) line.pop_back();
    if(line[line.length() - 1] == '\r' || line[line.length() - 1] == '\n' ) line.pop_back();
    return line;
}


//=================Word and Int test=================
while(bruh.hasNextInt() || bruh.hasNextWord()){
    std::cout << "started loop" << std::endl;
    if(bruh.hasNextInt()){
        std::cout << "Int: " << bruh.nextInt() << " " << std::endl;
    }else{
        std::cout << "Word: " << bruh.nextWord() << " " << std::endl;
    }
    bruh.nextWhitespace();
}
//===================Line test======================
for(int count = 1; bruh.hasNextLine(); ++count){
    std::string line = bruh.nextLine();
    std::cout << "Line " << count << ": " << line << std::endl;
}

Post Statistics