Skip to main content
Loading...

More JavaScript Posts

//Allegedly never tested on the twitter website
function scrapeScreen(){
    let articles = Array.from(document.getElementsByTagName("article"));
    let results = [];
    for(let tweet of articles){
        const isAnAd = Array.from(tweet.querySelectorAll("span")).map((e)=>e.textContent).includes("Ad");
        if(isAnAd){
            //console.log(tweet, "is an ad. Skipping...");
            continue;
        }
        const userName = tweet.querySelector("[data-testid='User-Name'] > div:nth-child(2) > div > div")?.textContent;
        const tweetContent = tweet.querySelector("[data-testid='tweetText']")?.textContent;
        const timeStamp = tweet.querySelector("time")?.getAttribute("datetime");
        const tweetLink = tweet.querySelector("time")?.parentElement?.getAttribute("href");
        if((!userName) || (!tweetContent)) continue;
        results.push({
            username: userName,
            tweetText: tweetContent,
            timeStamp: timeStamp,
            tweetLink: tweetLink
        });
    }
    return results;
}
let scraped = scrapeScreen();
setInterval(()=>{
    scraped = scraped.concat(
        scrapeScreen().filter((tweet)=>{
            for(let scrapedTweet of scraped){
                if(scrapedTweet.username == tweet.username && scrapedTweet.tweetText == tweet.tweetText) return false;
            }
            return true;
        })
    );
}, 500); //Scrape everything on the screen twice a second

window.scrollIntervalId = setInterval(function(){
    window.scrollBy(0, 1000);
}, 500); //Scroll for me


//http://bgrins.github.io/devtools-snippets/#console-save
(function(console){

console.save = function(data, filename){

 if(!data) {
 console.error('Console.save: No data')
 return;
 }

 if(!filename) filename = 'console.json'

 if(typeof data === "object"){
 data = JSON.stringify(data, undefined, '\t')
 }

 var blob = new Blob([data], {type: 'text/json'}),
 e = document.createEvent('MouseEvents'),
 a = document.createElement('a')

 a.download = filename
 a.href = window.URL.createObjectURL(blob)
 a.dataset.downloadurl = ['text/json', a.download, a.href].join(':')
 e.initMouseEvent('click', true, false, window, 0, 0, 0, 0, 0, false, false, false, false, 0, null)
 a.dispatchEvent(e)
 }
})(console)

setTimeout(()=>{
    clearTimeout(window.scrollIntervalId);
    delete window.scrollIntervalId;

    console.save(scraped, "TwitterScrape" + Date.now() + ".json");
}, 60 * 1000 * 20); //Twenty minutes
class Graph {
  constructor(directed = true) {
    this.directed = directed;
    this.nodes = [];
    this.edges = new Map();
  }

  addNode(key, value = key) {
    this.nodes.push({ key, value });
  }

  addEdge(a, b, weight) {
    this.edges.set(JSON.stringify([a, b]), { a, b, weight });
    if (!this.directed)
      this.edges.set(JSON.stringify([b, a]), { a: b, b: a, weight });
  }

  removeNode(key) {
    this.nodes = this.nodes.filter(n => n.key !== key);
    [...this.edges.values()].forEach(({ a, b }) => {
      if (a === key || b === key) this.edges.delete(JSON.stringify([a, b]));
    });
  }

  removeEdge(a, b) {
    this.edges.delete(JSON.stringify([a, b]));
    if (!this.directed) this.edges.delete(JSON.stringify([b, a]));
  }

  findNode(key) {
    return this.nodes.find(x => x.key === key);
  }

  hasEdge(a, b) {
    return this.edges.has(JSON.stringify([a, b]));
  }

  setEdgeWeight(a, b, weight) {
    this.edges.set(JSON.stringify([a, b]), { a, b, weight });
    if (!this.directed)
      this.edges.set(JSON.stringify([b, a]), { a: b, b: a, weight });
  }

  getEdgeWeight(a, b) {
    return this.edges.get(JSON.stringify([a, b])).weight;
  }

  adjacent(key) {
    return [...this.edges.values()].reduce((acc, { a, b }) => {
      if (a === key) acc.push(b);
      return acc;
    }, []);
  }

  indegree(key) {
    return [...this.edges.values()].reduce((acc, { a, b }) => {
      if (b === key) acc++;
      return acc;
    }, 0);
  }

  outdegree(key) {
    return [...this.edges.values()].reduce((acc, { a, b }) => {
      if (a === key) acc++;
      return acc;
    }, 0);
  }
}


const g = new Graph();

g.addNode('a');
g.addNode('b');
g.addNode('c');
g.addNode('d');

g.addEdge('a', 'c');
g.addEdge('b', 'c');
g.addEdge('c', 'b');
g.addEdge('d', 'a');

g.nodes.map(x => x.value);  // ['a', 'b', 'c', 'd']
[...g.edges.values()].map(({ a, b }) => `${a} => ${b}`);
// ['a => c', 'b => c', 'c => b', 'd => a']

g.adjacent('c');            // ['b']

g.indegree('c');            // 2
g.outdegree('c');           // 1

g.hasEdge('d', 'a');        // true
g.hasEdge('a', 'd');        // false

g.removeEdge('c', 'b');

[...g.edges.values()].map(({ a, b }) => `${a} => ${b}`);
// ['a => c', 'b => c', 'd => a']

g.removeNode('c');

g.nodes.map(x => x.value);  // ['a', 'b', 'd']
[...g.edges.values()].map(({ a, b }) => `${a} => ${b}`);
// ['d => a']

g.setEdgeWeight('d', 'a', 5);
g.getEdgeWeight('d', 'a');  // 5