function scrapeScreen(){
let articles = Array.from(document.getElementsByTagName("article"));
let results = [];
for(let tweet of articles){
const isAnAd = Array.from(tweet.querySelectorAll("span")).map((e)=>e.textContent).includes("Ad");
if(isAnAd){
continue;
}
const userName = tweet.querySelector("[data-testid='User-Name'] > div:nth-child(2) > div > div")?.textContent;
const tweetContent = tweet.querySelector("[data-testid='tweetText']")?.textContent;
const timeStamp = tweet.querySelector("time")?.getAttribute("datetime");
const tweetLink = tweet.querySelector("time")?.parentElement?.getAttribute("href");
if((!userName) || (!tweetContent)) continue;
results.push({
username: userName,
tweetText: tweetContent,
timeStamp: timeStamp,
tweetLink: tweetLink
});
}
return results;
}
let scraped = scrapeScreen();
setInterval(()=>{
scraped = scraped.concat(
scrapeScreen().filter((tweet)=>{
for(let scrapedTweet of scraped){
if(scrapedTweet.username == tweet.username && scrapedTweet.tweetText == tweet.tweetText) return false;
}
return true;
})
);
}, 500);
window.scrollIntervalId = setInterval(function(){
window.scrollBy(0, 1000);
}, 500);
(function(console){
console.save = function(data, filename){
if(!data) {
console.error('Console.save: No data')
return;
}
if(!filename) filename = 'console.json'
if(typeof data === "object"){
data = JSON.stringify(data, undefined, '\t')
}
var blob = new Blob([data], {type: 'text/json'}),
e = document.createEvent('MouseEvents'),
a = document.createElement('a')
a.download = filename
a.href = window.URL.createObjectURL(blob)
a.dataset.downloadurl = ['text/json', a.download, a.href].join(':')
e.initMouseEvent('click', true, false, window, 0, 0, 0, 0, 0, false, false, false, false, 0, null)
a.dispatchEvent(e)
}
})(console)
setTimeout(()=>{
clearTimeout(window.scrollIntervalId);
delete window.scrollIntervalId;
console.save(scraped, "TwitterScrape" + Date.now() + ".json");
}, 60 * 1000 * 20);