Install Dependencies
npm i google-search-results-nodejs
npm i puppeteer-extra-plugin-stealth
Scrape YouTube Keyword Suggestions Using Node.js Express Puppeteer
index.js
const puppeteer = require("puppeteer-extra");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");
const ObjectsToCsv = require('objects-to-csv');
const express = require('express')
puppeteer.use(StealthPlugin());
const app = express()
app.use(express.urlencoded({ extended: false }))
app.set('view engine','ejs')
app.get('/', (req, res) => {
res.render('index',{suggestions:[]})
})
app.post('/getsuggestions', (req, res) => {
let search = req.body.search.split('\n')
getYoutubeAutocomplete(search)
.then((data) => {
console.log(data)
res.render('index',{suggestions:data.autocompleteResults})
})
})
async function getYoutubeAutocomplete(search) {
const queries = search;
const URL = "https://www.youtube.com";
const browser = await puppeteer.launch({
headless: false,
args: ["--no-sandbox", "--disable-setuid-sandbox"],
});
const page = await browser.newPage();
await page.setDefaultNavigationTimeout(60000);
await page.waitForTimeout(4000)
await page.goto(URL);
await page.waitForSelector("#contents");
const autocompleteResults = [];
for (query of queries) {
await page.click("#search-input");
await page.keyboard.type(query);
await page.waitForTimeout(5000);
const results = {
query,
autocompleteResults: await page.evaluate(() => {
return Array.from(document.querySelectorAll(".sbdd_a li"))
.map((el) => el.querySelector(".sbqs_c")?.textContent.trim())
.filter((el) => el);
}),
};
autocompleteResults.push(results);
await page.click("#search-clear-button");
await page.waitForTimeout(2000);
}
await browser.close();
const csv = new ObjectsToCsv(autocompleteResults);
await csv.toDisk('./test.csv');
return autocompleteResults;
}
app.listen(5000,() => {
console.log("App is listening on port 5000")
})views/index.ejs
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Get Youtube Suggestions</title>
</head>
<body>
<form action="/getsuggestions" method="post">
<textarea cols="15" rows="15" name="search" required></textarea>
<button>Get Suggestions</button>
</form>
<%if(suggestions){%>
<table>
<thead>
<tr>
<th>Suggestions</th>
</tr>
</thead>
<tbody>
<%suggestions.forEach(item=> {%>
<tr>
<td>
<%=item%>
</td>
</tr>
<%})%>
</tbody>
</table>
<%}%>
</body>
</html>
