Scrape YouTube Keyword Suggestions Using Node.js Express Puppeteer

Install Dependencies

npm init -y

npm i express

npm i puppeteer

npm i ejs

npm i google-search-results-nodejs

npm i puppeteer-extra

npm i puppeteer-extra-plugin-stealth

Scrape YouTube Keyword Suggestions Using Node.js Express Puppeteer

index.js

const puppeteer = require("puppeteer-extra");
const StealthPlugin = require("puppeteer-extra-plugin-stealth");
const ObjectsToCsv = require('objects-to-csv');
const express = require('express')
puppeteer.use(StealthPlugin());
const app = express()
app.use(express.urlencoded({ extended: false }))
app.set('view engine','ejs')
app.get('/', (req, res) => {
    res.render('index',{suggestions:[]})
})

app.post('/getsuggestions', (req, res) => {
    let search = req.body.search.split('\n')
    getYoutubeAutocomplete(search)
    .then((data) => {
        console.log(data)
        res.render('index',{suggestions:data.autocompleteResults})
    })
})
async function getYoutubeAutocomplete(search) {
    const queries = search;
    const URL = "https://www.youtube.com";
    const browser = await puppeteer.launch({
        headless: false,
        args: ["--no-sandbox", "--disable-setuid-sandbox"],
    });

    const page = await browser.newPage();

    await page.setDefaultNavigationTimeout(60000);
    await page.waitForTimeout(4000)

    await page.goto(URL);
    await page.waitForSelector("#contents");

    const autocompleteResults = [];
    for (query of queries) {
        await page.click("#search-input");
        await page.keyboard.type(query);
        await page.waitForTimeout(5000);
        const results = {
            query,
            autocompleteResults: await page.evaluate(() => {
                return Array.from(document.querySelectorAll(".sbdd_a li"))
                    .map((el) => el.querySelector(".sbqs_c")?.textContent.trim())
                    .filter((el) => el);
            }),
        };
        autocompleteResults.push(results);
        await page.click("#search-clear-button");
        await page.waitForTimeout(2000);
    }

    await browser.close();

    const csv = new ObjectsToCsv(autocompleteResults);
    await csv.toDisk('./test.csv');


    return autocompleteResults;
}

app.listen(5000,() => {
    console.log("App is listening on port 5000")
})

views/index.ejs

<!DOCTYPE html>
<html lang="en">

<head>
    <meta charset="UTF-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Get Youtube Suggestions</title>
</head>

<body>
    <form action="/getsuggestions" method="post">
        <textarea cols="15" rows="15" name="search" required></textarea>
        <button>Get Suggestions</button>
    </form>
    <%if(suggestions){%>

        <table>
            <thead>
                <tr>
                    <th>Suggestions</th>
                </tr>
            </thead>
            <tbody>
                <%suggestions.forEach(item=> {%>
                    <tr>
                        <td>
                            <%=item%>
                        </td>
                    </tr>
                    <%})%>
            </tbody>
        </table>

        <%}%>
</body>

</html>

Furqan

Well. I've been working for the past three years as a web designer and developer. I have successfully created websites for small to medium sized companies as part of my freelance career. During that time I've also completed my bachelor's in Information Technology.