Node Web Crawler Practical Process
Setting Up the Environment
Ensure Node.js and npm are installed.
# Create project directory
mkdir node-crawler
cd node-crawler
# Initialize project
npm init -yInstalling Dependencies
We will use the following libraries:
- axios: For making HTTP requests.
- cheerio: For parsing HTML documents.
- puppeteer: For simulating browser behavior (optional).
npm install axios cheerio puppeteerWriting the Crawler Code
We will create a simple crawler to scrape titles and links from a webpage.
// crawler.js
const axios = require('axios');
const cheerio = require('cheerio');
async function fetchPage(url) {
try {
const response = await axios.get(url);
return response.data;
} catch (error) {
console.error(`Failed to fetch ${url}:`, error.message);
return null;
}
}
function parseHtml(html) {
if (!html) return [];
const $ = cheerio.load(html);
const links = [];
$('a').each((index, element) => {
const title = $(element).text();
const link = $(element).attr('href');
links.push({ title, link });
});
return links;
}
async function crawl(url) {
const html = await fetchPage(url);
const links = parseHtml(html);
console.log(links);
}
crawl('https://example.com');Using a Proxy
To avoid IP bans, you can use a proxy server for crawling.
// crawler-proxy.js
const axios = require('axios');
const cheerio = require('cheerio');
async function fetchPage(url, proxy) {
try {
const response = await axios.get(url, { proxy });
return response.data;
} catch (error) {
console.error(`Failed to fetch ${url}:`, error.message);
return null;
}
}
function parseHtml(html) {
if (!html) return [];
const $ = cheerio.load(html);
const links = [];
$('a').each((index, element) => {
const title = $(element).text();
const link = $(element).attr('href');
links.push({ title, link });
});
return links;
}
async function crawl(url, proxy) {
const html = await fetchPage(url, proxy);
const links = parseHtml(html);
console.log(links);
}
const proxy = {
host: 'proxy.example.com',
port: 8080,
};
crawl('https://example.com', proxy);Running the Crawler
node crawler.jsUsing Puppeteer for Browser Simulation
Puppeteer is useful for crawling dynamically loaded content by simulating browser behavior.
// crawler-puppeteer.js
const puppeteer = require('puppeteer');
async function launchBrowser() {
const browser = await puppeteer.launch();
const page = await browser.newPage();
await page.goto('https://example.com', { waitUntil: 'networkidle2' });
// Wait for AJAX-loaded content
await page.waitForSelector('.ajax-content-loaded');
const html = await page.content();
await browser.close();
return html;
}
async function crawlWithPuppeteer() {
const html = await launchBrowser();
const links = parseHtml(html);
console.log(links);
}
crawlWithPuppeteer();Storing Data
You can store crawled data in a file or database.
// storage.js
const fs = require('fs');
function saveToFile(data, filename) {
fs.writeFile(filename, JSON.stringify(data), (err) => {
if (err) throw err;
console.log(`Data saved to ${filename}`);
});
}
module.exports = {
saveToFile
};Integrating Storage
Combine the crawler with storage to save data to a file.
// crawler-storage.js
const axios = require('axios');
const cheerio = require('cheerio');
const { saveToFile } = require('./storage');
async function fetchPage(url) {
try {
const response = await axios.get(url);
return response.data;
} catch (error) {
console.error(`Failed to fetch ${url}:`, error.message);
return null;
}
}
function parseHtml(html) {
if (!html) return [];
const $ = cheerio.load(html);
const links = [];
$('a').each((index, element) => {
const title = $(element).text();
const link = $(element).attr('href');
links.push({ title, link });
});
return links;
}
async function crawl(url) {
const html = await fetchPage(url);
const links = parseHtml(html);
saveToFile(links, 'links.json');
}
crawl('https://example.com');Concurrency Control
To improve efficiency, use concurrency control to crawl multiple pages simultaneously.
// crawler-concurrency.js
const axios = require('axios');
const cheerio = require('cheerio');
const pLimit = require('p-limit'); // For limiting concurrency
const limit = pLimit(5); // Maximum 5 concurrent requests
async function fetchPage(url) {
try {
const response = await axios.get(url);
return response.data;
} catch (error) {
console.error(`Failed to fetch ${url}:`, error.message);
return null;
}
}
function parseHtml(html) {
if (!html) return [];
const $ = cheerio.load(html);
const links = [];
$('a').each((index, element) => {
const title = $(element).text();
const link = $(element).attr('href');
links.push({ title, link });
});
return links;
}
async function crawl(url) {
const html = await fetchPage(url);
const links = parseHtml(html);
console.log(links);
}
async function runCrawler(urls) {
await Promise.all(urls.map(url => limit(() => crawl(url))));
}
runCrawler(['https://example.com/page1', 'https://example.com/page2']);
