puppeteer-automation

npm init -y
npm install puppeteer

const puppeteer = require('puppeteer');
 
async function main() {
  const browser = await puppeteer.launch({
    headless: 'new',
    args: ['--no-sandbox', '--disable-setuid-sandbox']
  });
 
  try {
    const page = await browser.newPage();
    await page.goto('https://example.com');
    // Your automation code here
  } finally {
    await browser.close();
  }
}
 
main().catch(console.error);

const browser = await puppeteer.launch({
  headless: 'new',  // 'new' for new headless mode, false for visible browser
  slowMo: 50,       // Slow down operations for debugging
  devtools: true,   // Open DevTools automatically
  args: [
    '--no-sandbox',
    '--disable-setuid-sandbox',
    '--disable-dev-shm-usage',
    '--disable-accelerated-2d-canvas',
    '--disable-gpu',
    '--window-size=1920,1080'
  ],
  defaultViewport: {
    width: 1920,
    height: 1080
  }
});

// Navigate to URL
await page.goto('https://example.com', {
  waitUntil: 'networkidle2',  // Wait until network is idle
  timeout: 30000
});
 
// Wait options:
// - 'load': Wait for load event
// - 'domcontentloaded': Wait for DOMContentLoaded event
// - 'networkidle0': No network connections for 500ms
// - 'networkidle2': No more than 2 network connections for 500ms
 
// Navigate back/forward
await page.goBack();
await page.goForward();
 
// Reload page
await page.reload({ waitUntil: 'networkidle2' });

// Single element
const element = await page.$('selector');
 
// Multiple elements
const elements = await page.$$('selector');
 
// Wait for element
const element = await page.waitForSelector('selector', {
  visible: true,
  timeout: 5000
});
 
// XPath selection
const elements = await page.$x('//xpath/expression');

// Get text content
const text = await page.$eval('selector', el => el.textContent);
 
// Get attribute
const href = await page.$eval('a', el => el.getAttribute('href'));
 
// Multiple elements
const texts = await page.$$eval('.items', elements =>
  elements.map(el => el.textContent)
);
 
// Execute arbitrary JavaScript
const result = await page.evaluate(() => {
  return document.title;
});

await page.click('button#submit');
 
// Click with options
await page.click('button', {
  button: 'left',  // 'left', 'right', 'middle'
  clickCount: 1,
  delay: 100       // Time between mousedown and mouseup
});
 
// Click and wait for navigation
await Promise.all([
  page.waitForNavigation(),
  page.click('a.nav-link')
]);

// Type text
await page.type('input#username', 'myuser', { delay: 50 });
 
// Clear and type
await page.click('input#username', { clickCount: 3 });
await page.type('input#username', 'newvalue');
 
// Press keys
await page.keyboard.press('Enter');
await page.keyboard.down('Shift');
await page.keyboard.press('Tab');
await page.keyboard.up('Shift');

// Select dropdown
await page.select('select#country', 'us');
 
// Check checkbox
await page.click('input[type="checkbox"]');
 
// File upload
const inputFile = await page.$('input[type="file"]');
await inputFile.uploadFile('/path/to/file.pdf');

// Wait for selector
await page.waitForSelector('.loaded');
 
// Wait for selector to disappear
await page.waitForSelector('.loading', { hidden: true });
 
// Wait for function
await page.waitForFunction(
  () => document.querySelector('.count').textContent === '10'
);
 
// Wait for navigation
await page.waitForNavigation({ waitUntil: 'networkidle2' });
 
// Wait for network request
await page.waitForRequest(request =>
  request.url().includes('/api/data')
);
 
// Wait for network response
await page.waitForResponse(response =>
  response.url().includes('/api/data') && response.status() === 200
);
 
// Fixed timeout (use sparingly)
await page.waitForTimeout(1000);

// Full page screenshot
await page.screenshot({
  path: 'screenshot.png',
  fullPage: true
});
 
// Element screenshot
const element = await page.$('.chart');
await element.screenshot({ path: 'chart.png' });
 
// Screenshot options
await page.screenshot({
  path: 'screenshot.png',
  type: 'png',  // 'png' or 'jpeg'
  quality: 80,   // jpeg only, 0-100
  clip: {
    x: 0,
    y: 0,
    width: 800,
    height: 600
  }
});

await page.pdf({
  path: 'document.pdf',
  format: 'A4',
  printBackground: true,
  margin: {
    top: '20px',
    right: '20px',
    bottom: '20px',
    left: '20px'
  }
});

// Enable request interception
await page.setRequestInterception(true);
 
page.on('request', request => {
  // Block images and stylesheets
  if (['image', 'stylesheet'].includes(request.resourceType())) {
    request.abort();
  } else {
    request.continue();
  }
});
 
// Modify requests
page.on('request', request => {
  request.continue({
    headers: {
      ...request.headers(),
      'X-Custom-Header': 'value'
    }
  });
});
 
// Monitor responses
page.on('response', async response => {
  if (response.url().includes('/api/')) {
    const data = await response.json();
    console.log('API Response:', data);
  }
});

// Basic HTTP authentication
await page.authenticate({
  username: 'user',
  password: 'pass'
});
 
// Set cookies
await page.setCookie({
  name: 'session',
  value: 'abc123',
  domain: 'example.com'
});
 
// Get cookies
const cookies = await page.cookies();
 
// Clear cookies
await page.deleteCookie({ name: 'session' });

// Create incognito context
const context = await browser.createIncognitoBrowserContext();
const page = await context.newPage();
 
// Multiple pages
const page1 = await browser.newPage();
const page2 = await browser.newPage();
 
// Get all pages
const pages = await browser.pages();
 
// Handle popups
page.on('popup', async popup => {
  await popup.waitForLoadState();
  console.log('Popup URL:', popup.url());
});

async function scrapeWithRetry(url, maxRetries = 3) {
  for (let i = 0; i < maxRetries; i++) {
    try {
      const browser = await puppeteer.launch();
      const page = await browser.newPage();
 
      // Set timeout
      page.setDefaultTimeout(30000);
 
      await page.goto(url, { waitUntil: 'networkidle2' });
      const data = await page.$eval('.content', el => el.textContent);
 
      await browser.close();
      return data;
    } catch (error) {
      console.error(`Attempt ${i + 1} failed:`, error.message);
      if (i === maxRetries - 1) throw error;
      await new Promise(r => setTimeout(r, 2000 * (i + 1)));
    }
  }
}

// Disable unnecessary features
await page.setRequestInterception(true);
page.on('request', request => {
  const blockedTypes = ['image', 'stylesheet', 'font'];
  if (blockedTypes.includes(request.resourceType())) {
    request.abort();
  } else {
    request.continue();
  }
});
 
// Reuse browser instance
const browser = await puppeteer.launch();
 
async function scrape(url) {
  const page = await browser.newPage();
  try {
    await page.goto(url);
    // ... scraping logic
  } finally {
    await page.close();  // Close page, not browser
  }
}
 
// Use connection pool for parallel scraping
const cluster = require('puppeteer-cluster');