scrapfly-sdk-rust

use scrapfly_sdk::{Client, ScrapeConfig, Format, ProxyPool};
 
let config = ScrapeConfig::builder("https://web-scraping.dev/products")
    .unblocker(true)                             // bypass anti-bot
    .country("us")                               // proxy country (ISO 3166-1 alpha-2)
    .proxy_pool(ProxyPool::PublicResidentialPool) // or PublicDatacenterPool
    .render_js(true)                             // headless browser rendering
    .wait_for_selector("div.product")            // requires render_js
    .rendering_wait(5000)                        // ms, requires render_js
    .format(Format::Markdown)                    // clean, LLM-friendly content
    .build()?;
 
let result = client.scrape(&config).await?;

use scrapfly_sdk::{Client, ScreenshotConfig, ExtractionConfig};
 
let shot = client
    .screenshot(&ScreenshotConfig::builder("https://web-scraping.dev/products").build()?)
    .await?;
// shot.image is the raw bytes; shot.save("products", None)? writes it to disk
 
let html = result.result.content.into_bytes();
let extraction = ExtractionConfig::builder(html, "text/html")
    .extraction_prompt("extract all product names and prices")
    .build()?;
let data = client.extract(&extraction).await?;
println!("{}", serde_json::to_string_pretty(&data.data)?);

use futures_util::StreamExt;
 
let configs = vec![
    ScrapeConfig::builder("https://web-scraping.dev/products?page=1").build()?,
    ScrapeConfig::builder("https://web-scraping.dev/products?page=2").build()?,
];
 
let mut stream = client.concurrent_scrape(configs, 5); // concurrency = 5
while let Some(result) = stream.next().await {
    let result = result?;
    println!("{}", result.result.content.len());
}