AnyCrawl: Công cụ Web Scraping và SERP API tối ưu cho RAG & LLM Application

// Một dòng code, bạn có toàn bộ SERP từ Google
const results = await anycrawl.search({
  query: "best LLM frameworks 2026",
  engine: "google",
  pages: 2
});

git clone https://github.com/any4ai/AnyCrawl.git
cd AnyCrawl

# Cài pnpm nếu chưa có
npm install -g pnpm

# Cài dependencies
pnpm install

cp .env.example .env

# API Configuration
NODE_ENV=production
ANYCRAWL_PORT=3000

# Database (optional, cho self-hosting)
DATABASE_URL=postgresql://user:password@localhost:5432/anycrawl

# Redis (dùng cho caching và queue)
REDIS_URL=redis://localhost:6379

# Proxy (optional)
ANYCRAWL_PROXY_URL=http://proxy:port

pnpm start

import axios from 'axios';

const API_KEY = process.env.ANYCRAWL_API_KEY;
const API_BASE = 'https://api.anycrawl.dev';

// Scrape một trang
async function scrapeToMarkdown(url: string) {
  try {
    const response = await axios.post(
      `${API_BASE}/v1/scrape`,
      {
        url: url,
        engine: 'playwright', // JS rendering
        formats: ['markdown'], // Output format
      },
      {
        headers: {
          'Authorization': `Bearer ${API_KEY}`,
          'Content-Type': 'application/json',
        },
      }
    );

    // Kết quả chứa markdown sạch
    const markdown = response.data.markdown;
    console.log('Markdown Output:');
    console.log(markdown);

    return markdown;
  } catch (error) {
    console.error('Error scraping:', error.message);
  }
}

// Gọi function
scrapeToMarkdown('https://docs.example.com/getting-started');

import axios from 'axios';

const API_KEY = process.env.ANYCRAWL_API_KEY;
const API_BASE = 'https://api.anycrawl.dev';

async function extractProductData(url: string) {
  const schema = {
    type: 'object',
    properties: {
      product_name: { type: 'string' },
      price: { type: 'number' },
      rating: { type: 'number' },
      review_count: { type: 'number' },
      description: { type: 'string' },
      in_stock: { type: 'boolean' },
    },
    required: ['product_name', 'price', 'rating'],
  };

  try {
    const response = await axios.post(
      `${API_BASE}/v1/scrape`,
      {
        url: url,
        engine: 'playwright',
        json_options: {
          schema: schema,
        },
      },
      {
        headers: {
          'Authorization': `Bearer ${API_KEY}`,
          'Content-Type': 'application/json',
        },
      }
    );

    const productData = response.data.json; // Structured JSON
    console.log('Extracted Product Data:');
    console.log(JSON.stringify(productData, null, 2));

    return productData;
  } catch (error) {
    console.error('Error extracting data:', error.message);
  }
}

// Gọi function
extractProductData('https://example-store.com/product/laptop-pro');

import axios from 'axios';

const API_KEY = process.env.ANYCRAWL_API_KEY;
const API_BASE = 'https://api.anycrawl.dev';

async function crawlDocsSite(baseUrl: string) {
  try {
    const response = await axios.post(
      `${API_BASE}/v1/crawl`,
      {
        url: baseUrl,
        engine: 'playwright', // Xử lý JS
        max_depth: 3, // Giới hạn độ sâu (khôngcào vô hạn)
        limit: 100, // Max 100 trang
        strategy: 'same-domain', // Chỉ cào same domain
        include_paths: ['/docs', '/guide'], // Chỉ include /docs và /guide
        exclude_paths: ['/admin', '/settings'], // Loại bỏ admin pages
        scrape_options: {
          formats: ['markdown'], // Output format
        },
      },
      {
        headers: {
          'Authorization': `Bearer ${API_KEY}`,
          'Content-Type': 'application/json',
        },
      }
    );

    // Response chứa status và job ID (crawl chạy async)
    console.log('Crawl Job Started:');
    console.log(`Job ID: ${response.data.job_id}`);
    console.log(`Status: ${response.data.status}`);

    // Có thể polling để check status
    return response.data;
  } catch (error) {
    console.error('Error starting crawl:', error.message);
  }
}

// Gọi function
crawlDocsSite('https://docs.example.com');

import axios from 'axios';

const API_KEY = process.env.ANYCRAWL_API_KEY;
const API_BASE = 'https://api.anycrawl.dev';

async function getGoogleSerpResults(query: string, pages: number = 1) {
  try {
    const response = await axios.post(
      `${API_BASE}/v1/search`,
      {
        query: query,
        engine: 'google',
        pages: pages, // Số pages trong search results
        lang: 'vi-VN', // Ngôn ngữ Việt Nam
      },
      {
        headers: {
          'Authorization': `Bearer ${API_KEY}`,
          'Content-Type': 'application/json',
        },
      }
    );

    const results = response.data.results;
    console.log(`Found ${results.length} results for: "${query}"`);

    // Parse results
    results.forEach((result, index) => {
      console.log(`\n${index + 1}. ${result.title}`);
      console.log(`   URL: ${result.url}`);
      console.log(`   Description: ${result.description}`);
      console.log(`   Position: ${result.position}`);
    });

    return results;
  } catch (error) {
    console.error('Error fetching SERP:', error.message);
  }
}

// Gọi function
getGoogleSerpResults('best open source AI tools 2026', 2);

import axios from 'axios';

const API_KEY = process.env.ANYCRAWL_API_KEY;
const API_BASE = 'https://api.anycrawl.dev';

async function bulkScrapeUrls(urls: string[]) {
  try {
    // AnyCrawl hỗ trợ batch endpoint
    const response = await axios.post(
      `${API_BASE}/v1/batch`,
      {
        requests: urls.map((url) => ({
          url: url,
          engine: 'cheerio', // Static parsing, nhanh hơn
          formats: ['markdown'],
        })),
      },
      {
        headers: {
          'Authorization': `Bearer ${API_KEY}`,
          'Content-Type': 'application/json',
        },
      }
    );

    const batchId = response.data.batch_id;
    console.log(`Batch job created: ${batchId}`);
    console.log(`Processing ${urls.length} URLs...`);

    // Polling for results
    let completed = false;
    while (!completed) {
      const statusResponse = await axios.get(
        `${API_BASE}/v1/batch/${batchId}`,
        {
          headers: {
            'Authorization': `Bearer ${API_KEY}`,
          },
        }
      );

      const status = statusResponse.data;
      console.log(`Progress: ${status.completed}/${status.total} completed`);

      if (status.completed === status.total) {
        completed = true;
        console.log('Batch processing completed!');

        // Lấy results
        const results = status.results;
        results.forEach((result, index) => {
          console.log(`\n[${index + 1}] ${result.url}`);
          console.log(`Status: ${result.status}`);
          if (result.markdown) {
            console.log(`Content preview: ${result.markdown.substring(0, 100)}...`);
          }
        });
      } else {
        // Chờ 5 giây rồi check lại
        await new Promise((resolve) => setTimeout(resolve, 5000));
      }
    }
  } catch (error) {
    console.error('Error in bulk processing:', error.message);
  }
}

// Gọi function
const urls = [
  'https://example.com/page1',
  'https://example.com/page2',
  'https://example.com/page3',
  // ... 500+ URLs
];

bulkScrapeUrls(urls);