Web Crawler API: crawl a whole site, pay per page fetched

# 1. See what the site publishes before you pay to crawl it (1 credit)
curl -X POST "https://api.searlo.tech/api/v1/map" \
  -H "x-api-key: YOUR_API_KEY" \
  -H "content-type: application/json" \
  -d '{ "url": "https://docs.example.com", "search": "/guides/", "limit": 5000 }'

# 2. Start the crawl: 202 Accepted with a job id, billed per page as it runs
curl -X POST "https://api.searlo.tech/api/v1/crawl" \
  -H "x-api-key: YOUR_API_KEY" \
  -H "content-type: application/json" \
  -d '{
    "url": "https://docs.example.com/guides/",
    "limit": 500,
    "maxDepth": 3,
    "includePaths": ["/guides/**"],
    "excludePaths": ["/guides/archive/**"],
    "formats": ["markdown", "links"],
    "webhook": { "url": "https://your-app.example.com/hooks/crawl", "secret": "YOUR_WEBHOOK_SECRET" }
  }'

# 3. Check progress (free)
curl "https://api.searlo.tech/api/v1/crawl/crawl_5f2c9e0b7a41d3e8c6b1f0a2" \
  -H "x-api-key: YOUR_API_KEY"

# 4. Page through the stored pages, up to 200 per call (free)
curl "https://api.searlo.tech/api/v1/crawl/crawl_5f2c9e0b7a41d3e8c6b1f0a2/results?limit=200" \
  -H "x-api-key: YOUR_API_KEY"

import time
import requests

API = "https://api.searlo.tech/api/v1"
HEADERS = {"x-api-key": "YOUR_API_KEY"}

# 1. Size the job before paying for it (1 credit).
site = requests.post(f"{API}/map", headers=HEADERS,
                     json={"url": "https://docs.example.com", "limit": 5000}).json()
print(site["totalResults"], "URLs found via", site["discovery"])

# 2. Start the crawl. Credits are charged per page fetched, not per page allowed.
job = requests.post(f"{API}/crawl", headers=HEADERS, json={
    "url": "https://docs.example.com/guides/",
    "limit": 500,
    "maxDepth": 3,
    "includePaths": ["/guides/**"],
    "formats": ["markdown"],
}).json()
job_id = job["jobId"]

# 3. Poll until the job stops. Status reads are free.
while True:
    status = requests.get(f"{API}/crawl/{job_id}", headers=HEADERS).json()
    if status["status"] in ("completed", "failed", "cancelled"):
        break
    time.sleep(10)
print(status["progress"], status["creditsCharged"], status["stopReason"])

# 4. Read every stored page, 200 at a time. Also free: pages were billed when fetched.
cursor = None
while True:
    params = {"limit": 200, **({"cursor": cursor} if cursor else {})}
    batch = requests.get(f"{API}/crawl/{job_id}/results",
                         headers=HEADERS, params=params).json()
    for page in batch["pages"]:
        print(page["url"], page["statusCode"], page["wordCount"])
    if not batch["hasMore"]:
        break
    cursor = batch["nextCursor"]

const API = "https://api.searlo.tech/api/v1";
const headers = { "x-api-key": "YOUR_API_KEY", "content-type": "application/json" };
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

// Start the crawl: HTTP 202 with a job id.
const job = await fetch(`${API}/crawl`, {
  method: "POST",
  headers,
  body: JSON.stringify({
    url: "https://docs.example.com/guides/",
    limit: 500,
    maxDepth: 3,
    includePaths: ["/guides/**"],
    formats: ["markdown"],
  }),
}).then((r) => r.json());

// Poll the job (free) until it stops.
let status;
do {
  await sleep(10_000);
  status = await fetch(`${API}/crawl/${job.jobId}`, { headers }).then((r) => r.json());
} while (status.status === "queued" || status.status === "running");

console.log(status.progress, status.creditsCharged, status.stopReason);

// Page through the stored results with the cursor (free).
let cursor = null;
do {
  const qs = new URLSearchParams({ limit: "200", ...(cursor && { cursor }) });
  const batch = await fetch(`${API}/crawl/${job.jobId}/results?${qs}`, { headers })
    .then((r) => r.json());
  for (const page of batch.pages) console.log(page.url, page.title, page.wordCount);
  cursor = batch.hasMore ? batch.nextCursor : null;
} while (cursor);

{
  "success": true,
  "jobId": "crawl_5f2c9e0b7a41d3e8c6b1f0a2",
  "status": "queued",
  "url": "https://docs.example.com/guides",
  "options": {
    "maxDepth": 3,
    "limit": 500,
    "includePaths": ["/guides/**"],
    "excludePaths": ["/guides/archive/**"],
    "formats": ["markdown", "links"],
    "allowSubdomains": false,
    "allowExternal": false,
    "respectRobots": true,
    "concurrency": 3,
    "render": null,
    "onlyMainContent": true,
    "useSitemap": true,
    "storeContent": true
  },
  "maxCredits": 500,
  "statusUrl": "/api/v1/crawl/crawl_5f2c9e0b7a41d3e8c6b1f0a2",
  "resultsUrl": "/api/v1/crawl/crawl_5f2c9e0b7a41d3e8c6b1f0a2/results",
  "createdAt": "2026-09-29T09:00:00.000Z"
}

{
  "success": true,
  "jobId": "crawl_5f2c9e0b7a41d3e8c6b1f0a2",
  "status": "completed",
  "url": "https://docs.example.com/guides",
  "options": { "maxDepth": 3, "limit": 500, "formats": ["markdown", "links"] },
  "progress": { "discovered": 212, "completed": 187, "failed": 3, "skipped": 22 },
  "creditsCharged": 187,
  "stopReason": null,
  "webhook": {
    "url": "https://your-app.example.com/hooks/crawl",
    "deliveredAt": "2026-09-29T09:06:41.000Z",
    "attempts": 1
  },
  "createdAt": "2026-09-29T09:00:00.000Z",
  "startedAt": "2026-09-29T09:00:02.000Z",
  "finishedAt": "2026-09-29T09:06:40.000Z"
}

{
  "success": true,
  "jobId": "crawl_5f2c9e0b7a41d3e8c6b1f0a2",
  "status": "completed",
  "totalResults": 1,
  "hasMore": true,
  "nextCursor": "66f91a2b3c4d5e6f7a8b9c0d",
  "pages": [
    {
      "url": "https://docs.example.com/guides/install",
      "finalUrl": null,
      "depth": 1,
      "parentUrl": "https://docs.example.com/guides",
      "status": "completed",
      "statusCode": 200,
      "via": "http",
      "title": "Install the CLI",
      "description": "Install and configure the command-line tool.",
      "language": "en",
      "canonicalUrl": "https://docs.example.com/guides/install",
      "wordCount": 842,
      "markdown": "# Install the CLI\n\nRun the installer for your platform, then sign in...",
      "links": ["https://docs.example.com/guides/configure"],
      "fetchedAt": "2026-09-29T09:02:11.000Z"
    }
  ]
}

{
  "searchParameters": { "url": "https://docs.example.com/", "type": "map", "limit": 5000, "search": "/guides/" },
  "origin": "https://docs.example.com/",
  "discovery": "sitemap",
  "sitemaps": ["https://docs.example.com/sitemap.xml"],
  "robots": { "present": true, "sitemapCount": 1 },
  "totalResults": 2,
  "urls": [
    { "url": "https://docs.example.com/guides/install", "lastModified": "2026-09-12" },
    { "url": "https://docs.example.com/guides/configure", "lastModified": "2026-08-30" }
  ],
  "source": "searlo.tech(map)",
  "credits": 1
}