URL to Markdown API for LLM and RAG ingestion

# Convert one page to markdown, split into ~1,024-token chunks (1 credit)
curl -X POST "https://api.searlo.tech/api/v1/scrape" \
  -H "x-api-key: YOUR_API_KEY" \
  -H "content-type: application/json" \
  -d '{
    "url": "https://example.com/blog/how-we-ship",
    "formats": ["markdown"],
    "onlyMainContent": true,
    "excludeSelectors": [".newsletter-signup", "#comments"],
    "chunk": true,
    "chunkMaxTokens": 1024
  }'

# Accept a cached copy only if it is under an hour old.
# Served from cache: 1 credit. Fetched live because of maxAge: 2 credits.
curl -X POST "https://api.searlo.tech/api/v1/scrape" \
  -H "x-api-key: YOUR_API_KEY" \
  -H "content-type: application/json" \
  -d '{ "url": "https://example.com/pricing", "formats": ["markdown"], "maxAge": 3600 }'

import requests

r = requests.post(
    "https://api.searlo.tech/api/v1/scrape",
    headers={"x-api-key": "YOUR_API_KEY"},
    json={
        "url": "https://example.com/blog/how-we-ship",
        "formats": ["markdown"],
        "chunk": True,
        "chunkMaxTokens": 1024,
    },
    timeout=90,
)
r.raise_for_status()
page = r.json()

meta = page["metadata"]
source = meta["canonicalUrl"] or page["url"]
print(meta["title"], "|", meta["wordCount"], "words |", page["via"])
print("cache:", r.headers.get("X-Cache"), "| credits:", page["credits"])

# One record per chunk, each carrying its citation back to the page.
records = [
    {
        "id": f"{source}#{chunk['index']}",
        "text": chunk["text"],
        "tokens_estimate": chunk["tokens"],
        "source": source,
        "title": meta["title"],
    }
    for chunk in page.get("chunks", [])
]
print(len(records), "chunks ready to embed")

const res = await fetch("https://api.searlo.tech/api/v1/scrape", {
  method: "POST",
  headers: { "x-api-key": "YOUR_API_KEY", "content-type": "application/json" },
  body: JSON.stringify({
    url: "https://example.com/blog/how-we-ship",
    formats: ["markdown"],
    chunk: true,
    chunkMaxTokens: 1024,
  }),
});
if (!res.ok) throw new Error(`scrape failed: ${res.status}`); // failures are not billed
const page = await res.json();

console.log(page.metadata.title, page.via, res.headers.get("x-cache"), page.credits);

for (const chunk of page.chunks ?? []) {
  // tokens is an estimate (about 1.3 per word), not your model's tokenizer.
  console.log(`#${chunk.index} ~${chunk.tokens} tokens`, chunk.text.slice(0, 80));
}

{
  "searchParameters": {
    "url": "https://example.com/blog/how-we-ship",
    "type": "scrape",
    "formats": ["markdown"]
  },
  "url": "https://example.com/blog/how-we-ship",
  "requestedUrl": "https://example.com/blog/how-we-ship",
  "statusCode": 200,
  "via": "http",
  "bytes": 148213,
  "markdown": "# How we ship\n\nWe deploy small changes many times a day...\n\n## Every change starts as a draft\n\n...",
  "chunks": [
    { "index": 0, "text": "# How we ship\n\nWe deploy small changes many times a day...", "tokens": 1012 },
    { "index": 1, "text": "## Review happens in the open\n\n...", "tokens": 987 }
  ],
  "metadata": {
    "title": "How we ship | Example Blog",
    "description": "The release process behind a small engineering team.",
    "author": "Jane Doe",
    "publishedDate": "2026-08-14T09:00:00Z",
    "language": "en",
    "canonicalUrl": "https://example.com/blog/how-we-ship",
    "robots": null,
    "wordCount": 1874,
    "openGraph": { "title": "How we ship", "type": "article", "image": "https://example.com/og/how-we-ship.png" },
    "twitter": null
  },
  "links": [
    { "url": "https://example.com/blog", "text": "Blog", "rel": null, "internal": true }
  ],
  "source": "searlo.tech(scrape)",
  "credits": 1
}