URL features for your customers

import os, requests

def import_page(job_id: str, tenant_id: str, url: str) -> str:
    res = requests.post("https://urlpipe.dev/scrape",
        headers={
            "Authorization": f"Bearer {os.environ['URLPIPE_API_KEY']}",
            "Idempotency-Key": f"import-{job_id}",
        },
        json={
            "url": url,
            "operations": ["markdown", "meta"],
            "report_to": "https://app.example.com/webhooks/urlpipe",
            "labels": {"tenant": tenant_id, "feature": "import"},
        }, timeout=30)

    if res.status_code == 429:           # rate_limited or concurrency_limit
        raise RetryLater(res.json()["error"])
    res.raise_for_status()
    return res.json()["token"]

export async function importPage(jobId, tenantId, url) {
  const res = await fetch("https://urlpipe.dev/scrape", {
    method: "POST",
    headers: {
      Authorization: `Bearer ${process.env.URLPIPE_API_KEY}`,
      "Content-Type": "application/json",
      "Idempotency-Key": `import-${jobId}`,
    },
    body: JSON.stringify({
      url,
      operations: ["markdown", "meta"],
      report_to: "https://app.example.com/webhooks/urlpipe",
      labels: { tenant: tenantId, feature: "import" },
    }),
  });

  if (res.status === 429) {
    // rate_limited or concurrency_limit: back off and let the queue retry
    throw new RetryLater((await res.json()).error);
  }
  if (!res.ok) throw new Error(`URLpipe ${res.status}`);
  return (await res.json()).token;
}

curl -X POST https://urlpipe.dev/scrape \
  -H "Authorization: Bearer $URLPIPE_API_KEY" \
  -H "Idempotency-Key: import-job-58213" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://customer-site.example/about",
    "operations": ["markdown", "meta"],
    "report_to": "https://app.example.com/webhooks/urlpipe",
    "labels": {"tenant": "t_8812", "feature": "import"}
  }'