Crawl

Explorer plusieurs URL en fonction des options

curl --request POST \
  --url https://api.firecrawl.dev/v1/crawl \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "url": "<string>",
  "allowBackwardLinks": false,
  "allowExternalLinks": false,
  "allowSubdomains": false,
  "crawlEntireDomain": false,
  "delay": 123,
  "excludePaths": [
    "<string>"
  ],
  "ignoreQueryParameters": false,
  "ignoreSitemap": false,
  "includePaths": [
    "<string>"
  ],
  "limit": 10000,
  "maxConcurrency": 123,
  "maxDepth": 10,
  "maxDiscoveryDepth": 123,
  "regexOnFullURL": false,
  "scrapeOptions": {
    "actions": [
      {
        "type": "wait",
        "milliseconds": 2,
        "selector": "#my-element"
      }
    ],
    "blockAds": true,
    "excludeTags": [
      "<string>"
    ],
    "headers": {},
    "includeTags": [
      "<string>"
    ],
    "jsonOptions": {
      "checkPromptInjection": false,
      "prompt": "<string>",
      "schema": {},
      "systemPrompt": "<string>"
    },
    "location": {
      "country": "US",
      "languages": [
        "en-US"
      ]
    },
    "maxAge": 0,
    "mobile": false,
    "onlyMainContent": true,
    "parsePDF": true,
    "removeBase64Images": true,
    "skipTlsVerification": false,
    "storeInCache": true,
    "threatProtection": {
      "blacklist": [
        "<string>"
      ],
      "blockedTlds": [
        "<string>"
      ],
      "riskScoreThreshold": 75,
      "whitelist": [
        "<string>"
      ]
    },
    "timeout": 30000,
    "waitFor": 0,
    "changeTrackingOptions": {
      "modes": [],
      "prompt": "<string>",
      "schema": {},
      "tag": null
    },
    "formats": [
      "markdown"
    ]
  },
  "zeroDataRetention": false
}
'

import requests

url = "https://api.firecrawl.dev/v1/crawl"

payload = {
    "url": "<string>",
    "allowBackwardLinks": False,
    "allowExternalLinks": False,
    "allowSubdomains": False,
    "crawlEntireDomain": False,
    "delay": 123,
    "excludePaths": ["<string>"],
    "ignoreQueryParameters": False,
    "ignoreSitemap": False,
    "includePaths": ["<string>"],
    "limit": 10000,
    "maxConcurrency": 123,
    "maxDepth": 10,
    "maxDiscoveryDepth": 123,
    "regexOnFullURL": False,
    "scrapeOptions": {
        "actions": [
            {
                "type": "wait",
                "milliseconds": 2,
                "selector": "#my-element"
            }
        ],
        "blockAds": True,
        "excludeTags": ["<string>"],
        "headers": {},
        "includeTags": ["<string>"],
        "jsonOptions": {
            "checkPromptInjection": False,
            "prompt": "<string>",
            "schema": {},
            "systemPrompt": "<string>"
        },
        "location": {
            "country": "US",
            "languages": ["en-US"]
        },
        "maxAge": 0,
        "mobile": False,
        "onlyMainContent": True,
        "parsePDF": True,
        "removeBase64Images": True,
        "skipTlsVerification": False,
        "storeInCache": True,
        "threatProtection": {
            "blacklist": ["<string>"],
            "blockedTlds": ["<string>"],
            "riskScoreThreshold": 75,
            "whitelist": ["<string>"]
        },
        "timeout": 30000,
        "waitFor": 0,
        "changeTrackingOptions": {
            "modes": [],
            "prompt": "<string>",
            "schema": {},
            "tag": None
        },
        "formats": ["markdown"]
    },
    "zeroDataRetention": False
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    url: '<string>',
    allowBackwardLinks: false,
    allowExternalLinks: false,
    allowSubdomains: false,
    crawlEntireDomain: false,
    delay: 123,
    excludePaths: ['<string>'],
    ignoreQueryParameters: false,
    ignoreSitemap: false,
    includePaths: ['<string>'],
    limit: 10000,
    maxConcurrency: 123,
    maxDepth: 10,
    maxDiscoveryDepth: 123,
    regexOnFullURL: false,
    scrapeOptions: {
      actions: [{type: 'wait', milliseconds: 2, selector: '#my-element'}],
      blockAds: true,
      excludeTags: ['<string>'],
      headers: {},
      includeTags: ['<string>'],
      jsonOptions: {
        checkPromptInjection: false,
        prompt: '<string>',
        schema: {},
        systemPrompt: '<string>'
      },
      location: {country: 'US', languages: ['en-US']},
      maxAge: 0,
      mobile: false,
      onlyMainContent: true,
      parsePDF: true,
      removeBase64Images: true,
      skipTlsVerification: false,
      storeInCache: true,
      threatProtection: {
        blacklist: ['<string>'],
        blockedTlds: ['<string>'],
        riskScoreThreshold: 75,
        whitelist: ['<string>']
      },
      timeout: 30000,
      waitFor: 0,
      changeTrackingOptions: {modes: [], prompt: '<string>', schema: {}, tag: null},
      formats: ['markdown']
    },
    zeroDataRetention: false
  })
};

fetch('https://api.firecrawl.dev/v1/crawl', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.firecrawl.dev/v1/crawl",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'url' => '<string>',
    'allowBackwardLinks' => false,
    'allowExternalLinks' => false,
    'allowSubdomains' => false,
    'crawlEntireDomain' => false,
    'delay' => 123,
    'excludePaths' => [
        '<string>'
    ],
    'ignoreQueryParameters' => false,
    'ignoreSitemap' => false,
    'includePaths' => [
        '<string>'
    ],
    'limit' => 10000,
    'maxConcurrency' => 123,
    'maxDepth' => 10,
    'maxDiscoveryDepth' => 123,
    'regexOnFullURL' => false,
    'scrapeOptions' => [
        'actions' => [
                [
                                'type' => 'wait',
                                'milliseconds' => 2,
                                'selector' => '#my-element'
                ]
        ],
        'blockAds' => true,
        'excludeTags' => [
                '<string>'
        ],
        'headers' => [
                
        ],
        'includeTags' => [
                '<string>'
        ],
        'jsonOptions' => [
                'checkPromptInjection' => false,
                'prompt' => '<string>',
                'schema' => [
                                
                ],
                'systemPrompt' => '<string>'
        ],
        'location' => [
                'country' => 'US',
                'languages' => [
                                'en-US'
                ]
        ],
        'maxAge' => 0,
        'mobile' => false,
        'onlyMainContent' => true,
        'parsePDF' => true,
        'removeBase64Images' => true,
        'skipTlsVerification' => false,
        'storeInCache' => true,
        'threatProtection' => [
                'blacklist' => [
                                '<string>'
                ],
                'blockedTlds' => [
                                '<string>'
                ],
                'riskScoreThreshold' => 75,
                'whitelist' => [
                                '<string>'
                ]
        ],
        'timeout' => 30000,
        'waitFor' => 0,
        'changeTrackingOptions' => [
                'modes' => [
                                
                ],
                'prompt' => '<string>',
                'schema' => [
                                
                ],
                'tag' => null
        ],
        'formats' => [
                'markdown'
        ]
    ],
    'zeroDataRetention' => false
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.firecrawl.dev/v1/crawl"

	payload := strings.NewReader("{\n  \"url\": \"<string>\",\n  \"allowBackwardLinks\": false,\n  \"allowExternalLinks\": false,\n  \"allowSubdomains\": false,\n  \"crawlEntireDomain\": false,\n  \"delay\": 123,\n  \"excludePaths\": [\n    \"<string>\"\n  ],\n  \"ignoreQueryParameters\": false,\n  \"ignoreSitemap\": false,\n  \"includePaths\": [\n    \"<string>\"\n  ],\n  \"limit\": 10000,\n  \"maxConcurrency\": 123,\n  \"maxDepth\": 10,\n  \"maxDiscoveryDepth\": 123,\n  \"regexOnFullURL\": false,\n  \"scrapeOptions\": {\n    \"actions\": [\n      {\n        \"type\": \"wait\",\n        \"milliseconds\": 2,\n        \"selector\": \"#my-element\"\n      }\n    ],\n    \"blockAds\": true,\n    \"excludeTags\": [\n      \"<string>\"\n    ],\n    \"headers\": {},\n    \"includeTags\": [\n      \"<string>\"\n    ],\n    \"jsonOptions\": {\n      \"checkPromptInjection\": false,\n      \"prompt\": \"<string>\",\n      \"schema\": {},\n      \"systemPrompt\": \"<string>\"\n    },\n    \"location\": {\n      \"country\": \"US\",\n      \"languages\": [\n        \"en-US\"\n      ]\n    },\n    \"maxAge\": 0,\n    \"mobile\": false,\n    \"onlyMainContent\": true,\n    \"parsePDF\": true,\n    \"removeBase64Images\": true,\n    \"skipTlsVerification\": false,\n    \"storeInCache\": true,\n    \"threatProtection\": {\n      \"blacklist\": [\n        \"<string>\"\n      ],\n      \"blockedTlds\": [\n        \"<string>\"\n      ],\n      \"riskScoreThreshold\": 75,\n      \"whitelist\": [\n        \"<string>\"\n      ]\n    },\n    \"timeout\": 30000,\n    \"waitFor\": 0,\n    \"changeTrackingOptions\": {\n      \"modes\": [],\n      \"prompt\": \"<string>\",\n      \"schema\": {},\n      \"tag\": null\n    },\n    \"formats\": [\n      \"markdown\"\n    ]\n  },\n  \"zeroDataRetention\": false\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.firecrawl.dev/v1/crawl")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"url\": \"<string>\",\n  \"allowBackwardLinks\": false,\n  \"allowExternalLinks\": false,\n  \"allowSubdomains\": false,\n  \"crawlEntireDomain\": false,\n  \"delay\": 123,\n  \"excludePaths\": [\n    \"<string>\"\n  ],\n  \"ignoreQueryParameters\": false,\n  \"ignoreSitemap\": false,\n  \"includePaths\": [\n    \"<string>\"\n  ],\n  \"limit\": 10000,\n  \"maxConcurrency\": 123,\n  \"maxDepth\": 10,\n  \"maxDiscoveryDepth\": 123,\n  \"regexOnFullURL\": false,\n  \"scrapeOptions\": {\n    \"actions\": [\n      {\n        \"type\": \"wait\",\n        \"milliseconds\": 2,\n        \"selector\": \"#my-element\"\n      }\n    ],\n    \"blockAds\": true,\n    \"excludeTags\": [\n      \"<string>\"\n    ],\n    \"headers\": {},\n    \"includeTags\": [\n      \"<string>\"\n    ],\n    \"jsonOptions\": {\n      \"checkPromptInjection\": false,\n      \"prompt\": \"<string>\",\n      \"schema\": {},\n      \"systemPrompt\": \"<string>\"\n    },\n    \"location\": {\n      \"country\": \"US\",\n      \"languages\": [\n        \"en-US\"\n      ]\n    },\n    \"maxAge\": 0,\n    \"mobile\": false,\n    \"onlyMainContent\": true,\n    \"parsePDF\": true,\n    \"removeBase64Images\": true,\n    \"skipTlsVerification\": false,\n    \"storeInCache\": true,\n    \"threatProtection\": {\n      \"blacklist\": [\n        \"<string>\"\n      ],\n      \"blockedTlds\": [\n        \"<string>\"\n      ],\n      \"riskScoreThreshold\": 75,\n      \"whitelist\": [\n        \"<string>\"\n      ]\n    },\n    \"timeout\": 30000,\n    \"waitFor\": 0,\n    \"changeTrackingOptions\": {\n      \"modes\": [],\n      \"prompt\": \"<string>\",\n      \"schema\": {},\n      \"tag\": null\n    },\n    \"formats\": [\n      \"markdown\"\n    ]\n  },\n  \"zeroDataRetention\": false\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.firecrawl.dev/v1/crawl")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"url\": \"<string>\",\n  \"allowBackwardLinks\": false,\n  \"allowExternalLinks\": false,\n  \"allowSubdomains\": false,\n  \"crawlEntireDomain\": false,\n  \"delay\": 123,\n  \"excludePaths\": [\n    \"<string>\"\n  ],\n  \"ignoreQueryParameters\": false,\n  \"ignoreSitemap\": false,\n  \"includePaths\": [\n    \"<string>\"\n  ],\n  \"limit\": 10000,\n  \"maxConcurrency\": 123,\n  \"maxDepth\": 10,\n  \"maxDiscoveryDepth\": 123,\n  \"regexOnFullURL\": false,\n  \"scrapeOptions\": {\n    \"actions\": [\n      {\n        \"type\": \"wait\",\n        \"milliseconds\": 2,\n        \"selector\": \"#my-element\"\n      }\n    ],\n    \"blockAds\": true,\n    \"excludeTags\": [\n      \"<string>\"\n    ],\n    \"headers\": {},\n    \"includeTags\": [\n      \"<string>\"\n    ],\n    \"jsonOptions\": {\n      \"checkPromptInjection\": false,\n      \"prompt\": \"<string>\",\n      \"schema\": {},\n      \"systemPrompt\": \"<string>\"\n    },\n    \"location\": {\n      \"country\": \"US\",\n      \"languages\": [\n        \"en-US\"\n      ]\n    },\n    \"maxAge\": 0,\n    \"mobile\": false,\n    \"onlyMainContent\": true,\n    \"parsePDF\": true,\n    \"removeBase64Images\": true,\n    \"skipTlsVerification\": false,\n    \"storeInCache\": true,\n    \"threatProtection\": {\n      \"blacklist\": [\n        \"<string>\"\n      ],\n      \"blockedTlds\": [\n        \"<string>\"\n      ],\n      \"riskScoreThreshold\": 75,\n      \"whitelist\": [\n        \"<string>\"\n      ]\n    },\n    \"timeout\": 30000,\n    \"waitFor\": 0,\n    \"changeTrackingOptions\": {\n      \"modes\": [],\n      \"prompt\": \"<string>\",\n      \"schema\": {},\n      \"tag\": null\n    },\n    \"formats\": [\n      \"markdown\"\n    ]\n  },\n  \"zeroDataRetention\": false\n}"

response = http.request(request)
puts response.read_body

{
  "id": "<string>",
  "success": true,
  "url": "<string>"
}

{
  "error": "Payment required to access this resource."
}

{
  "error": "Request rate limit exceeded. Please wait and try again later."
}

{
  "error": "An unexpected error occurred on the server."
}

POST

crawl

Explorer plusieurs URL en fonction des options

curl --request POST \
  --url https://api.firecrawl.dev/v1/crawl \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "url": "<string>",
  "allowBackwardLinks": false,
  "allowExternalLinks": false,
  "allowSubdomains": false,
  "crawlEntireDomain": false,
  "delay": 123,
  "excludePaths": [
    "<string>"
  ],
  "ignoreQueryParameters": false,
  "ignoreSitemap": false,
  "includePaths": [
    "<string>"
  ],
  "limit": 10000,
  "maxConcurrency": 123,
  "maxDepth": 10,
  "maxDiscoveryDepth": 123,
  "regexOnFullURL": false,
  "scrapeOptions": {
    "actions": [
      {
        "type": "wait",
        "milliseconds": 2,
        "selector": "#my-element"
      }
    ],
    "blockAds": true,
    "excludeTags": [
      "<string>"
    ],
    "headers": {},
    "includeTags": [
      "<string>"
    ],
    "jsonOptions": {
      "checkPromptInjection": false,
      "prompt": "<string>",
      "schema": {},
      "systemPrompt": "<string>"
    },
    "location": {
      "country": "US",
      "languages": [
        "en-US"
      ]
    },
    "maxAge": 0,
    "mobile": false,
    "onlyMainContent": true,
    "parsePDF": true,
    "removeBase64Images": true,
    "skipTlsVerification": false,
    "storeInCache": true,
    "threatProtection": {
      "blacklist": [
        "<string>"
      ],
      "blockedTlds": [
        "<string>"
      ],
      "riskScoreThreshold": 75,
      "whitelist": [
        "<string>"
      ]
    },
    "timeout": 30000,
    "waitFor": 0,
    "changeTrackingOptions": {
      "modes": [],
      "prompt": "<string>",
      "schema": {},
      "tag": null
    },
    "formats": [
      "markdown"
    ]
  },
  "zeroDataRetention": false
}
'

import requests

url = "https://api.firecrawl.dev/v1/crawl"

payload = {
    "url": "<string>",
    "allowBackwardLinks": False,
    "allowExternalLinks": False,
    "allowSubdomains": False,
    "crawlEntireDomain": False,
    "delay": 123,
    "excludePaths": ["<string>"],
    "ignoreQueryParameters": False,
    "ignoreSitemap": False,
    "includePaths": ["<string>"],
    "limit": 10000,
    "maxConcurrency": 123,
    "maxDepth": 10,
    "maxDiscoveryDepth": 123,
    "regexOnFullURL": False,
    "scrapeOptions": {
        "actions": [
            {
                "type": "wait",
                "milliseconds": 2,
                "selector": "#my-element"
            }
        ],
        "blockAds": True,
        "excludeTags": ["<string>"],
        "headers": {},
        "includeTags": ["<string>"],
        "jsonOptions": {
            "checkPromptInjection": False,
            "prompt": "<string>",
            "schema": {},
            "systemPrompt": "<string>"
        },
        "location": {
            "country": "US",
            "languages": ["en-US"]
        },
        "maxAge": 0,
        "mobile": False,
        "onlyMainContent": True,
        "parsePDF": True,
        "removeBase64Images": True,
        "skipTlsVerification": False,
        "storeInCache": True,
        "threatProtection": {
            "blacklist": ["<string>"],
            "blockedTlds": ["<string>"],
            "riskScoreThreshold": 75,
            "whitelist": ["<string>"]
        },
        "timeout": 30000,
        "waitFor": 0,
        "changeTrackingOptions": {
            "modes": [],
            "prompt": "<string>",
            "schema": {},
            "tag": None
        },
        "formats": ["markdown"]
    },
    "zeroDataRetention": False
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.text)

const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: JSON.stringify({
    url: '<string>',
    allowBackwardLinks: false,
    allowExternalLinks: false,
    allowSubdomains: false,
    crawlEntireDomain: false,
    delay: 123,
    excludePaths: ['<string>'],
    ignoreQueryParameters: false,
    ignoreSitemap: false,
    includePaths: ['<string>'],
    limit: 10000,
    maxConcurrency: 123,
    maxDepth: 10,
    maxDiscoveryDepth: 123,
    regexOnFullURL: false,
    scrapeOptions: {
      actions: [{type: 'wait', milliseconds: 2, selector: '#my-element'}],
      blockAds: true,
      excludeTags: ['<string>'],
      headers: {},
      includeTags: ['<string>'],
      jsonOptions: {
        checkPromptInjection: false,
        prompt: '<string>',
        schema: {},
        systemPrompt: '<string>'
      },
      location: {country: 'US', languages: ['en-US']},
      maxAge: 0,
      mobile: false,
      onlyMainContent: true,
      parsePDF: true,
      removeBase64Images: true,
      skipTlsVerification: false,
      storeInCache: true,
      threatProtection: {
        blacklist: ['<string>'],
        blockedTlds: ['<string>'],
        riskScoreThreshold: 75,
        whitelist: ['<string>']
      },
      timeout: 30000,
      waitFor: 0,
      changeTrackingOptions: {modes: [], prompt: '<string>', schema: {}, tag: null},
      formats: ['markdown']
    },
    zeroDataRetention: false
  })
};

fetch('https://api.firecrawl.dev/v1/crawl', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.firecrawl.dev/v1/crawl",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'url' => '<string>',
    'allowBackwardLinks' => false,
    'allowExternalLinks' => false,
    'allowSubdomains' => false,
    'crawlEntireDomain' => false,
    'delay' => 123,
    'excludePaths' => [
        '<string>'
    ],
    'ignoreQueryParameters' => false,
    'ignoreSitemap' => false,
    'includePaths' => [
        '<string>'
    ],
    'limit' => 10000,
    'maxConcurrency' => 123,
    'maxDepth' => 10,
    'maxDiscoveryDepth' => 123,
    'regexOnFullURL' => false,
    'scrapeOptions' => [
        'actions' => [
                [
                                'type' => 'wait',
                                'milliseconds' => 2,
                                'selector' => '#my-element'
                ]
        ],
        'blockAds' => true,
        'excludeTags' => [
                '<string>'
        ],
        'headers' => [
                
        ],
        'includeTags' => [
                '<string>'
        ],
        'jsonOptions' => [
                'checkPromptInjection' => false,
                'prompt' => '<string>',
                'schema' => [
                                
                ],
                'systemPrompt' => '<string>'
        ],
        'location' => [
                'country' => 'US',
                'languages' => [
                                'en-US'
                ]
        ],
        'maxAge' => 0,
        'mobile' => false,
        'onlyMainContent' => true,
        'parsePDF' => true,
        'removeBase64Images' => true,
        'skipTlsVerification' => false,
        'storeInCache' => true,
        'threatProtection' => [
                'blacklist' => [
                                '<string>'
                ],
                'blockedTlds' => [
                                '<string>'
                ],
                'riskScoreThreshold' => 75,
                'whitelist' => [
                                '<string>'
                ]
        ],
        'timeout' => 30000,
        'waitFor' => 0,
        'changeTrackingOptions' => [
                'modes' => [
                                
                ],
                'prompt' => '<string>',
                'schema' => [
                                
                ],
                'tag' => null
        ],
        'formats' => [
                'markdown'
        ]
    ],
    'zeroDataRetention' => false
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.firecrawl.dev/v1/crawl"

	payload := strings.NewReader("{\n  \"url\": \"<string>\",\n  \"allowBackwardLinks\": false,\n  \"allowExternalLinks\": false,\n  \"allowSubdomains\": false,\n  \"crawlEntireDomain\": false,\n  \"delay\": 123,\n  \"excludePaths\": [\n    \"<string>\"\n  ],\n  \"ignoreQueryParameters\": false,\n  \"ignoreSitemap\": false,\n  \"includePaths\": [\n    \"<string>\"\n  ],\n  \"limit\": 10000,\n  \"maxConcurrency\": 123,\n  \"maxDepth\": 10,\n  \"maxDiscoveryDepth\": 123,\n  \"regexOnFullURL\": false,\n  \"scrapeOptions\": {\n    \"actions\": [\n      {\n        \"type\": \"wait\",\n        \"milliseconds\": 2,\n        \"selector\": \"#my-element\"\n      }\n    ],\n    \"blockAds\": true,\n    \"excludeTags\": [\n      \"<string>\"\n    ],\n    \"headers\": {},\n    \"includeTags\": [\n      \"<string>\"\n    ],\n    \"jsonOptions\": {\n      \"checkPromptInjection\": false,\n      \"prompt\": \"<string>\",\n      \"schema\": {},\n      \"systemPrompt\": \"<string>\"\n    },\n    \"location\": {\n      \"country\": \"US\",\n      \"languages\": [\n        \"en-US\"\n      ]\n    },\n    \"maxAge\": 0,\n    \"mobile\": false,\n    \"onlyMainContent\": true,\n    \"parsePDF\": true,\n    \"removeBase64Images\": true,\n    \"skipTlsVerification\": false,\n    \"storeInCache\": true,\n    \"threatProtection\": {\n      \"blacklist\": [\n        \"<string>\"\n      ],\n      \"blockedTlds\": [\n        \"<string>\"\n      ],\n      \"riskScoreThreshold\": 75,\n      \"whitelist\": [\n        \"<string>\"\n      ]\n    },\n    \"timeout\": 30000,\n    \"waitFor\": 0,\n    \"changeTrackingOptions\": {\n      \"modes\": [],\n      \"prompt\": \"<string>\",\n      \"schema\": {},\n      \"tag\": null\n    },\n    \"formats\": [\n      \"markdown\"\n    ]\n  },\n  \"zeroDataRetention\": false\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.post("https://api.firecrawl.dev/v1/crawl")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"url\": \"<string>\",\n  \"allowBackwardLinks\": false,\n  \"allowExternalLinks\": false,\n  \"allowSubdomains\": false,\n  \"crawlEntireDomain\": false,\n  \"delay\": 123,\n  \"excludePaths\": [\n    \"<string>\"\n  ],\n  \"ignoreQueryParameters\": false,\n  \"ignoreSitemap\": false,\n  \"includePaths\": [\n    \"<string>\"\n  ],\n  \"limit\": 10000,\n  \"maxConcurrency\": 123,\n  \"maxDepth\": 10,\n  \"maxDiscoveryDepth\": 123,\n  \"regexOnFullURL\": false,\n  \"scrapeOptions\": {\n    \"actions\": [\n      {\n        \"type\": \"wait\",\n        \"milliseconds\": 2,\n        \"selector\": \"#my-element\"\n      }\n    ],\n    \"blockAds\": true,\n    \"excludeTags\": [\n      \"<string>\"\n    ],\n    \"headers\": {},\n    \"includeTags\": [\n      \"<string>\"\n    ],\n    \"jsonOptions\": {\n      \"checkPromptInjection\": false,\n      \"prompt\": \"<string>\",\n      \"schema\": {},\n      \"systemPrompt\": \"<string>\"\n    },\n    \"location\": {\n      \"country\": \"US\",\n      \"languages\": [\n        \"en-US\"\n      ]\n    },\n    \"maxAge\": 0,\n    \"mobile\": false,\n    \"onlyMainContent\": true,\n    \"parsePDF\": true,\n    \"removeBase64Images\": true,\n    \"skipTlsVerification\": false,\n    \"storeInCache\": true,\n    \"threatProtection\": {\n      \"blacklist\": [\n        \"<string>\"\n      ],\n      \"blockedTlds\": [\n        \"<string>\"\n      ],\n      \"riskScoreThreshold\": 75,\n      \"whitelist\": [\n        \"<string>\"\n      ]\n    },\n    \"timeout\": 30000,\n    \"waitFor\": 0,\n    \"changeTrackingOptions\": {\n      \"modes\": [],\n      \"prompt\": \"<string>\",\n      \"schema\": {},\n      \"tag\": null\n    },\n    \"formats\": [\n      \"markdown\"\n    ]\n  },\n  \"zeroDataRetention\": false\n}")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.firecrawl.dev/v1/crawl")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"url\": \"<string>\",\n  \"allowBackwardLinks\": false,\n  \"allowExternalLinks\": false,\n  \"allowSubdomains\": false,\n  \"crawlEntireDomain\": false,\n  \"delay\": 123,\n  \"excludePaths\": [\n    \"<string>\"\n  ],\n  \"ignoreQueryParameters\": false,\n  \"ignoreSitemap\": false,\n  \"includePaths\": [\n    \"<string>\"\n  ],\n  \"limit\": 10000,\n  \"maxConcurrency\": 123,\n  \"maxDepth\": 10,\n  \"maxDiscoveryDepth\": 123,\n  \"regexOnFullURL\": false,\n  \"scrapeOptions\": {\n    \"actions\": [\n      {\n        \"type\": \"wait\",\n        \"milliseconds\": 2,\n        \"selector\": \"#my-element\"\n      }\n    ],\n    \"blockAds\": true,\n    \"excludeTags\": [\n      \"<string>\"\n    ],\n    \"headers\": {},\n    \"includeTags\": [\n      \"<string>\"\n    ],\n    \"jsonOptions\": {\n      \"checkPromptInjection\": false,\n      \"prompt\": \"<string>\",\n      \"schema\": {},\n      \"systemPrompt\": \"<string>\"\n    },\n    \"location\": {\n      \"country\": \"US\",\n      \"languages\": [\n        \"en-US\"\n      ]\n    },\n    \"maxAge\": 0,\n    \"mobile\": false,\n    \"onlyMainContent\": true,\n    \"parsePDF\": true,\n    \"removeBase64Images\": true,\n    \"skipTlsVerification\": false,\n    \"storeInCache\": true,\n    \"threatProtection\": {\n      \"blacklist\": [\n        \"<string>\"\n      ],\n      \"blockedTlds\": [\n        \"<string>\"\n      ],\n      \"riskScoreThreshold\": 75,\n      \"whitelist\": [\n        \"<string>\"\n      ]\n    },\n    \"timeout\": 30000,\n    \"waitFor\": 0,\n    \"changeTrackingOptions\": {\n      \"modes\": [],\n      \"prompt\": \"<string>\",\n      \"schema\": {},\n      \"tag\": null\n    },\n    \"formats\": [\n      \"markdown\"\n    ]\n  },\n  \"zeroDataRetention\": false\n}"

response = http.request(request)
puts response.read_body

{
  "id": "<string>",
  "success": true,
  "url": "<string>"
}

{
  "error": "Payment required to access this resource."
}

{
  "error": "Request rate limit exceeded. Please wait and try again later."
}

{
  "error": "An unexpected error occurred on the server."
}

Remarque : Une nouvelle version v2 de cette API est désormais disponible avec de nouvelles fonctionnalités et de meilleures performances.

Autorisations

Authorization

string

header

requis

Bearer authentication header of the form Bearer <token>, where <token> is your auth token.

Corps

application/json

url

string<uri>

requis

L’URL de base à partir de laquelle démarrer le crawl

allowBackwardLinks

boolean

défaut:false

obsolète

⚠️ OBSOLÈTE : utilisez plutôt « crawlEntireDomain ». Permet au robot d'exploration de suivre les liens internes vers des URL au même niveau ou parentes, et pas seulement vers des chemins enfants.

allowExternalLinks

boolean

défaut:false

Permet au crawler de suivre des liens pointant vers des sites web externes.

allowSubdomains

boolean

défaut:false

Permet au crawler de suivre les liens vers les sous-domaines du domaine principal.

crawlEntireDomain

boolean

défaut:false

Permet au crawler de suivre les liens internes vers des URL au même niveau ou de niveau supérieur, et pas seulement des chemins enfants.

false : Explore uniquement les URL plus profondes (enfants). → ex. /features/feature-1 → /features/feature-1/tips ✅ → Ne suivra pas /pricing ou / ❌

true : Explore tous les liens internes, y compris les URL au même niveau et de niveau supérieur. → ex. /features/feature-1 → /pricing, /, etc. ✅

Utilisez true pour une couverture interne plus large, au‑delà des chemins imbriqués.

delay

number

Intervalle en secondes entre deux opérations de scraping. Cela permet de respecter les limites de fréquence des sites web.

excludePaths

string[]

Motifs d’expressions régulières de chemins d’URL qui excluent du crawl les URL correspondantes. Par exemple, si vous définissez "excludePaths": ["blog/.*"] pour l’URL de base firecrawl.dev, tous les résultats correspondant à ce motif seront exclus, comme https://www.firecrawl.dev/blog/firecrawl-launch-week-1-recap. Les motifs utilisent la syntaxe des expressions régulières Rust (style RE2) : les assertions look-around et les références arrière ne sont pas prises en charge, et un motif qui ne se compile pas est rejeté avec un code 400. Chaque champ accepte au maximum 1 000 motifs de 2 000 caractères maximum chacun, et includePaths et excludePaths ne peuvent pas contenir ensemble plus de 1 000 motifs et 100 000 caractères au total. Pour le filtrage par mots-clés, envoyez un motif court par terme plutôt que de combiner les termes dans une seule longue alternance.

Maximum array length: 1000

Maximum string length: 2000

ignoreQueryParameters

boolean

défaut:false

Ne relancez pas le scraping du même chemin avec des paramètres de requête différents (ou sans aucun paramètre)

ignoreSitemap

boolean

défaut:false

Ignorer le sitemap du site lors du crawl

includePaths

string[]

Expressions régulières (regex) pour les chemins d’URL à inclure dans le crawl. Seuls les chemins qui correspondent aux motifs spécifiés seront inclus dans la réponse. Par exemple, si vous définissez "includePaths": ["blog/.*"] pour l’URL de base firecrawl.dev, seuls les résultats correspondant à ce motif seront inclus, comme https://www.firecrawl.dev/blog/firecrawl-launch-week-1-recap. Les motifs utilisent la syntaxe des expressions régulières Rust (de type RE2) : les assertions look-around et les références arrière ne sont pas prises en charge, et tout motif qui ne se compile pas est rejeté avec une erreur 400. Chaque champ accepte au maximum 1 000 motifs de 2 000 caractères chacun, et includePaths et excludePaths ne peuvent contenir ensemble plus de 1 000 motifs et 100 000 caractères au total. Pour le filtrage par mots-clés, envoyez un motif court par terme plutôt que de combiner les termes dans une seule longue alternance.

Maximum array length: 1000

Maximum string length: 2000

limit

integer

défaut:10000

Nombre maximal de pages à explorer. La limite par défaut est de 10 000.

maxConcurrency

integer

Nombre maximal d’opérations de scraping simultanées. Ce paramètre vous permet de définir une limite de parallélisme pour ce crawl. S’il n’est pas spécifié, le crawl respecte la limite de parallélisme de votre équipe.

maxDepth

integer

défaut:10

Profondeur absolue maximale à explorer à partir de la base de l’URL saisie. En pratique, il s’agit du nombre maximal de barres obliques que peut contenir le chemin (pathname) d’une URL explorée.

maxDiscoveryDepth

integer

Profondeur maximale d’exploration basée sur l’ordre de découverte. Le site racine et les pages issues du sitemap ont une profondeur de découverte de 0. Par exemple, si vous la définissez à 1 et que vous activez ignoreSitemap, seules l’URL saisie et toutes les URL liées depuis cette page seront explorées.

regexOnFullURL

boolean

défaut:false

Lorsque cette option est définie sur true, les expressions régulières includePaths et excludePaths sont évaluées par rapport à l’URL complète (y compris les paramètres de requête), et non uniquement au chemin de l’URL. Pratique lorsque vous devez filtrer des URL en fonction des chaînes de requête.

scrapeOptions

object

Show child attributes

webhook

object

Objet de spécification de webhook.

Show child attributes

zeroDataRetention

boolean

défaut:false

Si cette valeur est définie sur true, aucune donnée ne sera conservée pour ce crawl. Pour activer cette fonctionnalité, veuillez contacter help@firecrawl.dev.

Réponse

Réponse en cas de succès

string

success

boolean

url

string<uri>

Utiliser l’API

Points de terminaison de scraping

Points de terminaison de crawl

Points de terminaison de cartographie

Points de terminaison de recherche

Points de terminaison d’extraction

Points de terminaison du compte

Autorisations

Corps

Réponse