Camel AI

pip install "camel-ai[web-tools]"

export FIRECRAWL_API_KEY="fc-your-api-key"

from camel.loaders import Firecrawl

firecrawl = Firecrawl()

response = firecrawl.crawl(url="https://www.camel-ai.org/about")
print(response["status"])  # "completed" when the crawl has finished

print(response["data"][0]["markdown"])

from camel.loaders import Firecrawl

firecrawl = Firecrawl()

response = firecrawl.scrape(url="https://www.camel-ai.org/about")
print(response["markdown"])

from pydantic import BaseModel
from camel.loaders import Firecrawl


class About(BaseModel):
    mission: str


firecrawl = Firecrawl()

print(firecrawl.map_site(url="https://www.camel-ai.org")["links"][:5])
print(firecrawl.structured_scrape(url="https://www.camel-ai.org/about", response_format=About))