Skip to main content

Documentation Index

Fetch the complete documentation index at: https://docs.firecrawl.dev/llms.txt

Use this file to discover all available pages before exploring further.

Integrate Firecrawl with Microsoft AutoGen to give multi-agent conversations live web search, scrape, and crawl tools.

​
Setup

pip install -U "autogen-agentchat" "autogen-ext[openai]" firecrawl-py
Set your keys: 
export FIRECRAWL_API_KEY=fc-YOUR-API-KEY
export OPENAI_API_KEY=sk-YOUR-OPENAI-KEY

​
Firecrawl as an AutoGen Tool

This example wraps Firecrawl’s scrape and search as AutoGen function tools, then lets a single AssistantAgent use them to answer a question. 
import asyncio
import os
from firecrawl import FirecrawlApp
from autogen_agentchat.agents import AssistantAgent
from autogen_agentchat.ui import Console
from autogen_ext.models.openai import OpenAIChatCompletionClient

firecrawl = FirecrawlApp(api_key=os.environ["FIRECRAWL_API_KEY"])


def scrape_url(url: str) -> str:
    """Scrape a URL and return clean markdown."""
    result = firecrawl.scrape(url, formats=["markdown"])
    return result.markdown or ""


def web_search(query: str, limit: int = 5) -> list[dict]:
    """Search the web and return the top results."""
    result = firecrawl.search(query, limit=limit)
    return [
        {"title": r.title, "url": r.url, "snippet": r.description}
        for r in result.web or []
    ]


async def main() -> None:
    model = OpenAIChatCompletionClient(model="gpt-4o-mini")

    researcher = AssistantAgent(
        name="researcher",
        model_client=model,
        tools=[scrape_url, web_search],
        system_message=(
            "You are a web researcher. Use web_search to find candidate sources, "
            "then scrape_url to read the most relevant ones. Cite URLs in your answer."
        ),
    )

    await Console(
        researcher.run_stream(
            task="What does Firecrawl's /agent endpoint do? Cite the docs."
        )
    )


if __name__ == "__main__":
    asyncio.run(main())
Run it: 
python researcher.py

​
Multi-Agent: Researcher + Writer

Hand Firecrawl output from a researcher agent to a writer agent in a round-robin team. 
import asyncio
import os
from firecrawl import FirecrawlApp
from autogen_agentchat.agents import AssistantAgent
from autogen_agentchat.conditions import MaxMessageTermination
from autogen_agentchat.teams import RoundRobinGroupChat
from autogen_agentchat.ui import Console
from autogen_ext.models.openai import OpenAIChatCompletionClient

firecrawl = FirecrawlApp(api_key=os.environ["FIRECRAWL_API_KEY"])


def scrape_url(url: str) -> str:
    result = firecrawl.scrape(url, formats=["markdown"])
    return result.markdown or ""


def web_search(query: str, limit: int = 5) -> list[dict]:
    result = firecrawl.search(query, limit=limit)
    return [
        {"title": r.title, "url": r.url, "snippet": r.description}
        for r in result.web or []
    ]


async def main() -> None:
    model = OpenAIChatCompletionClient(model="gpt-4o-mini")

    researcher = AssistantAgent(
        name="researcher",
        model_client=model,
        tools=[scrape_url, web_search],
        system_message="Gather sources with web_search + scrape_url. Reply with bullet-point findings and URLs.",
    )

    writer = AssistantAgent(
        name="writer",
        model_client=model,
        system_message="Turn the researcher's findings into a 200-word briefing with inline citations.",
    )

    team = RoundRobinGroupChat(
        [researcher, writer],
        termination_condition=MaxMessageTermination(max_messages=6),
    )

    await Console(team.run_stream(task="Write a briefing on Firecrawl's crawl endpoint."))


if __name__ == "__main__":
    asyncio.run(main())

​
Notes

  • Firecrawl’s Python SDK is synchronous; AutoGen will call your wrappers inside its event loop without issues for small workloads. For heavy concurrent scraping, move calls off the main thread or use batch scrape.
  • Replace OpenAIChatCompletionClient with any AutoGen-supported model client (Azure OpenAI, Anthropic via autogen-ext, Ollama, etc.). Firecrawl is model-agnostic.
  • See the AutoGen docs for agent patterns beyond round-robin (selector, swarm, nested teams).