1
import asyncio
2
from crawl4ai import AsyncWebCrawler
3

4
async def main():
5
  async with AsyncWebCrawler() as crawler:
6
    result = await crawler.arun("https://example.com")
7
    print(result.markdown[:300])
8

9
if __name__ == "__main__":
10
  asyncio.run(main())

1
import asyncio
2
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
3

4
async def main():
5
  browser_conf = BrowserConfig(headless=True)
6
  run_conf = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
7

8
  async with AsyncWebCrawler(config=browser_conf) as crawler:
9
    result = await crawler.arun(
10
      url="https://example.com",
11
      config=run_conf,
12
    )
13
    print(result.markdown)
14

15
if __name__ == "__main__":
16
  asyncio.run(main())

1
import asyncio
2
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode
3
from crawl4ai.content_filter_strategy import PruningContentFilter
4
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
5

6
md_generator = DefaultMarkdownGenerator(
7
  content_filter=PruningContentFilter(threshold=0.4, threshold_type="fixed")
8
)
9

10
config = CrawlerRunConfig(
11
  cache_mode=CacheMode.BYPASS,
12
  markdown_generator=md_generator,
13
)
14

15
async def main():
16
  async with AsyncWebCrawler() as crawler:
17
    result = await crawler.arun("https://news.ycombinator.com", config=config)
18
    print("Raw:", len(result.markdown.raw_markdown))
19
    print("Fit:", len(result.markdown.fit_markdown))
20

21
if __name__ == "__main__":
22
  asyncio.run(main())

1
import asyncio
2
import json
3
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode
4
from crawl4ai import JsonCssExtractionStrategy
5

6
schema = {
7
  "name": "Example Items",
8
  "baseSelector": "div.item",
9
  "fields": [
10
    {"name": "title", "selector": "h2", "type": "text"},
11
    {"name": "link", "selector": "a", "type": "attribute", "attribute": "href"},
12
  ],
13
}
14

15
raw_html = "<div class='item'><h2>Item 1</h2><a href='https://example.com/item1'>Link 1</a></div>"
16

17
async def main():
18
  async with AsyncWebCrawler() as crawler:
19
    result = await crawler.arun(
20
      url="raw://" + raw_html,
21
      config=CrawlerRunConfig(
22
        cache_mode=CacheMode.BYPASS,
23
        extraction_strategy=JsonCssExtractionStrategy(schema),
24
      ),
25
    )
26
    data = json.loads(result.extracted_content)
27
    print(data)
28

29
if __name__ == "__main__":
30
  asyncio.run(main())