ScrapegraphLib-Examples/nemotron/rate_limit_nemotron.py at main · ScrapeGraphAI/ScrapegraphLib-Examples · GitHub

History

47 lines (38 loc) · 1.27 KB

Raw

"""

Basic example of scraping pipeline using SmartScraper with a custom rate limit

"""

import os

import json

from dotenv import load_dotenv

from scrapegraphai.graphs import SmartScraperGraph

from scrapegraphai.utils import prettify_exec_info

load_dotenv()

# ************************************************

# Define the configuration for the graph

# ************************************************

graph_config = {

"llm": {

"api_key": os.getenv("NEMOTRON_KEY"),

"model": "nvidia/meta/llama3-70b-instruct",

"rate_limit": {

"requests_per_second": 1

}

"verbose": True,

"headless": False,

}

# ************************************************

# Create the SmartScraperGraph instance and run it

# ************************************************

smart_scraper_graph = SmartScraperGraph(

prompt="Extract me the python code inside the page",

source="https://www.exploit-db.com/exploits/51447",

config=graph_config

)

result = smart_scraper_graph.run()

print(json.dumps(result, indent=4))

# ************************************************

# Get graph execution info

# ************************************************

graph_exec_info = smart_scraper_graph.get_execution_info()

print(prettify_exec_info(graph_exec_info))

ScrapegraphLib-Examples/nemotron/rate_limit_nemotron.py at main · ScrapeGraphAI/ScrapegraphLib-Examples · GitHub

Expand file tree

Breadcrumbs

rate_limit_nemotron.py

Latest commit

History

Breadcrumbs

rate_limit_nemotron.py

File metadata and controls