import json, os
from scrapegraphai.graphs import SmartScraperGraph
from dotenv import load_dotenv
load_dotenv()
# Define the configuration for the scraping pipeline
graph_config = {
"llm": {
"api_key": os.environ['AZURE_OPENAI_API_KEY'],
"model": "azure_openai/gpt-4o",
},
"verbose": True,
"headless": False,
}
# Create the SmartScraperGraph instance
smart_scraper_graph = SmartScraperGraph(
prompt="Find some information about what does the company do, the name and a contact email.",
source="https://scrapegraphai.com/",
config=graph_config
)
# Run the pipeline
result = smart_scraper_graph.run()
print(json.dumps(result, indent=4))RuntimeError Traceback (most recent call last)
Cell In[2], [line 25](vscode-notebook-cell:?execution_count=2&line=25)
[18](vscode-notebook-cell:?execution_count=2&line=18) smart_scraper_graph = SmartScraperGraph(
[19](vscode-notebook-cell:?execution_count=2&line=19) prompt="Find some information about what does the company do, the name and a contact email.",
[20](vscode-notebook-cell:?execution_count=2&line=20) source="https://scrapegraphai.com/",
[21](vscode-notebook-cell:?execution_count=2&line=21) config=graph_config
[22](vscode-notebook-cell:?execution_count=2&line=22) )
[24](vscode-notebook-cell:?execution_count=2&line=24) # Run the pipeline
---> [25](vscode-notebook-cell:?execution_count=2&line=25) result = smart_scraper_graph.run()
[26](vscode-notebook-cell:?execution_count=2&line=26) print(json.dumps(result, indent=4))
File ~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/smart_scraper_graph.py:114, in SmartScraperGraph.run(self)
[106](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/smart_scraper_graph.py:106) """
[107](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/smart_scraper_graph.py:107) Executes the scraping process and returns the answer to the prompt.
[108](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/smart_scraper_graph.py:108)
[109](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/smart_scraper_graph.py:109) Returns:
[110](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/smart_scraper_graph.py:110) str: The answer to the prompt.
[111](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/smart_scraper_graph.py:111) """
[113](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/smart_scraper_graph.py:113) inputs = {"user_prompt": self.prompt, self.input_key: self.source}
--> [114](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/smart_scraper_graph.py:114) self.final_state, self.execution_info = self.graph.execute(inputs)
[116](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/smart_scraper_graph.py:116) return self.final_state.get("answer", "No answer found.")
File ~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/base_graph.py:258, in BaseGraph.execute(self, initial_state)
[256](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/base_graph.py:256) return (result["_state"], [])
[257](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/base_graph.py:257) else:
--> [258](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/base_graph.py:258) return self._execute_standard(initial_state)
File ~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/base_graph.py:179, in BaseGraph._execute_standard(self, initial_state)
[166](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/base_graph.py:166) graph_execution_time = time.time() - start_time
[167](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/base_graph.py:167) log_graph_execution(
[168](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/base_graph.py:168) graph_name=self.graph_name,
[169](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/base_graph.py:169) source=source,
(...)
[177](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/base_graph.py:177) exception=str(e)
[178](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/base_graph.py:178) )
--> [179](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/base_graph.py:179) raise e
[180](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/base_graph.py:180) node_exec_time = time.time() - curr_time
[181](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/base_graph.py:181) total_exec_time += node_exec_time
File ~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/base_graph.py:163, in BaseGraph._execute_standard(self, initial_state)
[161](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/base_graph.py:161) with self.callback_manager.exclusive_get_callback(llm_model, llm_model_name) as cb:
[162](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/base_graph.py:162) try:
--> [163](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/base_graph.py:163) result = current_node.execute(state)
[164](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/base_graph.py:164) except Exception as e:
[165](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/graphs/base_graph.py:165) error_node = current_node.node_name
File ~/Documents/Code/Scrapegraph-ai/scrapegraphai/nodes/fetch_node.py:130, in FetchNode.execute(self, state)
[128](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/nodes/fetch_node.py:128) return self.handle_local_source(state, source)
[129](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/nodes/fetch_node.py:129) else:
--> [130](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/nodes/fetch_node.py:130) return self.handle_web_source(state, source)
File ~/Documents/Code/Scrapegraph-ai/scrapegraphai/nodes/fetch_node.py:303, in FetchNode.handle_web_source(self, state, source)
[301](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/nodes/fetch_node.py:301) else:
[302](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/nodes/fetch_node.py:302) loader = ChromiumLoader([source], headless=self.headless, **loader_kwargs)
--> [303](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/nodes/fetch_node.py:303) document = loader.load()
[305](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/nodes/fetch_node.py:305) if not document or not document[0].page_content.strip():
[306](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/nodes/fetch_node.py:306) raise ValueError("""No HTML body content found in
[307](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/nodes/fetch_node.py:307) the document fetched by ChromiumLoader.""")
File ~/Documents/Code/Scrapegraph-ai/.venv/lib/python3.11/site-packages/langchain_core/document_loaders/base.py:31, in BaseLoader.load(self)
[29](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/.venv/lib/python3.11/site-packages/langchain_core/document_loaders/base.py:29) def load(self) -> list[Document]:
[30](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/.venv/lib/python3.11/site-packages/langchain_core/document_loaders/base.py:30) """Load data into Document objects."""
---> [31](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/.venv/lib/python3.11/site-packages/langchain_core/document_loaders/base.py:31) return list(self.lazy_load())
File ~/Documents/Code/Scrapegraph-ai/scrapegraphai/docloaders/chromium.py:111, in ChromiumLoader.lazy_load(self)
[108](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/docloaders/chromium.py:108) scraping_fn = getattr(self, f"ascrape_{self.backend}")
[110](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/docloaders/chromium.py:110) for url in self.urls:
--> [111](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/docloaders/chromium.py:111) html_content = asyncio.run(scraping_fn(url))
[112](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/docloaders/chromium.py:112) metadata = {"source": url}
[113](https://file+.vscode-resource.vscode-cdn.net/home/anadirov/Documents/Code/Scrapegraph-ai/~/Documents/Code/Scrapegraph-ai/scrapegraphai/docloaders/chromium.py:113) yield Document(page_content=html_content, metadata=metadata)
File /usr/lib/python3.11/asyncio/runners.py:186, in run(main, debug)
[161](https://file+.vscode-resource.vscode-cdn.net/usr/lib/python3.11/asyncio/runners.py:161) """Execute the coroutine and return the result.
[162](https://file+.vscode-resource.vscode-cdn.net/usr/lib/python3.11/asyncio/runners.py:162)
[163](https://file+.vscode-resource.vscode-cdn.net/usr/lib/python3.11/asyncio/runners.py:163) This function runs the passed coroutine, taking care of
(...)
[182](https://file+.vscode-resource.vscode-cdn.net/usr/lib/python3.11/asyncio/runners.py:182) asyncio.run(main())
[183](https://file+.vscode-resource.vscode-cdn.net/usr/lib/python3.11/asyncio/runners.py:183) """
[184](https://file+.vscode-resource.vscode-cdn.net/usr/lib/python3.11/asyncio/runners.py:184) if events._get_running_loop() is not None:
[185](https://file+.vscode-resource.vscode-cdn.net/usr/lib/python3.11/asyncio/runners.py:185) # fail fast with short traceback
--> [186](https://file+.vscode-resource.vscode-cdn.net/usr/lib/python3.11/asyncio/runners.py:186) raise RuntimeError(
[187](https://file+.vscode-resource.vscode-cdn.net/usr/lib/python3.11/asyncio/runners.py:187) "asyncio.run() cannot be called from a running event loop")
[189](https://file+.vscode-resource.vscode-cdn.net/usr/lib/python3.11/asyncio/runners.py:189) with Runner(debug=debug) as runner:
[190](https://file+.vscode-resource.vscode-cdn.net/usr/lib/python3.11/asyncio/runners.py:190) return runner.run(main)
RuntimeError: asyncio.run() cannot be called from a running event loop
I try to start with code:
The error i get:
RuntimeError: asyncio.run() cannot be called from a running event loopFull stacktrace: