Settings¶

from onecrawler import Settings


settings = Settings(
    link_extraction_limit=500,
    include_link_patterns=["/docs/*"],
    concurrency=8,
    request_timeout=15,
    max_retries=3,
)

from onecrawler import BrowserSettings, Settings


settings = Settings(
    browser_settings=BrowserSettings(
        viewport={"width": 1440, "height": 900},
        locale="en-US",
        timezone_id="UTC",
    )
)

from onecrawler import BrowserSettings, Settings


settings = Settings(
    browser_settings=BrowserSettings(storage_state="auth-state.json")
)

from onecrawler import Settings, ProxySettings


settings = Settings(
    proxy=ProxySettings(
        server="http://proxy.example:8080",
        username="user",
        password="pass",
    )
)

settings = Settings(
    proxies=[
        ProxySettings(server="http://proxy-1.example:8080"),
        ProxySettings(server="http://proxy-2.example:8080"),
    ],
    proxy_rotation_method="round_robin",
)

from onecrawler import Settings, HumanBehaviorSettings


settings = Settings(
    enable_human_behaviors=True,
    human_behavior_settings=HumanBehaviorSettings(
        min_delay=0.5,
        max_delay=2.0,
        max_scrolls=20,
        min_mouse_moves=2,
        max_mouse_moves=8,
    ),
)

from pydantic import BaseModel
from onecrawler import Settings, GenerativeAISettings

class Product(BaseModel):
    name: str
    price: str | None = None
    availability: str | None = None

settings = Settings(
    scraping_strategy="genai",  # Required for GenAI extraction
    scraping_output_format="json",  # GenAI only supports JSON
    genai=GenerativeAISettings(
        provider="openai",  # Options: "openai", "google", "ollama"
        model_name="gpt-4o-mini",
        api_key="YOUR_API_KEY",  # Required for OpenAI/Google, optional for Ollama
        output_schema=Product,  # Pydantic model for structured output
    ),
    concurrency=2,  # Lower concurrency recommended for GenAI
    request_timeout=30,  # Increase timeout for model responses
)

genai=GenerativeAISettings(
    provider="openai",
    model_name="gpt-4o-mini",
    api_key="sk-...",  # Your OpenAI API key
    output_schema=Product,
)

genai=GenerativeAISettings(
    provider="google",
    model_name="gemini-1.5-pro",
    api_key="AIza...",  # Your Google API key
    output_schema=Product,
)

genai=GenerativeAISettings(
    provider="ollama",
    model_name="llama3:8b",
    base_url="http://localhost:11434/",  # Your Ollama instance
    output_schema=Product,
    # api_key optional for Ollama
)

settings = Settings(
    proxies=[
        ProxySettings(server="http://proxy1.example.com:8080"),
        ProxySettings(server="http://proxy2.example.com:8080"),
    ],
    proxy_rotation_method="round_robin",
)

from onecrawler.filters import by_date, by_keywords
from onecrawler.filters.chain import AND

content_filter = AND(
    by_date(start="2024-01-01", end="2024-12-31"),
    by_keywords(["python", "async"]),
)

async with Crawler(settings) as engine:
    results = await engine.run("https://example.com/news", filters=content_filter)

from onecrawler.filters.chain import AND, OR, NOT

f = AND(by_date(start="2024-01-01"), NOT(by_files(["pdf"])))

settings = Settings(
    enable_human_behaviors=True,
    human_behavior_settings=HumanBehaviorSettings(
        min_delay=1.0,        # Minimum delay between actions (seconds)
        max_delay=3.0,        # Maximum delay between actions (seconds)
        max_scrolls=5,        # Maximum scroll gestures per page
        min_mouse_moves=2,    # Minimum mouse movements
        max_mouse_moves=5,    # Maximum mouse movements
        mouse_width=100,      # Mouse movement area width
        mouse_height=100,     # Mouse movement area height
        min_mouse_steps=5,    # Minimum steps per movement
        max_mouse_steps=15,   # Maximum steps per movement
        min_mouse_sleep=0.1,  # Minimum sleep between steps
        max_mouse_sleep=0.3,  # Maximum sleep between steps
    ),
)