From 14cd4698c029fe55c8aea7ece1ddef5489e133ee Mon Sep 17 00:00:00 2001 From: chao ma Date: Sat, 15 Apr 2023 12:18:19 +0800 Subject: [PATCH 1/2] feat: Add support for running Chrome in Headless mode. Add headless mode support for Chrome and refactor web page text extraction --- .env.template | 3 +++ autogpt/config.py | 1 + autogpt/web.py | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/.env.template b/.env.template index 22bf8d74d970..89ae0a9b08dc 100644 --- a/.env.template +++ b/.env.template @@ -121,3 +121,6 @@ USE_BRIAN_TTS=False ELEVENLABS_API_KEY=your-elevenlabs-api-key ELEVENLABS_VOICE_1_ID=your-voice-id-1 ELEVENLABS_VOICE_2_ID=your-voice-id-2 + +# Chrome Headless Mode +HEADLESS_BROWSER=True diff --git a/autogpt/config.py b/autogpt/config.py index 26132a5abb5b..a2da05aa9d0f 100644 --- a/autogpt/config.py +++ b/autogpt/config.py @@ -86,6 +86,7 @@ def __init__(self): "USER_AGENT", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36", ) + self.headless_browser = os.getenv('HEADLESS_BROWSER',"True") == "True" self.redis_host = os.getenv("REDIS_HOST", "localhost") self.redis_port = os.getenv("REDIS_PORT", "6379") self.redis_password = os.getenv("REDIS_PASSWORD", "") diff --git a/autogpt/web.py b/autogpt/web.py index 355f7fd39b4c..fb25b9f28eb2 100644 --- a/autogpt/web.py +++ b/autogpt/web.py @@ -31,7 +31,12 @@ def scrape_text_with_selenium(url): logging.getLogger("selenium").setLevel(logging.CRITICAL) options = Options() + if cfg.headless_browser: + options.add_argument('--headless') + options.add_argument('--disable-gpu') + options.add_argument('--no-sandbox') options.add_argument( + "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.5615.49 Safari/537.36" ) driver = webdriver.Chrome( From 1cb2780d990f061889f1612c1972205a8596fd61 Mon Sep 17 00:00:00 2001 From: Reinier van der Leer Date: Tue, 18 Apr 2023 16:15:37 +0200 Subject: [PATCH 2/2] consolidate browser settings --- .env.template | 10 +++++----- autogpt/commands/web_selenium.py | 2 +- autogpt/config/config.py | 9 ++++++--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.env.template b/.env.template index 68392d769baf..855cb91ffdd0 100644 --- a/.env.template +++ b/.env.template @@ -9,9 +9,6 @@ BROWSE_CHUNK_MAX_LENGTH=8192 # USER_AGENT="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36" # AI_SETTINGS_FILE - Specifies which AI Settings file to use (defaults to ai_settings.yaml) AI_SETTINGS_FILE=ai_settings.yaml -# USE_WEB_BROWSER - Sets the web-browser drivers to use with selenium (defaults to chrome). -# Note: set this to either 'chrome', 'firefox', or 'safari' depending on your current browser -# USE_WEB_BROWSER=chrome ################################################################################ ### LLM PROVIDER @@ -138,8 +135,11 @@ GITHUB_USERNAME=your-github-username ################################################################################ ### BROWSER -# HEADLESS_BROWSER - Whether to run the browser in headless mode -HEADLESS_BROWSER=True +# USE_WEB_BROWSER - Sets the web-browser drivers to use with selenium (defaults to chrome). +# HEADLESS_BROWSER - Whether to run the browser in headless mode (defaults to True) +# Note: set this to either 'chrome', 'firefox', or 'safari' depending on your current browser +# USE_WEB_BROWSER=chrome +# HEADLESS_BROWSER=True ### GOOGLE # GOOGLE_API_KEY - Google API key (Example: my-google-api-key) diff --git a/autogpt/commands/web_selenium.py b/autogpt/commands/web_selenium.py index 277064034b38..11bdfeb1f163 100644 --- a/autogpt/commands/web_selenium.py +++ b/autogpt/commands/web_selenium.py @@ -83,7 +83,7 @@ def scrape_text_with_selenium(url: str) -> tuple[WebDriver, str]: options.add_argument("--remote-debugging-port=9222") options.add_argument("--no-sandbox") - if CFG.headless_browser: + if CFG.selenium_headless: options.add_argument("--headless") options.add_argument("--disable-gpu") diff --git a/autogpt/config/config.py b/autogpt/config/config.py index 7e82627df86e..acf41c44a7e3 100644 --- a/autogpt/config/config.py +++ b/autogpt/config/config.py @@ -25,7 +25,6 @@ def __init__(self) -> None: self.skip_reprompt = False self.allow_downloads = False - self.selenium_web_browser = os.getenv("USE_WEB_BROWSER", "chrome") self.ai_settings_file = os.getenv("AI_SETTINGS_FILE", "ai_settings.yaml") self.fast_llm_model = os.getenv("FAST_LLM_MODEL", "gpt-3.5-turbo") self.smart_llm_model = os.getenv("SMART_LLM_MODEL", "gpt-4") @@ -87,7 +86,11 @@ def __init__(self) -> None: "HUGGINGFACE_AUDIO_TO_TEXT_MODEL" ) - # User agent headers to use when browsing web + # Selenium browser settings + self.selenium_web_browser = os.getenv("USE_WEB_BROWSER", "chrome") + self.selenium_headless = os.getenv("HEADLESS_BROWSER", "True") == "True" + + # User agent header to use when making HTTP requests # Some websites might just completely deny request with an error code if # no user agent was found. self.user_agent = os.getenv( @@ -95,7 +98,7 @@ def __init__(self) -> None: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36" " (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36", ) - self.headless_browser = os.getenv('HEADLESS_BROWSER',"True") == "True" + self.redis_host = os.getenv("REDIS_HOST", "localhost") self.redis_port = os.getenv("REDIS_PORT", "6379") self.redis_password = os.getenv("REDIS_PASSWORD", "")