From 9c60eecce68f9718e48854ec51d7fda7af9e8387 Mon Sep 17 00:00:00 2001 From: Reinier van der Leer Date: Mon, 24 Apr 2023 15:27:53 +0200 Subject: [PATCH] Improve docker setup & config (#1843) * Improve docker setup & config * fix(browsing): Selenium needs access to home directory * fix(docker): allow overriding memory backend settings * simplify Dockerfile and docker-compose config * add .dockerignore * adjust Docker CI with release build type arg * replace Chrome by Chromium in devcontainer * update docs * update bulletin * use preinstalled chromedriver in web_selenium.py * update installation.md * fix code blocks for mkdocs * fix links to docs --- .devcontainer/Dockerfile | 8 ++- .dockerignore | 8 +++ .github/workflows/docker-image.yml | 2 +- BULLETIN.md | 5 +- Dockerfile | 37 ++++++------- README.md | 48 ++++++++--------- autogpt/commands/web_selenium.py | 7 ++- docker-compose.yml | 6 ++- docs/installation.md | 83 ++++++++++++++++-------------- docs/usage.md | 40 ++++++++------ 10 files changed, 136 insertions(+), 108 deletions(-) create mode 100644 .dockerignore diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 02f580a02e11..87ca39d27398 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,6 +1,6 @@ # [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3-bullseye, 3.10-bullseye, 3-buster, 3.10-buster ARG VARIANT=3-bullseye -FROM --platform=linux/amd64 python:3.10 +FROM python:3.10 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131 @@ -10,10 +10,8 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ # They are installed by the base image (python) which does not have the patch. RUN python3 -m pip install --upgrade setuptools -# Install Chrome for web browsing -RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && curl -sSL https://dl.google.com/linux/direct/google-chrome-stable_current_$(dpkg --print-architecture).deb -o /tmp/chrome.deb \ - && apt-get -y install /tmp/chrome.deb +# Install Chromium for web browsing +RUN apt-get install -y chromium-driver # [Optional] If your pip requirements rarely change, uncomment this section to add them to the image. # COPY requirements.txt /tmp/pip-tmp/ diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000000..9ac551095c77 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.* +*.template +*.yaml +*.yml + +*.md +*.png +!BULLETIN.md diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 2fcf6401efbf..d63ff1f4e765 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -18,4 +18,4 @@ jobs: steps: - uses: actions/checkout@v3 - name: Build the Docker image - run: docker build . --file Dockerfile --tag autogpt:$(date +%s) + run: docker build . --build-arg BUILD_TYPE=release --tag autogpt:$(date +%s) diff --git a/BULLETIN.md b/BULLETIN.md index 735048ddc87a..0962da4a85dd 100644 --- a/BULLETIN.md +++ b/BULLETIN.md @@ -1,2 +1,5 @@ Welcome to Auto-GPT! We'll keep you informed of the latest news and features by printing messages here. -If you don't wish to see this message, you can run Auto-GPT with the --skip-news flag \ No newline at end of file +If you don't wish to see this message, you can run Auto-GPT with the --skip-news flag + +## Changes to Docker configuration +The workdir has been changed from /home/appuser to /app. Be sure to update any volume mounts accordingly. diff --git a/Dockerfile b/Dockerfile index 8396154998f3..05ec1a429863 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,38 +1,33 @@ # Use an official Python base image from the Docker Hub FROM python:3.10-slim -# Install git -RUN apt-get -y update -RUN apt-get -y install git chromium-driver +# 'dev' or 'release' container build +ARG BUILD_TYPE=dev -# Install Xvfb and other dependencies for headless browser testing -RUN apt-get update \ - && apt-get install -y wget gnupg2 libgtk-3-0 libdbus-glib-1-2 dbus-x11 xvfb ca-certificates +# Install browsers +RUN apt-get update && apt-get install -y \ + chromium-driver firefox-esr \ + ca-certificates -# Install Firefox / Chromium -RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ - && echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list \ - && apt-get update \ - && apt-get install -y chromium firefox-esr +# Install utilities +RUN apt-get install -y curl jq wget git # Set environment variables ENV PIP_NO_CACHE_DIR=yes \ PYTHONUNBUFFERED=1 \ PYTHONDONTWRITEBYTECODE=1 -# Create a non-root user and set permissions -RUN useradd --create-home appuser -WORKDIR /home/appuser -RUN chown appuser:appuser /home/appuser -USER appuser +# Install the required python packages globally +ENV PATH="$PATH:/root/.local/bin" +COPY requirements.txt . -# Copy the requirements.txt file and install the requirements -COPY --chown=appuser:appuser requirements.txt . -RUN sed -i '/Items below this point will not be included in the Docker Image/,$d' requirements.txt && \ - pip install --no-cache-dir --user -r requirements.txt +# Only install dev dependencies in dev container builds +RUN [ '${BUILD_TYPE}' = 'dev' ] || sed -i '/Items below this point will not be included in the Docker Image/,$d' requirements.txt && \ + pip install --no-cache-dir -r requirements.txt # Copy the application files -COPY --chown=appuser:appuser autogpt/ ./autogpt +WORKDIR /app +COPY autogpt/ ./autogpt # Set the entrypoint ENTRYPOINT ["python", "-m", "autogpt"] diff --git a/README.md b/README.md index 497519adb57d..c0a9bf7dfe53 100644 --- a/README.md +++ b/README.md @@ -59,37 +59,37 @@ Your support is greatly appreciated. Development of this free, open-source proje ## 📋 Requirements -- Environment (pick one) +Choose an environment to run Auto-GPT in (pick one): + + - [Docker](https://docs.docker.com/get-docker/) (*recommended*) - Python 3.10 or later (instructions: [for Windows](https://www.tutorialspoint.com/how-to-install-python-in-windows)) - - Docker [Docker Hub](https://hub.docker.com/r/significantgravitas/auto-gpt) - [VSCode + devcontainer](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) ## Quickstart 1. Set up your OpenAI [API Keys](https://platform.openai.com/account/api-keys) 2. Download the [latest release](https://github.com/Significant-Gravitas/Auto-GPT/releases/latest) -3. Install the [dependencies](/docs/installation.md) and set your [environment variables](/docs/installation.md) -4. Configure any additional features you want, or install some [plugins](/docs/plugins.md) -5. [Run](/docs/usage.md) the app - -Please see the [documentation](/docs) linked below for full setup instructions and configuration options. - -## 💾 Documentation - -[Installation](/docs/installation.md) - -[Usage](/docs/usage.md) - -[Speech](/docs/configuration/voice.md) - -[Search](/docs/configuration/search.md) - -[Plugins](/docs/plugins.md) - -[Memory](/docs/configuration/memory.md) - -[Image Generation](/docs/configuration/imagegen.md) - +3. Follow the [installation instructions][docs/install] +4. Configure any additional features you want, or install some [plugins][docs/plugins] +5. [Run][docs/usage] the app + +Please see the [documentation][docs] linked below for full setup instructions and configuration options. + +[docs]: https://significant-gravitas.github.io/Auto-GPT/ + +## 📖 Documentation +* [⚙️ Installation][docs/install] +* [💻 Usage][docs/usage] +* [🔌 Plugins][docs/plugins] +* Configuration + * [🔍 Web Search](https://significant-gravitas.github.io/Auto-GPT/configuration/search/) + * [🧠 Memory](https://significant-gravitas.github.io/Auto-GPT/configuration/memory/) + * [🗣️ Voice (TTS)](https://significant-gravitas.github.io/Auto-GPT/configuration/voice/) + * [🖼️ Image Generation](https://significant-gravitas.github.io/Auto-GPT/configuration/imagegen/) + +[docs/install]: https://significant-gravitas.github.io/Auto-GPT/installation/ +[docs/usage]: https://significant-gravitas.github.io/Auto-GPT/usage/ +[docs/plugins]: https://significant-gravitas.github.io/Auto-GPT/plugins/ ## ⚠️ Limitations diff --git a/autogpt/commands/web_selenium.py b/autogpt/commands/web_selenium.py index 8f55c103e9ef..b435660c02c3 100644 --- a/autogpt/commands/web_selenium.py +++ b/autogpt/commands/web_selenium.py @@ -95,8 +95,13 @@ def scrape_text_with_selenium(url: str) -> tuple[WebDriver, str]: options.add_argument("--headless") options.add_argument("--disable-gpu") + chromium_driver_path = Path("/usr/bin/chromedriver") + driver = webdriver.Chrome( - executable_path=ChromeDriverManager().install(), options=options + executable_path=chromium_driver_path + if chromium_driver_path.exists() + else ChromeDriverManager().install(), + options=options, ) driver.get(url) diff --git a/docker-compose.yml b/docker-compose.yml index 50fa1a070721..4c6656dd6cf8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,9 +9,11 @@ services: build: ./ env_file: - .env + environment: + MEMORY_BACKEND: ${MEMORY_BACKEND:-redis} + REDIS_HOST: ${REDIS_HOST:-redis} volumes: - - "./autogpt:/home/appuser" - - ".env:/home/appuser/.env" + - ./:/app profiles: ["exclude-from-up"] redis: diff --git a/docs/installation.md b/docs/installation.md index 554ad711f437..579a246dfdad 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -11,48 +11,39 @@ You can set up paid account at https://platform.openai.com/account/billing/overv Important: It's highly recommended that you track your usage on [the Usage page](https://platform.openai.com/account/usage) You can also set limits on how much you spend on [the Usage limits page](https://platform.openai.com/account/billing/limits). -![For OpenAI API key to work, set up paid account at OpenAI API > Billing](./docs/imgs/openai-api-key-billing-paid-account.png) +![For OpenAI API key to work, set up paid account at OpenAI API > Billing](./imgs/openai-api-key-billing-paid-account.png) **PLEASE ENSURE YOU HAVE DONE THIS STEP BEFORE PROCEEDING. OTHERWISE, NOTHING WILL WORK!** -## Steps +## General setup -To install Auto-GPT, follow these steps: +1. Make sure you have one of the environments listed under [**requirements**](../README.md#-requirements) set up. -1. Make sure you have all the **requirements** listed in the [README](../README.md). If not, install/get them. + _To execute the following commands, open a CMD, Bash, or Powershell window by navigating to a folder on your computer and typing `CMD` in the folder path at the top, then press enter._ -_To execute the following commands, open a CMD, Bash, or Powershell window by navigating to a folder on your computer and typing `CMD` in the folder path at the top, then press enter._ +2. Clone the repository using Git, or download the [latest stable release](https://github.com/Significant-Gravitas/Auto-GPT/releases/latest) (`Source code (zip)`, at the bottom of the page). -2. Clone the repository: For this step, you need Git installed. -Note: If you don't have Git, you can just download the [latest stable release](https://github.com/Significant-Gravitas/Auto-GPT/releases/latest) instead (`Source code (zip)`, at the bottom of the page). - -``` shell + ``` shell git clone -b stable https://github.com/Significant-Gravitas/Auto-GPT.git -``` + ``` 3. Navigate to the directory where you downloaded the repository. -``` shell + ``` shell cd Auto-GPT -``` - -4. Install the required dependencies. - -``` shell - pip install -r requirements.txt -``` + ``` 5. Configure Auto-GPT: - 1. Find the file named `.env.template` in the main /Auto-GPT folder. This file may be hidden by default in some operating systems due to the dot prefix. To reveal hidden files, follow the instructions for your specific operating system (e.g., in Windows, click on the "View" tab in File Explorer and check the "Hidden items" box; in macOS, press Cmd + Shift + .). + 1. Find the file named `.env.template` in the main `Auto-GPT` folder. This file may be hidden by default in some operating systems due to the dot prefix. To reveal hidden files, follow the instructions for your specific operating system (e.g., in Windows, click on the "View" tab in File Explorer and check the "Hidden items" box; in macOS, press Cmd + Shift + .). 2. Create a copy of this file and call it `.env` by removing the `template` extension. The easiest way is to do this in a command prompt/terminal window `cp .env.template .env`. 3. Open the `.env` file in a text editor. 4. Find the line that says `OPENAI_API_KEY=`. 5. After the `"="`, enter your unique OpenAI API Key (without any quotes or spaces). - 6. Enter any other API keys or Tokens for services you would like to use. + 6. Enter any other API keys or Tokens for services you would like to use. To activate and adjust a setting, remove the `# ` prefix. 7. Save and close the `.env` file. - After you complete these steps, you'll have properly configured the API keys for your project. - + You have now configured Auto-GPT. + Notes: - See [OpenAI API Keys Configuration](#openai-api-keys-configuration) to get your OpenAI API key. - Get your ElevenLabs API key from: https://elevenlabs.io. You can view your xi-api-key using the "Profile" tab on the website. @@ -61,38 +52,54 @@ Note: If you don't have Git, you can just download the [latest stable release](h - `fast_llm_model_deployment_id` - your gpt-3.5-turbo or gpt-4 deployment ID - `smart_llm_model_deployment_id` - your gpt-4 deployment ID - `embedding_model_deployment_id` - your text-embedding-ada-002 v2 deployment ID + - Please specify all of these values as double-quoted strings - -``` shell -# Replace string in angled brackets (<>) to your own ID -azure_model_map: - fast_llm_model_deployment_id: "" - ... -``` + ``` shell + # Replace string in angled brackets (<>) to your own ID + azure_model_map: + fast_llm_model_deployment_id: "" + ... + ``` - Details can be found here: https://pypi.org/project/openai/ in the `Microsoft Azure Endpoints` section and here: https://learn.microsoft.com/en-us/azure/cognitive-services/openai/tutorials/embeddings?tabs=command-line for the embedding model. - If you're on Windows you may need to install https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist?view=msvc-170 -## Docker +4. Follow the further instructions for running Auto-GPT with [Docker](#run-with-docker) (*recommended*), or [Docker-less](#run-docker-less) -You can also build this into a docker image and run it: +### Run with Docker +Easiest is to run with `docker-compose`: ``` shell -docker build -t autogpt . -docker run -it --env-file=./.env -v $PWD/auto_gpt_workspace:/home/appuser/auto_gpt_workspace autogpt +docker-compose build auto-gpt +docker-compose run --rm auto-gpt ``` +By default, this will also start and attach a Redis memory backend. +For related settings, see [Memory > Redis setup](./configuration/memory.md#redis-setup). -Or if you have `docker-compose`: +You can also build and run it with "vanilla" docker commands: ``` shell -docker-compose run --build --rm auto-gpt +docker build -t auto-gpt . +docker run -it --env-file=.env -v $PWD:/app auto-gpt ``` You can pass extra arguments, for instance, running with `--gpt3only` and `--continuous` mode: ``` shell -docker run -it --env-file=./.env -v $PWD/auto_gpt_workspace:/home/appuser/auto_gpt_workspace autogpt --gpt3only --continuous +docker-compose run --rm auto-gpt --gpt3only --continuous +``` +``` shell +docker run -it --env-file=.env -v $PWD:/app --rm auto-gpt --gpt3only --continuous ``` +Alternatively, you can pull the latest release directly from [Docker Hub](https://hub.docker.com/r/significantgravitas/auto-gpt) and run that: ``` shell -docker-compose run --build --rm auto-gpt --gpt3only --continuous +docker run -it --env OPENAI_API_KEY='your-key-here' --rm significantgravitas/auto-gpt ``` -Alternatively, you can pull the latest release directly from [Docker Hub](https://hub.docker.com/r/significantgravitas/auto-gpt) +Or with `ai_settings.yml` presets mounted: +``` shell +docker run -it --env OPENAI_API_KEY='your-key-here' -v $PWD/ai_settings.yaml:/app/ai_settings.yaml --rm significantgravitas/auto-gpt +``` + + +### Run Docker-less + +Simply run `./run.sh` (Linux/macOS) or `.\run.bat` (Windows) in your terminal. diff --git a/docs/usage.md b/docs/usage.md index 9b2185ae0ae1..65adae74a34f 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -1,15 +1,25 @@ # Usage -1. Run the `autogpt` Python module in your terminal. -* On Linux/MacOS: - ``` - ./run.sh - ``` -* On Windows: - ``` - .\run.bat - ``` - Running with `--help` after `.\run.bat` lists all the possible command line arguments you can pass. +1. Open a terminal and run the startup script: + * On Linux/MacOS: + ``` shell + ./run.sh + ``` + * On Windows: + ``` shell + .\run.bat + ``` + * Using Docker: + ``` shell + docker-compose run --rm auto-gpt + ``` + Running with `--help` lists all the possible command line arguments you can pass: + ``` shell + ./run.sh --help + + # or with docker + docker-compose run --rm auto-gpt --help + ``` 2. After each response from Auto-GPT, choose from the options to authorize command(s), exit the program, or provide feedback to the AI. @@ -24,15 +34,15 @@ Here are some common arguments you can use when running Auto-GPT: > Replace anything in angled brackets (<>) to a value you want to specify * View all available command line arguments - ``` + ``` shell python -m autogpt --help ``` * Run Auto-GPT with a different AI Settings file - ``` + ``` shell python -m autogpt --ai-settings ``` * Specify a memory backend - ``` + ``` shell python -m autogpt --use-memory ``` @@ -55,7 +65,7 @@ Use at your own risk. 1. Run the `autogpt` python module in your terminal: - ``` + ``` shell python -m autogpt --continuous ``` @@ -85,4 +95,4 @@ To print out debug logs: ``` shell python -m autogpt --debug -``` \ No newline at end of file +```