Add a web surfer agent that can search and browse the web. (#1093)

* Initial commit of WebSurfer. Adds the browser_utils, and related tests. WebSurfer will be added in a subsequent commit.

* Added the web surfer agent, and related tests.

* Added a notebook to show how WebSurferAgent works.

* Fixed a typo.

* Updated test_web_surfer for compatibility with #1110.

* Updated skip_oai logic.

* Fixed code formatting.

* More pre-commit fixes.

* Added block to contrib-openai.yml

* Added block to contrib-openai.yml

* Added hook for BING_API_KEY

* Temporarily commented out other tests, per request.

* Fixed indentation (maybe?)

* Restoring contrib-openai.yml

.github/workflows/contrib-openai.yml

@@ -217,3 +217,42 @@ jobs:
         with:
           file: ./coverage.xml
           flags: unittests
+  WebSurfer:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.11"]
+    runs-on: ${{ matrix.os }}
+    environment: openai1
+    steps:
+      # checkout to pr branch
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install packages and dependencies
+        run: |
+          docker --version
+          python -m pip install --upgrade pip wheel
+          pip install -e .[websurfer]
+          python -c "import autogen"
+          pip install coverage pytest
+      - name: Coverage
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
+          AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
+          OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }}
+          BING_API_KEY: ${{ secrets.BING_API_KEY }}
+        run: |
+          coverage run -a -m pytest test/agentchat/contrib/test_web_surfer.py
+          coverage xml
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          file: ./coverage.xml
+          flags: unittests

.github/workflows/contrib-tests.yml

@@ -196,6 +196,49 @@ jobs:
           file: ./coverage.xml
           flags: unittests
 
+  WebSurfer:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-2019]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install packages and dependencies for all tests
+        run: |
+          python -m pip install --upgrade pip wheel
+          pip install pytest
+      - name: Install packages and dependencies for WebSurfer
+        run: |
+          pip install -e .[websurfer]
+      - name: Set AUTOGEN_USE_DOCKER based on OS
+        shell: bash
+        run: |
+          if [[ ${{ matrix.os }} != ubuntu-latest ]]; then
+            echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV
+          fi
+      - name: Test WebSurfer
+        if: matrix.python-version != '3.10'  # diversify the python versions
+        run: |
+          pytest test/test_browser_utils.py test/agentchat/contrib/test_web_surfer.py --skip-openai
+      - name: Coverage
+        if: matrix.python-version == '3.10'
+        run: |
+          pip install coverage>=5.3
+          coverage run -a -m pytest test/test_browser_utils.py test/agentchat/contrib/test_web_surfer.py --skip-openai
+          coverage xml
+      - name: Upload coverage to Codecov
+        if: matrix.python-version == '3.10'
+        uses: codecov/codecov-action@v3
+        with:
+          file: ./coverage.xml
+          flags: unittests
+
   LMMTest:
     runs-on: ${{ matrix.os }}
     strategy: