From 3d72e97831a874efeccdfe5776a2f4d6f09cf7f7 Mon Sep 17 00:00:00 2001 From: Ian Jennings Date: Tue, 2 Jul 2024 12:38:05 -0700 Subject: [PATCH] WIP: TestDriver: Update to v3 (#690) **Action Required:** We've moved to an authenticated system and you must now provide an authentication key within our script. 1. Create an account or log into https://app.dashcam.io. 2. Copy the API key from https://app.dashcam.io/team. You must be the team "owner" 3. Add the API key as `DASHCAM_API_KEY` within GitHub actions by following the guide [here](https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions). This PR moves to our v3 system which includes - Faster and more stable tests. Notice we're using a `yml` file to guide the test rather than open prompts. This means the AI will follow the same steps every time. - Smarter AI. Better text matching, and more commands. We optimized how the AI matches text and images, making it way faster and more fault-tolerant. - Better summaries. The AI will summarize the full test execution as well as errors - Assertions. We can now explicitly ask the AI to verify things on screen - Embedded tests. We can nest scripts for reusable components. --------- Co-authored-by: orliesaurus --- .github/workflows/regression.yml | 84 +++++++++++++------------------- .testdriver/prerun.sh | 18 ------- .testdriver/wave1.yml | 29 +++++++++++ 3 files changed, 63 insertions(+), 68 deletions(-) delete mode 100644 .testdriver/prerun.sh create mode 100644 .testdriver/wave1.yml diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index c1e774263..39fa98258 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -1,4 +1,4 @@ -name: TestDriver.ai Regression Testing +name: TestDriver.ai Regression Testing - Waveterm on: push: branches: @@ -14,53 +14,37 @@ permissions: contents: read # To allow the action to read repository contents pull-requests: write # To allow the action to create/update pull request comments + jobs: - test: - name: TestDriver - runs-on: ubuntu-latest - steps: - - uses: dashcamio/testdriver@main - id: testdriver - with: - version: v2.12.12 - prerun: | - rm ~/Desktop/WITH-LOVE-FROM-AMERICA.txt - cd ~/actions-runner/_work/testdriver/testdriver/ - brew install go - brew tap scripthaus-dev/scripthaus - brew install corepack - brew install scripthaus - corepack enable - yarn install - scripthaus run build-backend - echo "Yarn" - yarn - echo "Rebuild" - scripthaus run electron-rebuild - echo "Webpack" - scripthaus run webpack-build - echo "Starting Electron" - scripthaus run electron 1>/dev/null 2>&1 & - echo "Electron Done" - exit - prompt: | - 1. wait 10 seconds - 1. click "Get Started" - 1. validate that overlapping text does not appear in the application - 1. focus the Wave input with the keyboard shorcut Command + I - 1. type 'ls' into the input - 1. press return - 1. validate Wave shows the result of 'ls' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - uses: peter-evans/create-or-update-comment@v4 - if: ${{always()}} - with: - issue-number: ${{ github.event.pull_request.number }} - body: | - ## TestDriver Summary - ${{ steps.testdriver.outputs.markdown }} - ${{ steps.testdriver.outputs.summary }} - reactions: | - +1 - -1 + test: + name: "TestDriver" + runs-on: ubuntu-latest + steps: + - uses: dashcamio/testdriver@main + id: testdriver + with: + version: v3.5.6 + key: ${{secrets.DASHCAM_API}} + prerun: | + cd ~/actions-runner/_work/testdriver/testdriver/ + brew install go + brew tap scripthaus-dev/scripthaus + brew install corepack + brew install scripthaus + corepack enable + yarn install + scripthaus run build-backend + echo "Yarn" + yarn + echo "Rebuild" + scripthaus run electron-rebuild + echo "Webpack" + scripthaus run webpack-build + echo "Starting Electron" + scripthaus run electron 1>/dev/null 2>&1 & + echo "Electron Done" + cd /Users/ec2-user/Downloads/td/ + npm rebuild + exit + prompt: | + 1. /run /Users/ec2-user/actions-runner/_work/testdriver/testdriver/.testdriver/wave1.yml diff --git a/.testdriver/prerun.sh b/.testdriver/prerun.sh deleted file mode 100644 index 9567b41e6..000000000 --- a/.testdriver/prerun.sh +++ /dev/null @@ -1,18 +0,0 @@ -rm ~/Desktop/WITH-LOVE-FROM-AMERICA.txt -cd ~/actions-runner/_work/testdriver/testdriver/ -brew install go -brew tap scripthaus-dev/scripthaus -brew install scripthaus -corepack enable -yarn install -scripthaus run build-backend -echo "Yarn" -yarn -echo "Rebuild" -scripthaus run electron-rebuild -echo "Webpack" -scripthaus run webpack-build -echo "Starting Electron" -scripthaus run electron 1>/dev/null 2>&1 & -echo "Electron Done" -exit diff --git a/.testdriver/wave1.yml b/.testdriver/wave1.yml new file mode 100644 index 000000000..0baba0c52 --- /dev/null +++ b/.testdriver/wave1.yml @@ -0,0 +1,29 @@ +steps: + - step: + - command: focus-application + name: Electron + - command: hover-text + description: Get Started + action: click + - command: hover-text + description: Settings + action: click + - command: hover-text + description: 13px + action: click + - command: hover-text + description: 12px + action: click + - command: hover-text + description: Dark + action: click + - command: hover-text + description: Light + action: click + - command: hover-text + description: workspace-1 + action: click + - step: + - command: assert + description: the terminal is white +