From 4d3716d37534d5df78d42f2d7327ecc36d0c43f0 Mon Sep 17 00:00:00 2001 From: Matt Bishop Date: Mon, 11 Nov 2024 17:56:46 -0500 Subject: [PATCH] Patch build process --- .github/workflows/build.yml | 676 ++++++++++++++++++++++++++++++++++++ 1 file changed, 676 insertions(+) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000000..1d092d8b4d --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,676 @@ +name: Build + +on: + workflow_dispatch: + push: + branches: + - "main" + - "rc" + - "hotfix-rc" + pull_request_target: + types: [opened, synchronize] + +env: + _AZ_REGISTRY: "bitwardenprod.azurecr.io" + +jobs: + check-run: + name: Check PR run + uses: bitwarden/gh-actions/.github/workflows/check-run.yml@main + + lint: + name: Lint + runs-on: ubuntu-22.04 + needs: + - check-run + steps: + - name: Check out repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up .NET + uses: actions/setup-dotnet@3e891b0cb619bf60e2c25674b222b8940e2c1c25 # v4.1.0 + + - name: Verify format + run: dotnet format --verify-no-changes + + build-artifacts: + name: Build artifacts + runs-on: ubuntu-22.04 + needs: + - lint + strategy: + fail-fast: false + matrix: + include: + - project_name: Admin + base_path: ./src + node: true + - project_name: Api + base_path: ./src + - project_name: Billing + base_path: ./src + - project_name: Events + base_path: ./src + - project_name: EventsProcessor + base_path: ./src + - project_name: Icons + base_path: ./src + - project_name: Identity + base_path: ./src + - project_name: MsSqlMigratorUtility + base_path: ./util + dotnet: true + - project_name: Notifications + base_path: ./src + - project_name: Scim + base_path: ./bitwarden_license/src + dotnet: true + - project_name: Server + base_path: ./util + - project_name: Setup + base_path: ./util + - project_name: Sso + base_path: ./bitwarden_license/src + node: true + steps: + - name: Check out repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up .NET + uses: actions/setup-dotnet@3e891b0cb619bf60e2c25674b222b8940e2c1c25 # v4.1.0 + + - name: Set up Node + uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 + with: + cache: "npm" + cache-dependency-path: "**/package-lock.json" + node-version: "16" + + - name: Print environment + run: | + whoami + dotnet --info + node --version + npm --version + echo "GitHub ref: $GITHUB_REF" + echo "GitHub event: $GITHUB_EVENT" + + - name: Build node + if: ${{ matrix.node }} + working-directory: ${{ matrix.base_path }}/${{ matrix.project_name }} + run: | + npm ci + npm run build + + - name: Publish project + working-directory: ${{ matrix.base_path }}/${{ matrix.project_name }} + run: | + echo "Publish" + dotnet publish -c "Release" -o obj/build-output/publish + + cd obj/build-output/publish + zip -r ${{ matrix.project_name }}.zip . + mv ${{ matrix.project_name }}.zip ../../../ + + pwd + ls -atlh ../../../ + + - name: Upload project artifact + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: ${{ matrix.project_name }}.zip + path: ${{ matrix.base_path }}/${{ matrix.project_name }}/${{ matrix.project_name }}.zip + if-no-files-found: error + + build-docker: + name: Build Docker images + runs-on: ubuntu-22.04 + permissions: + security-events: write + needs: + - build-artifacts + strategy: + fail-fast: false + matrix: + include: + - project_name: Admin + base_path: ./src + dotnet: true + - project_name: Api + base_path: ./src + dotnet: true + - project_name: Attachments + base_path: ./util + - project_name: Billing + base_path: ./src + dotnet: true + - project_name: Events + base_path: ./src + dotnet: true + - project_name: EventsProcessor + base_path: ./src + dotnet: true + - project_name: Icons + base_path: ./src + dotnet: true + - project_name: Identity + base_path: ./src + dotnet: true + - project_name: MsSql + base_path: ./util + - project_name: MsSqlMigratorUtility + base_path: ./util + dotnet: true + - project_name: Nginx + base_path: ./util + - project_name: Notifications + base_path: ./src + dotnet: true + - project_name: Scim + base_path: ./bitwarden_license/src + dotnet: true + - project_name: Server + base_path: ./util + dotnet: true + - project_name: Setup + base_path: ./util + dotnet: true + - project_name: Sso + base_path: ./bitwarden_license/src + dotnet: true + steps: + - name: Check out repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Check branch to publish + env: + PUBLISH_BRANCHES: "main,rc,hotfix-rc" + id: publish-branch-check + run: | + IFS="," read -a publish_branches <<< $PUBLISH_BRANCHES + + if [[ " ${publish_branches[*]} " =~ " ${GITHUB_REF:11} " ]]; then + echo "is_publish_branch=true" >> $GITHUB_ENV + else + echo "is_publish_branch=false" >> $GITHUB_ENV + fi + + ########## ACRs ########## + - name: Log in to Azure - production subscription + uses: Azure/login@e15b166166a8746d1a47596803bd8c1b595455cf # v1.6.0 + with: + creds: ${{ secrets.AZURE_PROD_KV_CREDENTIALS }} + + - name: Log in to ACR - production subscription + run: az acr login -n bitwardenprod + + - name: Log in to Azure - CI subscription + uses: Azure/login@e15b166166a8746d1a47596803bd8c1b595455cf # v1.6.0 + with: + creds: ${{ secrets.AZURE_KV_CI_SERVICE_PRINCIPAL }} + + - name: Retrieve GitHub PAT secrets + id: retrieve-secret-pat + uses: bitwarden/gh-actions/get-keyvault-secrets@main + with: + keyvault: "bitwarden-ci" + secrets: "github-pat-bitwarden-devops-bot-repo-scope" + + ########## Generate image tag and build Docker image ########## + - name: Generate Docker image tag + id: tag + run: | + if [[ "${GITHUB_EVENT_NAME}" == "pull_request_target" ]]; then + IMAGE_TAG=$(echo "${GITHUB_HEAD_REF}" | sed "s#/#-#g") + else + IMAGE_TAG=$(echo "${GITHUB_REF:11}" | sed "s#/#-#g") + fi + + if [[ "$IMAGE_TAG" == "main" ]]; then + IMAGE_TAG=dev + fi + + echo "image_tag=$IMAGE_TAG" >> $GITHUB_OUTPUT + echo "### :mega: Docker Image Tag: $IMAGE_TAG" >> $GITHUB_STEP_SUMMARY + + - name: Set up project name + id: setup + run: | + PROJECT_NAME=$(echo "${{ matrix.project_name }}" | awk '{print tolower($0)}') + echo "Matrix name: ${{ matrix.project_name }}" + echo "PROJECT_NAME: $PROJECT_NAME" + echo "project_name=$PROJECT_NAME" >> $GITHUB_OUTPUT + + - name: Generate image tags(s) + id: image-tags + env: + IMAGE_TAG: ${{ steps.tag.outputs.image_tag }} + PROJECT_NAME: ${{ steps.setup.outputs.project_name }} + SHA: ${{ github.sha }} + run: | + TAGS="${_AZ_REGISTRY}/${PROJECT_NAME}:${IMAGE_TAG}" + echo "primary_tag=$TAGS" >> $GITHUB_OUTPUT + if [[ "${IMAGE_TAG}" == "dev" ]]; then + SHORT_SHA=$(git rev-parse --short ${SHA}) + TAGS=$TAGS",${_AZ_REGISTRY}/${PROJECT_NAME}:dev-${SHORT_SHA}" + fi + echo "tags=$TAGS" >> $GITHUB_OUTPUT + + - name: Get build artifact + if: ${{ matrix.dotnet }} + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: ${{ matrix.project_name }}.zip + + - name: Set up build artifact + if: ${{ matrix.dotnet }} + run: | + mkdir -p ${{ matrix.base_path}}/${{ matrix.project_name }}/obj/build-output/publish + unzip ${{ matrix.project_name }}.zip \ + -d ${{ matrix.base_path }}/${{ matrix.project_name }}/obj/build-output/publish + + - name: Build Docker image + uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6.9.0 + with: + context: ${{ matrix.base_path }}/${{ matrix.project_name }} + file: ${{ matrix.base_path }}/${{ matrix.project_name }}/Dockerfile + platforms: linux/amd64 + push: true + tags: ${{ steps.image-tags.outputs.tags }} + secrets: | + "GH_PAT=${{ steps.retrieve-secret-pat.outputs.github-pat-bitwarden-devops-bot-repo-scope }}" + + - name: Scan Docker image + id: container-scan + uses: anchore/scan-action@5ed195cc06065322983cae4bb31e2a751feb86fd # v5.2.0 + with: + image: ${{ steps.image-tags.outputs.primary_tag }} + fail-build: false + output-format: sarif + + - name: Upload Grype results to GitHub + uses: github/codeql-action/upload-sarif@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0 + with: + sarif_file: ${{ steps.container-scan.outputs.sarif }} + + upload: + name: Upload + runs-on: ubuntu-22.04 + needs: build-docker + steps: + - name: Check out repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up .NET + uses: actions/setup-dotnet@3e891b0cb619bf60e2c25674b222b8940e2c1c25 # v4.1.0 + + - name: Log in to Azure - production subscription + uses: Azure/login@e15b166166a8746d1a47596803bd8c1b595455cf # v1.6.0 + with: + creds: ${{ secrets.AZURE_PROD_KV_CREDENTIALS }} + + - name: Log in to ACR - production subscription + run: az acr login -n $_AZ_REGISTRY --only-show-errors + + - name: Make Docker stubs + if: | + github.event_name != 'pull_request_target' + && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/rc' || github.ref == 'refs/heads/hotfix-rc') + run: | + # Set proper setup image based on branch + case "$GITHUB_REF" in + "refs/heads/main") + SETUP_IMAGE="$_AZ_REGISTRY/setup:dev" + ;; + "refs/heads/rc") + SETUP_IMAGE="$_AZ_REGISTRY/setup:rc" + ;; + "refs/heads/hotfix-rc") + SETUP_IMAGE="$_AZ_REGISTRY/setup:hotfix-rc" + ;; + esac + + STUB_OUTPUT=$(pwd)/docker-stub + + # Run setup + docker run -i --rm --name setup -v $STUB_OUTPUT/US:/bitwarden $SETUP_IMAGE \ + dotnet Setup.dll -stub 1 -install 1 -domain bitwarden.example.com -os lin -cloud-region US + docker run -i --rm --name setup -v $STUB_OUTPUT/EU:/bitwarden $SETUP_IMAGE \ + dotnet Setup.dll -stub 1 -install 1 -domain bitwarden.example.com -os lin -cloud-region EU + + sudo chown -R $(whoami):$(whoami) $STUB_OUTPUT + + # Remove extra directories and files + rm -rf $STUB_OUTPUT/US/letsencrypt + rm -rf $STUB_OUTPUT/EU/letsencrypt + rm $STUB_OUTPUT/US/env/uid.env $STUB_OUTPUT/US/config.yml + rm $STUB_OUTPUT/EU/env/uid.env $STUB_OUTPUT/EU/config.yml + + # Create uid environment files + touch $STUB_OUTPUT/US/env/uid.env + touch $STUB_OUTPUT/EU/env/uid.env + + # Zip up the Docker stub files + cd docker-stub/US; zip -r ../../docker-stub-US.zip *; cd ../.. + cd docker-stub/EU; zip -r ../../docker-stub-EU.zip *; cd ../.. + + - name: Make Docker stub checksums + if: | + github.event_name != 'pull_request_target' + && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/rc' || github.ref == 'refs/heads/hotfix-rc') + run: | + sha256sum docker-stub-US.zip > docker-stub-US-sha256.txt + sha256sum docker-stub-EU.zip > docker-stub-EU-sha256.txt + + - name: Upload Docker stub US artifact + if: | + github.event_name != 'pull_request_target' + && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/rc' || github.ref == 'refs/heads/hotfix-rc') + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: docker-stub-US.zip + path: docker-stub-US.zip + if-no-files-found: error + + - name: Upload Docker stub EU artifact + if: | + github.event_name != 'pull_request_target' + && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/rc' || github.ref == 'refs/heads/hotfix-rc') + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: docker-stub-EU.zip + path: docker-stub-EU.zip + if-no-files-found: error + + - name: Upload Docker stub US checksum artifact + if: | + github.event_name != 'pull_request_target' + && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/rc' || github.ref == 'refs/heads/hotfix-rc') + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: docker-stub-US-sha256.txt + path: docker-stub-US-sha256.txt + if-no-files-found: error + + - name: Upload Docker stub EU checksum artifact + if: | + github.event_name != 'pull_request_target' + && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/rc' || github.ref == 'refs/heads/hotfix-rc') + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: docker-stub-EU-sha256.txt + path: docker-stub-EU-sha256.txt + if-no-files-found: error + + - name: Build Public API Swagger + run: | + cd ./src/Api + echo "Restore tools" + dotnet tool restore + echo "Publish" + dotnet publish -c "Release" -o obj/build-output/publish + + dotnet swagger tofile --output ../../swagger.json --host https://api.bitwarden.com \ + ./obj/build-output/publish/Api.dll public + cd ../.. + env: + ASPNETCORE_ENVIRONMENT: Production + swaggerGen: "True" + DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2 + GLOBALSETTINGS__SQLSERVER__CONNECTIONSTRING: "placeholder" + + - name: Upload Public API Swagger artifact + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: swagger.json + path: swagger.json + if-no-files-found: error + + - name: Build Internal API Swagger + run: | + cd ./src/Api + echo "Restore API tools" + dotnet tool restore + echo "Publish API" + dotnet publish -c "Release" -o obj/build-output/publish + + dotnet swagger tofile --output ../../internal.json --host https://api.bitwarden.com \ + ./obj/build-output/publish/Api.dll internal + + cd ../Identity + + echo "Restore Identity tools" + dotnet tool restore + echo "Publish Identity" + dotnet publish -c "Release" -o obj/build-output/publish + + dotnet swagger tofile --output ../../identity.json --host https://identity.bitwarden.com \ + ./obj/build-output/publish/Identity.dll v1 + cd ../.. + env: + ASPNETCORE_ENVIRONMENT: Development + swaggerGen: "True" + DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2 + GLOBALSETTINGS__SQLSERVER__CONNECTIONSTRING: "placeholder" + + - name: Upload Internal API Swagger artifact + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: internal.json + path: internal.json + if-no-files-found: error + + - name: Upload Identity Swagger artifact + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: identity.json + path: identity.json + if-no-files-found: error + + build-mssqlmigratorutility: + name: Build MSSQL migrator utility + runs-on: ubuntu-22.04 + needs: + - lint + defaults: + run: + shell: bash + working-directory: "util/MsSqlMigratorUtility" + strategy: + fail-fast: false + matrix: + target: + - osx-x64 + - linux-x64 + - win-x64 + steps: + - name: Check out repo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up .NET + uses: actions/setup-dotnet@3e891b0cb619bf60e2c25674b222b8940e2c1c25 # v4.1.0 + + - name: Print environment + run: | + whoami + dotnet --info + echo "GitHub ref: $GITHUB_REF" + echo "GitHub event: $GITHUB_EVENT" + + - name: Publish project + run: | + dotnet publish -c "Release" -o obj/build-output/publish -r ${{ matrix.target }} -p:PublishSingleFile=true \ + -p:IncludeNativeLibrariesForSelfExtract=true --self-contained true + + - name: Upload project artifact for Windows + if: ${{ contains(matrix.target, 'win') == true }} + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: MsSqlMigratorUtility-${{ matrix.target }} + path: util/MsSqlMigratorUtility/obj/build-output/publish/MsSqlMigratorUtility.exe + if-no-files-found: error + + - name: Upload project artifact + if: ${{ contains(matrix.target, 'win') == false }} + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: MsSqlMigratorUtility-${{ matrix.target }} + path: util/MsSqlMigratorUtility/obj/build-output/publish/MsSqlMigratorUtility + if-no-files-found: error + + self-host-build: + name: Trigger self-host build + if: github.event_name != 'pull_request_target' && github.ref == 'refs/heads/main' + runs-on: ubuntu-22.04 + needs: + - build-docker + steps: + - name: Log in to Azure - CI subscription + uses: Azure/login@e15b166166a8746d1a47596803bd8c1b595455cf # v1.6.0 + with: + creds: ${{ secrets.AZURE_KV_CI_SERVICE_PRINCIPAL }} + + - name: Retrieve GitHub PAT secrets + id: retrieve-secret-pat + uses: bitwarden/gh-actions/get-keyvault-secrets@main + with: + keyvault: "bitwarden-ci" + secrets: "github-pat-bitwarden-devops-bot-repo-scope" + + - name: Trigger self-host build + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + github-token: ${{ steps.retrieve-secret-pat.outputs.github-pat-bitwarden-devops-bot-repo-scope }} + script: | + await github.rest.actions.createWorkflowDispatch({ + owner: 'bitwarden', + repo: 'self-host', + workflow_id: 'build-unified.yml', + ref: 'main', + inputs: { + server_branch: process.env.GITHUB_REF + } + }); + + trigger-k8s-deploy: + name: Trigger k8s deploy + if: github.event_name != 'pull_request_target' && github.ref == 'refs/heads/main' + runs-on: ubuntu-22.04 + needs: + - build-docker + steps: + - name: Log in to Azure - CI subscription + uses: Azure/login@e15b166166a8746d1a47596803bd8c1b595455cf # v1.6.0 + with: + creds: ${{ secrets.AZURE_KV_CI_SERVICE_PRINCIPAL }} + + - name: Retrieve GitHub PAT secrets + id: retrieve-secret-pat + uses: bitwarden/gh-actions/get-keyvault-secrets@main + with: + keyvault: "bitwarden-ci" + secrets: "github-pat-bitwarden-devops-bot-repo-scope" + + - name: Trigger k8s deploy + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + github-token: ${{ steps.retrieve-secret-pat.outputs.github-pat-bitwarden-devops-bot-repo-scope }} + script: | + await github.rest.actions.createWorkflowDispatch({ + owner: 'bitwarden', + repo: 'devops', + workflow_id: 'deploy-k8s.yml', + ref: 'main', + inputs: { + environment: 'US-DEV Cloud', + tag: 'main' + } + }) + + trigger-ee-updates: + name: Trigger Ephemeral Environment updates + if: | + github.event_name == 'pull_request_target' + && contains(github.event.pull_request.labels.*.name, 'ephemeral-environment') + runs-on: ubuntu-24.04 + needs: + - build-docker + steps: + - name: Log in to Azure - CI subscription + uses: Azure/login@e15b166166a8746d1a47596803bd8c1b595455cf # v1.6.0 + with: + creds: ${{ secrets.AZURE_KV_CI_SERVICE_PRINCIPAL }} + + - name: Retrieve GitHub PAT secrets + id: retrieve-secret-pat + uses: bitwarden/gh-actions/get-keyvault-secrets@main + with: + keyvault: "bitwarden-ci" + secrets: "github-pat-bitwarden-devops-bot-repo-scope" + + - name: Trigger Ephemeral Environment update + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + github-token: ${{ steps.retrieve-secret-pat.outputs.github-pat-bitwarden-devops-bot-repo-scope }} + script: | + await github.rest.actions.createWorkflowDispatch({ + owner: 'bitwarden', + repo: 'devops', + workflow_id: '_update_ephemeral_tags.yml', + ref: 'main', + inputs: { + ephemeral_env_branch: process.env.GITHUB_HEAD_REF + } + }) + + check-failures: + name: Check for failures + if: always() + runs-on: ubuntu-22.04 + needs: + - lint + - build-artifacts + - build-docker + - upload + - build-mssqlmigratorutility + - self-host-build + - trigger-k8s-deploy + steps: + - name: Check if any job failed + if: | + github.event_name != 'pull_request_target' + && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/rc' || github.ref == 'refs/heads/hotfix-rc') + && contains(needs.*.result, 'failure') + run: exit 1 + + - name: Log in to Azure - CI subscription + uses: Azure/login@e15b166166a8746d1a47596803bd8c1b595455cf # v1.6.0 + if: failure() + with: + creds: ${{ secrets.AZURE_KV_CI_SERVICE_PRINCIPAL }} + + - name: Retrieve secrets + id: retrieve-secrets + uses: bitwarden/gh-actions/get-keyvault-secrets@main + if: failure() + with: + keyvault: "bitwarden-ci" + secrets: "devops-alerts-slack-webhook-url" + + - name: Notify Slack on failure + uses: act10ns/slack@44541246747a30eb3102d87f7a4cc5471b0ffb7d # v2.1.0 + if: failure() + env: + SLACK_WEBHOOK_URL: ${{ steps.retrieve-secrets.outputs.devops-alerts-slack-webhook-url }} + with: + status: ${{ job.status }}