diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
deleted file mode 100644
index 231ceaf4..00000000
--- a/.github/workflows/publish.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-name: Publish package to GitHub Packages
-
-on:
-  release:
-    types: [created]
-
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-java@v3
-        with:
-          java-version: '11'
-          distribution: 'adopt'
-          cache: gradle
-
-      - name: Get release
-        id: get_release
-        uses: bruceadams/get-release@v1.3.2
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-
-      - name: Validate Gradle wrapper
-        uses: gradle/wrapper-validation-action@v1
-
-      - name: Publish package
-        uses: gradle/gradle-build-action@v2
-        with:
-          arguments: publish
-        env:
-          GITHUB_VERSION: ${{ steps.get_release.outputs.tag }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Upload release artifact
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          upload_url: ${{ steps.get_release.outputs.upload_url }}
-          asset_path: build/libs/Multiverse-Core-${{ steps.get_release.outputs.tag }}.jar
-          asset_name: Multiverse-Core-${{ steps.get_release.outputs.tag }}.jar
-          asset_content_type: application/java-archive
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 1e4fb64d..5616b654 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -8,6 +8,24 @@ jobs:
   release_on_push:
     runs-on: ubuntu-latest
     steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-java@v3
+        with:
+          java-version: '11'
+          distribution: 'adopt'
+          cache: gradle
+
+      - name: Validate Gradle wrapper
+        uses: gradle/wrapper-validation-action@v1
+
+      - name: Test & Build
+        uses: gradle/gradle-build-action@v2
+        with:
+          arguments: clean build -x assemble -x shadowJar
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Create release
         uses: rymndhng/release-on-push-action@v0.27.0
         env:
@@ -17,3 +35,27 @@ jobs:
           tag_prefix: v
           release_name: "Release <RELEASE_VERSION>"
           use_github_release_notes: true
+
+      - name: Get release
+        id: get_release
+        uses: bruceadams/get-release@v1.3.2
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+
+      - name: Publish package
+        uses: gradle/gradle-build-action@v2
+        with:
+          arguments: publish
+        env:
+          GITHUB_VERSION: ${{ steps.get_release.outputs.tag }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload release artifact
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.get_release.outputs.upload_url }}
+          asset_path: build/libs/Multiverse-Core-${{ steps.get_release.outputs.tag }}.jar
+          asset_name: Multiverse-Core-${{ steps.get_release.outputs.tag }}.jar
+          asset_content_type: application/java-archive
diff --git a/build.gradle b/build.gradle
index 54e9122b..58c45821 100644
--- a/build.gradle
+++ b/build.gradle
@@ -132,6 +132,10 @@ processResources {
     filesMatching('plugin.yml') {
         expand props
     }
+
+    // This task should never be skipped. The tests depend on this having been run but we want the new version number
+    // that is created after tests are run and before we run again to publish.
+    outputs.upToDateWhen { false }
 }