diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml
index 2bae5a6116..2e9eb85780 100644
--- a/.github/workflows/labeler.yml
+++ b/.github/workflows/labeler.yml
@@ -139,6 +139,21 @@ jobs:
             const experiencedLabel = "experienced-contributor";
             const trustedThreshold = 4;
             const experiencedThreshold = 10;
+            const issueNumber = context.payload.pull_request.number;
+
+            const removeLabelIfPresent = async (name) => {
+              try {
+                await github.rest.issues.removeLabel({
+                  ...context.repo,
+                  issue_number: issueNumber,
+                  name,
+                });
+              } catch (error) {
+                if (error?.status !== 404) {
+                  throw error;
+                }
+              }
+            };
 
             let isMaintainer = false;
             try {
@@ -157,7 +172,7 @@ jobs:
             if (isMaintainer) {
               await github.rest.issues.addLabels({
                 ...context.repo,
-                issue_number: context.payload.pull_request.number,
+                issue_number: issueNumber,
                 labels: ["maintainer"],
               });
               return;
@@ -179,9 +194,10 @@ jobs:
             }
 
             if (mergedCount >= experiencedThreshold) {
+              await removeLabelIfPresent(trustedLabel);
               await github.rest.issues.addLabels({
                 ...context.repo,
-                issue_number: context.payload.pull_request.number,
+                issue_number: issueNumber,
                 labels: [experiencedLabel],
               });
               return;
@@ -190,7 +206,7 @@ jobs:
             if (mergedCount >= trustedThreshold) {
               await github.rest.issues.addLabels({
                 ...context.repo,
-                issue_number: context.payload.pull_request.number,
+                issue_number: issueNumber,
                 labels: [trustedLabel],
               });
             }
@@ -373,6 +389,22 @@ jobs:
                 return;
               }
 
+              if (label === experiencedLabel && labelNames.has(trustedLabel)) {
+                try {
+                  await github.rest.issues.removeLabel({
+                    owner,
+                    repo,
+                    issue_number: pullRequest.number,
+                    name: trustedLabel,
+                  });
+                } catch (error) {
+                  if (error?.status !== 404) {
+                    throw error;
+                  }
+                }
+                labelNames.delete(trustedLabel);
+              }
+
               if (labelNames.has(label)) {
                 return;
               }
@@ -462,6 +494,21 @@ jobs:
             const experiencedLabel = "experienced-contributor";
             const trustedThreshold = 4;
             const experiencedThreshold = 10;
+            const issueNumber = context.payload.issue.number;
+
+            const removeLabelIfPresent = async (name) => {
+              try {
+                await github.rest.issues.removeLabel({
+                  ...context.repo,
+                  issue_number: issueNumber,
+                  name,
+                });
+              } catch (error) {
+                if (error?.status !== 404) {
+                  throw error;
+                }
+              }
+            };
 
             let isMaintainer = false;
             try {
@@ -480,7 +527,7 @@ jobs:
             if (isMaintainer) {
               await github.rest.issues.addLabels({
                 ...context.repo,
-                issue_number: context.payload.issue.number,
+                issue_number: issueNumber,
                 labels: ["maintainer"],
               });
               return;
@@ -502,9 +549,10 @@ jobs:
             }
 
             if (mergedCount >= experiencedThreshold) {
+              await removeLabelIfPresent(trustedLabel);
               await github.rest.issues.addLabels({
                 ...context.repo,
-                issue_number: context.payload.issue.number,
+                issue_number: issueNumber,
                 labels: [experiencedLabel],
               });
               return;
@@ -513,7 +561,7 @@ jobs:
             if (mergedCount >= trustedThreshold) {
               await github.rest.issues.addLabels({
                 ...context.repo,
-                issue_number: context.payload.issue.number,
+                issue_number: issueNumber,
                 labels: [trustedLabel],
               });
             }