diff --git a/packages/components/src/color-palette/index.tsx b/packages/components/src/color-palette/index.tsx
index de4e4f4206fe3a..d9d7b82d542a7f 100644
--- a/packages/components/src/color-palette/index.tsx
+++ b/packages/components/src/color-palette/index.tsx
@@ -241,11 +241,11 @@ function UnforwardedColorPalette(
 		value,
 	};
 
-	const actions = !! clearable && (
+	const actions = (
 		<CircularOptionPicker.ButtonAction
 			onClick={ clearColor }
 			accessibleWhenDisabled
-			disabled={ ! value }
+			disabled={ ! value || ! clearable }
 		>
 			{ __( 'Clear' ) }
 		</CircularOptionPicker.ButtonAction>