diff --git a/packages/format-library/src/unknown/index.js b/packages/format-library/src/unknown/index.js
index 6bfe782c7d19a..b2da23388c79b 100644
--- a/packages/format-library/src/unknown/index.js
+++ b/packages/format-library/src/unknown/index.js
@@ -2,33 +2,39 @@
  * WordPress dependencies
  */
 import { __ } from '@wordpress/i18n';
-import { removeFormat, slice } from '@wordpress/rich-text';
+import { removeFormat, slice, isCollapsed } from '@wordpress/rich-text';
 import { RichTextToolbarButton } from '@wordpress/block-editor';
 import { help } from '@wordpress/icons';
 
 const name = 'core/unknown';
 const title = __( 'Clear Unknown Formatting' );
 
+function selectionContainsUnknownFormats( value ) {
+	if ( isCollapsed( value ) ) {
+		return false;
+	}
+
+	const selectedValue = slice( value );
+	return selectedValue.formats.some( ( formats ) => {
+		return formats.some( ( format ) => format.type === name );
+	} );
+}
+
 export const unknown = {
 	name,
 	title,
 	tagName: '*',
 	className: null,
 	edit( { isActive, value, onChange, onFocus } ) {
+		if ( ! isActive && ! selectionContainsUnknownFormats( value ) ) {
+			return null;
+		}
+
 		function onClick() {
 			onChange( removeFormat( value, name ) );
 			onFocus();
 		}
 
-		const selectedValue = slice( value );
-		const hasUnknownFormats = selectedValue.formats.some( ( formats ) => {
-			return formats.some( ( format ) => format.type === name );
-		} );
-
-		if ( ! isActive && ! hasUnknownFormats ) {
-			return null;
-		}
-
 		return (
 			<RichTextToolbarButton
 				name="unknown"