diff --git a/packages/block-editor/src/components/inserter/block-patterns-tab/index.js b/packages/block-editor/src/components/inserter/block-patterns-tab/index.js
index 141ebf8cc84016..e3bf1d96b831cd 100644
--- a/packages/block-editor/src/components/inserter/block-patterns-tab/index.js
+++ b/packages/block-editor/src/components/inserter/block-patterns-tab/index.js
@@ -1,7 +1,7 @@
 /**
  * WordPress dependencies
  */
-import { useState } from '@wordpress/element';
+import { useState, useEffect } from '@wordpress/element';
 import { useViewportMatch } from '@wordpress/compose';
 import { Button, Spinner } from '@wordpress/components';
 import { __ } from '@wordpress/i18n';
@@ -24,6 +24,7 @@ function BlockPatternsTab( {
 	selectedCategory,
 	onInsert,
 	rootClientId,
+	setHasCategories,
 	children,
 } ) {
 	const [ showPatternsExplorer, setShowPatternsExplorer ] = useState( false );
@@ -37,6 +38,10 @@ function BlockPatternsTab( {
 		[]
 	);
 
+	useEffect( () => {
+		setHasCategories( !! categories.length );
+	}, [ categories, setHasCategories ] );
+
 	if ( isResolvingPatterns ) {
 		return (
 			<div className="block-editor-inserter__patterns-loading">
diff --git a/packages/block-editor/src/components/inserter/media-tab/media-tab.js b/packages/block-editor/src/components/inserter/media-tab/media-tab.js
index ca901a127e5b42..02d60df64cd559 100644
--- a/packages/block-editor/src/components/inserter/media-tab/media-tab.js
+++ b/packages/block-editor/src/components/inserter/media-tab/media-tab.js
@@ -4,7 +4,7 @@
 import { __ } from '@wordpress/i18n';
 import { useViewportMatch } from '@wordpress/compose';
 import { Button } from '@wordpress/components';
-import { useCallback, useMemo } from '@wordpress/element';
+import { useCallback, useEffect, useMemo } from '@wordpress/element';
 
 /**
  * Internal dependencies
@@ -24,6 +24,7 @@ function MediaTab( {
 	rootClientId,
 	selectedCategory,
 	onSelectCategory,
+	setHasCategories,
 	onInsert,
 	children,
 } ) {
@@ -49,6 +50,10 @@ function MediaTab( {
 		[ mediaCategories ]
 	);
 
+	useEffect( () => {
+		setHasCategories( !! categories.length );
+	}, [ categories, setHasCategories ] );
+
 	if ( ! categories.length ) {
 		return <InserterNoResults />;
 	}
diff --git a/packages/block-editor/src/components/inserter/menu.js b/packages/block-editor/src/components/inserter/menu.js
index ea273f1284342b..fe62dfcd94dbab 100644
--- a/packages/block-editor/src/components/inserter/menu.js
+++ b/packages/block-editor/src/components/inserter/menu.js
@@ -67,6 +67,8 @@ function InserterMenu(
 	const [ patternFilter, setPatternFilter ] = useState( 'all' );
 	const [ selectedMediaCategory, setSelectedMediaCategory ] =
 		useState( null );
+
+	const [ hasCategories, setHasCategories ] = useState( true );
 	function getInitialTab() {
 		if ( __experimentalInitialTab ) {
 			return __experimentalInitialTab;
@@ -146,10 +148,12 @@ function InserterMenu(
 
 	const showPatternPanel =
 		selectedTab === 'patterns' &&
+		hasCategories &&
 		! delayedFilterValue &&
 		!! selectedPatternCategory;
 
-	const showMediaPanel = selectedTab === 'media' && !! selectedMediaCategory;
+	const showMediaPanel =
+		selectedTab === 'media' && !! selectedMediaCategory && hasCategories;
 
 	const inserterSearch = useMemo( () => {
 		if ( selectedTab === 'media' ) {
@@ -242,6 +246,7 @@ function InserterMenu(
 				onInsert={ onInsertPattern }
 				onSelectCategory={ onClickPatternCategory }
 				selectedCategory={ selectedPatternCategory }
+				setHasCategories={ setHasCategories }
 			>
 				{ showPatternPanel && (
 					<PatternCategoryPreviews
@@ -270,6 +275,7 @@ function InserterMenu(
 				selectedCategory={ selectedMediaCategory }
 				onSelectCategory={ setSelectedMediaCategory }
 				onInsert={ onInsert }
+				setHasCategories={ setHasCategories }
 			>
 				{ showMediaPanel && (
 					<MediaCategoryPanel