From 32ee7d9bfb81ba8672e33cde0ed6cb27e7776f2b Mon Sep 17 00:00:00 2001 From: JuanMa Garrido Date: Fri, 5 Jul 2024 17:07:59 +0100 Subject: [PATCH 1/2] formatter to raw and phpdoc-parser --- bin/api-docs/update-api-docs-json.js | 248 +++++++++++++++++++++ package.json | 1 + packages/docgen/lib/json/index.js | 3 + packages/docgen/lib/phpdoc-parser/index.js | 35 +++ packages/docgen/lib/phpdoc-parser/utils.js | 65 ++++++ 5 files changed, 352 insertions(+) create mode 100755 bin/api-docs/update-api-docs-json.js create mode 100644 packages/docgen/lib/json/index.js create mode 100644 packages/docgen/lib/phpdoc-parser/index.js create mode 100644 packages/docgen/lib/phpdoc-parser/utils.js diff --git a/bin/api-docs/update-api-docs-json.js b/bin/api-docs/update-api-docs-json.js new file mode 100755 index 0000000000000..969a79f322220 --- /dev/null +++ b/bin/api-docs/update-api-docs-json.js @@ -0,0 +1,248 @@ +/** + * External dependencies + */ +const { join, relative, resolve, sep, dirname } = require( 'path' ); +const glob = require( 'fast-glob' ); +const execa = require( 'execa' ); +const { Transform } = require( 'stream' ); +const { readFile } = require( 'fs' ).promises; + +/** + * README file tokens, defined as a tuple of token identifier, source path. + * + * @typedef {[string,string]} WPReadmeFileTokens + */ + +/** + * README file data, defined as a tuple of README file path, token details. + * + * @typedef {[string,WPReadmeFileTokens]} WPReadmeFileData + */ + +/** + * Path to root project directory. + * + * @type {string} + */ +const ROOT_DIR = resolve( __dirname, '../..' ).replace( /\\/g, '/' ); + +/** + * Path to packages directory. + * + * @type {string} + */ +const PACKAGES_DIR = resolve( ROOT_DIR, 'packages' ).replace( /\\/g, '/' ); + +/** + * Path to data documentation directory. + * + * @type {string} + */ +const DATA_DOCS_DIR = resolve( ROOT_DIR, 'docs/reference-guides/data' ).replace( + /\\/g, + '/' +); + +/** + * Pattern matching start token of a README file. + * + * @example Delimiter tokens that use the default source file: + * + * // content within will be filled by docgen + * + * + * @example Delimiter tokens that use a specific source file: + * + * // content within will be filled by docgen + * + * + * @type {RegExp} + */ +const TOKEN_PATTERN = //g; + +/** + * Given an absolute file path, returns the package name. + * + * @param {string} file Absolute path. + * + * @return {string} Package name. + */ +function getFilePackage( file ) { + return relative( PACKAGES_DIR, file ).split( sep )[ 0 ]; +} + +/** + * Returns an appropriate glob pattern for the packages directory to match + * relevant documentation files for a given set of files. + * + * @param {string[]} files Set of files to match. Pass an empty set to match + * all packages. + * + * @return {string} Packages glob pattern. + */ +function getPackagePattern( files ) { + if ( ! files.length ) { + return '*'; + } + + // Since brace expansion doesn't work with a single package, special-case + // the pattern for the singular match. + const packages = Array.from( new Set( files.map( getFilePackage ) ) ); + return packages.length === 1 ? packages[ 0 ] : '{' + packages.join() + '}'; +} + +/** + * Returns the conventional store name of a given package. + * + * @param {string} packageName Package name. + * + * @return {string} Store name. + */ +function getPackageStoreName( packageName ) { + let storeName = 'core'; + if ( packageName !== 'core-data' ) { + storeName += '/' + packageName; + } + + return storeName; +} + +/** + * Returns the conventional documentation file name of a given package. + * + * @param {string} packageName Package name. + * + * @return {string} Documentation file name. + */ +function getDataDocumentationFile( packageName ) { + const storeName = getPackageStoreName( packageName ); + return `data-${ storeName.replace( '/', '-' ) }.md`; +} + +/** + * Returns an appropriate glob pattern for the data documentation directory to + * match relevant documentation files for a given set of files. + * + * @param {string[]} files Set of files to match. Pass an empty set to match + * all packages. + * + * @return {string} Packages glob pattern. + */ +function getDataDocumentationPattern( files ) { + if ( ! files.length ) { + return '*'; + } + + // Since brace expansion doesn't work with a single package, special-case + // the pattern for the singular match. + const filePackages = Array.from( new Set( files.map( getFilePackage ) ) ); + const docFiles = filePackages.map( getDataDocumentationFile ); + + return docFiles.length === 1 ? docFiles[ 0 ] : '{' + docFiles.join() + '}'; +} + +/** + * Stream transform which filters out README files to include only those + * containing matched token pattern, yielding a tuple of the file and its + * matched tokens. + * + * @type {Transform} + */ +const filterTokenTransform = new Transform( { + objectMode: true, + + async transform( file, _encoding, callback ) { + let content; + try { + content = await readFile( file, 'utf8' ); + } catch {} + + if ( content ) { + const tokens = []; + + for ( const match of content.matchAll( TOKEN_PATTERN ) ) { + const [ , token, path ] = match; + tokens.push( [ token, path ] ); + } + + if ( tokens.length ) { + this.push( [ file, tokens ] ); + } + } + + callback(); + }, +} ); + +/** + * Find default source file (`src/index.{js,ts,tsx}`) in a specified package directory + * + * @param {string} dir Package directory to search in + * @return {string} Name of matching file + */ +function findDefaultSourcePath( dir ) { + const defaultPathMatches = glob.sync( 'src/index.{js,ts,tsx}', { + cwd: dir, + } ); + if ( ! defaultPathMatches.length ) { + throw new Error( `Cannot find default source file in ${ dir }` ); + } + // @ts-ignore + return defaultPathMatches[ 0 ]; +} + +/** + * Optional process arguments for which to generate documentation. + * + * @type {string[]} + */ +const files = process.argv.slice( 2 ); + +glob.stream( [ + `${ PACKAGES_DIR }/${ getPackagePattern( files ) }/README.md`, + `${ DATA_DOCS_DIR }/${ getDataDocumentationPattern( files ) }`, +] ) + .pipe( filterTokenTransform ) + .on( 'data', async ( /** @type {WPReadmeFileData} */ data ) => { + const [ file, tokens ] = data; + const output = relative( ROOT_DIR, file ); + + // Each file can have more than one placeholder content to update, each + // represented by tokens. The docgen script updates one token at a time, + // so the tokens must be replaced in sequence to prevent the processes + // from overriding each other. + try { + for ( const [ + , + path = findDefaultSourcePath( dirname( file ) ), + ] of tokens ) { + const { stdout } = await execa( + `"${ join( + __dirname, + '..', + '..', + 'node_modules', + '.bin', + 'docgen' + ) }"`, + [ + relative( ROOT_DIR, resolve( dirname( file ), path ) ), + `--output ${ output.replace( + 'README', + file + ) }.replace(".md", ".json") }`, + `--formatter=./packages/docgen/lib/phpdoc-parser/index.js`, + '--ignore "/unstable|experimental/i"', + ], + { shell: true } + ); + console.log( stdout ); + } + await execa( 'npm', [ 'run', 'format', output ], { + shell: true, + } ); + } catch ( error ) { + console.error( error ); + process.exit( 1 ); + } + } ); diff --git a/package.json b/package.json index bad9ebd48e92d..218a6c4dba039 100644 --- a/package.json +++ b/package.json @@ -280,6 +280,7 @@ "dev:packages": "cross-env NODE_ENV=development concurrently \"node ./bin/packages/watch.js\" \"tsc --build --watch\"", "distclean": "git clean --force -d -X", "docs:api-ref": "node ./bin/api-docs/update-api-docs.js", + "docs:api-ref-json": "node ./bin/api-docs/update-api-docs-json.js", "docs:blocks": "node ./bin/api-docs/gen-block-lib-list.js", "docs:build": "npm-run-all docs:gen docs:blocks docs:api-ref docs:theme-ref", "docs:gen": "node ./docs/tool/index.js", diff --git a/packages/docgen/lib/json/index.js b/packages/docgen/lib/json/index.js new file mode 100644 index 0000000000000..72b9ed3d50911 --- /dev/null +++ b/packages/docgen/lib/json/index.js @@ -0,0 +1,3 @@ +module.exports = ( rootDir, docPath, symbols ) => { + return JSON.stringify( symbols, null, 2 ); +}; diff --git a/packages/docgen/lib/phpdoc-parser/index.js b/packages/docgen/lib/phpdoc-parser/index.js new file mode 100644 index 0000000000000..87b64a4789520 --- /dev/null +++ b/packages/docgen/lib/phpdoc-parser/index.js @@ -0,0 +1,35 @@ +/** + * Internal dependencies + */ +const { + groupByPath, + getFirstPropertyName, + adaptFunctionsFormat, +} = require( './utils' ); + +module.exports = ( rootDir, docPath, symbols ) => { + const symbolsGroupedByPath = groupByPath( symbols ); + const path = getFirstPropertyName( symbolsGroupedByPath ); + + if ( symbolsGroupedByPath[ path ] === undefined ) { + return JSON.stringify( symbols, null, 2 ); + } + + const phpDocParserJSON = { + file: { + description: path, + tags: [ + { + name: 'package', + content: 'WordPress', + }, + ], + }, + path, + root: '/', + includes: [], + functions: symbolsGroupedByPath[ path ].map( adaptFunctionsFormat ), + }; + + return JSON.stringify( phpDocParserJSON, null, 2 ); +}; diff --git a/packages/docgen/lib/phpdoc-parser/utils.js b/packages/docgen/lib/phpdoc-parser/utils.js new file mode 100644 index 0000000000000..7274afaf1927f --- /dev/null +++ b/packages/docgen/lib/phpdoc-parser/utils.js @@ -0,0 +1,65 @@ +const getFirstPropertyName = ( obj ) => { + const keys = Object.keys( obj ); + return keys.length > 0 ? keys[ 0 ] : undefined; +}; + +const groupByPath = ( array ) => { + return array.reduce( ( acc, obj ) => { + const key = obj.path; + if ( ! acc[ key ] ) { + acc[ key ] = []; + } + acc[ key ].push( obj ); + return acc; + }, {} ); +}; + +const adaptFunctionsFormat = ( symbolFunction ) => { + const { + description: functionDescription, + name: functionName, + tags: functionTags, + } = symbolFunction; + return { + name: functionName, + line: symbolFunction.lineStart, + end_line: symbolFunction.lineEnd, + arguments: functionTags + .filter( ( tagItem ) => tagItem.tag === 'param' ) + .map( + ( { + name: paramName, + type: paramType, + description: paramDescription, + } ) => ( { + name: paramName, + type: paramType, + description: paramDescription, + } ) + ), + docs: { + description: functionDescription, + tags: functionTags + .filter( ( tagItem ) => + [ 'param', 'return' ].includes( tagItem.tag ) + ) + .map( + ( { + tag: paramOrReturn, + name: paramOrReturnName, + type: paramOrReturnDataType, + description: paramOrReturnDescription, + } ) => { + return { + name: paramOrReturn, + variable: paramOrReturnName, + type: paramOrReturnDataType, + content: paramOrReturnDescription, + }; + } + ), + }, + }; +}; + +module.exports = { groupByPath, getFirstPropertyName, adaptFunctionsFormat }; From d99ba99f3e229ea169c06cbd3ee96d8e72e51017 Mon Sep 17 00:00:00 2001 From: JuanMa Garrido Date: Fri, 5 Jul 2024 17:26:27 +0100 Subject: [PATCH 2/2] fixed issue and removed console log --- bin/api-docs/update-api-docs-json.js | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/bin/api-docs/update-api-docs-json.js b/bin/api-docs/update-api-docs-json.js index 969a79f322220..b68838b9eda74 100755 --- a/bin/api-docs/update-api-docs-json.js +++ b/bin/api-docs/update-api-docs-json.js @@ -216,7 +216,7 @@ glob.stream( [ , path = findDefaultSourcePath( dirname( file ) ), ] of tokens ) { - const { stdout } = await execa( + await execa( `"${ join( __dirname, '..', @@ -227,16 +227,12 @@ glob.stream( [ ) }"`, [ relative( ROOT_DIR, resolve( dirname( file ), path ) ), - `--output ${ output.replace( - 'README', - file - ) }.replace(".md", ".json") }`, + `--output ${ output.replace( '.md', '.json' ) }`, `--formatter=./packages/docgen/lib/phpdoc-parser/index.js`, '--ignore "/unstable|experimental/i"', ], { shell: true } ); - console.log( stdout ); } await execa( 'npm', [ 'run', 'format', output ], { shell: true,