From 2c1f88369ef65eb51edf3dfa38cfb5362535a338 Mon Sep 17 00:00:00 2001 From: Ben Keen Date: Wed, 9 Apr 2025 13:55:27 -0700 Subject: [PATCH 1/2] Add bridge-cache plugin to populate cache without running action --- apps/rush/package.json | 1 + apps/rush/src/start-dev-docs.ts | 2 +- apps/rush/src/start-dev.ts | 1 + .../rush/browser-approved-packages.json | 4 + .../config/subspaces/default/pnpm-lock.yaml | 24 ++++ common/reviews/api/rush-lib.api.md | 26 ++++- libraries/rush-lib/src/index.ts | 2 + .../src/logic/buildCache/ProjectBuildCache.ts | 3 + .../src/pluginFramework/PluginManager.ts | 1 + .../rush-bridge-cache-plugin/.eslintrc.js | 13 +++ rush-plugins/rush-bridge-cache-plugin/LICENSE | 24 ++++ .../rush-bridge-cache-plugin/README.md | 44 +++++++ .../command-line.json | 11 ++ .../config/jest.config.json | 3 + .../rush-bridge-cache-plugin/config/rig.json | 7 ++ .../rush-bridge-cache-plugin/package.json | 26 +++++ .../rush-plugin-manifest.json | 11 ++ .../src/BridgeCachePlugin.ts | 107 ++++++++++++++++++ .../rush-bridge-cache-plugin/src/index.ts | 4 + .../rush-bridge-cache-plugin/tsconfig.json | 3 + rush.json | 14 ++- 21 files changed, 322 insertions(+), 9 deletions(-) create mode 100644 rush-plugins/rush-bridge-cache-plugin/.eslintrc.js create mode 100644 rush-plugins/rush-bridge-cache-plugin/LICENSE create mode 100644 rush-plugins/rush-bridge-cache-plugin/README.md create mode 100644 rush-plugins/rush-bridge-cache-plugin/command-line.json create mode 100644 rush-plugins/rush-bridge-cache-plugin/config/jest.config.json create mode 100644 rush-plugins/rush-bridge-cache-plugin/config/rig.json create mode 100644 rush-plugins/rush-bridge-cache-plugin/package.json create mode 100644 rush-plugins/rush-bridge-cache-plugin/rush-plugin-manifest.json create mode 100644 rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts create mode 100644 rush-plugins/rush-bridge-cache-plugin/src/index.ts create mode 100644 rush-plugins/rush-bridge-cache-plugin/tsconfig.json diff --git a/apps/rush/package.json b/apps/rush/package.json index a8550f0026e..f42863edc54 100644 --- a/apps/rush/package.json +++ b/apps/rush/package.json @@ -47,6 +47,7 @@ "@rushstack/rush-amazon-s3-build-cache-plugin": "workspace:*", "@rushstack/rush-azure-storage-build-cache-plugin": "workspace:*", "@rushstack/rush-http-build-cache-plugin": "workspace:*", + "@rushstack/rush-bridge-cache-plugin": "workspace:*", "@types/heft-jest": "1.0.1", "@types/semver": "7.5.0" } diff --git a/apps/rush/src/start-dev-docs.ts b/apps/rush/src/start-dev-docs.ts index be5089d5130..4bd9539a8bb 100644 --- a/apps/rush/src/start-dev-docs.ts +++ b/apps/rush/src/start-dev-docs.ts @@ -6,4 +6,4 @@ import { Colorize, ConsoleTerminalProvider, Terminal } from '@rushstack/terminal const terminal: Terminal = new Terminal(new ConsoleTerminalProvider()); terminal.writeLine('For instructions on debugging Rush, please see this documentation:'); -terminal.writeLine(Colorize.bold('https://rushjs.io/pages/contributing/debugging/')); +terminal.writeLine(Colorize.bold('https://rushjs.io/pages/contributing/#debugging-rush')); diff --git a/apps/rush/src/start-dev.ts b/apps/rush/src/start-dev.ts index 1660da8628d..6b1f0e1201e 100644 --- a/apps/rush/src/start-dev.ts +++ b/apps/rush/src/start-dev.ts @@ -31,6 +31,7 @@ includePlugin('rush-azure-storage-build-cache-plugin'); includePlugin('rush-http-build-cache-plugin'); // Including this here so that developers can reuse it without installing the plugin a second time includePlugin('rush-azure-interactive-auth-plugin', '@rushstack/rush-azure-storage-build-cache-plugin'); +includePlugin('rush-bridge-cache-plugin'); const currentPackageVersion: string = PackageJsonLookup.loadOwnPackageJson(__dirname).version; RushCommandSelector.execute(currentPackageVersion, rushLib, { diff --git a/common/config/rush/browser-approved-packages.json b/common/config/rush/browser-approved-packages.json index 7a5edb8f445..f1a149df3cf 100644 --- a/common/config/rush/browser-approved-packages.json +++ b/common/config/rush/browser-approved-packages.json @@ -38,6 +38,10 @@ "name": "@reduxjs/toolkit", "allowedCategories": [ "libraries", "vscode-extensions" ] }, + { + "name": "@rushstack/rush-bridge-cache-plugin", + "allowedCategories": [ "libraries" ] + }, { "name": "@rushstack/rush-themed-ui", "allowedCategories": [ "libraries" ] diff --git a/common/config/subspaces/default/pnpm-lock.yaml b/common/config/subspaces/default/pnpm-lock.yaml index 91ecee53d3e..04793511b3f 100644 --- a/common/config/subspaces/default/pnpm-lock.yaml +++ b/common/config/subspaces/default/pnpm-lock.yaml @@ -332,6 +332,9 @@ importers: '@rushstack/rush-azure-storage-build-cache-plugin': specifier: workspace:* version: link:../../rush-plugins/rush-azure-storage-build-cache-plugin + '@rushstack/rush-bridge-cache-plugin': + specifier: workspace:* + version: link:../../rush-plugins/rush-bridge-cache-plugin '@rushstack/rush-http-build-cache-plugin': specifier: workspace:* version: link:../../rush-plugins/rush-http-build-cache-plugin @@ -3913,6 +3916,27 @@ importers: specifier: workspace:* version: link:../../rigs/local-node-rig + ../../../rush-plugins/rush-bridge-cache-plugin: + devDependencies: + '@microsoft/rush-lib': + specifier: workspace:* + version: link:../../libraries/rush-lib + '@rushstack/heft': + specifier: workspace:* + version: link:../../apps/heft + '@rushstack/node-core-library': + specifier: workspace:* + version: link:../../libraries/node-core-library + '@rushstack/rush-sdk': + specifier: workspace:* + version: link:../../libraries/rush-sdk + '@rushstack/terminal': + specifier: workspace:* + version: link:../../libraries/terminal + local-node-rig: + specifier: workspace:* + version: link:../../rigs/local-node-rig + ../../../rush-plugins/rush-buildxl-graph-plugin: dependencies: '@rushstack/node-core-library': diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 678053cc27c..846354cea6e 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -10,7 +10,7 @@ import { AsyncParallelHook } from 'tapable'; import { AsyncSeriesBailHook } from 'tapable'; import { AsyncSeriesHook } from 'tapable'; import { AsyncSeriesWaterfallHook } from 'tapable'; -import type { CollatedWriter } from '@rushstack/stream-collator'; +import { CollatedWriter } from '@rushstack/stream-collator'; import type { CommandLineParameter } from '@rushstack/ts-command-line'; import { CommandLineParameterKind } from '@rushstack/ts-command-line'; import { HookMap } from 'tapable'; @@ -23,7 +23,8 @@ import { JsonNull } from '@rushstack/node-core-library'; import { JsonObject } from '@rushstack/node-core-library'; import { LookupByPath } from '@rushstack/lookup-by-path'; import { PackageNameParser } from '@rushstack/node-core-library'; -import type { StdioSummarizer } from '@rushstack/terminal'; +import { StdioSummarizer } from '@rushstack/terminal'; +import { StreamCollator } from '@rushstack/stream-collator'; import { SyncHook } from 'tapable'; import { SyncWaterfallHook } from 'tapable'; import { Terminal } from '@rushstack/terminal'; @@ -1127,6 +1128,27 @@ export type PnpmStoreLocation = 'local' | 'global'; // @public @deprecated (undocumented) export type PnpmStoreOptions = PnpmStoreLocation; +// Warning: (ae-internal-missing-underscore) The name "ProjectBuildCache" should be prefixed with an underscore because the declaration is marked as @internal +// +// @internal (undocumented) +export class ProjectBuildCache { + // (undocumented) + get cacheId(): string | undefined; + // Warning: (ae-forgotten-export) The symbol "OperationExecutionRecord" needs to be exported by the entry point index.d.ts + // Warning: (ae-forgotten-export) The symbol "IOperationBuildCacheOptions" needs to be exported by the entry point index.d.ts + // + // (undocumented) + static forOperation(operation: OperationExecutionRecord, options: IOperationBuildCacheOptions): ProjectBuildCache; + // Warning: (ae-forgotten-export) The symbol "IProjectBuildCacheOptions" needs to be exported by the entry point index.d.ts + // + // (undocumented) + static getProjectBuildCache(options: IProjectBuildCacheOptions): ProjectBuildCache; + // (undocumented) + tryRestoreFromCacheAsync(terminal: ITerminal, specifiedCacheId?: string): Promise; + // (undocumented) + trySetCacheEntryAsync(terminal: ITerminal, specifiedCacheId?: string): Promise; +} + // @beta (undocumented) export class ProjectChangeAnalyzer { constructor(rushConfiguration: RushConfiguration); diff --git a/libraries/rush-lib/src/index.ts b/libraries/rush-lib/src/index.ts index d7ef26a364e..82c492cde9e 100644 --- a/libraries/rush-lib/src/index.ts +++ b/libraries/rush-lib/src/index.ts @@ -197,3 +197,5 @@ export { type IRushCommandLineParameter, type IRushCommandLineAction } from './api/RushCommandLine'; + +export { ProjectBuildCache } from './logic/buildCache/ProjectBuildCache'; diff --git a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts index 861d403f029..30199e8c1ac 100644 --- a/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts +++ b/libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts @@ -50,6 +50,9 @@ interface IPathsToCache { outputFilePaths: string[]; } +/** + * @internal + */ export class ProjectBuildCache { private static _tarUtilityPromise: Promise | undefined; diff --git a/libraries/rush-lib/src/pluginFramework/PluginManager.ts b/libraries/rush-lib/src/pluginFramework/PluginManager.ts index 9a5181e078c..a1818e180da 100644 --- a/libraries/rush-lib/src/pluginFramework/PluginManager.ts +++ b/libraries/rush-lib/src/pluginFramework/PluginManager.ts @@ -81,6 +81,7 @@ export class PluginManager { tryAddBuiltInPlugin('rush-amazon-s3-build-cache-plugin'); tryAddBuiltInPlugin('rush-azure-storage-build-cache-plugin'); tryAddBuiltInPlugin('rush-http-build-cache-plugin'); + tryAddBuiltInPlugin('rush-bridge-cache-plugin'); // This is a secondary plugin inside the `@rushstack/rush-azure-storage-build-cache-plugin` // package. Because that package comes with Rush (for now), it needs to get registered here. // If the necessary config file doesn't exist, this plugin doesn't do anything. diff --git a/rush-plugins/rush-bridge-cache-plugin/.eslintrc.js b/rush-plugins/rush-bridge-cache-plugin/.eslintrc.js new file mode 100644 index 00000000000..0b04796d1ee --- /dev/null +++ b/rush-plugins/rush-bridge-cache-plugin/.eslintrc.js @@ -0,0 +1,13 @@ +// This is a workaround for https://github.com/eslint/eslint/issues/3458 +require('local-node-rig/profiles/default/includes/eslint/patch/modern-module-resolution'); +// This is a workaround for https://github.com/microsoft/rushstack/issues/3021 +require('local-node-rig/profiles/default/includes/eslint/patch/custom-config-package-names'); + +module.exports = { + extends: [ + 'local-node-rig/profiles/default/includes/eslint/profile/node', + 'local-node-rig/profiles/default/includes/eslint/mixins/friendly-locals', + 'local-node-rig/profiles/default/includes/eslint/mixins/tsdoc' + ], + parserOptions: { tsconfigRootDir: __dirname } +}; diff --git a/rush-plugins/rush-bridge-cache-plugin/LICENSE b/rush-plugins/rush-bridge-cache-plugin/LICENSE new file mode 100644 index 00000000000..3d9be20289b --- /dev/null +++ b/rush-plugins/rush-bridge-cache-plugin/LICENSE @@ -0,0 +1,24 @@ +@rushstack/rush-serve-plugin + +Copyright (c) Microsoft Corporation. All rights reserved. + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/rush-plugins/rush-bridge-cache-plugin/README.md b/rush-plugins/rush-bridge-cache-plugin/README.md new file mode 100644 index 00000000000..8f801f82386 --- /dev/null +++ b/rush-plugins/rush-bridge-cache-plugin/README.md @@ -0,0 +1,44 @@ +# @rushstack/rush-bridge-cache-plugin + +This plugin allows for interaction with the Rush cache. It exposes some methods to set the cache. + + +## Installation + +`npm install @rushstack/rush-bridge-cache-plugin` + + + +-------------------- + +So this will: +- tap into the hooks and do all the necessary shit to figure out how to po +- expose a simple API for external users to tap into, e.g. if you want to populate the cache externally you'd import this plugin and use the methods to do so. You could then wrap that in a rush function, or however you want to do it. + + + + + +-------------------- + +Discussion about the solution for BuildXL here: + https://teams.microsoft.com/l/message/19:d85f52548ec74e8f8a0f107bd4e5ceb6@thread.v2/1740610046597?context=%7B%22contextType%22%3A%22chat%22%7D + + +"populate-cache": "..." <--- called after any cacheable unit of work is complete + +OperationExecutionRecord -> the smallest unit of work to be done. + - looks like it's strongly coupled to the runner. We need one-off method calls. + +CacheableOperationPlugin + _tryGetProjectBuildCache() -> this returns the project build cache + + diff --git a/rush-plugins/rush-bridge-cache-plugin/command-line.json b/rush-plugins/rush-bridge-cache-plugin/command-line.json new file mode 100644 index 00000000000..3741588d1d0 --- /dev/null +++ b/rush-plugins/rush-bridge-cache-plugin/command-line.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/command-line.schema.json", + "parameters": [ + { + "longName": "--set-cache-only", + "parameterKind": "flag", + "description": "...", + "associatedCommands": ["build"] // has to either apply to all phased command, or allow customization by user + } + ] +} diff --git a/rush-plugins/rush-bridge-cache-plugin/config/jest.config.json b/rush-plugins/rush-bridge-cache-plugin/config/jest.config.json new file mode 100644 index 00000000000..d1749681d90 --- /dev/null +++ b/rush-plugins/rush-bridge-cache-plugin/config/jest.config.json @@ -0,0 +1,3 @@ +{ + "extends": "local-node-rig/profiles/default/config/jest.config.json" +} diff --git a/rush-plugins/rush-bridge-cache-plugin/config/rig.json b/rush-plugins/rush-bridge-cache-plugin/config/rig.json new file mode 100644 index 00000000000..165ffb001f5 --- /dev/null +++ b/rush-plugins/rush-bridge-cache-plugin/config/rig.json @@ -0,0 +1,7 @@ +{ + // The "rig.json" file directs tools to look for their config files in an external package. + // Documentation for this system: https://www.npmjs.com/package/@rushstack/rig-package + "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json", + + "rigPackageName": "local-node-rig" +} diff --git a/rush-plugins/rush-bridge-cache-plugin/package.json b/rush-plugins/rush-bridge-cache-plugin/package.json new file mode 100644 index 00000000000..0e6dad8ddd4 --- /dev/null +++ b/rush-plugins/rush-bridge-cache-plugin/package.json @@ -0,0 +1,26 @@ +{ + "name": "@rushstack/rush-bridge-cache-plugin", + "version": "0.0.1", + "private": true, + "description": "A plugin to expose methods to interact with the Rush cache.", + "license": "MIT", + "main": "./lib/index.js", + "repository": { + "url": "https://github.com/microsoft/rushstack.git", + "type": "git", + "directory": "rush-plugins/rush-bridge-cache-plugin" + }, + "scripts": { + "build": "heft test --clean", + "_phase:build": "heft run --only build -- --clean", + "_phase:test": "heft run --only test -- --clean" + }, + "devDependencies": { + "@rushstack/node-core-library": "workspace:*", + "@microsoft/rush-lib": "workspace:*", + "@rushstack/rush-sdk": "workspace:*", + "@rushstack/terminal": "workspace:*", + "@rushstack/heft": "workspace:*", + "local-node-rig": "workspace:*" + } +} diff --git a/rush-plugins/rush-bridge-cache-plugin/rush-plugin-manifest.json b/rush-plugins/rush-bridge-cache-plugin/rush-plugin-manifest.json new file mode 100644 index 00000000000..4c7fedbe7be --- /dev/null +++ b/rush-plugins/rush-bridge-cache-plugin/rush-plugin-manifest.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/rush-plugin-manifest.schema.json", + "plugins": [ + { + "pluginName": "rush-bridge-cache-plugin", + "description": "Rush plugin to allow interactions with the Rush cache.", + "entryPoint": "./lib/index.js", + "commandLineJsonFilePath": "./command-line.json" + } + ] +} diff --git a/rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts b/rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts new file mode 100644 index 00000000000..ff4b79c3e87 --- /dev/null +++ b/rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { BuildCacheConfiguration, ProjectBuildCache } from '@rushstack/rush-sdk'; +import type { + ILogger, + IOperationExecutionResult, + IPhasedCommand, + IRushPlugin, + Operation, + RushConfiguration, + RushSession +} from '@rushstack/rush-sdk'; + +const PLUGIN_NAME: 'RushBridgeCachePlugin' = 'RushBridgeCachePlugin'; + +export class BridgeCachePlugin implements IRushPlugin { + public readonly pluginName: string = PLUGIN_NAME; + + public apply(session: RushSession, rushConfiguration: RushConfiguration): void { + // klutzy. Better way? + const actionName: string = process.argv[2]; + + // const isSetCacheOnly: boolean = process.argv.includes('--set-cache-only'); // has to be allowed for ANY phased command, or be customizable + // if (!isSetCacheOnly) { + // return; + // } + + // tracks the projects being targeted by the command (--to, --only etc.) + const targetProjects: string[] = []; + + const cancelOperations = (operations: Set): Set => { + operations.forEach((operation: Operation) => { + if (operation.enabled) { + targetProjects.push(operation.associatedProject.packageName); + } + + operation.enabled = false; + }); + return operations; + }; + + session.hooks.runPhasedCommand + .for(actionName) + .tapPromise(PLUGIN_NAME, async (command: IPhasedCommand) => { + // cancel the actual operations. We don't want to actually run the command, just cache the output folders from a previous run + command.hooks.createOperations.tap( + { name: PLUGIN_NAME, stage: Number.MAX_SAFE_INTEGER }, + cancelOperations + ); + + // now populate the cache for each operation + command.hooks.beforeExecuteOperations.tap( + PLUGIN_NAME, + async (recordByOperation: Map): Promise => { + await this._setCacheAsync(session, rushConfiguration, recordByOperation, targetProjects); + } + ); + }); + } + + private async _setCacheAsync( + session: RushSession, + rushConfiguration: RushConfiguration, + recordByOperation: Map, + targetProjects: string[] + ): Promise { + const logger: ILogger = session.getLogger(PLUGIN_NAME); + + // const isSetCacheOnly: boolean = process.argv.includes('--set-cache-only'); + + recordByOperation.forEach( + async (operationExecutionResult: IOperationExecutionResult, operation: Operation) => { + const { associatedProject, associatedPhase, settings } = operation; + + if (!targetProjects.includes(associatedProject.packageName)) { + return; + } + + const buildCacheConfiguration: BuildCacheConfiguration | undefined = + await BuildCacheConfiguration.tryLoadAsync(logger.terminal, rushConfiguration, session); + + if (!buildCacheConfiguration) { + return; + } + + const projectBuildCache: ProjectBuildCache = ProjectBuildCache.getProjectBuildCache({ + project: associatedProject, + projectOutputFolderNames: settings?.outputFolderNames || [], + buildCacheConfiguration, + terminal: logger.terminal, + operationStateHash: operationExecutionResult.getStateHash(), + phaseName: associatedPhase.name + }); + + const success: boolean = await projectBuildCache.trySetCacheEntryAsync(logger.terminal); + + // eslint-disable-next-line no-console + console.log('- setting cache for', { + success, + name: associatedPhase.name, + package: associatedProject.packageName + }); + } + ); + } +} diff --git a/rush-plugins/rush-bridge-cache-plugin/src/index.ts b/rush-plugins/rush-bridge-cache-plugin/src/index.ts new file mode 100644 index 00000000000..01e83887250 --- /dev/null +++ b/rush-plugins/rush-bridge-cache-plugin/src/index.ts @@ -0,0 +1,4 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +export { BridgeCachePlugin as default } from './BridgeCachePlugin'; diff --git a/rush-plugins/rush-bridge-cache-plugin/tsconfig.json b/rush-plugins/rush-bridge-cache-plugin/tsconfig.json new file mode 100644 index 00000000000..dac21d04081 --- /dev/null +++ b/rush-plugins/rush-bridge-cache-plugin/tsconfig.json @@ -0,0 +1,3 @@ +{ + "extends": "./node_modules/local-node-rig/profiles/default/tsconfig-base.json" +} diff --git a/rush.json b/rush.json index ca694d40f62..b2c255004a9 100644 --- a/rush.json +++ b/rush.json @@ -162,15 +162,13 @@ * IMPORTANT: Because these are regular expressions encoded as JSON string literals, * RegExp escapes need two backslashes, and ordinary periods should be "\\.". */ - "allowedEmailRegExps": ["[^@]+@users\\.noreply\\.github\\.com"], - + // "allowedEmailRegExps": ["[^@]+@users\\.noreply\\.github\\.com"], /** * When Rush reports that the address is malformed, the notice can include an example * of a recommended email. Make sure it conforms to one of the allowedEmailRegExps * expressions. */ - "sampleEmail": "example@users.noreply.github.com" - + // "sampleEmail": "example@users.noreply.github.com" /** * The commit message to use when committing changes during 'rush publish'. * @@ -179,7 +177,6 @@ * in the commit message, and then customize Rush's message to contain that string. */ // "versionBumpCommitMessage": "Bump versions [skip ci]", - /** * The commit message to use when committing changes during 'rush version'. * @@ -188,7 +185,6 @@ * in the commit message, and then customize Rush's message to contain that string. */ // "changeLogUpdateCommitMessage": "Update changelogs [skip ci]", - /** * The commit message to use when committing changefiles during 'rush change --commit' * @@ -1305,6 +1301,12 @@ "reviewCategory": "libraries", "versionPolicyName": "rush" }, + { + "packageName": "@rushstack/rush-bridge-cache-plugin", + "projectFolder": "rush-plugins/rush-bridge-cache-plugin", + "reviewCategory": "libraries", + "versionPolicyName": "rush" + }, // "vscode-extensions" folder (alphabetical order) { From c3fdcc92d92df6cfa57bfecc697e7374da9a63df Mon Sep 17 00:00:00 2001 From: Ben Keen Date: Thu, 10 Apr 2025 09:29:23 -0700 Subject: [PATCH 2/2] code review feedback --- common/reviews/api/rush-lib.api.md | 107 ++++++++++++--- libraries/rush-lib/src/index.ts | 3 +- .../operations/ShardedPhaseOperationPlugin.ts | 2 + .../rush-bridge-cache-plugin/README.md | 47 +++---- .../command-line.json | 11 -- .../rush-bridge-cache-plugin/package.json | 2 +- .../rush-plugin-manifest.json | 5 +- .../src/BridgeCachePlugin.ts | 128 ++++++++++-------- 8 files changed, 185 insertions(+), 120 deletions(-) delete mode 100644 rush-plugins/rush-bridge-cache-plugin/command-line.json diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 846354cea6e..28ddaade95e 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -946,6 +946,92 @@ export class Operation { weight: number; } +// Warning: (ae-internal-missing-underscore) The name "OperationBuildCache" should be prefixed with an underscore because the declaration is marked as @internal +// +// @internal (undocumented) +export class OperationBuildCache { + // (undocumented) + get cacheId(): string | undefined; + // Warning: (ae-forgotten-export) The symbol "IOperationBuildCacheOptions" needs to be exported by the entry point index.d.ts + // + // (undocumented) + static forOperation(operation: OperationExecutionRecord, options: IOperationBuildCacheOptions): OperationBuildCache; + // Warning: (ae-forgotten-export) The symbol "IProjectBuildCacheOptions" needs to be exported by the entry point index.d.ts + // + // (undocumented) + static getProjectBuildCache(options: IProjectBuildCacheOptions): OperationBuildCache; + // (undocumented) + tryRestoreFromCacheAsync(terminal: ITerminal, specifiedCacheId?: string): Promise; + // (undocumented) + trySetCacheEntryAsync(terminal: ITerminal, specifiedCacheId?: string): Promise; +} + +// Warning: (ae-internal-missing-underscore) The name "OperationExecutionRecord" should be prefixed with an underscore because the declaration is marked as @internal +// +// @internal +export class OperationExecutionRecord implements IOperationRunnerContext, IOperationExecutionResult { + // Warning: (ae-forgotten-export) The symbol "IOperationExecutionRecordContext" needs to be exported by the entry point index.d.ts + constructor(operation: Operation, context: IOperationExecutionRecordContext); + // (undocumented) + readonly associatedPhase: IPhase; + // (undocumented) + readonly associatedProject: RushConfigurationProject; + // (undocumented) + get cobuildRunnerId(): string | undefined; + // (undocumented) + get collatedWriter(): CollatedWriter; + readonly consumers: Set; + criticalPathLength: number | undefined; + // (undocumented) + get debugMode(): boolean; + readonly dependencies: Set; + // (undocumented) + get environment(): IEnvironment | undefined; + error: Error | undefined; + // (undocumented) + executeAsync({ onStart, onResult }: { + onStart: (record: OperationExecutionRecord) => Promise; + onResult: (record: OperationExecutionRecord) => Promise; + }): Promise; + // (undocumented) + getStateHash(): string; + // (undocumented) + getStateHashComponents(): ReadonlyArray; + // (undocumented) + get isTerminal(): boolean; + // (undocumented) + logFilePaths: ILogFilePaths | undefined; + // (undocumented) + get metadataFolderPath(): string | undefined; + // (undocumented) + get name(): string; + // (undocumented) + get nonCachedDurationMs(): number | undefined; + readonly operation: Operation; + // (undocumented) + readonly _operationMetadataManager: _OperationMetadataManager; + // (undocumented) + get quietMode(): boolean; + // (undocumented) + readonly runner: IOperationRunner; + runWithTerminalAsync(callback: (terminal: ITerminal, terminalProvider: ITerminalProvider) => Promise, options: { + createLogFile: boolean; + logFileSuffix: string; + }): Promise; + // (undocumented) + get silent(): boolean; + get status(): OperationStatus; + set status(newStatus: OperationStatus); + // (undocumented) + readonly stdioSummarizer: StdioSummarizer; + // Warning: (ae-forgotten-export) The symbol "Stopwatch" needs to be exported by the entry point index.d.ts + // + // (undocumented) + readonly stopwatch: Stopwatch; + // (undocumented) + get weight(): number; +} + // @internal export class _OperationMetadataManager { constructor(options: _IOperationMetadataManagerOptions); @@ -1128,27 +1214,6 @@ export type PnpmStoreLocation = 'local' | 'global'; // @public @deprecated (undocumented) export type PnpmStoreOptions = PnpmStoreLocation; -// Warning: (ae-internal-missing-underscore) The name "ProjectBuildCache" should be prefixed with an underscore because the declaration is marked as @internal -// -// @internal (undocumented) -export class ProjectBuildCache { - // (undocumented) - get cacheId(): string | undefined; - // Warning: (ae-forgotten-export) The symbol "OperationExecutionRecord" needs to be exported by the entry point index.d.ts - // Warning: (ae-forgotten-export) The symbol "IOperationBuildCacheOptions" needs to be exported by the entry point index.d.ts - // - // (undocumented) - static forOperation(operation: OperationExecutionRecord, options: IOperationBuildCacheOptions): ProjectBuildCache; - // Warning: (ae-forgotten-export) The symbol "IProjectBuildCacheOptions" needs to be exported by the entry point index.d.ts - // - // (undocumented) - static getProjectBuildCache(options: IProjectBuildCacheOptions): ProjectBuildCache; - // (undocumented) - tryRestoreFromCacheAsync(terminal: ITerminal, specifiedCacheId?: string): Promise; - // (undocumented) - trySetCacheEntryAsync(terminal: ITerminal, specifiedCacheId?: string): Promise; -} - // @beta (undocumented) export class ProjectChangeAnalyzer { constructor(rushConfiguration: RushConfiguration); diff --git a/libraries/rush-lib/src/index.ts b/libraries/rush-lib/src/index.ts index 82c492cde9e..9592a1bbecf 100644 --- a/libraries/rush-lib/src/index.ts +++ b/libraries/rush-lib/src/index.ts @@ -135,6 +135,7 @@ export type { IExecutionResult, IOperationExecutionResult } from './logic/operations/IOperationExecutionResult'; +export type { OperationExecutionRecord } from './logic/operations/OperationExecutionRecord'; export { type IOperationOptions, Operation } from './logic/operations/Operation'; export { OperationStatus } from './logic/operations/OperationStatus'; export type { ILogFilePaths } from './logic/operations/ProjectLogWritable'; @@ -198,4 +199,4 @@ export { type IRushCommandLineAction } from './api/RushCommandLine'; -export { ProjectBuildCache } from './logic/buildCache/ProjectBuildCache'; +export { ProjectBuildCache as OperationBuildCache } from './logic/buildCache/ProjectBuildCache'; diff --git a/libraries/rush-lib/src/logic/operations/ShardedPhaseOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/ShardedPhaseOperationPlugin.ts index 20a0ce44dbd..b28a1eeeeaf 100644 --- a/libraries/rush-lib/src/logic/operations/ShardedPhaseOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/ShardedPhaseOperationPlugin.ts @@ -128,6 +128,8 @@ function spliceShards(existingOperations: Set, context: ICreateOperat ]; const { scripts } = project.packageJson; + + // const commandToRun: string | undefined = phase.shellCommand ?? scripts?.[phase.name]; operation.logFilenameIdentifier = `${baseLogFilenameIdentifier}_collate`; diff --git a/rush-plugins/rush-bridge-cache-plugin/README.md b/rush-plugins/rush-bridge-cache-plugin/README.md index 8f801f82386..00ddac644b7 100644 --- a/rush-plugins/rush-bridge-cache-plugin/README.md +++ b/rush-plugins/rush-bridge-cache-plugin/README.md @@ -1,44 +1,45 @@ # @rushstack/rush-bridge-cache-plugin -This plugin allows for interaction with the Rush cache. It exposes some methods to set the cache. +This is a Rush plugin that adds an optional `--set-cache-only` flag to Rush's phased commands, so you can other tools to actual run the scripts and generate the build artifacts on disk, then separately populate the Rush cache for particular actions(s). For integrations with other build orchestrators, this allows the best of both worlds: using a different build tool to orchestrate the work, but still populate the Rush cache for benefiting local use of Rush. +## Here be dragons! -## Installation - -`npm install @rushstack/rush-bridge-cache-plugin` +This is a power-user sort of plugin. It assumes that the work for a particular task has already been completed and the build artifacts have been generated on disk. **If you run this command on a package where the command hasn't already been ran and the build artifacts are missing or incorrect, you will cache invalid content**. Be careful and beware! - +(TODO) --------------------- +`npm install @rushstack/rush-bridge-cache-plugin` -So this will: -- tap into the hooks and do all the necessary shit to figure out how to po -- expose a simple API for external users to tap into, e.g. if you want to populate the cache externally you'd import this plugin and use the methods to do so. You could then wrap that in a rush function, or however you want to do it. +## Configuration +First you need to update your `command-line.json` file to add the new flag. Configure it to target whatever specific commands you want to have this feature. Example: +```json +{ + "associatedCommands": ["build", "test", "lint", "a11y", "typecheck"], + "description": "When the flag is added to any associated command, it'll bypass running the command itself, but cache the result of a previous run. Beware! Only run when you know the build artifacts are in a valid state for the command.", + "parameterKind": "flag", + "longName": "--set-cache-only", + "required": false +} +``` --------------------- +## Usage -Discussion about the solution for BuildXL here: - https://teams.microsoft.com/l/message/19:d85f52548ec74e8f8a0f107bd4e5ceb6@thread.v2/1740610046597?context=%7B%22contextType%22%3A%22chat%22%7D +Any of the rush command can now just be given a `--set-cache-only` property, e.g. +`rush build --to your-packageX --set-cache-only` -"populate-cache": "..." <--- called after any cacheable unit of work is complete +That will examine `your-packageX` and all of its dependencies, then populate the cache. -OperationExecutionRecord -> the smallest unit of work to be done. - - looks like it's strongly coupled to the runner. We need one-off method calls. -CacheableOperationPlugin - _tryGetProjectBuildCache() -> this returns the project build cache +## Performance +When running within a pipeline, you'll want to populate the cache as fast as possible. So instead of waiting until the full build graph has been processed, you'll wnt to run it after each successful task. For that, just use Rush's `--only` and target whatever task had just completed, for example: +`rush lint --only your-packageY --set-cache-only` diff --git a/rush-plugins/rush-bridge-cache-plugin/command-line.json b/rush-plugins/rush-bridge-cache-plugin/command-line.json deleted file mode 100644 index 3741588d1d0..00000000000 --- a/rush-plugins/rush-bridge-cache-plugin/command-line.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/command-line.schema.json", - "parameters": [ - { - "longName": "--set-cache-only", - "parameterKind": "flag", - "description": "...", - "associatedCommands": ["build"] // has to either apply to all phased command, or allow customization by user - } - ] -} diff --git a/rush-plugins/rush-bridge-cache-plugin/package.json b/rush-plugins/rush-bridge-cache-plugin/package.json index 0e6dad8ddd4..09fbef2706a 100644 --- a/rush-plugins/rush-bridge-cache-plugin/package.json +++ b/rush-plugins/rush-bridge-cache-plugin/package.json @@ -2,7 +2,7 @@ "name": "@rushstack/rush-bridge-cache-plugin", "version": "0.0.1", "private": true, - "description": "A plugin to expose methods to interact with the Rush cache.", + "description": "Rush plugin that provides a --set-cache-only command flag to populates the cache from previous runs.", "license": "MIT", "main": "./lib/index.js", "repository": { diff --git a/rush-plugins/rush-bridge-cache-plugin/rush-plugin-manifest.json b/rush-plugins/rush-bridge-cache-plugin/rush-plugin-manifest.json index 4c7fedbe7be..f2345794e4d 100644 --- a/rush-plugins/rush-bridge-cache-plugin/rush-plugin-manifest.json +++ b/rush-plugins/rush-bridge-cache-plugin/rush-plugin-manifest.json @@ -3,9 +3,8 @@ "plugins": [ { "pluginName": "rush-bridge-cache-plugin", - "description": "Rush plugin to allow interactions with the Rush cache.", - "entryPoint": "./lib/index.js", - "commandLineJsonFilePath": "./command-line.json" + "description": "Rush plugin that provides a --set-cache-only command flag to populates the cache from previous runs.", + "entryPoint": "./lib/index.js" } ] } diff --git a/rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts b/rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts index ff4b79c3e87..664c2f15fb4 100644 --- a/rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts +++ b/rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts @@ -1,14 +1,16 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { BuildCacheConfiguration, ProjectBuildCache } from '@rushstack/rush-sdk'; +import { OperationBuildCache } from '@rushstack/rush-sdk'; import type { + BuildCacheConfiguration, + IExecuteOperationsContext, ILogger, IOperationExecutionResult, IPhasedCommand, IRushPlugin, Operation, - RushConfiguration, + OperationExecutionRecord, RushSession } from '@rushstack/rush-sdk'; @@ -17,90 +19,96 @@ const PLUGIN_NAME: 'RushBridgeCachePlugin' = 'RushBridgeCachePlugin'; export class BridgeCachePlugin implements IRushPlugin { public readonly pluginName: string = PLUGIN_NAME; - public apply(session: RushSession, rushConfiguration: RushConfiguration): void { - // klutzy. Better way? - const actionName: string = process.argv[2]; - - // const isSetCacheOnly: boolean = process.argv.includes('--set-cache-only'); // has to be allowed for ANY phased command, or be customizable - // if (!isSetCacheOnly) { - // return; - // } - - // tracks the projects being targeted by the command (--to, --only etc.) - const targetProjects: string[] = []; + public apply(session: RushSession): void { + const isSetCacheOnly: boolean = process.argv.includes('--set-cache-only'); + if (!isSetCacheOnly) { + return; + } const cancelOperations = (operations: Set): Set => { operations.forEach((operation: Operation) => { - if (operation.enabled) { - targetProjects.push(operation.associatedProject.packageName); - } - operation.enabled = false; }); return operations; }; - session.hooks.runPhasedCommand - .for(actionName) - .tapPromise(PLUGIN_NAME, async (command: IPhasedCommand) => { - // cancel the actual operations. We don't want to actually run the command, just cache the output folders from a previous run - command.hooks.createOperations.tap( - { name: PLUGIN_NAME, stage: Number.MAX_SAFE_INTEGER }, - cancelOperations - ); - - // now populate the cache for each operation - command.hooks.beforeExecuteOperations.tap( - PLUGIN_NAME, - async (recordByOperation: Map): Promise => { - await this._setCacheAsync(session, rushConfiguration, recordByOperation, targetProjects); + session.hooks.runAnyPhasedCommand.tapPromise(PLUGIN_NAME, async (command: IPhasedCommand) => { + // tracks the projects being targeted by the command (--to, --only etc.) + const targetProjects: Set = new Set(); + + // cancel the actual operations. We don't want to actually run the command, just cache the output folders from a previous run + command.hooks.createOperations.tap( + { name: PLUGIN_NAME, stage: Number.MAX_SAFE_INTEGER }, + (operations: Set): Set => { + operations.forEach((operation: Operation) => { + if (operation.enabled) { + targetProjects.add(operation); + } + }); + return cancelOperations(operations); + } + ); + + // now populate the cache for each operation + command.hooks.beforeExecuteOperations.tap( + PLUGIN_NAME, + async ( + recordByOperation: Map, + context: IExecuteOperationsContext + ): Promise => { + if (!context.buildCacheConfiguration) { + return; } - ); - }); + + await this._setCacheAsync( + session, + context.buildCacheConfiguration, + recordByOperation, + targetProjects + ); + } + ); + }); } private async _setCacheAsync( session: RushSession, - rushConfiguration: RushConfiguration, + buildCacheConfiguration: BuildCacheConfiguration, recordByOperation: Map, - targetProjects: string[] + targetProjects: Set ): Promise { const logger: ILogger = session.getLogger(PLUGIN_NAME); - // const isSetCacheOnly: boolean = process.argv.includes('--set-cache-only'); - recordByOperation.forEach( async (operationExecutionResult: IOperationExecutionResult, operation: Operation) => { - const { associatedProject, associatedPhase, settings } = operation; + const { associatedProject, associatedPhase } = operation; - if (!targetProjects.includes(associatedProject.packageName)) { + // omit operations that aren't targeted, or packages without a command for this phase + const hasCommand: boolean = !!associatedProject.packageJson.scripts?.[associatedPhase.name]; + if (!targetProjects.has(operation) || !hasCommand) { return; } - const buildCacheConfiguration: BuildCacheConfiguration | undefined = - await BuildCacheConfiguration.tryLoadAsync(logger.terminal, rushConfiguration, session); - - if (!buildCacheConfiguration) { - return; - } - - const projectBuildCache: ProjectBuildCache = ProjectBuildCache.getProjectBuildCache({ - project: associatedProject, - projectOutputFolderNames: settings?.outputFolderNames || [], - buildCacheConfiguration, - terminal: logger.terminal, - operationStateHash: operationExecutionResult.getStateHash(), - phaseName: associatedPhase.name - }); + const projectBuildCache: OperationBuildCache = OperationBuildCache.forOperation( + operationExecutionResult as OperationExecutionRecord, + { + buildCacheConfiguration, + terminal: logger.terminal + } + ); const success: boolean = await projectBuildCache.trySetCacheEntryAsync(logger.terminal); - // eslint-disable-next-line no-console - console.log('- setting cache for', { - success, - name: associatedPhase.name, - package: associatedProject.packageName - }); + if (success) { + logger.terminal.writeLine( + `Cache entry set for ${associatedPhase.name} (${associatedProject.packageName}) from previously generated output folders\n` + ); + } else { + logger.terminal.writeErrorLine( + `Error creating a cache entry set for ${associatedPhase.name} (${associatedProject.packageName}) from previously generated output folders\n` + ); + process.exit(1); + } } ); }