Add bridge-cache plugin to populate cache without running action

Author: Ben Keen

apps/rush/package.json

@@ -47,6 +47,7 @@
     "@rushstack/rush-amazon-s3-build-cache-plugin": "workspace:*",
     "@rushstack/rush-azure-storage-build-cache-plugin": "workspace:*",
     "@rushstack/rush-http-build-cache-plugin": "workspace:*",
+    "@rushstack/rush-bridge-cache-plugin": "workspace:*",
     "@types/heft-jest": "1.0.1",
     "@types/semver": "7.5.0"
   }

apps/rush/src/start-dev-docs.ts

@@ -6,4 +6,4 @@ import { Colorize, ConsoleTerminalProvider, Terminal } from '@rushstack/terminal
 const terminal: Terminal = new Terminal(new ConsoleTerminalProvider());
 
 terminal.writeLine('For instructions on debugging Rush, please see this documentation:');
-terminal.writeLine(Colorize.bold('https://rushjs.io/pages/contributing/debugging/'));
+terminal.writeLine(Colorize.bold('https://rushjs.io/pages/contributing/#debugging-rush'));

apps/rush/src/start-dev.ts

@@ -31,6 +31,7 @@ includePlugin('rush-azure-storage-build-cache-plugin');
 includePlugin('rush-http-build-cache-plugin');
 // Including this here so that developers can reuse it without installing the plugin a second time
 includePlugin('rush-azure-interactive-auth-plugin', '@rushstack/rush-azure-storage-build-cache-plugin');
+includePlugin('rush-bridge-cache-plugin');
 
 const currentPackageVersion: string = PackageJsonLookup.loadOwnPackageJson(__dirname).version;
 RushCommandSelector.execute(currentPackageVersion, rushLib, {

common/config/rush/browser-approved-packages.json

@@ -38,6 +38,10 @@
       "name": "@reduxjs/toolkit",
       "allowedCategories": [ "libraries", "vscode-extensions" ]
     },
+    {
+      "name": "@rushstack/rush-bridge-cache-plugin",
+      "allowedCategories": [ "libraries" ]
+    },
     {
       "name": "@rushstack/rush-themed-ui",
       "allowedCategories": [ "libraries" ]

common/config/subspaces/default/pnpm-lock.yaml

@@ -10,7 +10,7 @@ import { AsyncParallelHook } from 'tapable';
 import { AsyncSeriesBailHook } from 'tapable';
 import { AsyncSeriesHook } from 'tapable';
 import { AsyncSeriesWaterfallHook } from 'tapable';
-import type { CollatedWriter } from '@rushstack/stream-collator';
+import { CollatedWriter } from '@rushstack/stream-collator';
 import type { CommandLineParameter } from '@rushstack/ts-command-line';
 import { CommandLineParameterKind } from '@rushstack/ts-command-line';
 import { HookMap } from 'tapable';
@@ -23,7 +23,8 @@ import { JsonNull } from '@rushstack/node-core-library';
 import { JsonObject } from '@rushstack/node-core-library';
 import { LookupByPath } from '@rushstack/lookup-by-path';
 import { PackageNameParser } from '@rushstack/node-core-library';
-import type { StdioSummarizer } from '@rushstack/terminal';
+import { StdioSummarizer } from '@rushstack/terminal';
+import { StreamCollator } from '@rushstack/stream-collator';
 import { SyncHook } from 'tapable';
 import { SyncWaterfallHook } from 'tapable';
 import { Terminal } from '@rushstack/terminal';
@@ -1127,6 +1128,27 @@ export type PnpmStoreLocation = 'local' | 'global';
 // @public @deprecated (undocumented)
 export type PnpmStoreOptions = PnpmStoreLocation;
 
+// Warning: (ae-internal-missing-underscore) The name "ProjectBuildCache" should be prefixed with an underscore because the declaration is marked as @internal
+//
+// @internal (undocumented)
+export class ProjectBuildCache {
+    // (undocumented)
+    get cacheId(): string | undefined;
+    // Warning: (ae-forgotten-export) The symbol "OperationExecutionRecord" needs to be exported by the entry point index.d.ts
+    // Warning: (ae-forgotten-export) The symbol "IOperationBuildCacheOptions" needs to be exported by the entry point index.d.ts
+    //
+    // (undocumented)
+    static forOperation(operation: OperationExecutionRecord, options: IOperationBuildCacheOptions): ProjectBuildCache;
+    // Warning: (ae-forgotten-export) The symbol "IProjectBuildCacheOptions" needs to be exported by the entry point index.d.ts
+    //
+    // (undocumented)
+    static getProjectBuildCache(options: IProjectBuildCacheOptions): ProjectBuildCache;
+    // (undocumented)
+    tryRestoreFromCacheAsync(terminal: ITerminal, specifiedCacheId?: string): Promise<boolean>;
+    // (undocumented)
+    trySetCacheEntryAsync(terminal: ITerminal, specifiedCacheId?: string): Promise<boolean>;
+}
+
 // @beta (undocumented)
 export class ProjectChangeAnalyzer {
     constructor(rushConfiguration: RushConfiguration);

common/reviews/api/rush-lib.api.md

@@ -197,3 +197,5 @@ export {
   type IRushCommandLineParameter,
   type IRushCommandLineAction
 } from './api/RushCommandLine';
+
+export { ProjectBuildCache } from './logic/buildCache/ProjectBuildCache';

libraries/rush-lib/src/index.ts

@@ -50,6 +50,9 @@ interface IPathsToCache {
   outputFilePaths: string[];
 }
 
+/**
+ * @internal
+ */
 export class ProjectBuildCache {
   private static _tarUtilityPromise: Promise<TarExecutable | undefined> | undefined;
 

libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts

@@ -81,6 +81,7 @@ export class PluginManager {
     tryAddBuiltInPlugin('rush-amazon-s3-build-cache-plugin');
     tryAddBuiltInPlugin('rush-azure-storage-build-cache-plugin');
     tryAddBuiltInPlugin('rush-http-build-cache-plugin');
+    tryAddBuiltInPlugin('rush-bridge-cache-plugin');
     // This is a secondary plugin inside the `@rushstack/rush-azure-storage-build-cache-plugin`
     // package. Because that package comes with Rush (for now), it needs to get registered here.
     // If the necessary config file doesn't exist, this plugin doesn't do anything.

libraries/rush-lib/src/pluginFramework/PluginManager.ts

@@ -0,0 +1,13 @@
+// This is a workaround for https://github.com/eslint/eslint/issues/3458
+require('local-node-rig/profiles/default/includes/eslint/patch/modern-module-resolution');
+// This is a workaround for https://github.com/microsoft/rushstack/issues/3021
+require('local-node-rig/profiles/default/includes/eslint/patch/custom-config-package-names');
+
+module.exports = {
+  extends: [
+    'local-node-rig/profiles/default/includes/eslint/profile/node',
+    'local-node-rig/profiles/default/includes/eslint/mixins/friendly-locals',
+    'local-node-rig/profiles/default/includes/eslint/mixins/tsdoc'
+  ],
+  parserOptions: { tsconfigRootDir: __dirname }
+};

rush-plugins/rush-bridge-cache-plugin/.eslintrc.js

@@ -0,0 +1,24 @@
+@rushstack/rush-serve-plugin
+
+Copyright (c) Microsoft Corporation. All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

rush-plugins/rush-bridge-cache-plugin/LICENSE

@@ -0,0 +1,44 @@
+# @rushstack/rush-bridge-cache-plugin
+
+This plugin allows for interaction with the Rush cache. It exposes some methods to set the cache.
+
+
+## Installation
+
+`npm install @rushstack/rush-bridge-cache-plugin`
+
+<!--
+## Usage
+The package contains a binary you can run from the command line. It will auto-detect the location of your rush.json file.
+
+`populate-rush-cache --packageName=[package name] --phase=test:unit`
+
+- `--packageName` (required) - this should map to the `packageName` entry in your `rush.json` file.
+- `--phase` (required): the name of the phased command whose command has already been ran, and you want to cache the result on disk.
+-->
+
+--------------------
+
+So this will:
+- tap into the hooks and do all the necessary shit to figure out how to po
+- expose a simple API for external users to tap into, e.g. if you want to populate the cache externally you'd import this plugin and use the methods to do so. You could then wrap that in a rush function, or however you want to do it.
+
+
+
+
+
+--------------------
+
+Discussion about the solution for BuildXL here:
+  https://teams.microsoft.com/l/message/19:d85f52548ec74e8f8a0f107bd4e5ceb6@thread.v2/1740610046597?context=%7B%22contextType%22%3A%22chat%22%7D
+
+
+"populate-cache": "..." <--- called after any cacheable unit of work is complete
+
+OperationExecutionRecord -> the smallest unit of work to be done.
+  - looks like it's strongly coupled to the runner. We need one-off method calls.
+
+CacheableOperationPlugin
+  _tryGetProjectBuildCache()  -> this returns the project build cache
+
+

rush-plugins/rush-bridge-cache-plugin/README.md

@@ -0,0 +1,11 @@
+{
+  "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/command-line.schema.json",
+  "parameters": [
+    {
+      "longName": "--set-cache-only",
+      "parameterKind": "flag",
+      "description": "...",
+      "associatedCommands": ["build"] // has to either apply to all phased command, or allow customization by user
+    }
+  ]
+}

rush-plugins/rush-bridge-cache-plugin/command-line.json

@@ -0,0 +1,3 @@
+{
+  "extends": "local-node-rig/profiles/default/config/jest.config.json"
+}

rush-plugins/rush-bridge-cache-plugin/config/jest.config.json

@@ -0,0 +1,7 @@
+{
+  // The "rig.json" file directs tools to look for their config files in an external package.
+  // Documentation for this system: https://www.npmjs.com/package/@rushstack/rig-package
+  "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
+
+  "rigPackageName": "local-node-rig"
+}

rush-plugins/rush-bridge-cache-plugin/config/rig.json

@@ -0,0 +1,26 @@
+{
+  "name": "@rushstack/rush-bridge-cache-plugin",
+  "version": "0.0.1",
+  "private": true,
+  "description": "A plugin to expose methods to interact with the Rush cache.",
+  "license": "MIT",
+  "main": "./lib/index.js",
+  "repository": {
+    "url": "https://github.com/microsoft/rushstack.git",
+    "type": "git",
+    "directory": "rush-plugins/rush-bridge-cache-plugin"
+  },
+  "scripts": {
+    "build": "heft test --clean",
+    "_phase:build": "heft run --only build -- --clean",
+    "_phase:test": "heft run --only test -- --clean"
+  },
+  "devDependencies": {
+    "@rushstack/node-core-library": "workspace:*",
+    "@microsoft/rush-lib": "workspace:*",
+    "@rushstack/rush-sdk": "workspace:*",
+    "@rushstack/terminal": "workspace:*",
+    "@rushstack/heft": "workspace:*",
+    "local-node-rig": "workspace:*"
+  }
+}

rush-plugins/rush-bridge-cache-plugin/package.json

@@ -0,0 +1,11 @@
+{
+  "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/rush-plugin-manifest.schema.json",
+  "plugins": [
+    {
+      "pluginName": "rush-bridge-cache-plugin",
+      "description": "Rush plugin to allow interactions with the Rush cache.",
+      "entryPoint": "./lib/index.js",
+      "commandLineJsonFilePath": "./command-line.json"
+    }
+  ]
+}

rush-plugins/rush-bridge-cache-plugin/rush-plugin-manifest.json

@@ -0,0 +1,107 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
+// See LICENSE in the project root for license information.
+
+import { BuildCacheConfiguration, ProjectBuildCache } from '@rushstack/rush-sdk';
+import type {
+  ILogger,
+  IOperationExecutionResult,
+  IPhasedCommand,
+  IRushPlugin,
+  Operation,
+  RushConfiguration,
+  RushSession
+} from '@rushstack/rush-sdk';
+
+const PLUGIN_NAME: 'RushBridgeCachePlugin' = 'RushBridgeCachePlugin';
+
+export class BridgeCachePlugin implements IRushPlugin {
+  public readonly pluginName: string = PLUGIN_NAME;
+
+  public apply(session: RushSession, rushConfiguration: RushConfiguration): void {
+    // klutzy. Better way?
+    const actionName: string = process.argv[2];
+
+    // const isSetCacheOnly: boolean = process.argv.includes('--set-cache-only'); // has to be allowed for ANY phased command, or be customizable
+    // if (!isSetCacheOnly) {
+    //   return;
+    // }
+
+    // tracks the projects being targeted by the command (--to, --only etc.)
+    const targetProjects: string[] = [];
+
+    const cancelOperations = (operations: Set<Operation>): Set<Operation> => {
+      operations.forEach((operation: Operation) => {
+        if (operation.enabled) {
+          targetProjects.push(operation.associatedProject.packageName);
+        }
+
+        operation.enabled = false;
+      });
+      return operations;
+    };
+
+    session.hooks.runPhasedCommand
+      .for(actionName)
+      .tapPromise(PLUGIN_NAME, async (command: IPhasedCommand) => {
+        // cancel the actual operations. We don't want to actually run the command, just cache the output folders from a previous run
+        command.hooks.createOperations.tap(
+          { name: PLUGIN_NAME, stage: Number.MAX_SAFE_INTEGER },
+          cancelOperations
+        );
+
+        // now populate the cache for each operation
+        command.hooks.beforeExecuteOperations.tap(
+          PLUGIN_NAME,
+          async (recordByOperation: Map<Operation, IOperationExecutionResult>): Promise<void> => {
+            await this._setCacheAsync(session, rushConfiguration, recordByOperation, targetProjects);
+          }
+        );
+      });
+  }
+
+  private async _setCacheAsync(
+    session: RushSession,
+    rushConfiguration: RushConfiguration,
+    recordByOperation: Map<Operation, IOperationExecutionResult>,
+    targetProjects: string[]
+  ): Promise<void> {
+    const logger: ILogger = session.getLogger(PLUGIN_NAME);
+
+    // const isSetCacheOnly: boolean = process.argv.includes('--set-cache-only');
+
+    recordByOperation.forEach(
+      async (operationExecutionResult: IOperationExecutionResult, operation: Operation) => {
+        const { associatedProject, associatedPhase, settings } = operation;
+
+        if (!targetProjects.includes(associatedProject.packageName)) {
+          return;
+        }
+
+        const buildCacheConfiguration: BuildCacheConfiguration | undefined =
+          await BuildCacheConfiguration.tryLoadAsync(logger.terminal, rushConfiguration, session);
+
+        if (!buildCacheConfiguration) {
+          return;
+        }
+
+        const projectBuildCache: ProjectBuildCache = ProjectBuildCache.getProjectBuildCache({
+          project: associatedProject,
+          projectOutputFolderNames: settings?.outputFolderNames || [],
+          buildCacheConfiguration,
+          terminal: logger.terminal,
+          operationStateHash: operationExecutionResult.getStateHash(),
+          phaseName: associatedPhase.name
+        });
+
+        const success: boolean = await projectBuildCache.trySetCacheEntryAsync(logger.terminal);
+
+        // eslint-disable-next-line no-console
+        console.log('- setting cache for', {
+          success,
+          name: associatedPhase.name,
+          package: associatedProject.packageName
+        });
+      }
+    );
+  }
+}