[Incomplete - for discussion] Add bridge-cache plugin to populate cache without running action

apps/rush/package.json

@@ -47,6 +47,7 @@
     "@rushstack/rush-amazon-s3-build-cache-plugin": "workspace:*",
     "@rushstack/rush-azure-storage-build-cache-plugin": "workspace:*",
     "@rushstack/rush-http-build-cache-plugin": "workspace:*",
+    "@rushstack/rush-bridge-cache-plugin": "workspace:*",
     "@types/heft-jest": "1.0.1",
     "@types/semver": "7.5.0"
   }

apps/rush/src/start-dev-docs.ts

@@ -6,4 +6,4 @@ import { Colorize, ConsoleTerminalProvider, Terminal } from '@rushstack/terminal
 const terminal: Terminal = new Terminal(new ConsoleTerminalProvider());
 
 terminal.writeLine('For instructions on debugging Rush, please see this documentation:');
-terminal.writeLine(Colorize.bold('https://rushjs.io/pages/contributing/debugging/'));
+terminal.writeLine(Colorize.bold('https://rushjs.io/pages/contributing/#debugging-rush'));

apps/rush/src/start-dev.ts

@@ -31,6 +31,7 @@ includePlugin('rush-azure-storage-build-cache-plugin');
 includePlugin('rush-http-build-cache-plugin');
 // Including this here so that developers can reuse it without installing the plugin a second time
 includePlugin('rush-azure-interactive-auth-plugin', '@rushstack/rush-azure-storage-build-cache-plugin');
+includePlugin('rush-bridge-cache-plugin');
 
 const currentPackageVersion: string = PackageJsonLookup.loadOwnPackageJson(__dirname).version;
 RushCommandSelector.execute(currentPackageVersion, rushLib, {

common/config/rush/browser-approved-packages.json

@@ -38,6 +38,10 @@
       "name": "@reduxjs/toolkit",
       "allowedCategories": [ "libraries", "vscode-extensions" ]
     },
+    {
+      "name": "@rushstack/rush-bridge-cache-plugin",
+      "allowedCategories": [ "libraries" ]
+    },
     {
       "name": "@rushstack/rush-themed-ui",
       "allowedCategories": [ "libraries" ]

common/reviews/api/rush-lib.api.md

@@ -10,7 +10,7 @@ import { AsyncParallelHook } from 'tapable';
 import { AsyncSeriesBailHook } from 'tapable';
 import { AsyncSeriesHook } from 'tapable';
 import { AsyncSeriesWaterfallHook } from 'tapable';
-import type { CollatedWriter } from '@rushstack/stream-collator';
+import { CollatedWriter } from '@rushstack/stream-collator';
 import type { CommandLineParameter } from '@rushstack/ts-command-line';
 import { CommandLineParameterKind } from '@rushstack/ts-command-line';
 import { HookMap } from 'tapable';
@@ -23,7 +23,8 @@ import { JsonNull } from '@rushstack/node-core-library';
 import { JsonObject } from '@rushstack/node-core-library';
 import { LookupByPath } from '@rushstack/lookup-by-path';
 import { PackageNameParser } from '@rushstack/node-core-library';
-import type { StdioSummarizer } from '@rushstack/terminal';
+import { StdioSummarizer } from '@rushstack/terminal';
+import { StreamCollator } from '@rushstack/stream-collator';
 import { SyncHook } from 'tapable';
 import { SyncWaterfallHook } from 'tapable';
 import { Terminal } from '@rushstack/terminal';
@@ -945,6 +946,92 @@ export class Operation {
     weight: number;
 }
 
+// Warning: (ae-internal-missing-underscore) The name "OperationBuildCache" should be prefixed with an underscore because the declaration is marked as @internal
+//
+// @internal (undocumented)
+export class OperationBuildCache {
+    // (undocumented)
+    get cacheId(): string | undefined;
+    // Warning: (ae-forgotten-export) The symbol "IOperationBuildCacheOptions" needs to be exported by the entry point index.d.ts
+    //
+    // (undocumented)
+    static forOperation(operation: OperationExecutionRecord, options: IOperationBuildCacheOptions): OperationBuildCache;
+    // Warning: (ae-forgotten-export) The symbol "IProjectBuildCacheOptions" needs to be exported by the entry point index.d.ts
+    //
+    // (undocumented)
+    static getProjectBuildCache(options: IProjectBuildCacheOptions): OperationBuildCache;
+    // (undocumented)
+    tryRestoreFromCacheAsync(terminal: ITerminal, specifiedCacheId?: string): Promise<boolean>;
+    // (undocumented)
+    trySetCacheEntryAsync(terminal: ITerminal, specifiedCacheId?: string): Promise<boolean>;
+}
+
+// Warning: (ae-internal-missing-underscore) The name "OperationExecutionRecord" should be prefixed with an underscore because the declaration is marked as @internal
+//
+// @internal
+export class OperationExecutionRecord implements IOperationRunnerContext, IOperationExecutionResult {
+    // Warning: (ae-forgotten-export) The symbol "IOperationExecutionRecordContext" needs to be exported by the entry point index.d.ts
+    constructor(operation: Operation, context: IOperationExecutionRecordContext);
+    // (undocumented)
+    readonly associatedPhase: IPhase;
+    // (undocumented)
+    readonly associatedProject: RushConfigurationProject;
+    // (undocumented)
+    get cobuildRunnerId(): string | undefined;
+    // (undocumented)
+    get collatedWriter(): CollatedWriter;
+    readonly consumers: Set<OperationExecutionRecord>;
+    criticalPathLength: number | undefined;
+    // (undocumented)
+    get debugMode(): boolean;
+    readonly dependencies: Set<OperationExecutionRecord>;
+    // (undocumented)
+    get environment(): IEnvironment | undefined;
+    error: Error | undefined;
+    // (undocumented)
+    executeAsync({ onStart, onResult }: {
+        onStart: (record: OperationExecutionRecord) => Promise<OperationStatus | undefined>;
+        onResult: (record: OperationExecutionRecord) => Promise<void>;
+    }): Promise<void>;
+    // (undocumented)
+    getStateHash(): string;
+    // (undocumented)
+    getStateHashComponents(): ReadonlyArray<string>;
+    // (undocumented)
+    get isTerminal(): boolean;
+    // (undocumented)
+    logFilePaths: ILogFilePaths | undefined;
+    // (undocumented)
+    get metadataFolderPath(): string | undefined;
+    // (undocumented)
+    get name(): string;
+    // (undocumented)
+    get nonCachedDurationMs(): number | undefined;
+    readonly operation: Operation;
+    // (undocumented)
+    readonly _operationMetadataManager: _OperationMetadataManager;
+    // (undocumented)
+    get quietMode(): boolean;
+    // (undocumented)
+    readonly runner: IOperationRunner;
+    runWithTerminalAsync<T>(callback: (terminal: ITerminal, terminalProvider: ITerminalProvider) => Promise<T>, options: {
+        createLogFile: boolean;
+        logFileSuffix: string;
+    }): Promise<T>;
+    // (undocumented)
+    get silent(): boolean;
+    get status(): OperationStatus;
+    set status(newStatus: OperationStatus);
+    // (undocumented)
+    readonly stdioSummarizer: StdioSummarizer;
+    // Warning: (ae-forgotten-export) The symbol "Stopwatch" needs to be exported by the entry point index.d.ts
+    //
+    // (undocumented)
+    readonly stopwatch: Stopwatch;
+    // (undocumented)
+    get weight(): number;
+}
+
 // @internal
 export class _OperationMetadataManager {
     constructor(options: _IOperationMetadataManagerOptions);

libraries/rush-lib/src/index.ts

@@ -135,6 +135,7 @@ export type {
   IExecutionResult,
   IOperationExecutionResult
 } from './logic/operations/IOperationExecutionResult';
+export type { OperationExecutionRecord } from './logic/operations/OperationExecutionRecord';
 export { type IOperationOptions, Operation } from './logic/operations/Operation';
 export { OperationStatus } from './logic/operations/OperationStatus';
 export type { ILogFilePaths } from './logic/operations/ProjectLogWritable';
@@ -197,3 +198,5 @@ export {
   type IRushCommandLineParameter,
   type IRushCommandLineAction
 } from './api/RushCommandLine';
+
+export { ProjectBuildCache as OperationBuildCache } from './logic/buildCache/ProjectBuildCache';

libraries/rush-lib/src/logic/buildCache/ProjectBuildCache.ts

@@ -50,6 +50,9 @@ interface IPathsToCache {
   outputFilePaths: string[];
 }
 
+/**
+ * @internal
+ */
 export class ProjectBuildCache {
   private static _tarUtilityPromise: Promise<TarExecutable | undefined> | undefined;
 

libraries/rush-lib/src/logic/operations/ShardedPhaseOperationPlugin.ts

@@ -128,6 +128,8 @@ function spliceShards(existingOperations: Set<Operation>, context: ICreateOperat
       ];
 
       const { scripts } = project.packageJson;
+
+      //
       const commandToRun: string | undefined = phase.shellCommand ?? scripts?.[phase.name];
 
       operation.logFilenameIdentifier = `${baseLogFilenameIdentifier}_collate`;

libraries/rush-lib/src/pluginFramework/PluginManager.ts

@@ -81,6 +81,7 @@ export class PluginManager {
     tryAddBuiltInPlugin('rush-amazon-s3-build-cache-plugin');
     tryAddBuiltInPlugin('rush-azure-storage-build-cache-plugin');
     tryAddBuiltInPlugin('rush-http-build-cache-plugin');
+    tryAddBuiltInPlugin('rush-bridge-cache-plugin');
     // This is a secondary plugin inside the `@rushstack/rush-azure-storage-build-cache-plugin`
     // package. Because that package comes with Rush (for now), it needs to get registered here.
     // If the necessary config file doesn't exist, this plugin doesn't do anything.

rush-plugins/rush-bridge-cache-plugin/.eslintrc.js

@@ -0,0 +1,13 @@
+// This is a workaround for https://github.com/eslint/eslint/issues/3458
+require('local-node-rig/profiles/default/includes/eslint/patch/modern-module-resolution');
+// This is a workaround for https://github.com/microsoft/rushstack/issues/3021
+require('local-node-rig/profiles/default/includes/eslint/patch/custom-config-package-names');
+
+module.exports = {
+  extends: [
+    'local-node-rig/profiles/default/includes/eslint/profile/node',
+    'local-node-rig/profiles/default/includes/eslint/mixins/friendly-locals',
+    'local-node-rig/profiles/default/includes/eslint/mixins/tsdoc'
+  ],
+  parserOptions: { tsconfigRootDir: __dirname }
+};

rush-plugins/rush-bridge-cache-plugin/LICENSE

@@ -0,0 +1,24 @@
+@rushstack/rush-serve-plugin
+
+Copyright (c) Microsoft Corporation. All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

rush-plugins/rush-bridge-cache-plugin/README.md

@@ -0,0 +1,45 @@
+# @rushstack/rush-bridge-cache-plugin
+
+This is a Rush plugin that adds an optional `--set-cache-only` flag to Rush's phased commands, so you can other tools to actual run the scripts and generate the build artifacts on disk, then separately populate the Rush cache for particular actions(s). For integrations with other build orchestrators, this allows the best of both worlds: using a different build tool to orchestrate the work, but still populate the Rush cache for benefiting local use of Rush.
+
+## Here be dragons!
+
+This is a power-user sort of plugin. It assumes that the work for a particular task has already been completed and the build artifacts have been generated on disk. **If you run this command on a package where the command hasn't already been ran and the build artifacts are missing or incorrect, you will cache invalid content**. Be careful and beware!
+
+
+## Installation
+
+(TODO)
+
+`npm install @rushstack/rush-bridge-cache-plugin`
+
+
+## Configuration
+
+First you need to update your `command-line.json` file to add the new flag. Configure it to target whatever specific commands you want to have this feature. Example:
+
+```json
+{
+  "associatedCommands": ["build", "test", "lint", "a11y", "typecheck"],
+  "description": "When the flag is added to any associated command, it'll bypass running the command itself, but cache the result of a previous run. Beware! Only run when you know the build artifacts are in a valid state for the command.",
+  "parameterKind": "flag",
+  "longName": "--set-cache-only",
+  "required": false
+}
+```
+
+
+## Usage
+
+Any of the rush command can now just be given a `--set-cache-only` property, e.g.
+
+`rush build --to your-packageX --set-cache-only`
+
+That will examine `your-packageX` and all of its dependencies, then populate the cache.
+
+
+## Performance
+
+When running within a pipeline, you'll want to populate the cache as fast as possible. So instead of waiting until the full build graph has been processed, you'll wnt to run it after each successful task. For that, just use Rush's `--only` and target whatever task had just completed, for example:
+
+`rush lint --only your-packageY --set-cache-only`

rush-plugins/rush-bridge-cache-plugin/config/jest.config.json

@@ -0,0 +1,3 @@
+{
+  "extends": "local-node-rig/profiles/default/config/jest.config.json"
+}

rush-plugins/rush-bridge-cache-plugin/config/rig.json

@@ -0,0 +1,7 @@
+{
+  // The "rig.json" file directs tools to look for their config files in an external package.
+  // Documentation for this system: https://www.npmjs.com/package/@rushstack/rig-package
+  "$schema": "https://developer.microsoft.com/json-schemas/rig-package/rig.schema.json",
+
+  "rigPackageName": "local-node-rig"
+}

rush-plugins/rush-bridge-cache-plugin/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "@rushstack/rush-bridge-cache-plugin",
+  "version": "0.0.1",
+  "private": true,
+  "description": "Rush plugin that provides a --set-cache-only command flag to populates the cache from previous runs.",
+  "license": "MIT",
+  "main": "./lib/index.js",
+  "repository": {
+    "url": "https://github.com/microsoft/rushstack.git",
+    "type": "git",
+    "directory": "rush-plugins/rush-bridge-cache-plugin"
+  },
+  "scripts": {
+    "build": "heft test --clean",
+    "_phase:build": "heft run --only build -- --clean",
+    "_phase:test": "heft run --only test -- --clean"
+  },
+  "devDependencies": {
+    "@rushstack/node-core-library": "workspace:*",
+    "@microsoft/rush-lib": "workspace:*",
+    "@rushstack/rush-sdk": "workspace:*",
+    "@rushstack/terminal": "workspace:*",
+    "@rushstack/heft": "workspace:*",
+    "local-node-rig": "workspace:*"
+  }
+}

rush-plugins/rush-bridge-cache-plugin/rush-plugin-manifest.json

@@ -0,0 +1,10 @@
+{
+  "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/rush-plugin-manifest.schema.json",
+  "plugins": [
+    {
+      "pluginName": "rush-bridge-cache-plugin",
+      "description": "Rush plugin that provides a --set-cache-only command flag to populates the cache from previous runs.",
+      "entryPoint": "./lib/index.js"
+    }
+  ]
+}