From 2fabc1d1beb26cbeba1b7c886ef9f3732dd7e821 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 9 Jun 2023 15:08:21 +0200 Subject: Add support for sparse checkouts (#1369) * Add support for sparse checkouts * sparse-checkout: optionally turn off cone mode While it _is_ true that cone mode is the default nowadays (mainly for performance reasons: code mode is much faster than non-cone mode), there _are_ legitimate use cases where non-cone mode is really useful. Let's add a flag to optionally disable cone mode. Signed-off-by: Johannes Schindelin * Verify minimum Git version for sparse checkout The `git sparse-checkout` command is available only since Git version v2.25.0. The `actions/checkout` Action actually supports older Git versions than that; As of time of writing, the minimum version is v2.18.0. Instead of raising this minimum version even for users who do not require a sparse checkout, only check for this minimum version specifically when a sparse checkout was asked for. Suggested-by: Tingluo Huang Signed-off-by: Johannes Schindelin * Support sparse checkout/LFS better Instead of fetching all the LFS objects present in the current revision in a sparse checkout, whether they are needed inside the sparse cone or not, let's instead only pull the ones that are actually needed. To do that, let's avoid running that preemptive `git lfs fetch` call in case of a sparse checkout. An alternative that was considered during the development of this patch (and ultimately rejected) was to use `git lfs pull --include ...`, but it turned out to be too inflexible because it requires exact paths, not the patterns that are available via the sparse checkout definition, and that risks running into command-line length limitations. Signed-off-by: Johannes Schindelin --------- Signed-off-by: Johannes Schindelin Co-authored-by: Daniel --- src/git-command-manager.ts | 78 ++++++++++++++++++++++++++++++++++++++++------ src/git-source-provider.ts | 32 +++++++++++++++---- src/git-source-settings.ts | 10 ++++++ src/input-helper.ts | 11 +++++++ 4 files changed, 116 insertions(+), 15 deletions(-) (limited to 'src') diff --git a/src/git-command-manager.ts b/src/git-command-manager.ts index ab07524..e684dba 100644 --- a/src/git-command-manager.ts +++ b/src/git-command-manager.ts @@ -1,5 +1,6 @@ import * as core from '@actions/core' import * as exec from '@actions/exec' +import * as fs from 'fs' import * as fshelper from './fs-helper' import * as io from '@actions/io' import * as path from 'path' @@ -16,6 +17,8 @@ export interface IGitCommandManager { branchDelete(remote: boolean, branch: string): Promise branchExists(remote: boolean, pattern: string): Promise branchList(remote: boolean): Promise + sparseCheckout(sparseCheckout: string[]): Promise + sparseCheckoutNonConeMode(sparseCheckout: string[]): Promise checkout(ref: string, startPoint: string): Promise checkoutDetach(): Promise config( @@ -25,7 +28,13 @@ export interface IGitCommandManager { add?: boolean ): Promise configExists(configKey: string, globalConfig?: boolean): Promise - fetch(refSpec: string[], fetchDepth?: number): Promise + fetch( + refSpec: string[], + options: { + filter?: string + fetchDepth?: number + } + ): Promise getDefaultBranch(repositoryUrl: string): Promise getWorkingDirectory(): string init(): Promise @@ -52,9 +61,14 @@ export interface IGitCommandManager { export async function createCommandManager( workingDirectory: string, - lfs: boolean + lfs: boolean, + doSparseCheckout: boolean ): Promise { - return await GitCommandManager.createCommandManager(workingDirectory, lfs) + return await GitCommandManager.createCommandManager( + workingDirectory, + lfs, + doSparseCheckout + ) } class GitCommandManager { @@ -64,6 +78,7 @@ class GitCommandManager { } private gitPath = '' private lfs = false + private doSparseCheckout = false private workingDirectory = '' // Private constructor; use createCommandManager() @@ -154,6 +169,27 @@ class GitCommandManager { return result } + async sparseCheckout(sparseCheckout: string[]): Promise { + await this.execGit(['sparse-checkout', 'set', ...sparseCheckout]) + } + + async sparseCheckoutNonConeMode(sparseCheckout: string[]): Promise { + await this.execGit(['config', 'core.sparseCheckout', 'true']) + const output = await this.execGit([ + 'rev-parse', + '--git-path', + 'info/sparse-checkout' + ]) + const sparseCheckoutPath = path.join( + this.workingDirectory, + output.stdout.trimRight() + ) + await fs.promises.appendFile( + sparseCheckoutPath, + `\n${sparseCheckout.join('\n')}\n` + ) + } + async checkout(ref: string, startPoint: string): Promise { const args = ['checkout', '--progress', '--force'] if (startPoint) { @@ -202,15 +238,23 @@ class GitCommandManager { return output.exitCode === 0 } - async fetch(refSpec: string[], fetchDepth?: number): Promise { + async fetch( + refSpec: string[], + options: {filter?: string; fetchDepth?: number} + ): Promise { const args = ['-c', 'protocol.version=2', 'fetch'] if (!refSpec.some(x => x === refHelper.tagsRefSpec)) { args.push('--no-tags') } args.push('--prune', '--progress', '--no-recurse-submodules') - if (fetchDepth && fetchDepth > 0) { - args.push(`--depth=${fetchDepth}`) + + if (options.filter) { + args.push(`--filter=${options.filter}`) + } + + if (options.fetchDepth && options.fetchDepth > 0) { + args.push(`--depth=${options.fetchDepth}`) } else if ( fshelper.fileExistsSync( path.join(this.workingDirectory, '.git', 'shallow') @@ -423,10 +467,15 @@ class GitCommandManager { static async createCommandManager( workingDirectory: string, - lfs: boolean + lfs: boolean, + doSparseCheckout: boolean ): Promise { const result = new GitCommandManager() - await result.initializeCommandManager(workingDirectory, lfs) + await result.initializeCommandManager( + workingDirectory, + lfs, + doSparseCheckout + ) return result } @@ -476,7 +525,8 @@ class GitCommandManager { private async initializeCommandManager( workingDirectory: string, - lfs: boolean + lfs: boolean, + doSparseCheckout: boolean ): Promise { this.workingDirectory = workingDirectory @@ -539,6 +589,16 @@ class GitCommandManager { } } + this.doSparseCheckout = doSparseCheckout + if (this.doSparseCheckout) { + // The `git sparse-checkout` command was introduced in Git v2.25.0 + const minimumGitSparseCheckoutVersion = new GitVersion('2.25') + if (!gitVersion.checkMinimum(minimumGitSparseCheckoutVersion)) { + throw new Error( + `Minimum Git version required for sparse checkout is ${minimumGitSparseCheckoutVersion}. Your git ('${this.gitPath}') is ${gitVersion}` + ) + } + } // Set the user agent const gitHttpUserAgent = `git/${gitVersion} (github-actions-checkout)` core.debug(`Set git useragent to: ${gitHttpUserAgent}`) diff --git a/src/git-source-provider.ts b/src/git-source-provider.ts index 48f20da..8f9d63f 100644 --- a/src/git-source-provider.ts +++ b/src/git-source-provider.ts @@ -153,23 +153,26 @@ export async function getSource(settings: IGitSourceSettings): Promise { // Fetch core.startGroup('Fetching the repository') + const fetchOptions: {filter?: string; fetchDepth?: number} = {} + if (settings.sparseCheckout) fetchOptions.filter = 'blob:none' if (settings.fetchDepth <= 0) { // Fetch all branches and tags let refSpec = refHelper.getRefSpecForAllHistory( settings.ref, settings.commit ) - await git.fetch(refSpec) + await git.fetch(refSpec, fetchOptions) // When all history is fetched, the ref we're interested in may have moved to a different // commit (push or force push). If so, fetch again with a targeted refspec. if (!(await refHelper.testRef(git, settings.ref, settings.commit))) { refSpec = refHelper.getRefSpec(settings.ref, settings.commit) - await git.fetch(refSpec) + await git.fetch(refSpec, fetchOptions) } } else { + fetchOptions.fetchDepth = settings.fetchDepth const refSpec = refHelper.getRefSpec(settings.ref, settings.commit) - await git.fetch(refSpec, settings.fetchDepth) + await git.fetch(refSpec, fetchOptions) } core.endGroup() @@ -185,12 +188,24 @@ export async function getSource(settings: IGitSourceSettings): Promise { // LFS fetch // Explicit lfs-fetch to avoid slow checkout (fetches one lfs object at a time). // Explicit lfs fetch will fetch lfs objects in parallel. - if (settings.lfs) { + // For sparse checkouts, let `checkout` fetch the needed objects lazily. + if (settings.lfs && !settings.sparseCheckout) { core.startGroup('Fetching LFS objects') await git.lfsFetch(checkoutInfo.startPoint || checkoutInfo.ref) core.endGroup() } + // Sparse checkout + if (settings.sparseCheckout) { + core.startGroup('Setting up sparse checkout') + if (settings.sparseCheckoutConeMode) { + await git.sparseCheckout(settings.sparseCheckout) + } else { + await git.sparseCheckoutNonConeMode(settings.sparseCheckout) + } + core.endGroup() + } + // Checkout core.startGroup('Checking out the ref') await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint) @@ -261,7 +276,11 @@ export async function cleanup(repositoryPath: string): Promise { let git: IGitCommandManager try { - git = await gitCommandManager.createCommandManager(repositoryPath, false) + git = await gitCommandManager.createCommandManager( + repositoryPath, + false, + false + ) } catch { return } @@ -297,7 +316,8 @@ async function getGitCommandManager( try { return await gitCommandManager.createCommandManager( settings.repositoryPath, - settings.lfs + settings.lfs, + settings.sparseCheckout != null ) } catch (err) { // Git is required for LFS diff --git a/src/git-source-settings.ts b/src/git-source-settings.ts index 2da5622..3272e63 100644 --- a/src/git-source-settings.ts +++ b/src/git-source-settings.ts @@ -29,6 +29,16 @@ export interface IGitSourceSettings { */ clean: boolean + /** + * The array of folders to make the sparse checkout + */ + sparseCheckout: string[] + + /** + * Indicates whether to use cone mode in the sparse checkout (if any) + */ + sparseCheckoutConeMode: boolean + /** * The depth when fetching */ diff --git a/src/input-helper.ts b/src/input-helper.ts index 237b06a..410e480 100644 --- a/src/input-helper.ts +++ b/src/input-helper.ts @@ -82,6 +82,17 @@ export async function getInputs(): Promise { result.clean = (core.getInput('clean') || 'true').toUpperCase() === 'TRUE' core.debug(`clean = ${result.clean}`) + // Sparse checkout + const sparseCheckout = core.getMultilineInput('sparse-checkout') + if (sparseCheckout.length) { + result.sparseCheckout = sparseCheckout + core.debug(`sparse checkout = ${result.sparseCheckout}`) + } + + result.sparseCheckoutConeMode = + (core.getInput('sparse-checkout-cone-mode') || 'true').toUpperCase() === + 'TRUE' + // Fetch depth result.fetchDepth = Math.floor(Number(core.getInput('fetch-depth') || '1')) if (isNaN(result.fetchDepth) || result.fetchDepth < 0) { -- cgit v1.2.3