From 25ac60e43ded675f082e10f444b0ad95ea43622e Mon Sep 17 00:00:00 2001 From: eric sciple Date: Thu, 12 Dec 2019 13:16:16 -0500 Subject: fallback to REST API to download repo (#104) --- src/git-command-manager.ts | 63 +++------- src/git-source-provider.ts | 290 +++++++++++++++++++++++++-------------------- src/github-api-helper.ts | 92 ++++++++++++++ src/main.ts | 7 +- src/retry-helper.ts | 61 ++++++++++ src/state-helper.ts | 30 +++++ 6 files changed, 363 insertions(+), 180 deletions(-) create mode 100644 src/github-api-helper.ts create mode 100644 src/retry-helper.ts create mode 100644 src/state-helper.ts (limited to 'src') diff --git a/src/git-command-manager.ts b/src/git-command-manager.ts index 6e4f22f..ddd5ec6 100644 --- a/src/git-command-manager.ts +++ b/src/git-command-manager.ts @@ -3,8 +3,13 @@ import * as exec from '@actions/exec' import * as fshelper from './fs-helper' import * as io from '@actions/io' import * as path from 'path' +import * as retryHelper from './retry-helper' import {GitVersion} from './git-version' +// Auth header not supported before 2.9 +// Wire protocol v2 not supported before 2.18 +export const MinimumGitVersion = new GitVersion('2.18') + export interface IGitCommandManager { branchDelete(remote: boolean, branch: string): Promise branchExists(remote: boolean, pattern: string): Promise @@ -150,22 +155,10 @@ class GitCommandManager { args.push(arg) } - let attempt = 1 - const maxAttempts = 3 - while (attempt <= maxAttempts) { - const allowAllExitCodes = attempt < maxAttempts - const output = await this.execGit(args, allowAllExitCodes) - if (output.exitCode === 0) { - break - } - - const seconds = this.getRandomIntInclusive(1, 10) - core.warning( - `Git fetch failed with exit code ${output.exitCode}. Waiting ${seconds} seconds before trying again.` - ) - await this.sleep(seconds * 1000) - attempt++ - } + const that = this + await retryHelper.execute(async () => { + await that.execGit(args) + }) } getWorkingDirectory(): string { @@ -188,22 +181,10 @@ class GitCommandManager { async lfsFetch(ref: string): Promise { const args = ['lfs', 'fetch', 'origin', ref] - let attempt = 1 - const maxAttempts = 3 - while (attempt <= maxAttempts) { - const allowAllExitCodes = attempt < maxAttempts - const output = await this.execGit(args, allowAllExitCodes) - if (output.exitCode === 0) { - break - } - - const seconds = this.getRandomIntInclusive(1, 10) - core.warning( - `Git lfs fetch failed with exit code ${output.exitCode}. Waiting ${seconds} seconds before trying again.` - ) - await this.sleep(seconds * 1000) - attempt++ - } + const that = this + await retryHelper.execute(async () => { + await that.execGit(args) + }) } async lfsInstall(): Promise { @@ -338,13 +319,9 @@ class GitCommandManager { } // Minimum git version - // Note: - // - Auth header not supported before 2.9 - // - Wire protocol v2 not supported before 2.18 - const minimumGitVersion = new GitVersion('2.18') - if (!gitVersion.checkMinimum(minimumGitVersion)) { + if (!gitVersion.checkMinimum(MinimumGitVersion)) { throw new Error( - `Minimum required git version is ${minimumGitVersion}. Your git ('${this.gitPath}') is ${gitVersion}` + `Minimum required git version is ${MinimumGitVersion}. Your git ('${this.gitPath}') is ${gitVersion}` ) } @@ -381,16 +358,6 @@ class GitCommandManager { core.debug(`Set git useragent to: ${gitHttpUserAgent}`) this.gitEnv['GIT_HTTP_USER_AGENT'] = gitHttpUserAgent } - - private getRandomIntInclusive(minimum: number, maximum: number): number { - minimum = Math.floor(minimum) - maximum = Math.floor(maximum) - return Math.floor(Math.random() * (maximum - minimum + 1)) + minimum - } - - private async sleep(milliseconds): Promise { - return new Promise(resolve => setTimeout(resolve, milliseconds)) - } } class GitOutput { diff --git a/src/git-source-provider.ts b/src/git-source-provider.ts index fc41fd2..bf086ff 100644 --- a/src/git-source-provider.ts +++ b/src/git-source-provider.ts @@ -3,9 +3,11 @@ import * as coreCommand from '@actions/core/lib/command' import * as fs from 'fs' import * as fsHelper from './fs-helper' import * as gitCommandManager from './git-command-manager' +import * as githubApiHelper from './github-api-helper' import * as io from '@actions/io' import * as path from 'path' import * as refHelper from './ref-helper' +import * as stateHelper from './state-helper' import {IGitCommandManager} from './git-command-manager' const authConfigKey = `http.https://github.com/.extraheader` @@ -23,6 +25,7 @@ export interface ISourceSettings { } export async function getSource(settings: ISourceSettings): Promise { + // Repository URL core.info( `Syncing repository: ${settings.repositoryOwner}/${settings.repositoryName}` ) @@ -43,92 +46,92 @@ export async function getSource(settings: ISourceSettings): Promise { } // Git command manager - core.info(`Working directory is '${settings.repositoryPath}'`) - const git = await gitCommandManager.CreateCommandManager( - settings.repositoryPath, - settings.lfs - ) + const git = await getGitCommandManager(settings) - // Try prepare existing directory, otherwise recreate - if ( - isExisting && - !(await tryPrepareExistingDirectory( + // Prepare existing directory, otherwise recreate + if (isExisting) { + await prepareExistingDirectory( git, settings.repositoryPath, repositoryUrl, settings.clean - )) - ) { - // Delete the contents of the directory. Don't delete the directory itself - // since it may be the current working directory. - core.info(`Deleting the contents of '${settings.repositoryPath}'`) - for (const file of await fs.promises.readdir(settings.repositoryPath)) { - await io.rmRF(path.join(settings.repositoryPath, file)) - } - } - - // Initialize the repository - if ( - !fsHelper.directoryExistsSync(path.join(settings.repositoryPath, '.git')) - ) { - await git.init() - await git.remoteAdd('origin', repositoryUrl) - } - - // Disable automatic garbage collection - if (!(await git.tryDisableAutomaticGarbageCollection())) { - core.warning( - `Unable to turn off git automatic garbage collection. The git fetch operation may trigger garbage collection and cause a delay.` ) } - // Remove possible previous extraheader - await removeGitConfig(git, authConfigKey) + if (!git) { + // Downloading using REST API + core.info(`The repository will be downloaded using the GitHub REST API`) + core.info( + `To create a local Git repository instead, add Git ${gitCommandManager.MinimumGitVersion} or higher to the PATH` + ) + await githubApiHelper.downloadRepository( + settings.accessToken, + settings.repositoryOwner, + settings.repositoryName, + settings.ref, + settings.commit, + settings.repositoryPath + ) + } else { + // Save state for POST action + stateHelper.setRepositoryPath(settings.repositoryPath) + + // Initialize the repository + if ( + !fsHelper.directoryExistsSync(path.join(settings.repositoryPath, '.git')) + ) { + await git.init() + await git.remoteAdd('origin', repositoryUrl) + } - // Add extraheader (auth) - const base64Credentials = Buffer.from( - `x-access-token:${settings.accessToken}`, - 'utf8' - ).toString('base64') - core.setSecret(base64Credentials) - const authConfigValue = `AUTHORIZATION: basic ${base64Credentials}` - await git.config(authConfigKey, authConfigValue) - - // LFS install - if (settings.lfs) { - await git.lfsInstall() - } + // Disable automatic garbage collection + if (!(await git.tryDisableAutomaticGarbageCollection())) { + core.warning( + `Unable to turn off git automatic garbage collection. The git fetch operation may trigger garbage collection and cause a delay.` + ) + } - // Fetch - const refSpec = refHelper.getRefSpec(settings.ref, settings.commit) - await git.fetch(settings.fetchDepth, refSpec) + // Remove possible previous extraheader + await removeGitConfig(git, authConfigKey) + + // Add extraheader (auth) + const base64Credentials = Buffer.from( + `x-access-token:${settings.accessToken}`, + 'utf8' + ).toString('base64') + core.setSecret(base64Credentials) + const authConfigValue = `AUTHORIZATION: basic ${base64Credentials}` + await git.config(authConfigKey, authConfigValue) + + // LFS install + if (settings.lfs) { + await git.lfsInstall() + } - // Checkout info - const checkoutInfo = await refHelper.getCheckoutInfo( - git, - settings.ref, - settings.commit - ) + // Fetch + const refSpec = refHelper.getRefSpec(settings.ref, settings.commit) + await git.fetch(settings.fetchDepth, refSpec) - // LFS fetch - // Explicit lfs-fetch to avoid slow checkout (fetches one lfs object at a time). - // Explicit lfs fetch will fetch lfs objects in parallel. - if (settings.lfs) { - await git.lfsFetch(checkoutInfo.startPoint || checkoutInfo.ref) - } + // Checkout info + const checkoutInfo = await refHelper.getCheckoutInfo( + git, + settings.ref, + settings.commit + ) - // Checkout - await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint) + // LFS fetch + // Explicit lfs-fetch to avoid slow checkout (fetches one lfs object at a time). + // Explicit lfs fetch will fetch lfs objects in parallel. + if (settings.lfs) { + await git.lfsFetch(checkoutInfo.startPoint || checkoutInfo.ref) + } - // Dump some info about the checked out commit - await git.log1() + // Checkout + await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint) - // Set intra-task state for cleanup - coreCommand.issueCommand( - 'save-state', - {name: 'repositoryPath'}, - settings.repositoryPath - ) + // Dump some info about the checked out commit + await git.log1() + } } export async function cleanup(repositoryPath: string): Promise { @@ -146,79 +149,110 @@ export async function cleanup(repositoryPath: string): Promise { await removeGitConfig(git, authConfigKey) } -async function tryPrepareExistingDirectory( +async function getGitCommandManager( + settings: ISourceSettings +): Promise { + core.info(`Working directory is '${settings.repositoryPath}'`) + let git = (null as unknown) as IGitCommandManager + try { + return await gitCommandManager.CreateCommandManager( + settings.repositoryPath, + settings.lfs + ) + } catch (err) { + // Git is required for LFS + if (settings.lfs) { + throw err + } + + // Otherwise fallback to REST API + return (null as unknown) as IGitCommandManager + } +} + +async function prepareExistingDirectory( git: IGitCommandManager, repositoryPath: string, repositoryUrl: string, clean: boolean -): Promise { +): Promise { + let remove = false + + // Check whether using git or REST API + if (!git) { + remove = true + } // Fetch URL does not match - if ( + else if ( !fsHelper.directoryExistsSync(path.join(repositoryPath, '.git')) || repositoryUrl !== (await git.tryGetFetchUrl()) ) { - return false - } + remove = true + } else { + // Delete any index.lock and shallow.lock left by a previously canceled run or crashed git process + const lockPaths = [ + path.join(repositoryPath, '.git', 'index.lock'), + path.join(repositoryPath, '.git', 'shallow.lock') + ] + for (const lockPath of lockPaths) { + try { + await io.rmRF(lockPath) + } catch (error) { + core.debug(`Unable to delete '${lockPath}'. ${error.message}`) + } + } - // Delete any index.lock and shallow.lock left by a previously canceled run or crashed git process - const lockPaths = [ - path.join(repositoryPath, '.git', 'index.lock'), - path.join(repositoryPath, '.git', 'shallow.lock') - ] - for (const lockPath of lockPaths) { try { - await io.rmRF(lockPath) + // Checkout detached HEAD + if (!(await git.isDetached())) { + await git.checkoutDetach() + } + + // Remove all refs/heads/* + let branches = await git.branchList(false) + for (const branch of branches) { + await git.branchDelete(false, branch) + } + + // Remove all refs/remotes/origin/* to avoid conflicts + branches = await git.branchList(true) + for (const branch of branches) { + await git.branchDelete(true, branch) + } + + // Clean + if (clean) { + if (!(await git.tryClean())) { + core.debug( + `The clean command failed. This might be caused by: 1) path too long, 2) permission issue, or 3) file in use. For futher investigation, manually run 'git clean -ffdx' on the directory '${repositoryPath}'.` + ) + remove = true + } else if (!(await git.tryReset())) { + remove = true + } + + if (remove) { + core.warning( + `Unable to clean or reset the repository. The repository will be recreated instead.` + ) + } + } } catch (error) { - core.debug(`Unable to delete '${lockPath}'. ${error.message}`) - } - } - - try { - // Checkout detached HEAD - if (!(await git.isDetached())) { - await git.checkoutDetach() - } - - // Remove all refs/heads/* - let branches = await git.branchList(false) - for (const branch of branches) { - await git.branchDelete(false, branch) - } - - // Remove all refs/remotes/origin/* to avoid conflicts - branches = await git.branchList(true) - for (const branch of branches) { - await git.branchDelete(true, branch) - } - } catch (error) { - core.warning( - `Unable to prepare the existing repository. The repository will be recreated instead.` - ) - return false - } - - // Clean - if (clean) { - let succeeded = true - if (!(await git.tryClean())) { - core.debug( - `The clean command failed. This might be caused by: 1) path too long, 2) permission issue, or 3) file in use. For futher investigation, manually run 'git clean -ffdx' on the directory '${repositoryPath}'.` - ) - succeeded = false - } else if (!(await git.tryReset())) { - succeeded = false - } - - if (!succeeded) { core.warning( - `Unable to clean or reset the repository. The repository will be recreated instead.` + `Unable to prepare the existing repository. The repository will be recreated instead.` ) + remove = true } - - return succeeded } - return true + if (remove) { + // Delete the contents of the directory. Don't delete the directory itself + // since it might be the current working directory. + core.info(`Deleting the contents of '${repositoryPath}'`) + for (const file of await fs.promises.readdir(repositoryPath)) { + await io.rmRF(path.join(repositoryPath, file)) + } + } } async function removeGitConfig( diff --git a/src/github-api-helper.ts b/src/github-api-helper.ts new file mode 100644 index 0000000..b370d06 --- /dev/null +++ b/src/github-api-helper.ts @@ -0,0 +1,92 @@ +import * as assert from 'assert' +import * as core from '@actions/core' +import * as fs from 'fs' +import * as github from '@actions/github' +import * as io from '@actions/io' +import * as path from 'path' +import * as retryHelper from './retry-helper' +import * as toolCache from '@actions/tool-cache' +import {default as uuid} from 'uuid/v4' +import {ReposGetArchiveLinkParams} from '@octokit/rest' + +const IS_WINDOWS = process.platform === 'win32' + +export async function downloadRepository( + accessToken: string, + owner: string, + repo: string, + ref: string, + commit: string, + repositoryPath: string +): Promise { + // Download the archive + let archiveData = await retryHelper.execute(async () => { + core.info('Downloading the archive') + return await downloadArchive(accessToken, owner, repo, ref, commit) + }) + + // Write archive to disk + core.info('Writing archive to disk') + const uniqueId = uuid() + const archivePath = path.join(repositoryPath, `${uniqueId}.tar.gz`) + await fs.promises.writeFile(archivePath, archiveData) + archiveData = Buffer.from('') // Free memory + + // Extract archive + core.info('Extracting the archive') + const extractPath = path.join(repositoryPath, uniqueId) + await io.mkdirP(extractPath) + if (IS_WINDOWS) { + await toolCache.extractZip(archivePath, extractPath) + } else { + await toolCache.extractTar(archivePath, extractPath) + } + io.rmRF(archivePath) + + // Determine the path of the repository content. The archive contains + // a top-level folder and the repository content is inside. + const archiveFileNames = await fs.promises.readdir(extractPath) + assert.ok( + archiveFileNames.length == 1, + 'Expected exactly one directory inside archive' + ) + const archiveVersion = archiveFileNames[0] // The top-level folder name includes the short SHA + core.info(`Resolved version ${archiveVersion}`) + const tempRepositoryPath = path.join(extractPath, archiveVersion) + + // Move the files + for (const fileName of await fs.promises.readdir(tempRepositoryPath)) { + const sourcePath = path.join(tempRepositoryPath, fileName) + const targetPath = path.join(repositoryPath, fileName) + if (IS_WINDOWS) { + await io.cp(sourcePath, targetPath, {recursive: true}) // Copy on Windows (Windows Defender may have a lock) + } else { + await io.mv(sourcePath, targetPath) + } + } + io.rmRF(extractPath) +} + +async function downloadArchive( + accessToken: string, + owner: string, + repo: string, + ref: string, + commit: string +): Promise { + const octokit = new github.GitHub(accessToken) + const params: ReposGetArchiveLinkParams = { + owner: owner, + repo: repo, + archive_format: IS_WINDOWS ? 'zipball' : 'tarball', + ref: commit || ref + } + const response = await octokit.repos.getArchiveLink(params) + if (response.status != 200) { + throw new Error( + `Unexpected response from GitHub API. Status: ${response.status}, Data: ${response.data}` + ) + } + + return Buffer.from(response.data) // response.data is ArrayBuffer +} diff --git a/src/main.ts b/src/main.ts index 26da3aa..4702fe0 100644 --- a/src/main.ts +++ b/src/main.ts @@ -3,8 +3,7 @@ import * as coreCommand from '@actions/core/lib/command' import * as gitSourceProvider from './git-source-provider' import * as inputHelper from './input-helper' import * as path from 'path' - -const cleanupRepositoryPath = process.env['STATE_repositoryPath'] as string +import * as stateHelper from './state-helper' async function run(): Promise { try { @@ -31,14 +30,14 @@ async function run(): Promise { async function cleanup(): Promise { try { - await gitSourceProvider.cleanup(cleanupRepositoryPath) + await gitSourceProvider.cleanup(stateHelper.RepositoryPath) } catch (error) { core.warning(error.message) } } // Main -if (!cleanupRepositoryPath) { +if (!stateHelper.IsPost) { run() } // Post diff --git a/src/retry-helper.ts b/src/retry-helper.ts new file mode 100644 index 0000000..bbc20a1 --- /dev/null +++ b/src/retry-helper.ts @@ -0,0 +1,61 @@ +import * as core from '@actions/core' + +const defaultMaxAttempts = 3 +const defaultMinSeconds = 10 +const defaultMaxSeconds = 20 + +export class RetryHelper { + private maxAttempts: number + private minSeconds: number + private maxSeconds: number + + constructor( + maxAttempts: number = defaultMaxAttempts, + minSeconds: number = defaultMinSeconds, + maxSeconds: number = defaultMaxSeconds + ) { + this.maxAttempts = maxAttempts + this.minSeconds = Math.floor(minSeconds) + this.maxSeconds = Math.floor(maxSeconds) + if (this.minSeconds > this.maxSeconds) { + throw new Error('min seconds should be less than or equal to max seconds') + } + } + + async execute(action: () => Promise): Promise { + let attempt = 1 + while (attempt < this.maxAttempts) { + // Try + try { + return await action() + } catch (err) { + core.info(err.message) + } + + // Sleep + const seconds = this.getSleepAmount() + core.info(`Waiting ${seconds} seconds before trying again`) + await this.sleep(seconds) + attempt++ + } + + // Last attempt + return await action() + } + + private getSleepAmount(): number { + return ( + Math.floor(Math.random() * (this.maxSeconds - this.minSeconds + 1)) + + this.minSeconds + ) + } + + private async sleep(seconds: number): Promise { + return new Promise(resolve => setTimeout(resolve, seconds * 1000)) + } +} + +export async function execute(action: () => Promise): Promise { + const retryHelper = new RetryHelper() + return await retryHelper.execute(action) +} diff --git a/src/state-helper.ts b/src/state-helper.ts new file mode 100644 index 0000000..0f86a14 --- /dev/null +++ b/src/state-helper.ts @@ -0,0 +1,30 @@ +import * as core from '@actions/core' +import * as coreCommand from '@actions/core/lib/command' + +/** + * Indicates whether the POST action is running + */ +export const IsPost = !!process.env['STATE_isPost'] + +/** + * The repository path for the POST action. The value is empty during the MAIN action. + */ +export const RepositoryPath = + (process.env['STATE_repositoryPath'] as string) || '' + +/** + * Save the repository path so the POST action can retrieve the value. + */ +export function setRepositoryPath(repositoryPath: string) { + coreCommand.issueCommand( + 'save-state', + {name: 'repositoryPath'}, + repositoryPath + ) +} + +// Publish a variable so that when the POST action runs, it can determine it should run the cleanup logic. +// This is necessary since we don't have a separate entry point. +if (!IsPost) { + coreCommand.issueCommand('save-state', {name: 'isPost'}, 'true') +} -- cgit v1.2.3