Fix Install on Windows is very slow (#393)

* Fix Install on Windows is very slow

* Add unit test

* Improve readability

* Add e2e test

* fix lint

* Fix unit tests

* Fix unit tests

* limit to github hosted runners

* test hosted version of go

* AzDev environment

* rename lnkSrc

* refactor conditions

* improve tests

* refactoring

* Fix e2e test

* improve isHosted readability
This commit is contained in:
Sergey Dolin 2023-08-03 14:33:56 +02:00 committed by GitHub
parent 27eec5b982
commit 93397bea11
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 291 additions and 12 deletions

114
.github/workflows/windows-validation.yml vendored Normal file
View File

@ -0,0 +1,114 @@
name: Validate Windows installation
on:
push:
branches:
- main
paths-ignore:
- '**.md'
pull_request:
paths-ignore:
- '**.md'
jobs:
create-link-if-not-default:
runs-on: windows-latest
name: 'Validate if symlink is created'
strategy:
matrix:
cache: [false, true]
go: [1.20.1]
steps:
- uses: actions/checkout@v3
- name: 'Setup ${{ matrix.cache }}, cache: ${{ matrix.go }}'
uses: ./
with:
go-version: ${{ matrix.go }}
cache: ${{ matrix.cache }}
- name: 'Drive C: should have zero size link'
run: |
du -m -s 'C:\hostedtoolcache\windows\go\${{ matrix.go }}\x64'
# make sure drive c: contains only a link
size=$(du -m -s 'C:\hostedtoolcache\windows\go\${{ matrix.go }}\x64'|cut -f1 -d$'\t')
if [ $size -ne 0 ];then
echo 'Size of the link created on drive c: must be 0'
exit 1
fi
shell: bash
# Drive D: is small, take care the action does not eat up the space
- name: 'Drive D: space usage should be below 1G'
run: |
du -m -s 'D:\hostedtoolcache\windows\go\${{ matrix.go }}\x64'
size=$(du -m -s 'D:\hostedtoolcache\windows\go\${{ matrix.go }}\x64'|cut -f1 -d$'\t')
# make sure archive does not take lot of space
if [ $size -gt 999 ];then
echo 'Size of installed on drive d: go is too big';
exit 1
fi
shell: bash
# make sure the Go installation has not been changed to the end user
- name: Test paths and environments
run: |
echo $PATH
which go
go version
go env
if [ $(which go) != '/c/hostedtoolcache/windows/go/${{ matrix.go }}/x64/bin/go' ];then
echo 'which go should return "/c/hostedtoolcache/windows/go/${{ matrix.go }}/x64/bin/go"'
exit 1
fi
if [ $(go env GOROOT) != 'C:\hostedtoolcache\windows\go\${{ matrix.go }}\x64' ];then
echo 'go env GOROOT should return "C:\hostedtoolcache\windows\go\${{ matrix.go }}\x64"'
exit 1
fi
shell: bash
find-default-go:
name: 'Find default go version'
runs-on: windows-latest
outputs:
version: ${{ steps.goversion.outputs.version }}
steps:
- run: |
version=`go env GOVERSION|sed s/^go//`
echo "default go version: $version"
echo "version=$version" >> "$GITHUB_OUTPUT"
id: goversion
shell: bash
dont-create-link-if-default:
name: 'Validate if symlink is not created for default go'
runs-on: windows-latest
needs: find-default-go
strategy:
matrix:
cache: [false, true]
steps:
- uses: actions/checkout@v3
- name: 'Setup default go, cache: ${{ matrix.cache }}'
uses: ./
with:
go-version: ${{ needs.find-default-go.outputs.version }}
cache: ${{ matrix.cache }}
- name: 'Drive C: should have Go installation, cache: ${{ matrix.cache}}'
run: |
size=$(du -m -s 'C:\hostedtoolcache\windows\go\${{ needs.find-default-go.outputs.version }}\x64'|cut -f1 -d$'\t')
if [ $size -eq 0 ];then
echo 'Size of the hosted go installed on drive c: must be above zero'
exit 1
fi
shell: bash
- name: 'Drive D: should not have Go installation, cache: ${{ matrix.cache}}'
run: |
if [ -e 'D:\hostedtoolcache\windows\go\${{ needs.find-default-go.outputs.version }}\x64' ];then
echo 'D:\hostedtoolcache\windows\go\${{ needs.find-default-go.outputs.version }}\x64 should not exist for hosted version of go';
exit 1
fi
shell: bash

View File

@ -3,7 +3,7 @@ import * as io from '@actions/io';
import * as tc from '@actions/tool-cache'; import * as tc from '@actions/tool-cache';
import fs from 'fs'; import fs from 'fs';
import cp from 'child_process'; import cp from 'child_process';
import osm from 'os'; import osm, {type} from 'os';
import path from 'path'; import path from 'path';
import * as main from '../src/main'; import * as main from '../src/main';
import * as im from '../src/installer'; import * as im from '../src/installer';
@ -16,6 +16,8 @@ const matcherRegExp = new RegExp(matcherPattern.regexp);
const win32Join = path.win32.join; const win32Join = path.win32.join;
const posixJoin = path.posix.join; const posixJoin = path.posix.join;
jest.setTimeout(10000);
describe('setup-go', () => { describe('setup-go', () => {
let inputs = {} as any; let inputs = {} as any;
let os = {} as any; let os = {} as any;
@ -39,6 +41,8 @@ describe('setup-go', () => {
let existsSpy: jest.SpyInstance; let existsSpy: jest.SpyInstance;
let readFileSpy: jest.SpyInstance; let readFileSpy: jest.SpyInstance;
let mkdirpSpy: jest.SpyInstance; let mkdirpSpy: jest.SpyInstance;
let mkdirSpy: jest.SpyInstance;
let symlinkSpy: jest.SpyInstance;
let execSpy: jest.SpyInstance; let execSpy: jest.SpyInstance;
let getManifestSpy: jest.SpyInstance; let getManifestSpy: jest.SpyInstance;
let getAllVersionsSpy: jest.SpyInstance; let getAllVersionsSpy: jest.SpyInstance;
@ -92,6 +96,11 @@ describe('setup-go', () => {
readFileSpy = jest.spyOn(fs, 'readFileSync'); readFileSpy = jest.spyOn(fs, 'readFileSync');
mkdirpSpy = jest.spyOn(io, 'mkdirP'); mkdirpSpy = jest.spyOn(io, 'mkdirP');
// fs
mkdirSpy = jest.spyOn(fs, 'mkdir');
symlinkSpy = jest.spyOn(fs, 'symlinkSync');
symlinkSpy.mockImplementation(() => {});
// gets // gets
getManifestSpy.mockImplementation(() => <tc.IToolRelease[]>goTestManifest); getManifestSpy.mockImplementation(() => <tc.IToolRelease[]>goTestManifest);

View File

@ -0,0 +1,62 @@
import fs from 'fs';
import * as io from '@actions/io';
import * as tc from '@actions/tool-cache';
import path from 'path';
describe('Windows performance workaround', () => {
let mkdirSpy: jest.SpyInstance;
let symlinkSpy: jest.SpyInstance;
let statSpy: jest.SpyInstance;
let readdirSpy: jest.SpyInstance;
let writeFileSpy: jest.SpyInstance;
let rmRFSpy: jest.SpyInstance;
let mkdirPSpy: jest.SpyInstance;
let cpSpy: jest.SpyInstance;
let runnerToolCache: string | undefined;
beforeEach(() => {
mkdirSpy = jest.spyOn(fs, 'mkdir');
symlinkSpy = jest.spyOn(fs, 'symlinkSync');
statSpy = jest.spyOn(fs, 'statSync');
readdirSpy = jest.spyOn(fs, 'readdirSync');
writeFileSpy = jest.spyOn(fs, 'writeFileSync');
rmRFSpy = jest.spyOn(io, 'rmRF');
mkdirPSpy = jest.spyOn(io, 'mkdirP');
cpSpy = jest.spyOn(io, 'cp');
// default implementations
// @ts-ignore - not implement unused methods
statSpy.mockImplementation(() => ({
isDirectory: () => true
}));
readdirSpy.mockImplementation(() => []);
writeFileSpy.mockImplementation(() => {});
mkdirSpy.mockImplementation(() => {});
symlinkSpy.mockImplementation(() => {});
rmRFSpy.mockImplementation(() => Promise.resolve());
mkdirPSpy.mockImplementation(() => Promise.resolve());
cpSpy.mockImplementation(() => Promise.resolve());
runnerToolCache = process.env['RUNNER_TOOL_CACHE'];
});
afterEach(() => {
jest.clearAllMocks();
process.env['RUNNER_TOOL_CACHE'] = runnerToolCache;
});
// cacheWindowsToolkitDir depends on implementation of tc.cacheDir
// with the assumption that target dir is passed by RUNNER_TOOL_CACHE environment variable
// Make sure the implementation has not been changed
it('addExecutablesToCache should depend on env[RUNNER_TOOL_CACHE]', async () => {
process.env['RUNNER_TOOL_CACHE'] = '/faked-hostedtoolcache1';
const cacheDir1 = await tc.cacheDir('/qzx', 'go', '1.2.3', 'arch');
expect(cacheDir1).toBe(
path.join('/', 'faked-hostedtoolcache1', 'go', '1.2.3', 'arch')
);
process.env['RUNNER_TOOL_CACHE'] = '/faked-hostedtoolcache2';
const cacheDir2 = await tc.cacheDir('/qzx', 'go', '1.2.3', 'arch');
expect(cacheDir2).toBe(
path.join('/', 'faked-hostedtoolcache2', 'go', '1.2.3', 'arch')
);
});
});

46
dist/setup/index.js vendored
View File

@ -61488,6 +61488,46 @@ function resolveVersionFromManifest(versionSpec, stable, auth, arch, manifest) {
} }
}); });
} }
// for github hosted windows runner handle latency of OS drive
// by avoiding write operations to C:
function cacheWindowsDir(extPath, tool, version, arch) {
return __awaiter(this, void 0, void 0, function* () {
if (os_1.default.platform() !== 'win32')
return false;
// make sure the action runs in the hosted environment
if (process.env['RUNNER_ENVIRONMENT'] !== 'github-hosted' &&
process.env['AGENT_ISSELFHOSTED'] === '1')
return false;
const defaultToolCacheRoot = process.env['RUNNER_TOOL_CACHE'];
if (!defaultToolCacheRoot)
return false;
if (!fs_1.default.existsSync('d:\\') || !fs_1.default.existsSync('c:\\'))
return false;
const actualToolCacheRoot = defaultToolCacheRoot
.replace('C:', 'D:')
.replace('c:', 'd:');
// make toolcache root to be on drive d:
process.env['RUNNER_TOOL_CACHE'] = actualToolCacheRoot;
const actualToolCacheDir = yield tc.cacheDir(extPath, tool, version, arch);
// create a link from c: to d:
const defaultToolCacheDir = actualToolCacheDir.replace(actualToolCacheRoot, defaultToolCacheRoot);
fs_1.default.mkdirSync(path.dirname(defaultToolCacheDir), { recursive: true });
fs_1.default.symlinkSync(actualToolCacheDir, defaultToolCacheDir, 'junction');
core.info(`Created link ${defaultToolCacheDir} => ${actualToolCacheDir}`);
// make outer code to continue using toolcache as if it were installed on c:
// restore toolcache root to default drive c:
process.env['RUNNER_TOOL_CACHE'] = defaultToolCacheRoot;
return defaultToolCacheDir;
});
}
function addExecutablesToToolCache(extPath, info, arch) {
return __awaiter(this, void 0, void 0, function* () {
const tool = 'go';
const version = makeSemver(info.resolvedVersion);
return ((yield cacheWindowsDir(extPath, tool, version, arch)) ||
(yield tc.cacheDir(extPath, tool, version, arch)));
});
}
function installGoVersion(info, auth, arch) { function installGoVersion(info, auth, arch) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
core.info(`Acquiring ${info.resolvedVersion} from ${info.downloadUrl}`); core.info(`Acquiring ${info.resolvedVersion} from ${info.downloadUrl}`);
@ -61503,9 +61543,9 @@ function installGoVersion(info, auth, arch) {
extPath = path.join(extPath, 'go'); extPath = path.join(extPath, 'go');
} }
core.info('Adding to the cache ...'); core.info('Adding to the cache ...');
const cachedDir = yield tc.cacheDir(extPath, 'go', makeSemver(info.resolvedVersion), arch); const toolCacheDir = yield addExecutablesToToolCache(extPath, info, arch);
core.info(`Successfully cached go to ${cachedDir}`); core.info(`Successfully cached go to ${toolCacheDir}`);
return cachedDir; return toolCacheDir;
}); });
} }
function extractGoArchive(archivePath) { function extractGoArchive(archivePath) {

View File

@ -164,6 +164,64 @@ async function resolveVersionFromManifest(
} }
} }
// for github hosted windows runner handle latency of OS drive
// by avoiding write operations to C:
async function cacheWindowsDir(
extPath: string,
tool: string,
version: string,
arch: string
): Promise<string | false> {
if (os.platform() !== 'win32') return false;
// make sure the action runs in the hosted environment
if (
process.env['RUNNER_ENVIRONMENT'] !== 'github-hosted' &&
process.env['AGENT_ISSELFHOSTED'] === '1'
)
return false;
const defaultToolCacheRoot = process.env['RUNNER_TOOL_CACHE'];
if (!defaultToolCacheRoot) return false;
if (!fs.existsSync('d:\\') || !fs.existsSync('c:\\')) return false;
const actualToolCacheRoot = defaultToolCacheRoot
.replace('C:', 'D:')
.replace('c:', 'd:');
// make toolcache root to be on drive d:
process.env['RUNNER_TOOL_CACHE'] = actualToolCacheRoot;
const actualToolCacheDir = await tc.cacheDir(extPath, tool, version, arch);
// create a link from c: to d:
const defaultToolCacheDir = actualToolCacheDir.replace(
actualToolCacheRoot,
defaultToolCacheRoot
);
fs.mkdirSync(path.dirname(defaultToolCacheDir), {recursive: true});
fs.symlinkSync(actualToolCacheDir, defaultToolCacheDir, 'junction');
core.info(`Created link ${defaultToolCacheDir} => ${actualToolCacheDir}`);
// make outer code to continue using toolcache as if it were installed on c:
// restore toolcache root to default drive c:
process.env['RUNNER_TOOL_CACHE'] = defaultToolCacheRoot;
return defaultToolCacheDir;
}
async function addExecutablesToToolCache(
extPath: string,
info: IGoVersionInfo,
arch: string
): Promise<string> {
const tool = 'go';
const version = makeSemver(info.resolvedVersion);
return (
(await cacheWindowsDir(extPath, tool, version, arch)) ||
(await tc.cacheDir(extPath, tool, version, arch))
);
}
async function installGoVersion( async function installGoVersion(
info: IGoVersionInfo, info: IGoVersionInfo,
auth: string | undefined, auth: string | undefined,
@ -186,14 +244,10 @@ async function installGoVersion(
} }
core.info('Adding to the cache ...'); core.info('Adding to the cache ...');
const cachedDir = await tc.cacheDir( const toolCacheDir = await addExecutablesToToolCache(extPath, info, arch);
extPath, core.info(`Successfully cached go to ${toolCacheDir}`);
'go',
makeSemver(info.resolvedVersion), return toolCacheDir;
arch
);
core.info(`Successfully cached go to ${cachedDir}`);
return cachedDir;
} }
export async function extractGoArchive(archivePath: string): Promise<string> { export async function extractGoArchive(archivePath: string): Promise<string> {