src: update timeout on `setupStickyDisk` (#91)

This commit is contained in:
Aayush Shah 2025-01-01 15:09:21 -05:00 committed by GitHub
parent 4fe825e8c9
commit d8a061af73
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 39 additions and 33 deletions

2
dist/index.js generated vendored

File diff suppressed because one or more lines are too long

2
dist/index.js.map generated vendored

File diff suppressed because one or more lines are too long

View File

@ -74,7 +74,7 @@ async function writeBuildkitdTomlFile(parallelism: number): Promise<void> {
// has been seen to negatively affect startup times of the daemon. // has been seen to negatively affect startup times of the daemon.
gc: false, gc: false,
'max-parallelism': parallelism, 'max-parallelism': parallelism,
snapshotter: 'overlayfs', snapshotter: 'overlayfs'
}, },
containerd: { containerd: {
enabled: false enabled: false
@ -101,14 +101,7 @@ async function startBuildkitd(parallelism: number): Promise<string> {
const addr = 'unix:///run/buildkit/buildkitd.sock'; const addr = 'unix:///run/buildkit/buildkitd.sock';
const logStream = fs.createWriteStream('buildkitd.log'); const logStream = fs.createWriteStream('buildkitd.log');
const buildkitd = spawn('sudo', [ const buildkitd = spawn('sudo', ['buildkitd', '--debug', '--addr', addr, '--allow-insecure-entitlement', 'security.insecure', '--config=buildkitd.toml', '--allow-insecure-entitlement', 'network.host'], {
'buildkitd',
'--debug',
'--addr', addr,
'--allow-insecure-entitlement', 'security.insecure',
'--config=buildkitd.toml',
'--allow-insecure-entitlement', 'network.host'
], {
stdio: ['ignore', 'pipe', 'pipe'] stdio: ['ignore', 'pipe', 'pipe']
}); });
@ -116,7 +109,7 @@ async function startBuildkitd(parallelism: number): Promise<string> {
buildkitd.stdout.pipe(logStream); buildkitd.stdout.pipe(logStream);
buildkitd.stderr.pipe(logStream); buildkitd.stderr.pipe(logStream);
buildkitd.on('error', (error) => { buildkitd.on('error', error => {
throw new Error(`Failed to start buildkitd: ${error.message}`); throw new Error(`Failed to start buildkitd: ${error.message}`);
}); });
@ -168,31 +161,38 @@ export async function getStickyDisk(options?: {signal?: AbortSignal}): Promise<{
} }
core.debug(`Getting sticky disk for ${stickyDiskKey}`); core.debug(`Getting sticky disk for ${stickyDiskKey}`);
const response = await client.getStickyDisk(
const response = await client.getStickyDisk({ {
stickyDiskKey: stickyDiskKey, stickyDiskKey: stickyDiskKey,
region: process.env.BLACKSMITH_REGION || 'eu-central', region: process.env.BLACKSMITH_REGION || 'eu-central',
installationModelId: process.env.BLACKSMITH_INSTALLATION_MODEL_ID || '', installationModelId: process.env.BLACKSMITH_INSTALLATION_MODEL_ID || '',
vmId: process.env.VM_ID || '', vmId: process.env.VM_ID || '',
stickyDiskType: 'dockerfile', stickyDiskType: 'dockerfile',
repoName: process.env.GITHUB_REPO_NAME || '', repoName: process.env.GITHUB_REPO_NAME || '',
stickyDiskToken: process.env.BLACKSMITH_STICKYDISK_TOKEN || '' stickyDiskToken: process.env.BLACKSMITH_STICKYDISK_TOKEN || ''
}, { },
signal: options?.signal {
}); signal: options?.signal
}
);
return { return {
expose_id: response.exposeId || '', expose_id: response.exposeId || '',
device: response.diskIdentifier || '' device: response.diskIdentifier || ''
}; };
} }
// buildkitdTimeoutMs states the max amount of time this action will wait for the buildkitd
// daemon to start have its socket ready. It also additionally governs how long we will wait for
// the buildkitd workers to be ready.
const buildkitdTimeoutMs = 15000;
export async function startAndConfigureBuildkitd(parallelism: number): Promise<string> { export async function startAndConfigureBuildkitd(parallelism: number): Promise<string> {
const buildkitdAddr = await startBuildkitd(parallelism); const buildkitdAddr = await startBuildkitd(parallelism);
core.debug(`buildkitd daemon started at addr ${buildkitdAddr}`); core.debug(`buildkitd daemon started at addr ${buildkitdAddr}`);
// Change permissions on the buildkitd socket to allow non-root access // Change permissions on the buildkitd socket to allow non-root access
const startTime = Date.now(); const startTime = Date.now();
const timeout = 45000; // 45 seconds in milliseconds const timeout = buildkitdTimeoutMs;
while (Date.now() - startTime < timeout) { while (Date.now() - startTime < timeout) {
if (fs.existsSync('/run/buildkit/buildkitd.sock')) { if (fs.existsSync('/run/buildkit/buildkitd.sock')) {
@ -204,17 +204,18 @@ export async function startAndConfigureBuildkitd(parallelism: number): Promise<s
} }
if (!fs.existsSync('/run/buildkit/buildkitd.sock')) { if (!fs.existsSync('/run/buildkit/buildkitd.sock')) {
throw new Error('buildkitd socket not found after 30s timeout'); throw new Error('buildkitd socket not found after 15s timeout');
} }
// Check that buildkit instance is ready by querying workers for up to 30s // Check that buildkit instance is ready by querying workers for up to 30s
const startTimeBuildkitReady = Date.now(); const startTimeBuildkitReady = Date.now();
const timeoutBuildkitReady = 30000; // 30 seconds const timeoutBuildkitReady = buildkitdTimeoutMs;
while (Date.now() - startTimeBuildkitReady < timeoutBuildkitReady) { while (Date.now() - startTimeBuildkitReady < timeoutBuildkitReady) {
try { try {
const {stdout} = await execAsync('sudo buildctl debug workers'); const {stdout} = await execAsync('sudo buildctl debug workers');
const lines = stdout.trim().split('\n'); const lines = stdout.trim().split('\n');
if (lines.length > 1) { // Check if we have output lines beyond the header if (lines.length > 1) {
// Check if we have output lines beyond the header
break; break;
} }
} catch (error) { } catch (error) {
@ -228,7 +229,7 @@ export async function startAndConfigureBuildkitd(parallelism: number): Promise<s
const {stdout} = await execAsync('sudo buildctl debug workers'); const {stdout} = await execAsync('sudo buildctl debug workers');
const lines = stdout.trim().split('\n'); const lines = stdout.trim().split('\n');
if (lines.length <= 1) { if (lines.length <= 1) {
throw new Error('buildkit workers not ready after 30s timeout'); throw new Error('buildkit workers not ready after 15s timeout');
} }
} catch (error) { } catch (error) {
core.warning(`Error checking buildkit workers: ${error.message}`); core.warning(`Error checking buildkit workers: ${error.message}`);
@ -261,13 +262,17 @@ export async function pruneBuildkitCache(): Promise<void> {
} }
} }
// stickyDiskTimeoutMs states the max amount of time this action will wait for the VM agent to
// expose the sticky disk from the storage agent, map it onto the host and then patch the drive
// into the VM.
const stickyDiskTimeoutMs = 45000;
// setupStickyDisk mounts a sticky disk for the entity and returns the device information. // setupStickyDisk mounts a sticky disk for the entity and returns the device information.
// throws an error if it is unable to do so because of a timeout or an error // throws an error if it is unable to do so because of a timeout or an error
export async function setupStickyDisk(dockerfilePath: string): Promise<{device: string; buildId?: string | null; exposeId: string}> { export async function setupStickyDisk(dockerfilePath: string): Promise<{device: string; buildId?: string | null; exposeId: string}> {
try { try {
const controller = new AbortController(); const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 30000); const timeoutId = setTimeout(() => controller.abort(), stickyDiskTimeoutMs);
let buildResponse: {docker_build_id: string} | null = null; let buildResponse: {docker_build_id: string} | null = null;
let exposeId: string = ''; let exposeId: string = '';
@ -291,7 +296,8 @@ export async function setupStickyDisk(dockerfilePath: string): Promise<{device:
try { try {
const {stdout} = await execAsync(`df -i ${mountPoint} | tail -1 | awk '{print $5}' | sed 's/%//'`); const {stdout} = await execAsync(`df -i ${mountPoint} | tail -1 | awk '{print $5}' | sed 's/%//'`);
const inodePercentage = parseInt(stdout.trim()); const inodePercentage = parseInt(stdout.trim());
if (!isNaN(inodePercentage) && inodePercentage > 80) { // Report if over 80% if (!isNaN(inodePercentage) && inodePercentage > 80) {
// Report if over 80%
await reporter.reportBuildPushActionFailure(new Error(`High inode usage (${inodePercentage}%) detected at ${mountPoint}`), 'setupStickyDisk', true /* isWarning */); await reporter.reportBuildPushActionFailure(new Error(`High inode usage (${inodePercentage}%) detected at ${mountPoint}`), 'setupStickyDisk', true /* isWarning */);
core.warning(`High inode usage (${inodePercentage}%) detected at ${mountPoint}`); core.warning(`High inode usage (${inodePercentage}%) detected at ${mountPoint}`);
} }