src: update timeout on `setupStickyDisk` (#91)
This commit is contained in:
parent
4fe825e8c9
commit
d8a061af73
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
@ -74,7 +74,7 @@ async function writeBuildkitdTomlFile(parallelism: number): Promise<void> {
|
||||||
// has been seen to negatively affect startup times of the daemon.
|
// has been seen to negatively affect startup times of the daemon.
|
||||||
gc: false,
|
gc: false,
|
||||||
'max-parallelism': parallelism,
|
'max-parallelism': parallelism,
|
||||||
snapshotter: 'overlayfs',
|
snapshotter: 'overlayfs'
|
||||||
},
|
},
|
||||||
containerd: {
|
containerd: {
|
||||||
enabled: false
|
enabled: false
|
||||||
|
@ -101,14 +101,7 @@ async function startBuildkitd(parallelism: number): Promise<string> {
|
||||||
const addr = 'unix:///run/buildkit/buildkitd.sock';
|
const addr = 'unix:///run/buildkit/buildkitd.sock';
|
||||||
|
|
||||||
const logStream = fs.createWriteStream('buildkitd.log');
|
const logStream = fs.createWriteStream('buildkitd.log');
|
||||||
const buildkitd = spawn('sudo', [
|
const buildkitd = spawn('sudo', ['buildkitd', '--debug', '--addr', addr, '--allow-insecure-entitlement', 'security.insecure', '--config=buildkitd.toml', '--allow-insecure-entitlement', 'network.host'], {
|
||||||
'buildkitd',
|
|
||||||
'--debug',
|
|
||||||
'--addr', addr,
|
|
||||||
'--allow-insecure-entitlement', 'security.insecure',
|
|
||||||
'--config=buildkitd.toml',
|
|
||||||
'--allow-insecure-entitlement', 'network.host'
|
|
||||||
], {
|
|
||||||
stdio: ['ignore', 'pipe', 'pipe']
|
stdio: ['ignore', 'pipe', 'pipe']
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -116,7 +109,7 @@ async function startBuildkitd(parallelism: number): Promise<string> {
|
||||||
buildkitd.stdout.pipe(logStream);
|
buildkitd.stdout.pipe(logStream);
|
||||||
buildkitd.stderr.pipe(logStream);
|
buildkitd.stderr.pipe(logStream);
|
||||||
|
|
||||||
buildkitd.on('error', (error) => {
|
buildkitd.on('error', error => {
|
||||||
throw new Error(`Failed to start buildkitd: ${error.message}`);
|
throw new Error(`Failed to start buildkitd: ${error.message}`);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -168,31 +161,38 @@ export async function getStickyDisk(options?: {signal?: AbortSignal}): Promise<{
|
||||||
}
|
}
|
||||||
core.debug(`Getting sticky disk for ${stickyDiskKey}`);
|
core.debug(`Getting sticky disk for ${stickyDiskKey}`);
|
||||||
|
|
||||||
|
const response = await client.getStickyDisk(
|
||||||
const response = await client.getStickyDisk({
|
{
|
||||||
stickyDiskKey: stickyDiskKey,
|
stickyDiskKey: stickyDiskKey,
|
||||||
region: process.env.BLACKSMITH_REGION || 'eu-central',
|
region: process.env.BLACKSMITH_REGION || 'eu-central',
|
||||||
installationModelId: process.env.BLACKSMITH_INSTALLATION_MODEL_ID || '',
|
installationModelId: process.env.BLACKSMITH_INSTALLATION_MODEL_ID || '',
|
||||||
vmId: process.env.VM_ID || '',
|
vmId: process.env.VM_ID || '',
|
||||||
stickyDiskType: 'dockerfile',
|
stickyDiskType: 'dockerfile',
|
||||||
repoName: process.env.GITHUB_REPO_NAME || '',
|
repoName: process.env.GITHUB_REPO_NAME || '',
|
||||||
stickyDiskToken: process.env.BLACKSMITH_STICKYDISK_TOKEN || ''
|
stickyDiskToken: process.env.BLACKSMITH_STICKYDISK_TOKEN || ''
|
||||||
}, {
|
},
|
||||||
signal: options?.signal
|
{
|
||||||
});
|
signal: options?.signal
|
||||||
|
}
|
||||||
|
);
|
||||||
return {
|
return {
|
||||||
expose_id: response.exposeId || '',
|
expose_id: response.exposeId || '',
|
||||||
device: response.diskIdentifier || ''
|
device: response.diskIdentifier || ''
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// buildkitdTimeoutMs states the max amount of time this action will wait for the buildkitd
|
||||||
|
// daemon to start have its socket ready. It also additionally governs how long we will wait for
|
||||||
|
// the buildkitd workers to be ready.
|
||||||
|
const buildkitdTimeoutMs = 15000;
|
||||||
|
|
||||||
export async function startAndConfigureBuildkitd(parallelism: number): Promise<string> {
|
export async function startAndConfigureBuildkitd(parallelism: number): Promise<string> {
|
||||||
const buildkitdAddr = await startBuildkitd(parallelism);
|
const buildkitdAddr = await startBuildkitd(parallelism);
|
||||||
core.debug(`buildkitd daemon started at addr ${buildkitdAddr}`);
|
core.debug(`buildkitd daemon started at addr ${buildkitdAddr}`);
|
||||||
|
|
||||||
// Change permissions on the buildkitd socket to allow non-root access
|
// Change permissions on the buildkitd socket to allow non-root access
|
||||||
const startTime = Date.now();
|
const startTime = Date.now();
|
||||||
const timeout = 45000; // 45 seconds in milliseconds
|
const timeout = buildkitdTimeoutMs;
|
||||||
|
|
||||||
while (Date.now() - startTime < timeout) {
|
while (Date.now() - startTime < timeout) {
|
||||||
if (fs.existsSync('/run/buildkit/buildkitd.sock')) {
|
if (fs.existsSync('/run/buildkit/buildkitd.sock')) {
|
||||||
|
@ -204,17 +204,18 @@ export async function startAndConfigureBuildkitd(parallelism: number): Promise<s
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!fs.existsSync('/run/buildkit/buildkitd.sock')) {
|
if (!fs.existsSync('/run/buildkit/buildkitd.sock')) {
|
||||||
throw new Error('buildkitd socket not found after 30s timeout');
|
throw new Error('buildkitd socket not found after 15s timeout');
|
||||||
}
|
}
|
||||||
// Check that buildkit instance is ready by querying workers for up to 30s
|
// Check that buildkit instance is ready by querying workers for up to 30s
|
||||||
const startTimeBuildkitReady = Date.now();
|
const startTimeBuildkitReady = Date.now();
|
||||||
const timeoutBuildkitReady = 30000; // 30 seconds
|
const timeoutBuildkitReady = buildkitdTimeoutMs;
|
||||||
|
|
||||||
while (Date.now() - startTimeBuildkitReady < timeoutBuildkitReady) {
|
while (Date.now() - startTimeBuildkitReady < timeoutBuildkitReady) {
|
||||||
try {
|
try {
|
||||||
const {stdout} = await execAsync('sudo buildctl debug workers');
|
const {stdout} = await execAsync('sudo buildctl debug workers');
|
||||||
const lines = stdout.trim().split('\n');
|
const lines = stdout.trim().split('\n');
|
||||||
if (lines.length > 1) { // Check if we have output lines beyond the header
|
if (lines.length > 1) {
|
||||||
|
// Check if we have output lines beyond the header
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
@ -228,7 +229,7 @@ export async function startAndConfigureBuildkitd(parallelism: number): Promise<s
|
||||||
const {stdout} = await execAsync('sudo buildctl debug workers');
|
const {stdout} = await execAsync('sudo buildctl debug workers');
|
||||||
const lines = stdout.trim().split('\n');
|
const lines = stdout.trim().split('\n');
|
||||||
if (lines.length <= 1) {
|
if (lines.length <= 1) {
|
||||||
throw new Error('buildkit workers not ready after 30s timeout');
|
throw new Error('buildkit workers not ready after 15s timeout');
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
core.warning(`Error checking buildkit workers: ${error.message}`);
|
core.warning(`Error checking buildkit workers: ${error.message}`);
|
||||||
|
@ -261,13 +262,17 @@ export async function pruneBuildkitCache(): Promise<void> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// stickyDiskTimeoutMs states the max amount of time this action will wait for the VM agent to
|
||||||
|
// expose the sticky disk from the storage agent, map it onto the host and then patch the drive
|
||||||
|
// into the VM.
|
||||||
|
const stickyDiskTimeoutMs = 45000;
|
||||||
|
|
||||||
// setupStickyDisk mounts a sticky disk for the entity and returns the device information.
|
// setupStickyDisk mounts a sticky disk for the entity and returns the device information.
|
||||||
// throws an error if it is unable to do so because of a timeout or an error
|
// throws an error if it is unable to do so because of a timeout or an error
|
||||||
export async function setupStickyDisk(dockerfilePath: string): Promise<{device: string; buildId?: string | null; exposeId: string}> {
|
export async function setupStickyDisk(dockerfilePath: string): Promise<{device: string; buildId?: string | null; exposeId: string}> {
|
||||||
try {
|
try {
|
||||||
const controller = new AbortController();
|
const controller = new AbortController();
|
||||||
const timeoutId = setTimeout(() => controller.abort(), 30000);
|
const timeoutId = setTimeout(() => controller.abort(), stickyDiskTimeoutMs);
|
||||||
|
|
||||||
let buildResponse: {docker_build_id: string} | null = null;
|
let buildResponse: {docker_build_id: string} | null = null;
|
||||||
let exposeId: string = '';
|
let exposeId: string = '';
|
||||||
|
@ -291,7 +296,8 @@ export async function setupStickyDisk(dockerfilePath: string): Promise<{device:
|
||||||
try {
|
try {
|
||||||
const {stdout} = await execAsync(`df -i ${mountPoint} | tail -1 | awk '{print $5}' | sed 's/%//'`);
|
const {stdout} = await execAsync(`df -i ${mountPoint} | tail -1 | awk '{print $5}' | sed 's/%//'`);
|
||||||
const inodePercentage = parseInt(stdout.trim());
|
const inodePercentage = parseInt(stdout.trim());
|
||||||
if (!isNaN(inodePercentage) && inodePercentage > 80) { // Report if over 80%
|
if (!isNaN(inodePercentage) && inodePercentage > 80) {
|
||||||
|
// Report if over 80%
|
||||||
await reporter.reportBuildPushActionFailure(new Error(`High inode usage (${inodePercentage}%) detected at ${mountPoint}`), 'setupStickyDisk', true /* isWarning */);
|
await reporter.reportBuildPushActionFailure(new Error(`High inode usage (${inodePercentage}%) detected at ${mountPoint}`), 'setupStickyDisk', true /* isWarning */);
|
||||||
core.warning(`High inode usage (${inodePercentage}%) detected at ${mountPoint}`);
|
core.warning(`High inode usage (${inodePercentage}%) detected at ${mountPoint}`);
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue