Compare commits

..

8 Commits

Author SHA1 Message Date
darshanime 4a9081b71c move to using sha uploads 2026-07-08 13:50:46 +05:30
darshanime 713b909428 init warpbuild caching 2026-07-08 09:40:20 +05:30
dependabot[bot] 4f1f4aec02
Bump actions/upload-artifact from 4 to 7 (#2476)
Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4 to 7.
- [Release notes](https://github.com/actions/upload-artifact/releases)
- [Commits](https://github.com/actions/upload-artifact/compare/v4...v7)

---
updated-dependencies:
- dependency-name: actions/upload-artifact
  dependency-version: '7'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-06-30 10:34:21 -04:00
dependabot[bot] ba097532fb
Bump actions/checkout from 6 to 7 (#2488)
Bumps [actions/checkout](https://github.com/actions/checkout) from 6 to 7.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/v6...v7)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-version: '7'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-06-30 10:33:37 -04:00
dependabot[bot] b9e0990d21
Bump docker/login-action from 3.3.0 to 4.2.0 (#2479)
Bumps [docker/login-action](https://github.com/docker/login-action) from 3.3.0 to 4.2.0.
- [Release notes](https://github.com/docker/login-action/releases)
- [Commits](https://github.com/docker/login-action/compare/v3.3.0...v4.2.0)

---
updated-dependencies:
- dependency-name: docker/login-action
  dependency-version: 4.2.0
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-06-22 10:13:43 -04:00
dependabot[bot] e8cb398be4
Bump docker/build-push-action from 6.5.0 to 7.2.0 (#2478)
Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6.5.0 to 7.2.0.
- [Release notes](https://github.com/docker/build-push-action/releases)
- [Commits](https://github.com/docker/build-push-action/compare/v6.5.0...v7.2.0)

---
updated-dependencies:
- dependency-name: docker/build-push-action
  dependency-version: 7.2.0
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-06-22 10:12:50 -04:00
dependabot[bot] 5de26ee9b1
Bump actions/setup-node from 4 to 6 (#2477)
Bumps [actions/setup-node](https://github.com/actions/setup-node) from 4 to 6.
- [Release notes](https://github.com/actions/setup-node/releases)
- [Commits](https://github.com/actions/setup-node/compare/v4...v6)

---
updated-dependencies:
- dependency-name: actions/setup-node
  dependency-version: '6'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-06-22 10:12:03 -04:00
dependabot[bot] 79102f2503
Bump github/codeql-action from 3 to 4 (#2475)
Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3 to 4.
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](https://github.com/github/codeql-action/compare/v3...v4)

---
updated-dependencies:
- dependency-name: github/codeql-action
  dependency-version: '4'
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-06-22 10:11:28 -04:00
12 changed files with 39 additions and 185 deletions

View File

@ -22,10 +22,10 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v7
- name: Set Node.js 24.x
uses: actions/setup-node@v4
uses: actions/setup-node@v6
with:
node-version: 24.x
@ -44,7 +44,7 @@ jobs:
fi
# If dist/ was different than expected, upload the expected version as an artifact
- uses: actions/upload-artifact@v4
- uses: actions/upload-artifact@v7
if: ${{ failure() && steps.diff.conclusion == 'failure' }}
with:
name: dist

View File

@ -39,10 +39,10 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@v7
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
uses: github/codeql-action/init@v4
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
@ -55,4 +55,4 @@ jobs:
- run: rm -rf dist # We want code scanning to analyze lib instead (individual .js files)
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v3
uses: github/codeql-action/analyze@v4

View File

@ -9,6 +9,6 @@ jobs:
runs-on: ubuntu-latest
name: Check licenses
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@v7
- run: npm ci
- run: npm run licensed-check

View File

@ -14,7 +14,7 @@ jobs:
steps:
- name: Checking out
uses: actions/checkout@v6
uses: actions/checkout@v7
- name: Publish
id: publish
uses: actions/publish-immutable-action@v0.0.4

View File

@ -16,10 +16,10 @@ jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/setup-node@v4
- uses: actions/setup-node@v6
with:
node-version: 24.x
- uses: actions/checkout@v6
- uses: actions/checkout@v7
- run: npm ci
- run: npm run build
- run: npm run format-check
@ -37,7 +37,7 @@ jobs:
steps:
# Clone this repo
- name: Checkout
uses: actions/checkout@v6
uses: actions/checkout@v7
# Basic checkout
- name: Checkout basic
@ -229,7 +229,7 @@ jobs:
steps:
# Clone this repo
- name: Checkout
uses: actions/checkout@v6
uses: actions/checkout@v7
# Basic checkout using git
- name: Checkout basic
@ -261,7 +261,7 @@ jobs:
steps:
# Clone this repo
- name: Checkout
uses: actions/checkout@v6
uses: actions/checkout@v7
# Basic checkout using git
- name: Checkout basic
@ -291,7 +291,7 @@ jobs:
steps:
# Clone this repo
- name: Checkout
uses: actions/checkout@v6
uses: actions/checkout@v7
with:
path: localClone
@ -319,7 +319,7 @@ jobs:
# needed to make checkout post cleanup succeed
- name: Fix Checkout v6
uses: actions/checkout@v6
uses: actions/checkout@v7
with:
path: localClone
@ -328,7 +328,7 @@ jobs:
steps:
# Clone this repo
- name: Checkout
uses: actions/checkout@v6
uses: actions/checkout@v7
with:
path: actions-checkout

View File

@ -23,7 +23,7 @@ jobs:
# Note this update workflow can also be used as a rollback tool.
# For that reason, it's best to pin `actions/checkout` to a known, stable version
# (typically, about two releases back).
- uses: actions/checkout@v6
- uses: actions/checkout@v7
with:
fetch-depth: 0
- name: Git config

View File

@ -26,12 +26,12 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@v7
# Use `docker/login-action` to log in to GHCR.io.
# Once published, the packages are scoped to the account defined here.
- name: Log in to the ghcr.io container registry
uses: docker/login-action@v3.3.0
uses: docker/login-action@v4.2.0
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
@ -48,7 +48,7 @@ jobs:
# Use `docker/build-push-action` to build (and optionally publish) the image.
- name: Build Docker Image (with optional Push)
uses: docker/build-push-action@v6.5.0
uses: docker/build-push-action@v7.2.0
with:
context: .
file: images/test-ubuntu-git.Dockerfile

View File

@ -1,14 +1,12 @@
# WarpBuild Checkout
This is [WarpBuild's](https://warpbuild.com) fork of `actions/checkout`, a drop-in
replacement that adds a **checkout snapshot cache**: on WarpBuild runners, the `.git`
objects a checkout produces are tarred and stored in S3, keyed by the exact commit SHA.
A later job checking out the same commit restores that snapshot and skips the fetch from
GitHub entirely — cutting request load (the main source of GitHub rate-limiting on
matrix builds and busy repos). SHA keys are immutable, so there is no TTL or refresh.
replacement that adds a **git mirror cache**: on WarpBuild runners, a tar of the repo's
bare mirror is restored from S3 into `.git/wb-mirror.git` and wired up via git
alternates, so the fetch from GitHub downloads only the delta instead of the whole
repository. On a cache miss, the run hydrates the mirror once (full clone + upload);
mirrors expire on a server-configured TTL and re-hydrate.
- Only the default checkout shape is cached (`fetch-depth: 1`, no tags/filter/sparse/LFS);
everything else runs exactly like upstream.
- Zero new inputs — behavior is identical to upstream everywhere except WarpBuild runners.
- Fail-open — any cache error degrades to stock `actions/checkout` behavior.
- All fork code lives in `src/warpbuild/`

View File

@ -6,13 +6,8 @@ import {
afterEach,
afterAll
} from '@jest/globals'
import * as exec from '@actions/exec'
import * as fs from 'fs'
import * as os from 'os'
import * as path from 'path'
import {
SKIP_NOT_WARPBUILD,
assertSafeTarMembers,
computeDestinationRef,
getMirrorCacheSkipReason
} from '../src/warpbuild/mirror-cache.js'
@ -136,69 +131,6 @@ describe('warpbuild snapshot cache', () => {
})
})
describe('assertSafeTarMembers', () => {
async function makeTar(
dir: string,
members: string[],
writeFiles = true
): Promise<string> {
if (writeFiles) {
for (const m of members) {
const abs = path.join(dir, m)
await fs.promises.mkdir(path.dirname(abs), {recursive: true})
await fs.promises.writeFile(abs, 'x')
}
}
const tar = path.join(dir, 'out.tar')
await exec.exec('tar', ['-cf', tar, '-C', dir, ...members])
return tar
}
it('accepts objects/ and shallow members', async () => {
const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-'))
try {
const tar = await makeTar(dir, ['objects/pack/p.pack', 'shallow'])
await expect(assertSafeTarMembers(tar)).resolves.toBeUndefined()
} finally {
await fs.promises.rm(dir, {recursive: true, force: true})
}
})
it('rejects members outside objects/ and shallow (e.g. hooks)', async () => {
const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-'))
try {
const tar = await makeTar(dir, ['objects/x', 'hooks/pre-commit'])
await expect(assertSafeTarMembers(tar)).rejects.toThrow(
/unexpected snapshot tar member/
)
} finally {
await fs.promises.rm(dir, {recursive: true, force: true})
}
})
it('rejects path traversal members', async () => {
const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'wb-tar-'))
try {
await fs.promises.mkdir(path.join(dir, 'sub'))
const tar = path.join(dir, 'evil.tar')
// -P preserves the ../ member instead of stripping it.
await fs.promises.writeFile(path.join(dir, 'evil'), 'x')
await exec.exec('tar', [
'-cPf',
tar,
'-C',
path.join(dir, 'sub'),
'../evil'
])
await expect(assertSafeTarMembers(tar)).rejects.toThrow(
/unexpected snapshot tar member/
)
} finally {
await fs.promises.rm(dir, {recursive: true, force: true})
}
})
})
describe('computeDestinationRef', () => {
it('maps the refs the fetch would have created', () => {
expect(computeDestinationRef('refs/heads/main')).toBe(
@ -240,11 +172,6 @@ describe('warpbuild snapshot cache', () => {
expect((await lookupSnapshot('123', SHA)).kind).toBe('miss')
})
it('maps 409 upload responses to locked (another job is uploading)', async () => {
stubFetch(409, {sub_code: 'FVE_GITMIRROR_LOCKED'})
expect((await requestUploadURL('123', SHA)).kind).toBe('locked')
})
it('maps 403 to disabled (backend kill switch)', async () => {
stubFetch(403, {sub_code: 'PDE_GITMIRROR_DISABLED'})
expect((await lookupSnapshot('123', SHA)).kind).toBe('disabled')

47
dist/index.js vendored
View File

@ -41699,8 +41699,7 @@ const promises_namespaceObject = __WEBPACK_EXTERNAL_createRequire(import.meta.ur
// Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner
// verification token. Contract: 200 = presigned URL; 404 = miss (upload after the
// stock fetch); 403 = unservable org (skip cache + upload); 409 = another job is
// uploading this snapshot (skip upload); else = fall back.
// stock fetch); 403 = unservable org (skip cache + upload); else = fall back.
const API_TIMEOUT_MS = 10_000;
function backend_api_baseUrl() {
return (process.env['WARPBUILD_HOST_URL'] || '').replace(/\/+$/, '');
@ -41754,10 +41753,6 @@ async function requestUploadURL(repoKey, sha) {
core_debug(`[wb-cache] upload-url answered 403 (disabled)`);
return { kind: 'disabled' };
}
if (res.status === 409) {
core_debug(`[wb-cache] upload-url answered 409 (locked)`);
return { kind: 'locked' };
}
core_debug(`[wb-cache] upload-url answered ${res.status}`);
return { kind: 'error' };
}
@ -41777,7 +41772,6 @@ async function requestUploadURL(repoKey, sha) {
// WarpBuild checkout snapshot cache: SHA-keyed tars of what the stock shallow fetch
// produces. Hit = restore + skip the fetch; miss = upload after checkout. Keys are
// immutable (no expiry). Fail-open: any error degrades to stock behavior.
@ -41796,6 +41790,9 @@ function getMirrorCacheSkipReason(settings) {
!process.env['WARPBUILD_HOST_URL']) {
return SKIP_NOT_WARPBUILD;
}
if (process.platform === 'win32') {
return 'Windows is not supported by the snapshot cache yet';
}
const repoKey = process.env['GITHUB_REPOSITORY_ID'] || '';
if (!repoKey) {
return 'GITHUB_REPOSITORY_ID is not set';
@ -41876,11 +41873,6 @@ async function setup(settings) {
async function setupInner(settings) {
const repoKey = process.env['GITHUB_REPOSITORY_ID'];
const sha = settings.commit;
// The restore/upload paths shell out to `tar`; without it, fall back cleanly.
if (!(await which('tar', false))) {
info('tar not found on PATH; using standard checkout');
return false;
}
const lookup = await lookupSnapshot(repoKey, sha);
if (lookup.kind === 'disabled') {
info('Snapshot cache is disabled by the backend for this organization');
@ -41918,28 +41910,6 @@ async function contribute(settings) {
endGroup();
}
}
// Refuse any tar member outside objects/ or shallow, absolute, or with a `..`
// component — the tar is remote and extracted into .git, so a crafted member could
// escape (e.g. hooks/, ../) and run code during checkout.
async function assertSafeTarMembers(tar) {
let listing = '';
await exec_exec('tar', ['-tf', tar], {
silent: true,
listeners: { stdout: (d) => (listing += d.toString()) }
});
for (const raw of listing.split('\n')) {
const member = raw.trim();
if (!member) {
continue;
}
const top = member.replace(/^\.\//, '').split('/')[0];
if (member.startsWith('/') ||
member.split('/').includes('..') ||
(top !== 'objects' && top !== 'shallow')) {
throw new Error(`unexpected snapshot tar member: ${member}`);
}
}
}
async function restoreSnapshot(settings, url, sha) {
const gitDir = external_path_namespaceObject.join(settings.repositoryPath, '.git');
const tmpTar = external_path_namespaceObject.join(external_os_namespaceObject.tmpdir(), `wb-snapshot-${process.pid}.tar`);
@ -41951,7 +41921,6 @@ async function restoreSnapshot(settings, url, sha) {
throw new Error(`snapshot download answered ${res.status}`);
}
await (0,promises_namespaceObject.pipeline)(external_stream_namespaceObject.Readable.fromWeb(res.body), external_fs_namespaceObject.createWriteStream(tmpTar));
await assertSafeTarMembers(tmpTar);
await exec_exec('tar', ['-xf', tmpTar, '-C', gitDir]);
const check = await exec_exec('git', ['-C', settings.repositoryPath, 'cat-file', '-e', `${sha}^{commit}`], { ignoreReturnCode: true });
if (check !== 0) {
@ -42005,11 +41974,9 @@ async function uploadSnapshot(settings) {
const size = (await external_fs_namespaceObject.promises.stat(tmpTar)).size;
const upload = await requestUploadURL(repoKey, sha);
if (upload.kind !== 'ok') {
info(upload.kind === 'locked'
? 'Another job is already uploading this snapshot; skipping'
: upload.kind === 'disabled'
? 'Snapshot cache is disabled by the backend; not uploading'
: 'Snapshot cache backend unavailable; not uploading');
info(upload.kind === 'disabled'
? 'Snapshot cache is disabled by the backend; not uploading'
: 'Snapshot cache backend unavailable; not uploading');
return;
}
const init = {

View File

@ -3,8 +3,7 @@ import * as core from '@actions/core'
// Client for backend-core's /api/v1/git-mirrors endpoints, authed by the runner
// verification token. Contract: 200 = presigned URL; 404 = miss (upload after the
// stock fetch); 403 = unservable org (skip cache + upload); 409 = another job is
// uploading this snapshot (skip upload); else = fall back.
// stock fetch); 403 = unservable org (skip cache + upload); else = fall back.
const API_TIMEOUT_MS = 10_000
@ -23,7 +22,6 @@ export type SnapshotLookup =
export type UploadURLResult =
| {kind: 'ok'; url: string}
| {kind: 'disabled'}
| {kind: 'locked'}
| {kind: 'error'}
function baseUrl(): string {
@ -91,10 +89,6 @@ export async function requestUploadURL(
core.debug(`[wb-cache] upload-url answered 403 (disabled)`)
return {kind: 'disabled'}
}
if (res.status === 409) {
core.debug(`[wb-cache] upload-url answered 409 (locked)`)
return {kind: 'locked'}
}
core.debug(`[wb-cache] upload-url answered ${res.status}`)
return {kind: 'error'}
} catch (error) {

View File

@ -1,7 +1,6 @@
/* eslint-disable i18n-text/no-en, import/no-unresolved -- upstream conventions; no TS import resolver configured */
import * as core from '@actions/core'
import * as exec from '@actions/exec'
import * as io from '@actions/io'
import * as fs from 'fs'
import * as os from 'os'
import * as path from 'path'
@ -39,6 +38,9 @@ export function getMirrorCacheSkipReason(
) {
return SKIP_NOT_WARPBUILD
}
if (process.platform === 'win32') {
return 'Windows is not supported by the snapshot cache yet'
}
const repoKey = process.env['GITHUB_REPOSITORY_ID'] || ''
if (!repoKey) {
return 'GITHUB_REPOSITORY_ID is not set'
@ -125,12 +127,6 @@ async function setupInner(settings: IGitSourceSettings): Promise<boolean> {
const repoKey = process.env['GITHUB_REPOSITORY_ID'] as string
const sha = settings.commit
// The restore/upload paths shell out to `tar`; without it, fall back cleanly.
if (!(await io.which('tar', false))) {
core.info('tar not found on PATH; using standard checkout')
return false
}
const lookup = await api.lookupSnapshot(repoKey, sha)
if (lookup.kind === 'disabled') {
@ -175,31 +171,6 @@ export async function contribute(settings: IGitSourceSettings): Promise<void> {
}
}
// Refuse any tar member outside objects/ or shallow, absolute, or with a `..`
// component — the tar is remote and extracted into .git, so a crafted member could
// escape (e.g. hooks/, ../) and run code during checkout.
export async function assertSafeTarMembers(tar: string): Promise<void> {
let listing = ''
await exec.exec('tar', ['-tf', tar], {
silent: true,
listeners: {stdout: (d: Buffer) => (listing += d.toString())}
})
for (const raw of listing.split('\n')) {
const member = raw.trim()
if (!member) {
continue
}
const top = member.replace(/^\.\//, '').split('/')[0]
if (
member.startsWith('/') ||
member.split('/').includes('..') ||
(top !== 'objects' && top !== 'shallow')
) {
throw new Error(`unexpected snapshot tar member: ${member}`)
}
}
}
async function restoreSnapshot(
settings: IGitSourceSettings,
url: string,
@ -218,7 +189,6 @@ async function restoreSnapshot(
Readable.fromWeb(res.body as import('stream/web').ReadableStream),
fs.createWriteStream(tmpTar)
)
await assertSafeTarMembers(tmpTar)
await exec.exec('tar', ['-xf', tmpTar, '-C', gitDir])
const check = await exec.exec(
@ -283,11 +253,9 @@ async function uploadSnapshot(settings: IGitSourceSettings): Promise<void> {
const upload = await api.requestUploadURL(repoKey, sha)
if (upload.kind !== 'ok') {
core.info(
upload.kind === 'locked'
? 'Another job is already uploading this snapshot; skipping'
: upload.kind === 'disabled'
? 'Snapshot cache is disabled by the backend; not uploading'
: 'Snapshot cache backend unavailable; not uploading'
upload.kind === 'disabled'
? 'Snapshot cache is disabled by the backend; not uploading'
: 'Snapshot cache backend unavailable; not uploading'
)
return
}