mirror of
https://github.com/RayLabsHQ/gitea-mirror.git
synced 2026-08-14 18:39:34 +08:00
Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 632bbd0d4a | |||
| 0b65e40784 | |||
| 4a8b4f6ff3 | |||
| 1d9dfdeb70 |
+1
-1
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "gitea-mirror",
|
||||
"type": "module",
|
||||
"version": "3.20.0",
|
||||
"version": "3.20.2",
|
||||
"engines": {
|
||||
"bun": ">=1.2.9"
|
||||
},
|
||||
|
||||
@@ -27,7 +27,10 @@
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from "bun:test";
|
||||
import { classifyReleasesForReconciliation } from "@/lib/gitea";
|
||||
import {
|
||||
classifyReleasesForReconciliation,
|
||||
classifyAssetsForReconciliation,
|
||||
} from "@/lib/gitea";
|
||||
|
||||
describe("classifyReleasesForReconciliation", () => {
|
||||
describe("normal repo — published_at order matches tag-commit order", () => {
|
||||
@@ -148,3 +151,82 @@ describe("classifyReleasesForReconciliation", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Asset reconciliation — regression for #331.
|
||||
*
|
||||
* Root cause: assets were uploaded only on the create path. When a Gitea release
|
||||
* already existed (every re-sync, or after an interrupted first upload), the update
|
||||
* path PATCHed the body and `continue`d without ever touching assets — so a release
|
||||
* that existed without its full asset set stayed permanently asset-less and re-syncing
|
||||
* could never heal it. Reproduced on a real Forgejo pull-mirror: GitHub release with
|
||||
* two 35-40MB binaries → Gitea release with 0 assets → re-sync logged "Updating
|
||||
* existing release" and left it at 0.
|
||||
*
|
||||
* Fix: reconcile assets idempotently on both paths via classifyAssetsForReconciliation.
|
||||
*/
|
||||
describe("classifyAssetsForReconciliation", () => {
|
||||
it("uploads all assets when the Gitea release has none (the #331 broken state)", () => {
|
||||
const github = [
|
||||
{ name: "base.zip", size: 40_264_954 },
|
||||
{ name: "extras.zip", size: 37_098_528 },
|
||||
];
|
||||
const gitea: Array<{ id: number; name: string; size: number }> = [];
|
||||
|
||||
const { toUpload, toSkip } = classifyAssetsForReconciliation(github, gitea);
|
||||
|
||||
expect(toSkip).toEqual([]);
|
||||
expect(toUpload).toEqual([
|
||||
{ name: "base.zip", replaceAssetId: null },
|
||||
{ name: "extras.zip", replaceAssetId: null },
|
||||
]);
|
||||
});
|
||||
|
||||
it("backfills only the missing asset when one already exists", () => {
|
||||
const github = [
|
||||
{ name: "base.zip", size: 40_264_954 },
|
||||
{ name: "extras.zip", size: 37_098_528 },
|
||||
];
|
||||
const gitea = [{ id: 9, name: "base.zip", size: 40_264_954 }];
|
||||
|
||||
const { toUpload, toSkip } = classifyAssetsForReconciliation(github, gitea);
|
||||
|
||||
expect(toSkip).toEqual(["base.zip"]);
|
||||
expect(toUpload).toEqual([{ name: "extras.zip", replaceAssetId: null }]);
|
||||
});
|
||||
|
||||
it("is idempotent — skips everything when all assets already match by name+size", () => {
|
||||
const github = [
|
||||
{ name: "base.zip", size: 40_264_954 },
|
||||
{ name: "extras.zip", size: 37_098_528 },
|
||||
];
|
||||
const gitea = [
|
||||
{ id: 9, name: "base.zip", size: 40_264_954 },
|
||||
{ id: 10, name: "extras.zip", size: 37_098_528 },
|
||||
];
|
||||
|
||||
const { toUpload, toSkip } = classifyAssetsForReconciliation(github, gitea);
|
||||
|
||||
expect(toUpload).toEqual([]);
|
||||
expect(toSkip).toEqual(["base.zip", "extras.zip"]);
|
||||
});
|
||||
|
||||
it("replaces an asset whose size changed upstream (re-upload over the stale copy)", () => {
|
||||
const github = [{ name: "firmware.bin", size: 2048 }];
|
||||
const gitea = [{ id: 42, name: "firmware.bin", size: 1024 }]; // truncated/stale
|
||||
|
||||
const { toUpload, toSkip } = classifyAssetsForReconciliation(github, gitea);
|
||||
|
||||
expect(toSkip).toEqual([]);
|
||||
expect(toUpload).toEqual([{ name: "firmware.bin", replaceAssetId: 42 }]);
|
||||
});
|
||||
|
||||
it("handles a release with no GitHub assets", () => {
|
||||
const { toUpload, toSkip } = classifyAssetsForReconciliation(
|
||||
[],
|
||||
[{ id: 1, name: "leftover.zip", size: 10 }]
|
||||
);
|
||||
expect(toUpload).toEqual([]);
|
||||
expect(toSkip).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
+242
-50
@@ -2691,6 +2691,168 @@ export function classifyReleasesForReconciliation(
|
||||
return { toCreate, toSkip };
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide which of a GitHub release's assets need (re)uploading to Gitea.
|
||||
*
|
||||
* Compared by name:
|
||||
* - present in Gitea with a matching size -> skip (already mirrored)
|
||||
* - present with a different size -> upload, replacing the stale copy
|
||||
* - absent -> upload (fresh)
|
||||
*
|
||||
* Pure function so the create/update reconciliation decision is unit-testable
|
||||
* without hitting the network (regression guard for #331).
|
||||
*/
|
||||
export function classifyAssetsForReconciliation(
|
||||
githubAssets: Array<{ name: string; size: number }>,
|
||||
giteaAssets: Array<{ id: number; name: string; size: number }>
|
||||
): {
|
||||
toUpload: Array<{ name: string; replaceAssetId: number | null }>;
|
||||
toSkip: string[];
|
||||
} {
|
||||
const existingByName = new Map(giteaAssets.map((a) => [a.name, a]));
|
||||
const toUpload: Array<{ name: string; replaceAssetId: number | null }> = [];
|
||||
const toSkip: string[] = [];
|
||||
|
||||
for (const asset of githubAssets) {
|
||||
const existing = existingByName.get(asset.name);
|
||||
if (existing && existing.size === asset.size) {
|
||||
toSkip.push(asset.name);
|
||||
} else {
|
||||
toUpload.push({ name: asset.name, replaceAssetId: existing ? existing.id : null });
|
||||
}
|
||||
}
|
||||
|
||||
return { toUpload, toSkip };
|
||||
}
|
||||
|
||||
/**
|
||||
* Idempotently mirror a GitHub release's assets onto the matching Gitea release.
|
||||
*
|
||||
* Runs on BOTH the create and update paths so assets are reconciled on every sync,
|
||||
* not only on the single sync where the Gitea release is first created. Previously
|
||||
* assets were uploaded inline in the create path only; the update path PATCHed the
|
||||
* body and `continue`d without ever looking at assets. Any release whose assets
|
||||
* failed or were interrupted on first creation therefore stayed permanently
|
||||
* asset-less, and re-syncing could never heal it (#331).
|
||||
*
|
||||
* Strategy: compare by asset name. Skip assets already present with a matching size;
|
||||
* (re)upload anything missing, and replace an existing asset whose size differs
|
||||
* (truncated/changed upstream). Returns per-release counts so the caller can report.
|
||||
*/
|
||||
async function reconcileReleaseAssets({
|
||||
config,
|
||||
decryptedConfig,
|
||||
repoOwner,
|
||||
repoName,
|
||||
giteaReleaseId,
|
||||
githubAssets,
|
||||
tagName,
|
||||
}: {
|
||||
config: Partial<Config>;
|
||||
decryptedConfig: Config;
|
||||
repoOwner: string;
|
||||
repoName: string;
|
||||
giteaReleaseId: number;
|
||||
githubAssets: Array<{ name: string; size: number; browser_download_url: string }>;
|
||||
tagName: string;
|
||||
}): Promise<{ uploaded: number; failed: number; skipped: number }> {
|
||||
let uploaded = 0;
|
||||
let failed = 0;
|
||||
let skipped = 0;
|
||||
|
||||
if (!githubAssets || githubAssets.length === 0) {
|
||||
return { uploaded, failed, skipped };
|
||||
}
|
||||
|
||||
const giteaBaseUrl = config.giteaConfig!.url;
|
||||
const giteaAuth = { Authorization: `token ${decryptedConfig.giteaConfig!.token}` };
|
||||
|
||||
// Fetch existing attachments so we only transfer what's missing or changed.
|
||||
const existingAssets: Array<{ id: number; name: string; size: number }> = await httpGet(
|
||||
`${giteaBaseUrl}/api/v1/repos/${repoOwner}/${repoName}/releases/${giteaReleaseId}/assets`,
|
||||
giteaAuth
|
||||
)
|
||||
.then((r) => (Array.isArray(r?.data) ? r.data : []))
|
||||
.catch(() => []);
|
||||
|
||||
const { toUpload, toSkip } = classifyAssetsForReconciliation(
|
||||
githubAssets,
|
||||
existingAssets
|
||||
);
|
||||
skipped = toSkip.length;
|
||||
|
||||
const githubByName = new Map(githubAssets.map((a) => [a.name, a]));
|
||||
|
||||
for (const { name, replaceAssetId } of toUpload) {
|
||||
const asset = githubByName.get(name)!;
|
||||
try {
|
||||
// Download from GitHub. fetch strips the Authorization header on the
|
||||
// cross-host redirect to GitHub's object storage, so this works for both
|
||||
// public and private release assets.
|
||||
console.log(
|
||||
`[Releases] Downloading asset: ${asset.name} (${asset.size} bytes) for ${tagName}`
|
||||
);
|
||||
const assetResponse = await fetch(asset.browser_download_url, {
|
||||
headers: {
|
||||
Accept: "application/octet-stream",
|
||||
Authorization: `token ${decryptedConfig.githubConfig!.token}`,
|
||||
},
|
||||
});
|
||||
|
||||
if (!assetResponse.ok) {
|
||||
console.error(
|
||||
`[Releases] Failed to download asset ${asset.name}: ${assetResponse.status} ${assetResponse.statusText}`
|
||||
);
|
||||
failed++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const assetData = await assetResponse.arrayBuffer();
|
||||
|
||||
// Gitea rejects a duplicate attachment name, so drop a stale/mismatched
|
||||
// copy before re-uploading.
|
||||
if (replaceAssetId !== null) {
|
||||
await httpDelete(
|
||||
`${giteaBaseUrl}/api/v1/repos/${repoOwner}/${repoName}/releases/${giteaReleaseId}/assets/${replaceAssetId}`,
|
||||
giteaAuth
|
||||
).catch(() => null);
|
||||
}
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append("attachment", new Blob([assetData]), asset.name);
|
||||
|
||||
const uploadResponse = await fetch(
|
||||
`${giteaBaseUrl}/api/v1/repos/${repoOwner}/${repoName}/releases/${giteaReleaseId}/assets?name=${encodeURIComponent(asset.name)}`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: { Authorization: `token ${decryptedConfig.giteaConfig!.token}` },
|
||||
body: formData,
|
||||
}
|
||||
);
|
||||
|
||||
if (uploadResponse.ok) {
|
||||
console.log(`[Releases] Successfully uploaded asset: ${asset.name}`);
|
||||
uploaded++;
|
||||
} else {
|
||||
const errorText = await uploadResponse.text();
|
||||
console.error(
|
||||
`[Releases] Failed to upload asset ${asset.name}: ${uploadResponse.status} ${errorText}`
|
||||
);
|
||||
failed++;
|
||||
}
|
||||
} catch (assetError) {
|
||||
console.error(
|
||||
`[Releases] Error processing asset ${asset.name}: ${
|
||||
assetError instanceof Error ? assetError.message : String(assetError)
|
||||
}`
|
||||
);
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
|
||||
return { uploaded, failed, skipped };
|
||||
}
|
||||
|
||||
export async function mirrorGitHubReleasesToGitea({
|
||||
octokit,
|
||||
repository,
|
||||
@@ -2776,6 +2938,9 @@ export async function mirrorGitHubReleasesToGitea({
|
||||
|
||||
let mirroredCount = 0;
|
||||
let skippedCount = 0;
|
||||
let skippedMissingTagCount = 0;
|
||||
let totalAssetsUploaded = 0;
|
||||
let totalAssetsFailed = 0;
|
||||
|
||||
// Process releases in their GitHub API order (newest first by default)
|
||||
const releasesToProcess = limitedReleases.slice();
|
||||
@@ -2832,7 +2997,8 @@ export async function mirrorGitHubReleasesToGitea({
|
||||
`${config.giteaConfig.url}/api/v1/repos/${repoOwner}/${repoName}/releases/${existingRelease.id}`,
|
||||
{
|
||||
tag_name: release.tag_name,
|
||||
target: release.target_commitish,
|
||||
// Omit `target` — the release already exists and is anchored to its tag;
|
||||
// re-sending target_commitish risks the same "target not found" 404 (#331).
|
||||
title: release.name || release.tag_name,
|
||||
body: releaseNote,
|
||||
draft: release.draft,
|
||||
@@ -2853,6 +3019,49 @@ export async function mirrorGitHubReleasesToGitea({
|
||||
console.log(`[Releases] Release ${release.tag_name} already up-to-date, skipping`);
|
||||
skippedCount++;
|
||||
}
|
||||
|
||||
// Reconcile assets on every sync — backfill any that are missing or changed.
|
||||
// The update path used to `continue` here without touching assets, so a
|
||||
// release that existed without its full asset set stayed broken forever (#331).
|
||||
const assetResult = await reconcileReleaseAssets({
|
||||
config,
|
||||
decryptedConfig,
|
||||
repoOwner,
|
||||
repoName,
|
||||
giteaReleaseId: existingRelease.id,
|
||||
githubAssets: release.assets || [],
|
||||
tagName: release.tag_name,
|
||||
});
|
||||
if (assetResult.uploaded > 0) {
|
||||
console.log(
|
||||
`[Releases] Backfilled ${assetResult.uploaded} missing/changed asset(s) for existing release ${release.tag_name}`
|
||||
);
|
||||
}
|
||||
totalAssetsUploaded += assetResult.uploaded;
|
||||
totalAssetsFailed += assetResult.failed;
|
||||
continue;
|
||||
}
|
||||
|
||||
// The git tag must already exist in Gitea before we create a release for it.
|
||||
// For a mirror, tags are synced from upstream by Gitea's own git mirror, which
|
||||
// can lag behind this metadata sync (e.g. a large/slow initial clone). If the
|
||||
// tag isn't present yet, skip and let a later sync pick it up — do NOT ask Gitea
|
||||
// to create the release against a `target` branch:
|
||||
// - if the target can't be resolved Gitea returns 404 "The target couldn't be
|
||||
// found" and the release is lost (#331),
|
||||
// - if it can, Gitea would create a brand-new tag at the wrong commit.
|
||||
const tagExists = await httpGet(
|
||||
`${config.giteaConfig.url}/api/v1/repos/${repoOwner}/${repoName}/tags/${encodeURIComponent(release.tag_name)}`,
|
||||
{ Authorization: `token ${decryptedConfig.giteaConfig.token}` }
|
||||
)
|
||||
.then(() => true)
|
||||
.catch(() => false);
|
||||
|
||||
if (!tagExists) {
|
||||
console.warn(
|
||||
`[Releases] Tag ${release.tag_name} is not present in Gitea yet — skipping release for now (the git mirror may still be syncing; it will be retried on the next sync)`
|
||||
);
|
||||
skippedMissingTagCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -2862,12 +3071,14 @@ export async function mirrorGitHubReleasesToGitea({
|
||||
} else {
|
||||
console.log(`[Releases] Creating release ${release.tag_name} with GitHub date header (no changelog)`);
|
||||
}
|
||||
|
||||
|
||||
const createReleaseResponse = await httpPost(
|
||||
`${config.giteaConfig.url}/api/v1/repos/${repoOwner}/${repoName}/releases`,
|
||||
{
|
||||
tag_name: release.tag_name,
|
||||
target: release.target_commitish,
|
||||
// Intentionally omit `target`: the tag already exists (verified above), so
|
||||
// Gitea attaches the release to it. Sending target_commitish can 404 with
|
||||
// "The target couldn't be found" on some Gitea/Forgejo versions (#331).
|
||||
title: release.name || release.tag_name,
|
||||
body: releaseNote,
|
||||
draft: release.draft,
|
||||
@@ -2878,55 +3089,22 @@ export async function mirrorGitHubReleasesToGitea({
|
||||
}
|
||||
);
|
||||
|
||||
// Mirror release assets if they exist
|
||||
// Mirror release assets if they exist (idempotent — see reconcileReleaseAssets)
|
||||
if (release.assets && release.assets.length > 0) {
|
||||
console.log(`[Releases] Mirroring ${release.assets.length} assets for release ${release.tag_name}`);
|
||||
|
||||
for (const asset of release.assets) {
|
||||
try {
|
||||
// Download the asset from GitHub
|
||||
console.log(`[Releases] Downloading asset: ${asset.name} (${asset.size} bytes)`);
|
||||
const assetResponse = await fetch(asset.browser_download_url, {
|
||||
headers: {
|
||||
'Accept': 'application/octet-stream',
|
||||
'Authorization': `token ${decryptedConfig.githubConfig.token}`,
|
||||
},
|
||||
});
|
||||
|
||||
if (!assetResponse.ok) {
|
||||
console.error(`[Releases] Failed to download asset ${asset.name}: ${assetResponse.statusText}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const assetData = await assetResponse.arrayBuffer();
|
||||
|
||||
// Upload the asset to Gitea release
|
||||
const formData = new FormData();
|
||||
formData.append('attachment', new Blob([assetData]), asset.name);
|
||||
|
||||
const uploadResponse = await fetch(
|
||||
`${config.giteaConfig.url}/api/v1/repos/${repoOwner}/${repoName}/releases/${createReleaseResponse.data.id}/assets?name=${encodeURIComponent(asset.name)}`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `token ${decryptedConfig.giteaConfig.token}`,
|
||||
},
|
||||
body: formData,
|
||||
}
|
||||
);
|
||||
|
||||
if (uploadResponse.ok) {
|
||||
console.log(`[Releases] Successfully uploaded asset: ${asset.name}`);
|
||||
} else {
|
||||
const errorText = await uploadResponse.text();
|
||||
console.error(`[Releases] Failed to upload asset ${asset.name}: ${errorText}`);
|
||||
}
|
||||
} catch (assetError) {
|
||||
console.error(`[Releases] Error processing asset ${asset.name}: ${assetError instanceof Error ? assetError.message : String(assetError)}`);
|
||||
}
|
||||
}
|
||||
const assetResult = await reconcileReleaseAssets({
|
||||
config,
|
||||
decryptedConfig,
|
||||
repoOwner,
|
||||
repoName,
|
||||
giteaReleaseId: createReleaseResponse.data.id,
|
||||
githubAssets: release.assets,
|
||||
tagName: release.tag_name,
|
||||
});
|
||||
totalAssetsUploaded += assetResult.uploaded;
|
||||
totalAssetsFailed += assetResult.failed;
|
||||
}
|
||||
|
||||
|
||||
mirroredCount++;
|
||||
const noteInfo = originalReleaseNote ? ` with ${originalReleaseNote.length} character changelog` : " without changelog";
|
||||
console.log(`[Releases] Successfully mirrored release: ${release.tag_name}${noteInfo}`);
|
||||
@@ -2935,7 +3113,21 @@ export async function mirrorGitHubReleasesToGitea({
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`✅ Mirrored/Updated ${mirroredCount} releases to Gitea (${skippedCount} already up-to-date)`);
|
||||
console.log(
|
||||
`✅ Mirrored/Updated ${mirroredCount} releases to Gitea (${skippedCount} already up-to-date, ${skippedMissingTagCount} skipped: tag not synced yet); assets uploaded: ${totalAssetsUploaded}, failed: ${totalAssetsFailed}`
|
||||
);
|
||||
|
||||
if (skippedMissingTagCount > 0) {
|
||||
console.warn(
|
||||
`[Releases] ${skippedMissingTagCount} release(s) skipped because their git tag is not in Gitea yet for ${repository.fullName} — these will be created automatically once the git mirror finishes syncing the tags`
|
||||
);
|
||||
}
|
||||
|
||||
if (totalAssetsFailed > 0) {
|
||||
console.error(
|
||||
`[Releases] ⚠️ ${totalAssetsFailed} release asset(s) failed to mirror for ${repository.fullName} — they will be retried on the next sync`
|
||||
);
|
||||
}
|
||||
|
||||
// Enforce release retention limit by removing the oldest excess releases from Gitea
|
||||
try {
|
||||
|
||||
Reference in New Issue
Block a user