first commit

This commit is contained in:
KodekaTeam committed 2026-06-02 00:17:50 +07:00
1 parent abb09fbea8
commit 72f6d49818
355 files changed
+121239 -3

No files matched your search

+23
View File
@@ -0,0 +1,23 @@
# App Catalog Modules
Folder ini berisi modularisasi business logic untuk `app-catalog.service.ts`.
Struktur:
- `types.ts`: kontrak publik dan type cache.
- `catalog-options.ts`: opsi sumber katalog yang diexpose ke route.
- `cache.ts`: cache remote catalog dan helper result cache.
- `utils.ts`: helper generik untuk parsing value dan structured text.
- `normalize.ts`: normalisasi field template generik.
- `manifest-parsers.ts`: parser manifest CasaOS, Umbrel, dan generic manifest.
- `remote.ts`: normalisasi URL remote, deteksi ZIP response, dan fetch wrapper.
- `archive.ts`: scan archive ZIP dan ekstraksi katalog.
- `service.ts`: orchestration utama `getCatalogTemplates()`.
- `index.ts`: public barrel untuk API modul ini.
Pattern:
- Helper generik disimpan di file kecil sesuai concern.
- `service.ts` hanya mengorkestrasi alur, tidak memuat detail parsing besar.
- Hanya `index.ts` yang menjadi surface API publik modul baru.
- `../app-catalog.service.ts` dipertahankan sebagai compatibility facade untuk callsite lama.
+214
View File
@@ -0,0 +1,214 @@
import AdmZip from "adm-zip";
import { AppTemplate } from "../../config/app-templates";
import { CatalogSourceMode, CatalogSyncResult } from "./types";
import { dedupeTemplates, normalizePorts } from "./normalize";
import {
normalizeSingleManifest,
parseCasaOsCompose,
parseCasaOsConfigJson,
} from "./manifest-parsers";
import { asRecord, parseStructuredText, slugify, toStringArray } from "./utils";
function getAppDirKey(entryName: string): string | undefined {
const match = entryName.match(/(^|\/)Apps\/([^/]+)\//i);
return match?.[2]?.toLowerCase();
}
function getGithubRawArchiveBase(
archiveUrl: string,
): { owner: string; repo: string; ref: string } | undefined {
try {
const parsed = new URL(archiveUrl);
if (parsed.hostname === "codeload.github.com") {
const match = parsed.pathname.match(
/^\/([^/]+)\/([^/]+)\/zip\/refs\/(?:heads|tags)\/(.+)$/i,
);
if (match) {
return { owner: match[1], repo: match[2], ref: match[3] };
}
}
if (parsed.hostname === "github.com") {
const refMatch = parsed.pathname.match(
/^\/([^/]+)\/([^/]+)\/archive\/refs\/(?:heads|tags)\/(.+)\.zip$/i,
);
if (refMatch) {
return { owner: refMatch[1], repo: refMatch[2], ref: refMatch[3] };
}
const commitMatch = parsed.pathname.match(
/^\/([^/]+)\/([^/]+)\/archive\/([a-f0-9]{7,40})\.zip$/i,
);
if (commitMatch) {
return {
owner: commitMatch[1],
repo: commitMatch[2],
ref: commitMatch[3],
};
}
}
} catch {
return undefined;
}
return undefined;
}
function getArchiveEntryBaseUrl(
archiveUrl: string,
entryName: string,
): string | undefined {
const github = getGithubRawArchiveBase(archiveUrl);
if (!github) return undefined;
const entryPath = entryName.split("/").filter(Boolean).slice(1).join("/");
if (!entryPath) return undefined;
return `https://raw.githubusercontent.com/${github.owner}/${github.repo}/${github.ref}/${entryPath}`;
}
export function parseCasaOsArchiveCatalog(
zip: AdmZip,
archiveUrl = "",
): AppTemplate[] {
const configEntries = new Map<string, string>();
const composeEntries = new Map<string, string>();
const results: AppTemplate[] = [];
zip.getEntries().forEach((entry) => {
if (entry.isDirectory) return;
const appDirKey = getAppDirKey(entry.entryName);
if (!appDirKey) return;
const lower = entry.entryName.toLowerCase();
if (lower.endsWith("/config.json")) {
configEntries.set(appDirKey, entry.entryName);
}
if (
lower.endsWith("/docker-compose.yml") ||
lower.endsWith("/docker-compose.yaml")
) {
composeEntries.set(appDirKey, entry.entryName);
}
});
Array.from(
new Set([...configEntries.keys(), ...composeEntries.keys()]),
).forEach((appDirKey) => {
const configPath = configEntries.get(appDirKey);
const composePath = composeEntries.get(appDirKey);
try {
const configPayload = configPath
? JSON.parse(zip.readAsText(configPath))
: undefined;
const composePayload = composePath
? parseStructuredText(zip.readAsText(composePath))
: undefined;
let template: AppTemplate | null = null;
if (composePayload) {
template = parseCasaOsCompose(
asRecord(composePayload) ?? {},
appDirKey,
composePath
? getArchiveEntryBaseUrl(archiveUrl, composePath)
: undefined,
);
}
if (!template && configPayload) {
template = parseCasaOsConfigJson(
asRecord(configPayload) ?? {},
appDirKey,
configPath
? getArchiveEntryBaseUrl(archiveUrl, configPath)
: undefined,
);
}
if (template) {
if (configPayload && !template.defaultPort) {
template.defaultPort = normalizePorts(asRecord(configPayload)?.port);
}
if (configPayload && template.tags.length === 0) {
template.tags = toStringArray(asRecord(configPayload)?.categories);
}
results.push(template);
}
} catch {
// Ignore malformed app folder data.
}
});
return results;
}
export function parseGenericArchiveCatalog(
zip: AdmZip,
archiveUrl = "",
): AppTemplate[] {
const templates: AppTemplate[] = [];
const candidateEntries = zip
.getEntries()
.filter((entry) => {
if (entry.isDirectory) return false;
const lower = entry.entryName.toLowerCase();
if (!/\.(json|ya?ml)$/.test(lower)) return false;
return /(app|catalog|manifest|template|store|config|compose|umbrel)/.test(
lower,
);
})
.slice(0, 300);
candidateEntries.forEach((entry) => {
try {
const payload = parseStructuredText(entry.getData().toString("utf8"));
const fallbackId = slugify(
entry.entryName.split("/").slice(-2, -1)[0] ?? entry.name,
);
templates.push(
...normalizeSingleManifest(
payload,
fallbackId,
getArchiveEntryBaseUrl(archiveUrl, entry.entryName),
),
);
} catch {
// Ignore malformed files.
}
});
return templates;
}
export async function parseArchiveResponse(
response: Response,
source: CatalogSourceMode,
label: string,
url: string,
): Promise<CatalogSyncResult> {
const buffer = Buffer.from(await response.arrayBuffer());
const zip = new AdmZip(buffer);
const casaOsTemplates = parseCasaOsArchiveCatalog(zip, url);
const genericTemplates = parseGenericArchiveCatalog(zip, url);
return {
templates: dedupeTemplates([...casaOsTemplates, ...genericTemplates]),
meta: {
source,
label,
url,
fetchedAt: new Date().toISOString(),
isRemote: true,
format: casaOsTemplates.length > 0 ? "casaos-archive" : "archive",
},
};
}
+53
View File
@@ -0,0 +1,53 @@
import { AppTemplate } from "../../config/app-templates";
import { CacheEntry, CatalogMeta, CatalogSyncResult } from "./types";
const REMOTE_CATALOG_CACHE = new Map<string, CacheEntry>();
const CACHE_TTL_MS = 5 * 60 * 1000;
export function getCacheKey(source: string, url?: string): string {
return `${source}:${(url ?? "").trim()}`;
}
export function getCachedCatalog(key: string): CacheEntry | undefined {
const entry = REMOTE_CATALOG_CACHE.get(key);
if (!entry) return undefined;
if (entry.expiresAt <= Date.now()) {
REMOTE_CATALOG_CACHE.delete(key);
return undefined;
}
return entry;
}
export function setCachedCatalog(
key: string,
templates: AppTemplate[],
meta: CatalogMeta,
): CatalogSyncResult {
const expiresAt = Date.now() + CACHE_TTL_MS;
const cacheMeta: CatalogMeta = {
...meta,
cached: false,
expiresAt: new Date(expiresAt).toISOString(),
};
REMOTE_CATALOG_CACHE.set(key, {
templates,
meta: cacheMeta,
expiresAt,
});
return { templates, meta: cacheMeta };
}
export function makeCachedResult(entry: CacheEntry): CatalogSyncResult {
return {
templates: entry.templates,
meta: {
...entry.meta,
cached: true,
expiresAt: new Date(entry.expiresAt).toISOString(),
},
};
}
@@ -0,0 +1,41 @@
import { CatalogSourceOption } from "./types";
export const CATALOG_SOURCE_OPTIONS: CatalogSourceOption[] = [
{
id: "local",
label: "Built-in Catalog",
description: "Use the local catalog bundled with this backend.",
requiresUrl: false,
},
{
id: "auto-detect",
label: "Custom Source",
description:
"Paste a remote URL and let the backend detect whether it is a manifest, compose/config file, or ZIP app store archive.",
requiresUrl: true,
placeholder:
"https://github.com/bigbeartechworld/big-bear-casaos/archive/refs/heads/master.zip",
exampleUrl: "https://casaos-appstore.paodayag.dev/linuxserver.zip",
},
{
id: "manifest-url",
label: "Manifest URL",
description:
"Fetch app entries from a remote JSON, YAML, CasaOS config, compose, or Umbrel manifest URL.",
requiresUrl: true,
placeholder: "https://example.com/apps.json",
exampleUrl:
"https://raw.githubusercontent.com/bigbeartechworld/big-bear-casaos/master/Apps/2fauth/config.json",
},
{
id: "github-archive",
label: "GitHub Archive ZIP",
description:
"Download a GitHub archive ZIP and scan CasaOS app folders, config.json, docker-compose.yml, JSON, or YAML manifests.",
requiresUrl: true,
placeholder:
"https://github.com/bigbeartechworld/big-bear-casaos/archive/refs/heads/master.zip",
exampleUrl:
"https://github.com/bigbeartechworld/big-bear-casaos/archive/refs/heads/master.zip",
},
];
+3
View File
@@ -0,0 +1,3 @@
export * from "./types";
export * from "./catalog-options";
export * from "./service";
@@ -0,0 +1,207 @@
import { AppTemplate } from "../../config/app-templates";
import {
asNumber,
asRecord,
asString,
buildImageWithVersion,
slugify,
toStringArray,
} from "./utils";
import {
extractCandidateEntries,
normalizeEntry,
normalizeEnv,
normalizeIcon,
normalizePorts,
normalizeVolumes,
} from "./normalize";
function getMainService(
composeRecord: Record<string, unknown>,
): Record<string, unknown> | undefined {
const services = asRecord(composeRecord.services);
if (!services) return undefined;
const casaOs = asRecord(composeRecord["x-casaos"]);
const mainName = asString(casaOs?.main);
if (mainName) {
const service = asRecord(services[mainName]);
if (service) return service;
}
const firstKey = Object.keys(services)[0];
return firstKey ? asRecord(services[firstKey]) : undefined;
}
function isGenericAppTemplateRecord(record: Record<string, unknown>): boolean {
return [
"desc",
"category",
"tags",
"defaultPort",
"defaultEnv",
"defaultVolumes",
"defaultCommand",
"defaultNetwork",
"restartPolicy",
"presets",
"presentation",
].some((key) => key in record);
}
export function parseCasaOsCompose(
composeRecord: Record<string, unknown>,
fallbackId: string,
baseUrl?: string,
): AppTemplate | null {
const casaOs = asRecord(composeRecord["x-casaos"]);
const mainService = getMainService(composeRecord);
const titleBlock = asRecord(casaOs?.title);
const descBlock = asRecord(casaOs?.description);
const taglineBlock = asRecord(casaOs?.tagline);
const name =
asString(titleBlock?.en_us) ??
asString(casaOs?.title) ??
asString(casaOs?.name) ??
fallbackId;
const description =
asString(descBlock?.en_us) ??
asString(taglineBlock?.en_us) ??
asString(casaOs?.description) ??
"Community app compose manifest";
const image =
asString(mainService?.image) ??
buildImageWithVersion(asString(casaOs?.image), asString(casaOs?.version));
if (!image) return null;
return {
id: asString(casaOs?.id) ?? slugify(fallbackId),
name,
desc: description,
category: asString(casaOs?.category) ?? "Community",
icon: normalizeIcon({ icon: casaOs?.icon, logo: casaOs?.logo }, baseUrl),
image,
defaultPort: normalizePorts(casaOs?.port_map ?? mainService?.ports),
defaultEnv: normalizeEnv(mainService?.environment),
defaultVolumes: normalizeVolumes(mainService?.volumes) || undefined,
defaultCommand: asString(mainService?.command),
defaultNetwork:
asString(mainService?.network_mode) ?? asString(casaOs?.network),
restartPolicy:
asString(mainService?.restart) ??
asString(casaOs?.restart_policy) ??
"unless-stopped",
popular: false,
tags: [
...new Set(
[
asString(casaOs?.category),
"community",
...toStringArray(casaOs?.architectures),
].filter((tag): tag is string => Boolean(tag)),
),
],
};
}
export function parseCasaOsConfigJson(
record: Record<string, unknown>,
fallbackId: string,
baseUrl?: string,
): AppTemplate | null {
const id = asString(record.id) ?? slugify(fallbackId);
const name =
asString(record.name) ??
asString(asRecord(record.title)?.en_us) ??
asString(record.id) ??
fallbackId;
const image = buildImageWithVersion(
asString(record.image),
asString(record.version),
);
if (!image) return null;
return {
id,
name,
icon: normalizeIcon(record, baseUrl),
desc:
asString(record.description) ??
asString(record.short_desc) ??
"Community app config",
category: toStringArray(record.categories)[0] ?? "Community",
image,
defaultPort: normalizePorts(record.port),
defaultEnv: normalizeEnv(record.form_fields),
restartPolicy: "unless-stopped",
popular: Boolean(record.featured ?? false),
tags: [...new Set([...toStringArray(record.categories), "community"])],
};
}
export function parseUmbrelManifest(
record: Record<string, unknown>,
fallbackId: string,
baseUrl?: string,
): AppTemplate | null {
const name = asString(record.name) ?? fallbackId;
const id = asString(record.id) ?? slugify(fallbackId);
const port = asNumber(record.port);
const image =
asString(record.image) ??
buildImageWithVersion(asString(record.repo), asString(record.version));
if (!name || !image) return null;
return {
id,
name,
icon: normalizeIcon(record, baseUrl),
desc:
asString(record.description) ??
asString(record.tagline) ??
"Umbrel manifest",
category: asString(record.category) ?? "Umbrel",
image,
defaultPort: port ? `${port}:${port}` : normalizePorts(record.port),
defaultEnv: "",
defaultVolumes: undefined,
defaultCommand: undefined,
defaultNetwork: undefined,
restartPolicy: "unless-stopped",
popular: false,
tags: [
...new Set(["umbrel", asString(record.category) ?? ""].filter(Boolean)),
],
};
}
export function normalizeSingleManifest(
payload: unknown,
fallbackId: string,
baseUrl?: string,
): AppTemplate[] {
const record = asRecord(payload);
if (!record) return [];
const compose = parseCasaOsCompose(record, fallbackId, baseUrl);
if (compose) return [compose];
if (isGenericAppTemplateRecord(record)) {
const generic = normalizeEntry(record, 0, baseUrl);
if (generic) return [generic];
}
const casaConfig = parseCasaOsConfigJson(record, fallbackId, baseUrl);
if (casaConfig) return [casaConfig];
const umbrel = parseUmbrelManifest(record, fallbackId, baseUrl);
if (umbrel) return [umbrel];
return extractCandidateEntries(payload)
.map((entry, index) => normalizeEntry(entry, index, baseUrl))
.filter((entry): entry is AppTemplate => Boolean(entry));
}
@@ -0,0 +1,362 @@
import { AppTemplate } from "../../config/app-templates";
import {
asNumber,
asRecord,
asString,
buildImageWithVersion,
slugify,
toStringArray,
} from "./utils";
function isAbsoluteHttpUrl(value: string): boolean {
return /^https?:\/\//i.test(value.trim());
}
function resolveRemoteAssetUrl(
value: unknown,
baseUrl?: string,
): string | undefined {
const raw = asString(value);
if (!raw) return undefined;
if (isAbsoluteHttpUrl(raw)) return raw;
if (!baseUrl) return undefined;
try {
return new URL(raw, baseUrl).toString();
} catch {
return undefined;
}
}
export function normalizeTags(entry: Record<string, unknown>): string[] {
const explicit = toStringArray(
entry.tags ?? entry.keywords ?? entry.categories ?? entry.category,
);
return explicit.slice(0, 8);
}
export function normalizePorts(value: unknown): string {
if (typeof value === "string") {
return value.trim();
}
if (typeof value === "number") {
return `${value}:${value}`;
}
if (Array.isArray(value)) {
return value
.map((item) => {
if (typeof item === "string") return item.trim();
if (typeof item === "number") return `${item}:${item}`;
const record = asRecord(item);
if (!record) return "";
const host = String(
record.host ?? record.hostPort ?? record.public ?? record.port ?? "",
).trim();
const container = String(
record.container ??
record.containerPort ??
record.private ??
record.target ??
record.port ??
"",
).trim();
const protocol = String(record.protocol ?? "tcp")
.trim()
.toLowerCase();
if (!host || !container) return "";
return protocol === "udp"
? `${host}:${container}/udp`
: `${host}:${container}`;
})
.filter(Boolean)
.join(",");
}
const record = asRecord(value);
if (!record) return "";
return Object.entries(record)
.map(([container, host]) => {
const hostStr = String(host ?? "").trim();
const containerStr = container.trim();
return hostStr && containerStr ? `${hostStr}:${containerStr}` : "";
})
.filter(Boolean)
.join(",");
}
export function normalizeEnv(value: unknown): string {
if (typeof value === "string") {
return value.trim();
}
if (Array.isArray(value)) {
return value
.map((item) => {
if (typeof item === "string") return item.trim();
const record = asRecord(item);
if (!record) return "";
const key = String(
record.key ?? record.name ?? record.label ?? "",
).trim();
const val = String(
record.value ?? record.default ?? record.defaultValue ?? "",
).trim();
return key ? `${key}=${val}` : "";
})
.filter(Boolean)
.join("\n");
}
const record = asRecord(value);
if (!record) return "";
return Object.entries(record)
.map(([key, val]) => `${key}=${String(val ?? "").trim()}`)
.filter(Boolean)
.join("\n");
}
export function normalizeVolumes(value: unknown): string {
if (typeof value === "string") {
return value.trim();
}
if (Array.isArray(value)) {
return value
.map((item) => {
if (typeof item === "string") return item.trim();
const record = asRecord(item);
if (!record) return "";
const host = String(
record.host ?? record.source ?? record.src ?? record.bind ?? "",
).trim();
const container = String(
record.container ??
record.target ??
record.dst ??
record.destination ??
"",
).trim();
return host && container ? `${host}:${container}` : "";
})
.filter(Boolean)
.join(",");
}
const record = asRecord(value);
if (!record) return "";
return Object.entries(record)
.map(([container, host]) => {
const hostStr = String(host ?? "").trim();
return hostStr && container.trim()
? `${hostStr}:${container.trim()}`
: "";
})
.filter(Boolean)
.join(",");
}
function normalizePresetList(entry: Record<string, unknown>) {
const presetsRaw = Array.isArray(entry.presets) ? entry.presets : [];
return presetsRaw
.map((preset) => {
const record = asRecord(preset);
if (!record) return null;
const name =
asString(record.name) ??
asString(record.title) ??
asString(record.id) ??
"Preset";
const id = asString(record.id) ?? slugify(name);
return {
id,
name,
desc: asString(record.desc) ?? asString(record.description) ?? "",
defaultPort: normalizePorts(
record.defaultPort ?? record.ports ?? record.port_map,
),
defaultEnv: normalizeEnv(
record.defaultEnv ?? record.env ?? record.environment,
),
defaultVolumes: normalizeVolumes(
record.defaultVolumes ?? record.volumes ?? record.storage,
),
defaultCommand:
asString(record.defaultCommand) ?? asString(record.command),
defaultNetwork:
asString(record.defaultNetwork) ?? asString(record.network),
restartPolicy:
asString(record.restartPolicy) ?? asString(record.restart),
};
})
.filter((preset): preset is NonNullable<typeof preset> => Boolean(preset));
}
function normalizePresentation(
entry: Record<string, unknown>,
baseUrl?: string,
): AppTemplate["presentation"] | undefined {
const presentation = asRecord(entry.presentation);
if (!presentation) return undefined;
const icon = resolveRemoteAssetUrl(presentation.icon, baseUrl);
const color = asString(presentation.color);
const installs = asNumber(presentation.installs);
if (!icon && !color && installs === undefined) return undefined;
return {
...(icon ? { icon } : {}),
...(color ? { color } : {}),
...(installs !== undefined ? { installs } : {}),
};
}
export function normalizeIcon(
entry: Record<string, unknown>,
baseUrl?: string,
): string | undefined {
const iconBlock = asRecord(entry.icon);
const logoBlock = asRecord(entry.logo);
return (
resolveRemoteAssetUrl(entry.icon, baseUrl) ??
resolveRemoteAssetUrl(entry.iconUrl, baseUrl) ??
resolveRemoteAssetUrl(entry.icon_url, baseUrl) ??
resolveRemoteAssetUrl(entry.logo, baseUrl) ??
resolveRemoteAssetUrl(entry.logoUrl, baseUrl) ??
resolveRemoteAssetUrl(entry.logo_url, baseUrl) ??
resolveRemoteAssetUrl(entry.thumbnail, baseUrl) ??
resolveRemoteAssetUrl(entry.thumb, baseUrl) ??
resolveRemoteAssetUrl(entry.avatar, baseUrl) ??
resolveRemoteAssetUrl(iconBlock?.en_us, baseUrl) ??
resolveRemoteAssetUrl(logoBlock?.en_us, baseUrl)
);
}
export function normalizeEntry(
entry: Record<string, unknown>,
index: number,
baseUrl?: string,
): AppTemplate | null {
const name =
asString(entry.name) ??
asString(entry.title) ??
asString(entry.app_name) ??
"";
const image =
asString(entry.image) ??
asString(entry.docker_image) ??
asString(entry.container_image) ??
asString(entry.repo) ??
buildImageWithVersion(asString(entry.image), asString(entry.version));
if (!name || !image) {
return null;
}
const id =
(asString(entry.id) ??
asString(entry.slug) ??
asString(entry.appid) ??
slugify(name)) ||
`remote-${index}`;
const presets = normalizePresetList(entry);
const presentation = normalizePresentation(entry, baseUrl);
return {
id,
name,
desc:
asString(entry.desc) ??
asString(entry.description) ??
asString(entry.short_desc) ??
asString(entry.subtitle) ??
"Remote app catalog entry",
category:
asString(entry.category) ??
asString(entry.type) ??
asString(entry.section) ??
"Remote",
icon: normalizeIcon(entry, baseUrl),
image,
defaultPort: normalizePorts(
entry.defaultPort ??
entry.ports ??
entry.port_map ??
entry.portMap ??
entry.port,
),
defaultEnv: normalizeEnv(
entry.defaultEnv ?? entry.env ?? entry.environment ?? entry.envs,
),
defaultVolumes:
normalizeVolumes(
entry.defaultVolumes ?? entry.volumes ?? entry.storage ?? entry.mounts,
) || undefined,
defaultCommand: asString(entry.defaultCommand) ?? asString(entry.command),
defaultNetwork: asString(entry.defaultNetwork) ?? asString(entry.network),
restartPolicy:
asString(entry.restartPolicy) ??
asString(entry.restart) ??
asString(entry.restart_policy) ??
"unless-stopped",
popular: Boolean(entry.popular ?? entry.featured ?? false),
tags: normalizeTags(entry),
presentation,
presets: presets.length > 0 ? presets : undefined,
};
}
export function extractCandidateEntries(
payload: unknown,
): Record<string, unknown>[] {
if (Array.isArray(payload)) {
return payload
.map((item) => asRecord(item))
.filter((item): item is Record<string, unknown> => Boolean(item));
}
const record = asRecord(payload);
if (!record) return [];
for (const key of [
"apps",
"templates",
"data",
"items",
"catalog",
"entries",
]) {
const value = record[key];
if (Array.isArray(value)) {
return value
.map((item) => asRecord(item))
.filter((item): item is Record<string, unknown> => Boolean(item));
}
}
return [record];
}
export function dedupeTemplates(templates: AppTemplate[]): AppTemplate[] {
const seen = new Set<string>();
return templates.filter((template) => {
if (seen.has(template.id)) return false;
seen.add(template.id);
return true;
});
}
+136
View File
@@ -0,0 +1,136 @@
import { CatalogSourceMode } from "./types";
import { slugify } from "./utils";
export function normalizeGithubManifestUrl(url: string): string {
const parsed = new URL(url);
if (parsed.hostname === "github.com" && parsed.pathname.includes("/blob/")) {
const parts = parsed.pathname.split("/").filter(Boolean);
const owner = parts[0];
const repo = parts[1];
const branch = parts[3];
const filePath = parts.slice(4).join("/");
return `https://raw.githubusercontent.com/${owner}/${repo}/${branch}/${filePath}`;
}
return url;
}
export function normalizeGithubArchiveUrl(url: string): string {
const parsed = new URL(url);
if (parsed.hostname === "github.com") {
const match = parsed.pathname.match(
/^\/([^/]+)\/([^/]+)\/archive\/refs\/(heads|tags)\/([^/]+)\.zip$/,
);
if (match) {
const [, owner, repo, refType, refName] = match;
return `https://codeload.github.com/${owner}/${repo}/zip/refs/${refType}/${refName}`;
}
}
return url;
}
export function getFallbackIdFromUrl(url: string): string {
try {
const parsed = new URL(url);
const segments = parsed.pathname.split("/").filter(Boolean);
const lastSegment = segments.at(-1) ?? "remote-manifest";
const normalizedLast = lastSegment.toLowerCase();
if (
/^(config\.json|docker-compose\.ya?ml|compose\.ya?ml|umbrel-app\.ya?ml|app\.json|manifest\.json|manifest\.ya?ml)$/.test(
normalizedLast,
)
) {
return slugify(segments.at(-2) ?? lastSegment);
}
return slugify(lastSegment.replace(/\.(json|ya?ml)$/i, ""));
} catch {
return slugify(url.split("/").pop() ?? "remote-manifest");
}
}
function isZipUrl(url: string): boolean {
try {
const parsed = new URL(url);
return /\.zip$/i.test(parsed.pathname);
} catch {
return /\.zip($|\?)/i.test(url);
}
}
function isZipContentType(contentType?: string | null): boolean {
if (!contentType) return false;
const normalized = contentType.toLowerCase();
return (
normalized.includes("zip") ||
normalized.includes("compressed") ||
normalized.includes("x-zip")
);
}
function getContentDispositionFileName(header?: string | null): string {
if (!header) return "";
const utf8Match = header.match(/filename\*=UTF-8''([^;]+)/i);
if (utf8Match?.[1]) {
try {
return decodeURIComponent(utf8Match[1]);
} catch {
return utf8Match[1];
}
}
const plainMatch = header.match(/filename="?([^";]+)"?/i);
return plainMatch?.[1] ?? "";
}
export function isZipResponse(response: Response, url: string): boolean {
const contentType = response.headers.get("content-type");
const fileName = getContentDispositionFileName(
response.headers.get("content-disposition"),
);
return (
isZipUrl(url) || isZipContentType(contentType) || /\.zip$/i.test(fileName)
);
}
export async function fetchRemoteResponse(url: string): Promise<Response> {
const parsedUrl = new URL(url);
if (!["http:", "https:"].includes(parsedUrl.protocol)) {
throw new Error("Remote catalog URL must use HTTP or HTTPS");
}
const response = await fetch(url, {
headers: {
Accept:
"application/json, application/yaml, text/yaml, text/x-yaml, text/plain, application/octet-stream, */*",
"User-Agent": "vps-panel-app-catalog/1.0",
},
redirect: "follow",
});
if (!response.ok) {
throw new Error(
`Remote request failed with status ${response.status} for ${response.url}`,
);
}
return response;
}
export function normalizeRemoteUrl(
source: CatalogSourceMode,
url: string,
): string {
const manifestNormalized = normalizeGithubManifestUrl(url);
return source === "github-archive"
? normalizeGithubArchiveUrl(url)
: normalizeGithubArchiveUrl(manifestNormalized);
}
+122
View File
@@ -0,0 +1,122 @@
import { APP_TEMPLATES } from "../../config/app-templates";
import {
getCachedCatalog,
getCacheKey,
makeCachedResult,
setCachedCatalog,
} from "./cache";
import { CatalogSyncResult, CatalogSourceMode } from "./types";
import { parseArchiveResponse } from "./archive";
import { dedupeTemplates } from "./normalize";
import { normalizeSingleManifest } from "./manifest-parsers";
import {
fetchRemoteResponse,
getFallbackIdFromUrl,
isZipResponse,
normalizeRemoteUrl,
} from "./remote";
import {
CatalogSourceValidationError,
isLikelyHtmlDocument,
parseStructuredText,
} from "./utils";
function detectTextFormat(text: string): string {
return text.trim().startsWith("{") || text.trim().startsWith("[")
? "json"
: "yaml";
}
async function loadRemoteTextCatalog(
source: Extract<CatalogSourceMode, "auto-detect" | "manifest-url">,
url: string,
label: string,
): Promise<CatalogSyncResult> {
const normalizedUrl = normalizeRemoteUrl(source, url);
const response = await fetchRemoteResponse(normalizedUrl);
if (isZipResponse(response, normalizedUrl)) {
return parseArchiveResponse(response, source, label, normalizedUrl);
}
const text = await response.text();
if (isLikelyHtmlDocument(text, response.headers.get("content-type"))) {
throw new CatalogSourceValidationError();
}
const payload = parseStructuredText(text);
const fallbackId = getFallbackIdFromUrl(normalizedUrl);
return {
templates: dedupeTemplates(
normalizeSingleManifest(payload, fallbackId, normalizedUrl),
),
meta: {
source,
label,
url: normalizedUrl,
fetchedAt: new Date().toISOString(),
isRemote: true,
format: detectTextFormat(text),
},
};
}
async function loadGithubArchiveCatalog(
url: string,
): Promise<CatalogSyncResult> {
const normalizedUrl = normalizeRemoteUrl("github-archive", url);
const response = await fetchRemoteResponse(normalizedUrl);
return parseArchiveResponse(
response,
"github-archive",
"GitHub Archive ZIP",
normalizedUrl,
);
}
export async function getCatalogTemplates(
source: CatalogSourceMode,
url?: string,
): Promise<CatalogSyncResult> {
const fetchedAt = new Date().toISOString();
if (source === "local") {
return {
templates: APP_TEMPLATES,
meta: {
source,
label: "Built-in Catalog",
fetchedAt,
isRemote: false,
cached: false,
},
};
}
if (!url?.trim()) {
throw new Error("Remote catalog URL is required");
}
const trimmedUrl = url.trim();
const cacheKey = getCacheKey(source, trimmedUrl);
const cachedEntry = getCachedCatalog(cacheKey);
if (cachedEntry) {
return makeCachedResult(cachedEntry);
}
const result =
source === "auto-detect"
? await loadRemoteTextCatalog(source, trimmedUrl, "Custom Source")
: source === "manifest-url"
? await loadRemoteTextCatalog(source, trimmedUrl, "Remote Manifest URL")
: await loadGithubArchiveCatalog(trimmedUrl);
if (result.templates.length === 0) {
throw new Error(
"No app templates could be extracted from the remote source",
);
}
return setCachedCatalog(cacheKey, result.templates, result.meta);
}
+38
View File
@@ -0,0 +1,38 @@
import { AppTemplate } from "../../config/app-templates";
export type CatalogSourceMode =
| "local"
| "auto-detect"
| "manifest-url"
| "github-archive";
export interface CatalogMeta {
source: CatalogSourceMode;
label: string;
url?: string;
fetchedAt: string;
isRemote: boolean;
cached?: boolean;
expiresAt?: string;
format?: string;
}
export interface CatalogSyncResult {
templates: AppTemplate[];
meta: CatalogMeta;
}
export interface CatalogSourceOption {
id: CatalogSourceMode;
label: string;
description: string;
requiresUrl: boolean;
placeholder?: string;
exampleUrl?: string;
}
export interface CacheEntry {
templates: AppTemplate[];
meta: CatalogMeta;
expiresAt: number;
}
+126
View File
@@ -0,0 +1,126 @@
import yaml from "js-yaml";
export const REMOTE_CATALOG_SOURCE_HELP =
"Custom source must point to a raw JSON/YAML app manifest or a direct ZIP archive URL, not a browser HTML page. Use a raw file URL or a repository archive URL.";
export class CatalogSourceValidationError extends Error {
constructor(message = REMOTE_CATALOG_SOURCE_HELP) {
super(message);
this.name = "CatalogSourceValidationError";
}
}
export function slugify(value: string): string {
return value
.toLowerCase()
.replace(/[^a-z0-9]+/g, "-")
.replace(/^-+|-+$/g, "")
.slice(0, 60);
}
export function asString(value: unknown): string | undefined {
return typeof value === "string" ? value.trim() : undefined;
}
export function asNumber(value: unknown): number | undefined {
if (typeof value === "number" && Number.isFinite(value)) {
return value;
}
if (typeof value === "string") {
const parsed = Number(value.trim());
return Number.isFinite(parsed) ? parsed : undefined;
}
return undefined;
}
export function asRecord(value: unknown): Record<string, unknown> | undefined {
if (!value || typeof value !== "object" || Array.isArray(value)) {
return undefined;
}
return value as Record<string, unknown>;
}
export function toStringArray(value: unknown): string[] {
if (Array.isArray(value)) {
return value
.map((item) => (typeof item === "string" ? item.trim() : ""))
.filter(Boolean);
}
if (typeof value === "string") {
return value
.split(",")
.map((item) => item.trim())
.filter(Boolean);
}
return [];
}
export function buildImageWithVersion(
image?: string,
version?: string,
): string {
const imageValue = image?.trim() ?? "";
const versionValue = version?.trim() ?? "";
if (!imageValue) return "";
if (!versionValue) return imageValue;
const lastColon = imageValue.lastIndexOf(":");
const lastSlash = imageValue.lastIndexOf("/");
if (lastColon > lastSlash) {
return imageValue;
}
return `${imageValue}:${versionValue}`;
}
export function isLikelyHtmlDocument(
text: string,
contentType?: string | null,
): boolean {
if (contentType?.toLowerCase().includes("text/html")) {
return true;
}
const preview = text.trimStart().slice(0, 4096).toLowerCase();
if (!preview) return false;
return (
preview.startsWith("<!doctype html") ||
preview.startsWith("<html") ||
preview.includes("<head") ||
preview.includes("<body") ||
preview.includes("<style") ||
preview.includes("<script")
);
}
export function parseStructuredText(text: string): unknown {
const trimmed = text.trim();
if (!trimmed) return {};
if (isLikelyHtmlDocument(trimmed)) {
throw new CatalogSourceValidationError();
}
try {
if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
return JSON.parse(trimmed);
}
return yaml.load(trimmed);
} catch (error) {
if (error instanceof CatalogSourceValidationError) {
throw error;
}
throw new CatalogSourceValidationError(
"Custom source could not be parsed as JSON or YAML. Check that the URL returns a raw app manifest or use a direct ZIP archive URL.",
);
}
}