import fs from 'node:fs' import path from 'node:path' import crypto from 'node:crypto' import { fileURLToPath } from 'node:url' const buildDirectory = path.dirname(fileURLToPath(import.meta.url)) export const SOURCE_MANIFEST_PATH = path.join(buildDirectory, 'tang-detective-source-manifest.json') export const DEFAULT_PRUNE_RECEIPT_PATH = path.join(buildDirectory, 'tang-detective-media-prune-receipt.json') export const ORIGINAL_SOURCE_MANIFEST_SHA256 = '3b203406a31fdbe17c770aeea0bab19755acfedf3fa1595641ee4e15a9712e20' export const MEDIA_TYPES = { '.jpg': ['image', 'image/jpeg'], '.jpeg': ['image', 'image/jpeg'], '.png': ['image', 'image/png'], '.webp': ['image', 'image/webp'], '.gif': ['image', 'image/gif'], '.svg': ['image', 'image/svg+xml'], '.avif': ['image', 'image/avif'], '.mp3': ['audio', 'audio/mpeg'], '.wav': ['audio', 'audio/wav'], '.aac': ['audio', 'audio/aac'], '.m4a': ['audio', 'audio/mp4'], '.ogg': ['audio', 'audio/ogg'], '.mp4': ['video', 'video/mp4'], '.webm': ['video', 'video/webm'], '.mov': ['video', 'video/quicktime'], } export const isMediaFile = value => Boolean(MEDIA_TYPES[path.extname(value).toLowerCase()]) export const sha256 = value => crypto.createHash('sha256').update(value).digest('hex') export const canonicalJsonSha256 = value => sha256(`${JSON.stringify(value, null, 2)}\n`) const SHA256 = /^[a-f0-9]{64}$/ function assert(condition, message) { if (!condition) throw new Error(`Invalid Tang Detective source snapshot: ${message}`) } export function safeSourcePath(value) { return typeof value === 'string' && value.length > 0 && !path.isAbsolute(value) && !/[\\?#\u0000-\u0020]/.test(value) && value.split('/').every(part => part && part !== '.' && part !== '..') } function lstatOrNull(filename) { try { return fs.lstatSync(filename) } catch (error) { if (error.code === 'ENOENT') return null throw error } } export function readSourceManifest(sourceManifestPath = SOURCE_MANIFEST_PATH) { const stat = lstatOrNull(sourceManifestPath) assert(stat?.isFile() && !stat.isSymbolicLink(), 'source manifest must be a regular file') const bytes = fs.readFileSync(sourceManifestPath) const manifestSha256 = sha256(bytes) if (path.resolve(sourceManifestPath) === SOURCE_MANIFEST_PATH) { assert(manifestSha256 === ORIGINAL_SOURCE_MANIFEST_SHA256, 'original source manifest changed') } const manifest = JSON.parse(bytes) assert(Array.isArray(manifest.files) && manifest.files.length > 0, 'source manifest files are required') const files = new Map() for (const record of manifest.files) { assert(record && safeSourcePath(record.path) && !files.has(record.path), 'unsafe or duplicate source manifest path') assert(Number.isSafeInteger(record.bytes) && record.bytes >= 0 && SHA256.test(record.sha256), `invalid source metadata: ${record.path}`) files.set(record.path, record) } return { manifest, manifestSha256, files } } /** Inspect the entire tree, including dangling links and unexpected file types. * No media omission may hide an extra file, source edit or symlink. */ export function inspectSourceFiles(sourceDirectory, source = readSourceManifest()) { const root = path.resolve(sourceDirectory) const rootStat = lstatOrNull(root) assert(rootStat?.isDirectory() && !rootStat.isSymbolicLink(), 'source root must be a real directory, not a symbolic link') const actual = new Map() const expectedDirectories = new Set([...source.files.keys()].flatMap(relative => { const parts = relative.split('/') return parts.slice(0, -1).map((_, index) => parts.slice(0, index + 1).join('/')) })) function visit(directory, prefix = '') { for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { const relative = prefix ? `${prefix}/${entry.name}` : entry.name const filename = path.join(directory, entry.name) assert(!entry.isSymbolicLink(), `source symbolic links are forbidden: ${relative}`) if (entry.isDirectory()) { assert(expectedDirectories.has(relative), `unexpected source directory: ${relative}`) visit(filename, relative) continue } assert(entry.isFile(), `unexpected source file type: ${relative}`) assert(source.files.has(relative), `unexpected source file: ${relative}`) const record = source.files.get(relative) const bytes = fs.readFileSync(filename) assert(bytes.length === record.bytes && sha256(bytes) === record.sha256, `source bytes changed: ${relative}`) actual.set(relative, record) } } visit(root) const missingMedia = [] for (const record of source.files.values()) { if (actual.has(record.path)) continue assert(isMediaFile(record.path), `missing non-media source: ${record.path}`) missingMedia.push(record.path) } return { actual, missingMedia } } export function validatePruneReceipt(receipt, { source, media }) { assert(receipt?.schemaVersion === 1 && receipt.status === 'completed', 'completed project-local media prune receipt is required') assert(receipt.sourceManifestSha256 === source.manifestSha256, 'prune receipt source manifest hash mismatch') assert(receipt.mediaManifestSha256 === media.manifestSha256, 'prune receipt media manifest hash mismatch') assert(typeof media.uploadRunId === 'string' && /^[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}$/.test(media.uploadRunId) && receipt.uploadRunId === media.uploadRunId, 'prune receipt upload run mismatch') assert(receipt.backup?.format === 'tar.gz' && SHA256.test(receipt.backup.sha256) && Number.isSafeInteger(receipt.backup.bytes) && receipt.backup.bytes > 0, 'prune receipt verified backup hash and size are required') const expectedMedia = [...source.files.values()].filter(record => isMediaFile(record.path)) assert(media.entries instanceof Map && media.entries.size === expectedMedia.length && expectedMedia.every(record => { const entry = media.entries.get(record.path) return entry?.sourcePath === record.path && entry.sha256 === record.sha256 && entry.bytes === record.bytes && entry.uploaded === true && entry.publicReadVerified === true && entry.remoteVerifiedSha256 === record.sha256 && (MEDIA_TYPES[path.extname(record.path)][0] === 'image' || entry.rangeVerified === true) }), 'prune receipt requires complete verified source media coverage') assert(Array.isArray(receipt.entries) && receipt.entries.length === expectedMedia.length, 'prune receipt media coverage is incomplete') const seen = new Set() for (const entry of receipt.entries) { assert(entry && safeSourcePath(entry.sourcePath) && !seen.has(entry.sourcePath), 'unsafe or duplicate prune receipt entry') const expected = media.entries.get(entry.sourcePath) assert(expected && entry.sha256 === expected.sha256 && entry.bytes === expected.bytes && entry.url === expected.url && entry.objectKey === expected.objectKey && entry.backupSha256 === receipt.backup.sha256, `prune receipt entry mismatch: ${entry.sourcePath}`) seen.add(entry.sourcePath) } return { receiptSha256: canonicalJsonSha256(receipt), uploadRunId: receipt.uploadRunId, backupSha256: receipt.backup.sha256 } } export function validateSourceSnapshot({ sourceDirectory, sourceManifestPath = SOURCE_MANIFEST_PATH, media = null, pruneReceipt, pruneReceiptPath = DEFAULT_PRUNE_RECEIPT_PATH } = {}) { const source = readSourceManifest(sourceManifestPath) const inspection = inspectSourceFiles(sourceDirectory, source) let prune = null if (inspection.missingMedia.length) { assert(media, `local mode requires all media; ${inspection.missingMedia.length} source media files are missing`) assert(media.sourceManifestSha256 === source.manifestSha256, 'remote source manifest binding mismatch') if (pruneReceipt === undefined) { const stat = lstatOrNull(pruneReceiptPath) assert(stat?.isFile() && !stat.isSymbolicLink(), 'completed project-local media prune receipt is required') pruneReceipt = JSON.parse(fs.readFileSync(pruneReceiptPath, 'utf8')) } prune = validatePruneReceipt(pruneReceipt, { source, media }) } return { ...source, ...inspection, prune } }