140 lines
8.0 KiB
JavaScript
140 lines
8.0 KiB
JavaScript
import fs from 'node:fs'
|
|
import path from 'node:path'
|
|
import crypto from 'node:crypto'
|
|
import { fileURLToPath } from 'node:url'
|
|
|
|
const buildDirectory = path.dirname(fileURLToPath(import.meta.url))
|
|
export const SOURCE_MANIFEST_PATH = path.join(buildDirectory, 'tang-detective-source-manifest.json')
|
|
export const DEFAULT_PRUNE_RECEIPT_PATH = path.join(buildDirectory, 'tang-detective-media-prune-receipt.json')
|
|
export const ORIGINAL_SOURCE_MANIFEST_SHA256 = '3b203406a31fdbe17c770aeea0bab19755acfedf3fa1595641ee4e15a9712e20'
|
|
export const MEDIA_TYPES = {
|
|
'.jpg': ['image', 'image/jpeg'], '.jpeg': ['image', 'image/jpeg'],
|
|
'.png': ['image', 'image/png'], '.webp': ['image', 'image/webp'],
|
|
'.gif': ['image', 'image/gif'], '.svg': ['image', 'image/svg+xml'], '.avif': ['image', 'image/avif'],
|
|
'.mp3': ['audio', 'audio/mpeg'], '.wav': ['audio', 'audio/wav'],
|
|
'.aac': ['audio', 'audio/aac'], '.m4a': ['audio', 'audio/mp4'], '.ogg': ['audio', 'audio/ogg'],
|
|
'.mp4': ['video', 'video/mp4'], '.webm': ['video', 'video/webm'], '.mov': ['video', 'video/quicktime'],
|
|
}
|
|
export const isMediaFile = value => Boolean(MEDIA_TYPES[path.extname(value).toLowerCase()])
|
|
export const sha256 = value => crypto.createHash('sha256').update(value).digest('hex')
|
|
export const canonicalJsonSha256 = value => sha256(`${JSON.stringify(value, null, 2)}\n`)
|
|
const SHA256 = /^[a-f0-9]{64}$/
|
|
function assert(condition, message) {
|
|
if (!condition) throw new Error(`Invalid Tang Detective source snapshot: ${message}`)
|
|
}
|
|
export function safeSourcePath(value) {
|
|
return typeof value === 'string' && value.length > 0 && !path.isAbsolute(value)
|
|
&& !/[\\?#\u0000-\u0020]/.test(value)
|
|
&& value.split('/').every(part => part && part !== '.' && part !== '..')
|
|
}
|
|
function lstatOrNull(filename) {
|
|
try { return fs.lstatSync(filename) } catch (error) {
|
|
if (error.code === 'ENOENT') return null
|
|
throw error
|
|
}
|
|
}
|
|
export function readSourceManifest(sourceManifestPath = SOURCE_MANIFEST_PATH) {
|
|
const stat = lstatOrNull(sourceManifestPath)
|
|
assert(stat?.isFile() && !stat.isSymbolicLink(), 'source manifest must be a regular file')
|
|
const bytes = fs.readFileSync(sourceManifestPath)
|
|
const manifestSha256 = sha256(bytes)
|
|
if (path.resolve(sourceManifestPath) === SOURCE_MANIFEST_PATH) {
|
|
assert(manifestSha256 === ORIGINAL_SOURCE_MANIFEST_SHA256, 'original source manifest changed')
|
|
}
|
|
const manifest = JSON.parse(bytes)
|
|
assert(Array.isArray(manifest.files) && manifest.files.length > 0, 'source manifest files are required')
|
|
const files = new Map()
|
|
for (const record of manifest.files) {
|
|
assert(record && safeSourcePath(record.path) && !files.has(record.path), 'unsafe or duplicate source manifest path')
|
|
assert(Number.isSafeInteger(record.bytes) && record.bytes >= 0 && SHA256.test(record.sha256), `invalid source metadata: ${record.path}`)
|
|
files.set(record.path, record)
|
|
}
|
|
return { manifest, manifestSha256, files }
|
|
}
|
|
|
|
/** Inspect the entire tree, including dangling links and unexpected file types.
|
|
* No media omission may hide an extra file, source edit or symlink. */
|
|
export function inspectSourceFiles(sourceDirectory, source = readSourceManifest()) {
|
|
const root = path.resolve(sourceDirectory)
|
|
const rootStat = lstatOrNull(root)
|
|
assert(rootStat?.isDirectory() && !rootStat.isSymbolicLink(), 'source root must be a real directory, not a symbolic link')
|
|
const actual = new Map()
|
|
const expectedDirectories = new Set([...source.files.keys()].flatMap(relative => {
|
|
const parts = relative.split('/')
|
|
return parts.slice(0, -1).map((_, index) => parts.slice(0, index + 1).join('/'))
|
|
}))
|
|
function visit(directory, prefix = '') {
|
|
for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
|
|
const relative = prefix ? `${prefix}/${entry.name}` : entry.name
|
|
const filename = path.join(directory, entry.name)
|
|
assert(!entry.isSymbolicLink(), `source symbolic links are forbidden: ${relative}`)
|
|
if (entry.isDirectory()) {
|
|
assert(expectedDirectories.has(relative), `unexpected source directory: ${relative}`)
|
|
visit(filename, relative)
|
|
continue
|
|
}
|
|
assert(entry.isFile(), `unexpected source file type: ${relative}`)
|
|
assert(source.files.has(relative), `unexpected source file: ${relative}`)
|
|
const record = source.files.get(relative)
|
|
const bytes = fs.readFileSync(filename)
|
|
assert(bytes.length === record.bytes && sha256(bytes) === record.sha256, `source bytes changed: ${relative}`)
|
|
actual.set(relative, record)
|
|
}
|
|
}
|
|
visit(root)
|
|
const missingMedia = []
|
|
for (const record of source.files.values()) {
|
|
if (actual.has(record.path)) continue
|
|
assert(isMediaFile(record.path), `missing non-media source: ${record.path}`)
|
|
missingMedia.push(record.path)
|
|
}
|
|
return { actual, missingMedia }
|
|
}
|
|
|
|
export function validatePruneReceipt(receipt, { source, media }) {
|
|
assert(receipt?.schemaVersion === 1 && receipt.status === 'completed', 'completed project-local media prune receipt is required')
|
|
assert(receipt.sourceManifestSha256 === source.manifestSha256, 'prune receipt source manifest hash mismatch')
|
|
assert(receipt.mediaManifestSha256 === media.manifestSha256, 'prune receipt media manifest hash mismatch')
|
|
assert(typeof media.uploadRunId === 'string' && /^[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}$/.test(media.uploadRunId)
|
|
&& receipt.uploadRunId === media.uploadRunId, 'prune receipt upload run mismatch')
|
|
assert(receipt.backup?.format === 'tar.gz' && SHA256.test(receipt.backup.sha256)
|
|
&& Number.isSafeInteger(receipt.backup.bytes) && receipt.backup.bytes > 0, 'prune receipt verified backup hash and size are required')
|
|
const expectedMedia = [...source.files.values()].filter(record => isMediaFile(record.path))
|
|
assert(media.entries instanceof Map && media.entries.size === expectedMedia.length
|
|
&& expectedMedia.every(record => {
|
|
const entry = media.entries.get(record.path)
|
|
return entry?.sourcePath === record.path && entry.sha256 === record.sha256 && entry.bytes === record.bytes
|
|
&& entry.uploaded === true && entry.publicReadVerified === true && entry.remoteVerifiedSha256 === record.sha256
|
|
&& (MEDIA_TYPES[path.extname(record.path)][0] === 'image' || entry.rangeVerified === true)
|
|
}), 'prune receipt requires complete verified source media coverage')
|
|
assert(Array.isArray(receipt.entries) && receipt.entries.length === expectedMedia.length, 'prune receipt media coverage is incomplete')
|
|
const seen = new Set()
|
|
for (const entry of receipt.entries) {
|
|
assert(entry && safeSourcePath(entry.sourcePath) && !seen.has(entry.sourcePath), 'unsafe or duplicate prune receipt entry')
|
|
const expected = media.entries.get(entry.sourcePath)
|
|
assert(expected && entry.sha256 === expected.sha256 && entry.bytes === expected.bytes
|
|
&& entry.url === expected.url && entry.objectKey === expected.objectKey
|
|
&& entry.backupSha256 === receipt.backup.sha256, `prune receipt entry mismatch: ${entry.sourcePath}`)
|
|
seen.add(entry.sourcePath)
|
|
}
|
|
return { receiptSha256: canonicalJsonSha256(receipt), uploadRunId: receipt.uploadRunId, backupSha256: receipt.backup.sha256 }
|
|
}
|
|
|
|
export function validateSourceSnapshot({ sourceDirectory, sourceManifestPath = SOURCE_MANIFEST_PATH,
|
|
media = null, pruneReceipt, pruneReceiptPath = DEFAULT_PRUNE_RECEIPT_PATH } = {}) {
|
|
const source = readSourceManifest(sourceManifestPath)
|
|
const inspection = inspectSourceFiles(sourceDirectory, source)
|
|
let prune = null
|
|
if (inspection.missingMedia.length) {
|
|
assert(media, `local mode requires all media; ${inspection.missingMedia.length} source media files are missing`)
|
|
assert(media.sourceManifestSha256 === source.manifestSha256, 'remote source manifest binding mismatch')
|
|
if (pruneReceipt === undefined) {
|
|
const stat = lstatOrNull(pruneReceiptPath)
|
|
assert(stat?.isFile() && !stat.isSymbolicLink(), 'completed project-local media prune receipt is required')
|
|
pruneReceipt = JSON.parse(fs.readFileSync(pruneReceiptPath, 'utf8'))
|
|
}
|
|
prune = validatePruneReceipt(pruneReceipt, { source, media })
|
|
}
|
|
return { ...source, ...inspection, prune }
|
|
}
|