import fs from 'node:fs' import path from 'node:path' import crypto from 'node:crypto' import { fileURLToPath, pathToFileURL } from 'node:url' import { dependency, readCosConfig, safeError } from './config.mjs' export const PROJECT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..') const MEDIA = { '.jpg': ['image', 'image/jpeg'], '.jpeg': ['image', 'image/jpeg'], '.png': ['image', 'image/png'], '.webp': ['image', 'image/webp'], '.gif': ['image', 'image/gif'], '.svg': ['image', 'image/svg+xml'], '.mp3': ['audio', 'audio/mpeg'], '.wav': ['audio', 'audio/wav'], '.m4a': ['audio', 'audio/mp4'], '.aac': ['audio', 'audio/aac'], '.ogg': ['audio', 'audio/ogg'], '.mp4': ['video', 'video/mp4'], '.webm': ['video', 'video/webm'], '.mov': ['video', 'video/quicktime'], } export const hash = bytes => crypto.createHash('sha256').update(bytes).digest('hex') function walk(directory, prefix = '') { return fs.readdirSync(directory, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)).flatMap(entry => { const relative = prefix ? `${prefix}/${entry.name}` : entry.name if (entry.isSymbolicLink()) throw new Error('SOURCE_SYMLINK_NOT_ALLOWED') return entry.isDirectory() ? walk(path.join(directory, entry.name), relative) : [relative] }) } export function inventory(project = PROJECT) { const sourceDirectory = path.join(project, 'native/tang-detective') const manifestBytes = fs.readFileSync(path.join(project, 'build/tang-detective-source-manifest.json')) const source = JSON.parse(manifestBytes) const files = walk(sourceDirectory) const expected = new Map(source.files.map(entry => [entry.path, entry])) if (files.length !== expected.size || files.some(file => !expected.has(file))) throw new Error('SOURCE_FILE_SET_CHANGED') const entries = [] for (const relative of files) { const bytes = fs.readFileSync(path.join(sourceDirectory, relative)) const sha256 = hash(bytes) const original = expected.get(relative) if (original.bytes !== bytes.length || original.sha256 !== sha256) throw new Error('SOURCE_BYTES_CHANGED') const extension = path.extname(relative).toLowerCase() if (!MEDIA[extension]) continue const [kind, contentType] = MEDIA[extension] entries.push({ sourcePath: relative, kind, contentType, bytes: bytes.length, sha256, objectKey: `tang-detective/season-01/media-v1/${sha256}${extension}` }) } if (!entries.length) throw new Error('NO_SOURCE_MEDIA') return { sourceDirectory, sourceManifestSha256: hash(manifestBytes), entries } } function writeJson(filename, value) { fs.mkdirSync(path.dirname(filename), { recursive: true }) const temp = `${filename}.${process.pid}.tmp` fs.writeFileSync(temp, `${JSON.stringify(value, null, 2)}\n`, { flag: 'wx' }) fs.renameSync(temp, filename) } export function objectUrl(baseUrl, objectKey) { if (!/^tang-detective\/season-01\/media-v1\/[a-f0-9]{64}\.[a-z0-9]+$/.test(objectKey)) { throw new Error('UNSAFE_OBJECT_KEY') } const url = new URL(`${baseUrl}/${objectKey}`) if (url.protocol !== 'https:' || url.username || url.password || url.search || url.hash) throw new Error('UNSAFE_OBJECT_URL') return url.href } async function boundedBody(response, limit) { let length = 0 const chunks = [] for await (const chunk of response.body) { length += chunk.length if (length > limit) throw new Error('REMOTE_BODY_TOO_LARGE') chunks.push(chunk) } return Buffer.concat(chunks) } export async function verifyPublicObject(entry, fetchImpl = fetch) { const response = await fetchImpl(entry.url, { redirect: 'error', signal: AbortSignal.timeout(20000) }) if (response.status !== 200) { await response.body?.cancel(); throw new Error('PUBLIC_MEDIA_GET_FAILED') } const bytes = await boundedBody(response, entry.bytes) if (bytes.length !== entry.bytes || hash(bytes) !== entry.sha256) throw new Error('PUBLIC_MEDIA_HASH_MISMATCH') if (response.headers.get('content-type')?.split(';')[0].trim() !== entry.contentType) throw new Error('PUBLIC_MEDIA_TYPE_MISMATCH') let rangeVerified = null if (entry.kind === 'audio' || entry.kind === 'video') { const last = Math.min(1023, entry.bytes - 1) const range = await fetchImpl(entry.url, { headers: { Range: `bytes=0-${last}` }, redirect: 'error', signal: AbortSignal.timeout(20000) }) if (range.status !== 206 || range.headers.get('content-range') !== `bytes 0-${last}/${entry.bytes}`) { await range.body?.cancel(); throw new Error('PUBLIC_MEDIA_RANGE_FAILED') } const fragment = await boundedBody(range, last + 1) if (!fragment.equals(bytes.subarray(0, last + 1))) throw new Error('PUBLIC_MEDIA_RANGE_HASH_MISMATCH') rangeVerified = true } return { remoteVerifiedSha256: hash(bytes), rangeVerified, verifiedAt: new Date().toISOString() } } export async function uploadObject(cos, destination, entry, body, onState = () => {}) { const params = { Bucket: destination.bucket, Region: destination.region, Key: entry.objectKey } let exists = false try { const head = await cos.headObject(params) if (Number(head.headers?.['content-length']) !== entry.bytes || head.headers?.['x-cos-meta-sha256'] !== entry.sha256) { throw new Error('EXISTING_OBJECT_NOT_OWNED_OR_DIFFERENT') } exists = true } catch (error) { if (Number(error.statusCode) !== 404) throw error } if (!exists) { onState({ uploaded: 'unknown', uploadStatus: 'put-started-outcome-unknown' }) await cos.putObject({ ...params, Body: body, ContentLength: body.length, ContentType: entry.contentType, CacheControl: 'public, max-age=31536000, immutable', Headers: { 'Content-MD5': crypto.createHash('md5').update(body).digest('base64'), 'x-cos-meta-sha256': entry.sha256, 'x-cos-forbid-overwrite': 'true', } }) } return { uploaded: true, uploadStatus: 'confirmed', action: exists ? 'reused-identical' : 'uploaded' } } export async function main(mode, dependencies = {}) { if (!['inventory', 'inspect', 'upload'].includes(mode)) throw new Error('USE_INVENTORY_INSPECT_OR_UPLOAD') const project = dependencies.project || PROJECT const readConfig = dependencies.readConfig || readCosConfig const verify = dependencies.verify || verifyPublicObject const log = dependencies.log || (value => console.log(JSON.stringify(value))) if (mode === 'upload') return runUpload({ project, readConfig, verify, log, createCos: dependencies.createCos }) const input = inventory(project) const summary = { files: input.entries.length, uniqueObjects: new Set(input.entries.map(e => e.objectKey)).size, bytes: input.entries.reduce((sum, entry) => sum + entry.bytes, 0), kinds: Object.fromEntries(['image', 'audio', 'video'].map(kind => [kind, input.entries.filter(e => e.kind === kind).length])) } if (mode === 'inventory') { log(summary); return } const config = await readConfig(path.resolve(project, '../server')) const destination = { bucket: config.bucket, region: config.region, baseUrl: config.baseUrl } log({ ...summary, destination, credentialsPresent: true, databaseReadOnly: true, tlsVerified: true }) } async function runUpload({ project, readConfig, verify, log, createCos }) { const runId = crypto.randomUUID() const receipt = { schemaVersion: 1, runId, startedAt: new Date().toISOString(), phase: 'inventory', complete: false, bucketPermissionsChanged: false, originalFilesChanged: false, objects: [] } const receiptPath = path.join(project, 'build/tang-detective-cos-upload-receipt.json') const historyDirectory = path.join(project, 'build/tang-detective-cos-upload-attempts') fs.mkdirSync(historyDirectory, { recursive: true }) if (fs.existsSync(receiptPath)) { const previous = fs.readFileSync(receiptPath) const archive = path.join(historyDirectory, `previous-${hash(previous)}.json`) if (!fs.existsSync(archive)) fs.writeFileSync(archive, previous, { flag: 'wx' }) } const persist = () => { writeJson(path.join(historyDirectory, `${runId}.json`), receipt) writeJson(receiptPath, receipt) } persist() const completed = new Map() try { const input = inventory(project) const summary = { files: input.entries.length, uniqueObjects: new Set(input.entries.map(e => e.objectKey)).size, bytes: input.entries.reduce((sum, entry) => sum + entry.bytes, 0) } receipt.sourceManifestSha256 = input.sourceManifestSha256 receipt.phase = 'configuration' persist() const config = await readConfig(path.resolve(project, '../server')) const destination = { bucket: config.bucket, region: config.region, baseUrl: config.baseUrl } receipt.destination = destination receipt.phase = 'sdk-initialization' persist() const options = { SecretId: config.access_key, SecretKey: config.secret_key, Protocol: 'https:', Timeout: 20000, MaxRetryTimes: 0, UploadCheckContentMd5: true } const cos = createCos ? createCos(options) : new (dependency('cos-nodejs-sdk-v5'))(options) log({ ...summary, destination, credentialsPresent: true, databaseReadOnly: true, tlsVerified: true }) receipt.phase = 'upload-and-verify' // Sequential first-object verification stops immediately if this bucket/domain isn't anonymously readable. for (const item of input.entries) { if (completed.has(item.objectKey)) continue const entry = { ...item, url: objectUrl(config.baseUrl, item.objectKey) } const body = fs.readFileSync(path.join(input.sourceDirectory, item.sourcePath)) if (body.length !== item.bytes || hash(body) !== item.sha256) throw new Error('SOURCE_CHANGED_DURING_UPLOAD') const result = { ...entry, uploaded: false, uploadStatus: 'not-attempted' } receipt.objects.push(result) persist() try { Object.assign(result, await uploadObject(cos, destination, entry, body, state => { Object.assign(result, state) persist() })) persist() Object.assign(result, await verify(entry)) completed.set(item.objectKey, result) persist() } catch (error) { result.error = safeError(error) // A lost PUT response is not proof that nothing was uploaded. Read-only reconciliation can establish it. if (result.uploaded === 'unknown') { try { const head = await cos.headObject({ Bucket: destination.bucket, Region: destination.region, Key: entry.objectKey }) if (Number(head.headers?.['content-length']) === entry.bytes && head.headers?.['x-cos-meta-sha256'] === entry.sha256) { Object.assign(result, { uploaded: true, uploadStatus: 'confirmed-by-readback' }, await verify(entry)) } } catch { /* Keep unknown; retry checks the exact content-addressed key without overwriting. */ } } throw error } log({ verified: completed.size, total: summary.uniqueObjects }) } // Recheck all source bytes immediately before publishing the build activation manifest. if (inventory(project).sourceManifestSha256 !== input.sourceManifestSha256) throw new Error('SOURCE_MANIFEST_CHANGED') const entries = input.entries.map(entry => ({ ...entry, url: completed.get(entry.objectKey).url, uploaded: true, remoteVerifiedSha256: completed.get(entry.objectKey).remoteVerifiedSha256, rangeVerified: completed.get(entry.objectKey).rangeVerified, verifiedAt: completed.get(entry.objectKey).verifiedAt })) writeJson(path.join(project, 'build/tang-detective-cos-manifest.json'), { schemaVersion: 1, sourceManifestSha256: input.sourceManifestSha256, destination, entries, }) receipt.complete = true receipt.phase = 'complete' receipt.completedAt = new Date().toISOString() log({ complete: true, ...summary }) } catch (error) { receipt.error = safeError(error) receipt.failedAt = new Date().toISOString() throw error } finally { persist() } } if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { main(process.argv[2]).catch(error => { console.error(JSON.stringify({ complete: false, error: safeError(error) })); process.exitCode = 1 }) }