229 lines
12 KiB
JavaScript
229 lines
12 KiB
JavaScript
import fs from 'node:fs'
|
|
import path from 'node:path'
|
|
import crypto from 'node:crypto'
|
|
import { fileURLToPath, pathToFileURL } from 'node:url'
|
|
import { dependency, readCosConfig, safeError } from './config.mjs'
|
|
|
|
export const PROJECT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..')
|
|
const MEDIA = {
|
|
'.jpg': ['image', 'image/jpeg'], '.jpeg': ['image', 'image/jpeg'], '.png': ['image', 'image/png'],
|
|
'.webp': ['image', 'image/webp'], '.gif': ['image', 'image/gif'], '.svg': ['image', 'image/svg+xml'],
|
|
'.mp3': ['audio', 'audio/mpeg'], '.wav': ['audio', 'audio/wav'], '.m4a': ['audio', 'audio/mp4'],
|
|
'.aac': ['audio', 'audio/aac'], '.ogg': ['audio', 'audio/ogg'], '.mp4': ['video', 'video/mp4'],
|
|
'.webm': ['video', 'video/webm'], '.mov': ['video', 'video/quicktime'],
|
|
}
|
|
export const hash = bytes => crypto.createHash('sha256').update(bytes).digest('hex')
|
|
|
|
function walk(directory, prefix = '') {
|
|
return fs.readdirSync(directory, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name)).flatMap(entry => {
|
|
const relative = prefix ? `${prefix}/${entry.name}` : entry.name
|
|
if (entry.isSymbolicLink()) throw new Error('SOURCE_SYMLINK_NOT_ALLOWED')
|
|
return entry.isDirectory() ? walk(path.join(directory, entry.name), relative) : [relative]
|
|
})
|
|
}
|
|
|
|
export function inventory(project = PROJECT) {
|
|
const sourceDirectory = path.join(project, 'native/tang-detective')
|
|
const manifestBytes = fs.readFileSync(path.join(project, 'build/tang-detective-source-manifest.json'))
|
|
const source = JSON.parse(manifestBytes)
|
|
const files = walk(sourceDirectory)
|
|
const expected = new Map(source.files.map(entry => [entry.path, entry]))
|
|
if (files.length !== expected.size || files.some(file => !expected.has(file))) throw new Error('SOURCE_FILE_SET_CHANGED')
|
|
const entries = []
|
|
for (const relative of files) {
|
|
const bytes = fs.readFileSync(path.join(sourceDirectory, relative))
|
|
const sha256 = hash(bytes)
|
|
const original = expected.get(relative)
|
|
if (original.bytes !== bytes.length || original.sha256 !== sha256) throw new Error('SOURCE_BYTES_CHANGED')
|
|
const extension = path.extname(relative).toLowerCase()
|
|
if (!MEDIA[extension]) continue
|
|
const [kind, contentType] = MEDIA[extension]
|
|
entries.push({ sourcePath: relative, kind, contentType, bytes: bytes.length, sha256,
|
|
objectKey: `tang-detective/season-01/media-v1/${sha256}${extension}` })
|
|
}
|
|
if (!entries.length) throw new Error('NO_SOURCE_MEDIA')
|
|
return { sourceDirectory, sourceManifestSha256: hash(manifestBytes), entries }
|
|
}
|
|
|
|
function writeJson(filename, value) {
|
|
fs.mkdirSync(path.dirname(filename), { recursive: true })
|
|
const temp = `${filename}.${process.pid}.tmp`
|
|
fs.writeFileSync(temp, `${JSON.stringify(value, null, 2)}\n`, { flag: 'wx' })
|
|
fs.renameSync(temp, filename)
|
|
}
|
|
|
|
export function objectUrl(baseUrl, objectKey) {
|
|
if (!/^tang-detective\/season-01\/media-v1\/[a-f0-9]{64}\.[a-z0-9]+$/.test(objectKey)) {
|
|
throw new Error('UNSAFE_OBJECT_KEY')
|
|
}
|
|
const url = new URL(`${baseUrl}/${objectKey}`)
|
|
if (url.protocol !== 'https:' || url.username || url.password || url.search || url.hash) throw new Error('UNSAFE_OBJECT_URL')
|
|
return url.href
|
|
}
|
|
|
|
async function boundedBody(response, limit) {
|
|
let length = 0
|
|
const chunks = []
|
|
for await (const chunk of response.body) {
|
|
length += chunk.length
|
|
if (length > limit) throw new Error('REMOTE_BODY_TOO_LARGE')
|
|
chunks.push(chunk)
|
|
}
|
|
return Buffer.concat(chunks)
|
|
}
|
|
|
|
export async function verifyPublicObject(entry, fetchImpl = fetch) {
|
|
const response = await fetchImpl(entry.url, { redirect: 'error', signal: AbortSignal.timeout(20000) })
|
|
if (response.status !== 200) { await response.body?.cancel(); throw new Error('PUBLIC_MEDIA_GET_FAILED') }
|
|
const bytes = await boundedBody(response, entry.bytes)
|
|
if (bytes.length !== entry.bytes || hash(bytes) !== entry.sha256) throw new Error('PUBLIC_MEDIA_HASH_MISMATCH')
|
|
if (response.headers.get('content-type')?.split(';')[0].trim() !== entry.contentType) throw new Error('PUBLIC_MEDIA_TYPE_MISMATCH')
|
|
let rangeVerified = null
|
|
if (entry.kind === 'audio' || entry.kind === 'video') {
|
|
const last = Math.min(1023, entry.bytes - 1)
|
|
const range = await fetchImpl(entry.url, { headers: { Range: `bytes=0-${last}` }, redirect: 'error', signal: AbortSignal.timeout(20000) })
|
|
if (range.status !== 206 || range.headers.get('content-range') !== `bytes 0-${last}/${entry.bytes}`) {
|
|
await range.body?.cancel(); throw new Error('PUBLIC_MEDIA_RANGE_FAILED')
|
|
}
|
|
const fragment = await boundedBody(range, last + 1)
|
|
if (!fragment.equals(bytes.subarray(0, last + 1))) throw new Error('PUBLIC_MEDIA_RANGE_HASH_MISMATCH')
|
|
rangeVerified = true
|
|
}
|
|
return { remoteVerifiedSha256: hash(bytes), rangeVerified, verifiedAt: new Date().toISOString() }
|
|
}
|
|
|
|
export async function uploadObject(cos, destination, entry, body, onState = () => {}) {
|
|
const params = { Bucket: destination.bucket, Region: destination.region, Key: entry.objectKey }
|
|
let exists = false
|
|
try {
|
|
const head = await cos.headObject(params)
|
|
if (Number(head.headers?.['content-length']) !== entry.bytes || head.headers?.['x-cos-meta-sha256'] !== entry.sha256) {
|
|
throw new Error('EXISTING_OBJECT_NOT_OWNED_OR_DIFFERENT')
|
|
}
|
|
exists = true
|
|
} catch (error) {
|
|
if (Number(error.statusCode) !== 404) throw error
|
|
}
|
|
if (!exists) {
|
|
onState({ uploaded: 'unknown', uploadStatus: 'put-started-outcome-unknown' })
|
|
await cos.putObject({ ...params, Body: body, ContentLength: body.length, ContentType: entry.contentType,
|
|
CacheControl: 'public, max-age=31536000, immutable', Headers: {
|
|
'Content-MD5': crypto.createHash('md5').update(body).digest('base64'),
|
|
'x-cos-meta-sha256': entry.sha256,
|
|
'x-cos-forbid-overwrite': 'true',
|
|
} })
|
|
}
|
|
return { uploaded: true, uploadStatus: 'confirmed', action: exists ? 'reused-identical' : 'uploaded' }
|
|
}
|
|
|
|
export async function main(mode, dependencies = {}) {
|
|
if (!['inventory', 'inspect', 'upload'].includes(mode)) throw new Error('USE_INVENTORY_INSPECT_OR_UPLOAD')
|
|
const project = dependencies.project || PROJECT
|
|
const readConfig = dependencies.readConfig || readCosConfig
|
|
const verify = dependencies.verify || verifyPublicObject
|
|
const log = dependencies.log || (value => console.log(JSON.stringify(value)))
|
|
if (mode === 'upload') return runUpload({ project, readConfig, verify, log, createCos: dependencies.createCos })
|
|
const input = inventory(project)
|
|
const summary = { files: input.entries.length, uniqueObjects: new Set(input.entries.map(e => e.objectKey)).size,
|
|
bytes: input.entries.reduce((sum, entry) => sum + entry.bytes, 0),
|
|
kinds: Object.fromEntries(['image', 'audio', 'video'].map(kind => [kind, input.entries.filter(e => e.kind === kind).length])) }
|
|
if (mode === 'inventory') { log(summary); return }
|
|
const config = await readConfig(path.resolve(project, '../server'))
|
|
const destination = { bucket: config.bucket, region: config.region, baseUrl: config.baseUrl }
|
|
log({ ...summary, destination, credentialsPresent: true, databaseReadOnly: true, tlsVerified: true })
|
|
}
|
|
|
|
async function runUpload({ project, readConfig, verify, log, createCos }) {
|
|
const runId = crypto.randomUUID()
|
|
const receipt = { schemaVersion: 1, runId, startedAt: new Date().toISOString(), phase: 'inventory',
|
|
complete: false, bucketPermissionsChanged: false, originalFilesChanged: false, objects: [] }
|
|
const receiptPath = path.join(project, 'build/tang-detective-cos-upload-receipt.json')
|
|
const historyDirectory = path.join(project, 'build/tang-detective-cos-upload-attempts')
|
|
fs.mkdirSync(historyDirectory, { recursive: true })
|
|
if (fs.existsSync(receiptPath)) {
|
|
const previous = fs.readFileSync(receiptPath)
|
|
const archive = path.join(historyDirectory, `previous-${hash(previous)}.json`)
|
|
if (!fs.existsSync(archive)) fs.writeFileSync(archive, previous, { flag: 'wx' })
|
|
}
|
|
const persist = () => {
|
|
writeJson(path.join(historyDirectory, `${runId}.json`), receipt)
|
|
writeJson(receiptPath, receipt)
|
|
}
|
|
persist()
|
|
const completed = new Map()
|
|
try {
|
|
const input = inventory(project)
|
|
const summary = { files: input.entries.length, uniqueObjects: new Set(input.entries.map(e => e.objectKey)).size,
|
|
bytes: input.entries.reduce((sum, entry) => sum + entry.bytes, 0) }
|
|
receipt.sourceManifestSha256 = input.sourceManifestSha256
|
|
receipt.phase = 'configuration'
|
|
persist()
|
|
const config = await readConfig(path.resolve(project, '../server'))
|
|
const destination = { bucket: config.bucket, region: config.region, baseUrl: config.baseUrl }
|
|
receipt.destination = destination
|
|
receipt.phase = 'sdk-initialization'
|
|
persist()
|
|
const options = { SecretId: config.access_key, SecretKey: config.secret_key, Protocol: 'https:',
|
|
Timeout: 20000, MaxRetryTimes: 0, UploadCheckContentMd5: true }
|
|
const cos = createCos ? createCos(options) : new (dependency('cos-nodejs-sdk-v5'))(options)
|
|
log({ ...summary, destination, credentialsPresent: true, databaseReadOnly: true, tlsVerified: true })
|
|
receipt.phase = 'upload-and-verify'
|
|
// Sequential first-object verification stops immediately if this bucket/domain isn't anonymously readable.
|
|
for (const item of input.entries) {
|
|
if (completed.has(item.objectKey)) continue
|
|
const entry = { ...item, url: objectUrl(config.baseUrl, item.objectKey) }
|
|
const body = fs.readFileSync(path.join(input.sourceDirectory, item.sourcePath))
|
|
if (body.length !== item.bytes || hash(body) !== item.sha256) throw new Error('SOURCE_CHANGED_DURING_UPLOAD')
|
|
const result = { ...entry, uploaded: false, uploadStatus: 'not-attempted' }
|
|
receipt.objects.push(result)
|
|
persist()
|
|
try {
|
|
Object.assign(result, await uploadObject(cos, destination, entry, body, state => {
|
|
Object.assign(result, state)
|
|
persist()
|
|
}))
|
|
persist()
|
|
Object.assign(result, await verify(entry))
|
|
completed.set(item.objectKey, result)
|
|
persist()
|
|
} catch (error) {
|
|
result.error = safeError(error)
|
|
// A lost PUT response is not proof that nothing was uploaded. Read-only reconciliation can establish it.
|
|
if (result.uploaded === 'unknown') {
|
|
try {
|
|
const head = await cos.headObject({ Bucket: destination.bucket, Region: destination.region, Key: entry.objectKey })
|
|
if (Number(head.headers?.['content-length']) === entry.bytes && head.headers?.['x-cos-meta-sha256'] === entry.sha256) {
|
|
Object.assign(result, { uploaded: true, uploadStatus: 'confirmed-by-readback' }, await verify(entry))
|
|
}
|
|
} catch { /* Keep unknown; retry checks the exact content-addressed key without overwriting. */ }
|
|
}
|
|
throw error
|
|
}
|
|
log({ verified: completed.size, total: summary.uniqueObjects })
|
|
}
|
|
// Recheck all source bytes immediately before publishing the build activation manifest.
|
|
if (inventory(project).sourceManifestSha256 !== input.sourceManifestSha256) throw new Error('SOURCE_MANIFEST_CHANGED')
|
|
const entries = input.entries.map(entry => ({ ...entry, url: completed.get(entry.objectKey).url, uploaded: true,
|
|
remoteVerifiedSha256: completed.get(entry.objectKey).remoteVerifiedSha256,
|
|
rangeVerified: completed.get(entry.objectKey).rangeVerified,
|
|
verifiedAt: completed.get(entry.objectKey).verifiedAt }))
|
|
writeJson(path.join(project, 'build/tang-detective-cos-manifest.json'), {
|
|
schemaVersion: 1, sourceManifestSha256: input.sourceManifestSha256, destination, entries,
|
|
})
|
|
receipt.complete = true
|
|
receipt.phase = 'complete'
|
|
receipt.completedAt = new Date().toISOString()
|
|
log({ complete: true, ...summary })
|
|
} catch (error) {
|
|
receipt.error = safeError(error)
|
|
receipt.failedAt = new Date().toISOString()
|
|
throw error
|
|
} finally {
|
|
persist()
|
|
}
|
|
}
|
|
|
|
if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
|
|
main(process.argv[2]).catch(error => { console.error(JSON.stringify({ complete: false, error: safeError(error) })); process.exitCode = 1 })
|
|
}
|