Migrate Val Town blobs

import { blob } from "https://esm.town/v/std/blob/main.ts";

const PREFIX = "uploads/";

export default async function (request: Request): Promise<Response> {
  const token = Deno.env.get("BLOB_EXPORT_TOKEN");
  if (!token || request.headers.get("authorization") !== `Bearer ${token}`) {
    return new Response("Unauthorized", { status: 401 });
  }
  if (request.method !== "GET") return new Response(null, { status: 405 });

  const url = new URL(request.url);
  if (url.searchParams.has("list")) {
    const entries = await blob.list(PREFIX);
    return Response.json(
      { prefix: PREFIX, keys: entries.map((entry) => entry.key) },
      { headers: { "cache-control": "no-store" } },
    );
  }

  const key = url.searchParams.get("key") ?? "";
  if (!key.startsWith(PREFIX)) return new Response("Not found", { status: 404 });

  try {
    const result = await blob.get(key);
    return new Response(result.body, {
      headers: {
        "content-type": result.headers.get("content-type") ?? "application/octet-stream",
        "cache-control": "no-store",
      },
    });
  } catch {
    return new Response("Not found", { status: 404 });
  }
}

import { createHash } from "node:crypto";
import { appendFileSync } from "node:fs";
import {
  DeleteObjectCommand,
  GetObjectCommand,
  ListObjectsV2Command,
  PutObjectCommand,
  S3Client,
} from "@aws-sdk/client-s3";

const final = process.argv.includes("--final");
const apply = process.argv.includes("--apply");
const env = (name) => {
  const value = process.env[name];
  if (!value) throw new Error(`${name} is required`);
  return value;
};

const exportUrl = new URL(env("BLOB_EXPORT_URL"));
if (exportUrl.protocol !== "https:") throw new Error("BLOB_EXPORT_URL must be HTTPS");
const auth = { authorization: `Bearer ${env("BLOB_EXPORT_TOKEN")}` };
const bucket = env("S3_BUCKET");
const prefix = env("BLOB_PREFIX");
const s3 = new S3Client({
  region: process.env.S3_REGION ?? "auto",
  endpoint: process.env.S3_ENDPOINT,
  credentials: {
    accessKeyId: env("S3_ACCESS_KEY_ID"),
    secretAccessKey: env("S3_SECRET_ACCESS_KEY"),
  },
});

const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex");

async function exported(params) {
  const url = new URL(exportUrl);
  for (const [name, value] of Object.entries(params)) url.searchParams.set(name, value);
  const response = await fetch(url, { headers: auth, redirect: "error" });
  if (!response.ok) throw new Error(`Export failed for ${url.search}: ${response.status}`);
  return response;
}

async function readDestination(key) {
  try {
    const object = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
    return Buffer.from(await object.Body.transformToByteArray());
  } catch (error) {
    if (error.name === "NoSuchKey") return null;
    throw error;
  }
}

async function listDestination() {
  const keys = [];
  let token;
  do {
    const page = await s3.send(
      new ListObjectsV2Command({ Bucket: bucket, Prefix: prefix, ContinuationToken: token }),
    );
    for (const { Key } of page.Contents ?? []) keys.push(Key);
    token = page.NextContinuationToken;
  } while (token);
  return keys;
}

const manifest = await (await exported({ list: "1" })).json();
if (manifest.prefix !== prefix) {
  throw new Error(`The export serves "${manifest.prefix}", but BLOB_PREFIX is "${prefix}"`);
}
const keys = manifest.keys;
if (keys.length === 0) throw new Error("The export listed no keys; check the store and prefix");

for (const key of keys) {
  const response = await exported({ key });
  const bytes = Buffer.from(await response.arrayBuffer());
  const hash = sha256(bytes);

  const existing = await readDestination(key);
  if (existing && sha256(existing) === hash) continue;
  if (existing && !final) {
    throw new Error(
      `${key} changed on Val Town since the last copy. Freeze writes on Val Town, then run with --final.`,
    );
  }

  await s3.send(
    new PutObjectCommand({
      Bucket: bucket,
      Key: key,
      Body: bytes,
      ContentType: response.headers.get("content-type") ?? undefined,
      ...(existing ? {} : { IfNoneMatch: "*" }),
    }),
  );
  const copied = await readDestination(key);
  if (!copied || sha256(copied) !== hash) throw new Error(`Verification failed for ${key}`);

  appendFileSync("verified.jsonl", `${JSON.stringify({ key, sha256: hash })}\n`, { mode: 0o600 });
  console.log(`${existing ? "Updated" : "Copied"} ${key}`);
}

if (final) {
  const wanted = new Set(keys);
  const extra = (await listDestination()).filter((key) => !wanted.has(key));
  console.log(`Bucket ${bucket}, prefix ${prefix}: ${keys.length} objects on Val Town`);
  console.log(`Objects to delete: ${extra.length}`);
  for (const key of extra) console.log(`  ${key}`);
  if (!apply) {
    console.log("Nothing deleted. Rerun with --final --apply to delete them.");
  } else {
    for (const key of extra) {
      await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key }));
      console.log(`Deleted ${key}`);
    }
  }
}