website

#!/usr/bin/env node
// Asks public web archives to snapshot the key pages of the site.
//
// Providers:
//  - Wayback Machine (web.archive.org) via its "Save Page Now" endpoint.
//    Set ARCHIVE_ORG_ACCESS_KEY / ARCHIVE_ORG_SECRET_KEY (from
//    https://archive.org/account/s3.php) to use the authenticated SPN2 API,
//    which is less aggressively rate-limited and reports a job status.
//    Without them, falls back to the unauthenticated /save/<url> endpoint.
//  - archive.today (archive.ph) via its public /submit/ form endpoint.
//    There's no official API; archive.today occasionally interposes a
//    captcha/"please wait" page under load, in which case the submission
//    just gets logged as needing a manual follow-up.
//
// Usage:
//   node scripts/submit-to-archives.js [--provider=wayback|archivetoday|all] [--dry-run] [--url=/some/path/]

import { readdirSync } from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { setTimeout as sleep } from "node:timers/promises";

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const SITE_URL = "https://pyrossh.dev";
const DELAY_BETWEEN_REQUESTS_MS = 4000;

function parseArgs(argv) {
  const args = { provider: "all", dryRun: false, urls: [] };
  for (const arg of argv) {
    if (arg === "--dry-run") args.dryRun = true;
    else if (arg.startsWith("--provider=")) args.provider = arg.slice("--provider=".length);
    else if (arg.startsWith("--url=")) args.urls.push(arg.slice("--url=".length));
  }
  return args;
}

function discoverPostSlugs() {
  const postsDir = path.join(__dirname, "..", "src", "posts");
  return readdirSync(postsDir)
    .filter((file) => file.endsWith(".md"))
    .map((file) => file.replace(/\.md$/, ""));
}

function discoverUrls() {
  const staticPaths = [
    "/",
    "/cv/",
    "/posts/",
    "/only-bible-app/",
    "/only-bible-app/privacy-policy/",
    "/only-bible-app/terms-and-conditions/",
  ];
  const postPaths = discoverPostSlugs().map((slug) => `/posts/${slug}/`);
  return [...staticPaths, ...postPaths].map((p) => `${SITE_URL}${p}`);
}

async function submitToWaybackMachine(url) {
  const accessKey = process.env.ARCHIVE_ORG_ACCESS_KEY;
  const secretKey = process.env.ARCHIVE_ORG_SECRET_KEY;

  if (accessKey && secretKey) {
    const response = await fetch("https://web.archive.org/save", {
      method: "POST",
      headers: {
        Authorization: `LOW ${accessKey}:${secretKey}`,
        "content-type": "application/x-www-form-urlencoded",
        Accept: "application/json",
      },
      body: new URLSearchParams({ url, skip_first_archive: "1" }),
    });
    const body = await response.json().catch(() => ({}));
    return { ok: response.ok, status: response.status, detail: body.job_id ? `job ${body.job_id}` : JSON.stringify(body) };
  }

  const response = await fetch(`https://web.archive.org/save/${url}`, { method: "GET", redirect: "follow" });
  return { ok: response.ok, status: response.status, detail: response.url };
}

async function submitToArchiveToday(url) {
  const response = await fetch("https://archive.ph/submit/", {
    method: "POST",
    headers: {
      "content-type": "application/x-www-form-urlencoded",
      // archive.today's anti-bot layer rejects requests without a
      // browser-like User-Agent outright, independent of any real rate limit.
      "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",
    },
    body: new URLSearchParams({ url, anyway: "1" }),
    redirect: "manual",
  });
  const location = response.headers.get("location");
  const isRedirect = response.status >= 300 && response.status < 400;
  if (isRedirect) return { ok: true, status: response.status, detail: location };

  const body = await response.text().catch(() => "");
  if (/captcha/i.test(body)) {
    return { ok: false, status: response.status, detail: "blocked by archive.today's CAPTCHA/anti-bot check — submit this URL manually at https://archive.ph/" };
  }
  return { ok: response.ok, status: response.status, detail: "submitted — check https://archive.ph/ manually for the resulting snapshot" };
}

const PROVIDERS = {
  wayback: { label: "Wayback Machine", submit: submitToWaybackMachine },
  archivetoday: { label: "archive.today", submit: submitToArchiveToday },
};

async function main() {
  const args = parseArgs(process.argv.slice(2));
  const urls = args.urls.length ? args.urls.map((u) => (u.startsWith("http") ? u : `${SITE_URL}${u}`)) : discoverUrls();
  const providerNames = args.provider === "all" ? Object.keys(PROVIDERS) : [args.provider];

  for (const name of providerNames) {
    if (!PROVIDERS[name]) throw new Error(`Unknown provider "${name}". Expected one of: wayback, archivetoday, all`);
  }

  console.log(`Submitting ${urls.length} URL(s) to: ${providerNames.map((n) => PROVIDERS[n].label).join(", ")}`);
  if (args.dryRun) {
    for (const url of urls) console.log(`[dry-run] ${url}`);
    return;
  }

  for (const name of providerNames) {
    const provider = PROVIDERS[name];
    console.log(`\n--- ${provider.label} ---`);
    for (const [index, url] of urls.entries()) {
      try {
        const result = await provider.submit(url);
        const icon = result.ok ? "✔" : "✖";
        console.log(`${icon} [${result.status}] ${url} — ${result.detail}`);
      } catch (err) {
        console.log(`✖ [error] ${url} — ${err.message}`);
      }
      if (index < urls.length - 1) await sleep(DELAY_BETWEEN_REQUESTS_MS);
    }
  }
}

main().catch((err) => {
  console.error(err.message);
  process.exit(1);
});