website

#!/usr/bin/env node

// Asks public web archives to snapshot the key pages of the site.

//

// Providers:

//  - Wayback Machine (web.archive.org) via its "Save Page Now" endpoint.

//    Set ARCHIVE_ORG_ACCESS_KEY / ARCHIVE_ORG_SECRET_KEY (from

//    https://archive.org/account/s3.php) to use the authenticated SPN2 API,

//    which is less aggressively rate-limited and reports a job status.

//    Without them, falls back to the unauthenticated /save/<url> endpoint.

//  - archive.today (archive.ph) via its public /submit/ form endpoint.

//    There's no official API; archive.today occasionally interposes a

//    captcha/"please wait" page under load, in which case the submission

//    just gets logged as needing a manual follow-up.

//

// Usage:

//   node scripts/submit-to-archives.js [--provider=wayback|archivetoday|all] [--dry-run] [--url=/some/path/]

import { readdirSync } from "node:fs";

import path from "node:path";

import { fileURLToPath } from "node:url";

import { setTimeout as sleep } from "node:timers/promises";

const __dirname = path.dirname(fileURLToPath(import.meta.url));

const SITE_URL = "https://pyrossh.dev";

const DELAY_BETWEEN_REQUESTS_MS = 4000;

function parseArgs(argv) {

  const args = { provider: "all", dryRun: false, urls: [] };

  for (const arg of argv) {

    if (arg === "--dry-run") args.dryRun = true;

    else if (arg.startsWith("--provider=")) args.provider = arg.slice("--provider=".length);

    else if (arg.startsWith("--url=")) args.urls.push(arg.slice("--url=".length));

  }

  return args;

}

function discoverPostSlugs() {

  const postsDir = path.join(__dirname, "..", "src", "posts");

  return readdirSync(postsDir)

    .filter((file) => file.endsWith(".md"))

    .map((file) => file.replace(/\.md$/, ""));

}

function discoverUrls() {

  const staticPaths = [

    "/",

    "/cv/",

    "/posts/",

    "/only-bible-app/",

    "/only-bible-app/privacy-policy/",

    "/only-bible-app/terms-and-conditions/",

  ];

  const postPaths = discoverPostSlugs().map((slug) => `/posts/${slug}/`);

  return [...staticPaths, ...postPaths].map((p) => `${SITE_URL}${p}`);

}

async function submitToWaybackMachine(url) {

  const accessKey = process.env.ARCHIVE_ORG_ACCESS_KEY;

  const secretKey = process.env.ARCHIVE_ORG_SECRET_KEY;

  if (accessKey && secretKey) {

    const response = await fetch("https://web.archive.org/save", {

      method: "POST",

      headers: {

        Authorization: `LOW ${accessKey}:${secretKey}`,

        "content-type": "application/x-www-form-urlencoded",

        Accept: "application/json",

      },

      body: new URLSearchParams({ url, skip_first_archive: "1" }),

    });

    const body = await response.json().catch(() => ({}));

    return { ok: response.ok, status: response.status, detail: body.job_id ? `job ${body.job_id}` : JSON.stringify(body) };

  }

  const response = await fetch(`https://web.archive.org/save/${url}`, { method: "GET", redirect: "follow" });

  return { ok: response.ok, status: response.status, detail: response.url };

}

async function submitToArchiveToday(url) {

  const response = await fetch("https://archive.ph/submit/", {

    method: "POST",

    headers: {

      "content-type": "application/x-www-form-urlencoded",

      // archive.today's anti-bot layer rejects requests without a

      // browser-like User-Agent outright, independent of any real rate limit.

      "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",

    },

    body: new URLSearchParams({ url, anyway: "1" }),

    redirect: "manual",

  });

  const location = response.headers.get("location");

  const isRedirect = response.status >= 300 && response.status < 400;

  if (isRedirect) return { ok: true, status: response.status, detail: location };

  const body = await response.text().catch(() => "");

  if (/captcha/i.test(body)) {

    return { ok: false, status: response.status, detail: "blocked by archive.today's CAPTCHA/anti-bot check — submit this URL manually at https://archive.ph/" };

  }

  return { ok: response.ok, status: response.status, detail: "submitted — check https://archive.ph/ manually for the resulting snapshot" };

}

const PROVIDERS = {

  wayback: { label: "Wayback Machine", submit: submitToWaybackMachine },

  archivetoday: { label: "archive.today", submit: submitToArchiveToday },

};

async function main() {

  const args = parseArgs(process.argv.slice(2));

  const urls = args.urls.length ? args.urls.map((u) => (u.startsWith("http") ? u : `${SITE_URL}${u}`)) : discoverUrls();

  const providerNames = args.provider === "all" ? Object.keys(PROVIDERS) : [args.provider];

  for (const name of providerNames) {

    if (!PROVIDERS[name]) throw new Error(`Unknown provider "${name}". Expected one of: wayback, archivetoday, all`);

  }

  console.log(`Submitting ${urls.length} URL(s) to: ${providerNames.map((n) => PROVIDERS[n].label).join(", ")}`);

  if (args.dryRun) {

    for (const url of urls) console.log(`[dry-run] ${url}`);

    return;

  }

  for (const name of providerNames) {

    const provider = PROVIDERS[name];

    console.log(`\n--- ${provider.label} ---`);

    for (const [index, url] of urls.entries()) {

      try {

        const result = await provider.submit(url);

        const icon = result.ok ? "✔" : "✖";

        console.log(`${icon} [${result.status}] ${url} — ${result.detail}`);

      } catch (err) {

        console.log(`✖ [error] ${url} — ${err.message}`);

      }

      if (index < urls.length - 1) await sleep(DELAY_BETWEEN_REQUESTS_MS);

    }

  }

}

main().catch((err) => {

  console.error(err.message);

  process.exit(1);

});