~repos /only-bible-app

const https = require("https");
const fs = require("fs");
const path = require("path");
const cheerio = require("cheerio");

// Wordproject uses its own book numbering (1-66 maps to standard Protestant canon)
// bookId = 0-based index in ja.txt, wpBook = wordproject book number (1-66)
const BOOKS = [
  { bookId: 0, wpBook: 1, chapters: 50, jaName: "創世記" },
  { bookId: 1, wpBook: 2, chapters: 40, jaName: "出エジプト記" },
  { bookId: 2, wpBook: 3, chapters: 27, jaName: "レビ記" },
  { bookId: 3, wpBook: 4, chapters: 36, jaName: "民数記" },
  { bookId: 4, wpBook: 5, chapters: 34, jaName: "申命記" },
  { bookId: 5, wpBook: 6, chapters: 24, jaName: "ヨシュア記" },
  { bookId: 6, wpBook: 7, chapters: 21, jaName: "士師記" },
  { bookId: 7, wpBook: 8, chapters: 4, jaName: "ルツ記" },
  { bookId: 8, wpBook: 9, chapters: 31, jaName: "サムエル記上" },
  { bookId: 9, wpBook: 10, chapters: 24, jaName: "サムエル記下" },
  { bookId: 10, wpBook: 11, chapters: 22, jaName: "列王紀上" },
  { bookId: 11, wpBook: 12, chapters: 25, jaName: "列王紀下" },
  { bookId: 12, wpBook: 13, chapters: 29, jaName: "歴代志上" },
  { bookId: 13, wpBook: 14, chapters: 36, jaName: "歴代志下" },
  { bookId: 14, wpBook: 15, chapters: 10, jaName: "エズラ記" },
  { bookId: 15, wpBook: 16, chapters: 13, jaName: "ネヘミヤ記" },
  { bookId: 16, wpBook: 17, chapters: 10, jaName: "エステル記" },
  { bookId: 17, wpBook: 18, chapters: 42, jaName: "ヨブ記" },
  { bookId: 18, wpBook: 19, chapters: 150, jaName: "詩篇" },
  { bookId: 19, wpBook: 20, chapters: 31, jaName: "箴言" },
  { bookId: 20, wpBook: 21, chapters: 12, jaName: "伝道の書" },
  { bookId: 21, wpBook: 22, chapters: 8, jaName: "雅歌" },
  { bookId: 22, wpBook: 23, chapters: 66, jaName: "イザヤ書" },
  { bookId: 23, wpBook: 24, chapters: 52, jaName: "エレミヤ書" },
  { bookId: 24, wpBook: 25, chapters: 5, jaName: "哀歌" },
  { bookId: 25, wpBook: 26, chapters: 48, jaName: "エゼキエル書" },
  { bookId: 26, wpBook: 27, chapters: 12, jaName: "ダニエル書" },
  { bookId: 27, wpBook: 28, chapters: 14, jaName: "ホセア書" },
  { bookId: 28, wpBook: 29, chapters: 3, jaName: "ヨエル書" },
  { bookId: 29, wpBook: 30, chapters: 9, jaName: "アモス書" },
  { bookId: 30, wpBook: 31, chapters: 1, jaName: "オバデヤ書" },
  { bookId: 31, wpBook: 32, chapters: 4, jaName: "ヨナ書" },
  { bookId: 32, wpBook: 33, chapters: 7, jaName: "ミカ書" },
  { bookId: 33, wpBook: 34, chapters: 3, jaName: "ナホム書" },
  { bookId: 34, wpBook: 35, chapters: 3, jaName: "ハバクク書" },
  { bookId: 35, wpBook: 36, chapters: 3, jaName: "ゼパニヤ書" },
  { bookId: 36, wpBook: 37, chapters: 2, jaName: "ハガイ書" },
  { bookId: 37, wpBook: 38, chapters: 14, jaName: "ゼカリヤ書" },
  { bookId: 38, wpBook: 39, chapters: 4, jaName: "マラキ書" },
  { bookId: 39, wpBook: 40, chapters: 28, jaName: "マタイによる福音書" },
  { bookId: 40, wpBook: 41, chapters: 16, jaName: "マルコによる福音書" },
  { bookId: 41, wpBook: 42, chapters: 24, jaName: "ルカによる福音書" },
  { bookId: 42, wpBook: 43, chapters: 21, jaName: "ヨハネによる福音書" },
  { bookId: 43, wpBook: 44, chapters: 28, jaName: "使徒行伝" },
  { bookId: 44, wpBook: 45, chapters: 16, jaName: "ローマ人への手紙" },
  { bookId: 45, wpBook: 46, chapters: 16, jaName: "コリント人への第一の手紙" },
  { bookId: 46, wpBook: 47, chapters: 13, jaName: "コリント人への第二の手紙" },
  { bookId: 47, wpBook: 48, chapters: 6, jaName: "ガラテヤ人への手紙" },
  { bookId: 48, wpBook: 49, chapters: 6, jaName: "エペソ人への手紙" },
  { bookId: 49, wpBook: 50, chapters: 4, jaName: "ピリピ人への手紙" },
  { bookId: 50, wpBook: 51, chapters: 4, jaName: "コロサイ人への手紙" },
  { bookId: 51, wpBook: 52, chapters: 5, jaName: "テサロニケ人への第一の手紙" },
  { bookId: 52, wpBook: 53, chapters: 3, jaName: "テサロニケ人への第二の手紙" },
  { bookId: 53, wpBook: 54, chapters: 6, jaName: "テモテへの第一の手紙" },
  { bookId: 54, wpBook: 55, chapters: 4, jaName: "テモテへの第二の手紙" },
  { bookId: 55, wpBook: 56, chapters: 3, jaName: "テトスへの手紙" },
  { bookId: 56, wpBook: 57, chapters: 1, jaName: "ピレモンへの手紙" },
  { bookId: 57, wpBook: 58, chapters: 13, jaName: "ヘブル人への手紙" },
  { bookId: 58, wpBook: 59, chapters: 5, jaName: "ヤコブの手紙" },
  { bookId: 59, wpBook: 60, chapters: 5, jaName: "ペテロの第一の手紙" },
  { bookId: 60, wpBook: 61, chapters: 3, jaName: "ペテロの第二の手紙" },
  { bookId: 61, wpBook: 62, chapters: 5, jaName: "ヨハネの第一の手紙" },
  { bookId: 62, wpBook: 63, chapters: 1, jaName: "ヨハネの第二の手紙" },
  { bookId: 63, wpBook: 64, chapters: 1, jaName: "ヨハネの第三の手紙" },
  { bookId: 64, wpBook: 65, chapters: 1, jaName: "ユダの手紙" },
  { bookId: 65, wpBook: 66, chapters: 22, jaName: "ヨハネの黙示録" },
];

function fetchPage(url) {
  return new Promise((resolve, reject) => {
    const req = https.get(url, (res) => {
      if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
        fetchPage(res.headers.location).then(resolve).catch(reject);
        return;
      }
      const chunks = [];
      res.on("data", (chunk) => chunks.push(chunk));
      res.on("end", () => resolve(Buffer.concat(chunks).toString("utf8")));
      res.on("error", reject);
    });
    req.on("error", reject);
    req.setTimeout(15000, () => { req.destroy(); reject(new Error("timeout")); });
  });
}

function parseVerses(html) {
  const $ = cheerio.load(html);

  // Remove chapter navigation links and dimver spans
  $("p.ym-noprint").remove();
  $(".dimver").remove();

  // Get all <p> content after <h3>
  const h3 = $("h3").first();
  if (!h3.length) return [];

  let contentHtml = "";
  h3.nextAll("p").each((_, el) => {
    contentHtml += $.html(el);
  });

  if (!contentHtml) return [];

  // Find all verse spans: <span class="verse" id="N">N </span>
  const spanRegex = /<span[^>]*class="verse"[^>]*id="(\d+)"[^>]*>\s*\d+\s*<\/span>/g;
  const allSpans = [...contentHtml.matchAll(spanRegex)];

  const verses = [];

  if (allSpans.length > 0) {
    // Verse 1 text = everything from start of content to first verse span
    const v1Html = contentHtml.substring(0, allSpans[0].index);
    const v1Text = cheerio.load(v1Html).text().replace(/\s+/g, " ").trim();
    if (v1Text) verses.push(v1Text);

    // Remaining verses: text between consecutive spans
    for (let i = 0; i < allSpans.length; i++) {
      const spanEnd = allSpans[i].index + allSpans[i][0].length;
      const nextStart =
        i + 1 < allSpans.length ? allSpans[i + 1].index : contentHtml.length;
      const segHtml = contentHtml.substring(spanEnd, nextStart);
      const segText = cheerio.load(segHtml).text().replace(/\s+/g, " ").trim();
      verses.push(segText);
    }
  } else {
    // No verse spans at all — just get all text
    const allText = cheerio.load(contentHtml).text().replace(/\s+/g, " ").trim();
    if (allText) verses.push(allText);
  }

  return verses;
}

// Load en_kjv.txt to get headings and references
function loadHeadingsAndRefs(enFile) {
  const lines = fs.readFileSync(enFile, "utf-8").split("\n");
  const map = {};
  for (const line of lines) {
    if (!line.trim()) continue;
    const parts = line.split("|");
    if (parts.length < 7) continue;
    const [, bookId, chIdx, verseIdx, heading, refs] = parts;
    const key = `${bookId}|${chIdx}|${verseIdx}`;
    map[key] = { heading: heading || "", refs: refs || "" };
  }
  return map;
}

function delay(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

async function main() {
  const enFile = path.join(__dirname, "files", "en_kjv.txt");
  const outFile = path.join(__dirname, "files", "ja_new.txt");
  const headingMap = loadHeadingsAndRefs(enFile);

  const allLines = [];
  let totalVerses = 0;
  let totalChapters = 0;
  const issues = [];

  for (const book of BOOKS) {
    console.error(`\n=== ${book.jaName} (${book.chapters} chapters) ===`);

    for (let ch = 1; ch <= book.chapters; ch++) {
      const chIdx = ch - 1; // 0-based
      const wpBookStr = String(book.wpBook).padStart(2, "0");
      const url = `https://www.wordproject.org/bibles/jp/${wpBookStr}/${ch}.htm`;

      let retries = 3;
      let verses = [];
      while (retries > 0) {
        try {
          const html = await fetchPage(url);
          verses = parseVerses(html);
          break;
        } catch (err) {
          retries--;
          if (retries > 0) {
            console.error(`  Retry ${book.jaName} ch${ch}: ${err.message}`);
            await delay(2000);
          } else {
            console.error(`  FAILED ${book.jaName} ch${ch}: ${err.message}`);
            issues.push(`FAILED: ${book.jaName} ch${ch}`);
          }
        }
      }

      // Check expected verse count from en_kjv
      const enKey = `${book.bookId}|${chIdx}`;
      const expectedCount = Object.keys(headingMap).filter((k) =>
        k.startsWith(enKey + "|")
      ).length;

      if (verses.length !== expectedCount) {
        console.error(
          `  ${book.jaName} ch${ch}: got ${verses.length} verses, expected ${expectedCount}`
        );
        issues.push(
          `MISMATCH: ${book.jaName} ch${ch}: got ${verses.length}, expected ${expectedCount}`
        );
      }

      for (let i = 0; i < verses.length; i++) {
        const key = `${book.bookId}|${chIdx}|${i}`;
        const meta = headingMap[key] || { heading: "", refs: "" };
        const line = `${book.jaName}|${book.bookId}|${chIdx}|${i}|${meta.heading}|${meta.refs}|${verses[i]}`;
        allLines.push(line);
      }

      totalVerses += verses.length;
      totalChapters++;

      // Small delay to be respectful to the server
      if (totalChapters % 10 === 0) {
        await delay(200);
      }
    }

    console.error(`  Done. Total so far: ${totalVerses} verses in ${totalChapters} chapters`);
  }

  // Write output
  fs.writeFileSync(outFile, allLines.join("\n") + "\n");

  console.error(`\n=== COMPLETE ===`);
  console.error(`Total chapters: ${totalChapters}`);
  console.error(`Total verses/lines: ${totalVerses}`);
  console.error(`Output: ${outFile}`);

  if (issues.length > 0) {
    console.error(`\n=== ISSUES (${issues.length}) ===`);
    for (const issue of issues) {
      console.error(`  ${issue}`);
    }
  }
}

main().catch((err) => {
  console.error("Fatal error:", err);
  process.exit(1);
});