ブラウザーでsitemapインデックスからURLを収集する

(async () => {
  const MAX_SITEMAPS = 500;
  const MAX_URLS = 1000000;
  const MAX_SITEMAP_BYTES = 25 * 1024 * 1024;
  const MAX_RUNTIME_MS = 180000;
  const FETCH_TIMEOUT_MS = 20000;
  const allowedOrigin = window.location.origin;
  const startedAt = Date.now();
  const deadlineAt = startedAt + MAX_RUNTIME_MS;
  const queue = [];
  const queued = new Set();
  const visited = new Set();
  const pages = new Map();
  const errors = [];
  const limits = { sitemaps: false, urls: false, fileSize: false, runtime: false };

  function enqueue(value, base) {
    try {
      const url = new URL(value, base);
      if (!/^https?:$/.test(url.protocol) || url.username || url.password || queued.has(url.href)) return;
      if (queued.size >= MAX_SITEMAPS) {
        limits.sitemaps = true;
        return;
      }
      queued.add(url.href);
      queue.push(url.href);
    } catch {
      // Ignore malformed sitemap locations.
    }
  }

  async function fetchText(url) {
    if (Date.now() >= deadlineAt) {
      limits.runtime = true;
      throw new Error('Runtime limit reached');
    }
    const controller = new AbortController();
    const timeout = setTimeout(() => controller.abort(), Math.min(FETCH_TIMEOUT_MS, deadlineAt - Date.now()));
    try {
      const response = await fetch(url, { credentials: 'omit', redirect: 'follow', signal: controller.signal });
      if (!response.ok) throw new Error('HTTP ' + response.status);
      if (!response.body) {
        const text = await response.text();
        if (new TextEncoder().encode(text).byteLength > MAX_SITEMAP_BYTES) {
          limits.fileSize = true;
          throw new Error('Sitemap exceeds the 25 MB file limit');
        }
        return text;
      }
      const sourceReader = response.body.getReader();
      const prefixChunks = [];
      let prefixBytes = 0;
      let firstChunk;
      while (prefixBytes < 2) {
        firstChunk = await sourceReader.read();
        if (firstChunk.done) break;
        prefixChunks.push(firstChunk.value);
        prefixBytes += firstChunk.value.byteLength;
      }
      let bodyStream = new ReadableStream({
        start(controller) {
          if (firstChunk?.done) controller.close();
          else prefixChunks.forEach((chunk) => controller.enqueue(chunk));
        },
        async pull(controller) {
          const { done, value } = await sourceReader.read();
          if (done) controller.close();
          else controller.enqueue(value);
        },
        cancel(reason) {
          return sourceReader.cancel(reason);
        },
      });
      const firstBytes = prefixChunks.length ? new Uint8Array(prefixChunks.reduce((size, chunk) => size + chunk.length, 0)) : null;
      if (firstBytes) {
        let offset = 0;
        for (const chunk of prefixChunks) {
          firstBytes.set(chunk, offset);
          offset += chunk.length;
        }
      }
      const isGzipFile = firstBytes?.length >= 2 && firstBytes[0] === 0x1f && firstBytes[1] === 0x8b;
      if (isGzipFile) {
        if (!window.DecompressionStream) {
          await sourceReader.cancel();
          throw new Error('This browser cannot decompress gzip sitemap files');
        }
        bodyStream = bodyStream.pipeThrough(new window.DecompressionStream('gzip'));
      }
      const reader = bodyStream.getReader();
      const chunks = [];
      let bytes = 0;
      while (true) {
        if (Date.now() >= deadlineAt) {
          limits.runtime = true;
          await reader.cancel();
          throw new Error('Runtime limit reached');
        }
        const { done, value } = await reader.read();
        if (done) break;
        bytes += value.byteLength;
        if (bytes > MAX_SITEMAP_BYTES) {
          limits.fileSize = true;
          await reader.cancel();
          throw new Error('Sitemap exceeds the 25 MB file limit');
        }
        chunks.push(value);
      }
      return await new Blob(chunks).text();
    } finally {
      clearTimeout(timeout);
    }
  }

  let robotsSitemaps = [];
  try {
    const robots = await fetchText(new URL('/robots.txt', allowedOrigin).href);
    robotsSitemaps = robots.split(/\r?\n/)
      .map((line) => line.match(/^\s*sitemap:\s*(\S+)/i)?.[1])
      .filter(Boolean);
  } catch (error) {
    errors.push('/robots.txt: ' + error.message);
  }

  robotsSitemaps.forEach((url) => enqueue(url, allowedOrigin));
  if (!queue.length) enqueue('/sitemap.xml', allowedOrigin);

  while (queue.length && visited.size < MAX_SITEMAPS && pages.size < MAX_URLS && Date.now() < deadlineAt) {
    const sitemapUrl = queue.shift();
    if (visited.has(sitemapUrl)) continue;
    visited.add(sitemapUrl);
    try {
      const xml = new DOMParser().parseFromString(await fetchText(sitemapUrl), 'application/xml');
      if (xml.querySelector('parsererror')) throw new Error('Invalid XML');
      const rootName = xml.documentElement.localName.toLowerCase();
      if (rootName === 'sitemapindex') {
        for (const node of xml.getElementsByTagNameNS('*', 'sitemap')) {
          if (Date.now() >= deadlineAt) {
            limits.runtime = true;
            break;
          }
          const loc = node.getElementsByTagNameNS('*', 'loc')[0]?.textContent?.trim();
          if (loc) enqueue(loc, sitemapUrl);
        }
      } else if (rootName === 'urlset') {
        for (const node of xml.getElementsByTagNameNS('*', 'url')) {
          if (Date.now() >= deadlineAt) {
            limits.runtime = true;
            break;
          }
          const loc = node.getElementsByTagNameNS('*', 'loc')[0]?.textContent?.trim();
          if (!loc) continue;
          try {
            const url = new URL(loc, sitemapUrl);
            if (!/^https?:$/.test(url.protocol)) continue;
            const lastmod = node.getElementsByTagNameNS('*', 'lastmod')[0]?.textContent?.trim() || '';
            if (!pages.has(url.href) || lastmod) pages.set(url.href, { url: url.href, lastmod });
          } catch {
            // Ignore malformed page URLs.
          }
          if (pages.size >= MAX_URLS) {
            limits.urls = true;
            break;
          }
        }
      } else {
        throw new Error('Expected a sitemap index or URL set');
      }
    } catch (error) {
      errors.push(sitemapUrl + ': ' + error.message);
    }
    console.info('Sitemaps: ' + visited.size + '; URLs: ' + pages.size);
  }

  if (!pages.size) {
    console.warn('No sitemap URLs collected. Check that robots.txt and the sitemap are readable from this page.');
    if (errors.length) console.table(errors.map((message) => ({ error: message })));
    return;
  }

  const quote = (value) => {
    const text = String(value);
    const safeText = /^[\u0000-\u0020]*[=+\-@]/.test(text) ? "'" + text : text;
    return '"' + safeText.replace(/"/g, '""') + '"';
  };
  const rows = [['URL', 'Last modified'], ...Array.from(pages.values(), ({ url, lastmod }) => [url, lastmod])];
  const csv = '\uFEFF' + rows.map((row) => row.map(quote).join(',')).join('\r\n');
  const blobUrl = URL.createObjectURL(new Blob([csv], { type: 'text/csv;charset=utf-8' }));
  const link = document.createElement('a');
  link.href = blobUrl;
  link.download = 'sitemap-urls.csv';
  link.click();
  setTimeout(() => URL.revokeObjectURL(blobUrl), 1000);

  console.info('Downloaded ' + pages.size + ' unique URLs from ' + visited.size + ' sitemaps.');
  if (queue.length || limits.sitemaps || limits.urls || limits.fileSize || limits.runtime || visited.size >= MAX_SITEMAPS) {
    console.warn('A safety limit was reached; the CSV may be partial. Limits: ' + MAX_SITEMAPS + ' sitemap files, ' + MAX_URLS + ' URLs, 25 MB per file, and three minutes.');
  }
  if (errors.length) console.table(errors.map((message) => ({ error: message })));
})().catch((error) => console.error('Sitemap collection failed:', error));

実行前にコードを確認してください。このスクリプトはサイトマップを読み取り、CSVをダウンロードします。サイトは変更しません。