从 sitemap 收集页面元数据

(async () => {
  const MAX_SITEMAPS = 100;
  const MAX_PAGES = 100;
  const MAX_BYTES = 10 * 1024 * 1024;
  const deadline = Date.now() + 120000;
  const origin = location.origin;
  const queue = [];
  const queuedSitemaps = new Set();
  const seenSitemaps = new Set();
  const pages = new Set();
  const issues = [];
  const add = (value, base) => {
    try {
      const url = new URL(value, base);
      if (url.origin === origin && /^https?:$/.test(url.protocol) && !queuedSitemaps.has(url.href) && queue.length + seenSitemaps.size < MAX_SITEMAPS) {
        queuedSitemaps.add(url.href);
        queue.push(url.href);
      }
    } catch {}
  };
  const readLimitedText = async (response, limit, decompressGzip = false) => {
    const declaredSize = Number(response.headers.get('content-length'));
    if (Number.isFinite(declaredSize) && declaredSize > limit) throw new Error('Response exceeds size limit');
    if (!response.body) {
      const text = await response.text();
      if (new TextEncoder().encode(text).byteLength > limit) throw new Error('Response exceeds size limit');
      return text;
    }
    const sourceReader = response.body.getReader();
    const prefix = [];
    let prefixBytes = 0;
    let first;
    while (prefixBytes < 2) { first = await sourceReader.read(); if (first.done) break; prefix.push(first.value); prefixBytes += first.value.byteLength; }
    let stream = new ReadableStream({
      start(target) { if (first?.done) target.close(); else prefix.forEach((chunk) => target.enqueue(chunk)); },
      async pull(target) { const item = await sourceReader.read(); if (item.done) target.close(); else target.enqueue(item.value); },
      cancel(reason) { return sourceReader.cancel(reason); },
    });
    const magic = prefix.length ? new Uint8Array(prefix.reduce((sum, chunk) => sum + chunk.length, 0)) : new Uint8Array();
    let offset = 0;
    for (const chunk of prefix) { magic.set(chunk, offset); offset += chunk.length; }
    if (decompressGzip && magic[0] === 0x1f && magic[1] === 0x8b) {
      if (!window.DecompressionStream) { await sourceReader.cancel(); throw new Error('gzip unsupported in this browser'); }
      stream = stream.pipeThrough(new DecompressionStream('gzip'));
    }
    const reader = stream.getReader();
    const chunks = [];
    let size = 0;
    while (true) {
      if (Date.now() >= deadline) { await reader.cancel(); throw new Error('Two-minute runtime limit reached'); }
      const item = await reader.read();
      if (item.done) break;
      size += item.value.byteLength;
      if (size > limit) { await reader.cancel(); throw new Error('Response exceeds size limit'); }
      chunks.push(item.value);
    }
    return new Blob(chunks).text();
  };
  const readXml = async (url) => {
    if (Date.now() >= deadline) throw new Error('Two-minute runtime limit reached');
    const controller = new AbortController();
    const timer = setTimeout(() => controller.abort(), Math.min(10000, deadline - Date.now()));
    try {
      const response = await fetch(url, { credentials: 'omit', redirect: 'follow', signal: controller.signal });
      if (!response.ok || new URL(response.url).origin !== origin) throw new Error('HTTP ' + response.status + ' or cross-origin redirect');
      const xmlText = await readLimitedText(response, MAX_BYTES, true);
      return new DOMParser().parseFromString(xmlText, 'application/xml');
    } catch (error) {
      if (error.name === 'AbortError') throw new Error('Sitemap request timed out');
      throw error;
    } finally { clearTimeout(timer); }
  };
  try {
    const controller = new AbortController();
    const timer = setTimeout(() => controller.abort(), 5000);
    try {
      const robotsResponse = await fetch(new URL('/robots.txt', origin), { credentials: 'omit', signal: controller.signal });
      if (robotsResponse.ok) (await readLimitedText(robotsResponse, 1024 * 1024)).split(/\r?\n/).forEach((line) => { const value = line.match(/^\s*sitemap:\s*(\S+)/i)?.[1]; if (value) add(value, origin); });
    } finally { clearTimeout(timer); }
  } catch (error) { issues.push('/robots.txt: ' + error.message); }
  if (!queue.length) add('/sitemap.xml', origin);
  while (queue.length && seenSitemaps.size < MAX_SITEMAPS && pages.size < MAX_PAGES && Date.now() < deadline) {
    const url = queue.shift();
    if (seenSitemaps.has(url)) continue;
    seenSitemaps.add(url);
    try {
      const xml = await readXml(url);
      if (xml.querySelector('parsererror')) throw new Error('Invalid XML');
      if (xml.documentElement.localName.toLowerCase() === 'sitemapindex') {
        for (const node of xml.getElementsByTagNameNS('*', 'sitemap')) { if (queue.length + seenSitemaps.size >= MAX_SITEMAPS) break; add(node.getElementsByTagNameNS('*', 'loc')[0]?.textContent.trim(), url); }
      } else if (xml.documentElement.localName.toLowerCase() === 'urlset') {
        for (const node of xml.getElementsByTagNameNS('*', 'url')) {
          if (pages.size >= MAX_PAGES) break;
          try { const page = new URL(node.getElementsByTagNameNS('*', 'loc')[0]?.textContent.trim(), url); if (page.origin === origin) pages.add(page.href); } catch {}
        }
      } else throw new Error('Expected sitemap index or URL set');
    } catch (error) { issues.push(url + ': ' + error.message); }
  }
  const urls = [...pages];
  const results = new Array(urls.length);
  let cursor = 0;
  async function worker() {
    while (cursor < urls.length && Date.now() < deadline) {
      const index = cursor++;
      const controller = new AbortController();
      const timer = setTimeout(() => controller.abort(), Math.min(8000, deadline - Date.now()));
      try {
        const response = await fetch(urls[index], { credentials: 'omit', redirect: 'follow', signal: controller.signal });
        if (!response.ok || new URL(response.url).origin !== origin) throw new Error('HTTP ' + response.status);
        const html = await readLimitedText(response, MAX_BYTES);
        const doc = new DOMParser().parseFromString(html, 'text/html');
        results[index] = { url: urls[index], status: response.status, title: doc.title, description: doc.querySelector('meta[name="description"]')?.content || '', h1: doc.querySelector('h1')?.innerText.trim() || '', canonical: doc.querySelector('link[rel="canonical"]')?.href || '', robots: doc.querySelector('meta[name="robots"]')?.content || '' };
      } catch (error) { results[index] = { url: urls[index], status: '', title: '', description: '', h1: '', canonical: '', robots: error.name === 'AbortError' ? 'timeout' : error.message }; }
      finally { clearTimeout(timer); }
    }
  }
  await Promise.all(Array.from({ length: Math.min(3, urls.length) }, worker));
  const completed = results.filter(Boolean).length;
  console.table(results);
  const quote = (value) => '"' + String(value).replace(/[\u0000-\u0020]*[=+\-@]/, (match) => "'" + match).replace(/"/g, '""') + '"';
  const columns = ['url', 'status', 'title', 'description', 'h1', 'canonical', 'robots'];
  for (let index = 0; index < results.length; index++) if (!results[index]) results[index] = { url: urls[index], status: '', title: '', description: '', h1: '', canonical: '', robots: 'not checked: runtime limit' };
  const csv = '\uFEFF' + [columns, ...results.map((row) => columns.map((column) => row[column] || ''))].map((row) => row.map(quote).join(',')).join('\r\n');
  const a = document.createElement('a'); a.href = URL.createObjectURL(new Blob([csv], { type: 'text/csv;charset=utf-8' })); a.download = 'sitemap-page-metadata.csv'; a.click(); setTimeout(() => URL.revokeObjectURL(a.href), 1000);
  console.info('Collected metadata for ' + completed + ' of ' + results.length + ' URLs from ' + seenSitemaps.size + ' same-origin sitemaps. Limits: ' + MAX_SITEMAPS + ' sitemaps, ' + MAX_PAGES + ' URLs, 10 MB per file, and two minutes.');
  if (queue.length || pages.size >= MAX_PAGES || seenSitemaps.size >= MAX_SITEMAPS || Date.now() >= deadline || completed < results.length) console.warn('A safety limit or request issue was reached; the CSV may be partial.');
  if (issues.length) console.table(issues.map((error) => ({ error })));
})();

运行前请先审查代码。脚本读取当前页面；只有 sitemap 和链接审查工具会在有限范围内发出同源请求。请求不携带 cookies；脚本不会更改网站内容。