جمع بيانات الصفحات الوصفية من sitemap

(async () => {
  const MAX_SITEMAPS = 100;
  const MAX_PAGES = 100;
  const MAX_BYTES = 10 * 1024 * 1024;
  const deadline = Date.now() + 120000;
  const origin = location.origin;
  const queue = [];
  const queuedSitemaps = new Set();
  const seenSitemaps = new Set();
  const pages = new Set();
  const issues = [];
  const add = (value, base) => {
    try {
      const url = new URL(value, base);
      if (url.origin === origin && /^https?:$/.test(url.protocol) && !queuedSitemaps.has(url.href) && queue.length + seenSitemaps.size < MAX_SITEMAPS) {
        queuedSitemaps.add(url.href);
        queue.push(url.href);
      }
    } catch {}
  };
  const readLimitedText = async (response, limit, decompressGzip = false) => {
    const declaredSize = Number(response.headers.get('content-length'));
    if (Number.isFinite(declaredSize) && declaredSize > limit) throw new Error('Response exceeds size limit');
    if (!response.body) {
      const text = await response.text();
      if (new TextEncoder().encode(text).byteLength > limit) throw new Error('Response exceeds size limit');
      return text;
    }
    const sourceReader = response.body.getReader();
    const prefix = [];
    let prefixBytes = 0;
    let first;
    while (prefixBytes < 2) { first = await sourceReader.read(); if (first.done) break; prefix.push(first.value); prefixBytes += first.value.byteLength; }
    let stream = new ReadableStream({
      start(target) { if (first?.done) target.close(); else prefix.forEach((chunk) => target.enqueue(chunk)); },
      async pull(target) { const item = await sourceReader.read(); if (item.done) target.close(); else target.enqueue(item.value); },
      cancel(reason) { return sourceReader.cancel(reason); },
    });
    const magic = prefix.length ? new Uint8Array(prefix.reduce((sum, chunk) => sum + chunk.length, 0)) : new Uint8Array();
    let offset = 0;
    for (const chunk of prefix) { magic.set(chunk, offset); offset += chunk.length; }
    if (decompressGzip && magic[0] === 0x1f && magic[1] === 0x8b) {
      if (!window.DecompressionStream) { await sourceReader.cancel(); throw new Error('gzip unsupported in this browser'); }
      stream = stream.pipeThrough(new DecompressionStream('gzip'));
    }
    const reader = stream.getReader();
    const chunks = [];
    let size = 0;
    while (true) {
      if (Date.now() >= deadline) { await reader.cancel(); throw new Error('Two-minute runtime limit reached'); }
      const item = await reader.read();
      if (item.done) break;
      size += item.value.byteLength;
      if (size > limit) { await reader.cancel(); throw new Error('Response exceeds size limit'); }
      chunks.push(item.value);
    }
    return new Blob(chunks).text();
  };
  const readXml = async (url) => {
    if (Date.now() >= deadline) throw new Error('Two-minute runtime limit reached');
    const controller = new AbortController();
    const timer = setTimeout(() => controller.abort(), Math.min(10000, deadline - Date.now()));
    try {
      const response = await fetch(url, { credentials: 'omit', redirect: 'follow', signal: controller.signal });
      if (!response.ok || new URL(response.url).origin !== origin) throw new Error('HTTP ' + response.status + ' or cross-origin redirect');
      const xmlText = await readLimitedText(response, MAX_BYTES, true);
      return new DOMParser().parseFromString(xmlText, 'application/xml');
    } catch (error) {
      if (error.name === 'AbortError') throw new Error('Sitemap request timed out');
      throw error;
    } finally { clearTimeout(timer); }
  };
  try {
    const controller = new AbortController();
    const timer = setTimeout(() => controller.abort(), 5000);
    try {
      const robotsResponse = await fetch(new URL('/robots.txt', origin), { credentials: 'omit', signal: controller.signal });
      if (robotsResponse.ok) (await readLimitedText(robotsResponse, 1024 * 1024)).split(/\r?\n/).forEach((line) => { const value = line.match(/^\s*sitemap:\s*(\S+)/i)?.[1]; if (value) add(value, origin); });
    } finally { clearTimeout(timer); }
  } catch (error) { issues.push('/robots.txt: ' + error.message); }
  if (!queue.length) add('/sitemap.xml', origin);
  while (queue.length && seenSitemaps.size < MAX_SITEMAPS && pages.size < MAX_PAGES && Date.now() < deadline) {
    const url = queue.shift();
    if (seenSitemaps.has(url)) continue;
    seenSitemaps.add(url);
    try {
      const xml = await readXml(url);
      if (xml.querySelector('parsererror')) throw new Error('Invalid XML');
      if (xml.documentElement.localName.toLowerCase() === 'sitemapindex') {
        for (const node of xml.getElementsByTagNameNS('*', 'sitemap')) { if (queue.length + seenSitemaps.size >= MAX_SITEMAPS) break; add(node.getElementsByTagNameNS('*', 'loc')[0]?.textContent.trim(), url); }
      } else if (xml.documentElement.localName.toLowerCase() === 'urlset') {
        for (const node of xml.getElementsByTagNameNS('*', 'url')) {
          if (pages.size >= MAX_PAGES) break;
          try { const page = new URL(node.getElementsByTagNameNS('*', 'loc')[0]?.textContent.trim(), url); if (page.origin === origin) pages.add(page.href); } catch {}
        }
      } else throw new Error('Expected sitemap index or URL set');
    } catch (error) { issues.push(url + ': ' + error.message); }
  }
  const urls = [...pages];
  const results = new Array(urls.length);
  let cursor = 0;
  async function worker() {
    while (cursor < urls.length && Date.now() < deadline) {
      const index = cursor++;
      const controller = new AbortController();
      const timer = setTimeout(() => controller.abort(), Math.min(8000, deadline - Date.now()));
      try {
        const response = await fetch(urls[index], { credentials: 'omit', redirect: 'follow', signal: controller.signal });
        if (!response.ok || new URL(response.url).origin !== origin) throw new Error('HTTP ' + response.status);
        const html = await readLimitedText(response, MAX_BYTES);
        const doc = new DOMParser().parseFromString(html, 'text/html');
        results[index] = { url: urls[index], status: response.status, title: doc.title, description: doc.querySelector('meta[name="description"]')?.content || '', h1: doc.querySelector('h1')?.innerText.trim() || '', canonical: doc.querySelector('link[rel="canonical"]')?.href || '', robots: doc.querySelector('meta[name="robots"]')?.content || '' };
      } catch (error) { results[index] = { url: urls[index], status: '', title: '', description: '', h1: '', canonical: '', robots: error.name === 'AbortError' ? 'timeout' : error.message }; }
      finally { clearTimeout(timer); }
    }
  }
  await Promise.all(Array.from({ length: Math.min(3, urls.length) }, worker));
  const completed = results.filter(Boolean).length;
  console.table(results);
  const quote = (value) => '"' + String(value).replace(/[\u0000-\u0020]*[=+\-@]/, (match) => "'" + match).replace(/"/g, '""') + '"';
  const columns = ['url', 'status', 'title', 'description', 'h1', 'canonical', 'robots'];
  for (let index = 0; index < results.length; index++) if (!results[index]) results[index] = { url: urls[index], status: '', title: '', description: '', h1: '', canonical: '', robots: 'not checked: runtime limit' };
  const csv = '\uFEFF' + [columns, ...results.map((row) => columns.map((column) => row[column] || ''))].map((row) => row.map(quote).join(',')).join('\r\n');
  const a = document.createElement('a'); a.href = URL.createObjectURL(new Blob([csv], { type: 'text/csv;charset=utf-8' })); a.download = 'sitemap-page-metadata.csv'; a.click(); setTimeout(() => URL.revokeObjectURL(a.href), 1000);
  console.info('Collected metadata for ' + completed + ' of ' + results.length + ' URLs from ' + seenSitemaps.size + ' same-origin sitemaps. Limits: ' + MAX_SITEMAPS + ' sitemaps, ' + MAX_PAGES + ' URLs, 10 MB per file, and two minutes.');
  if (queue.length || pages.size >= MAX_PAGES || seenSitemaps.size >= MAX_SITEMAPS || Date.now() >= deadline || completed < results.length) console.warn('A safety limit or request issue was reached; the CSV may be partial.');
  if (issues.length) console.table(issues.map((error) => ({ error })));
})();

راجع الشيفرة قبل تشغيلها. تقرأ الصفحة الحالية؛ ولا تجري طلبات محدودة إلى الأصل نفسه (البروتوكول والنطاق والمنفذ) إلا فحوص sitemap والروابط. لا ترسل الطلبات ملفات تعريف الارتباط، ولا تغيّر الشيفرة محتوى الموقع.