Use these steps as a working checklist and adapt them to the project’s evidence, scope, and constraints.
Practical steps
- Open a page on the target site and open the browser developer console. This script starts from that site’s robots.txt.
- If browser developer tools block pasting, type the browser’s displayed confirmation phrase yourself. Never paste code you have not reviewed.
- Paste the full script below and press Enter. It reads Sitemap entries from /robots.txt; if none are found, it tries /sitemap.xml.
- The script follows nested sitemap indexes, collects page URLs and last-modified dates, removes duplicate URLs, and downloads sitemap-urls.csv. It also decompresses gzip sitemap files in supported browsers.
- It can read same-origin sitemap files and cross-origin files listed by the site when those servers allow CORS. Requests omit cookies and credentials. Sign-in-only, blocked, or malformed files are reported in the console.
- Limits: 500 sitemap files, 1,000,000 URLs, 25 MB uncompressed per sitemap file, and three minutes. If a limit is reached, the downloaded CSV may be partial. For very large sites, use a crawler or a command-line parser instead.
(async () => {
const MAX_SITEMAPS = 500;
const MAX_URLS = 1000000;
const MAX_SITEMAP_BYTES = 25 * 1024 * 1024;
const MAX_RUNTIME_MS = 180000;
const FETCH_TIMEOUT_MS = 20000;
const allowedOrigin = window.location.origin;
const startedAt = Date.now();
const deadlineAt = startedAt + MAX_RUNTIME_MS;
const queue = [];
const queued = new Set();
const visited = new Set();
const pages = new Map();
const errors = [];
const limits = { sitemaps: false, urls: false, fileSize: false, runtime: false };
function enqueue(value, base) {
try {
const url = new URL(value, base);
if (!/^https?:$/.test(url.protocol) || url.username || url.password || queued.has(url.href)) return;
if (queued.size >= MAX_SITEMAPS) {
limits.sitemaps = true;
return;
}
queued.add(url.href);
queue.push(url.href);
} catch {
// Ignore malformed sitemap locations.
}
}
async function fetchText(url) {
if (Date.now() >= deadlineAt) {
limits.runtime = true;
throw new Error('Runtime limit reached');
}
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), Math.min(FETCH_TIMEOUT_MS, deadlineAt - Date.now()));
try {
const response = await fetch(url, { credentials: 'omit', redirect: 'follow', signal: controller.signal });
if (!response.ok) throw new Error('HTTP ' + response.status);
if (!response.body) {
const text = await response.text();
if (new TextEncoder().encode(text).byteLength > MAX_SITEMAP_BYTES) {
limits.fileSize = true;
throw new Error('Sitemap exceeds the 25 MB file limit');
}
return text;
}
const sourceReader = response.body.getReader();
const prefixChunks = [];
let prefixBytes = 0;
let firstChunk;
while (prefixBytes < 2) {
firstChunk = await sourceReader.read();
if (firstChunk.done) break;
prefixChunks.push(firstChunk.value);
prefixBytes += firstChunk.value.byteLength;
}
let bodyStream = new ReadableStream({
start(controller) {
if (firstChunk?.done) controller.close();
else prefixChunks.forEach((chunk) => controller.enqueue(chunk));
},
async pull(controller) {
const { done, value } = await sourceReader.read();
if (done) controller.close();
else controller.enqueue(value);
},
cancel(reason) {
return sourceReader.cancel(reason);
},
});
const firstBytes = prefixChunks.length ? new Uint8Array(prefixChunks.reduce((size, chunk) => size + chunk.length, 0)) : null;
if (firstBytes) {
let offset = 0;
for (const chunk of prefixChunks) {
firstBytes.set(chunk, offset);
offset += chunk.length;
}
}
const isGzipFile = firstBytes?.length >= 2 && firstBytes[0] === 0x1f && firstBytes[1] === 0x8b;
if (isGzipFile) {
if (!window.DecompressionStream) {
await sourceReader.cancel();
throw new Error('This browser cannot decompress gzip sitemap files');
}
bodyStream = bodyStream.pipeThrough(new window.DecompressionStream('gzip'));
}
const reader = bodyStream.getReader();
const chunks = [];
let bytes = 0;
while (true) {
if (Date.now() >= deadlineAt) {
limits.runtime = true;
await reader.cancel();
throw new Error('Runtime limit reached');
}
const { done, value } = await reader.read();
if (done) break;
bytes += value.byteLength;
if (bytes > MAX_SITEMAP_BYTES) {
limits.fileSize = true;
await reader.cancel();
throw new Error('Sitemap exceeds the 25 MB file limit');
}
chunks.push(value);
}
return await new Blob(chunks).text();
} finally {
clearTimeout(timeout);
}
}
let robotsSitemaps = [];
try {
const robots = await fetchText(new URL('/robots.txt', allowedOrigin).href);
robotsSitemaps = robots.split(/\r?\n/)
.map((line) => line.match(/^\s*sitemap:\s*(\S+)/i)?.[1])
.filter(Boolean);
} catch (error) {
errors.push('/robots.txt: ' + error.message);
}
robotsSitemaps.forEach((url) => enqueue(url, allowedOrigin));
if (!queue.length) enqueue('/sitemap.xml', allowedOrigin);
while (queue.length && visited.size < MAX_SITEMAPS && pages.size < MAX_URLS && Date.now() < deadlineAt) {
const sitemapUrl = queue.shift();
if (visited.has(sitemapUrl)) continue;
visited.add(sitemapUrl);
try {
const xml = new DOMParser().parseFromString(await fetchText(sitemapUrl), 'application/xml');
if (xml.querySelector('parsererror')) throw new Error('Invalid XML');
const rootName = xml.documentElement.localName.toLowerCase();
if (rootName === 'sitemapindex') {
for (const node of xml.getElementsByTagNameNS('*', 'sitemap')) {
if (Date.now() >= deadlineAt) {
limits.runtime = true;
break;
}
const loc = node.getElementsByTagNameNS('*', 'loc')[0]?.textContent?.trim();
if (loc) enqueue(loc, sitemapUrl);
}
} else if (rootName === 'urlset') {
for (const node of xml.getElementsByTagNameNS('*', 'url')) {
if (Date.now() >= deadlineAt) {
limits.runtime = true;
break;
}
const loc = node.getElementsByTagNameNS('*', 'loc')[0]?.textContent?.trim();
if (!loc) continue;
try {
const url = new URL(loc, sitemapUrl);
if (!/^https?:$/.test(url.protocol)) continue;
const lastmod = node.getElementsByTagNameNS('*', 'lastmod')[0]?.textContent?.trim() || '';
if (!pages.has(url.href) || lastmod) pages.set(url.href, { url: url.href, lastmod });
} catch {
// Ignore malformed page URLs.
}
if (pages.size >= MAX_URLS) {
limits.urls = true;
break;
}
}
} else {
throw new Error('Expected a sitemap index or URL set');
}
} catch (error) {
errors.push(sitemapUrl + ': ' + error.message);
}
console.info('Sitemaps: ' + visited.size + '; URLs: ' + pages.size);
}
if (!pages.size) {
console.warn('No sitemap URLs collected. Check that robots.txt and the sitemap are readable from this page.');
if (errors.length) console.table(errors.map((message) => ({ error: message })));
return;
}
const quote = (value) => {
const text = String(value);
const safeText = /^[\u0000-\u0020]*[=+\-@]/.test(text) ? "'" + text : text;
return '"' + safeText.replace(/"/g, '""') + '"';
};
const rows = [['URL', 'Last modified'], ...Array.from(pages.values(), ({ url, lastmod }) => [url, lastmod])];
const csv = '\uFEFF' + rows.map((row) => row.map(quote).join(',')).join('\r\n');
const blobUrl = URL.createObjectURL(new Blob([csv], { type: 'text/csv;charset=utf-8' }));
const link = document.createElement('a');
link.href = blobUrl;
link.download = 'sitemap-urls.csv';
link.click();
setTimeout(() => URL.revokeObjectURL(blobUrl), 1000);
console.info('Downloaded ' + pages.size + ' unique URLs from ' + visited.size + ' sitemaps.');
if (queue.length || limits.sitemaps || limits.urls || limits.fileSize || limits.runtime || visited.size >= MAX_SITEMAPS) {
console.warn('A safety limit was reached; the CSV may be partial. Limits: ' + MAX_SITEMAPS + ' sitemap files, ' + MAX_URLS + ' URLs, 25 MB per file, and three minutes.');
}
if (errors.length) console.table(errors.map((message) => ({ error: message })));
})().catch((error) => console.error('Sitemap collection failed:', error));Review the code before running it. It reads sitemap files and downloads a CSV; it does not change the site.