Merge pull request #202 from bartvdbraak/feat/sitemapxml-robotstxt

Create robots.txt and sitemap.xml programmatically

package.json

@@ -38,6 +38,7 @@
 		"vite": "^5.0.11"
 	},
 	"dependencies": {
+		"@vercel/analytics": "^1.1.1",
 		"bits-ui": "^0.14.0",
 		"clsx": "^2.1.0",
 		"lucide-svelte": "^0.309.0",
@@ -45,7 +46,8 @@
 		"radix-icons-svelte": "^1.2.1",
 		"svelte-wrap-balancer": "^0.0.4",
 		"tailwind-merge": "^2.2.0",
-		"tailwind-variants": "^0.1.20"
+		"tailwind-variants": "^0.1.20",
+		"web-vitals": "^3.5.1"
 	},
 	"lint-staged": {
 		"*.{js,ts,svelte,css,scss,postcss,md,json}": [

pnpm-lock.yaml

@@ -5,6 +5,9 @@ settings:
   excludeLinksFromLockfile: false
 
 dependencies:
+  '@vercel/analytics':
+    specifier: ^1.1.1
+    version: 1.1.1
   bits-ui:
     specifier: ^0.14.0
     version: 0.14.0(svelte@4.2.8)
@@ -29,6 +32,9 @@ dependencies:
   tailwind-variants:
     specifier: ^0.1.20
     version: 0.1.20(tailwindcss@3.4.1)
+  web-vitals:
+    specifier: ^3.5.1
+    version: 3.5.1
 
 devDependencies:
   '@sveltejs/adapter-vercel':
@@ -1076,6 +1082,12 @@ packages:
     resolution: {integrity: sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==}
     dev: true
 
+  /@vercel/analytics@1.1.1:
+    resolution: {integrity: sha512-+NqgNmSabg3IFfxYhrWCfB/H+RCUOCR5ExRudNG2+pcRehq628DJB5e1u1xqwpLtn4pAYii4D98w7kofORAGQA==}
+    dependencies:
+      server-only: 0.0.1
+    dev: false
+
   /@vercel/nft@0.26.2:
     resolution: {integrity: sha512-bxe2iShmKZi7476xYamyKvhhKwQ6JPEtQ2FSq1AjMUH2buMd8LQMkdoHinTqZYc+1sMTh3G0ARdjzNvV1FEisA==}
     engines: {node: '>=16'}
@@ -2857,6 +2869,10 @@ packages:
       lru-cache: 6.0.0
     dev: true
 
+  /server-only@0.0.1:
+    resolution: {integrity: sha512-qepMx2JxAa5jjfzxG79yPPq+8BuFToHd1hm7kI+Z4zAq1ftQiP7HcxMhDDItrbtwVeLg/cY2JnKnrcFkmiswNA==}
+    dev: false
+
   /set-blocking@2.0.0:
     resolution: {integrity: sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==}
     dev: true
@@ -3401,6 +3417,10 @@ packages:
       vite: 5.0.11
     dev: true
 
+  /web-vitals@3.5.1:
+    resolution: {integrity: sha512-xQ9lvIpfLxUj0eSmT79ZjRoU5wIRfIr7pNukL7ZE4EcWZSmfZQqOlhuAGfkVa3EFmzPHZhWhXfm2i5ys+THVPg==}
+    dev: false
+
   /webidl-conversions@3.0.1:
     resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==}
     dev: true

src/lib/types/nav.ts

@@ -2,7 +2,7 @@ import type { Icons } from '$lib/components/site/icons';
 
 export type NavItem = {
 	title: string;
-	href?: string;
+	href: string;
 	disabled?: boolean;
 	external?: boolean;
 	icon?: keyof typeof Icons;

src/lib/vitals.ts

@@ -0,0 +1,82 @@
+import type { Metric } from 'web-vitals';
+import { onCLS, onFCP, onFID, onLCP, onTTFB } from 'web-vitals';
+
+const vitalsUrl = 'https://vitals.vercel-analytics.com/v1/vitals';
+
+interface NavigatorWithConnection extends Navigator {
+	connection: {
+		effectiveType: string;
+	};
+}
+
+type Params = Record<string, string>;
+
+function getConnectionSpeed() {
+	return 'connection' in navigator &&
+		'connection' &&
+		'effectiveType' in (navigator as NavigatorWithConnection).connection
+		? (navigator as NavigatorWithConnection).connection.effectiveType
+		: '';
+}
+
+function sendToAnalytics(
+	metric: Metric,
+	options: {
+		params: Params;
+		path: string;
+		analyticsId: string;
+		debug: boolean;
+	}
+) {
+	const page = (Object.entries(options.params) as [string, string][]).reduce(
+		(acc: string, [key, value]: [string, string]) => acc.replace(value, `[${key}]`),
+		options.path
+	);
+
+	const body = {
+		dsn: options.analyticsId,
+		id: metric.id,
+		page,
+		href: location.href,
+		event_name: metric.name,
+		value: metric.value.toString(),
+		speed: getConnectionSpeed()
+	};
+
+	if (options.debug) {
+		console.log('[Web Vitals]', metric.name, JSON.stringify(body, null, 2));
+	}
+
+	const searchParams = new URLSearchParams(body);
+
+	const blob = new Blob([searchParams.toString()], {
+		type: 'application/x-www-form-urlencoded'
+	});
+	if (navigator.sendBeacon) {
+		navigator.sendBeacon(vitalsUrl, blob);
+	} else {
+		fetch(vitalsUrl, {
+			body: blob,
+			method: 'POST',
+			credentials: 'omit',
+			keepalive: true
+		});
+	}
+}
+
+export function webVitals(options: {
+	params: Params;
+	path: string;
+	analyticsId: string;
+	debug: boolean;
+}) {
+	try {
+		onFID((metric) => sendToAnalytics(metric, options));
+		onTTFB((metric) => sendToAnalytics(metric, options));
+		onLCP((metric) => sendToAnalytics(metric, options));
+		onCLS((metric) => sendToAnalytics(metric, options));
+		onFCP((metric) => sendToAnalytics(metric, options));
+	} catch (err) {
+		console.error('[Web Vitals]', err);
+	}
+}

src/routes/+layout.svelte

@@ -4,6 +4,23 @@
 	import '../styles/globals.css';
 	import { ModeWatcher } from 'mode-watcher';
 	import { fly } from 'svelte/transition';
+	import { inject } from '@vercel/analytics';
+	import { webVitals } from '$lib/vitals';
+	import { browser } from '$app/environment';
+	import { page } from '$app/stores';
+
+	inject({ mode: dev ? 'development' : 'production' });
+
+	let analyticsId = import.meta.env.VERCEL_ANALYTICS_ID;
+
+	$: if (browser && analyticsId) {
+		webVitals({
+			path: $page.url.pathname,
+			params: $page.params,
+			analyticsId,
+			debug: false
+		});
+	}
 
 	export let data;
 </script>

src/routes/robots.txt/+server.ts

@@ -0,0 +1,51 @@
+import { siteConfig } from '$lib/config/site';
+
+const SITE_URL = siteConfig.url;
+
+/**
+ * SvelteKit RequestHandler for generating robots.txt.
+ */
+export async function GET() {
+	// Define the robots.txt configuration
+	const robotsConfig = [
+		{
+			agent: '*',
+			disallow: ['/']
+		}
+	];
+
+	const body = generateRobotsTxt(SITE_URL, robotsConfig);
+
+	return new Response(body, {
+		headers: {
+			'Cache-Control': `public, max-age=${86400}`, // 24 hours
+			'Content-Type': 'text/plain' // Corrected MIME type for robots.txt
+		}
+	});
+}
+
+/**
+ * Generates robots.txt content.
+ * @param {string} siteUrl Base site URL.
+ * @param {Array} config Robots.txt configuration array.
+ * @returns {string} robots.txt content.
+ */
+function generateRobotsTxt(siteUrl: string, config: { agent: string; disallow: string[] }[]) {
+	return `Sitemap: ${siteUrl}/sitemap.xml
+
+# https://developers.google.com/search/docs/advanced/sitemaps/build-sitemap#addsitemap
+# https://www.robotstxt.org/robotstxt.html
+${config.map((item) => generateRobotsTxtAgent(item.agent, item.disallow)).join('\n')}
+`;
+}
+
+/**
+ * Generates a user-agent section for robots.txt.
+ * @param {string} agent User-agent string.
+ * @param {string[]} disallow Array of paths to disallow.
+ * @returns {string} User-agent section of robots.txt.
+ */
+function generateRobotsTxtAgent(agent: string, disallow: string[]) {
+	const disallowEntries = disallow.map((path) => `Disallow: ${path}`).join('\n');
+	return `User-agent: ${agent}\n${disallowEntries}`;
+}

src/routes/sitemap.xml/+server.ts

@@ -0,0 +1,67 @@
+import { siteConfig } from '$lib/config/site';
+import { navConfig } from '$lib/config/nav';
+import type { NavItem } from '$lib/types/nav';
+
+const SITE_URL = siteConfig.url;
+
+/**
+ * SvelteKit RequestHandler for generating sitemap.
+ */
+export async function GET() {
+	const pages = generatePagePaths(navConfig.mainNav);
+	const body = generateSitemapXml(SITE_URL, pages);
+
+	return new Response(body, {
+		headers: {
+			'Cache-Control': `public, max-age=${86400}`, // 24 hours
+			'Content-Type': 'application/xml'
+		}
+	});
+}
+
+/**
+ * Generates paths for pages from navigation configuration.
+ * @param {Array} navItems Navigation items configuration.
+ * @returns {Array} Filtered and transformed page paths.
+ */
+function generatePagePaths(navItems: NavItem[]) {
+	return navItems
+		.map((item: { href: string }) => item.href.replace(/^\//, ''))
+		.filter((href: string) => href !== '');
+}
+
+/**
+ * Generates XML sitemap content.
+ * @param {string} siteUrl Base site URL.
+ * @param {Array} pages Array of page paths.
+ * @returns {string} Sitemap XML content.
+ */
+function generateSitemapXml(siteUrl: string, pages: string[]) {
+	return `<?xml version="1.0" encoding="UTF-8" ?>
+    <urlset
+      xmlns="https://www.sitemaps.org/schemas/sitemap/0.9"
+      xmlns:news="https://www.google.com/schemas/sitemap-news/0.9"
+      xmlns:xhtml="https://www.w3.org/1999/xhtml"
+      xmlns:mobile="https://www.google.com/schemas/sitemap-mobile/1.0"
+      xmlns:image="https://www.google.com/schemas/sitemap-image/1.1"
+      xmlns:video="https://www.google.com/schemas/sitemap-video/1.1"
+    >
+      ${generateUrlElement(siteUrl)}
+      ${pages.map((page: string) => generateUrlElement(`${siteUrl}/${page}`)).join('')}
+    </urlset>`;
+}
+
+/**
+ * Generates a URL element for sitemap XML.
+ * @param {string} url URL for the sitemap entry.
+ * @returns {string} URL element XML string.
+ */
+function generateUrlElement(url: string) {
+	return `
+    <url>
+      <loc>${url}</loc>
+      <changefreq>daily</changefreq>
+      <priority>0.7</priority>
+    </url>
+    `;
+}

static/robots.txt

@@ -1,6 +0,0 @@
-Sitemap: https://hellob.art/sitemap.xml
-
-# https://developers.google.com/search/docs/advanced/sitemaps/build-sitemap#addsitemap
-# https://www.robotstxt.org/robotstxt.html
-User-agent: *
-Disallow: