Add server-level protections to prevent search engine indexing

Author: dvXcode

.htaccess

@@ -0,0 +1,20 @@
+# Additional server-level protection against indexing
+# This works on Apache servers (GitHub Pages uses different servers, but good to have)
+
+<IfModule mod_headers.c>
+    # Send X-Robots-Tag header to prevent indexing
+    Header set X-Robots-Tag "noindex, nofollow, noarchive, nosnippet, noimageindex, notranslate"
+</IfModule>
+
+# Block common crawler user agents
+<RequireAll>
+    Require all granted
+    Require not expr "%{HTTP_USER_AGENT} =~ /googlebot/i"
+    Require not expr "%{HTTP_USER_AGENT} =~ /bingbot/i"
+    Require not expr "%{HTTP_USER_AGENT} =~ /slurp/i"
+    Require not expr "%{HTTP_USER_AGENT} =~ /duckduckbot/i"
+    Require not expr "%{HTTP_USER_AGENT} =~ /baiduspider/i"
+    Require not expr "%{HTTP_USER_AGENT} =~ /yandexbot/i"
+    Require not expr "%{HTTP_USER_AGENT} =~ /facebookexternalhit/i"
+    Require not expr "%{HTTP_USER_AGENT} =~ /twitterbot/i"
+</RequireAll>

_config.yml

@@ -3,6 +3,8 @@ permalink: pretty
 title: 'Volovyk'
 description: 'Volovyk provides consulting in Azure, .NET Core, Delphi, Dart & Flutter, plus startup and mobile app development from Norway.'
 
+# Prevent search engine indexing
+robots: noindex, nofollow
 
 home: 
   limit_services: 0

_layouts/default.html

@@ -5,6 +5,8 @@
   <meta charset="utf-8">
   <title>{% if page.title %}{{page.title}}{% else %}{{ site.title | escape }}{% endif %}</title>
   <meta name="viewport" content="width=device-width, initial-scale=1" />
+  <!-- Prevent search engine indexing -->
+  <meta name="robots" content="noindex, nofollow, noarchive, nosnippet, noimageindex, notranslate" />
   <link rel="icon" type="image/png" href="{{ '/images/favicon-32x32.svg' | relative_url }}">
   <!-- Google Fonts CDN -->
   <link rel="preconnect" href="https://fonts.googleapis.com">

robots.txt

@@ -0,0 +1,5 @@
+User-agent: *
+Disallow: /
+
+# Block all crawlers from the entire site
+# This prevents indexing by Google, Bing, Yahoo, and other search engines