Appease our great search engine overlords

This hopefully fixes all translated pages showing up in search engine results, and makes the STABLE version of each page canonical.
In turn, this allows us to re-enable indexing of the version-specific pages (see robots.txt changes), as search engines should prefer the canonical (stable) version, and only show the other versions if no canonical (stable) version exists (i.e. because that feature is only in latest, or was removed in stable).

It adds proper canonical links for all generated pages, and fixes the existing links between the various translations of a page by both ensuring the pages links to itself with the proper language tag, and by properly linking to the full path of other translated versions.

Author: mhilbrunner

_templates/layout.html

@@ -1,14 +1,12 @@
-{% extends "!layout.html" %}
-{% block linktags %}
-    <link rel="alternate" hreflang="en" href="https://docs.godotengine.org/en/" />
-    <link rel="alternate" hreflang="de" href="https://docs.godotengine.org/de/" />
-    <link rel="alternate" hreflang="es" href="https://docs.godotengine.org/es/" />
-    <link rel="alternate" hreflang="fr" href="https://docs.godotengine.org/fr/" />
-    <link rel="alternate" hreflang="ko" href="https://docs.godotengine.org/ko/" />
-    <link rel="alternate" hreflang="pl" href="https://docs.godotengine.org/pl/" />
-    <link rel="alternate" hreflang="pt-br" href="https://docs.godotengine.org/pt-br/" />
-    <link rel="alternate" hreflang="uk" href="https://docs.godotengine.org/uk/" />
-    <link rel="alternate" hreflang="zh-cn" href="https://docs.godotengine.org/zh-cn/" />
-    <link rel="alternate" hreflang="x-default" href="https://docs.godotengine.org/" />
-    {{ super() }}
-{% endblock %}
+{% extends "!layout.html" -%}
+{% block linktags -%}
+  {% if godot_inject_language_links -%}
+  {% for alternate_lang in godot_docs_supported_languages -%}
+  <link rel="alternate" hreflang="{{ alternate_lang }}" href="{{ godot_docs_basepath }}{{ alternate_lang }}/{{ godot_canonical_version }}/{{ pagename }}{{ godot_docs_suffix }}" />
+  {% endfor -%}
+  <link rel="alternate" hreflang="x-default" href="{{ godot_docs_basepath }}{{ godot_default_lang }}/{{ godot_canonical_version }}/{{ pagename }}{{ godot_docs_suffix }}" />
+  
+  <link rel="canonical" href="{{ godot_docs_basepath }}{{ lang_attr }}/{{ godot_canonical_version }}/{{ pagename }}{{ godot_docs_suffix }}" />
+  {% endif -%}
+  {{ super() }}
+{% endblock -%}

conf.py

@@ -49,7 +49,13 @@
         tags.add(tag.strip())  # noqa: F821
 
 # Language / i18n
+supported_languages = ['en', 'de', 'es', 'fr', 'fi', 'it', 'ja', 'ko', 'pl', 'pt-br', 'ru', 'uk', 'zh-cn']
 language = os.getenv("READTHEDOCS_LANGUAGE", "en")
+if not language in supported_languages:
+    print("Unknown language: " + language)
+    print("Supported languages: " + ", ".join(supported_languages))
+    print("This is an error or needs to be added to supported_languages in conf.py")
+
 is_i18n = tags.has("i18n")  # noqa: F821
 
 exclude_patterns = ["_build"]
@@ -94,6 +100,12 @@
     "github_repo": "godot-docs",  # Repo name
     "github_version": "master",  # Version
     "conf_py_path": "/",  # Path in the checkout to the docs root
+    "godot_inject_language_links": True,
+    "godot_docs_supported_languages": supported_languages,
+    "godot_docs_basepath": "https://docs.godotengine.org/",
+    "godot_docs_suffix": ".html",
+    "godot_default_lang": "en",
+    "godot_canonical_version": "stable",
 }
 
 html_logo = "img/docs_logo.png"

robots.txt

@@ -1,7 +1,3 @@
 user-agent: *
-disallow: /*/3.2
-disallow: /*/3.1
-disallow: /*/3.0
-disallow: /*/2.1
 
 sitemap: https://docs.godotengine.org/sitemap.xml