Merge pull request #1220 from nodiscc/doc-robots-noindex-nofollow

add "noindex, nofollow" HTML robots meta-tag to documentation pages

diff --git a/doc/custom_theme/main.html b/doc/custom_theme/main.html
new file mode 100644 (file)
index 0000000..69b398f
--- /dev/null
+++ b/doc/custom_theme/main.html
@@ -0,0 +1,25 @@
+{% extends "base.html" %}
+
+{#
+The entry point for the ReadTheDocs Theme.
+ 
+Any theme customisations should override this file to redefine blocks defined in
+the various templates. The custom theme should only need to define a main.html
+which `{% extends "base.html" %}` and defines various blocks which will replace
+the blocks defined in base.html and its included child templates.
+#}
+
+{%- block site_meta %}
+<meta charset="utf-8">
+<meta http-equiv="X-UA-Compatible" content="IE=edge">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+{%- if 'media.readthedocs.org' not in config.extra_css[0] %}
+<meta name="robots" content="noindex, nofollow">
+{%- endif %}
+
+{% if page and page.is_homepage %}<meta name="description" content="{{ config.site_description }}">{% endif %}
+{% if config.site_author %}<meta name="author" content="{{ config.site_author }}">{% endif %}
+{% if config.site_favicon %}<link rel="shortcut icon" href="{{ config.site_favicon|url }}">
+{% else %}<link rel="shortcut icon" href="{{ 'img/favicon.ico'|url }}">{% endif %}
+{%- endblock %}

diff --git a/doc/md/Server-configuration.md b/doc/md/Server-configuration.md
index e281dc859239cae2ead0801671e6c8ccf1dc197a..cf44ecf5924099ee78405fee4ca4b1bf3e322c8e 100644 (file)
--- a/doc/md/Server-configuration.md
+++ b/doc/md/Server-configuration.md
@@ -397,6 +397,7 @@ http {
 ```
 
 ## Proxies
+
 If Shaarli is served behind a proxy (i.e. there is a proxy server between clients and the web server hosting Shaarli), please refer to the proxy server documentation for proper configuration. In particular, you have to ensure that the following server variables are properly set:
 
 - `X-Forwarded-Proto`
@@ -405,6 +406,12 @@ If Shaarli is served behind a proxy (i.e. there is a proxy server between client
 
 See also [proxy-related](https://github.com/shaarli/Shaarli/issues?utf8=%E2%9C%93&q=label%3Aproxy+) issues.
 
+## Robots and crawlers
+
+Shaarli disallows indexing and crawling of your local documentation pages by search engines, using `<meta name="robots">` HTML tags.
+Your Shaarli instance and other pages you host may still be indexed by various robots on the public Internet.
+You may want to setup a robots.txt file or other crawler control mechanism on your server.
+See [[1]](https://en.wikipedia.org/wiki/Robots_exclusion_standard), [[2]](https://support.google.com/webmasters/answer/6062608?hl=en) and [[3]](https://developers.google.com/search/reference/robots_meta_tag)
 
 ## See also
 

diff --git a/mkdocs.yml b/mkdocs.yml
index 941fce3aad1ad0f6a779d2789ce666419a4029c6..248fdbfe3923dafab6cca711a4f4278eaf6a583b 100644 (file)
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -2,7 +2,9 @@ site_name: Shaarli Documentation
 repo_url: https://github.com/shaarli/Shaarli
 edit_uri: edit/master/doc/md
 site_description: The personal, minimalist, super-fast, database free, bookmarking service
-theme: readthedocs
+theme:
+  name: readthedocs
+  custom_dir: doc/custom_theme/
 docs_dir: doc/md
 site_dir: doc/html
 # Disable strict mode until ReadTheDocs provides up-to-date MkDocs settings: