robots.txt config and route

diff --git a/config/default.yaml b/config/default.yaml
index 2826e76f8220dfb7b88dae9e64cbac72f7512740..995638281159857b69869345ada0a43ae1215221 100644 (file)
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -90,6 +90,10 @@ instance:
   customizations:
     javascript: '' # Directly your JavaScript code (without <script> tags). Will be eval at runtime
     css: '' # Directly your CSS code (without <style> tags). Will be injected at runtime
+  # Robot.txt rules. To allow robots to crawl your instance and allow indexation of your site, remove the '/'
+  robots: |
+    User-agent: *
+    Disallow: /
 
 services:
   # Cards configuration to format video in Twitter

diff --git a/config/production.yaml.example b/config/production.yaml.example
index a6f1740fedde17ee8be663acb955928f40969128..71344fa35dabcef634cce708c22471870d87bbb6 100644 (file)
--- a/config/production.yaml.example
+++ b/config/production.yaml.example
@@ -106,6 +106,10 @@ instance:
   customizations:
     javascript: '' # Directly your JavaScript code (without <script> tags). Will be eval at runtime
     css: '' # Directly your CSS code (without <style> tags). Will be injected at runtime
+  # Robot.txt rules. To allow robots to crawl your instance and allow indexation of your site, remove the '/'
+  robots: |
+    User-agent: *
+    Disallow: /
 
 services:
   # Cards configuration to format video in Twitter

diff --git a/server/controllers/static.ts b/server/controllers/static.ts
index eece9c06b5103b39728f0db37030f137b97adfe6..c1bf384a49c07dc749dcc7a4d20fd9edf871a5ab 100644 (file)
--- a/server/controllers/static.ts
+++ b/server/controllers/static.ts
@@ -44,6 +44,12 @@ staticRouter.use(
   asyncMiddleware(getPreview)
 )
 
+// robots.txt service
+staticRouter.get('/robots.txt', (req: express.Request, res: express.Response) => {
+  res.type('text/plain')
+  return res.send(CONFIG.INSTANCE.ROBOTS)
+})
+
 // ---------------------------------------------------------------------------
 
 export {

diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts
index 9bf53e940e7869d596eca90731dfef4057d700b2..c11dc7a89c65f19c43f8f40d0c4b5bc433dd6c09 100644 (file)
--- a/server/initializers/checker.ts
+++ b/server/initializers/checker.ts
@@ -29,7 +29,7 @@ function checkMissedConfig () {
     'user.video_quota',
     'cache.previews.size', 'admin.email', 'signup.enabled', 'signup.limit', 'transcoding.enabled', 'transcoding.threads',
     'instance.name', 'instance.short_description', 'instance.description', 'instance.terms', 'instance.default_client_route',
-    'instance.default_nsfw_policy',
+    'instance.default_nsfw_policy', 'instance.robots',
     'services.twitter.username', 'services.twitter.whitelisted'
   ]
   const miss: string[] = []

diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 5a7d91160d8fd495323e9a512fabe1d916de8036..51d27289519652246b10e1ef0a0c623b55fcfa09 100644 (file)
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -180,7 +180,8 @@ const CONFIG = {
     CUSTOMIZATIONS: {
       get JAVASCRIPT () { return config.get<string>('instance.customizations.javascript') },
       get CSS () { return config.get<string>('instance.customizations.css') }
-    }
+    },
+    get ROBOTS () { return config.get<string>('instance.robots') }
   },
   SERVICES: {
     TWITTER: {