server: serve files from storage/well-known (#5214)

* server: serve files from storage/well-known

closes #5206

* well-known: add tests

* test: try to skip new tests

* test: another try

* fix(config/prod): well_known path

* test: fix broken tests

* Update misc-endpoints.ts

* Use getDirectoryPath for tests

* Fix tests

Co-authored-by: Chocobozzz <me@florianbigard.com>

21 files changed, 83 insertions, 41 deletions

diff --git a/config/default.yaml b/config/default.yaml
index 3a0b494fb..2d8aaf1ea 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -122,6 +122,7 @@ storage:
   captions: 'storage/captions/'
   cache: 'storage/cache/'
   plugins: 'storage/plugins/'
+  well_known: 'storage/well-known/'
   # Overridable client files in client/dist/assets/images:
   # - logo.svg
   # - favicon.png
diff --git a/config/production.yaml.example b/config/production.yaml.example
index dafc15915..46d574e42 100644
--- a/config/production.yaml.example
+++ b/config/production.yaml.example
@@ -120,6 +120,7 @@ storage:
   captions: '/var/www/peertube/storage/captions/'
   cache: '/var/www/peertube/storage/cache/'
   plugins: '/var/www/peertube/storage/plugins/'
+  well_known: '/var/www/peertube/storage/well-known/'
   # Overridable client files in client/dist/assets/images:
   # - logo.svg
   # - favicon.png
diff --git a/config/test-1.yaml b/config/test-1.yaml
index fd6a5a341..1402f3048 100644
--- a/config/test-1.yaml
+++ b/config/test-1.yaml
@@ -23,6 +23,7 @@ storage:
   captions: 'test1/captions/'
   cache: 'test1/cache/'
   plugins: 'test1/plugins/'
+  well_known: 'test1/well-known/'
   client_overrides: 'test1/client-overrides/'
 
 admin:
diff --git a/config/test-2.yaml b/config/test-2.yaml
index 5a4ba0abd..5d9db762f 100644
--- a/config/test-2.yaml
+++ b/config/test-2.yaml
@@ -23,6 +23,7 @@ storage:
   captions: 'test2/captions/'
   cache: 'test2/cache/'
   plugins: 'test2/plugins/'
+  well_known: 'test2/well-known/'
   client_overrides: 'test2/client-overrides/'
 
 admin:
diff --git a/config/test-3.yaml b/config/test-3.yaml
index c04df6029..97fe9cd44 100644
--- a/config/test-3.yaml
+++ b/config/test-3.yaml
@@ -23,6 +23,7 @@ storage:
   captions: 'test3/captions/'
   cache: 'test3/cache/'
   plugins: 'test3/plugins/'
+  well_known: 'test3/well-known/'
   client_overrides: 'test3/client-overrides/'
 
 admin:
diff --git a/config/test-4.yaml b/config/test-4.yaml
index 963dee547..328e70fa6 100644
--- a/config/test-4.yaml
+++ b/config/test-4.yaml
@@ -23,6 +23,7 @@ storage:
   captions: 'test4/captions/'
   cache: 'test4/cache/'
   plugins: 'test4/plugins/'
+  well_known: 'test4/well-known/'
   client_overrides: 'test4/client-overrides/'
 
 admin:
diff --git a/config/test-5.yaml b/config/test-5.yaml
index f41378c24..3e6ca9c12 100644
--- a/config/test-5.yaml
+++ b/config/test-5.yaml
@@ -23,6 +23,7 @@ storage:
   captions: 'test5/captions/'
   cache: 'test5/cache/'
   plugins: 'test5/plugins/'
+  well_known: 'test5/well-known/'
   client_overrides: 'test5/client-overrides/'
 
 admin:
diff --git a/config/test-6.yaml b/config/test-6.yaml
index 574f89d19..89a71593f 100644
--- a/config/test-6.yaml
+++ b/config/test-6.yaml
@@ -23,6 +23,7 @@ storage:
   captions: 'test6/captions/'
   cache: 'test6/cache/'
   plugins: 'test6/plugins/'
+  well_known: 'test6/well-known/'
   client_overrides: 'test6/client-overrides/'
 
 admin:
diff --git a/server/controllers/well-known.ts b/server/controllers/well-known.ts
index f467bd629..ce5883571 100644
--- a/server/controllers/well-known.ts
+++ b/server/controllers/well-known.ts
@@ -5,6 +5,7 @@ import { root } from '@shared/core-utils'
 import { CONFIG } from '../initializers/config'
 import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants'
 import { cacheRoute } from '../middlewares/cache/cache'
+import { handleStaticError } from '@server/middlewares'
 
 const wellKnownRouter = express.Router()
 
@@ -69,6 +70,12 @@ wellKnownRouter.use('/.well-known/host-meta',
   }
 )
 
+wellKnownRouter.use('/.well-known/',
+  cacheRoute(ROUTE_CACHE_LIFETIME.WELL_KNOWN),
+  express.static(CONFIG.STORAGE.WELL_KNOWN_DIR, { fallthrough: false }),
+  handleStaticError
+)
+
 // ---------------------------------------------------------------------------
 
 export {
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts
index 3188903be..1fd4ba248 100644
--- a/server/initializers/checker-before-init.ts
+++ b/server/initializers/checker-before-init.ts
@@ -16,7 +16,7 @@ function checkMissedConfig () {
     'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address',
     'email.body.signature', 'email.subject.prefix',
     'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache',
-    'storage.redundancy', 'storage.tmp', 'storage.streaming_playlists', 'storage.plugins',
+    'storage.redundancy', 'storage.tmp', 'storage.streaming_playlists', 'storage.plugins', 'storage.well_known',
     'log.level',
     'user.video_quota', 'user.video_quota_daily',
     'video_channels.max_per_user',
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 2c92bea22..287bf6f6d 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -107,7 +107,8 @@ const CONFIG = {
     TORRENTS_DIR: buildPath(config.get<string>('storage.torrents')),
     CACHE_DIR: buildPath(config.get<string>('storage.cache')),
     PLUGINS_DIR: buildPath(config.get<string>('storage.plugins')),
-    CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides'))
+    CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides')),
+    WELL_KNOWN_DIR: buildPath(config.get<string>('storage.well_known'))
   },
   OBJECT_STORAGE: {
     ENABLED: config.get<boolean>('object_storage.enabled'),
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 7039ab457..9257ebf93 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -116,7 +116,8 @@ const ROUTE_CACHE_LIFETIME = {
   ACTIVITY_PUB: {
     VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example
   },
-  STATS: '4 hours'
+  STATS: '4 hours',
+  WELL_KNOWN: '1 day'
 }
 
 // ---------------------------------------------------------------------------
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts
index 5abed358f..0f4973184 100644
--- a/server/tests/api/redundancy/redundancy.ts
+++ b/server/tests/api/redundancy/redundancy.ts
@@ -159,8 +159,8 @@ async function check2Webseeds (videoUUID?: string) {
   const { webtorrentFilenames } = await ensureSameFilenames(videoUUID)
 
   const directories = [
-    'test' + servers[0].internalServerNumber + '/redundancy',
-    'test' + servers[1].internalServerNumber + '/videos'
+    servers[0].getDirectoryPath('redundancy'),
+    servers[1].getDirectoryPath('videos')
   ]
 
   for (const directory of directories) {
@@ -214,8 +214,8 @@ async function check1PlaylistRedundancies (videoUUID?: string) {
   const { hlsFilenames } = await ensureSameFilenames(videoUUID)
 
   const directories = [
-    'test' + servers[0].internalServerNumber + '/redundancy/hls',
-    'test' + servers[1].internalServerNumber + '/streaming-playlists/hls'
+    servers[0].getDirectoryPath('redundancy/hls'),
+    servers[1].getDirectoryPath('streaming-playlists/hls')
   ]
 
   for (const directory of directories) {
diff --git a/server/tests/api/users/users-multiple-servers.ts b/server/tests/api/users/users-multiple-servers.ts
index 62d668d1e..188e6f137 100644
--- a/server/tests/api/users/users-multiple-servers.ts
+++ b/server/tests/api/users/users-multiple-servers.ts
@@ -197,7 +197,7 @@ describe('Test users with multiple servers', function () {
   it('Should not have actor files', async () => {
     for (const server of servers) {
       for (const userAvatarFilename of userAvatarFilenames) {
-        await checkActorFilesWereRemoved(userAvatarFilename, server.internalServerNumber)
+        await checkActorFilesWereRemoved(userAvatarFilename, server)
       }
     }
   })
diff --git a/server/tests/api/videos/video-playlists.ts b/server/tests/api/videos/video-playlists.ts
index a0c743170..9d223de48 100644
--- a/server/tests/api/videos/video-playlists.ts
+++ b/server/tests/api/videos/video-playlists.ts
@@ -1049,7 +1049,7 @@ describe('Test video playlists', function () {
       this.timeout(30000)
 
       for (const server of servers) {
-        await checkPlaylistFilesWereRemoved(playlistServer1UUID, server.internalServerNumber)
+        await checkPlaylistFilesWereRemoved(playlistServer1UUID, server)
       }
     })
 
diff --git a/server/tests/misc-endpoints.ts b/server/tests/misc-endpoints.ts
index 663ac044a..d2072342e 100644
--- a/server/tests/misc-endpoints.ts
+++ b/server/tests/misc-endpoints.ts
@@ -1,18 +1,24 @@
 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
 
 import { expect } from 'chai'
-import { cleanupTests, createSingleServer, makeGetRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands'
+import { writeJson } from 'fs-extra'
+import { join } from 'path'
 import { HttpStatusCode, VideoPrivacy } from '@shared/models'
+import { cleanupTests, createSingleServer, makeGetRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands'
 import { expectLogDoesNotContain } from './shared'
 
 describe('Test misc endpoints', function () {
   let server: PeerTubeServer
+  let wellKnownPath: string
 
   before(async function () {
     this.timeout(120000)
 
     server = await createSingleServer(1)
+
     await setAccessTokensToServers([ server ])
+
+    wellKnownPath = server.getDirectoryPath('well-known')
   })
 
   describe('Test a well known endpoints', function () {
@@ -93,6 +99,28 @@ describe('Test misc endpoints', function () {
       expect(remoteInteract).to.exist
       expect(remoteInteract.template).to.equal(server.url + '/remote-interaction?uri={uri}')
     })
+
+    it('Should return 404 for non-existing files in /.well-known', async function () {
+      await makeGetRequest({
+        url: server.url,
+        path: '/.well-known/non-existing-file',
+        expectedStatus: HttpStatusCode.NOT_FOUND_404
+      })
+    })
+
+    it('Should return custom file from /.well-known', async function () {
+      const filename = 'existing-file.json'
+
+      await writeJson(join(wellKnownPath, filename), { iThink: 'therefore I am' })
+
+      const { body } = await makeGetRequest({
+        url: server.url,
+        path: '/.well-known/' + filename,
+        expectedStatus: HttpStatusCode.OK_200
+      })
+
+      expect(body.iThink).to.equal('therefore I am')
+    })
   })
 
   describe('Test classic static endpoints', function () {
diff --git a/server/tests/shared/actors.ts b/server/tests/shared/actors.ts
index f8f4a5137..41fd72e89 100644
--- a/server/tests/shared/actors.ts
+++ b/server/tests/shared/actors.ts
@@ -2,8 +2,6 @@
 
 import { expect } from 'chai'
 import { pathExists, readdir } from 'fs-extra'
-import { join } from 'path'
-import { root } from '@shared/core-utils'
 import { Account, VideoChannel } from '@shared/models'
 import { PeerTubeServer } from '@shared/server-commands'
 
@@ -31,11 +29,9 @@ async function expectAccountFollows (options: {
   return expectActorFollow({ ...options, data })
 }
 
-async function checkActorFilesWereRemoved (filename: string, serverNumber: number) {
-  const testDirectory = 'test' + serverNumber
-
+async function checkActorFilesWereRemoved (filename: string, server: PeerTubeServer) {
   for (const directory of [ 'avatars' ]) {
-    const directoryPath = join(root(), testDirectory, directory)
+    const directoryPath = server.getDirectoryPath(directory)
 
     const directoryExists = await pathExists(directoryPath)
     expect(directoryExists).to.be.true
diff --git a/server/tests/shared/directories.ts b/server/tests/shared/directories.ts
index c7065a767..90d534a06 100644
--- a/server/tests/shared/directories.ts
+++ b/server/tests/shared/directories.ts
@@ -2,22 +2,18 @@
 
 import { expect } from 'chai'
 import { pathExists, readdir } from 'fs-extra'
-import { join } from 'path'
-import { root } from '@shared/core-utils'
 import { PeerTubeServer } from '@shared/server-commands'
 
 async function checkTmpIsEmpty (server: PeerTubeServer) {
   await checkDirectoryIsEmpty(server, 'tmp', [ 'plugins-global.css', 'hls', 'resumable-uploads' ])
 
-  if (await pathExists(join('test' + server.internalServerNumber, 'tmp', 'hls'))) {
+  if (await pathExists(server.getDirectoryPath('tmp/hls'))) {
     await checkDirectoryIsEmpty(server, 'tmp/hls')
   }
 }
 
 async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) {
-  const testDirectory = 'test' + server.internalServerNumber
-
-  const directoryPath = join(root(), testDirectory, directory)
+  const directoryPath = server.getDirectoryPath(directory)
 
   const directoryExists = await pathExists(directoryPath)
   expect(directoryExists).to.be.true
diff --git a/server/tests/shared/playlists.ts b/server/tests/shared/playlists.ts
index fdd541d20..8db303fd8 100644
--- a/server/tests/shared/playlists.ts
+++ b/server/tests/shared/playlists.ts
@@ -1,17 +1,14 @@
 import { expect } from 'chai'
 import { readdir } from 'fs-extra'
-import { join } from 'path'
-import { root } from '@shared/core-utils'
+import { PeerTubeServer } from '@shared/server-commands'
 
 async function checkPlaylistFilesWereRemoved (
   playlistUUID: string,
-  internalServerNumber: number,
+  server: PeerTubeServer,
   directories = [ 'thumbnails' ]
 ) {
-  const testDirectory = 'test' + internalServerNumber
-
   for (const directory of directories) {
-    const directoryPath = join(root(), testDirectory, directory)
+    const directoryPath = server.getDirectoryPath(directory)
 
     const files = await readdir(directoryPath)
     for (const file of files) {
diff --git a/shared/server-commands/server/server.ts b/shared/server-commands/server/server.ts
index 2b4c9c9f8..a8f8c1d84 100644
--- a/shared/server-commands/server/server.ts
+++ b/shared/server-commands/server/server.ts
@@ -182,6 +182,12 @@ export class PeerTubeServer {
     this.port = parseInt(parsed.port)
   }
 
+  getDirectoryPath (directoryName: string) {
+    const testDirectory = 'test' + this.internalServerNumber
+
+    return join(root(), testDirectory, directoryName)
+  }
+
   async flushAndRun (configOverride?: Object, options: RunServerOptions = {}) {
     await ServersCommand.flushTests(this.internalServerNumber)
 
@@ -341,19 +347,20 @@ export class PeerTubeServer {
         suffix: '_test' + this.internalServerNumber
       },
       storage: {
-        tmp: `test${this.internalServerNumber}/tmp/`,
-        bin: `test${this.internalServerNumber}/bin/`,
-        avatars: `test${this.internalServerNumber}/avatars/`,
-        videos: `test${this.internalServerNumber}/videos/`,
-        streaming_playlists: `test${this.internalServerNumber}/streaming-playlists/`,
-        redundancy: `test${this.internalServerNumber}/redundancy/`,
-        logs: `test${this.internalServerNumber}/logs/`,
-        previews: `test${this.internalServerNumber}/previews/`,
-        thumbnails: `test${this.internalServerNumber}/thumbnails/`,
-        torrents: `test${this.internalServerNumber}/torrents/`,
-        captions: `test${this.internalServerNumber}/captions/`,
-        cache: `test${this.internalServerNumber}/cache/`,
-        plugins: `test${this.internalServerNumber}/plugins/`
+        tmp: this.getDirectoryPath('tmp') + '/',
+        bin: this.getDirectoryPath('bin') + '/',
+        avatars: this.getDirectoryPath('avatars') + '/',
+        videos: this.getDirectoryPath('videos') + '/',
+        streaming_playlists: this.getDirectoryPath('streaming-playlists') + '/',
+        redundancy: this.getDirectoryPath('redundancy') + '/',
+        logs: this.getDirectoryPath('logs') + '/',
+        previews: this.getDirectoryPath('previews') + '/',
+        thumbnails: this.getDirectoryPath('thumbnails') + '/',
+        torrents: this.getDirectoryPath('torrents') + '/',
+        captions: this.getDirectoryPath('captions') + '/',
+        cache: this.getDirectoryPath('cache') + '/',
+        plugins: this.getDirectoryPath('plugins') + '/',
+        well_known: this.getDirectoryPath('well-known') + '/'
       },
       admin: {
         email: `admin${this.internalServerNumber}@example.com`
diff --git a/support/docker/production/config/production.yaml b/support/docker/production/config/production.yaml
index 2a9117242..f1d698580 100644
--- a/support/docker/production/config/production.yaml
+++ b/support/docker/production/config/production.yaml
@@ -56,6 +56,7 @@ storage:
   captions: '../data/captions/'
   cache: '../data/cache/'
   plugins: '../data/plugins/'
+  well_known: '../data/well-known/'
   # Overridable client files in client/dist/assets/images :
   # - logo.svg
   # - favicon.png