server: serve files from storage/well-known (#5214)

* server: serve files from storage/well-known

closes #5206

* well-known: add tests

* test: try to skip new tests

* test: another try

* fix(config/prod): well_known path

* test: fix broken tests

* Update misc-endpoints.ts

* Use getDirectoryPath for tests

* Fix tests

Co-authored-by: Chocobozzz <me@florianbigard.com>

11 files changed, 54 insertions, 28 deletions

diff --git a/server/controllers/well-known.ts b/server/controllers/well-known.ts
index f467bd629..ce5883571 100644
--- a/server/controllers/well-known.ts
+++ b/server/controllers/well-known.ts
@@ -5,6 +5,7 @@ import { root } from '@shared/core-utils'
 import { CONFIG } from '../initializers/config'
 import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants'
 import { cacheRoute } from '../middlewares/cache/cache'
+import { handleStaticError } from '@server/middlewares'
 
 const wellKnownRouter = express.Router()
 
@@ -69,6 +70,12 @@ wellKnownRouter.use('/.well-known/host-meta',
   }
 )
 
+wellKnownRouter.use('/.well-known/',
+  cacheRoute(ROUTE_CACHE_LIFETIME.WELL_KNOWN),
+  express.static(CONFIG.STORAGE.WELL_KNOWN_DIR, { fallthrough: false }),
+  handleStaticError
+)
+
 // ---------------------------------------------------------------------------
 
 export {
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts
index 3188903be..1fd4ba248 100644
--- a/server/initializers/checker-before-init.ts
+++ b/server/initializers/checker-before-init.ts
@@ -16,7 +16,7 @@ function checkMissedConfig () {
     'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address',
     'email.body.signature', 'email.subject.prefix',
     'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache',
-    'storage.redundancy', 'storage.tmp', 'storage.streaming_playlists', 'storage.plugins',
+    'storage.redundancy', 'storage.tmp', 'storage.streaming_playlists', 'storage.plugins', 'storage.well_known',
     'log.level',
     'user.video_quota', 'user.video_quota_daily',
     'video_channels.max_per_user',
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 2c92bea22..287bf6f6d 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -107,7 +107,8 @@ const CONFIG = {
     TORRENTS_DIR: buildPath(config.get<string>('storage.torrents')),
     CACHE_DIR: buildPath(config.get<string>('storage.cache')),
     PLUGINS_DIR: buildPath(config.get<string>('storage.plugins')),
-    CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides'))
+    CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides')),
+    WELL_KNOWN_DIR: buildPath(config.get<string>('storage.well_known'))
   },
   OBJECT_STORAGE: {
     ENABLED: config.get<boolean>('object_storage.enabled'),
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 7039ab457..9257ebf93 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -116,7 +116,8 @@ const ROUTE_CACHE_LIFETIME = {
   ACTIVITY_PUB: {
     VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example
   },
-  STATS: '4 hours'
+  STATS: '4 hours',
+  WELL_KNOWN: '1 day'
 }
 
 // ---------------------------------------------------------------------------
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts
index 5abed358f..0f4973184 100644
--- a/server/tests/api/redundancy/redundancy.ts
+++ b/server/tests/api/redundancy/redundancy.ts
@@ -159,8 +159,8 @@ async function check2Webseeds (videoUUID?: string) {
   const { webtorrentFilenames } = await ensureSameFilenames(videoUUID)
 
   const directories = [
-    'test' + servers[0].internalServerNumber + '/redundancy',
-    'test' + servers[1].internalServerNumber + '/videos'
+    servers[0].getDirectoryPath('redundancy'),
+    servers[1].getDirectoryPath('videos')
   ]
 
   for (const directory of directories) {
@@ -214,8 +214,8 @@ async function check1PlaylistRedundancies (videoUUID?: string) {
   const { hlsFilenames } = await ensureSameFilenames(videoUUID)
 
   const directories = [
-    'test' + servers[0].internalServerNumber + '/redundancy/hls',
-    'test' + servers[1].internalServerNumber + '/streaming-playlists/hls'
+    servers[0].getDirectoryPath('redundancy/hls'),
+    servers[1].getDirectoryPath('streaming-playlists/hls')
   ]
 
   for (const directory of directories) {
diff --git a/server/tests/api/users/users-multiple-servers.ts b/server/tests/api/users/users-multiple-servers.ts
index 62d668d1e..188e6f137 100644
--- a/server/tests/api/users/users-multiple-servers.ts
+++ b/server/tests/api/users/users-multiple-servers.ts
@@ -197,7 +197,7 @@ describe('Test users with multiple servers', function () {
   it('Should not have actor files', async () => {
     for (const server of servers) {
       for (const userAvatarFilename of userAvatarFilenames) {
-        await checkActorFilesWereRemoved(userAvatarFilename, server.internalServerNumber)
+        await checkActorFilesWereRemoved(userAvatarFilename, server)
       }
     }
   })
diff --git a/server/tests/api/videos/video-playlists.ts b/server/tests/api/videos/video-playlists.ts
index a0c743170..9d223de48 100644
--- a/server/tests/api/videos/video-playlists.ts
+++ b/server/tests/api/videos/video-playlists.ts
@@ -1049,7 +1049,7 @@ describe('Test video playlists', function () {
       this.timeout(30000)
 
       for (const server of servers) {
-        await checkPlaylistFilesWereRemoved(playlistServer1UUID, server.internalServerNumber)
+        await checkPlaylistFilesWereRemoved(playlistServer1UUID, server)
       }
     })
 
diff --git a/server/tests/misc-endpoints.ts b/server/tests/misc-endpoints.ts
index 663ac044a..d2072342e 100644
--- a/server/tests/misc-endpoints.ts
+++ b/server/tests/misc-endpoints.ts
@@ -1,18 +1,24 @@
 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
 
 import { expect } from 'chai'
-import { cleanupTests, createSingleServer, makeGetRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands'
+import { writeJson } from 'fs-extra'
+import { join } from 'path'
 import { HttpStatusCode, VideoPrivacy } from '@shared/models'
+import { cleanupTests, createSingleServer, makeGetRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands'
 import { expectLogDoesNotContain } from './shared'
 
 describe('Test misc endpoints', function () {
   let server: PeerTubeServer
+  let wellKnownPath: string
 
   before(async function () {
     this.timeout(120000)
 
     server = await createSingleServer(1)
+
     await setAccessTokensToServers([ server ])
+
+    wellKnownPath = server.getDirectoryPath('well-known')
   })
 
   describe('Test a well known endpoints', function () {
@@ -93,6 +99,28 @@ describe('Test misc endpoints', function () {
       expect(remoteInteract).to.exist
       expect(remoteInteract.template).to.equal(server.url + '/remote-interaction?uri={uri}')
     })
+
+    it('Should return 404 for non-existing files in /.well-known', async function () {
+      await makeGetRequest({
+        url: server.url,
+        path: '/.well-known/non-existing-file',
+        expectedStatus: HttpStatusCode.NOT_FOUND_404
+      })
+    })
+
+    it('Should return custom file from /.well-known', async function () {
+      const filename = 'existing-file.json'
+
+      await writeJson(join(wellKnownPath, filename), { iThink: 'therefore I am' })
+
+      const { body } = await makeGetRequest({
+        url: server.url,
+        path: '/.well-known/' + filename,
+        expectedStatus: HttpStatusCode.OK_200
+      })
+
+      expect(body.iThink).to.equal('therefore I am')
+    })
   })
 
   describe('Test classic static endpoints', function () {
diff --git a/server/tests/shared/actors.ts b/server/tests/shared/actors.ts
index f8f4a5137..41fd72e89 100644
--- a/server/tests/shared/actors.ts
+++ b/server/tests/shared/actors.ts
@@ -2,8 +2,6 @@
 
 import { expect } from 'chai'
 import { pathExists, readdir } from 'fs-extra'
-import { join } from 'path'
-import { root } from '@shared/core-utils'
 import { Account, VideoChannel } from '@shared/models'
 import { PeerTubeServer } from '@shared/server-commands'
 
@@ -31,11 +29,9 @@ async function expectAccountFollows (options: {
   return expectActorFollow({ ...options, data })
 }
 
-async function checkActorFilesWereRemoved (filename: string, serverNumber: number) {
-  const testDirectory = 'test' + serverNumber
-
+async function checkActorFilesWereRemoved (filename: string, server: PeerTubeServer) {
   for (const directory of [ 'avatars' ]) {
-    const directoryPath = join(root(), testDirectory, directory)
+    const directoryPath = server.getDirectoryPath(directory)
 
     const directoryExists = await pathExists(directoryPath)
     expect(directoryExists).to.be.true
diff --git a/server/tests/shared/directories.ts b/server/tests/shared/directories.ts
index c7065a767..90d534a06 100644
--- a/server/tests/shared/directories.ts
+++ b/server/tests/shared/directories.ts
@@ -2,22 +2,18 @@
 
 import { expect } from 'chai'
 import { pathExists, readdir } from 'fs-extra'
-import { join } from 'path'
-import { root } from '@shared/core-utils'
 import { PeerTubeServer } from '@shared/server-commands'
 
 async function checkTmpIsEmpty (server: PeerTubeServer) {
   await checkDirectoryIsEmpty(server, 'tmp', [ 'plugins-global.css', 'hls', 'resumable-uploads' ])
 
-  if (await pathExists(join('test' + server.internalServerNumber, 'tmp', 'hls'))) {
+  if (await pathExists(server.getDirectoryPath('tmp/hls'))) {
     await checkDirectoryIsEmpty(server, 'tmp/hls')
   }
 }
 
 async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) {
-  const testDirectory = 'test' + server.internalServerNumber
-
-  const directoryPath = join(root(), testDirectory, directory)
+  const directoryPath = server.getDirectoryPath(directory)
 
   const directoryExists = await pathExists(directoryPath)
   expect(directoryExists).to.be.true
diff --git a/server/tests/shared/playlists.ts b/server/tests/shared/playlists.ts
index fdd541d20..8db303fd8 100644
--- a/server/tests/shared/playlists.ts
+++ b/server/tests/shared/playlists.ts
@@ -1,17 +1,14 @@
 import { expect } from 'chai'
 import { readdir } from 'fs-extra'
-import { join } from 'path'
-import { root } from '@shared/core-utils'
+import { PeerTubeServer } from '@shared/server-commands'
 
 async function checkPlaylistFilesWereRemoved (
   playlistUUID: string,
-  internalServerNumber: number,
+  server: PeerTubeServer,
   directories = [ 'thumbnails' ]
 ) {
-  const testDirectory = 'test' + internalServerNumber
-
   for (const directory of directories) {
-    const directoryPath = join(root(), testDirectory, directory)
+    const directoryPath = server.getDirectoryPath(directory)
 
     const files = await readdir(directoryPath)
     for (const file of files) {