server: serve files from storage/well-known (#5214)

* server: serve files from storage/well-known

closes #5206

* well-known: add tests

* test: try to skip new tests

* test: another try

* fix(config/prod): well_known path

* test: fix broken tests

* Update misc-endpoints.ts

* Use getDirectoryPath for tests

* Fix tests

Co-authored-by: Chocobozzz <me@florianbigard.com>

diff --git a/config/default.yaml b/config/default.yaml
index 3a0b494fbd45c6e28aa6ec0f18290d94fe34518a..2d8aaf1eaa61279d4b8699bf3b4a5c049fc11a2b 100644 (file)
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -122,6 +122,7 @@ storage:
   captions: 'storage/captions/'
   cache: 'storage/cache/'
   plugins: 'storage/plugins/'
+  well_known: 'storage/well-known/'
   # Overridable client files in client/dist/assets/images:
   # - logo.svg
   # - favicon.png

diff --git a/config/production.yaml.example b/config/production.yaml.example
index dafc15915e4a65b2f90c7fa49c7692d3ec2e702e..46d574e4237252a182fd020f6acfc87111f20925 100644 (file)
--- a/config/production.yaml.example
+++ b/config/production.yaml.example
@@ -120,6 +120,7 @@ storage:
   captions: '/var/www/peertube/storage/captions/'
   cache: '/var/www/peertube/storage/cache/'
   plugins: '/var/www/peertube/storage/plugins/'
+  well_known: '/var/www/peertube/storage/well-known/'
   # Overridable client files in client/dist/assets/images:
   # - logo.svg
   # - favicon.png

diff --git a/config/test-1.yaml b/config/test-1.yaml
index fd6a5a3416b1e1f3fea965cafc2c6ab3c64ff4b9..1402f30484a52c50ca3dbc7ed072767ba6c36f5e 100644 (file)
--- a/config/test-1.yaml
+++ b/config/test-1.yaml
@@ -23,6 +23,7 @@ storage:
   captions: 'test1/captions/'
   cache: 'test1/cache/'
   plugins: 'test1/plugins/'
+  well_known: 'test1/well-known/'
   client_overrides: 'test1/client-overrides/'
 
 admin:

diff --git a/config/test-2.yaml b/config/test-2.yaml
index 5a4ba0abd3d73723a12d47669db67e19dcdc4d52..5d9db762fba853ed3cf4a695d22f6075647d87f4 100644 (file)
--- a/config/test-2.yaml
+++ b/config/test-2.yaml
@@ -23,6 +23,7 @@ storage:
   captions: 'test2/captions/'
   cache: 'test2/cache/'
   plugins: 'test2/plugins/'
+  well_known: 'test2/well-known/'
   client_overrides: 'test2/client-overrides/'
 
 admin:

diff --git a/config/test-3.yaml b/config/test-3.yaml
index c04df6029c51c6da74ccf6de331dec26575d8fdb..97fe9cd44e77619ea7427d1cc701fee8ded82d19 100644 (file)
--- a/config/test-3.yaml
+++ b/config/test-3.yaml
@@ -23,6 +23,7 @@ storage:
   captions: 'test3/captions/'
   cache: 'test3/cache/'
   plugins: 'test3/plugins/'
+  well_known: 'test3/well-known/'
   client_overrides: 'test3/client-overrides/'
 
 admin:

diff --git a/config/test-4.yaml b/config/test-4.yaml
index 963dee547f87e3af4c753b7e7099add04ae1422b..328e70fa6e4e118e973d4fd8b872807a7d879837 100644 (file)
--- a/config/test-4.yaml
+++ b/config/test-4.yaml
@@ -23,6 +23,7 @@ storage:
   captions: 'test4/captions/'
   cache: 'test4/cache/'
   plugins: 'test4/plugins/'
+  well_known: 'test4/well-known/'
   client_overrides: 'test4/client-overrides/'
 
 admin:

diff --git a/config/test-5.yaml b/config/test-5.yaml
index f41378c2478ea304244d3fda0e2ff04f0faa3358..3e6ca9c12f689a39fb8a145070dec27baed0be53 100644 (file)
--- a/config/test-5.yaml
+++ b/config/test-5.yaml
@@ -23,6 +23,7 @@ storage:
   captions: 'test5/captions/'
   cache: 'test5/cache/'
   plugins: 'test5/plugins/'
+  well_known: 'test5/well-known/'
   client_overrides: 'test5/client-overrides/'
 
 admin:

diff --git a/config/test-6.yaml b/config/test-6.yaml
index 574f89d19f8f33ea66efd641639239379747d540..89a71593fec43f929df5ebcf1ac0eb958d3a4ceb 100644 (file)
--- a/config/test-6.yaml
+++ b/config/test-6.yaml
@@ -23,6 +23,7 @@ storage:
   captions: 'test6/captions/'
   cache: 'test6/cache/'
   plugins: 'test6/plugins/'
+  well_known: 'test6/well-known/'
   client_overrides: 'test6/client-overrides/'
 
 admin:

diff --git a/server/controllers/well-known.ts b/server/controllers/well-known.ts
index f467bd629835e2299b922b8d7a6c6a14899b0f35..ce5883571cf66e1f7961f9f08f53c57b9e322fe5 100644 (file)
--- a/server/controllers/well-known.ts
+++ b/server/controllers/well-known.ts
@@ -5,6 +5,7 @@ import { root } from '@shared/core-utils'
 import { CONFIG } from '../initializers/config'
 import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants'
 import { cacheRoute } from '../middlewares/cache/cache'
+import { handleStaticError } from '@server/middlewares'
 
 const wellKnownRouter = express.Router()
 
@@ -69,6 +70,12 @@ wellKnownRouter.use('/.well-known/host-meta',
   }
 )
 
+wellKnownRouter.use('/.well-known/',
+  cacheRoute(ROUTE_CACHE_LIFETIME.WELL_KNOWN),
+  express.static(CONFIG.STORAGE.WELL_KNOWN_DIR, { fallthrough: false }),
+  handleStaticError
+)
+
 // ---------------------------------------------------------------------------
 
 export {

diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts
index 3188903be56dadbcbf8704e5655eb380f713ee0a..1fd4ba248372f6cdb15b94320e6e648bd04e8ff7 100644 (file)
--- a/server/initializers/checker-before-init.ts
+++ b/server/initializers/checker-before-init.ts
@@ -16,7 +16,7 @@ function checkMissedConfig () {
     'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address',
     'email.body.signature', 'email.subject.prefix',
     'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache',
-    'storage.redundancy', 'storage.tmp', 'storage.streaming_playlists', 'storage.plugins',
+    'storage.redundancy', 'storage.tmp', 'storage.streaming_playlists', 'storage.plugins', 'storage.well_known',
     'log.level',
     'user.video_quota', 'user.video_quota_daily',
     'video_channels.max_per_user',

diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 2c92bea2297b04de415d83604a779ca9c41eb1c3..287bf6f6df69c1efced967133d0d58a920ac1ed0 100644 (file)
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -107,7 +107,8 @@ const CONFIG = {
     TORRENTS_DIR: buildPath(config.get<string>('storage.torrents')),
     CACHE_DIR: buildPath(config.get<string>('storage.cache')),
     PLUGINS_DIR: buildPath(config.get<string>('storage.plugins')),
-    CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides'))
+    CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides')),
+    WELL_KNOWN_DIR: buildPath(config.get<string>('storage.well_known'))
   },
   OBJECT_STORAGE: {
     ENABLED: config.get<boolean>('object_storage.enabled'),

diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 7039ab457ceebb0ec61648d9158dd8642f0cefce..9257ebf93ff6c1691827e25ec88c4e1130075f5f 100644 (file)
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -116,7 +116,8 @@ const ROUTE_CACHE_LIFETIME = {
   ACTIVITY_PUB: {
     VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example
   },
-  STATS: '4 hours'
+  STATS: '4 hours',
+  WELL_KNOWN: '1 day'
 }
 
 // ---------------------------------------------------------------------------

diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts
index 5abed358f99737ee3f885f25ea273f09a63a56f5..0f4973184ae0337ad818e48fa73344a5100f7445 100644 (file)
--- a/server/tests/api/redundancy/redundancy.ts
+++ b/server/tests/api/redundancy/redundancy.ts
@@ -159,8 +159,8 @@ async function check2Webseeds (videoUUID?: string) {
   const { webtorrentFilenames } = await ensureSameFilenames(videoUUID)
 
   const directories = [
-    'test' + servers[0].internalServerNumber + '/redundancy',
-    'test' + servers[1].internalServerNumber + '/videos'
+    servers[0].getDirectoryPath('redundancy'),
+    servers[1].getDirectoryPath('videos')
   ]
 
   for (const directory of directories) {
@@ -214,8 +214,8 @@ async function check1PlaylistRedundancies (videoUUID?: string) {
   const { hlsFilenames } = await ensureSameFilenames(videoUUID)
 
   const directories = [
-    'test' + servers[0].internalServerNumber + '/redundancy/hls',
-    'test' + servers[1].internalServerNumber + '/streaming-playlists/hls'
+    servers[0].getDirectoryPath('redundancy/hls'),
+    servers[1].getDirectoryPath('streaming-playlists/hls')
   ]
 
   for (const directory of directories) {

diff --git a/server/tests/api/users/users-multiple-servers.ts b/server/tests/api/users/users-multiple-servers.ts
index 62d668d1e69f1d0918585005a238b10dbe784c57..188e6f1377733b18767d1280965fb26c7e1fbdb5 100644 (file)
--- a/server/tests/api/users/users-multiple-servers.ts
+++ b/server/tests/api/users/users-multiple-servers.ts
@@ -197,7 +197,7 @@ describe('Test users with multiple servers', function () {
   it('Should not have actor files', async () => {
     for (const server of servers) {
       for (const userAvatarFilename of userAvatarFilenames) {
-        await checkActorFilesWereRemoved(userAvatarFilename, server.internalServerNumber)
+        await checkActorFilesWereRemoved(userAvatarFilename, server)
       }
     }
   })

diff --git a/server/tests/api/videos/video-playlists.ts b/server/tests/api/videos/video-playlists.ts
index a0c743170410255886aaaf227593641b5f89b834..9d223de485f646fdcf5b21c4ad8d6881effea8b8 100644 (file)
--- a/server/tests/api/videos/video-playlists.ts
+++ b/server/tests/api/videos/video-playlists.ts
@@ -1049,7 +1049,7 @@ describe('Test video playlists', function () {
       this.timeout(30000)
 
       for (const server of servers) {
-        await checkPlaylistFilesWereRemoved(playlistServer1UUID, server.internalServerNumber)
+        await checkPlaylistFilesWereRemoved(playlistServer1UUID, server)
       }
     })
 

diff --git a/server/tests/misc-endpoints.ts b/server/tests/misc-endpoints.ts
index 663ac044a91bea5f1e6aa7a30fa56fee58a079a0..d2072342efc1ad86a093c44c0923e87b9cb71f3d 100644 (file)
--- a/server/tests/misc-endpoints.ts
+++ b/server/tests/misc-endpoints.ts
@@ -1,18 +1,24 @@
 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
 
 import { expect } from 'chai'
-import { cleanupTests, createSingleServer, makeGetRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands'
+import { writeJson } from 'fs-extra'
+import { join } from 'path'
 import { HttpStatusCode, VideoPrivacy } from '@shared/models'
+import { cleanupTests, createSingleServer, makeGetRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands'
 import { expectLogDoesNotContain } from './shared'
 
 describe('Test misc endpoints', function () {
   let server: PeerTubeServer
+  let wellKnownPath: string
 
   before(async function () {
     this.timeout(120000)
 
     server = await createSingleServer(1)
+
     await setAccessTokensToServers([ server ])
+
+    wellKnownPath = server.getDirectoryPath('well-known')
   })
 
   describe('Test a well known endpoints', function () {
@@ -93,6 +99,28 @@ describe('Test misc endpoints', function () {
       expect(remoteInteract).to.exist
       expect(remoteInteract.template).to.equal(server.url + '/remote-interaction?uri={uri}')
     })
+
+    it('Should return 404 for non-existing files in /.well-known', async function () {
+      await makeGetRequest({
+        url: server.url,
+        path: '/.well-known/non-existing-file',
+        expectedStatus: HttpStatusCode.NOT_FOUND_404
+      })
+    })
+
+    it('Should return custom file from /.well-known', async function () {
+      const filename = 'existing-file.json'
+
+      await writeJson(join(wellKnownPath, filename), { iThink: 'therefore I am' })
+
+      const { body } = await makeGetRequest({
+        url: server.url,
+        path: '/.well-known/' + filename,
+        expectedStatus: HttpStatusCode.OK_200
+      })
+
+      expect(body.iThink).to.equal('therefore I am')
+    })
   })
 
   describe('Test classic static endpoints', function () {

diff --git a/server/tests/shared/actors.ts b/server/tests/shared/actors.ts
index f8f4a51370227abf4ca7d5d2c5be5857077fb7cd..41fd72e89410dc53d433981141df59e172e6ee15 100644 (file)
--- a/server/tests/shared/actors.ts
+++ b/server/tests/shared/actors.ts
@@ -2,8 +2,6 @@
 
 import { expect } from 'chai'
 import { pathExists, readdir } from 'fs-extra'
-import { join } from 'path'
-import { root } from '@shared/core-utils'
 import { Account, VideoChannel } from '@shared/models'
 import { PeerTubeServer } from '@shared/server-commands'
 
@@ -31,11 +29,9 @@ async function expectAccountFollows (options: {
   return expectActorFollow({ ...options, data })
 }
 
-async function checkActorFilesWereRemoved (filename: string, serverNumber: number) {
-  const testDirectory = 'test' + serverNumber
-
+async function checkActorFilesWereRemoved (filename: string, server: PeerTubeServer) {
   for (const directory of [ 'avatars' ]) {
-    const directoryPath = join(root(), testDirectory, directory)
+    const directoryPath = server.getDirectoryPath(directory)
 
     const directoryExists = await pathExists(directoryPath)
     expect(directoryExists).to.be.true

diff --git a/server/tests/shared/directories.ts b/server/tests/shared/directories.ts
index c7065a767f6d04e3d4a26db14f3c15fad4118922..90d534a065ed1bdd87df70f17817e7f8e0df31de 100644 (file)
--- a/server/tests/shared/directories.ts
+++ b/server/tests/shared/directories.ts
@@ -2,22 +2,18 @@
 
 import { expect } from 'chai'
 import { pathExists, readdir } from 'fs-extra'
-import { join } from 'path'
-import { root } from '@shared/core-utils'
 import { PeerTubeServer } from '@shared/server-commands'
 
 async function checkTmpIsEmpty (server: PeerTubeServer) {
   await checkDirectoryIsEmpty(server, 'tmp', [ 'plugins-global.css', 'hls', 'resumable-uploads' ])
 
-  if (await pathExists(join('test' + server.internalServerNumber, 'tmp', 'hls'))) {
+  if (await pathExists(server.getDirectoryPath('tmp/hls'))) {
     await checkDirectoryIsEmpty(server, 'tmp/hls')
   }
 }
 
 async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) {
-  const testDirectory = 'test' + server.internalServerNumber
-
-  const directoryPath = join(root(), testDirectory, directory)
+  const directoryPath = server.getDirectoryPath(directory)
 
   const directoryExists = await pathExists(directoryPath)
   expect(directoryExists).to.be.true

diff --git a/server/tests/shared/playlists.ts b/server/tests/shared/playlists.ts
index fdd541d20e25417144ba77b0da09d1c4c23e8d54..8db303fd86f1f558b3ecd101ac72796558b29aaa 100644 (file)
--- a/server/tests/shared/playlists.ts
+++ b/server/tests/shared/playlists.ts
@@ -1,17 +1,14 @@
 import { expect } from 'chai'
 import { readdir } from 'fs-extra'
-import { join } from 'path'
-import { root } from '@shared/core-utils'
+import { PeerTubeServer } from '@shared/server-commands'
 
 async function checkPlaylistFilesWereRemoved (
   playlistUUID: string,
-  internalServerNumber: number,
+  server: PeerTubeServer,
   directories = [ 'thumbnails' ]
 ) {
-  const testDirectory = 'test' + internalServerNumber
-
   for (const directory of directories) {
-    const directoryPath = join(root(), testDirectory, directory)
+    const directoryPath = server.getDirectoryPath(directory)
 
     const files = await readdir(directoryPath)
     for (const file of files) {

diff --git a/shared/server-commands/server/server.ts b/shared/server-commands/server/server.ts
index 2b4c9c9f83c4091b3ea631117017411ba3db37c9..a8f8c1d8415f39ca13237d552acd6761c84179ac 100644 (file)
--- a/shared/server-commands/server/server.ts
+++ b/shared/server-commands/server/server.ts
@@ -182,6 +182,12 @@ export class PeerTubeServer {
     this.port = parseInt(parsed.port)
   }
 
+  getDirectoryPath (directoryName: string) {
+    const testDirectory = 'test' + this.internalServerNumber
+
+    return join(root(), testDirectory, directoryName)
+  }
+
   async flushAndRun (configOverride?: Object, options: RunServerOptions = {}) {
     await ServersCommand.flushTests(this.internalServerNumber)
 
@@ -341,19 +347,20 @@ export class PeerTubeServer {
         suffix: '_test' + this.internalServerNumber
       },
       storage: {
-        tmp: `test${this.internalServerNumber}/tmp/`,
-        bin: `test${this.internalServerNumber}/bin/`,
-        avatars: `test${this.internalServerNumber}/avatars/`,
-        videos: `test${this.internalServerNumber}/videos/`,
-        streaming_playlists: `test${this.internalServerNumber}/streaming-playlists/`,
-        redundancy: `test${this.internalServerNumber}/redundancy/`,
-        logs: `test${this.internalServerNumber}/logs/`,
-        previews: `test${this.internalServerNumber}/previews/`,
-        thumbnails: `test${this.internalServerNumber}/thumbnails/`,
-        torrents: `test${this.internalServerNumber}/torrents/`,
-        captions: `test${this.internalServerNumber}/captions/`,
-        cache: `test${this.internalServerNumber}/cache/`,
-        plugins: `test${this.internalServerNumber}/plugins/`
+        tmp: this.getDirectoryPath('tmp') + '/',
+        bin: this.getDirectoryPath('bin') + '/',
+        avatars: this.getDirectoryPath('avatars') + '/',
+        videos: this.getDirectoryPath('videos') + '/',
+        streaming_playlists: this.getDirectoryPath('streaming-playlists') + '/',
+        redundancy: this.getDirectoryPath('redundancy') + '/',
+        logs: this.getDirectoryPath('logs') + '/',
+        previews: this.getDirectoryPath('previews') + '/',
+        thumbnails: this.getDirectoryPath('thumbnails') + '/',
+        torrents: this.getDirectoryPath('torrents') + '/',
+        captions: this.getDirectoryPath('captions') + '/',
+        cache: this.getDirectoryPath('cache') + '/',
+        plugins: this.getDirectoryPath('plugins') + '/',
+        well_known: this.getDirectoryPath('well-known') + '/'
       },
       admin: {
         email: `admin${this.internalServerNumber}@example.com`

diff --git a/support/docker/production/config/production.yaml b/support/docker/production/config/production.yaml
index 2a91172421997cd5584786787b46fa28ae927ac4..f1d698580cde2aa7954408f91bf307af045f04f4 100644 (file)
--- a/support/docker/production/config/production.yaml
+++ b/support/docker/production/config/production.yaml
@@ -56,6 +56,7 @@ storage:
   captions: '../data/captions/'
   cache: '../data/cache/'
   plugins: '../data/plugins/'
+  well_known: '../data/well-known/'
   # Overridable client files in client/dist/assets/images :
   # - logo.svg
   # - favicon.png