From 6c5f0d3aebbd9debcd33a9aab306b130547852a5 Mon Sep 17 00:00:00 2001 From: kontrollanten <6680299+kontrollanten@users.noreply.github.com> Date: Tue, 4 Oct 2022 10:53:00 +0200 Subject: [PATCH] server: serve files from storage/well-known (#5214) * server: serve files from storage/well-known closes #5206 * well-known: add tests * test: try to skip new tests * test: another try * fix(config/prod): well_known path * test: fix broken tests * Update misc-endpoints.ts * Use getDirectoryPath for tests * Fix tests Co-authored-by: Chocobozzz --- config/default.yaml | 1 + config/production.yaml.example | 1 + config/test-1.yaml | 1 + config/test-2.yaml | 1 + config/test-3.yaml | 1 + config/test-4.yaml | 1 + config/test-5.yaml | 1 + config/test-6.yaml | 1 + server/controllers/well-known.ts | 7 ++++ server/initializers/checker-before-init.ts | 2 +- server/initializers/config.ts | 3 +- server/initializers/constants.ts | 3 +- server/tests/api/redundancy/redundancy.ts | 8 ++--- .../tests/api/users/users-multiple-servers.ts | 2 +- server/tests/api/videos/video-playlists.ts | 2 +- server/tests/misc-endpoints.ts | 30 ++++++++++++++++- server/tests/shared/actors.ts | 8 ++--- server/tests/shared/directories.ts | 8 ++--- server/tests/shared/playlists.ts | 9 ++--- shared/server-commands/server/server.ts | 33 +++++++++++-------- .../docker/production/config/production.yaml | 1 + 21 files changed, 83 insertions(+), 41 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index 3a0b494fb..2d8aaf1ea 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -122,6 +122,7 @@ storage: captions: 'storage/captions/' cache: 'storage/cache/' plugins: 'storage/plugins/' + well_known: 'storage/well-known/' # Overridable client files in client/dist/assets/images: # - logo.svg # - favicon.png diff --git a/config/production.yaml.example b/config/production.yaml.example index dafc15915..46d574e42 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -120,6 +120,7 @@ storage: captions: '/var/www/peertube/storage/captions/' cache: '/var/www/peertube/storage/cache/' plugins: '/var/www/peertube/storage/plugins/' + well_known: '/var/www/peertube/storage/well-known/' # Overridable client files in client/dist/assets/images: # - logo.svg # - favicon.png diff --git a/config/test-1.yaml b/config/test-1.yaml index fd6a5a341..1402f3048 100644 --- a/config/test-1.yaml +++ b/config/test-1.yaml @@ -23,6 +23,7 @@ storage: captions: 'test1/captions/' cache: 'test1/cache/' plugins: 'test1/plugins/' + well_known: 'test1/well-known/' client_overrides: 'test1/client-overrides/' admin: diff --git a/config/test-2.yaml b/config/test-2.yaml index 5a4ba0abd..5d9db762f 100644 --- a/config/test-2.yaml +++ b/config/test-2.yaml @@ -23,6 +23,7 @@ storage: captions: 'test2/captions/' cache: 'test2/cache/' plugins: 'test2/plugins/' + well_known: 'test2/well-known/' client_overrides: 'test2/client-overrides/' admin: diff --git a/config/test-3.yaml b/config/test-3.yaml index c04df6029..97fe9cd44 100644 --- a/config/test-3.yaml +++ b/config/test-3.yaml @@ -23,6 +23,7 @@ storage: captions: 'test3/captions/' cache: 'test3/cache/' plugins: 'test3/plugins/' + well_known: 'test3/well-known/' client_overrides: 'test3/client-overrides/' admin: diff --git a/config/test-4.yaml b/config/test-4.yaml index 963dee547..328e70fa6 100644 --- a/config/test-4.yaml +++ b/config/test-4.yaml @@ -23,6 +23,7 @@ storage: captions: 'test4/captions/' cache: 'test4/cache/' plugins: 'test4/plugins/' + well_known: 'test4/well-known/' client_overrides: 'test4/client-overrides/' admin: diff --git a/config/test-5.yaml b/config/test-5.yaml index f41378c24..3e6ca9c12 100644 --- a/config/test-5.yaml +++ b/config/test-5.yaml @@ -23,6 +23,7 @@ storage: captions: 'test5/captions/' cache: 'test5/cache/' plugins: 'test5/plugins/' + well_known: 'test5/well-known/' client_overrides: 'test5/client-overrides/' admin: diff --git a/config/test-6.yaml b/config/test-6.yaml index 574f89d19..89a71593f 100644 --- a/config/test-6.yaml +++ b/config/test-6.yaml @@ -23,6 +23,7 @@ storage: captions: 'test6/captions/' cache: 'test6/cache/' plugins: 'test6/plugins/' + well_known: 'test6/well-known/' client_overrides: 'test6/client-overrides/' admin: diff --git a/server/controllers/well-known.ts b/server/controllers/well-known.ts index f467bd629..ce5883571 100644 --- a/server/controllers/well-known.ts +++ b/server/controllers/well-known.ts @@ -5,6 +5,7 @@ import { root } from '@shared/core-utils' import { CONFIG } from '../initializers/config' import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants' import { cacheRoute } from '../middlewares/cache/cache' +import { handleStaticError } from '@server/middlewares' const wellKnownRouter = express.Router() @@ -69,6 +70,12 @@ wellKnownRouter.use('/.well-known/host-meta', } ) +wellKnownRouter.use('/.well-known/', + cacheRoute(ROUTE_CACHE_LIFETIME.WELL_KNOWN), + express.static(CONFIG.STORAGE.WELL_KNOWN_DIR, { fallthrough: false }), + handleStaticError +) + // --------------------------------------------------------------------------- export { diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index 3188903be..1fd4ba248 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts @@ -16,7 +16,7 @@ function checkMissedConfig () { 'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address', 'email.body.signature', 'email.subject.prefix', 'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache', - 'storage.redundancy', 'storage.tmp', 'storage.streaming_playlists', 'storage.plugins', + 'storage.redundancy', 'storage.tmp', 'storage.streaming_playlists', 'storage.plugins', 'storage.well_known', 'log.level', 'user.video_quota', 'user.video_quota_daily', 'video_channels.max_per_user', diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 2c92bea22..287bf6f6d 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts @@ -107,7 +107,8 @@ const CONFIG = { TORRENTS_DIR: buildPath(config.get('storage.torrents')), CACHE_DIR: buildPath(config.get('storage.cache')), PLUGINS_DIR: buildPath(config.get('storage.plugins')), - CLIENT_OVERRIDES_DIR: buildPath(config.get('storage.client_overrides')) + CLIENT_OVERRIDES_DIR: buildPath(config.get('storage.client_overrides')), + WELL_KNOWN_DIR: buildPath(config.get('storage.well_known')) }, OBJECT_STORAGE: { ENABLED: config.get('object_storage.enabled'), diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 7039ab457..9257ebf93 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -116,7 +116,8 @@ const ROUTE_CACHE_LIFETIME = { ACTIVITY_PUB: { VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example }, - STATS: '4 hours' + STATS: '4 hours', + WELL_KNOWN: '1 day' } // --------------------------------------------------------------------------- diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index 5abed358f..0f4973184 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts @@ -159,8 +159,8 @@ async function check2Webseeds (videoUUID?: string) { const { webtorrentFilenames } = await ensureSameFilenames(videoUUID) const directories = [ - 'test' + servers[0].internalServerNumber + '/redundancy', - 'test' + servers[1].internalServerNumber + '/videos' + servers[0].getDirectoryPath('redundancy'), + servers[1].getDirectoryPath('videos') ] for (const directory of directories) { @@ -214,8 +214,8 @@ async function check1PlaylistRedundancies (videoUUID?: string) { const { hlsFilenames } = await ensureSameFilenames(videoUUID) const directories = [ - 'test' + servers[0].internalServerNumber + '/redundancy/hls', - 'test' + servers[1].internalServerNumber + '/streaming-playlists/hls' + servers[0].getDirectoryPath('redundancy/hls'), + servers[1].getDirectoryPath('streaming-playlists/hls') ] for (const directory of directories) { diff --git a/server/tests/api/users/users-multiple-servers.ts b/server/tests/api/users/users-multiple-servers.ts index 62d668d1e..188e6f137 100644 --- a/server/tests/api/users/users-multiple-servers.ts +++ b/server/tests/api/users/users-multiple-servers.ts @@ -197,7 +197,7 @@ describe('Test users with multiple servers', function () { it('Should not have actor files', async () => { for (const server of servers) { for (const userAvatarFilename of userAvatarFilenames) { - await checkActorFilesWereRemoved(userAvatarFilename, server.internalServerNumber) + await checkActorFilesWereRemoved(userAvatarFilename, server) } } }) diff --git a/server/tests/api/videos/video-playlists.ts b/server/tests/api/videos/video-playlists.ts index a0c743170..9d223de48 100644 --- a/server/tests/api/videos/video-playlists.ts +++ b/server/tests/api/videos/video-playlists.ts @@ -1049,7 +1049,7 @@ describe('Test video playlists', function () { this.timeout(30000) for (const server of servers) { - await checkPlaylistFilesWereRemoved(playlistServer1UUID, server.internalServerNumber) + await checkPlaylistFilesWereRemoved(playlistServer1UUID, server) } }) diff --git a/server/tests/misc-endpoints.ts b/server/tests/misc-endpoints.ts index 663ac044a..d2072342e 100644 --- a/server/tests/misc-endpoints.ts +++ b/server/tests/misc-endpoints.ts @@ -1,18 +1,24 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import { expect } from 'chai' -import { cleanupTests, createSingleServer, makeGetRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' +import { writeJson } from 'fs-extra' +import { join } from 'path' import { HttpStatusCode, VideoPrivacy } from '@shared/models' +import { cleanupTests, createSingleServer, makeGetRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' import { expectLogDoesNotContain } from './shared' describe('Test misc endpoints', function () { let server: PeerTubeServer + let wellKnownPath: string before(async function () { this.timeout(120000) server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + + wellKnownPath = server.getDirectoryPath('well-known') }) describe('Test a well known endpoints', function () { @@ -93,6 +99,28 @@ describe('Test misc endpoints', function () { expect(remoteInteract).to.exist expect(remoteInteract.template).to.equal(server.url + '/remote-interaction?uri={uri}') }) + + it('Should return 404 for non-existing files in /.well-known', async function () { + await makeGetRequest({ + url: server.url, + path: '/.well-known/non-existing-file', + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should return custom file from /.well-known', async function () { + const filename = 'existing-file.json' + + await writeJson(join(wellKnownPath, filename), { iThink: 'therefore I am' }) + + const { body } = await makeGetRequest({ + url: server.url, + path: '/.well-known/' + filename, + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(body.iThink).to.equal('therefore I am') + }) }) describe('Test classic static endpoints', function () { diff --git a/server/tests/shared/actors.ts b/server/tests/shared/actors.ts index f8f4a5137..41fd72e89 100644 --- a/server/tests/shared/actors.ts +++ b/server/tests/shared/actors.ts @@ -2,8 +2,6 @@ import { expect } from 'chai' import { pathExists, readdir } from 'fs-extra' -import { join } from 'path' -import { root } from '@shared/core-utils' import { Account, VideoChannel } from '@shared/models' import { PeerTubeServer } from '@shared/server-commands' @@ -31,11 +29,9 @@ async function expectAccountFollows (options: { return expectActorFollow({ ...options, data }) } -async function checkActorFilesWereRemoved (filename: string, serverNumber: number) { - const testDirectory = 'test' + serverNumber - +async function checkActorFilesWereRemoved (filename: string, server: PeerTubeServer) { for (const directory of [ 'avatars' ]) { - const directoryPath = join(root(), testDirectory, directory) + const directoryPath = server.getDirectoryPath(directory) const directoryExists = await pathExists(directoryPath) expect(directoryExists).to.be.true diff --git a/server/tests/shared/directories.ts b/server/tests/shared/directories.ts index c7065a767..90d534a06 100644 --- a/server/tests/shared/directories.ts +++ b/server/tests/shared/directories.ts @@ -2,22 +2,18 @@ import { expect } from 'chai' import { pathExists, readdir } from 'fs-extra' -import { join } from 'path' -import { root } from '@shared/core-utils' import { PeerTubeServer } from '@shared/server-commands' async function checkTmpIsEmpty (server: PeerTubeServer) { await checkDirectoryIsEmpty(server, 'tmp', [ 'plugins-global.css', 'hls', 'resumable-uploads' ]) - if (await pathExists(join('test' + server.internalServerNumber, 'tmp', 'hls'))) { + if (await pathExists(server.getDirectoryPath('tmp/hls'))) { await checkDirectoryIsEmpty(server, 'tmp/hls') } } async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) { - const testDirectory = 'test' + server.internalServerNumber - - const directoryPath = join(root(), testDirectory, directory) + const directoryPath = server.getDirectoryPath(directory) const directoryExists = await pathExists(directoryPath) expect(directoryExists).to.be.true diff --git a/server/tests/shared/playlists.ts b/server/tests/shared/playlists.ts index fdd541d20..8db303fd8 100644 --- a/server/tests/shared/playlists.ts +++ b/server/tests/shared/playlists.ts @@ -1,17 +1,14 @@ import { expect } from 'chai' import { readdir } from 'fs-extra' -import { join } from 'path' -import { root } from '@shared/core-utils' +import { PeerTubeServer } from '@shared/server-commands' async function checkPlaylistFilesWereRemoved ( playlistUUID: string, - internalServerNumber: number, + server: PeerTubeServer, directories = [ 'thumbnails' ] ) { - const testDirectory = 'test' + internalServerNumber - for (const directory of directories) { - const directoryPath = join(root(), testDirectory, directory) + const directoryPath = server.getDirectoryPath(directory) const files = await readdir(directoryPath) for (const file of files) { diff --git a/shared/server-commands/server/server.ts b/shared/server-commands/server/server.ts index 2b4c9c9f8..a8f8c1d84 100644 --- a/shared/server-commands/server/server.ts +++ b/shared/server-commands/server/server.ts @@ -182,6 +182,12 @@ export class PeerTubeServer { this.port = parseInt(parsed.port) } + getDirectoryPath (directoryName: string) { + const testDirectory = 'test' + this.internalServerNumber + + return join(root(), testDirectory, directoryName) + } + async flushAndRun (configOverride?: Object, options: RunServerOptions = {}) { await ServersCommand.flushTests(this.internalServerNumber) @@ -341,19 +347,20 @@ export class PeerTubeServer { suffix: '_test' + this.internalServerNumber }, storage: { - tmp: `test${this.internalServerNumber}/tmp/`, - bin: `test${this.internalServerNumber}/bin/`, - avatars: `test${this.internalServerNumber}/avatars/`, - videos: `test${this.internalServerNumber}/videos/`, - streaming_playlists: `test${this.internalServerNumber}/streaming-playlists/`, - redundancy: `test${this.internalServerNumber}/redundancy/`, - logs: `test${this.internalServerNumber}/logs/`, - previews: `test${this.internalServerNumber}/previews/`, - thumbnails: `test${this.internalServerNumber}/thumbnails/`, - torrents: `test${this.internalServerNumber}/torrents/`, - captions: `test${this.internalServerNumber}/captions/`, - cache: `test${this.internalServerNumber}/cache/`, - plugins: `test${this.internalServerNumber}/plugins/` + tmp: this.getDirectoryPath('tmp') + '/', + bin: this.getDirectoryPath('bin') + '/', + avatars: this.getDirectoryPath('avatars') + '/', + videos: this.getDirectoryPath('videos') + '/', + streaming_playlists: this.getDirectoryPath('streaming-playlists') + '/', + redundancy: this.getDirectoryPath('redundancy') + '/', + logs: this.getDirectoryPath('logs') + '/', + previews: this.getDirectoryPath('previews') + '/', + thumbnails: this.getDirectoryPath('thumbnails') + '/', + torrents: this.getDirectoryPath('torrents') + '/', + captions: this.getDirectoryPath('captions') + '/', + cache: this.getDirectoryPath('cache') + '/', + plugins: this.getDirectoryPath('plugins') + '/', + well_known: this.getDirectoryPath('well-known') + '/' }, admin: { email: `admin${this.internalServerNumber}@example.com` diff --git a/support/docker/production/config/production.yaml b/support/docker/production/config/production.yaml index 2a9117242..f1d698580 100644 --- a/support/docker/production/config/production.yaml +++ b/support/docker/production/config/production.yaml @@ -56,6 +56,7 @@ storage: captions: '../data/captions/' cache: '../data/cache/' plugins: '../data/plugins/' + well_known: '../data/well-known/' # Overridable client files in client/dist/assets/images : # - logo.svg # - favicon.png -- 2.41.0