From e2600d8b261994abbbeb1ff921edaefd267fc122 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 9 Aug 2019 15:04:36 +0200 Subject: Add avatar to prune script --- server/controllers/lazy-static.ts | 7 +- server/lib/files-cache/videos-preview-cache.ts | 2 +- server/models/video/thumbnail.ts | 10 ++ server/models/video/video.ts | 23 ++- server/tests/cli/index.ts | 1 + server/tests/cli/prune-storage.ts | 199 +++++++++++++++++++++++++ 6 files changed, 236 insertions(+), 6 deletions(-) create mode 100644 server/tests/cli/prune-storage.ts (limited to 'server') diff --git a/server/controllers/lazy-static.ts b/server/controllers/lazy-static.ts index 4285fd727..28d2f862a 100644 --- a/server/controllers/lazy-static.ts +++ b/server/controllers/lazy-static.ts @@ -49,7 +49,12 @@ async function getAvatar (req: express.Request, res: express.Response) { logger.info('Lazy serve remote avatar image %s.', avatar.fileUrl) - await pushAvatarProcessInQueue({ filename: avatar.filename, fileUrl: avatar.fileUrl }) + try { + await pushAvatarProcessInQueue({ filename: avatar.filename, fileUrl: avatar.fileUrl }) + } catch (err) { + logger.warn('Cannot process remote avatar %s.', avatar.fileUrl, { err }) + return res.sendStatus(404) + } avatar.onDisk = true avatar.save() diff --git a/server/lib/files-cache/videos-preview-cache.ts b/server/lib/files-cache/videos-preview-cache.ts index a68619d07..3da6bb138 100644 --- a/server/lib/files-cache/videos-preview-cache.ts +++ b/server/lib/files-cache/videos-preview-cache.ts @@ -18,7 +18,7 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache { } async getFilePathImpl (videoUUID: string) { - const video = await VideoModel.loadByUUIDWithFile(videoUUID) + const video = await VideoModel.loadByUUID(videoUUID) if (!video) return undefined if (video.isOwned()) return { isOwned: true, path: video.getPreview().getPath() } diff --git a/server/models/video/thumbnail.ts b/server/models/video/thumbnail.ts index cf2040cbf..f1952dcc1 100644 --- a/server/models/video/thumbnail.ts +++ b/server/models/video/thumbnail.ts @@ -100,6 +100,16 @@ export class ThumbnailModel extends Model { .catch(err => logger.error('Cannot remove thumbnail file %s.', instance.filename, err)) } + static loadByName (filename: string) { + const query = { + where: { + filename + } + } + + return ThumbnailModel.findOne(query) + } + static generateDefaultPreviewName (videoUUID: string) { return videoUUID + '.jpg' } diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 1321337ff..b59df397d 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -119,6 +119,7 @@ import { CONFIG } from '../../initializers/config' import { ThumbnailModel } from './thumbnail' import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type' import { createTorrentPromise } from '../../helpers/webtorrent' +import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation const indexes: (ModelIndexesOptions & { where?: WhereOptions })[] = [ @@ -1422,15 +1423,23 @@ export class VideoModel extends Model { return VideoModel.scope(ScopeNames.WITH_THUMBNAILS).findOne(options) } - static loadWithFiles (id: number, t?: Transaction, logging?: boolean) { + static loadWithFiles (id: number | string, t?: Transaction, logging?: boolean) { + const where = buildWhereIdOrUUID(id) + + const query = { + where, + transaction: t, + logging + } + return VideoModel.scope([ ScopeNames.WITH_FILES, ScopeNames.WITH_STREAMING_PLAYLISTS, ScopeNames.WITH_THUMBNAILS - ]).findByPk(id, { transaction: t, logging }) + ]).findOne(query) } - static loadByUUIDWithFile (uuid: string) { + static loadByUUID (uuid: string) { const options = { where: { uuid @@ -1754,7 +1763,7 @@ export class VideoModel extends Model { return maxBy(this.VideoFiles, file => file.resolution) } - getFile (resolution: VideoResolution) { + getFile (resolution: number) { if (Array.isArray(this.VideoFiles) === false) return undefined return this.VideoFiles.find(f => f.resolution === resolution) @@ -1893,6 +1902,12 @@ export class VideoModel extends Model { return `/api/${API_VERSION}/videos/${this.uuid}/description` } + getHLSPlaylist () { + if (!this.VideoStreamingPlaylists) return undefined + + return this.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) + } + removeFile (videoFile: VideoFileModel, isRedundancy = false) { const baseDir = isRedundancy ? CONFIG.STORAGE.REDUNDANCY_DIR : CONFIG.STORAGE.VIDEOS_DIR diff --git a/server/tests/cli/index.ts b/server/tests/cli/index.ts index 5af286fe2..029cd5196 100644 --- a/server/tests/cli/index.ts +++ b/server/tests/cli/index.ts @@ -4,5 +4,6 @@ import './create-transcoding-job' import './optimize-old-videos' import './peertube' import './plugins' +import './prune-storage' import './reset-password' import './update-host' diff --git a/server/tests/cli/prune-storage.ts b/server/tests/cli/prune-storage.ts new file mode 100644 index 000000000..67a5c564e --- /dev/null +++ b/server/tests/cli/prune-storage.ts @@ -0,0 +1,199 @@ +/* tslint:disable:no-unused-expression */ + +import 'mocha' +import * as chai from 'chai' +import { waitJobs } from '../../../shared/extra-utils/server/jobs' +import { + buildServerDirectory, + cleanupTests, + createVideoPlaylist, + doubleFollow, + execCLI, + flushAndRunMultipleServers, + getAccount, + getEnvCli, + ServerInfo, + setAccessTokensToServers, setDefaultVideoChannel, + updateMyAvatar, + uploadVideo, + wait +} from '../../../shared/extra-utils' +import { Account, VideoPlaylistPrivacy } from '../../../shared/models' +import { createFile, readdir } from 'fs-extra' +import * as uuidv4 from 'uuid/v4' +import { join } from 'path' +import * as request from 'supertest' + +const expect = chai.expect + +async function countFiles (internalServerNumber: number, directory: string) { + const files = await readdir(buildServerDirectory(internalServerNumber, directory)) + + return files.length +} + +async function assertNotExists (internalServerNumber: number, directory: string, substring: string) { + const files = await readdir(buildServerDirectory(internalServerNumber, directory)) + + for (const f of files) { + expect(f).to.not.contain(substring) + } +} + +async function assertCountAreOkay (servers: ServerInfo[]) { + for (const server of servers) { + const videosCount = await countFiles(server.internalServerNumber, 'videos') + expect(videosCount).to.equal(8) + + const torrentsCount = await countFiles(server.internalServerNumber, 'torrents') + expect(torrentsCount).to.equal(8) + + const previewsCount = await countFiles(server.internalServerNumber, 'previews') + expect(previewsCount).to.equal(2) + + const thumbnailsCount = await countFiles(server.internalServerNumber, 'thumbnails') + expect(thumbnailsCount).to.equal(6) + + const avatarsCount = await countFiles(server.internalServerNumber, 'avatars') + expect(avatarsCount).to.equal(2) + } +} + +describe('Test prune storage scripts', function () { + let servers: ServerInfo[] + const badNames: { [ directory: string ]: string[] } = {} + + before(async function () { + this.timeout(120000) + + servers = await flushAndRunMultipleServers(2, { transcoding: { enabled: true } }) + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + for (const server of servers) { + await uploadVideo(server.url, server.accessToken, { name: 'video 1' }) + await uploadVideo(server.url, server.accessToken, { name: 'video 2' }) + + await updateMyAvatar({ url: server.url, accessToken: server.accessToken, fixture: 'avatar.png' }) + + await createVideoPlaylist({ + url: server.url, + token: server.accessToken, + playlistAttrs: { + displayName: 'playlist', + privacy: VideoPlaylistPrivacy.PUBLIC, + videoChannelId: server.videoChannel.id, + thumbnailfile: 'thumbnail.jpg' + } + }) + } + + await doubleFollow(servers[0], servers[1]) + + // Lazy load the remote avatar + { + const res = await getAccount(servers[ 0 ].url, 'root@localhost:' + servers[ 1 ].port) + const account: Account = res.body + await request('http://localhost:' + servers[ 0 ].port).get(account.avatar.path).expect(200) + } + + { + const res = await getAccount(servers[ 1 ].url, 'root@localhost:' + servers[ 0 ].port) + const account: Account = res.body + await request('http://localhost:' + servers[ 1 ].port).get(account.avatar.path).expect(200) + } + + await wait(1000) + + await waitJobs(servers) + }) + + it('Should have the files on the disk', async function () { + await assertCountAreOkay(servers) + }) + + it('Should create some dirty files', async function () { + for (let i = 0; i < 2; i++) { + { + const base = buildServerDirectory(servers[0].internalServerNumber, 'videos') + + const n1 = uuidv4() + '.mp4' + const n2 = uuidv4() + '.webm' + + await createFile(join(base, n1)) + await createFile(join(base, n2)) + + badNames['videos'] = [ n1, n2 ] + } + + { + const base = buildServerDirectory(servers[0].internalServerNumber, 'torrents') + + const n1 = uuidv4() + '-240.torrent' + const n2 = uuidv4() + '-480.torrent' + + await createFile(join(base, n1)) + await createFile(join(base, n2)) + + badNames['torrents'] = [ n1, n2 ] + } + + { + const base = buildServerDirectory(servers[0].internalServerNumber, 'thumbnails') + + const n1 = uuidv4() + '.jpg' + const n2 = uuidv4() + '.jpg' + + await createFile(join(base, n1)) + await createFile(join(base, n2)) + + badNames['thumbnails'] = [ n1, n2 ] + } + + { + const base = buildServerDirectory(servers[0].internalServerNumber, 'previews') + + const n1 = uuidv4() + '.jpg' + const n2 = uuidv4() + '.jpg' + + await createFile(join(base, n1)) + await createFile(join(base, n2)) + + badNames['previews'] = [ n1, n2 ] + } + + { + const base = buildServerDirectory(servers[0].internalServerNumber, 'avatars') + + const n1 = uuidv4() + '.png' + const n2 = uuidv4() + '.jpg' + + await createFile(join(base, n1)) + await createFile(join(base, n2)) + + badNames['avatars'] = [ n1, n2 ] + } + } + }) + + it('Should run prune storage', async function () { + this.timeout(30000) + + const env = getEnvCli(servers[0]) + await execCLI(`echo y | ${env} npm run prune-storage`) + }) + + it('Should have removed files', async function () { + await assertCountAreOkay(servers) + + for (const directory of Object.keys(badNames)) { + for (const name of badNames[directory]) { + await assertNotExists(servers[0].internalServerNumber, directory, name) + } + } + }) + + after(async function () { + await cleanupTests(servers) + }) +}) -- cgit v1.2.3