From 66fb2aa39b6f8e4677f80128c27fbafd3a8fe2e7 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 21 Nov 2019 12:16:27 +0100 Subject: [PATCH] Don't always replace actor avatar --- scripts/optimize-old-videos.ts | 4 +-- server/lib/activitypub/actor.ts | 33 ++++++++++--------- .../schedulers/videos-redundancy-scheduler.ts | 15 +++++---- server/lib/video-paths.ts | 16 +++++++-- server/models/video/video.ts | 9 +++-- server/tests/api/activitypub/refresher.ts | 2 +- server/tests/cli/prune-storage.ts | 16 ++++++--- 7 files changed, 59 insertions(+), 36 deletions(-) diff --git a/scripts/optimize-old-videos.ts b/scripts/optimize-old-videos.ts index 107483c50..a84845068 100644 --- a/scripts/optimize-old-videos.ts +++ b/scripts/optimize-old-videos.ts @@ -1,4 +1,6 @@ import { registerTSPaths } from '../server/helpers/register-ts-paths' +registerTSPaths() + import { VIDEO_TRANSCODING_FPS } from '../server/initializers/constants' import { getDurationFromVideoFile, getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '../server/helpers/ffmpeg-utils' import { getMaxBitrate } from '../shared/models/videos' @@ -10,8 +12,6 @@ import { copy, move, remove } from 'fs-extra' import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' import { getVideoFilePath } from '@server/lib/video-paths' -registerTSPaths() - run() .then(() => process.exit(0)) .catch(err => { diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index 74241aba9..14dd1b9b9 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -173,25 +173,28 @@ async function updateActorInstance (actorInstance: ActorModel, attributes: Activ type AvatarInfo = { name: string, onDisk: boolean, fileUrl: string } async function updateActorAvatarInstance (actor: MActorDefault, info: AvatarInfo, t: Transaction) { - if (info.name !== undefined) { - if (actor.avatarId) { - try { - await actor.Avatar.destroy({ transaction: t }) - } catch (err) { - logger.error('Cannot remove old avatar of actor %s.', actor.url, { err }) - } - } + if (!info.name) return actor - const avatar = await AvatarModel.create({ - filename: info.name, - onDisk: info.onDisk, - fileUrl: info.fileUrl - }, { transaction: t }) + if (actor.Avatar) { + // Don't update the avatar if the filename did not change + if (actor.Avatar.filename === info.name) return actor - actor.avatarId = avatar.id - actor.Avatar = avatar + try { + await actor.Avatar.destroy({ transaction: t }) + } catch (err) { + logger.error('Cannot remove old avatar of actor %s.', actor.url, { err }) + } } + const avatar = await AvatarModel.create({ + filename: info.name, + onDisk: info.onDisk, + fileUrl: info.fileUrl + }, { transaction: t }) + + actor.avatarId = avatar.id + actor.Avatar = avatar + return actor } diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index f2bd75cb4..c1c91b656 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -14,7 +14,7 @@ import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' import { downloadPlaylistSegments } from '../hls' import { CONFIG } from '../../initializers/config' import { - MStreamingPlaylist, + MStreamingPlaylist, MStreamingPlaylistFiles, MStreamingPlaylistVideo, MVideoAccountLight, MVideoFile, @@ -30,7 +30,7 @@ type CandidateToDuplicate = { redundancy: VideosRedundancy, video: MVideoWithAllFiles, files: MVideoFile[], - streamingPlaylists: MStreamingPlaylist[] + streamingPlaylists: MStreamingPlaylistFiles[] } function isMVideoRedundancyFileVideo ( @@ -196,7 +196,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() - const magnetUri = await generateMagnetUri(video, file, baseUrlHttp, baseUrlWs) + const magnetUri = generateMagnetUri(video, file, baseUrlHttp, baseUrlWs) const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) @@ -290,12 +290,15 @@ export class VideosRedundancyScheduler extends AbstractScheduler { return `${object.VideoStreamingPlaylist.playlistUrl}` } - private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylist[]) { + private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]) { const fileReducer = (previous: number, current: MVideoFile) => previous + current.size - const totalSize = files.reduce(fileReducer, 0) + let allFiles = files + for (const p of playlists) { + allFiles = allFiles.concat(p.VideoFiles) + } - return totalSize + (totalSize * playlists.length) + return allFiles.reduce(fileReducer, 0) } private async loadAndRefreshVideo (videoUrl: string) { diff --git a/server/lib/video-paths.ts b/server/lib/video-paths.ts index 63011cdb2..fe0a004e4 100644 --- a/server/lib/video-paths.ts +++ b/server/lib/video-paths.ts @@ -1,8 +1,8 @@ -import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/typings/models' +import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/typings/models' import { extractVideo } from './videos' import { join } from 'path' import { CONFIG } from '@server/initializers/config' -import { HLS_STREAMING_PLAYLIST_DIRECTORY } from '@server/initializers/constants' +import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY } from '@server/initializers/constants' // ################## Video file name ################## @@ -34,6 +34,14 @@ function getVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, vi return join(baseDir, getVideoFilename(videoOrPlaylist, videoFile)) } +// ################## Streaming playlist ################## + +function getHLSDirectory (video: MVideoUUID, isRedundancy = false) { + const baseDir = isRedundancy ? HLS_REDUNDANCY_DIRECTORY : HLS_STREAMING_PLAYLIST_DIRECTORY + + return join(baseDir, video.uuid) +} + // ################## Torrents ################## function getTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { @@ -60,5 +68,7 @@ export { getVideoFilePath, getTorrentFileName, - getTorrentFilePath + getTorrentFilePath, + + getHLSDirectory } diff --git a/server/models/video/video.ts b/server/models/video/video.ts index f84a90992..7e3512fe1 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -143,7 +143,7 @@ import { import { MVideoFile, MVideoFileStreamingPlaylistVideo } from '../../typings/models/video/video-file' import { MThumbnail } from '../../typings/models/video/thumbnail' import { VideoFile } from '@shared/models/videos/video-file.model' -import { getTorrentFileName, getTorrentFilePath, getVideoFilename, getVideoFilePath } from '@server/lib/video-paths' +import { getTorrentFileName, getTorrentFilePath, getVideoFilename, getVideoFilePath, getHLSDirectory } from '@server/lib/video-paths' // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation const indexes: (ModelIndexesOptions & { where?: WhereOptions })[] = [ @@ -1950,11 +1950,10 @@ export class VideoModel extends Model { } removeStreamingPlaylist (isRedundancy = false) { - const baseDir = isRedundancy ? HLS_REDUNDANCY_DIRECTORY : HLS_STREAMING_PLAYLIST_DIRECTORY + const directoryPath = getHLSDirectory(this, isRedundancy) - const filePath = join(baseDir, this.uuid) - return remove(filePath) - .catch(err => logger.warn('Cannot delete playlist directory %s.', filePath, { err })) + return remove(directoryPath) + .catch(err => logger.warn('Cannot delete playlist directory %s.', directoryPath, { err })) } isOutdated () { diff --git a/server/tests/api/activitypub/refresher.ts b/server/tests/api/activitypub/refresher.ts index 921ee874c..2a6be97c6 100644 --- a/server/tests/api/activitypub/refresher.ts +++ b/server/tests/api/activitypub/refresher.ts @@ -106,7 +106,7 @@ describe('Test AP refresher', function () { await reRunServer(servers[ 1 ]) - // Should not refresh the video, even if the last refresh failed (to avoir a loop on dead instances) + // Should not refresh the video, even if the last refresh failed (to avoid a loop on dead instances) await getVideo(servers[ 0 ].url, videoUUID3) await waitJobs(servers) diff --git a/server/tests/cli/prune-storage.ts b/server/tests/cli/prune-storage.ts index 67a5c564e..144e67c44 100644 --- a/server/tests/cli/prune-storage.ts +++ b/server/tests/cli/prune-storage.ts @@ -11,7 +11,7 @@ import { execCLI, flushAndRunMultipleServers, getAccount, - getEnvCli, + getEnvCli, makeGetRequest, makeRawRequest, ServerInfo, setAccessTokensToServers, setDefaultVideoChannel, updateMyAvatar, @@ -46,7 +46,7 @@ async function assertCountAreOkay (servers: ServerInfo[]) { expect(videosCount).to.equal(8) const torrentsCount = await countFiles(server.internalServerNumber, 'torrents') - expect(torrentsCount).to.equal(8) + expect(torrentsCount).to.equal(16) const previewsCount = await countFiles(server.internalServerNumber, 'previews') expect(previewsCount).to.equal(2) @@ -94,13 +94,21 @@ describe('Test prune storage scripts', function () { { const res = await getAccount(servers[ 0 ].url, 'root@localhost:' + servers[ 1 ].port) const account: Account = res.body - await request('http://localhost:' + servers[ 0 ].port).get(account.avatar.path).expect(200) + await makeGetRequest({ + url: servers[ 0 ].url, + path: account.avatar.path, + statusCodeExpected: 200 + }) } { const res = await getAccount(servers[ 1 ].url, 'root@localhost:' + servers[ 0 ].port) const account: Account = res.body - await request('http://localhost:' + servers[ 1 ].port).get(account.avatar.path).expect(200) + await makeGetRequest({ + url: servers[ 1 ].url, + path: account.avatar.path, + statusCodeExpected: 200 + }) } await wait(1000) -- 2.41.0