diff options
-rw-r--r-- | scripts/optimize-old-videos.ts | 4 | ||||
-rw-r--r-- | server/lib/activitypub/actor.ts | 33 | ||||
-rw-r--r-- | server/lib/schedulers/videos-redundancy-scheduler.ts | 15 | ||||
-rw-r--r-- | server/lib/video-paths.ts | 16 | ||||
-rw-r--r-- | server/models/video/video.ts | 9 | ||||
-rw-r--r-- | server/tests/api/activitypub/refresher.ts | 2 | ||||
-rw-r--r-- | server/tests/cli/prune-storage.ts | 16 |
7 files changed, 59 insertions, 36 deletions
diff --git a/scripts/optimize-old-videos.ts b/scripts/optimize-old-videos.ts index 107483c50..a84845068 100644 --- a/scripts/optimize-old-videos.ts +++ b/scripts/optimize-old-videos.ts | |||
@@ -1,4 +1,6 @@ | |||
1 | import { registerTSPaths } from '../server/helpers/register-ts-paths' | 1 | import { registerTSPaths } from '../server/helpers/register-ts-paths' |
2 | registerTSPaths() | ||
3 | |||
2 | import { VIDEO_TRANSCODING_FPS } from '../server/initializers/constants' | 4 | import { VIDEO_TRANSCODING_FPS } from '../server/initializers/constants' |
3 | import { getDurationFromVideoFile, getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '../server/helpers/ffmpeg-utils' | 5 | import { getDurationFromVideoFile, getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '../server/helpers/ffmpeg-utils' |
4 | import { getMaxBitrate } from '../shared/models/videos' | 6 | import { getMaxBitrate } from '../shared/models/videos' |
@@ -10,8 +12,6 @@ import { copy, move, remove } from 'fs-extra' | |||
10 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | 12 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' |
11 | import { getVideoFilePath } from '@server/lib/video-paths' | 13 | import { getVideoFilePath } from '@server/lib/video-paths' |
12 | 14 | ||
13 | registerTSPaths() | ||
14 | |||
15 | run() | 15 | run() |
16 | .then(() => process.exit(0)) | 16 | .then(() => process.exit(0)) |
17 | .catch(err => { | 17 | .catch(err => { |
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index 74241aba9..14dd1b9b9 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts | |||
@@ -173,25 +173,28 @@ async function updateActorInstance (actorInstance: ActorModel, attributes: Activ | |||
173 | 173 | ||
174 | type AvatarInfo = { name: string, onDisk: boolean, fileUrl: string } | 174 | type AvatarInfo = { name: string, onDisk: boolean, fileUrl: string } |
175 | async function updateActorAvatarInstance (actor: MActorDefault, info: AvatarInfo, t: Transaction) { | 175 | async function updateActorAvatarInstance (actor: MActorDefault, info: AvatarInfo, t: Transaction) { |
176 | if (info.name !== undefined) { | 176 | if (!info.name) return actor |
177 | if (actor.avatarId) { | ||
178 | try { | ||
179 | await actor.Avatar.destroy({ transaction: t }) | ||
180 | } catch (err) { | ||
181 | logger.error('Cannot remove old avatar of actor %s.', actor.url, { err }) | ||
182 | } | ||
183 | } | ||
184 | 177 | ||
185 | const avatar = await AvatarModel.create({ | 178 | if (actor.Avatar) { |
186 | filename: info.name, | 179 | // Don't update the avatar if the filename did not change |
187 | onDisk: info.onDisk, | 180 | if (actor.Avatar.filename === info.name) return actor |
188 | fileUrl: info.fileUrl | ||
189 | }, { transaction: t }) | ||
190 | 181 | ||
191 | actor.avatarId = avatar.id | 182 | try { |
192 | actor.Avatar = avatar | 183 | await actor.Avatar.destroy({ transaction: t }) |
184 | } catch (err) { | ||
185 | logger.error('Cannot remove old avatar of actor %s.', actor.url, { err }) | ||
186 | } | ||
193 | } | 187 | } |
194 | 188 | ||
189 | const avatar = await AvatarModel.create({ | ||
190 | filename: info.name, | ||
191 | onDisk: info.onDisk, | ||
192 | fileUrl: info.fileUrl | ||
193 | }, { transaction: t }) | ||
194 | |||
195 | actor.avatarId = avatar.id | ||
196 | actor.Avatar = avatar | ||
197 | |||
195 | return actor | 198 | return actor |
196 | } | 199 | } |
197 | 200 | ||
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index f2bd75cb4..c1c91b656 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts | |||
@@ -14,7 +14,7 @@ import { getOrCreateVideoAndAccountAndChannel } from '../activitypub' | |||
14 | import { downloadPlaylistSegments } from '../hls' | 14 | import { downloadPlaylistSegments } from '../hls' |
15 | import { CONFIG } from '../../initializers/config' | 15 | import { CONFIG } from '../../initializers/config' |
16 | import { | 16 | import { |
17 | MStreamingPlaylist, | 17 | MStreamingPlaylist, MStreamingPlaylistFiles, |
18 | MStreamingPlaylistVideo, | 18 | MStreamingPlaylistVideo, |
19 | MVideoAccountLight, | 19 | MVideoAccountLight, |
20 | MVideoFile, | 20 | MVideoFile, |
@@ -30,7 +30,7 @@ type CandidateToDuplicate = { | |||
30 | redundancy: VideosRedundancy, | 30 | redundancy: VideosRedundancy, |
31 | video: MVideoWithAllFiles, | 31 | video: MVideoWithAllFiles, |
32 | files: MVideoFile[], | 32 | files: MVideoFile[], |
33 | streamingPlaylists: MStreamingPlaylist[] | 33 | streamingPlaylists: MStreamingPlaylistFiles[] |
34 | } | 34 | } |
35 | 35 | ||
36 | function isMVideoRedundancyFileVideo ( | 36 | function isMVideoRedundancyFileVideo ( |
@@ -196,7 +196,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
196 | logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) | 196 | logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy) |
197 | 197 | ||
198 | const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() | 198 | const { baseUrlHttp, baseUrlWs } = video.getBaseUrls() |
199 | const magnetUri = await generateMagnetUri(video, file, baseUrlHttp, baseUrlWs) | 199 | const magnetUri = generateMagnetUri(video, file, baseUrlHttp, baseUrlWs) |
200 | 200 | ||
201 | const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) | 201 | const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT) |
202 | 202 | ||
@@ -290,12 +290,15 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
290 | return `${object.VideoStreamingPlaylist.playlistUrl}` | 290 | return `${object.VideoStreamingPlaylist.playlistUrl}` |
291 | } | 291 | } |
292 | 292 | ||
293 | private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylist[]) { | 293 | private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]) { |
294 | const fileReducer = (previous: number, current: MVideoFile) => previous + current.size | 294 | const fileReducer = (previous: number, current: MVideoFile) => previous + current.size |
295 | 295 | ||
296 | const totalSize = files.reduce(fileReducer, 0) | 296 | let allFiles = files |
297 | for (const p of playlists) { | ||
298 | allFiles = allFiles.concat(p.VideoFiles) | ||
299 | } | ||
297 | 300 | ||
298 | return totalSize + (totalSize * playlists.length) | 301 | return allFiles.reduce(fileReducer, 0) |
299 | } | 302 | } |
300 | 303 | ||
301 | private async loadAndRefreshVideo (videoUrl: string) { | 304 | private async loadAndRefreshVideo (videoUrl: string) { |
diff --git a/server/lib/video-paths.ts b/server/lib/video-paths.ts index 63011cdb2..fe0a004e4 100644 --- a/server/lib/video-paths.ts +++ b/server/lib/video-paths.ts | |||
@@ -1,8 +1,8 @@ | |||
1 | import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/typings/models' | 1 | import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/typings/models' |
2 | import { extractVideo } from './videos' | 2 | import { extractVideo } from './videos' |
3 | import { join } from 'path' | 3 | import { join } from 'path' |
4 | import { CONFIG } from '@server/initializers/config' | 4 | import { CONFIG } from '@server/initializers/config' |
5 | import { HLS_STREAMING_PLAYLIST_DIRECTORY } from '@server/initializers/constants' | 5 | import { HLS_REDUNDANCY_DIRECTORY, HLS_STREAMING_PLAYLIST_DIRECTORY } from '@server/initializers/constants' |
6 | 6 | ||
7 | // ################## Video file name ################## | 7 | // ################## Video file name ################## |
8 | 8 | ||
@@ -34,6 +34,14 @@ function getVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, vi | |||
34 | return join(baseDir, getVideoFilename(videoOrPlaylist, videoFile)) | 34 | return join(baseDir, getVideoFilename(videoOrPlaylist, videoFile)) |
35 | } | 35 | } |
36 | 36 | ||
37 | // ################## Streaming playlist ################## | ||
38 | |||
39 | function getHLSDirectory (video: MVideoUUID, isRedundancy = false) { | ||
40 | const baseDir = isRedundancy ? HLS_REDUNDANCY_DIRECTORY : HLS_STREAMING_PLAYLIST_DIRECTORY | ||
41 | |||
42 | return join(baseDir, video.uuid) | ||
43 | } | ||
44 | |||
37 | // ################## Torrents ################## | 45 | // ################## Torrents ################## |
38 | 46 | ||
39 | function getTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { | 47 | function getTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { |
@@ -60,5 +68,7 @@ export { | |||
60 | getVideoFilePath, | 68 | getVideoFilePath, |
61 | 69 | ||
62 | getTorrentFileName, | 70 | getTorrentFileName, |
63 | getTorrentFilePath | 71 | getTorrentFilePath, |
72 | |||
73 | getHLSDirectory | ||
64 | } | 74 | } |
diff --git a/server/models/video/video.ts b/server/models/video/video.ts index f84a90992..7e3512fe1 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts | |||
@@ -143,7 +143,7 @@ import { | |||
143 | import { MVideoFile, MVideoFileStreamingPlaylistVideo } from '../../typings/models/video/video-file' | 143 | import { MVideoFile, MVideoFileStreamingPlaylistVideo } from '../../typings/models/video/video-file' |
144 | import { MThumbnail } from '../../typings/models/video/thumbnail' | 144 | import { MThumbnail } from '../../typings/models/video/thumbnail' |
145 | import { VideoFile } from '@shared/models/videos/video-file.model' | 145 | import { VideoFile } from '@shared/models/videos/video-file.model' |
146 | import { getTorrentFileName, getTorrentFilePath, getVideoFilename, getVideoFilePath } from '@server/lib/video-paths' | 146 | import { getTorrentFileName, getTorrentFilePath, getVideoFilename, getVideoFilePath, getHLSDirectory } from '@server/lib/video-paths' |
147 | 147 | ||
148 | // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation | 148 | // FIXME: Define indexes here because there is an issue with TS and Sequelize.literal when called directly in the annotation |
149 | const indexes: (ModelIndexesOptions & { where?: WhereOptions })[] = [ | 149 | const indexes: (ModelIndexesOptions & { where?: WhereOptions })[] = [ |
@@ -1950,11 +1950,10 @@ export class VideoModel extends Model<VideoModel> { | |||
1950 | } | 1950 | } |
1951 | 1951 | ||
1952 | removeStreamingPlaylist (isRedundancy = false) { | 1952 | removeStreamingPlaylist (isRedundancy = false) { |
1953 | const baseDir = isRedundancy ? HLS_REDUNDANCY_DIRECTORY : HLS_STREAMING_PLAYLIST_DIRECTORY | 1953 | const directoryPath = getHLSDirectory(this, isRedundancy) |
1954 | 1954 | ||
1955 | const filePath = join(baseDir, this.uuid) | 1955 | return remove(directoryPath) |
1956 | return remove(filePath) | 1956 | .catch(err => logger.warn('Cannot delete playlist directory %s.', directoryPath, { err })) |
1957 | .catch(err => logger.warn('Cannot delete playlist directory %s.', filePath, { err })) | ||
1958 | } | 1957 | } |
1959 | 1958 | ||
1960 | isOutdated () { | 1959 | isOutdated () { |
diff --git a/server/tests/api/activitypub/refresher.ts b/server/tests/api/activitypub/refresher.ts index 921ee874c..2a6be97c6 100644 --- a/server/tests/api/activitypub/refresher.ts +++ b/server/tests/api/activitypub/refresher.ts | |||
@@ -106,7 +106,7 @@ describe('Test AP refresher', function () { | |||
106 | 106 | ||
107 | await reRunServer(servers[ 1 ]) | 107 | await reRunServer(servers[ 1 ]) |
108 | 108 | ||
109 | // Should not refresh the video, even if the last refresh failed (to avoir a loop on dead instances) | 109 | // Should not refresh the video, even if the last refresh failed (to avoid a loop on dead instances) |
110 | await getVideo(servers[ 0 ].url, videoUUID3) | 110 | await getVideo(servers[ 0 ].url, videoUUID3) |
111 | await waitJobs(servers) | 111 | await waitJobs(servers) |
112 | 112 | ||
diff --git a/server/tests/cli/prune-storage.ts b/server/tests/cli/prune-storage.ts index 67a5c564e..144e67c44 100644 --- a/server/tests/cli/prune-storage.ts +++ b/server/tests/cli/prune-storage.ts | |||
@@ -11,7 +11,7 @@ import { | |||
11 | execCLI, | 11 | execCLI, |
12 | flushAndRunMultipleServers, | 12 | flushAndRunMultipleServers, |
13 | getAccount, | 13 | getAccount, |
14 | getEnvCli, | 14 | getEnvCli, makeGetRequest, makeRawRequest, |
15 | ServerInfo, | 15 | ServerInfo, |
16 | setAccessTokensToServers, setDefaultVideoChannel, | 16 | setAccessTokensToServers, setDefaultVideoChannel, |
17 | updateMyAvatar, | 17 | updateMyAvatar, |
@@ -46,7 +46,7 @@ async function assertCountAreOkay (servers: ServerInfo[]) { | |||
46 | expect(videosCount).to.equal(8) | 46 | expect(videosCount).to.equal(8) |
47 | 47 | ||
48 | const torrentsCount = await countFiles(server.internalServerNumber, 'torrents') | 48 | const torrentsCount = await countFiles(server.internalServerNumber, 'torrents') |
49 | expect(torrentsCount).to.equal(8) | 49 | expect(torrentsCount).to.equal(16) |
50 | 50 | ||
51 | const previewsCount = await countFiles(server.internalServerNumber, 'previews') | 51 | const previewsCount = await countFiles(server.internalServerNumber, 'previews') |
52 | expect(previewsCount).to.equal(2) | 52 | expect(previewsCount).to.equal(2) |
@@ -94,13 +94,21 @@ describe('Test prune storage scripts', function () { | |||
94 | { | 94 | { |
95 | const res = await getAccount(servers[ 0 ].url, 'root@localhost:' + servers[ 1 ].port) | 95 | const res = await getAccount(servers[ 0 ].url, 'root@localhost:' + servers[ 1 ].port) |
96 | const account: Account = res.body | 96 | const account: Account = res.body |
97 | await request('http://localhost:' + servers[ 0 ].port).get(account.avatar.path).expect(200) | 97 | await makeGetRequest({ |
98 | url: servers[ 0 ].url, | ||
99 | path: account.avatar.path, | ||
100 | statusCodeExpected: 200 | ||
101 | }) | ||
98 | } | 102 | } |
99 | 103 | ||
100 | { | 104 | { |
101 | const res = await getAccount(servers[ 1 ].url, 'root@localhost:' + servers[ 0 ].port) | 105 | const res = await getAccount(servers[ 1 ].url, 'root@localhost:' + servers[ 0 ].port) |
102 | const account: Account = res.body | 106 | const account: Account = res.body |
103 | await request('http://localhost:' + servers[ 1 ].port).get(account.avatar.path).expect(200) | 107 | await makeGetRequest({ |
108 | url: servers[ 1 ].url, | ||
109 | path: account.avatar.path, | ||
110 | statusCodeExpected: 200 | ||
111 | }) | ||
104 | } | 112 | } |
105 | 113 | ||
106 | await wait(1000) | 114 | await wait(1000) |