diff options
Diffstat (limited to 'server/lib')
69 files changed, 1084 insertions, 488 deletions
diff --git a/server/lib/activitypub/activity.ts b/server/lib/activitypub/activity.ts index 1f6ec221e..0fed3e8fd 100644 --- a/server/lib/activitypub/activity.ts +++ b/server/lib/activitypub/activity.ts | |||
@@ -1,4 +1,5 @@ | |||
1 | import { ActivityType } from '@shared/models' | 1 | import { doJSONRequest } from '@server/helpers/requests' |
2 | import { APObjectId, ActivityObject, ActivityPubActor, ActivityType } from '@shared/models' | ||
2 | 3 | ||
3 | function getAPId (object: string | { id: string }) { | 4 | function getAPId (object: string | { id: string }) { |
4 | if (typeof object === 'string') return object | 5 | if (typeof object === 'string') return object |
@@ -32,8 +33,19 @@ function buildAvailableActivities (): ActivityType[] { | |||
32 | ] | 33 | ] |
33 | } | 34 | } |
34 | 35 | ||
36 | async function fetchAPObject <T extends (ActivityObject | ActivityPubActor)> (object: APObjectId) { | ||
37 | if (typeof object === 'string') { | ||
38 | const { body } = await doJSONRequest<Exclude<T, string>>(object, { activityPub: true }) | ||
39 | |||
40 | return body | ||
41 | } | ||
42 | |||
43 | return object as Exclude<T, string> | ||
44 | } | ||
45 | |||
35 | export { | 46 | export { |
36 | getAPId, | 47 | getAPId, |
48 | fetchAPObject, | ||
37 | getActivityStreamDuration, | 49 | getActivityStreamDuration, |
38 | buildAvailableActivities, | 50 | buildAvailableActivities, |
39 | getDurationFromActivityStream | 51 | getDurationFromActivityStream |
diff --git a/server/lib/activitypub/actors/get.ts b/server/lib/activitypub/actors/get.ts index e73b7d707..b2be3f5fb 100644 --- a/server/lib/activitypub/actors/get.ts +++ b/server/lib/activitypub/actors/get.ts | |||
@@ -3,8 +3,9 @@ import { logger } from '@server/helpers/logger' | |||
3 | import { JobQueue } from '@server/lib/job-queue' | 3 | import { JobQueue } from '@server/lib/job-queue' |
4 | import { ActorLoadByUrlType, loadActorByUrl } from '@server/lib/model-loaders' | 4 | import { ActorLoadByUrlType, loadActorByUrl } from '@server/lib/model-loaders' |
5 | import { MActor, MActorAccountChannelId, MActorAccountChannelIdActor, MActorAccountId, MActorFullActor } from '@server/types/models' | 5 | import { MActor, MActorAccountChannelId, MActorAccountChannelIdActor, MActorAccountId, MActorFullActor } from '@server/types/models' |
6 | import { ActivityPubActor } from '@shared/models' | 6 | import { arrayify } from '@shared/core-utils' |
7 | import { getAPId } from '../activity' | 7 | import { ActivityPubActor, APObjectId } from '@shared/models' |
8 | import { fetchAPObject, getAPId } from '../activity' | ||
8 | import { checkUrlsSameHost } from '../url' | 9 | import { checkUrlsSameHost } from '../url' |
9 | import { refreshActorIfNeeded } from './refresh' | 10 | import { refreshActorIfNeeded } from './refresh' |
10 | import { APActorCreator, fetchRemoteActor } from './shared' | 11 | import { APActorCreator, fetchRemoteActor } from './shared' |
@@ -40,7 +41,7 @@ async function getOrCreateAPActor ( | |||
40 | const { actorObject } = await fetchRemoteActor(actorUrl) | 41 | const { actorObject } = await fetchRemoteActor(actorUrl) |
41 | if (actorObject === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl) | 42 | if (actorObject === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl) |
42 | 43 | ||
43 | // actorUrl is just an alias/rediraction, so process object id instead | 44 | // actorUrl is just an alias/redirection, so process object id instead |
44 | if (actorObject.id !== actorUrl) return getOrCreateAPActor(actorObject, 'all', recurseIfNeeded, updateCollections) | 45 | if (actorObject.id !== actorUrl) return getOrCreateAPActor(actorObject, 'all', recurseIfNeeded, updateCollections) |
45 | 46 | ||
46 | // Create the attributed to actor | 47 | // Create the attributed to actor |
@@ -68,29 +69,48 @@ async function getOrCreateAPActor ( | |||
68 | return actorRefreshed | 69 | return actorRefreshed |
69 | } | 70 | } |
70 | 71 | ||
71 | function getOrCreateAPOwner (actorObject: ActivityPubActor, actorUrl: string) { | 72 | async function getOrCreateAPOwner (actorObject: ActivityPubActor, actorUrl: string) { |
72 | const accountAttributedTo = actorObject.attributedTo.find(a => a.type === 'Person') | 73 | const accountAttributedTo = await findOwner(actorUrl, actorObject.attributedTo, 'Person') |
73 | if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actorUrl) | 74 | if (!accountAttributedTo) { |
74 | 75 | throw new Error(`Cannot find account attributed to video channel ${actorUrl}`) | |
75 | if (checkUrlsSameHost(accountAttributedTo.id, actorUrl) !== true) { | ||
76 | throw new Error(`Account attributed to ${accountAttributedTo.id} does not have the same host than actor url ${actorUrl}`) | ||
77 | } | 76 | } |
78 | 77 | ||
79 | try { | 78 | try { |
80 | // Don't recurse another time | 79 | // Don't recurse another time |
81 | const recurseIfNeeded = false | 80 | const recurseIfNeeded = false |
82 | return getOrCreateAPActor(accountAttributedTo.id, 'all', recurseIfNeeded) | 81 | return getOrCreateAPActor(accountAttributedTo, 'all', recurseIfNeeded) |
83 | } catch (err) { | 82 | } catch (err) { |
84 | logger.error('Cannot get or create account attributed to video channel ' + actorUrl) | 83 | logger.error('Cannot get or create account attributed to video channel ' + actorUrl) |
85 | throw new Error(err) | 84 | throw new Error(err) |
86 | } | 85 | } |
87 | } | 86 | } |
88 | 87 | ||
88 | async function findOwner (rootUrl: string, attributedTo: APObjectId[] | APObjectId, type: 'Person' | 'Group') { | ||
89 | for (const actorToCheck of arrayify(attributedTo)) { | ||
90 | const actorObject = await fetchAPObject<ActivityPubActor>(getAPId(actorToCheck)) | ||
91 | |||
92 | if (!actorObject) { | ||
93 | logger.warn('Unknown attributed to actor %s for owner %s', actorToCheck, rootUrl) | ||
94 | continue | ||
95 | } | ||
96 | |||
97 | if (checkUrlsSameHost(actorObject.id, rootUrl) !== true) { | ||
98 | logger.warn(`Account attributed to ${actorObject.id} does not have the same host than owner actor url ${rootUrl}`) | ||
99 | continue | ||
100 | } | ||
101 | |||
102 | if (actorObject.type === type) return actorObject | ||
103 | } | ||
104 | |||
105 | return undefined | ||
106 | } | ||
107 | |||
89 | // --------------------------------------------------------------------------- | 108 | // --------------------------------------------------------------------------- |
90 | 109 | ||
91 | export { | 110 | export { |
92 | getOrCreateAPOwner, | 111 | getOrCreateAPOwner, |
93 | getOrCreateAPActor | 112 | getOrCreateAPActor, |
113 | findOwner | ||
94 | } | 114 | } |
95 | 115 | ||
96 | // --------------------------------------------------------------------------- | 116 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/activitypub/actors/refresh.ts b/server/lib/activitypub/actors/refresh.ts index 6d8428d66..d15cb5e90 100644 --- a/server/lib/activitypub/actors/refresh.ts +++ b/server/lib/activitypub/actors/refresh.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | 1 | import { logger, loggerTagsFactory } from '@server/helpers/logger' |
2 | import { PromiseCache } from '@server/helpers/promise-cache' | 2 | import { CachePromiseFactory } from '@server/helpers/promise-cache' |
3 | import { PeerTubeRequestError } from '@server/helpers/requests' | 3 | import { PeerTubeRequestError } from '@server/helpers/requests' |
4 | import { ActorLoadByUrlType } from '@server/lib/model-loaders' | 4 | import { ActorLoadByUrlType } from '@server/lib/model-loaders' |
5 | import { ActorModel } from '@server/models/actor/actor' | 5 | import { ActorModel } from '@server/models/actor/actor' |
@@ -16,7 +16,7 @@ type RefreshOptions <T> = { | |||
16 | fetchedType: ActorLoadByUrlType | 16 | fetchedType: ActorLoadByUrlType |
17 | } | 17 | } |
18 | 18 | ||
19 | const promiseCache = new PromiseCache(doRefresh, (options: RefreshOptions<MActorFull | MActorAccountChannelId>) => options.actor.url) | 19 | const promiseCache = new CachePromiseFactory(doRefresh, (options: RefreshOptions<MActorFull | MActorAccountChannelId>) => options.actor.url) |
20 | 20 | ||
21 | function refreshActorIfNeeded <T extends MActorFull | MActorAccountChannelId> (options: RefreshOptions<T>): RefreshResult <T> { | 21 | function refreshActorIfNeeded <T extends MActorFull | MActorAccountChannelId> (options: RefreshOptions<T>): RefreshResult <T> { |
22 | const actorArg = options.actor | 22 | const actorArg = options.actor |
diff --git a/server/lib/activitypub/context.ts b/server/lib/activitypub/context.ts index a3ca52a31..750276a11 100644 --- a/server/lib/activitypub/context.ts +++ b/server/lib/activitypub/context.ts | |||
@@ -46,6 +46,19 @@ const contextStore: { [ id in ContextType ]: (string | { [ id: string ]: string | |||
46 | 46 | ||
47 | Infohash: 'pt:Infohash', | 47 | Infohash: 'pt:Infohash', |
48 | 48 | ||
49 | tileWidth: { | ||
50 | '@type': 'sc:Number', | ||
51 | '@id': 'pt:tileWidth' | ||
52 | }, | ||
53 | tileHeight: { | ||
54 | '@type': 'sc:Number', | ||
55 | '@id': 'pt:tileHeight' | ||
56 | }, | ||
57 | tileDuration: { | ||
58 | '@type': 'sc:Number', | ||
59 | '@id': 'pt:tileDuration' | ||
60 | }, | ||
61 | |||
49 | originallyPublishedAt: 'sc:datePublished', | 62 | originallyPublishedAt: 'sc:datePublished', |
50 | views: { | 63 | views: { |
51 | '@type': 'sc:Number', | 64 | '@type': 'sc:Number', |
diff --git a/server/lib/activitypub/playlists/create-update.ts b/server/lib/activitypub/playlists/create-update.ts index 9339e8ea4..b24299f29 100644 --- a/server/lib/activitypub/playlists/create-update.ts +++ b/server/lib/activitypub/playlists/create-update.ts | |||
@@ -4,7 +4,7 @@ import { retryTransactionWrapper } from '@server/helpers/database-utils' | |||
4 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | 4 | import { logger, loggerTagsFactory } from '@server/helpers/logger' |
5 | import { CRAWL_REQUEST_CONCURRENCY } from '@server/initializers/constants' | 5 | import { CRAWL_REQUEST_CONCURRENCY } from '@server/initializers/constants' |
6 | import { sequelizeTypescript } from '@server/initializers/database' | 6 | import { sequelizeTypescript } from '@server/initializers/database' |
7 | import { updatePlaylistMiniatureFromUrl } from '@server/lib/thumbnail' | 7 | import { updateRemotePlaylistMiniatureFromUrl } from '@server/lib/thumbnail' |
8 | import { VideoPlaylistModel } from '@server/models/video/video-playlist' | 8 | import { VideoPlaylistModel } from '@server/models/video/video-playlist' |
9 | import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element' | 9 | import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element' |
10 | import { FilteredModelAttributes } from '@server/types' | 10 | import { FilteredModelAttributes } from '@server/types' |
@@ -77,7 +77,7 @@ async function setVideoChannel (playlistObject: PlaylistObject, playlistAttribut | |||
77 | throw new Error('Not attributed to for playlist object ' + getAPId(playlistObject)) | 77 | throw new Error('Not attributed to for playlist object ' + getAPId(playlistObject)) |
78 | } | 78 | } |
79 | 79 | ||
80 | const actor = await getOrCreateAPActor(playlistObject.attributedTo[0], 'all') | 80 | const actor = await getOrCreateAPActor(getAPId(playlistObject.attributedTo[0]), 'all') |
81 | 81 | ||
82 | if (!actor.VideoChannel) { | 82 | if (!actor.VideoChannel) { |
83 | logger.warn('Playlist "attributedTo" %s is not a video channel.', playlistObject.id, { playlistObject, ...lTags(playlistObject.id) }) | 83 | logger.warn('Playlist "attributedTo" %s is not a video channel.', playlistObject.id, { playlistObject, ...lTags(playlistObject.id) }) |
@@ -104,7 +104,7 @@ async function updatePlaylistThumbnail (playlistObject: PlaylistObject, playlist | |||
104 | let thumbnailModel: MThumbnail | 104 | let thumbnailModel: MThumbnail |
105 | 105 | ||
106 | try { | 106 | try { |
107 | thumbnailModel = await updatePlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist }) | 107 | thumbnailModel = await updateRemotePlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist }) |
108 | await playlist.setAndSaveThumbnail(thumbnailModel, undefined) | 108 | await playlist.setAndSaveThumbnail(thumbnailModel, undefined) |
109 | } catch (err) { | 109 | } catch (err) { |
110 | logger.warn('Cannot set thumbnail of %s.', playlistObject.id, { err, ...lTags(playlistObject.id, playlist.uuid, playlist.url) }) | 110 | logger.warn('Cannot set thumbnail of %s.', playlistObject.id, { err, ...lTags(playlistObject.id, playlist.uuid, playlist.url) }) |
diff --git a/server/lib/activitypub/playlists/get.ts b/server/lib/activitypub/playlists/get.ts index bfaf52cc9..c34554d69 100644 --- a/server/lib/activitypub/playlists/get.ts +++ b/server/lib/activitypub/playlists/get.ts | |||
@@ -1,12 +1,12 @@ | |||
1 | import { VideoPlaylistModel } from '@server/models/video/video-playlist' | 1 | import { VideoPlaylistModel } from '@server/models/video/video-playlist' |
2 | import { MVideoPlaylistFullSummary } from '@server/types/models' | 2 | import { MVideoPlaylistFullSummary } from '@server/types/models' |
3 | import { APObject } from '@shared/models' | 3 | import { APObjectId } from '@shared/models' |
4 | import { getAPId } from '../activity' | 4 | import { getAPId } from '../activity' |
5 | import { createOrUpdateVideoPlaylist } from './create-update' | 5 | import { createOrUpdateVideoPlaylist } from './create-update' |
6 | import { scheduleRefreshIfNeeded } from './refresh' | 6 | import { scheduleRefreshIfNeeded } from './refresh' |
7 | import { fetchRemoteVideoPlaylist } from './shared' | 7 | import { fetchRemoteVideoPlaylist } from './shared' |
8 | 8 | ||
9 | async function getOrCreateAPVideoPlaylist (playlistObjectArg: APObject): Promise<MVideoPlaylistFullSummary> { | 9 | async function getOrCreateAPVideoPlaylist (playlistObjectArg: APObjectId): Promise<MVideoPlaylistFullSummary> { |
10 | const playlistUrl = getAPId(playlistObjectArg) | 10 | const playlistUrl = getAPId(playlistObjectArg) |
11 | 11 | ||
12 | const playlistFromDatabase = await VideoPlaylistModel.loadByUrlWithAccountAndChannelSummary(playlistUrl) | 12 | const playlistFromDatabase = await VideoPlaylistModel.loadByUrlWithAccountAndChannelSummary(playlistUrl) |
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 1e6e8956c..e89d1ab45 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts | |||
@@ -1,13 +1,24 @@ | |||
1 | import { isBlockedByServerOrAccount } from '@server/lib/blocklist' | 1 | import { isBlockedByServerOrAccount } from '@server/lib/blocklist' |
2 | import { isRedundancyAccepted } from '@server/lib/redundancy' | 2 | import { isRedundancyAccepted } from '@server/lib/redundancy' |
3 | import { VideoModel } from '@server/models/video/video' | 3 | import { VideoModel } from '@server/models/video/video' |
4 | import { ActivityCreate, CacheFileObject, PlaylistObject, VideoCommentObject, VideoObject, WatchActionObject } from '@shared/models' | 4 | import { |
5 | AbuseObject, | ||
6 | ActivityCreate, | ||
7 | ActivityCreateObject, | ||
8 | ActivityObject, | ||
9 | CacheFileObject, | ||
10 | PlaylistObject, | ||
11 | VideoCommentObject, | ||
12 | VideoObject, | ||
13 | WatchActionObject | ||
14 | } from '@shared/models' | ||
5 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | 15 | import { retryTransactionWrapper } from '../../../helpers/database-utils' |
6 | import { logger } from '../../../helpers/logger' | 16 | import { logger } from '../../../helpers/logger' |
7 | import { sequelizeTypescript } from '../../../initializers/database' | 17 | import { sequelizeTypescript } from '../../../initializers/database' |
8 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | 18 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' |
9 | import { MActorSignature, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../../types/models' | 19 | import { MActorSignature, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../../types/models' |
10 | import { Notifier } from '../../notifier' | 20 | import { Notifier } from '../../notifier' |
21 | import { fetchAPObject } from '../activity' | ||
11 | import { createOrUpdateCacheFile } from '../cache-file' | 22 | import { createOrUpdateCacheFile } from '../cache-file' |
12 | import { createOrUpdateLocalVideoViewer } from '../local-video-viewer' | 23 | import { createOrUpdateLocalVideoViewer } from '../local-video-viewer' |
13 | import { createOrUpdateVideoPlaylist } from '../playlists' | 24 | import { createOrUpdateVideoPlaylist } from '../playlists' |
@@ -15,35 +26,35 @@ import { forwardVideoRelatedActivity } from '../send/shared/send-utils' | |||
15 | import { resolveThread } from '../video-comments' | 26 | import { resolveThread } from '../video-comments' |
16 | import { getOrCreateAPVideo } from '../videos' | 27 | import { getOrCreateAPVideo } from '../videos' |
17 | 28 | ||
18 | async function processCreateActivity (options: APProcessorOptions<ActivityCreate>) { | 29 | async function processCreateActivity (options: APProcessorOptions<ActivityCreate<ActivityCreateObject>>) { |
19 | const { activity, byActor } = options | 30 | const { activity, byActor } = options |
20 | 31 | ||
21 | // Only notify if it is not from a fetcher job | 32 | // Only notify if it is not from a fetcher job |
22 | const notify = options.fromFetch !== true | 33 | const notify = options.fromFetch !== true |
23 | const activityObject = activity.object | 34 | const activityObject = await fetchAPObject<Exclude<ActivityObject, AbuseObject>>(activity.object) |
24 | const activityType = activityObject.type | 35 | const activityType = activityObject.type |
25 | 36 | ||
26 | if (activityType === 'Video') { | 37 | if (activityType === 'Video') { |
27 | return processCreateVideo(activity, notify) | 38 | return processCreateVideo(activityObject, notify) |
28 | } | 39 | } |
29 | 40 | ||
30 | if (activityType === 'Note') { | 41 | if (activityType === 'Note') { |
31 | // Comments will be fetched from videos | 42 | // Comments will be fetched from videos |
32 | if (options.fromFetch) return | 43 | if (options.fromFetch) return |
33 | 44 | ||
34 | return retryTransactionWrapper(processCreateVideoComment, activity, byActor, notify) | 45 | return retryTransactionWrapper(processCreateVideoComment, activity, activityObject, byActor, notify) |
35 | } | 46 | } |
36 | 47 | ||
37 | if (activityType === 'WatchAction') { | 48 | if (activityType === 'WatchAction') { |
38 | return retryTransactionWrapper(processCreateWatchAction, activity) | 49 | return retryTransactionWrapper(processCreateWatchAction, activityObject) |
39 | } | 50 | } |
40 | 51 | ||
41 | if (activityType === 'CacheFile') { | 52 | if (activityType === 'CacheFile') { |
42 | return retryTransactionWrapper(processCreateCacheFile, activity, byActor) | 53 | return retryTransactionWrapper(processCreateCacheFile, activity, activityObject, byActor) |
43 | } | 54 | } |
44 | 55 | ||
45 | if (activityType === 'Playlist') { | 56 | if (activityType === 'Playlist') { |
46 | return retryTransactionWrapper(processCreatePlaylist, activity, byActor) | 57 | return retryTransactionWrapper(processCreatePlaylist, activity, activityObject, byActor) |
47 | } | 58 | } |
48 | 59 | ||
49 | logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) | 60 | logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) |
@@ -58,9 +69,7 @@ export { | |||
58 | 69 | ||
59 | // --------------------------------------------------------------------------- | 70 | // --------------------------------------------------------------------------- |
60 | 71 | ||
61 | async function processCreateVideo (activity: ActivityCreate, notify: boolean) { | 72 | async function processCreateVideo (videoToCreateData: VideoObject, notify: boolean) { |
62 | const videoToCreateData = activity.object as VideoObject | ||
63 | |||
64 | const syncParam = { rates: false, shares: false, comments: false, thumbnail: true, refreshVideo: false } | 73 | const syncParam = { rates: false, shares: false, comments: false, thumbnail: true, refreshVideo: false } |
65 | const { video, created } = await getOrCreateAPVideo({ videoObject: videoToCreateData, syncParam }) | 74 | const { video, created } = await getOrCreateAPVideo({ videoObject: videoToCreateData, syncParam }) |
66 | 75 | ||
@@ -69,11 +78,13 @@ async function processCreateVideo (activity: ActivityCreate, notify: boolean) { | |||
69 | return video | 78 | return video |
70 | } | 79 | } |
71 | 80 | ||
72 | async function processCreateCacheFile (activity: ActivityCreate, byActor: MActorSignature) { | 81 | async function processCreateCacheFile ( |
82 | activity: ActivityCreate<CacheFileObject | string>, | ||
83 | cacheFile: CacheFileObject, | ||
84 | byActor: MActorSignature | ||
85 | ) { | ||
73 | if (await isRedundancyAccepted(activity, byActor) !== true) return | 86 | if (await isRedundancyAccepted(activity, byActor) !== true) return |
74 | 87 | ||
75 | const cacheFile = activity.object as CacheFileObject | ||
76 | |||
77 | const { video } = await getOrCreateAPVideo({ videoObject: cacheFile.object }) | 88 | const { video } = await getOrCreateAPVideo({ videoObject: cacheFile.object }) |
78 | 89 | ||
79 | await sequelizeTypescript.transaction(async t => { | 90 | await sequelizeTypescript.transaction(async t => { |
@@ -87,9 +98,7 @@ async function processCreateCacheFile (activity: ActivityCreate, byActor: MActor | |||
87 | } | 98 | } |
88 | } | 99 | } |
89 | 100 | ||
90 | async function processCreateWatchAction (activity: ActivityCreate) { | 101 | async function processCreateWatchAction (watchAction: WatchActionObject) { |
91 | const watchAction = activity.object as WatchActionObject | ||
92 | |||
93 | if (watchAction.actionStatus !== 'CompletedActionStatus') return | 102 | if (watchAction.actionStatus !== 'CompletedActionStatus') return |
94 | 103 | ||
95 | const video = await VideoModel.loadByUrl(watchAction.object) | 104 | const video = await VideoModel.loadByUrl(watchAction.object) |
@@ -100,8 +109,12 @@ async function processCreateWatchAction (activity: ActivityCreate) { | |||
100 | }) | 109 | }) |
101 | } | 110 | } |
102 | 111 | ||
103 | async function processCreateVideoComment (activity: ActivityCreate, byActor: MActorSignature, notify: boolean) { | 112 | async function processCreateVideoComment ( |
104 | const commentObject = activity.object as VideoCommentObject | 113 | activity: ActivityCreate<VideoCommentObject | string>, |
114 | commentObject: VideoCommentObject, | ||
115 | byActor: MActorSignature, | ||
116 | notify: boolean | ||
117 | ) { | ||
105 | const byAccount = byActor.Account | 118 | const byAccount = byActor.Account |
106 | 119 | ||
107 | if (!byAccount) throw new Error('Cannot create video comment with the non account actor ' + byActor.url) | 120 | if (!byAccount) throw new Error('Cannot create video comment with the non account actor ' + byActor.url) |
@@ -144,8 +157,11 @@ async function processCreateVideoComment (activity: ActivityCreate, byActor: MAc | |||
144 | if (created && notify) Notifier.Instance.notifyOnNewComment(comment) | 157 | if (created && notify) Notifier.Instance.notifyOnNewComment(comment) |
145 | } | 158 | } |
146 | 159 | ||
147 | async function processCreatePlaylist (activity: ActivityCreate, byActor: MActorSignature) { | 160 | async function processCreatePlaylist ( |
148 | const playlistObject = activity.object as PlaylistObject | 161 | activity: ActivityCreate<PlaylistObject | string>, |
162 | playlistObject: PlaylistObject, | ||
163 | byActor: MActorSignature | ||
164 | ) { | ||
149 | const byAccount = byActor.Account | 165 | const byAccount = byActor.Account |
150 | 166 | ||
151 | if (!byAccount) throw new Error('Cannot create video playlist with the non account actor ' + byActor.url) | 167 | if (!byAccount) throw new Error('Cannot create video playlist with the non account actor ' + byActor.url) |
diff --git a/server/lib/activitypub/process/process-dislike.ts b/server/lib/activitypub/process/process-dislike.ts index 44e349b22..4e270f917 100644 --- a/server/lib/activitypub/process/process-dislike.ts +++ b/server/lib/activitypub/process/process-dislike.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import { VideoModel } from '@server/models/video/video' | 1 | import { VideoModel } from '@server/models/video/video' |
2 | import { ActivityCreate, ActivityDislike, DislikeObject } from '@shared/models' | 2 | import { ActivityDislike } from '@shared/models' |
3 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | 3 | import { retryTransactionWrapper } from '../../../helpers/database-utils' |
4 | import { sequelizeTypescript } from '../../../initializers/database' | 4 | import { sequelizeTypescript } from '../../../initializers/database' |
5 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' | 5 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' |
@@ -7,7 +7,7 @@ import { APProcessorOptions } from '../../../types/activitypub-processor.model' | |||
7 | import { MActorSignature } from '../../../types/models' | 7 | import { MActorSignature } from '../../../types/models' |
8 | import { federateVideoIfNeeded, getOrCreateAPVideo } from '../videos' | 8 | import { federateVideoIfNeeded, getOrCreateAPVideo } from '../videos' |
9 | 9 | ||
10 | async function processDislikeActivity (options: APProcessorOptions<ActivityCreate | ActivityDislike>) { | 10 | async function processDislikeActivity (options: APProcessorOptions<ActivityDislike>) { |
11 | const { activity, byActor } = options | 11 | const { activity, byActor } = options |
12 | return retryTransactionWrapper(processDislike, activity, byActor) | 12 | return retryTransactionWrapper(processDislike, activity, byActor) |
13 | } | 13 | } |
@@ -20,11 +20,8 @@ export { | |||
20 | 20 | ||
21 | // --------------------------------------------------------------------------- | 21 | // --------------------------------------------------------------------------- |
22 | 22 | ||
23 | async function processDislike (activity: ActivityCreate | ActivityDislike, byActor: MActorSignature) { | 23 | async function processDislike (activity: ActivityDislike, byActor: MActorSignature) { |
24 | const dislikeObject = activity.type === 'Dislike' | 24 | const dislikeObject = activity.object |
25 | ? activity.object | ||
26 | : (activity.object as DislikeObject).object | ||
27 | |||
28 | const byAccount = byActor.Account | 25 | const byAccount = byActor.Account |
29 | 26 | ||
30 | if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) | 27 | if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) |
diff --git a/server/lib/activitypub/process/process-flag.ts b/server/lib/activitypub/process/process-flag.ts index 10f58ef27..bea285670 100644 --- a/server/lib/activitypub/process/process-flag.ts +++ b/server/lib/activitypub/process/process-flag.ts | |||
@@ -3,7 +3,7 @@ import { AccountModel } from '@server/models/account/account' | |||
3 | import { VideoModel } from '@server/models/video/video' | 3 | import { VideoModel } from '@server/models/video/video' |
4 | import { VideoCommentModel } from '@server/models/video/video-comment' | 4 | import { VideoCommentModel } from '@server/models/video/video-comment' |
5 | import { abusePredefinedReasonsMap } from '@shared/core-utils/abuse' | 5 | import { abusePredefinedReasonsMap } from '@shared/core-utils/abuse' |
6 | import { AbuseObject, AbuseState, ActivityCreate, ActivityFlag } from '@shared/models' | 6 | import { AbuseState, ActivityFlag } from '@shared/models' |
7 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | 7 | import { retryTransactionWrapper } from '../../../helpers/database-utils' |
8 | import { logger } from '../../../helpers/logger' | 8 | import { logger } from '../../../helpers/logger' |
9 | import { sequelizeTypescript } from '../../../initializers/database' | 9 | import { sequelizeTypescript } from '../../../initializers/database' |
@@ -11,7 +11,7 @@ import { getAPId } from '../../../lib/activitypub/activity' | |||
11 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | 11 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' |
12 | import { MAccountDefault, MActorSignature, MCommentOwnerVideo } from '../../../types/models' | 12 | import { MAccountDefault, MActorSignature, MCommentOwnerVideo } from '../../../types/models' |
13 | 13 | ||
14 | async function processFlagActivity (options: APProcessorOptions<ActivityCreate | ActivityFlag>) { | 14 | async function processFlagActivity (options: APProcessorOptions<ActivityFlag>) { |
15 | const { activity, byActor } = options | 15 | const { activity, byActor } = options |
16 | 16 | ||
17 | return retryTransactionWrapper(processCreateAbuse, activity, byActor) | 17 | return retryTransactionWrapper(processCreateAbuse, activity, byActor) |
@@ -25,9 +25,7 @@ export { | |||
25 | 25 | ||
26 | // --------------------------------------------------------------------------- | 26 | // --------------------------------------------------------------------------- |
27 | 27 | ||
28 | async function processCreateAbuse (activity: ActivityCreate | ActivityFlag, byActor: MActorSignature) { | 28 | async function processCreateAbuse (flag: ActivityFlag, byActor: MActorSignature) { |
29 | const flag = activity.type === 'Flag' ? activity : (activity.object as AbuseObject) | ||
30 | |||
31 | const account = byActor.Account | 29 | const account = byActor.Account |
32 | if (!account) throw new Error('Cannot create abuse with the non account actor ' + byActor.url) | 30 | if (!account) throw new Error('Cannot create abuse with the non account actor ' + byActor.url) |
33 | 31 | ||
diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts index 99423a72b..25f68724d 100644 --- a/server/lib/activitypub/process/process-undo.ts +++ b/server/lib/activitypub/process/process-undo.ts | |||
@@ -1,6 +1,14 @@ | |||
1 | import { VideoModel } from '@server/models/video/video' | 1 | import { VideoModel } from '@server/models/video/video' |
2 | import { ActivityAnnounce, ActivityFollow, ActivityLike, ActivityUndo, CacheFileObject } from '../../../../shared/models/activitypub' | 2 | import { |
3 | import { DislikeObject } from '../../../../shared/models/activitypub/objects' | 3 | ActivityAnnounce, |
4 | ActivityCreate, | ||
5 | ActivityDislike, | ||
6 | ActivityFollow, | ||
7 | ActivityLike, | ||
8 | ActivityUndo, | ||
9 | ActivityUndoObject, | ||
10 | CacheFileObject | ||
11 | } from '../../../../shared/models/activitypub' | ||
4 | import { retryTransactionWrapper } from '../../../helpers/database-utils' | 12 | import { retryTransactionWrapper } from '../../../helpers/database-utils' |
5 | import { logger } from '../../../helpers/logger' | 13 | import { logger } from '../../../helpers/logger' |
6 | import { sequelizeTypescript } from '../../../initializers/database' | 14 | import { sequelizeTypescript } from '../../../initializers/database' |
@@ -11,10 +19,11 @@ import { VideoRedundancyModel } from '../../../models/redundancy/video-redundanc | |||
11 | import { VideoShareModel } from '../../../models/video/video-share' | 19 | import { VideoShareModel } from '../../../models/video/video-share' |
12 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | 20 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' |
13 | import { MActorSignature } from '../../../types/models' | 21 | import { MActorSignature } from '../../../types/models' |
22 | import { fetchAPObject } from '../activity' | ||
14 | import { forwardVideoRelatedActivity } from '../send/shared/send-utils' | 23 | import { forwardVideoRelatedActivity } from '../send/shared/send-utils' |
15 | import { federateVideoIfNeeded, getOrCreateAPVideo } from '../videos' | 24 | import { federateVideoIfNeeded, getOrCreateAPVideo } from '../videos' |
16 | 25 | ||
17 | async function processUndoActivity (options: APProcessorOptions<ActivityUndo>) { | 26 | async function processUndoActivity (options: APProcessorOptions<ActivityUndo<ActivityUndoObject>>) { |
18 | const { activity, byActor } = options | 27 | const { activity, byActor } = options |
19 | const activityToUndo = activity.object | 28 | const activityToUndo = activity.object |
20 | 29 | ||
@@ -23,8 +32,10 @@ async function processUndoActivity (options: APProcessorOptions<ActivityUndo>) { | |||
23 | } | 32 | } |
24 | 33 | ||
25 | if (activityToUndo.type === 'Create') { | 34 | if (activityToUndo.type === 'Create') { |
26 | if (activityToUndo.object.type === 'CacheFile') { | 35 | const objectToUndo = await fetchAPObject<CacheFileObject>(activityToUndo.object) |
27 | return retryTransactionWrapper(processUndoCacheFile, byActor, activity) | 36 | |
37 | if (objectToUndo.type === 'CacheFile') { | ||
38 | return retryTransactionWrapper(processUndoCacheFile, byActor, activity, objectToUndo) | ||
28 | } | 39 | } |
29 | } | 40 | } |
30 | 41 | ||
@@ -53,8 +64,8 @@ export { | |||
53 | 64 | ||
54 | // --------------------------------------------------------------------------- | 65 | // --------------------------------------------------------------------------- |
55 | 66 | ||
56 | async function processUndoLike (byActor: MActorSignature, activity: ActivityUndo) { | 67 | async function processUndoLike (byActor: MActorSignature, activity: ActivityUndo<ActivityLike>) { |
57 | const likeActivity = activity.object as ActivityLike | 68 | const likeActivity = activity.object |
58 | 69 | ||
59 | const { video: onlyVideo } = await getOrCreateAPVideo({ videoObject: likeActivity.object }) | 70 | const { video: onlyVideo } = await getOrCreateAPVideo({ videoObject: likeActivity.object }) |
60 | // We don't care about likes of remote videos | 71 | // We don't care about likes of remote videos |
@@ -78,12 +89,10 @@ async function processUndoLike (byActor: MActorSignature, activity: ActivityUndo | |||
78 | }) | 89 | }) |
79 | } | 90 | } |
80 | 91 | ||
81 | async function processUndoDislike (byActor: MActorSignature, activity: ActivityUndo) { | 92 | async function processUndoDislike (byActor: MActorSignature, activity: ActivityUndo<ActivityDislike>) { |
82 | const dislike = activity.object.type === 'Dislike' | 93 | const dislikeActivity = activity.object |
83 | ? activity.object | ||
84 | : activity.object.object as DislikeObject | ||
85 | 94 | ||
86 | const { video: onlyVideo } = await getOrCreateAPVideo({ videoObject: dislike.object }) | 95 | const { video: onlyVideo } = await getOrCreateAPVideo({ videoObject: dislikeActivity.object }) |
87 | // We don't care about likes of remote videos | 96 | // We don't care about likes of remote videos |
88 | if (!onlyVideo.isOwned()) return | 97 | if (!onlyVideo.isOwned()) return |
89 | 98 | ||
@@ -91,7 +100,7 @@ async function processUndoDislike (byActor: MActorSignature, activity: ActivityU | |||
91 | if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) | 100 | if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) |
92 | 101 | ||
93 | const video = await VideoModel.loadFull(onlyVideo.id, t) | 102 | const video = await VideoModel.loadFull(onlyVideo.id, t) |
94 | const rate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byActor.Account.id, video.id, dislike.id, t) | 103 | const rate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byActor.Account.id, video.id, dislikeActivity.id, t) |
95 | if (!rate || rate.type !== 'dislike') { | 104 | if (!rate || rate.type !== 'dislike') { |
96 | logger.warn(`Unknown dislike by account %d for video %d.`, byActor.Account.id, video.id) | 105 | logger.warn(`Unknown dislike by account %d for video %d.`, byActor.Account.id, video.id) |
97 | return | 106 | return |
@@ -107,9 +116,11 @@ async function processUndoDislike (byActor: MActorSignature, activity: ActivityU | |||
107 | 116 | ||
108 | // --------------------------------------------------------------------------- | 117 | // --------------------------------------------------------------------------- |
109 | 118 | ||
110 | async function processUndoCacheFile (byActor: MActorSignature, activity: ActivityUndo) { | 119 | async function processUndoCacheFile ( |
111 | const cacheFileObject = activity.object.object as CacheFileObject | 120 | byActor: MActorSignature, |
112 | 121 | activity: ActivityUndo<ActivityCreate<CacheFileObject>>, | |
122 | cacheFileObject: CacheFileObject | ||
123 | ) { | ||
113 | const { video } = await getOrCreateAPVideo({ videoObject: cacheFileObject.object }) | 124 | const { video } = await getOrCreateAPVideo({ videoObject: cacheFileObject.object }) |
114 | 125 | ||
115 | return sequelizeTypescript.transaction(async t => { | 126 | return sequelizeTypescript.transaction(async t => { |
diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index 4afdbd430..9caa74e04 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import { isRedundancyAccepted } from '@server/lib/redundancy' | 1 | import { isRedundancyAccepted } from '@server/lib/redundancy' |
2 | import { ActivityUpdate, CacheFileObject, VideoObject } from '../../../../shared/models/activitypub' | 2 | import { ActivityUpdate, ActivityUpdateObject, CacheFileObject, VideoObject } from '../../../../shared/models/activitypub' |
3 | import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor' | 3 | import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor' |
4 | import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object' | 4 | import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object' |
5 | import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file' | 5 | import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file' |
@@ -10,16 +10,18 @@ import { sequelizeTypescript } from '../../../initializers/database' | |||
10 | import { ActorModel } from '../../../models/actor/actor' | 10 | import { ActorModel } from '../../../models/actor/actor' |
11 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' | 11 | import { APProcessorOptions } from '../../../types/activitypub-processor.model' |
12 | import { MActorFull, MActorSignature } from '../../../types/models' | 12 | import { MActorFull, MActorSignature } from '../../../types/models' |
13 | import { fetchAPObject } from '../activity' | ||
13 | import { APActorUpdater } from '../actors/updater' | 14 | import { APActorUpdater } from '../actors/updater' |
14 | import { createOrUpdateCacheFile } from '../cache-file' | 15 | import { createOrUpdateCacheFile } from '../cache-file' |
15 | import { createOrUpdateVideoPlaylist } from '../playlists' | 16 | import { createOrUpdateVideoPlaylist } from '../playlists' |
16 | import { forwardVideoRelatedActivity } from '../send/shared/send-utils' | 17 | import { forwardVideoRelatedActivity } from '../send/shared/send-utils' |
17 | import { APVideoUpdater, getOrCreateAPVideo } from '../videos' | 18 | import { APVideoUpdater, getOrCreateAPVideo } from '../videos' |
18 | 19 | ||
19 | async function processUpdateActivity (options: APProcessorOptions<ActivityUpdate>) { | 20 | async function processUpdateActivity (options: APProcessorOptions<ActivityUpdate<ActivityUpdateObject>>) { |
20 | const { activity, byActor } = options | 21 | const { activity, byActor } = options |
21 | 22 | ||
22 | const objectType = activity.object.type | 23 | const object = await fetchAPObject(activity.object) |
24 | const objectType = object.type | ||
23 | 25 | ||
24 | if (objectType === 'Video') { | 26 | if (objectType === 'Video') { |
25 | return retryTransactionWrapper(processUpdateVideo, activity) | 27 | return retryTransactionWrapper(processUpdateVideo, activity) |
@@ -28,17 +30,17 @@ async function processUpdateActivity (options: APProcessorOptions<ActivityUpdate | |||
28 | if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') { | 30 | if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') { |
29 | // We need more attributes | 31 | // We need more attributes |
30 | const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url) | 32 | const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url) |
31 | return retryTransactionWrapper(processUpdateActor, byActorFull, activity) | 33 | return retryTransactionWrapper(processUpdateActor, byActorFull, object) |
32 | } | 34 | } |
33 | 35 | ||
34 | if (objectType === 'CacheFile') { | 36 | if (objectType === 'CacheFile') { |
35 | // We need more attributes | 37 | // We need more attributes |
36 | const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url) | 38 | const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url) |
37 | return retryTransactionWrapper(processUpdateCacheFile, byActorFull, activity) | 39 | return retryTransactionWrapper(processUpdateCacheFile, byActorFull, activity, object) |
38 | } | 40 | } |
39 | 41 | ||
40 | if (objectType === 'Playlist') { | 42 | if (objectType === 'Playlist') { |
41 | return retryTransactionWrapper(processUpdatePlaylist, byActor, activity) | 43 | return retryTransactionWrapper(processUpdatePlaylist, byActor, activity, object) |
42 | } | 44 | } |
43 | 45 | ||
44 | return undefined | 46 | return undefined |
@@ -52,7 +54,7 @@ export { | |||
52 | 54 | ||
53 | // --------------------------------------------------------------------------- | 55 | // --------------------------------------------------------------------------- |
54 | 56 | ||
55 | async function processUpdateVideo (activity: ActivityUpdate) { | 57 | async function processUpdateVideo (activity: ActivityUpdate<VideoObject | string>) { |
56 | const videoObject = activity.object as VideoObject | 58 | const videoObject = activity.object as VideoObject |
57 | 59 | ||
58 | if (sanitizeAndCheckVideoTorrentObject(videoObject) === false) { | 60 | if (sanitizeAndCheckVideoTorrentObject(videoObject) === false) { |
@@ -72,11 +74,13 @@ async function processUpdateVideo (activity: ActivityUpdate) { | |||
72 | return updater.update(activity.to) | 74 | return updater.update(activity.to) |
73 | } | 75 | } |
74 | 76 | ||
75 | async function processUpdateCacheFile (byActor: MActorSignature, activity: ActivityUpdate) { | 77 | async function processUpdateCacheFile ( |
78 | byActor: MActorSignature, | ||
79 | activity: ActivityUpdate<CacheFileObject | string>, | ||
80 | cacheFileObject: CacheFileObject | ||
81 | ) { | ||
76 | if (await isRedundancyAccepted(activity, byActor) !== true) return | 82 | if (await isRedundancyAccepted(activity, byActor) !== true) return |
77 | 83 | ||
78 | const cacheFileObject = activity.object as CacheFileObject | ||
79 | |||
80 | if (!isCacheFileObjectValid(cacheFileObject)) { | 84 | if (!isCacheFileObjectValid(cacheFileObject)) { |
81 | logger.debug('Cache file object sent by update is not valid.', { cacheFileObject }) | 85 | logger.debug('Cache file object sent by update is not valid.', { cacheFileObject }) |
82 | return undefined | 86 | return undefined |
@@ -96,19 +100,19 @@ async function processUpdateCacheFile (byActor: MActorSignature, activity: Activ | |||
96 | } | 100 | } |
97 | } | 101 | } |
98 | 102 | ||
99 | async function processUpdateActor (actor: MActorFull, activity: ActivityUpdate) { | 103 | async function processUpdateActor (actor: MActorFull, actorObject: ActivityPubActor) { |
100 | const actorObject = activity.object as ActivityPubActor | ||
101 | |||
102 | logger.debug('Updating remote account "%s".', actorObject.url) | 104 | logger.debug('Updating remote account "%s".', actorObject.url) |
103 | 105 | ||
104 | const updater = new APActorUpdater(actorObject, actor) | 106 | const updater = new APActorUpdater(actorObject, actor) |
105 | return updater.update() | 107 | return updater.update() |
106 | } | 108 | } |
107 | 109 | ||
108 | async function processUpdatePlaylist (byActor: MActorSignature, activity: ActivityUpdate) { | 110 | async function processUpdatePlaylist ( |
109 | const playlistObject = activity.object as PlaylistObject | 111 | byActor: MActorSignature, |
112 | activity: ActivityUpdate<PlaylistObject | string>, | ||
113 | playlistObject: PlaylistObject | ||
114 | ) { | ||
110 | const byAccount = byActor.Account | 115 | const byAccount = byActor.Account |
111 | |||
112 | if (!byAccount) throw new Error('Cannot update video playlist with the non account actor ' + byActor.url) | 116 | if (!byAccount) throw new Error('Cannot update video playlist with the non account actor ' + byActor.url) |
113 | 117 | ||
114 | await createOrUpdateVideoPlaylist(playlistObject, activity.to) | 118 | await createOrUpdateVideoPlaylist(playlistObject, activity.to) |
diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts index 0e996ab80..2cd4db14d 100644 --- a/server/lib/activitypub/send/send-create.ts +++ b/server/lib/activitypub/send/send-create.ts | |||
@@ -1,6 +1,14 @@ | |||
1 | import { Transaction } from 'sequelize' | 1 | import { Transaction } from 'sequelize' |
2 | import { getServerActor } from '@server/models/application/application' | 2 | import { getServerActor } from '@server/models/application/application' |
3 | import { ActivityAudience, ActivityCreate, ContextType, VideoPlaylistPrivacy, VideoPrivacy } from '@shared/models' | 3 | import { |
4 | ActivityAudience, | ||
5 | ActivityCreate, | ||
6 | ActivityCreateObject, | ||
7 | ContextType, | ||
8 | VideoCommentObject, | ||
9 | VideoPlaylistPrivacy, | ||
10 | VideoPrivacy | ||
11 | } from '@shared/models' | ||
4 | import { logger, loggerTagsFactory } from '../../../helpers/logger' | 12 | import { logger, loggerTagsFactory } from '../../../helpers/logger' |
5 | import { VideoCommentModel } from '../../../models/video/video-comment' | 13 | import { VideoCommentModel } from '../../../models/video/video-comment' |
6 | import { | 14 | import { |
@@ -107,7 +115,7 @@ async function sendCreateVideoComment (comment: MCommentOwnerVideo, transaction: | |||
107 | 115 | ||
108 | const byActor = comment.Account.Actor | 116 | const byActor = comment.Account.Actor |
109 | const threadParentComments = await VideoCommentModel.listThreadParentComments(comment, transaction) | 117 | const threadParentComments = await VideoCommentModel.listThreadParentComments(comment, transaction) |
110 | const commentObject = comment.toActivityPubObject(threadParentComments) | 118 | const commentObject = comment.toActivityPubObject(threadParentComments) as VideoCommentObject |
111 | 119 | ||
112 | const actorsInvolvedInComment = await getActorsInvolvedInVideo(comment.Video, transaction) | 120 | const actorsInvolvedInComment = await getActorsInvolvedInVideo(comment.Video, transaction) |
113 | // Add the actor that commented too | 121 | // Add the actor that commented too |
@@ -168,7 +176,12 @@ async function sendCreateVideoComment (comment: MCommentOwnerVideo, transaction: | |||
168 | }) | 176 | }) |
169 | } | 177 | } |
170 | 178 | ||
171 | function buildCreateActivity (url: string, byActor: MActorLight, object: any, audience?: ActivityAudience): ActivityCreate { | 179 | function buildCreateActivity <T extends ActivityCreateObject> ( |
180 | url: string, | ||
181 | byActor: MActorLight, | ||
182 | object: T, | ||
183 | audience?: ActivityAudience | ||
184 | ): ActivityCreate<T> { | ||
172 | if (!audience) audience = getAudience(byActor) | 185 | if (!audience) audience = getAudience(byActor) |
173 | 186 | ||
174 | return audiencify( | 187 | return audiencify( |
@@ -176,7 +189,9 @@ function buildCreateActivity (url: string, byActor: MActorLight, object: any, au | |||
176 | type: 'Create' as 'Create', | 189 | type: 'Create' as 'Create', |
177 | id: url + '/activity', | 190 | id: url + '/activity', |
178 | actor: byActor.url, | 191 | actor: byActor.url, |
179 | object: audiencify(object, audience) | 192 | object: typeof object === 'string' |
193 | ? object | ||
194 | : audiencify(object, audience) | ||
180 | }, | 195 | }, |
181 | audience | 196 | audience |
182 | ) | 197 | ) |
diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts index b8eb47ff6..b0b48c9c4 100644 --- a/server/lib/activitypub/send/send-undo.ts +++ b/server/lib/activitypub/send/send-undo.ts | |||
@@ -1,14 +1,5 @@ | |||
1 | import { Transaction } from 'sequelize' | 1 | import { Transaction } from 'sequelize' |
2 | import { | 2 | import { ActivityAudience, ActivityDislike, ActivityLike, ActivityUndo, ActivityUndoObject, ContextType } from '@shared/models' |
3 | ActivityAnnounce, | ||
4 | ActivityAudience, | ||
5 | ActivityCreate, | ||
6 | ActivityDislike, | ||
7 | ActivityFollow, | ||
8 | ActivityLike, | ||
9 | ActivityUndo, | ||
10 | ContextType | ||
11 | } from '@shared/models' | ||
12 | import { logger } from '../../../helpers/logger' | 3 | import { logger } from '../../../helpers/logger' |
13 | import { VideoModel } from '../../../models/video/video' | 4 | import { VideoModel } from '../../../models/video/video' |
14 | import { | 5 | import { |
@@ -128,12 +119,12 @@ export { | |||
128 | 119 | ||
129 | // --------------------------------------------------------------------------- | 120 | // --------------------------------------------------------------------------- |
130 | 121 | ||
131 | function undoActivityData ( | 122 | function undoActivityData <T extends ActivityUndoObject> ( |
132 | url: string, | 123 | url: string, |
133 | byActor: MActorAudience, | 124 | byActor: MActorAudience, |
134 | object: ActivityFollow | ActivityLike | ActivityDislike | ActivityCreate | ActivityAnnounce, | 125 | object: T, |
135 | audience?: ActivityAudience | 126 | audience?: ActivityAudience |
136 | ): ActivityUndo { | 127 | ): ActivityUndo<T> { |
137 | if (!audience) audience = getAudience(byActor) | 128 | if (!audience) audience = getAudience(byActor) |
138 | 129 | ||
139 | return audiencify( | 130 | return audiencify( |
@@ -151,7 +142,7 @@ async function sendUndoVideoRelatedActivity (options: { | |||
151 | byActor: MActor | 142 | byActor: MActor |
152 | video: MVideoAccountLight | 143 | video: MVideoAccountLight |
153 | url: string | 144 | url: string |
154 | activity: ActivityFollow | ActivityCreate | ActivityAnnounce | 145 | activity: ActivityUndoObject |
155 | contextType: ContextType | 146 | contextType: ContextType |
156 | transaction: Transaction | 147 | transaction: Transaction |
157 | }) { | 148 | }) { |
diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts index 379e2d9d8..f3fb741c6 100644 --- a/server/lib/activitypub/send/send-update.ts +++ b/server/lib/activitypub/send/send-update.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import { Transaction } from 'sequelize' | 1 | import { Transaction } from 'sequelize' |
2 | import { getServerActor } from '@server/models/application/application' | 2 | import { getServerActor } from '@server/models/application/application' |
3 | import { ActivityAudience, ActivityUpdate, VideoPlaylistPrivacy, VideoPrivacy } from '@shared/models' | 3 | import { ActivityAudience, ActivityUpdate, ActivityUpdateObject, VideoPlaylistPrivacy, VideoPrivacy } from '@shared/models' |
4 | import { logger } from '../../../helpers/logger' | 4 | import { logger } from '../../../helpers/logger' |
5 | import { AccountModel } from '../../../models/account/account' | 5 | import { AccountModel } from '../../../models/account/account' |
6 | import { VideoModel } from '../../../models/video/video' | 6 | import { VideoModel } from '../../../models/video/video' |
@@ -10,8 +10,7 @@ import { | |||
10 | MActor, | 10 | MActor, |
11 | MActorLight, | 11 | MActorLight, |
12 | MChannelDefault, | 12 | MChannelDefault, |
13 | MVideoAP, | 13 | MVideoAPLight, |
14 | MVideoAPWithoutCaption, | ||
15 | MVideoPlaylistFull, | 14 | MVideoPlaylistFull, |
16 | MVideoRedundancyVideo | 15 | MVideoRedundancyVideo |
17 | } from '../../../types/models' | 16 | } from '../../../types/models' |
@@ -20,10 +19,10 @@ import { getUpdateActivityPubUrl } from '../url' | |||
20 | import { getActorsInvolvedInVideo } from './shared' | 19 | import { getActorsInvolvedInVideo } from './shared' |
21 | import { broadcastToFollowers, sendVideoRelatedActivity } from './shared/send-utils' | 20 | import { broadcastToFollowers, sendVideoRelatedActivity } from './shared/send-utils' |
22 | 21 | ||
23 | async function sendUpdateVideo (videoArg: MVideoAPWithoutCaption, transaction: Transaction, overriddenByActor?: MActor) { | 22 | async function sendUpdateVideo (videoArg: MVideoAPLight, transaction: Transaction, overriddenByActor?: MActor) { |
24 | const video = videoArg as MVideoAP | 23 | if (!videoArg.hasPrivacyForFederation()) return undefined |
25 | 24 | ||
26 | if (!video.hasPrivacyForFederation()) return undefined | 25 | const video = await videoArg.lightAPToFullAP(transaction) |
27 | 26 | ||
28 | logger.info('Creating job to update video %s.', video.url) | 27 | logger.info('Creating job to update video %s.', video.url) |
29 | 28 | ||
@@ -31,11 +30,6 @@ async function sendUpdateVideo (videoArg: MVideoAPWithoutCaption, transaction: T | |||
31 | 30 | ||
32 | const url = getUpdateActivityPubUrl(video.url, video.updatedAt.toISOString()) | 31 | const url = getUpdateActivityPubUrl(video.url, video.updatedAt.toISOString()) |
33 | 32 | ||
34 | // Needed to build the AP object | ||
35 | if (!video.VideoCaptions) { | ||
36 | video.VideoCaptions = await video.$get('VideoCaptions', { transaction }) | ||
37 | } | ||
38 | |||
39 | const videoObject = await video.toActivityPubObject() | 33 | const videoObject = await video.toActivityPubObject() |
40 | const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC) | 34 | const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC) |
41 | 35 | ||
@@ -143,7 +137,12 @@ export { | |||
143 | 137 | ||
144 | // --------------------------------------------------------------------------- | 138 | // --------------------------------------------------------------------------- |
145 | 139 | ||
146 | function buildUpdateActivity (url: string, byActor: MActorLight, object: any, audience?: ActivityAudience): ActivityUpdate { | 140 | function buildUpdateActivity ( |
141 | url: string, | ||
142 | byActor: MActorLight, | ||
143 | object: ActivityUpdateObject, | ||
144 | audience?: ActivityAudience | ||
145 | ): ActivityUpdate<ActivityUpdateObject> { | ||
147 | if (!audience) audience = getAudience(byActor) | 146 | if (!audience) audience = getAudience(byActor) |
148 | 147 | ||
149 | return audiencify( | 148 | return audiencify( |
diff --git a/server/lib/activitypub/videos/federate.ts b/server/lib/activitypub/videos/federate.ts index bd0c54b0c..d7e251153 100644 --- a/server/lib/activitypub/videos/federate.ts +++ b/server/lib/activitypub/videos/federate.ts | |||
@@ -1,10 +1,9 @@ | |||
1 | import { Transaction } from 'sequelize/types' | 1 | import { Transaction } from 'sequelize/types' |
2 | import { isArray } from '@server/helpers/custom-validators/misc' | 2 | import { MVideoAP, MVideoAPLight } from '@server/types/models' |
3 | import { MVideoAP, MVideoAPWithoutCaption } from '@server/types/models' | ||
4 | import { sendCreateVideo, sendUpdateVideo } from '../send' | 3 | import { sendCreateVideo, sendUpdateVideo } from '../send' |
5 | import { shareVideoByServerAndChannel } from '../share' | 4 | import { shareVideoByServerAndChannel } from '../share' |
6 | 5 | ||
7 | async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVideo: boolean, transaction?: Transaction) { | 6 | async function federateVideoIfNeeded (videoArg: MVideoAPLight, isNewVideo: boolean, transaction?: Transaction) { |
8 | const video = videoArg as MVideoAP | 7 | const video = videoArg as MVideoAP |
9 | 8 | ||
10 | if ( | 9 | if ( |
@@ -13,13 +12,7 @@ async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVid | |||
13 | // Check the video is public/unlisted and published | 12 | // Check the video is public/unlisted and published |
14 | video.hasPrivacyForFederation() && video.hasStateForFederation() | 13 | video.hasPrivacyForFederation() && video.hasStateForFederation() |
15 | ) { | 14 | ) { |
16 | // Fetch more attributes that we will need to serialize in AP object | 15 | const video = await videoArg.lightAPToFullAP(transaction) |
17 | if (isArray(video.VideoCaptions) === false) { | ||
18 | video.VideoCaptions = await video.$get('VideoCaptions', { | ||
19 | attributes: [ 'filename', 'language' ], | ||
20 | transaction | ||
21 | }) | ||
22 | } | ||
23 | 16 | ||
24 | if (isNewVideo) { | 17 | if (isNewVideo) { |
25 | // Now we'll add the video's meta data to our followers | 18 | // Now we'll add the video's meta data to our followers |
diff --git a/server/lib/activitypub/videos/get.ts b/server/lib/activitypub/videos/get.ts index 14ba55034..92387c5d4 100644 --- a/server/lib/activitypub/videos/get.ts +++ b/server/lib/activitypub/videos/get.ts | |||
@@ -3,7 +3,7 @@ import { logger } from '@server/helpers/logger' | |||
3 | import { JobQueue } from '@server/lib/job-queue' | 3 | import { JobQueue } from '@server/lib/job-queue' |
4 | import { loadVideoByUrl, VideoLoadByUrlType } from '@server/lib/model-loaders' | 4 | import { loadVideoByUrl, VideoLoadByUrlType } from '@server/lib/model-loaders' |
5 | import { MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from '@server/types/models' | 5 | import { MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from '@server/types/models' |
6 | import { APObject } from '@shared/models' | 6 | import { APObjectId } from '@shared/models' |
7 | import { getAPId } from '../activity' | 7 | import { getAPId } from '../activity' |
8 | import { refreshVideoIfNeeded } from './refresh' | 8 | import { refreshVideoIfNeeded } from './refresh' |
9 | import { APVideoCreator, fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared' | 9 | import { APVideoCreator, fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared' |
@@ -15,21 +15,21 @@ type GetVideoResult <T> = Promise<{ | |||
15 | }> | 15 | }> |
16 | 16 | ||
17 | type GetVideoParamAll = { | 17 | type GetVideoParamAll = { |
18 | videoObject: APObject | 18 | videoObject: APObjectId |
19 | syncParam?: SyncParam | 19 | syncParam?: SyncParam |
20 | fetchType?: 'all' | 20 | fetchType?: 'all' |
21 | allowRefresh?: boolean | 21 | allowRefresh?: boolean |
22 | } | 22 | } |
23 | 23 | ||
24 | type GetVideoParamImmutable = { | 24 | type GetVideoParamImmutable = { |
25 | videoObject: APObject | 25 | videoObject: APObjectId |
26 | syncParam?: SyncParam | 26 | syncParam?: SyncParam |
27 | fetchType: 'only-immutable-attributes' | 27 | fetchType: 'only-immutable-attributes' |
28 | allowRefresh: false | 28 | allowRefresh: false |
29 | } | 29 | } |
30 | 30 | ||
31 | type GetVideoParamOther = { | 31 | type GetVideoParamOther = { |
32 | videoObject: APObject | 32 | videoObject: APObjectId |
33 | syncParam?: SyncParam | 33 | syncParam?: SyncParam |
34 | fetchType?: 'all' | 'only-video' | 34 | fetchType?: 'all' | 'only-video' |
35 | allowRefresh?: boolean | 35 | allowRefresh?: boolean |
diff --git a/server/lib/activitypub/videos/shared/abstract-builder.ts b/server/lib/activitypub/videos/shared/abstract-builder.ts index c0b92c93d..98c2f58eb 100644 --- a/server/lib/activitypub/videos/shared/abstract-builder.ts +++ b/server/lib/activitypub/videos/shared/abstract-builder.ts | |||
@@ -1,8 +1,9 @@ | |||
1 | import { CreationAttributes, Transaction } from 'sequelize/types' | 1 | import { CreationAttributes, Transaction } from 'sequelize/types' |
2 | import { deleteAllModels, filterNonExistingModels } from '@server/helpers/database-utils' | 2 | import { deleteAllModels, filterNonExistingModels } from '@server/helpers/database-utils' |
3 | import { logger, LoggerTagsFn } from '@server/helpers/logger' | 3 | import { logger, LoggerTagsFn } from '@server/helpers/logger' |
4 | import { updatePlaceholderThumbnail, updateVideoMiniatureFromUrl } from '@server/lib/thumbnail' | 4 | import { updateRemoteVideoThumbnail } from '@server/lib/thumbnail' |
5 | import { setVideoTags } from '@server/lib/video' | 5 | import { setVideoTags } from '@server/lib/video' |
6 | import { StoryboardModel } from '@server/models/video/storyboard' | ||
6 | import { VideoCaptionModel } from '@server/models/video/video-caption' | 7 | import { VideoCaptionModel } from '@server/models/video/video-caption' |
7 | import { VideoFileModel } from '@server/models/video/video-file' | 8 | import { VideoFileModel } from '@server/models/video/video-file' |
8 | import { VideoLiveModel } from '@server/models/video/video-live' | 9 | import { VideoLiveModel } from '@server/models/video/video-live' |
@@ -10,20 +11,19 @@ import { VideoStreamingPlaylistModel } from '@server/models/video/video-streamin | |||
10 | import { | 11 | import { |
11 | MStreamingPlaylistFiles, | 12 | MStreamingPlaylistFiles, |
12 | MStreamingPlaylistFilesVideo, | 13 | MStreamingPlaylistFilesVideo, |
13 | MThumbnail, | ||
14 | MVideoCaption, | 14 | MVideoCaption, |
15 | MVideoFile, | 15 | MVideoFile, |
16 | MVideoFullLight, | 16 | MVideoFullLight, |
17 | MVideoThumbnail | 17 | MVideoThumbnail |
18 | } from '@server/types/models' | 18 | } from '@server/types/models' |
19 | import { ActivityTagObject, ThumbnailType, VideoObject, VideoStreamingPlaylistType } from '@shared/models' | 19 | import { ActivityTagObject, ThumbnailType, VideoObject, VideoStreamingPlaylistType } from '@shared/models' |
20 | import { getOrCreateAPActor } from '../../actors' | 20 | import { findOwner, getOrCreateAPActor } from '../../actors' |
21 | import { checkUrlsSameHost } from '../../url' | ||
22 | import { | 21 | import { |
23 | getCaptionAttributesFromObject, | 22 | getCaptionAttributesFromObject, |
24 | getFileAttributesFromUrl, | 23 | getFileAttributesFromUrl, |
25 | getLiveAttributesFromObject, | 24 | getLiveAttributesFromObject, |
26 | getPreviewFromIcons, | 25 | getPreviewFromIcons, |
26 | getStoryboardAttributeFromObject, | ||
27 | getStreamingPlaylistAttributesFromObject, | 27 | getStreamingPlaylistAttributesFromObject, |
28 | getTagsFromObject, | 28 | getTagsFromObject, |
29 | getThumbnailFromIcons | 29 | getThumbnailFromIcons |
@@ -35,38 +35,40 @@ export abstract class APVideoAbstractBuilder { | |||
35 | protected abstract lTags: LoggerTagsFn | 35 | protected abstract lTags: LoggerTagsFn |
36 | 36 | ||
37 | protected async getOrCreateVideoChannelFromVideoObject () { | 37 | protected async getOrCreateVideoChannelFromVideoObject () { |
38 | const channel = this.videoObject.attributedTo.find(a => a.type === 'Group') | 38 | const channel = await findOwner(this.videoObject.id, this.videoObject.attributedTo, 'Group') |
39 | if (!channel) throw new Error('Cannot find associated video channel to video ' + this.videoObject.url) | 39 | if (!channel) throw new Error('Cannot find associated video channel to video ' + this.videoObject.url) |
40 | 40 | ||
41 | if (checkUrlsSameHost(channel.id, this.videoObject.id) !== true) { | ||
42 | throw new Error(`Video channel url ${channel.id} does not have the same host than video object id ${this.videoObject.id}`) | ||
43 | } | ||
44 | |||
45 | return getOrCreateAPActor(channel.id, 'all') | 41 | return getOrCreateAPActor(channel.id, 'all') |
46 | } | 42 | } |
47 | 43 | ||
48 | protected tryToGenerateThumbnail (video: MVideoThumbnail): Promise<MThumbnail> { | 44 | protected async setThumbnail (video: MVideoThumbnail, t?: Transaction) { |
49 | return updateVideoMiniatureFromUrl({ | 45 | const miniatureIcon = getThumbnailFromIcons(this.videoObject) |
50 | downloadUrl: getThumbnailFromIcons(this.videoObject).url, | 46 | if (!miniatureIcon) { |
51 | video, | 47 | logger.warn('Cannot find thumbnail in video object', { object: this.videoObject }) |
52 | type: ThumbnailType.MINIATURE | ||
53 | }).catch(err => { | ||
54 | logger.warn('Cannot generate thumbnail of %s.', this.videoObject.id, { err, ...this.lTags() }) | ||
55 | |||
56 | return undefined | 48 | return undefined |
49 | } | ||
50 | |||
51 | const miniatureModel = updateRemoteVideoThumbnail({ | ||
52 | fileUrl: miniatureIcon.url, | ||
53 | video, | ||
54 | type: ThumbnailType.MINIATURE, | ||
55 | size: miniatureIcon, | ||
56 | onDisk: false // Lazy download remote thumbnails | ||
57 | }) | 57 | }) |
58 | |||
59 | await video.addAndSaveThumbnail(miniatureModel, t) | ||
58 | } | 60 | } |
59 | 61 | ||
60 | protected async setPreview (video: MVideoFullLight, t?: Transaction) { | 62 | protected async setPreview (video: MVideoFullLight, t?: Transaction) { |
61 | // Don't fetch the preview that could be big, create a placeholder instead | ||
62 | const previewIcon = getPreviewFromIcons(this.videoObject) | 63 | const previewIcon = getPreviewFromIcons(this.videoObject) |
63 | if (!previewIcon) return | 64 | if (!previewIcon) return |
64 | 65 | ||
65 | const previewModel = updatePlaceholderThumbnail({ | 66 | const previewModel = updateRemoteVideoThumbnail({ |
66 | fileUrl: previewIcon.url, | 67 | fileUrl: previewIcon.url, |
67 | video, | 68 | video, |
68 | type: ThumbnailType.PREVIEW, | 69 | type: ThumbnailType.PREVIEW, |
69 | size: previewIcon | 70 | size: previewIcon, |
71 | onDisk: false // Lazy download remote previews | ||
70 | }) | 72 | }) |
71 | 73 | ||
72 | await video.addAndSaveThumbnail(previewModel, t) | 74 | await video.addAndSaveThumbnail(previewModel, t) |
@@ -107,6 +109,16 @@ export abstract class APVideoAbstractBuilder { | |||
107 | } | 109 | } |
108 | } | 110 | } |
109 | 111 | ||
112 | protected async insertOrReplaceStoryboard (video: MVideoFullLight, t: Transaction) { | ||
113 | const existingStoryboard = await StoryboardModel.loadByVideo(video.id, t) | ||
114 | if (existingStoryboard) await existingStoryboard.destroy({ transaction: t }) | ||
115 | |||
116 | const storyboardAttributes = getStoryboardAttributeFromObject(video, this.videoObject) | ||
117 | if (!storyboardAttributes) return | ||
118 | |||
119 | return StoryboardModel.create(storyboardAttributes, { transaction: t }) | ||
120 | } | ||
121 | |||
110 | protected async insertOrReplaceLive (video: MVideoFullLight, transaction: Transaction) { | 122 | protected async insertOrReplaceLive (video: MVideoFullLight, transaction: Transaction) { |
111 | const attributes = getLiveAttributesFromObject(video, this.videoObject) | 123 | const attributes = getLiveAttributesFromObject(video, this.videoObject) |
112 | const [ videoLive ] = await VideoLiveModel.upsert(attributes, { transaction, returning: true }) | 124 | const [ videoLive ] = await VideoLiveModel.upsert(attributes, { transaction, returning: true }) |
@@ -114,7 +126,7 @@ export abstract class APVideoAbstractBuilder { | |||
114 | video.VideoLive = videoLive | 126 | video.VideoLive = videoLive |
115 | } | 127 | } |
116 | 128 | ||
117 | protected async setWebTorrentFiles (video: MVideoFullLight, t: Transaction) { | 129 | protected async setWebVideoFiles (video: MVideoFullLight, t: Transaction) { |
118 | const videoFileAttributes = getFileAttributesFromUrl(video, this.videoObject.url) | 130 | const videoFileAttributes = getFileAttributesFromUrl(video, this.videoObject.url) |
119 | const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a)) | 131 | const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a)) |
120 | 132 | ||
diff --git a/server/lib/activitypub/videos/shared/creator.ts b/server/lib/activitypub/videos/shared/creator.ts index 77321d8a5..bc139e4fa 100644 --- a/server/lib/activitypub/videos/shared/creator.ts +++ b/server/lib/activitypub/videos/shared/creator.ts | |||
@@ -4,7 +4,7 @@ import { sequelizeTypescript } from '@server/initializers/database' | |||
4 | import { Hooks } from '@server/lib/plugins/hooks' | 4 | import { Hooks } from '@server/lib/plugins/hooks' |
5 | import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist' | 5 | import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist' |
6 | import { VideoModel } from '@server/models/video/video' | 6 | import { VideoModel } from '@server/models/video/video' |
7 | import { MThumbnail, MVideoFullLight, MVideoThumbnail } from '@server/types/models' | 7 | import { MVideoFullLight, MVideoThumbnail } from '@server/types/models' |
8 | import { VideoObject } from '@shared/models' | 8 | import { VideoObject } from '@shared/models' |
9 | import { APVideoAbstractBuilder } from './abstract-builder' | 9 | import { APVideoAbstractBuilder } from './abstract-builder' |
10 | import { getVideoAttributesFromObject } from './object-to-model-attributes' | 10 | import { getVideoAttributesFromObject } from './object-to-model-attributes' |
@@ -27,64 +27,38 @@ export class APVideoCreator extends APVideoAbstractBuilder { | |||
27 | const videoData = getVideoAttributesFromObject(channel, this.videoObject, this.videoObject.to) | 27 | const videoData = getVideoAttributesFromObject(channel, this.videoObject, this.videoObject.to) |
28 | const video = VideoModel.build({ ...videoData, likes: 0, dislikes: 0 }) as MVideoThumbnail | 28 | const video = VideoModel.build({ ...videoData, likes: 0, dislikes: 0 }) as MVideoThumbnail |
29 | 29 | ||
30 | const promiseThumbnail = this.tryToGenerateThumbnail(video) | ||
31 | |||
32 | let thumbnailModel: MThumbnail | ||
33 | if (waitThumbnail === true) { | ||
34 | thumbnailModel = await promiseThumbnail | ||
35 | } | ||
36 | |||
37 | const { autoBlacklisted, videoCreated } = await sequelizeTypescript.transaction(async t => { | 30 | const { autoBlacklisted, videoCreated } = await sequelizeTypescript.transaction(async t => { |
38 | try { | 31 | const videoCreated = await video.save({ transaction: t }) as MVideoFullLight |
39 | const videoCreated = await video.save({ transaction: t }) as MVideoFullLight | 32 | videoCreated.VideoChannel = channel |
40 | videoCreated.VideoChannel = channel | 33 | |
41 | 34 | await this.setThumbnail(videoCreated, t) | |
42 | if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t) | 35 | await this.setPreview(videoCreated, t) |
43 | 36 | await this.setWebVideoFiles(videoCreated, t) | |
44 | await this.setPreview(videoCreated, t) | 37 | await this.setStreamingPlaylists(videoCreated, t) |
45 | await this.setWebTorrentFiles(videoCreated, t) | 38 | await this.setTags(videoCreated, t) |
46 | await this.setStreamingPlaylists(videoCreated, t) | 39 | await this.setTrackers(videoCreated, t) |
47 | await this.setTags(videoCreated, t) | 40 | await this.insertOrReplaceCaptions(videoCreated, t) |
48 | await this.setTrackers(videoCreated, t) | 41 | await this.insertOrReplaceLive(videoCreated, t) |
49 | await this.insertOrReplaceCaptions(videoCreated, t) | 42 | await this.insertOrReplaceStoryboard(videoCreated, t) |
50 | await this.insertOrReplaceLive(videoCreated, t) | 43 | |
51 | 44 | // We added a video in this channel, set it as updated | |
52 | // We added a video in this channel, set it as updated | 45 | await channel.setAsUpdated(t) |
53 | await channel.setAsUpdated(t) | 46 | |
54 | 47 | const autoBlacklisted = await autoBlacklistVideoIfNeeded({ | |
55 | const autoBlacklisted = await autoBlacklistVideoIfNeeded({ | 48 | video: videoCreated, |
56 | video: videoCreated, | 49 | user: undefined, |
57 | user: undefined, | 50 | isRemote: true, |
58 | isRemote: true, | 51 | isNew: true, |
59 | isNew: true, | 52 | transaction: t |
60 | transaction: t | 53 | }) |
61 | }) | ||
62 | |||
63 | logger.info('Remote video with uuid %s inserted.', this.videoObject.uuid, this.lTags()) | ||
64 | 54 | ||
65 | Hooks.runAction('action:activity-pub.remote-video.created', { video: videoCreated, videoAPObject: this.videoObject }) | 55 | logger.info('Remote video with uuid %s inserted.', this.videoObject.uuid, this.lTags()) |
66 | 56 | ||
67 | return { autoBlacklisted, videoCreated } | 57 | Hooks.runAction('action:activity-pub.remote-video.created', { video: videoCreated, videoAPObject: this.videoObject }) |
68 | } catch (err) { | ||
69 | // FIXME: Use rollback hook when https://github.com/sequelize/sequelize/pull/13038 is released | ||
70 | if (thumbnailModel) await thumbnailModel.removeThumbnail() | ||
71 | 58 | ||
72 | throw err | 59 | return { autoBlacklisted, videoCreated } |
73 | } | ||
74 | }) | 60 | }) |
75 | 61 | ||
76 | if (waitThumbnail === false) { | ||
77 | // Error is already caught above | ||
78 | // eslint-disable-next-line @typescript-eslint/no-floating-promises | ||
79 | promiseThumbnail.then(thumbnailModel => { | ||
80 | if (!thumbnailModel) return | ||
81 | |||
82 | thumbnailModel = videoCreated.id | ||
83 | |||
84 | return thumbnailModel.save() | ||
85 | }) | ||
86 | } | ||
87 | |||
88 | return { autoBlacklisted, videoCreated } | 62 | return { autoBlacklisted, videoCreated } |
89 | } | 63 | } |
90 | } | 64 | } |
diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts index 8fd0a816c..a9e0bed97 100644 --- a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts +++ b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import { maxBy, minBy } from 'lodash' | 1 | import { maxBy, minBy } from 'lodash' |
2 | import { decode as magnetUriDecode } from 'magnet-uri' | 2 | import { decode as magnetUriDecode } from 'magnet-uri' |
3 | import { basename } from 'path' | 3 | import { basename, extname } from 'path' |
4 | import { isAPVideoFileUrlMetadataObject } from '@server/helpers/custom-validators/activitypub/videos' | 4 | import { isAPVideoFileUrlMetadataObject } from '@server/helpers/custom-validators/activitypub/videos' |
5 | import { isVideoFileInfoHashValid } from '@server/helpers/custom-validators/videos' | 5 | import { isVideoFileInfoHashValid } from '@server/helpers/custom-validators/videos' |
6 | import { logger } from '@server/helpers/logger' | 6 | import { logger } from '@server/helpers/logger' |
@@ -25,6 +25,9 @@ import { | |||
25 | VideoStreamingPlaylistType | 25 | VideoStreamingPlaylistType |
26 | } from '@shared/models' | 26 | } from '@shared/models' |
27 | import { getDurationFromActivityStream } from '../../activity' | 27 | import { getDurationFromActivityStream } from '../../activity' |
28 | import { isArray } from '@server/helpers/custom-validators/misc' | ||
29 | import { generateImageFilename } from '@server/helpers/image-utils' | ||
30 | import { arrayify } from '@shared/core-utils' | ||
28 | 31 | ||
29 | function getThumbnailFromIcons (videoObject: VideoObject) { | 32 | function getThumbnailFromIcons (videoObject: VideoObject) { |
30 | let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth) | 33 | let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth) |
@@ -166,6 +169,26 @@ function getCaptionAttributesFromObject (video: MVideoId, videoObject: VideoObje | |||
166 | })) | 169 | })) |
167 | } | 170 | } |
168 | 171 | ||
172 | function getStoryboardAttributeFromObject (video: MVideoId, videoObject: VideoObject) { | ||
173 | if (!isArray(videoObject.preview)) return undefined | ||
174 | |||
175 | const storyboard = videoObject.preview.find(p => p.rel.includes('storyboard')) | ||
176 | if (!storyboard) return undefined | ||
177 | |||
178 | const url = arrayify(storyboard.url).find(u => u.mediaType === 'image/jpeg') | ||
179 | |||
180 | return { | ||
181 | filename: generateImageFilename(extname(url.href)), | ||
182 | totalHeight: url.height, | ||
183 | totalWidth: url.width, | ||
184 | spriteHeight: url.tileHeight, | ||
185 | spriteWidth: url.tileWidth, | ||
186 | spriteDuration: getDurationFromActivityStream(url.tileDuration), | ||
187 | fileUrl: url.href, | ||
188 | videoId: video.id | ||
189 | } | ||
190 | } | ||
191 | |||
169 | function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: VideoObject, to: string[] = []) { | 192 | function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: VideoObject, to: string[] = []) { |
170 | const privacy = to.includes(ACTIVITY_PUB.PUBLIC) | 193 | const privacy = to.includes(ACTIVITY_PUB.PUBLIC) |
171 | ? VideoPrivacy.PUBLIC | 194 | ? VideoPrivacy.PUBLIC |
@@ -228,6 +251,7 @@ export { | |||
228 | 251 | ||
229 | getLiveAttributesFromObject, | 252 | getLiveAttributesFromObject, |
230 | getCaptionAttributesFromObject, | 253 | getCaptionAttributesFromObject, |
254 | getStoryboardAttributeFromObject, | ||
231 | 255 | ||
232 | getVideoAttributesFromObject | 256 | getVideoAttributesFromObject |
233 | } | 257 | } |
diff --git a/server/lib/activitypub/videos/updater.ts b/server/lib/activitypub/videos/updater.ts index 6ddd2301b..522d7b043 100644 --- a/server/lib/activitypub/videos/updater.ts +++ b/server/lib/activitypub/videos/updater.ts | |||
@@ -41,7 +41,7 @@ export class APVideoUpdater extends APVideoAbstractBuilder { | |||
41 | try { | 41 | try { |
42 | const channelActor = await this.getOrCreateVideoChannelFromVideoObject() | 42 | const channelActor = await this.getOrCreateVideoChannelFromVideoObject() |
43 | 43 | ||
44 | const thumbnailModel = await this.tryToGenerateThumbnail(this.video) | 44 | const thumbnailModel = await this.setThumbnail(this.video) |
45 | 45 | ||
46 | this.checkChannelUpdateOrThrow(channelActor) | 46 | this.checkChannelUpdateOrThrow(channelActor) |
47 | 47 | ||
@@ -50,15 +50,21 @@ export class APVideoUpdater extends APVideoAbstractBuilder { | |||
50 | if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel) | 50 | if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel) |
51 | 51 | ||
52 | await runInReadCommittedTransaction(async t => { | 52 | await runInReadCommittedTransaction(async t => { |
53 | await this.setWebTorrentFiles(videoUpdated, t) | 53 | await this.setWebVideoFiles(videoUpdated, t) |
54 | await this.setStreamingPlaylists(videoUpdated, t) | 54 | await this.setStreamingPlaylists(videoUpdated, t) |
55 | }) | 55 | }) |
56 | 56 | ||
57 | await Promise.all([ | 57 | await Promise.all([ |
58 | runInReadCommittedTransaction(t => this.setTags(videoUpdated, t)), | 58 | runInReadCommittedTransaction(t => this.setTags(videoUpdated, t)), |
59 | runInReadCommittedTransaction(t => this.setTrackers(videoUpdated, t)), | 59 | runInReadCommittedTransaction(t => this.setTrackers(videoUpdated, t)), |
60 | this.setOrDeleteLive(videoUpdated), | 60 | runInReadCommittedTransaction(t => this.setStoryboard(videoUpdated, t)), |
61 | this.setPreview(videoUpdated) | 61 | runInReadCommittedTransaction(t => { |
62 | return Promise.all([ | ||
63 | this.setPreview(videoUpdated, t), | ||
64 | this.setThumbnail(videoUpdated, t) | ||
65 | ]) | ||
66 | }), | ||
67 | this.setOrDeleteLive(videoUpdated) | ||
62 | ]) | 68 | ]) |
63 | 69 | ||
64 | await runInReadCommittedTransaction(t => this.setCaptions(videoUpdated, t)) | 70 | await runInReadCommittedTransaction(t => this.setCaptions(videoUpdated, t)) |
@@ -138,6 +144,10 @@ export class APVideoUpdater extends APVideoAbstractBuilder { | |||
138 | await this.insertOrReplaceCaptions(videoUpdated, t) | 144 | await this.insertOrReplaceCaptions(videoUpdated, t) |
139 | } | 145 | } |
140 | 146 | ||
147 | private async setStoryboard (videoUpdated: MVideoFullLight, t: Transaction) { | ||
148 | await this.insertOrReplaceStoryboard(videoUpdated, t) | ||
149 | } | ||
150 | |||
141 | private async setOrDeleteLive (videoUpdated: MVideoFullLight, transaction?: Transaction) { | 151 | private async setOrDeleteLive (videoUpdated: MVideoFullLight, transaction?: Transaction) { |
142 | if (!this.video.isLive) return | 152 | if (!this.video.isLive) return |
143 | 153 | ||
diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts index 18b16bee1..be6df1792 100644 --- a/server/lib/client-html.ts +++ b/server/lib/client-html.ts | |||
@@ -32,6 +32,7 @@ import { getActivityStreamDuration } from './activitypub/activity' | |||
32 | import { getBiggestActorImage } from './actor-image' | 32 | import { getBiggestActorImage } from './actor-image' |
33 | import { Hooks } from './plugins/hooks' | 33 | import { Hooks } from './plugins/hooks' |
34 | import { ServerConfigManager } from './server-config-manager' | 34 | import { ServerConfigManager } from './server-config-manager' |
35 | import { isVideoInPrivateDirectory } from './video-privacy' | ||
35 | 36 | ||
36 | type Tags = { | 37 | type Tags = { |
37 | ogType: string | 38 | ogType: string |
@@ -106,7 +107,7 @@ class ClientHtml { | |||
106 | ]) | 107 | ]) |
107 | 108 | ||
108 | // Let Angular application handle errors | 109 | // Let Angular application handle errors |
109 | if (!video || video.privacy === VideoPrivacy.PRIVATE || video.privacy === VideoPrivacy.INTERNAL || video.VideoBlacklist) { | 110 | if (!video || isVideoInPrivateDirectory(video.privacy) || video.VideoBlacklist) { |
110 | res.status(HttpStatusCode.NOT_FOUND_404) | 111 | res.status(HttpStatusCode.NOT_FOUND_404) |
111 | return html | 112 | return html |
112 | } | 113 | } |
diff --git a/server/lib/files-cache/avatar-permanent-file-cache.ts b/server/lib/files-cache/avatar-permanent-file-cache.ts new file mode 100644 index 000000000..0c508b063 --- /dev/null +++ b/server/lib/files-cache/avatar-permanent-file-cache.ts | |||
@@ -0,0 +1,27 @@ | |||
1 | import { CONFIG } from '@server/initializers/config' | ||
2 | import { ACTOR_IMAGES_SIZE } from '@server/initializers/constants' | ||
3 | import { ActorImageModel } from '@server/models/actor/actor-image' | ||
4 | import { MActorImage } from '@server/types/models' | ||
5 | import { AbstractPermanentFileCache } from './shared' | ||
6 | |||
7 | export class AvatarPermanentFileCache extends AbstractPermanentFileCache<MActorImage> { | ||
8 | |||
9 | constructor () { | ||
10 | super(CONFIG.STORAGE.ACTOR_IMAGES_DIR) | ||
11 | } | ||
12 | |||
13 | protected loadModel (filename: string) { | ||
14 | return ActorImageModel.loadByName(filename) | ||
15 | } | ||
16 | |||
17 | protected getImageSize (image: MActorImage): { width: number, height: number } { | ||
18 | if (image.width && image.height) { | ||
19 | return { | ||
20 | height: image.height, | ||
21 | width: image.width | ||
22 | } | ||
23 | } | ||
24 | |||
25 | return ACTOR_IMAGES_SIZE[image.type][0] | ||
26 | } | ||
27 | } | ||
diff --git a/server/lib/files-cache/index.ts b/server/lib/files-cache/index.ts index e5853f7d6..5630a9b80 100644 --- a/server/lib/files-cache/index.ts +++ b/server/lib/files-cache/index.ts | |||
@@ -1,3 +1,6 @@ | |||
1 | export * from './videos-preview-cache' | 1 | export * from './avatar-permanent-file-cache' |
2 | export * from './videos-caption-cache' | 2 | export * from './video-miniature-permanent-file-cache' |
3 | export * from './videos-torrent-cache' | 3 | export * from './video-captions-simple-file-cache' |
4 | export * from './video-previews-simple-file-cache' | ||
5 | export * from './video-storyboards-simple-file-cache' | ||
6 | export * from './video-torrents-simple-file-cache' | ||
diff --git a/server/lib/files-cache/shared/abstract-permanent-file-cache.ts b/server/lib/files-cache/shared/abstract-permanent-file-cache.ts new file mode 100644 index 000000000..f990e9872 --- /dev/null +++ b/server/lib/files-cache/shared/abstract-permanent-file-cache.ts | |||
@@ -0,0 +1,132 @@ | |||
1 | import express from 'express' | ||
2 | import { LRUCache } from 'lru-cache' | ||
3 | import { Model } from 'sequelize' | ||
4 | import { logger } from '@server/helpers/logger' | ||
5 | import { CachePromise } from '@server/helpers/promise-cache' | ||
6 | import { LRU_CACHE, STATIC_MAX_AGE } from '@server/initializers/constants' | ||
7 | import { downloadImageFromWorker } from '@server/lib/worker/parent-process' | ||
8 | import { HttpStatusCode } from '@shared/models' | ||
9 | |||
10 | type ImageModel = { | ||
11 | fileUrl: string | ||
12 | filename: string | ||
13 | onDisk: boolean | ||
14 | |||
15 | isOwned (): boolean | ||
16 | getPath (): string | ||
17 | |||
18 | save (): Promise<Model> | ||
19 | } | ||
20 | |||
21 | export abstract class AbstractPermanentFileCache <M extends ImageModel> { | ||
22 | // Unsafe because it can return paths that do not exist anymore | ||
23 | private readonly filenameToPathUnsafeCache = new LRUCache<string, string>({ | ||
24 | max: LRU_CACHE.FILENAME_TO_PATH_PERMANENT_FILE_CACHE.MAX_SIZE | ||
25 | }) | ||
26 | |||
27 | protected abstract getImageSize (image: M): { width: number, height: number } | ||
28 | protected abstract loadModel (filename: string): Promise<M> | ||
29 | |||
30 | constructor (private readonly directory: string) { | ||
31 | |||
32 | } | ||
33 | |||
34 | async lazyServe (options: { | ||
35 | filename: string | ||
36 | res: express.Response | ||
37 | next: express.NextFunction | ||
38 | }) { | ||
39 | const { filename, res, next } = options | ||
40 | |||
41 | if (this.filenameToPathUnsafeCache.has(filename)) { | ||
42 | return res.sendFile(this.filenameToPathUnsafeCache.get(filename), { maxAge: STATIC_MAX_AGE.SERVER }) | ||
43 | } | ||
44 | |||
45 | const image = await this.lazyLoadIfNeeded(filename) | ||
46 | if (!image) return res.status(HttpStatusCode.NOT_FOUND_404).end() | ||
47 | |||
48 | const path = image.getPath() | ||
49 | this.filenameToPathUnsafeCache.set(filename, path) | ||
50 | |||
51 | return res.sendFile(path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }, (err: any) => { | ||
52 | if (!err) return | ||
53 | |||
54 | this.onServeError({ err, image, next, filename }) | ||
55 | }) | ||
56 | } | ||
57 | |||
58 | @CachePromise({ | ||
59 | keyBuilder: filename => filename | ||
60 | }) | ||
61 | private async lazyLoadIfNeeded (filename: string) { | ||
62 | const image = await this.loadModel(filename) | ||
63 | if (!image) return undefined | ||
64 | |||
65 | if (image.onDisk === false) { | ||
66 | if (!image.fileUrl) return undefined | ||
67 | |||
68 | try { | ||
69 | await this.downloadRemoteFile(image) | ||
70 | } catch (err) { | ||
71 | logger.warn('Cannot process remote image %s.', image.fileUrl, { err }) | ||
72 | |||
73 | return undefined | ||
74 | } | ||
75 | } | ||
76 | |||
77 | return image | ||
78 | } | ||
79 | |||
80 | async downloadRemoteFile (image: M) { | ||
81 | logger.info('Download remote image %s lazily.', image.fileUrl) | ||
82 | |||
83 | const destination = await this.downloadImage({ | ||
84 | filename: image.filename, | ||
85 | fileUrl: image.fileUrl, | ||
86 | size: this.getImageSize(image) | ||
87 | }) | ||
88 | |||
89 | image.onDisk = true | ||
90 | image.save() | ||
91 | .catch(err => logger.error('Cannot save new image disk state.', { err })) | ||
92 | |||
93 | return destination | ||
94 | } | ||
95 | |||
96 | private onServeError (options: { | ||
97 | err: any | ||
98 | image: M | ||
99 | filename: string | ||
100 | next: express.NextFunction | ||
101 | }) { | ||
102 | const { err, image, filename, next } = options | ||
103 | |||
104 | // It seems this actor image is not on the disk anymore | ||
105 | if (err.status === HttpStatusCode.NOT_FOUND_404 && !image.isOwned()) { | ||
106 | logger.error('Cannot lazy serve image %s.', filename, { err }) | ||
107 | |||
108 | this.filenameToPathUnsafeCache.delete(filename) | ||
109 | |||
110 | image.onDisk = false | ||
111 | image.save() | ||
112 | .catch(err => logger.error('Cannot save new image disk state.', { err })) | ||
113 | } | ||
114 | |||
115 | return next(err) | ||
116 | } | ||
117 | |||
118 | private downloadImage (options: { | ||
119 | fileUrl: string | ||
120 | filename: string | ||
121 | size: { width: number, height: number } | ||
122 | }) { | ||
123 | const downloaderOptions = { | ||
124 | url: options.fileUrl, | ||
125 | destDir: this.directory, | ||
126 | destName: options.filename, | ||
127 | size: options.size | ||
128 | } | ||
129 | |||
130 | return downloadImageFromWorker(downloaderOptions) | ||
131 | } | ||
132 | } | ||
diff --git a/server/lib/files-cache/abstract-video-static-file-cache.ts b/server/lib/files-cache/shared/abstract-simple-file-cache.ts index a7ac88525..6fab322cd 100644 --- a/server/lib/files-cache/abstract-video-static-file-cache.ts +++ b/server/lib/files-cache/shared/abstract-simple-file-cache.ts | |||
@@ -1,10 +1,10 @@ | |||
1 | import { remove } from 'fs-extra' | 1 | import { remove } from 'fs-extra' |
2 | import { logger } from '../../helpers/logger' | 2 | import { logger } from '../../../helpers/logger' |
3 | import memoizee from 'memoizee' | 3 | import memoizee from 'memoizee' |
4 | 4 | ||
5 | type GetFilePathResult = { isOwned: boolean, path: string, downloadName?: string } | undefined | 5 | type GetFilePathResult = { isOwned: boolean, path: string, downloadName?: string } | undefined |
6 | 6 | ||
7 | export abstract class AbstractVideoStaticFileCache <T> { | 7 | export abstract class AbstractSimpleFileCache <T> { |
8 | 8 | ||
9 | getFilePath: (params: T) => Promise<GetFilePathResult> | 9 | getFilePath: (params: T) => Promise<GetFilePathResult> |
10 | 10 | ||
diff --git a/server/lib/files-cache/shared/index.ts b/server/lib/files-cache/shared/index.ts new file mode 100644 index 000000000..61c4aacc7 --- /dev/null +++ b/server/lib/files-cache/shared/index.ts | |||
@@ -0,0 +1,2 @@ | |||
1 | export * from './abstract-permanent-file-cache' | ||
2 | export * from './abstract-simple-file-cache' | ||
diff --git a/server/lib/files-cache/videos-caption-cache.ts b/server/lib/files-cache/video-captions-simple-file-cache.ts index d21acf4ef..cbeeff732 100644 --- a/server/lib/files-cache/videos-caption-cache.ts +++ b/server/lib/files-cache/video-captions-simple-file-cache.ts | |||
@@ -5,11 +5,11 @@ import { CONFIG } from '../../initializers/config' | |||
5 | import { FILES_CACHE } from '../../initializers/constants' | 5 | import { FILES_CACHE } from '../../initializers/constants' |
6 | import { VideoModel } from '../../models/video/video' | 6 | import { VideoModel } from '../../models/video/video' |
7 | import { VideoCaptionModel } from '../../models/video/video-caption' | 7 | import { VideoCaptionModel } from '../../models/video/video-caption' |
8 | import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache' | 8 | import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache' |
9 | 9 | ||
10 | class VideosCaptionCache extends AbstractVideoStaticFileCache <string> { | 10 | class VideoCaptionsSimpleFileCache extends AbstractSimpleFileCache <string> { |
11 | 11 | ||
12 | private static instance: VideosCaptionCache | 12 | private static instance: VideoCaptionsSimpleFileCache |
13 | 13 | ||
14 | private constructor () { | 14 | private constructor () { |
15 | super() | 15 | super() |
@@ -23,7 +23,9 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache <string> { | |||
23 | const videoCaption = await VideoCaptionModel.loadWithVideoByFilename(filename) | 23 | const videoCaption = await VideoCaptionModel.loadWithVideoByFilename(filename) |
24 | if (!videoCaption) return undefined | 24 | if (!videoCaption) return undefined |
25 | 25 | ||
26 | if (videoCaption.isOwned()) return { isOwned: true, path: join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.filename) } | 26 | if (videoCaption.isOwned()) { |
27 | return { isOwned: true, path: join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.filename) } | ||
28 | } | ||
27 | 29 | ||
28 | return this.loadRemoteFile(filename) | 30 | return this.loadRemoteFile(filename) |
29 | } | 31 | } |
@@ -55,5 +57,5 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache <string> { | |||
55 | } | 57 | } |
56 | 58 | ||
57 | export { | 59 | export { |
58 | VideosCaptionCache | 60 | VideoCaptionsSimpleFileCache |
59 | } | 61 | } |
diff --git a/server/lib/files-cache/video-miniature-permanent-file-cache.ts b/server/lib/files-cache/video-miniature-permanent-file-cache.ts new file mode 100644 index 000000000..35d9466f7 --- /dev/null +++ b/server/lib/files-cache/video-miniature-permanent-file-cache.ts | |||
@@ -0,0 +1,28 @@ | |||
1 | import { CONFIG } from '@server/initializers/config' | ||
2 | import { THUMBNAILS_SIZE } from '@server/initializers/constants' | ||
3 | import { ThumbnailModel } from '@server/models/video/thumbnail' | ||
4 | import { MThumbnail } from '@server/types/models' | ||
5 | import { ThumbnailType } from '@shared/models' | ||
6 | import { AbstractPermanentFileCache } from './shared' | ||
7 | |||
8 | export class VideoMiniaturePermanentFileCache extends AbstractPermanentFileCache<MThumbnail> { | ||
9 | |||
10 | constructor () { | ||
11 | super(CONFIG.STORAGE.THUMBNAILS_DIR) | ||
12 | } | ||
13 | |||
14 | protected loadModel (filename: string) { | ||
15 | return ThumbnailModel.loadByFilename(filename, ThumbnailType.MINIATURE) | ||
16 | } | ||
17 | |||
18 | protected getImageSize (image: MThumbnail): { width: number, height: number } { | ||
19 | if (image.width && image.height) { | ||
20 | return { | ||
21 | height: image.height, | ||
22 | width: image.width | ||
23 | } | ||
24 | } | ||
25 | |||
26 | return THUMBNAILS_SIZE | ||
27 | } | ||
28 | } | ||
diff --git a/server/lib/files-cache/videos-preview-cache.ts b/server/lib/files-cache/video-previews-simple-file-cache.ts index d19c3f4f4..a05e80e16 100644 --- a/server/lib/files-cache/videos-preview-cache.ts +++ b/server/lib/files-cache/video-previews-simple-file-cache.ts | |||
@@ -1,15 +1,15 @@ | |||
1 | import { join } from 'path' | 1 | import { join } from 'path' |
2 | import { FILES_CACHE } from '../../initializers/constants' | 2 | import { FILES_CACHE } from '../../initializers/constants' |
3 | import { VideoModel } from '../../models/video/video' | 3 | import { VideoModel } from '../../models/video/video' |
4 | import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache' | 4 | import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache' |
5 | import { doRequestAndSaveToFile } from '@server/helpers/requests' | 5 | import { doRequestAndSaveToFile } from '@server/helpers/requests' |
6 | import { ThumbnailModel } from '@server/models/video/thumbnail' | 6 | import { ThumbnailModel } from '@server/models/video/thumbnail' |
7 | import { ThumbnailType } from '@shared/models' | 7 | import { ThumbnailType } from '@shared/models' |
8 | import { logger } from '@server/helpers/logger' | 8 | import { logger } from '@server/helpers/logger' |
9 | 9 | ||
10 | class VideosPreviewCache extends AbstractVideoStaticFileCache <string> { | 10 | class VideoPreviewsSimpleFileCache extends AbstractSimpleFileCache <string> { |
11 | 11 | ||
12 | private static instance: VideosPreviewCache | 12 | private static instance: VideoPreviewsSimpleFileCache |
13 | 13 | ||
14 | private constructor () { | 14 | private constructor () { |
15 | super() | 15 | super() |
@@ -54,5 +54,5 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache <string> { | |||
54 | } | 54 | } |
55 | 55 | ||
56 | export { | 56 | export { |
57 | VideosPreviewCache | 57 | VideoPreviewsSimpleFileCache |
58 | } | 58 | } |
diff --git a/server/lib/files-cache/video-storyboards-simple-file-cache.ts b/server/lib/files-cache/video-storyboards-simple-file-cache.ts new file mode 100644 index 000000000..4cd96e70c --- /dev/null +++ b/server/lib/files-cache/video-storyboards-simple-file-cache.ts | |||
@@ -0,0 +1,53 @@ | |||
1 | import { join } from 'path' | ||
2 | import { logger } from '@server/helpers/logger' | ||
3 | import { doRequestAndSaveToFile } from '@server/helpers/requests' | ||
4 | import { StoryboardModel } from '@server/models/video/storyboard' | ||
5 | import { FILES_CACHE } from '../../initializers/constants' | ||
6 | import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache' | ||
7 | |||
8 | class VideoStoryboardsSimpleFileCache extends AbstractSimpleFileCache <string> { | ||
9 | |||
10 | private static instance: VideoStoryboardsSimpleFileCache | ||
11 | |||
12 | private constructor () { | ||
13 | super() | ||
14 | } | ||
15 | |||
16 | static get Instance () { | ||
17 | return this.instance || (this.instance = new this()) | ||
18 | } | ||
19 | |||
20 | async getFilePathImpl (filename: string) { | ||
21 | const storyboard = await StoryboardModel.loadWithVideoByFilename(filename) | ||
22 | if (!storyboard) return undefined | ||
23 | |||
24 | if (storyboard.Video.isOwned()) return { isOwned: true, path: storyboard.getPath() } | ||
25 | |||
26 | return this.loadRemoteFile(storyboard.filename) | ||
27 | } | ||
28 | |||
29 | // Key is the storyboard filename | ||
30 | protected async loadRemoteFile (key: string) { | ||
31 | const storyboard = await StoryboardModel.loadWithVideoByFilename(key) | ||
32 | if (!storyboard) return undefined | ||
33 | |||
34 | const destPath = join(FILES_CACHE.STORYBOARDS.DIRECTORY, storyboard.filename) | ||
35 | const remoteUrl = storyboard.getOriginFileUrl(storyboard.Video) | ||
36 | |||
37 | try { | ||
38 | await doRequestAndSaveToFile(remoteUrl, destPath) | ||
39 | |||
40 | logger.debug('Fetched remote storyboard %s to %s.', remoteUrl, destPath) | ||
41 | |||
42 | return { isOwned: false, path: destPath } | ||
43 | } catch (err) { | ||
44 | logger.info('Cannot fetch remote storyboard file %s.', remoteUrl, { err }) | ||
45 | |||
46 | return undefined | ||
47 | } | ||
48 | } | ||
49 | } | ||
50 | |||
51 | export { | ||
52 | VideoStoryboardsSimpleFileCache | ||
53 | } | ||
diff --git a/server/lib/files-cache/videos-torrent-cache.ts b/server/lib/files-cache/video-torrents-simple-file-cache.ts index a6bf98dd4..8bcd0b9bf 100644 --- a/server/lib/files-cache/videos-torrent-cache.ts +++ b/server/lib/files-cache/video-torrents-simple-file-cache.ts | |||
@@ -6,11 +6,11 @@ import { MVideo, MVideoFile } from '@server/types/models' | |||
6 | import { CONFIG } from '../../initializers/config' | 6 | import { CONFIG } from '../../initializers/config' |
7 | import { FILES_CACHE } from '../../initializers/constants' | 7 | import { FILES_CACHE } from '../../initializers/constants' |
8 | import { VideoModel } from '../../models/video/video' | 8 | import { VideoModel } from '../../models/video/video' |
9 | import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache' | 9 | import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache' |
10 | 10 | ||
11 | class VideosTorrentCache extends AbstractVideoStaticFileCache <string> { | 11 | class VideoTorrentsSimpleFileCache extends AbstractSimpleFileCache <string> { |
12 | 12 | ||
13 | private static instance: VideosTorrentCache | 13 | private static instance: VideoTorrentsSimpleFileCache |
14 | 14 | ||
15 | private constructor () { | 15 | private constructor () { |
16 | super() | 16 | super() |
@@ -66,5 +66,5 @@ class VideosTorrentCache extends AbstractVideoStaticFileCache <string> { | |||
66 | } | 66 | } |
67 | 67 | ||
68 | export { | 68 | export { |
69 | VideosTorrentCache | 69 | VideoTorrentsSimpleFileCache |
70 | } | 70 | } |
diff --git a/server/lib/hls.ts b/server/lib/hls.ts index fc1d7e1b0..19044d7c2 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts | |||
@@ -8,7 +8,7 @@ import { sha256 } from '@shared/extra-utils' | |||
8 | import { getVideoStreamDimensionsInfo } from '@shared/ffmpeg' | 8 | import { getVideoStreamDimensionsInfo } from '@shared/ffmpeg' |
9 | import { VideoStorage } from '@shared/models' | 9 | import { VideoStorage } from '@shared/models' |
10 | import { getAudioStreamCodec, getVideoStreamCodec } from '../helpers/ffmpeg' | 10 | import { getAudioStreamCodec, getVideoStreamCodec } from '../helpers/ffmpeg' |
11 | import { logger } from '../helpers/logger' | 11 | import { logger, loggerTagsFactory } from '../helpers/logger' |
12 | import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' | 12 | import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' |
13 | import { generateRandomString } from '../helpers/utils' | 13 | import { generateRandomString } from '../helpers/utils' |
14 | import { CONFIG } from '../initializers/config' | 14 | import { CONFIG } from '../initializers/config' |
@@ -20,6 +20,8 @@ import { storeHLSFileFromFilename } from './object-storage' | |||
20 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths' | 20 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths' |
21 | import { VideoPathManager } from './video-path-manager' | 21 | import { VideoPathManager } from './video-path-manager' |
22 | 22 | ||
23 | const lTags = loggerTagsFactory('hls') | ||
24 | |||
23 | async function updateStreamingPlaylistsInfohashesIfNeeded () { | 25 | async function updateStreamingPlaylistsInfohashesIfNeeded () { |
24 | const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() | 26 | const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() |
25 | 27 | ||
@@ -48,7 +50,7 @@ async function updatePlaylistAfterFileChange (video: MVideo, playlist: MStreamin | |||
48 | 50 | ||
49 | video.setHLSPlaylist(playlistWithFiles) | 51 | video.setHLSPlaylist(playlistWithFiles) |
50 | } catch (err) { | 52 | } catch (err) { |
51 | logger.info('Cannot update playlist after file change. Maybe due to concurrent transcoding', { err }) | 53 | logger.warn('Cannot update playlist after file change. Maybe due to concurrent transcoding', { err }) |
52 | } | 54 | } |
53 | } | 55 | } |
54 | 56 | ||
@@ -95,6 +97,8 @@ function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingPlaylist | |||
95 | const masterPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.playlistFilename) | 97 | const masterPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.playlistFilename) |
96 | await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') | 98 | await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') |
97 | 99 | ||
100 | logger.info('Updating %s master playlist file of video %s', masterPlaylistPath, video.uuid, lTags(video.uuid)) | ||
101 | |||
98 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) { | 102 | if (playlist.storage === VideoStorage.OBJECT_STORAGE) { |
99 | playlist.playlistUrl = await storeHLSFileFromFilename(playlist, playlist.playlistFilename) | 103 | playlist.playlistUrl = await storeHLSFileFromFilename(playlist, playlist.playlistFilename) |
100 | await remove(masterPlaylistPath) | 104 | await remove(masterPlaylistPath) |
diff --git a/server/lib/job-queue/handlers/generate-storyboard.ts b/server/lib/job-queue/handlers/generate-storyboard.ts new file mode 100644 index 000000000..ec07c568c --- /dev/null +++ b/server/lib/job-queue/handlers/generate-storyboard.ts | |||
@@ -0,0 +1,149 @@ | |||
1 | import { Job } from 'bullmq' | ||
2 | import { join } from 'path' | ||
3 | import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg' | ||
4 | import { generateImageFilename, getImageSize } from '@server/helpers/image-utils' | ||
5 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
6 | import { CONFIG } from '@server/initializers/config' | ||
7 | import { STORYBOARD } from '@server/initializers/constants' | ||
8 | import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' | ||
9 | import { VideoPathManager } from '@server/lib/video-path-manager' | ||
10 | import { StoryboardModel } from '@server/models/video/storyboard' | ||
11 | import { VideoModel } from '@server/models/video/video' | ||
12 | import { MVideo } from '@server/types/models' | ||
13 | import { FFmpegImage, isAudioFile } from '@shared/ffmpeg' | ||
14 | import { GenerateStoryboardPayload } from '@shared/models' | ||
15 | |||
16 | const lTagsBase = loggerTagsFactory('storyboard') | ||
17 | |||
18 | async function processGenerateStoryboard (job: Job): Promise<void> { | ||
19 | const payload = job.data as GenerateStoryboardPayload | ||
20 | const lTags = lTagsBase(payload.videoUUID) | ||
21 | |||
22 | logger.info('Processing generate storyboard of %s in job %s.', payload.videoUUID, job.id, lTags) | ||
23 | |||
24 | const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID) | ||
25 | |||
26 | try { | ||
27 | const video = await VideoModel.loadFull(payload.videoUUID) | ||
28 | if (!video) { | ||
29 | logger.info('Video %s does not exist anymore, skipping storyboard generation.', payload.videoUUID, lTags) | ||
30 | return | ||
31 | } | ||
32 | |||
33 | const inputFile = video.getMaxQualityFile() | ||
34 | |||
35 | await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => { | ||
36 | const isAudio = await isAudioFile(videoPath) | ||
37 | |||
38 | if (isAudio) { | ||
39 | logger.info('Do not generate a storyboard of %s since the video does not have a video stream', payload.videoUUID, lTags) | ||
40 | return | ||
41 | } | ||
42 | |||
43 | const ffmpeg = new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')) | ||
44 | |||
45 | const filename = generateImageFilename() | ||
46 | const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename) | ||
47 | |||
48 | const totalSprites = buildTotalSprites(video) | ||
49 | if (totalSprites === 0) { | ||
50 | logger.info('Do not generate a storyboard of %s because the video is not long enough', payload.videoUUID, lTags) | ||
51 | return | ||
52 | } | ||
53 | |||
54 | const spriteDuration = Math.round(video.duration / totalSprites) | ||
55 | |||
56 | const spritesCount = findGridSize({ | ||
57 | toFind: totalSprites, | ||
58 | maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT | ||
59 | }) | ||
60 | |||
61 | logger.debug( | ||
62 | 'Generating storyboard from video of %s to %s', video.uuid, destination, | ||
63 | { ...lTags, spritesCount, spriteDuration, videoDuration: video.duration } | ||
64 | ) | ||
65 | |||
66 | await ffmpeg.generateStoryboardFromVideo({ | ||
67 | destination, | ||
68 | path: videoPath, | ||
69 | sprites: { | ||
70 | size: STORYBOARD.SPRITE_SIZE, | ||
71 | count: spritesCount, | ||
72 | duration: spriteDuration | ||
73 | } | ||
74 | }) | ||
75 | |||
76 | const imageSize = await getImageSize(destination) | ||
77 | |||
78 | const existing = await StoryboardModel.loadByVideo(video.id) | ||
79 | if (existing) await existing.destroy() | ||
80 | |||
81 | await StoryboardModel.create({ | ||
82 | filename, | ||
83 | totalHeight: imageSize.height, | ||
84 | totalWidth: imageSize.width, | ||
85 | spriteHeight: STORYBOARD.SPRITE_SIZE.height, | ||
86 | spriteWidth: STORYBOARD.SPRITE_SIZE.width, | ||
87 | spriteDuration, | ||
88 | videoId: video.id | ||
89 | }) | ||
90 | |||
91 | logger.info('Storyboard generation %s ended for video %s.', destination, video.uuid, lTags) | ||
92 | }) | ||
93 | |||
94 | if (payload.federate) { | ||
95 | await federateVideoIfNeeded(video, false) | ||
96 | } | ||
97 | } finally { | ||
98 | inputFileMutexReleaser() | ||
99 | } | ||
100 | } | ||
101 | |||
102 | // --------------------------------------------------------------------------- | ||
103 | |||
104 | export { | ||
105 | processGenerateStoryboard | ||
106 | } | ||
107 | |||
108 | function buildTotalSprites (video: MVideo) { | ||
109 | const maxSprites = STORYBOARD.SPRITE_SIZE.height * STORYBOARD.SPRITE_SIZE.width | ||
110 | const totalSprites = Math.min(Math.ceil(video.duration), maxSprites) | ||
111 | |||
112 | // We can generate a single line | ||
113 | if (totalSprites <= STORYBOARD.SPRITES_MAX_EDGE_COUNT) return totalSprites | ||
114 | |||
115 | return findGridFit(totalSprites, STORYBOARD.SPRITES_MAX_EDGE_COUNT) | ||
116 | } | ||
117 | |||
118 | function findGridSize (options: { | ||
119 | toFind: number | ||
120 | maxEdgeCount: number | ||
121 | }) { | ||
122 | const { toFind, maxEdgeCount } = options | ||
123 | |||
124 | for (let i = 1; i <= maxEdgeCount; i++) { | ||
125 | for (let j = i; j <= maxEdgeCount; j++) { | ||
126 | if (toFind === i * j) return { width: j, height: i } | ||
127 | } | ||
128 | } | ||
129 | |||
130 | throw new Error(`Could not find grid size (to find: ${toFind}, max edge count: ${maxEdgeCount}`) | ||
131 | } | ||
132 | |||
133 | function findGridFit (value: number, maxMultiplier: number) { | ||
134 | for (let i = value; i--; i > 0) { | ||
135 | if (!isPrimeWithin(i, maxMultiplier)) return i | ||
136 | } | ||
137 | |||
138 | throw new Error('Could not find prime number below ' + value) | ||
139 | } | ||
140 | |||
141 | function isPrimeWithin (value: number, maxMultiplier: number) { | ||
142 | if (value < 2) return false | ||
143 | |||
144 | for (let i = 2, end = Math.min(Math.sqrt(value), maxMultiplier); i <= end; i++) { | ||
145 | if (value % i === 0 && value / i <= maxMultiplier) return false | ||
146 | } | ||
147 | |||
148 | return true | ||
149 | } | ||
diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts index 26752ff37..9a99b6722 100644 --- a/server/lib/job-queue/handlers/move-to-object-storage.ts +++ b/server/lib/job-queue/handlers/move-to-object-storage.ts | |||
@@ -4,7 +4,7 @@ import { join } from 'path' | |||
4 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | 4 | import { logger, loggerTagsFactory } from '@server/helpers/logger' |
5 | import { updateTorrentMetadata } from '@server/helpers/webtorrent' | 5 | import { updateTorrentMetadata } from '@server/helpers/webtorrent' |
6 | import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants' | 6 | import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants' |
7 | import { storeHLSFileFromFilename, storeWebTorrentFile } from '@server/lib/object-storage' | 7 | import { storeHLSFileFromFilename, storeWebVideoFile } from '@server/lib/object-storage' |
8 | import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' | 8 | import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' |
9 | import { VideoPathManager } from '@server/lib/video-path-manager' | 9 | import { VideoPathManager } from '@server/lib/video-path-manager' |
10 | import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state' | 10 | import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state' |
@@ -33,9 +33,9 @@ export async function processMoveToObjectStorage (job: Job) { | |||
33 | 33 | ||
34 | try { | 34 | try { |
35 | if (video.VideoFiles) { | 35 | if (video.VideoFiles) { |
36 | logger.debug('Moving %d webtorrent files for video %s.', video.VideoFiles.length, video.uuid, lTags) | 36 | logger.debug('Moving %d web video files for video %s.', video.VideoFiles.length, video.uuid, lTags) |
37 | 37 | ||
38 | await moveWebTorrentFiles(video) | 38 | await moveWebVideoFiles(video) |
39 | } | 39 | } |
40 | 40 | ||
41 | if (video.VideoStreamingPlaylists) { | 41 | if (video.VideoStreamingPlaylists) { |
@@ -75,11 +75,11 @@ export async function onMoveToObjectStorageFailure (job: Job, err: any) { | |||
75 | 75 | ||
76 | // --------------------------------------------------------------------------- | 76 | // --------------------------------------------------------------------------- |
77 | 77 | ||
78 | async function moveWebTorrentFiles (video: MVideoWithAllFiles) { | 78 | async function moveWebVideoFiles (video: MVideoWithAllFiles) { |
79 | for (const file of video.VideoFiles) { | 79 | for (const file of video.VideoFiles) { |
80 | if (file.storage !== VideoStorage.FILE_SYSTEM) continue | 80 | if (file.storage !== VideoStorage.FILE_SYSTEM) continue |
81 | 81 | ||
82 | const fileUrl = await storeWebTorrentFile(video, file) | 82 | const fileUrl = await storeWebVideoFile(video, file) |
83 | 83 | ||
84 | const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file) | 84 | const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file) |
85 | await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath }) | 85 | await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath }) |
diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts index 9a4550e4d..d221e8968 100644 --- a/server/lib/job-queue/handlers/video-file-import.ts +++ b/server/lib/job-queue/handlers/video-file-import.ts | |||
@@ -3,7 +3,7 @@ import { copy, stat } from 'fs-extra' | |||
3 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' | 3 | import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' |
4 | import { CONFIG } from '@server/initializers/config' | 4 | import { CONFIG } from '@server/initializers/config' |
5 | import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' | 5 | import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' |
6 | import { generateWebTorrentVideoFilename } from '@server/lib/paths' | 6 | import { generateWebVideoFilename } from '@server/lib/paths' |
7 | import { buildMoveToObjectStorageJob } from '@server/lib/video' | 7 | import { buildMoveToObjectStorageJob } from '@server/lib/video' |
8 | import { VideoPathManager } from '@server/lib/video-path-manager' | 8 | import { VideoPathManager } from '@server/lib/video-path-manager' |
9 | import { VideoModel } from '@server/models/video/video' | 9 | import { VideoModel } from '@server/models/video/video' |
@@ -56,7 +56,7 @@ async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) { | |||
56 | 56 | ||
57 | if (currentVideoFile) { | 57 | if (currentVideoFile) { |
58 | // Remove old file and old torrent | 58 | // Remove old file and old torrent |
59 | await video.removeWebTorrentFile(currentVideoFile) | 59 | await video.removeWebVideoFile(currentVideoFile) |
60 | // Remove the old video file from the array | 60 | // Remove the old video file from the array |
61 | video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile) | 61 | video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile) |
62 | 62 | ||
@@ -66,7 +66,7 @@ async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) { | |||
66 | const newVideoFile = new VideoFileModel({ | 66 | const newVideoFile = new VideoFileModel({ |
67 | resolution, | 67 | resolution, |
68 | extname: fileExt, | 68 | extname: fileExt, |
69 | filename: generateWebTorrentVideoFilename(resolution, fileExt), | 69 | filename: generateWebVideoFilename(resolution, fileExt), |
70 | storage: VideoStorage.FILE_SYSTEM, | 70 | storage: VideoStorage.FILE_SYSTEM, |
71 | size, | 71 | size, |
72 | fps, | 72 | fps, |
diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts index cdd362f6e..e5cd258d6 100644 --- a/server/lib/job-queue/handlers/video-import.ts +++ b/server/lib/job-queue/handlers/video-import.ts | |||
@@ -4,7 +4,7 @@ import { retryTransactionWrapper } from '@server/helpers/database-utils' | |||
4 | import { YoutubeDLWrapper } from '@server/helpers/youtube-dl' | 4 | import { YoutubeDLWrapper } from '@server/helpers/youtube-dl' |
5 | import { CONFIG } from '@server/initializers/config' | 5 | import { CONFIG } from '@server/initializers/config' |
6 | import { isPostImportVideoAccepted } from '@server/lib/moderation' | 6 | import { isPostImportVideoAccepted } from '@server/lib/moderation' |
7 | import { generateWebTorrentVideoFilename } from '@server/lib/paths' | 7 | import { generateWebVideoFilename } from '@server/lib/paths' |
8 | import { Hooks } from '@server/lib/plugins/hooks' | 8 | import { Hooks } from '@server/lib/plugins/hooks' |
9 | import { ServerConfigManager } from '@server/lib/server-config-manager' | 9 | import { ServerConfigManager } from '@server/lib/server-config-manager' |
10 | import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job' | 10 | import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job' |
@@ -39,7 +39,7 @@ import { VideoFileModel } from '../../../models/video/video-file' | |||
39 | import { VideoImportModel } from '../../../models/video/video-import' | 39 | import { VideoImportModel } from '../../../models/video/video-import' |
40 | import { federateVideoIfNeeded } from '../../activitypub/videos' | 40 | import { federateVideoIfNeeded } from '../../activitypub/videos' |
41 | import { Notifier } from '../../notifier' | 41 | import { Notifier } from '../../notifier' |
42 | import { generateVideoMiniature } from '../../thumbnail' | 42 | import { generateLocalVideoMiniature } from '../../thumbnail' |
43 | import { JobQueue } from '../job-queue' | 43 | import { JobQueue } from '../job-queue' |
44 | 44 | ||
45 | async function processVideoImport (job: Job): Promise<VideoImportPreventExceptionResult> { | 45 | async function processVideoImport (job: Job): Promise<VideoImportPreventExceptionResult> { |
@@ -148,7 +148,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid | |||
148 | extname: fileExt, | 148 | extname: fileExt, |
149 | resolution, | 149 | resolution, |
150 | size: stats.size, | 150 | size: stats.size, |
151 | filename: generateWebTorrentVideoFilename(resolution, fileExt), | 151 | filename: generateWebVideoFilename(resolution, fileExt), |
152 | fps, | 152 | fps, |
153 | videoId: videoImport.videoId | 153 | videoId: videoImport.videoId |
154 | } | 154 | } |
@@ -274,7 +274,7 @@ async function generateMiniature (videoImportWithFiles: MVideoImportDefaultFiles | |||
274 | } | 274 | } |
275 | } | 275 | } |
276 | 276 | ||
277 | const miniatureModel = await generateVideoMiniature({ | 277 | const miniatureModel = await generateLocalVideoMiniature({ |
278 | video: videoImportWithFiles.Video, | 278 | video: videoImportWithFiles.Video, |
279 | videoFile, | 279 | videoFile, |
280 | type: thumbnailType | 280 | type: thumbnailType |
@@ -306,6 +306,15 @@ async function afterImportSuccess (options: { | |||
306 | Notifier.Instance.notifyOnNewVideoIfNeeded(video) | 306 | Notifier.Instance.notifyOnNewVideoIfNeeded(video) |
307 | } | 307 | } |
308 | 308 | ||
309 | // Generate the storyboard in the job queue, and don't forget to federate an update after | ||
310 | await JobQueue.Instance.createJob({ | ||
311 | type: 'generate-video-storyboard' as 'generate-video-storyboard', | ||
312 | payload: { | ||
313 | videoUUID: video.uuid, | ||
314 | federate: true | ||
315 | } | ||
316 | }) | ||
317 | |||
309 | if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { | 318 | if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { |
310 | await JobQueue.Instance.createJob( | 319 | await JobQueue.Instance.createJob( |
311 | await buildMoveToObjectStorageJob({ video, previousVideoState: VideoState.TO_IMPORT }) | 320 | await buildMoveToObjectStorageJob({ video, previousVideoState: VideoState.TO_IMPORT }) |
diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts index 49feb53f2..ae886de35 100644 --- a/server/lib/job-queue/handlers/video-live-ending.ts +++ b/server/lib/job-queue/handlers/video-live-ending.ts | |||
@@ -1,11 +1,13 @@ | |||
1 | import { Job } from 'bullmq' | 1 | import { Job } from 'bullmq' |
2 | import { readdir, remove } from 'fs-extra' | 2 | import { readdir, remove } from 'fs-extra' |
3 | import { join } from 'path' | 3 | import { join } from 'path' |
4 | import { peertubeTruncate } from '@server/helpers/core-utils' | ||
5 | import { CONSTRAINTS_FIELDS } from '@server/initializers/constants' | ||
4 | import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' | 6 | import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' |
5 | import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' | 7 | import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' |
6 | import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live' | 8 | import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live' |
7 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths' | 9 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths' |
8 | import { generateVideoMiniature } from '@server/lib/thumbnail' | 10 | import { generateLocalVideoMiniature } from '@server/lib/thumbnail' |
9 | import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/hls-transcoding' | 11 | import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/hls-transcoding' |
10 | import { VideoPathManager } from '@server/lib/video-path-manager' | 12 | import { VideoPathManager } from '@server/lib/video-path-manager' |
11 | import { moveToNextState } from '@server/lib/video-state' | 13 | import { moveToNextState } from '@server/lib/video-state' |
@@ -20,8 +22,7 @@ import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@serv | |||
20 | import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg' | 22 | import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg' |
21 | import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' | 23 | import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' |
22 | import { logger, loggerTagsFactory } from '../../../helpers/logger' | 24 | import { logger, loggerTagsFactory } from '../../../helpers/logger' |
23 | import { peertubeTruncate } from '@server/helpers/core-utils' | 25 | import { JobQueue } from '../job-queue' |
24 | import { CONSTRAINTS_FIELDS } from '@server/initializers/constants' | ||
25 | 26 | ||
26 | const lTags = loggerTagsFactory('live', 'job') | 27 | const lTags = loggerTagsFactory('live', 'job') |
27 | 28 | ||
@@ -142,11 +143,13 @@ async function saveReplayToExternalVideo (options: { | |||
142 | await remove(replayDirectory) | 143 | await remove(replayDirectory) |
143 | 144 | ||
144 | for (const type of [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ]) { | 145 | for (const type of [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ]) { |
145 | const image = await generateVideoMiniature({ video: replayVideo, videoFile: replayVideo.getMaxQualityFile(), type }) | 146 | const image = await generateLocalVideoMiniature({ video: replayVideo, videoFile: replayVideo.getMaxQualityFile(), type }) |
146 | await replayVideo.addAndSaveThumbnail(image) | 147 | await replayVideo.addAndSaveThumbnail(image) |
147 | } | 148 | } |
148 | 149 | ||
149 | await moveToNextState({ video: replayVideo, isNewVideo: true }) | 150 | await moveToNextState({ video: replayVideo, isNewVideo: true }) |
151 | |||
152 | await createStoryboardJob(replayVideo) | ||
150 | } | 153 | } |
151 | 154 | ||
152 | async function replaceLiveByReplay (options: { | 155 | async function replaceLiveByReplay (options: { |
@@ -186,6 +189,7 @@ async function replaceLiveByReplay (options: { | |||
186 | 189 | ||
187 | await assignReplayFilesToVideo({ video: videoWithFiles, replayDirectory }) | 190 | await assignReplayFilesToVideo({ video: videoWithFiles, replayDirectory }) |
188 | 191 | ||
192 | // FIXME: should not happen in this function | ||
189 | if (permanentLive) { // Remove session replay | 193 | if (permanentLive) { // Remove session replay |
190 | await remove(replayDirectory) | 194 | await remove(replayDirectory) |
191 | } else { // We won't stream again in this live, we can delete the base replay directory | 195 | } else { // We won't stream again in this live, we can delete the base replay directory |
@@ -194,7 +198,7 @@ async function replaceLiveByReplay (options: { | |||
194 | 198 | ||
195 | // Regenerate the thumbnail & preview? | 199 | // Regenerate the thumbnail & preview? |
196 | if (videoWithFiles.getMiniature().automaticallyGenerated === true) { | 200 | if (videoWithFiles.getMiniature().automaticallyGenerated === true) { |
197 | const miniature = await generateVideoMiniature({ | 201 | const miniature = await generateLocalVideoMiniature({ |
198 | video: videoWithFiles, | 202 | video: videoWithFiles, |
199 | videoFile: videoWithFiles.getMaxQualityFile(), | 203 | videoFile: videoWithFiles.getMaxQualityFile(), |
200 | type: ThumbnailType.MINIATURE | 204 | type: ThumbnailType.MINIATURE |
@@ -203,7 +207,7 @@ async function replaceLiveByReplay (options: { | |||
203 | } | 207 | } |
204 | 208 | ||
205 | if (videoWithFiles.getPreview().automaticallyGenerated === true) { | 209 | if (videoWithFiles.getPreview().automaticallyGenerated === true) { |
206 | const preview = await generateVideoMiniature({ | 210 | const preview = await generateLocalVideoMiniature({ |
207 | video: videoWithFiles, | 211 | video: videoWithFiles, |
208 | videoFile: videoWithFiles.getMaxQualityFile(), | 212 | videoFile: videoWithFiles.getMaxQualityFile(), |
209 | type: ThumbnailType.PREVIEW | 213 | type: ThumbnailType.PREVIEW |
@@ -213,6 +217,8 @@ async function replaceLiveByReplay (options: { | |||
213 | 217 | ||
214 | // We consider this is a new video | 218 | // We consider this is a new video |
215 | await moveToNextState({ video: videoWithFiles, isNewVideo: true }) | 219 | await moveToNextState({ video: videoWithFiles, isNewVideo: true }) |
220 | |||
221 | await createStoryboardJob(videoWithFiles) | ||
216 | } | 222 | } |
217 | 223 | ||
218 | async function assignReplayFilesToVideo (options: { | 224 | async function assignReplayFilesToVideo (options: { |
@@ -277,3 +283,13 @@ async function cleanupLiveAndFederate (options: { | |||
277 | logger.warn('Cannot federate live after cleanup', { videoId: video.id, err }) | 283 | logger.warn('Cannot federate live after cleanup', { videoId: video.id, err }) |
278 | } | 284 | } |
279 | } | 285 | } |
286 | |||
287 | function createStoryboardJob (video: MVideo) { | ||
288 | return JobQueue.Instance.createJob({ | ||
289 | type: 'generate-video-storyboard' as 'generate-video-storyboard', | ||
290 | payload: { | ||
291 | videoUUID: video.uuid, | ||
292 | federate: true | ||
293 | } | ||
294 | }) | ||
295 | } | ||
diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts index f8758f170..1c8f4fd9f 100644 --- a/server/lib/job-queue/handlers/video-transcoding.ts +++ b/server/lib/job-queue/handlers/video-transcoding.ts | |||
@@ -1,8 +1,8 @@ | |||
1 | import { Job } from 'bullmq' | 1 | import { Job } from 'bullmq' |
2 | import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding' | 2 | import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding' |
3 | import { generateHlsPlaylistResolution } from '@server/lib/transcoding/hls-transcoding' | 3 | import { generateHlsPlaylistResolution } from '@server/lib/transcoding/hls-transcoding' |
4 | import { mergeAudioVideofile, optimizeOriginalVideofile, transcodeNewWebTorrentResolution } from '@server/lib/transcoding/web-transcoding' | 4 | import { mergeAudioVideofile, optimizeOriginalVideofile, transcodeNewWebVideoResolution } from '@server/lib/transcoding/web-transcoding' |
5 | import { removeAllWebTorrentFiles } from '@server/lib/video-file' | 5 | import { removeAllWebVideoFiles } from '@server/lib/video-file' |
6 | import { VideoPathManager } from '@server/lib/video-path-manager' | 6 | import { VideoPathManager } from '@server/lib/video-path-manager' |
7 | import { moveToFailedTranscodingState } from '@server/lib/video-state' | 7 | import { moveToFailedTranscodingState } from '@server/lib/video-state' |
8 | import { UserModel } from '@server/models/user/user' | 8 | import { UserModel } from '@server/models/user/user' |
@@ -11,7 +11,7 @@ import { MUser, MUserId, MVideoFullLight } from '@server/types/models' | |||
11 | import { | 11 | import { |
12 | HLSTranscodingPayload, | 12 | HLSTranscodingPayload, |
13 | MergeAudioTranscodingPayload, | 13 | MergeAudioTranscodingPayload, |
14 | NewWebTorrentResolutionTranscodingPayload, | 14 | NewWebVideoResolutionTranscodingPayload, |
15 | OptimizeTranscodingPayload, | 15 | OptimizeTranscodingPayload, |
16 | VideoTranscodingPayload | 16 | VideoTranscodingPayload |
17 | } from '@shared/models' | 17 | } from '@shared/models' |
@@ -22,9 +22,9 @@ type HandlerFunction = (job: Job, payload: VideoTranscodingPayload, video: MVide | |||
22 | 22 | ||
23 | const handlers: { [ id in VideoTranscodingPayload['type'] ]: HandlerFunction } = { | 23 | const handlers: { [ id in VideoTranscodingPayload['type'] ]: HandlerFunction } = { |
24 | 'new-resolution-to-hls': handleHLSJob, | 24 | 'new-resolution-to-hls': handleHLSJob, |
25 | 'new-resolution-to-webtorrent': handleNewWebTorrentResolutionJob, | 25 | 'new-resolution-to-web-video': handleNewWebVideoResolutionJob, |
26 | 'merge-audio-to-webtorrent': handleWebTorrentMergeAudioJob, | 26 | 'merge-audio-to-web-video': handleWebVideoMergeAudioJob, |
27 | 'optimize-to-webtorrent': handleWebTorrentOptimizeJob | 27 | 'optimize-to-web-video': handleWebVideoOptimizeJob |
28 | } | 28 | } |
29 | 29 | ||
30 | const lTags = loggerTagsFactory('transcoding') | 30 | const lTags = loggerTagsFactory('transcoding') |
@@ -74,7 +74,7 @@ export { | |||
74 | // Job handlers | 74 | // Job handlers |
75 | // --------------------------------------------------------------------------- | 75 | // --------------------------------------------------------------------------- |
76 | 76 | ||
77 | async function handleWebTorrentMergeAudioJob (job: Job, payload: MergeAudioTranscodingPayload, video: MVideoFullLight, user: MUserId) { | 77 | async function handleWebVideoMergeAudioJob (job: Job, payload: MergeAudioTranscodingPayload, video: MVideoFullLight, user: MUserId) { |
78 | logger.info('Handling merge audio transcoding job for %s.', video.uuid, lTags(video.uuid), { payload }) | 78 | logger.info('Handling merge audio transcoding job for %s.', video.uuid, lTags(video.uuid), { payload }) |
79 | 79 | ||
80 | await mergeAudioVideofile({ video, resolution: payload.resolution, fps: payload.fps, job }) | 80 | await mergeAudioVideofile({ video, resolution: payload.resolution, fps: payload.fps, job }) |
@@ -84,7 +84,7 @@ async function handleWebTorrentMergeAudioJob (job: Job, payload: MergeAudioTrans | |||
84 | await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: !payload.hasChildren, video }) | 84 | await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: !payload.hasChildren, video }) |
85 | } | 85 | } |
86 | 86 | ||
87 | async function handleWebTorrentOptimizeJob (job: Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) { | 87 | async function handleWebVideoOptimizeJob (job: Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) { |
88 | logger.info('Handling optimize transcoding job for %s.', video.uuid, lTags(video.uuid), { payload }) | 88 | logger.info('Handling optimize transcoding job for %s.', video.uuid, lTags(video.uuid), { payload }) |
89 | 89 | ||
90 | await optimizeOriginalVideofile({ video, inputVideoFile: video.getMaxQualityFile(), quickTranscode: payload.quickTranscode, job }) | 90 | await optimizeOriginalVideofile({ video, inputVideoFile: video.getMaxQualityFile(), quickTranscode: payload.quickTranscode, job }) |
@@ -96,12 +96,12 @@ async function handleWebTorrentOptimizeJob (job: Job, payload: OptimizeTranscodi | |||
96 | 96 | ||
97 | // --------------------------------------------------------------------------- | 97 | // --------------------------------------------------------------------------- |
98 | 98 | ||
99 | async function handleNewWebTorrentResolutionJob (job: Job, payload: NewWebTorrentResolutionTranscodingPayload, video: MVideoFullLight) { | 99 | async function handleNewWebVideoResolutionJob (job: Job, payload: NewWebVideoResolutionTranscodingPayload, video: MVideoFullLight) { |
100 | logger.info('Handling WebTorrent transcoding job for %s.', video.uuid, lTags(video.uuid), { payload }) | 100 | logger.info('Handling Web Video transcoding job for %s.', video.uuid, lTags(video.uuid), { payload }) |
101 | 101 | ||
102 | await transcodeNewWebTorrentResolution({ video, resolution: payload.resolution, fps: payload.fps, job }) | 102 | await transcodeNewWebVideoResolution({ video, resolution: payload.resolution, fps: payload.fps, job }) |
103 | 103 | ||
104 | logger.info('WebTorrent transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload }) | 104 | logger.info('Web Video transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload }) |
105 | 105 | ||
106 | await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video }) | 106 | await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video }) |
107 | } | 107 | } |
@@ -118,7 +118,7 @@ async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, videoArg: | |||
118 | video = await VideoModel.loadFull(videoArg.uuid) | 118 | video = await VideoModel.loadFull(videoArg.uuid) |
119 | 119 | ||
120 | const videoFileInput = payload.copyCodecs | 120 | const videoFileInput = payload.copyCodecs |
121 | ? video.getWebTorrentFile(payload.resolution) | 121 | ? video.getWebVideoFile(payload.resolution) |
122 | : video.getMaxQualityFile() | 122 | : video.getMaxQualityFile() |
123 | 123 | ||
124 | const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist() | 124 | const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist() |
@@ -140,10 +140,10 @@ async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, videoArg: | |||
140 | 140 | ||
141 | logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload }) | 141 | logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload }) |
142 | 142 | ||
143 | if (payload.deleteWebTorrentFiles === true) { | 143 | if (payload.deleteWebVideoFiles === true) { |
144 | logger.info('Removing WebTorrent files of %s now we have a HLS version of it.', video.uuid, lTags(video.uuid)) | 144 | logger.info('Removing Web Video files of %s now we have a HLS version of it.', video.uuid, lTags(video.uuid)) |
145 | 145 | ||
146 | await removeAllWebTorrentFiles(video) | 146 | await removeAllWebVideoFiles(video) |
147 | } | 147 | } |
148 | 148 | ||
149 | await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video }) | 149 | await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video }) |
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index 03f6fbea7..177bca285 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts | |||
@@ -25,6 +25,7 @@ import { | |||
25 | DeleteResumableUploadMetaFilePayload, | 25 | DeleteResumableUploadMetaFilePayload, |
26 | EmailPayload, | 26 | EmailPayload, |
27 | FederateVideoPayload, | 27 | FederateVideoPayload, |
28 | GenerateStoryboardPayload, | ||
28 | JobState, | 29 | JobState, |
29 | JobType, | 30 | JobType, |
30 | ManageVideoTorrentPayload, | 31 | ManageVideoTorrentPayload, |
@@ -65,6 +66,7 @@ import { processVideoLiveEnding } from './handlers/video-live-ending' | |||
65 | import { processVideoStudioEdition } from './handlers/video-studio-edition' | 66 | import { processVideoStudioEdition } from './handlers/video-studio-edition' |
66 | import { processVideoTranscoding } from './handlers/video-transcoding' | 67 | import { processVideoTranscoding } from './handlers/video-transcoding' |
67 | import { processVideosViewsStats } from './handlers/video-views-stats' | 68 | import { processVideosViewsStats } from './handlers/video-views-stats' |
69 | import { processGenerateStoryboard } from './handlers/generate-storyboard' | ||
68 | 70 | ||
69 | export type CreateJobArgument = | 71 | export type CreateJobArgument = |
70 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | | 72 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | |
@@ -91,7 +93,8 @@ export type CreateJobArgument = | |||
91 | { type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } | | 93 | { type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } | |
92 | { type: 'notify', payload: NotifyPayload } | | 94 | { type: 'notify', payload: NotifyPayload } | |
93 | { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } | | 95 | { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } | |
94 | { type: 'federate-video', payload: FederateVideoPayload } | 96 | { type: 'federate-video', payload: FederateVideoPayload } | |
97 | { type: 'generate-video-storyboard', payload: GenerateStoryboardPayload } | ||
95 | 98 | ||
96 | export type CreateJobOptions = { | 99 | export type CreateJobOptions = { |
97 | delay?: number | 100 | delay?: number |
@@ -122,7 +125,8 @@ const handlers: { [id in JobType]: (job: Job) => Promise<any> } = { | |||
122 | 'video-redundancy': processVideoRedundancy, | 125 | 'video-redundancy': processVideoRedundancy, |
123 | 'video-studio-edition': processVideoStudioEdition, | 126 | 'video-studio-edition': processVideoStudioEdition, |
124 | 'video-transcoding': processVideoTranscoding, | 127 | 'video-transcoding': processVideoTranscoding, |
125 | 'videos-views-stats': processVideosViewsStats | 128 | 'videos-views-stats': processVideosViewsStats, |
129 | 'generate-video-storyboard': processGenerateStoryboard | ||
126 | } | 130 | } |
127 | 131 | ||
128 | const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> } = { | 132 | const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise<any> } = { |
@@ -141,10 +145,11 @@ const jobTypes: JobType[] = [ | |||
141 | 'after-video-channel-import', | 145 | 'after-video-channel-import', |
142 | 'email', | 146 | 'email', |
143 | 'federate-video', | 147 | 'federate-video', |
144 | 'transcoding-job-builder', | 148 | 'generate-video-storyboard', |
145 | 'manage-video-torrent', | 149 | 'manage-video-torrent', |
146 | 'move-to-object-storage', | 150 | 'move-to-object-storage', |
147 | 'notify', | 151 | 'notify', |
152 | 'transcoding-job-builder', | ||
148 | 'video-channel-import', | 153 | 'video-channel-import', |
149 | 'video-file-import', | 154 | 'video-file-import', |
150 | 'video-import', | 155 | 'video-import', |
diff --git a/server/lib/local-actor.ts b/server/lib/local-actor.ts index 16dc265a3..611e6d0af 100644 --- a/server/lib/local-actor.ts +++ b/server/lib/local-actor.ts | |||
@@ -1,5 +1,4 @@ | |||
1 | import { remove } from 'fs-extra' | 1 | import { remove } from 'fs-extra' |
2 | import { LRUCache } from 'lru-cache' | ||
3 | import { join } from 'path' | 2 | import { join } from 'path' |
4 | import { Transaction } from 'sequelize/types' | 3 | import { Transaction } from 'sequelize/types' |
5 | import { ActorModel } from '@server/models/actor/actor' | 4 | import { ActorModel } from '@server/models/actor/actor' |
@@ -8,14 +7,14 @@ import { buildUUID } from '@shared/extra-utils' | |||
8 | import { ActivityPubActorType, ActorImageType } from '@shared/models' | 7 | import { ActivityPubActorType, ActorImageType } from '@shared/models' |
9 | import { retryTransactionWrapper } from '../helpers/database-utils' | 8 | import { retryTransactionWrapper } from '../helpers/database-utils' |
10 | import { CONFIG } from '../initializers/config' | 9 | import { CONFIG } from '../initializers/config' |
11 | import { ACTOR_IMAGES_SIZE, LRU_CACHE, WEBSERVER } from '../initializers/constants' | 10 | import { ACTOR_IMAGES_SIZE, WEBSERVER } from '../initializers/constants' |
12 | import { sequelizeTypescript } from '../initializers/database' | 11 | import { sequelizeTypescript } from '../initializers/database' |
13 | import { MAccountDefault, MActor, MChannelDefault } from '../types/models' | 12 | import { MAccountDefault, MActor, MChannelDefault } from '../types/models' |
14 | import { deleteActorImages, updateActorImages } from './activitypub/actors' | 13 | import { deleteActorImages, updateActorImages } from './activitypub/actors' |
15 | import { sendUpdateActor } from './activitypub/send' | 14 | import { sendUpdateActor } from './activitypub/send' |
16 | import { downloadImageFromWorker, processImageFromWorker } from './worker/parent-process' | 15 | import { processImageFromWorker } from './worker/parent-process' |
17 | 16 | ||
18 | function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string) { | 17 | export function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string) { |
19 | return new ActorModel({ | 18 | return new ActorModel({ |
20 | type, | 19 | type, |
21 | url, | 20 | url, |
@@ -32,7 +31,7 @@ function buildActorInstance (type: ActivityPubActorType, url: string, preferredU | |||
32 | }) as MActor | 31 | }) as MActor |
33 | } | 32 | } |
34 | 33 | ||
35 | async function updateLocalActorImageFiles ( | 34 | export async function updateLocalActorImageFiles ( |
36 | accountOrChannel: MAccountDefault | MChannelDefault, | 35 | accountOrChannel: MAccountDefault | MChannelDefault, |
37 | imagePhysicalFile: Express.Multer.File, | 36 | imagePhysicalFile: Express.Multer.File, |
38 | type: ActorImageType | 37 | type: ActorImageType |
@@ -41,7 +40,7 @@ async function updateLocalActorImageFiles ( | |||
41 | const extension = getLowercaseExtension(imagePhysicalFile.filename) | 40 | const extension = getLowercaseExtension(imagePhysicalFile.filename) |
42 | 41 | ||
43 | const imageName = buildUUID() + extension | 42 | const imageName = buildUUID() + extension |
44 | const destination = join(CONFIG.STORAGE.ACTOR_IMAGES, imageName) | 43 | const destination = join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, imageName) |
45 | await processImageFromWorker({ path: imagePhysicalFile.path, destination, newSize: imageSize, keepOriginal: true }) | 44 | await processImageFromWorker({ path: imagePhysicalFile.path, destination, newSize: imageSize, keepOriginal: true }) |
46 | 45 | ||
47 | return { | 46 | return { |
@@ -73,7 +72,7 @@ async function updateLocalActorImageFiles ( | |||
73 | })) | 72 | })) |
74 | } | 73 | } |
75 | 74 | ||
76 | async function deleteLocalActorImageFile (accountOrChannel: MAccountDefault | MChannelDefault, type: ActorImageType) { | 75 | export async function deleteLocalActorImageFile (accountOrChannel: MAccountDefault | MChannelDefault, type: ActorImageType) { |
77 | return retryTransactionWrapper(() => { | 76 | return retryTransactionWrapper(() => { |
78 | return sequelizeTypescript.transaction(async t => { | 77 | return sequelizeTypescript.transaction(async t => { |
79 | const updatedActor = await deleteActorImages(accountOrChannel.Actor, type, t) | 78 | const updatedActor = await deleteActorImages(accountOrChannel.Actor, type, t) |
@@ -88,7 +87,7 @@ async function deleteLocalActorImageFile (accountOrChannel: MAccountDefault | MC | |||
88 | 87 | ||
89 | // --------------------------------------------------------------------------- | 88 | // --------------------------------------------------------------------------- |
90 | 89 | ||
91 | async function findAvailableLocalActorName (baseActorName: string, transaction?: Transaction) { | 90 | export async function findAvailableLocalActorName (baseActorName: string, transaction?: Transaction) { |
92 | let actor = await ActorModel.loadLocalByName(baseActorName, transaction) | 91 | let actor = await ActorModel.loadLocalByName(baseActorName, transaction) |
93 | if (!actor) return baseActorName | 92 | if (!actor) return baseActorName |
94 | 93 | ||
@@ -101,34 +100,3 @@ async function findAvailableLocalActorName (baseActorName: string, transaction?: | |||
101 | 100 | ||
102 | throw new Error('Cannot find available actor local name (too much iterations).') | 101 | throw new Error('Cannot find available actor local name (too much iterations).') |
103 | } | 102 | } |
104 | |||
105 | // --------------------------------------------------------------------------- | ||
106 | |||
107 | function downloadActorImageFromWorker (options: { | ||
108 | fileUrl: string | ||
109 | filename: string | ||
110 | type: ActorImageType | ||
111 | size: typeof ACTOR_IMAGES_SIZE[ActorImageType][0] | ||
112 | }) { | ||
113 | const downloaderOptions = { | ||
114 | url: options.fileUrl, | ||
115 | destDir: CONFIG.STORAGE.ACTOR_IMAGES, | ||
116 | destName: options.filename, | ||
117 | size: options.size | ||
118 | } | ||
119 | |||
120 | return downloadImageFromWorker(downloaderOptions) | ||
121 | } | ||
122 | |||
123 | // Unsafe so could returns paths that does not exist anymore | ||
124 | const actorImagePathUnsafeCache = new LRUCache<string, string>({ max: LRU_CACHE.ACTOR_IMAGE_STATIC.MAX_SIZE }) | ||
125 | |||
126 | export { | ||
127 | actorImagePathUnsafeCache, | ||
128 | updateLocalActorImageFiles, | ||
129 | findAvailableLocalActorName, | ||
130 | downloadActorImageFromWorker, | ||
131 | deleteLocalActorImageFile, | ||
132 | downloadImageFromWorker, | ||
133 | buildActorInstance | ||
134 | } | ||
diff --git a/server/lib/object-storage/index.ts b/server/lib/object-storage/index.ts index 6525f8dfb..3ad6cab63 100644 --- a/server/lib/object-storage/index.ts +++ b/server/lib/object-storage/index.ts | |||
@@ -1,4 +1,5 @@ | |||
1 | export * from './keys' | 1 | export * from './keys' |
2 | export * from './proxy' | 2 | export * from './proxy' |
3 | export * from './pre-signed-urls' | ||
3 | export * from './urls' | 4 | export * from './urls' |
4 | export * from './videos' | 5 | export * from './videos' |
diff --git a/server/lib/object-storage/keys.ts b/server/lib/object-storage/keys.ts index 4f17073f4..6d2098298 100644 --- a/server/lib/object-storage/keys.ts +++ b/server/lib/object-storage/keys.ts | |||
@@ -9,12 +9,12 @@ function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylistVideo) { | |||
9 | return join(playlist.getStringType(), playlist.Video.uuid) | 9 | return join(playlist.getStringType(), playlist.Video.uuid) |
10 | } | 10 | } |
11 | 11 | ||
12 | function generateWebTorrentObjectStorageKey (filename: string) { | 12 | function generateWebVideoObjectStorageKey (filename: string) { |
13 | return filename | 13 | return filename |
14 | } | 14 | } |
15 | 15 | ||
16 | export { | 16 | export { |
17 | generateHLSObjectStorageKey, | 17 | generateHLSObjectStorageKey, |
18 | generateHLSObjectBaseStorageKey, | 18 | generateHLSObjectBaseStorageKey, |
19 | generateWebTorrentObjectStorageKey | 19 | generateWebVideoObjectStorageKey |
20 | } | 20 | } |
diff --git a/server/lib/object-storage/pre-signed-urls.ts b/server/lib/object-storage/pre-signed-urls.ts new file mode 100644 index 000000000..caf149bb8 --- /dev/null +++ b/server/lib/object-storage/pre-signed-urls.ts | |||
@@ -0,0 +1,46 @@ | |||
1 | import { GetObjectCommand } from '@aws-sdk/client-s3' | ||
2 | import { getSignedUrl } from '@aws-sdk/s3-request-presigner' | ||
3 | import { CONFIG } from '@server/initializers/config' | ||
4 | import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models' | ||
5 | import { generateHLSObjectStorageKey, generateWebVideoObjectStorageKey } from './keys' | ||
6 | import { buildKey, getClient } from './shared' | ||
7 | import { getHLSPublicFileUrl, getWebVideoPublicFileUrl } from './urls' | ||
8 | |||
9 | export async function generateWebVideoPresignedUrl (options: { | ||
10 | file: MVideoFile | ||
11 | downloadFilename: string | ||
12 | }) { | ||
13 | const { file, downloadFilename } = options | ||
14 | |||
15 | const key = generateWebVideoObjectStorageKey(file.filename) | ||
16 | |||
17 | const command = new GetObjectCommand({ | ||
18 | Bucket: CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BUCKET_NAME, | ||
19 | Key: buildKey(key, CONFIG.OBJECT_STORAGE.WEB_VIDEOS), | ||
20 | ResponseContentDisposition: `attachment; filename=${downloadFilename}` | ||
21 | }) | ||
22 | |||
23 | const url = await getSignedUrl(getClient(), command, { expiresIn: 3600 * 24 }) | ||
24 | |||
25 | return getWebVideoPublicFileUrl(url) | ||
26 | } | ||
27 | |||
28 | export async function generateHLSFilePresignedUrl (options: { | ||
29 | streamingPlaylist: MStreamingPlaylistVideo | ||
30 | file: MVideoFile | ||
31 | downloadFilename: string | ||
32 | }) { | ||
33 | const { streamingPlaylist, file, downloadFilename } = options | ||
34 | |||
35 | const key = generateHLSObjectStorageKey(streamingPlaylist, file.filename) | ||
36 | |||
37 | const command = new GetObjectCommand({ | ||
38 | Bucket: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME, | ||
39 | Key: buildKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS), | ||
40 | ResponseContentDisposition: `attachment; filename=${downloadFilename}` | ||
41 | }) | ||
42 | |||
43 | const url = await getSignedUrl(getClient(), command, { expiresIn: 3600 * 24 }) | ||
44 | |||
45 | return getHLSPublicFileUrl(url) | ||
46 | } | ||
diff --git a/server/lib/object-storage/proxy.ts b/server/lib/object-storage/proxy.ts index c782a8a25..c09a0d1b0 100644 --- a/server/lib/object-storage/proxy.ts +++ b/server/lib/object-storage/proxy.ts | |||
@@ -7,19 +7,19 @@ import { StreamReplacer } from '@server/helpers/stream-replacer' | |||
7 | import { MStreamingPlaylist, MVideo } from '@server/types/models' | 7 | import { MStreamingPlaylist, MVideo } from '@server/types/models' |
8 | import { HttpStatusCode } from '@shared/models' | 8 | import { HttpStatusCode } from '@shared/models' |
9 | import { injectQueryToPlaylistUrls } from '../hls' | 9 | import { injectQueryToPlaylistUrls } from '../hls' |
10 | import { getHLSFileReadStream, getWebTorrentFileReadStream } from './videos' | 10 | import { getHLSFileReadStream, getWebVideoFileReadStream } from './videos' |
11 | 11 | ||
12 | export async function proxifyWebTorrentFile (options: { | 12 | export async function proxifyWebVideoFile (options: { |
13 | req: express.Request | 13 | req: express.Request |
14 | res: express.Response | 14 | res: express.Response |
15 | filename: string | 15 | filename: string |
16 | }) { | 16 | }) { |
17 | const { req, res, filename } = options | 17 | const { req, res, filename } = options |
18 | 18 | ||
19 | logger.debug('Proxifying WebTorrent file %s from object storage.', filename) | 19 | logger.debug('Proxifying Web Video file %s from object storage.', filename) |
20 | 20 | ||
21 | try { | 21 | try { |
22 | const { response: s3Response, stream } = await getWebTorrentFileReadStream({ | 22 | const { response: s3Response, stream } = await getWebVideoFileReadStream({ |
23 | filename, | 23 | filename, |
24 | rangeHeader: req.header('range') | 24 | rangeHeader: req.header('range') |
25 | }) | 25 | }) |
diff --git a/server/lib/object-storage/urls.ts b/server/lib/object-storage/urls.ts index b8ef94559..40619cd5a 100644 --- a/server/lib/object-storage/urls.ts +++ b/server/lib/object-storage/urls.ts | |||
@@ -9,8 +9,8 @@ function getInternalUrl (config: BucketInfo, keyWithoutPrefix: string) { | |||
9 | 9 | ||
10 | // --------------------------------------------------------------------------- | 10 | // --------------------------------------------------------------------------- |
11 | 11 | ||
12 | function getWebTorrentPublicFileUrl (fileUrl: string) { | 12 | function getWebVideoPublicFileUrl (fileUrl: string) { |
13 | const baseUrl = CONFIG.OBJECT_STORAGE.VIDEOS.BASE_URL | 13 | const baseUrl = CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BASE_URL |
14 | if (!baseUrl) return fileUrl | 14 | if (!baseUrl) return fileUrl |
15 | 15 | ||
16 | return replaceByBaseUrl(fileUrl, baseUrl) | 16 | return replaceByBaseUrl(fileUrl, baseUrl) |
@@ -29,8 +29,8 @@ function getHLSPrivateFileUrl (video: MVideoUUID, filename: string) { | |||
29 | return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + video.uuid + `/${filename}` | 29 | return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + video.uuid + `/${filename}` |
30 | } | 30 | } |
31 | 31 | ||
32 | function getWebTorrentPrivateFileUrl (filename: string) { | 32 | function getWebVideoPrivateFileUrl (filename: string) { |
33 | return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEBSEED + filename | 33 | return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEB_VIDEOS + filename |
34 | } | 34 | } |
35 | 35 | ||
36 | // --------------------------------------------------------------------------- | 36 | // --------------------------------------------------------------------------- |
@@ -38,11 +38,11 @@ function getWebTorrentPrivateFileUrl (filename: string) { | |||
38 | export { | 38 | export { |
39 | getInternalUrl, | 39 | getInternalUrl, |
40 | 40 | ||
41 | getWebTorrentPublicFileUrl, | 41 | getWebVideoPublicFileUrl, |
42 | getHLSPublicFileUrl, | 42 | getHLSPublicFileUrl, |
43 | 43 | ||
44 | getHLSPrivateFileUrl, | 44 | getHLSPrivateFileUrl, |
45 | getWebTorrentPrivateFileUrl, | 45 | getWebVideoPrivateFileUrl, |
46 | 46 | ||
47 | replaceByBaseUrl | 47 | replaceByBaseUrl |
48 | } | 48 | } |
diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts index 9152c5352..891e9ff76 100644 --- a/server/lib/object-storage/videos.ts +++ b/server/lib/object-storage/videos.ts | |||
@@ -4,7 +4,7 @@ import { CONFIG } from '@server/initializers/config' | |||
4 | import { MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/types/models' | 4 | import { MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/types/models' |
5 | import { getHLSDirectory } from '../paths' | 5 | import { getHLSDirectory } from '../paths' |
6 | import { VideoPathManager } from '../video-path-manager' | 6 | import { VideoPathManager } from '../video-path-manager' |
7 | import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys' | 7 | import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebVideoObjectStorageKey } from './keys' |
8 | import { | 8 | import { |
9 | createObjectReadStream, | 9 | createObjectReadStream, |
10 | listKeysOfPrefix, | 10 | listKeysOfPrefix, |
@@ -55,21 +55,21 @@ function storeHLSFileFromContent (playlist: MStreamingPlaylistVideo, path: strin | |||
55 | 55 | ||
56 | // --------------------------------------------------------------------------- | 56 | // --------------------------------------------------------------------------- |
57 | 57 | ||
58 | function storeWebTorrentFile (video: MVideo, file: MVideoFile) { | 58 | function storeWebVideoFile (video: MVideo, file: MVideoFile) { |
59 | return storeObject({ | 59 | return storeObject({ |
60 | inputPath: VideoPathManager.Instance.getFSVideoFileOutputPath(video, file), | 60 | inputPath: VideoPathManager.Instance.getFSVideoFileOutputPath(video, file), |
61 | objectStorageKey: generateWebTorrentObjectStorageKey(file.filename), | 61 | objectStorageKey: generateWebVideoObjectStorageKey(file.filename), |
62 | bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS, | 62 | bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS, |
63 | isPrivate: video.hasPrivateStaticPath() | 63 | isPrivate: video.hasPrivateStaticPath() |
64 | }) | 64 | }) |
65 | } | 65 | } |
66 | 66 | ||
67 | // --------------------------------------------------------------------------- | 67 | // --------------------------------------------------------------------------- |
68 | 68 | ||
69 | async function updateWebTorrentFileACL (video: MVideo, file: MVideoFile) { | 69 | async function updateWebVideoFileACL (video: MVideo, file: MVideoFile) { |
70 | await updateObjectACL({ | 70 | await updateObjectACL({ |
71 | objectStorageKey: generateWebTorrentObjectStorageKey(file.filename), | 71 | objectStorageKey: generateWebVideoObjectStorageKey(file.filename), |
72 | bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS, | 72 | bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS, |
73 | isPrivate: video.hasPrivateStaticPath() | 73 | isPrivate: video.hasPrivateStaticPath() |
74 | }) | 74 | }) |
75 | } | 75 | } |
@@ -102,8 +102,8 @@ function removeHLSFileObjectStorageByFullKey (key: string) { | |||
102 | 102 | ||
103 | // --------------------------------------------------------------------------- | 103 | // --------------------------------------------------------------------------- |
104 | 104 | ||
105 | function removeWebTorrentObjectStorage (videoFile: MVideoFile) { | 105 | function removeWebVideoObjectStorage (videoFile: MVideoFile) { |
106 | return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS) | 106 | return removeObject(generateWebVideoObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.WEB_VIDEOS) |
107 | } | 107 | } |
108 | 108 | ||
109 | // --------------------------------------------------------------------------- | 109 | // --------------------------------------------------------------------------- |
@@ -122,15 +122,15 @@ async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename | |||
122 | return destination | 122 | return destination |
123 | } | 123 | } |
124 | 124 | ||
125 | async function makeWebTorrentFileAvailable (filename: string, destination: string) { | 125 | async function makeWebVideoFileAvailable (filename: string, destination: string) { |
126 | const key = generateWebTorrentObjectStorageKey(filename) | 126 | const key = generateWebVideoObjectStorageKey(filename) |
127 | 127 | ||
128 | logger.info('Fetching WebTorrent file %s from object storage to %s.', key, destination, lTags()) | 128 | logger.info('Fetching Web Video file %s from object storage to %s.', key, destination, lTags()) |
129 | 129 | ||
130 | await makeAvailable({ | 130 | await makeAvailable({ |
131 | key, | 131 | key, |
132 | destination, | 132 | destination, |
133 | bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS | 133 | bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS |
134 | }) | 134 | }) |
135 | 135 | ||
136 | return destination | 136 | return destination |
@@ -138,17 +138,17 @@ async function makeWebTorrentFileAvailable (filename: string, destination: strin | |||
138 | 138 | ||
139 | // --------------------------------------------------------------------------- | 139 | // --------------------------------------------------------------------------- |
140 | 140 | ||
141 | function getWebTorrentFileReadStream (options: { | 141 | function getWebVideoFileReadStream (options: { |
142 | filename: string | 142 | filename: string |
143 | rangeHeader: string | 143 | rangeHeader: string |
144 | }) { | 144 | }) { |
145 | const { filename, rangeHeader } = options | 145 | const { filename, rangeHeader } = options |
146 | 146 | ||
147 | const key = generateWebTorrentObjectStorageKey(filename) | 147 | const key = generateWebVideoObjectStorageKey(filename) |
148 | 148 | ||
149 | return createObjectReadStream({ | 149 | return createObjectReadStream({ |
150 | key, | 150 | key, |
151 | bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS, | 151 | bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS, |
152 | rangeHeader | 152 | rangeHeader |
153 | }) | 153 | }) |
154 | } | 154 | } |
@@ -174,12 +174,12 @@ function getHLSFileReadStream (options: { | |||
174 | export { | 174 | export { |
175 | listHLSFileKeysOf, | 175 | listHLSFileKeysOf, |
176 | 176 | ||
177 | storeWebTorrentFile, | 177 | storeWebVideoFile, |
178 | storeHLSFileFromFilename, | 178 | storeHLSFileFromFilename, |
179 | storeHLSFileFromPath, | 179 | storeHLSFileFromPath, |
180 | storeHLSFileFromContent, | 180 | storeHLSFileFromContent, |
181 | 181 | ||
182 | updateWebTorrentFileACL, | 182 | updateWebVideoFileACL, |
183 | updateHLSFilesACL, | 183 | updateHLSFilesACL, |
184 | 184 | ||
185 | removeHLSObjectStorage, | 185 | removeHLSObjectStorage, |
@@ -187,11 +187,11 @@ export { | |||
187 | removeHLSFileObjectStorageByPath, | 187 | removeHLSFileObjectStorageByPath, |
188 | removeHLSFileObjectStorageByFullKey, | 188 | removeHLSFileObjectStorageByFullKey, |
189 | 189 | ||
190 | removeWebTorrentObjectStorage, | 190 | removeWebVideoObjectStorage, |
191 | 191 | ||
192 | makeWebTorrentFileAvailable, | 192 | makeWebVideoFileAvailable, |
193 | makeHLSFileAvailable, | 193 | makeHLSFileAvailable, |
194 | 194 | ||
195 | getWebTorrentFileReadStream, | 195 | getWebVideoFileReadStream, |
196 | getHLSFileReadStream | 196 | getHLSFileReadStream |
197 | } | 197 | } |
diff --git a/server/lib/paths.ts b/server/lib/paths.ts index 470970f55..db1cdede2 100644 --- a/server/lib/paths.ts +++ b/server/lib/paths.ts | |||
@@ -8,7 +8,7 @@ import { isVideoInPrivateDirectory } from './video-privacy' | |||
8 | 8 | ||
9 | // ################## Video file name ################## | 9 | // ################## Video file name ################## |
10 | 10 | ||
11 | function generateWebTorrentVideoFilename (resolution: number, extname: string) { | 11 | function generateWebVideoFilename (resolution: number, extname: string) { |
12 | return buildUUID() + '-' + resolution + extname | 12 | return buildUUID() + '-' + resolution + extname |
13 | } | 13 | } |
14 | 14 | ||
@@ -76,7 +76,7 @@ function getFSTorrentFilePath (videoFile: MVideoFile) { | |||
76 | 76 | ||
77 | export { | 77 | export { |
78 | generateHLSVideoFilename, | 78 | generateHLSVideoFilename, |
79 | generateWebTorrentVideoFilename, | 79 | generateWebVideoFilename, |
80 | 80 | ||
81 | generateTorrentFileName, | 81 | generateTorrentFileName, |
82 | getFSTorrentFilePath, | 82 | getFSTorrentFilePath, |
diff --git a/server/lib/plugins/plugin-helpers-builder.ts b/server/lib/plugins/plugin-helpers-builder.ts index d235f52c0..b4e3eece4 100644 --- a/server/lib/plugins/plugin-helpers-builder.ts +++ b/server/lib/plugins/plugin-helpers-builder.ts | |||
@@ -104,7 +104,7 @@ function buildVideosHelpers () { | |||
104 | const video = await VideoModel.loadFull(id) | 104 | const video = await VideoModel.loadFull(id) |
105 | if (!video) return undefined | 105 | if (!video) return undefined |
106 | 106 | ||
107 | const webtorrentVideoFiles = (video.VideoFiles || []).map(f => ({ | 107 | const webVideoFiles = (video.VideoFiles || []).map(f => ({ |
108 | path: f.storage === VideoStorage.FILE_SYSTEM | 108 | path: f.storage === VideoStorage.FILE_SYSTEM |
109 | ? VideoPathManager.Instance.getFSVideoFileOutputPath(video, f) | 109 | ? VideoPathManager.Instance.getFSVideoFileOutputPath(video, f) |
110 | : null, | 110 | : null, |
@@ -138,8 +138,12 @@ function buildVideosHelpers () { | |||
138 | })) | 138 | })) |
139 | 139 | ||
140 | return { | 140 | return { |
141 | webtorrent: { | 141 | webtorrent: { // TODO: remove in v7 |
142 | videoFiles: webtorrentVideoFiles | 142 | videoFiles: webVideoFiles |
143 | }, | ||
144 | |||
145 | webVideo: { | ||
146 | videoFiles: webVideoFiles | ||
143 | }, | 147 | }, |
144 | 148 | ||
145 | hls: { | 149 | hls: { |
diff --git a/server/lib/redis.ts b/server/lib/redis.ts index 8430b2227..48d9986b5 100644 --- a/server/lib/redis.ts +++ b/server/lib/redis.ts | |||
@@ -325,8 +325,8 @@ class Redis { | |||
325 | const value = await this.getValue('resumable-upload-' + uploadId) | 325 | const value = await this.getValue('resumable-upload-' + uploadId) |
326 | 326 | ||
327 | return value | 327 | return value |
328 | ? JSON.parse(value) | 328 | ? JSON.parse(value) as { video: { id: number, shortUUID: string, uuid: string } } |
329 | : '' | 329 | : undefined |
330 | } | 330 | } |
331 | 331 | ||
332 | deleteUploadSession (uploadId: string) { | 332 | deleteUploadSession (uploadId: string) { |
diff --git a/server/lib/runners/job-handlers/shared/vod-helpers.ts b/server/lib/runners/job-handlers/shared/vod-helpers.ts index 93ae89ff8..1a2ad02ca 100644 --- a/server/lib/runners/job-handlers/shared/vod-helpers.ts +++ b/server/lib/runners/job-handlers/shared/vod-helpers.ts | |||
@@ -2,7 +2,7 @@ import { move } from 'fs-extra' | |||
2 | import { dirname, join } from 'path' | 2 | import { dirname, join } from 'path' |
3 | import { logger, LoggerTagsFn } from '@server/helpers/logger' | 3 | import { logger, LoggerTagsFn } from '@server/helpers/logger' |
4 | import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding' | 4 | import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding' |
5 | import { onWebTorrentVideoFileTranscoding } from '@server/lib/transcoding/web-transcoding' | 5 | import { onWebVideoFileTranscoding } from '@server/lib/transcoding/web-transcoding' |
6 | import { buildNewFile } from '@server/lib/video-file' | 6 | import { buildNewFile } from '@server/lib/video-file' |
7 | import { VideoModel } from '@server/models/video/video' | 7 | import { VideoModel } from '@server/models/video/video' |
8 | import { MVideoFullLight } from '@server/types/models' | 8 | import { MVideoFullLight } from '@server/types/models' |
@@ -22,7 +22,7 @@ export async function onVODWebVideoOrAudioMergeTranscodingJob (options: { | |||
22 | const newVideoFilePath = join(dirname(videoFilePath), videoFile.filename) | 22 | const newVideoFilePath = join(dirname(videoFilePath), videoFile.filename) |
23 | await move(videoFilePath, newVideoFilePath) | 23 | await move(videoFilePath, newVideoFilePath) |
24 | 24 | ||
25 | await onWebTorrentVideoFileTranscoding({ | 25 | await onWebVideoFileTranscoding({ |
26 | video, | 26 | video, |
27 | videoFile, | 27 | videoFile, |
28 | videoOutputPath: newVideoFilePath | 28 | videoOutputPath: newVideoFilePath |
diff --git a/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts index 5f247d792..905007db9 100644 --- a/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts +++ b/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts | |||
@@ -83,7 +83,7 @@ export class VODAudioMergeTranscodingJobHandler extends AbstractVODTranscodingJo | |||
83 | 83 | ||
84 | // We can remove the old audio file | 84 | // We can remove the old audio file |
85 | const oldAudioFile = video.VideoFiles[0] | 85 | const oldAudioFile = video.VideoFiles[0] |
86 | await video.removeWebTorrentFile(oldAudioFile) | 86 | await video.removeWebVideoFile(oldAudioFile) |
87 | await oldAudioFile.destroy() | 87 | await oldAudioFile.destroy() |
88 | video.VideoFiles = [] | 88 | video.VideoFiles = [] |
89 | 89 | ||
diff --git a/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts index cc94bcbda..02845952c 100644 --- a/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts +++ b/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts | |||
@@ -5,7 +5,7 @@ import { renameVideoFileInPlaylist } from '@server/lib/hls' | |||
5 | import { getHlsResolutionPlaylistFilename } from '@server/lib/paths' | 5 | import { getHlsResolutionPlaylistFilename } from '@server/lib/paths' |
6 | import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding' | 6 | import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding' |
7 | import { onHLSVideoFileTranscoding } from '@server/lib/transcoding/hls-transcoding' | 7 | import { onHLSVideoFileTranscoding } from '@server/lib/transcoding/hls-transcoding' |
8 | import { buildNewFile, removeAllWebTorrentFiles } from '@server/lib/video-file' | 8 | import { buildNewFile, removeAllWebVideoFiles } from '@server/lib/video-file' |
9 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' | 9 | import { VideoJobInfoModel } from '@server/models/video/video-job-info' |
10 | import { MVideo } from '@server/types/models' | 10 | import { MVideo } from '@server/types/models' |
11 | import { MRunnerJob } from '@server/types/models/runners' | 11 | import { MRunnerJob } from '@server/types/models/runners' |
@@ -106,7 +106,7 @@ export class VODHLSTranscodingJobHandler extends AbstractVODTranscodingJobHandle | |||
106 | if (privatePayload.deleteWebVideoFiles === true) { | 106 | if (privatePayload.deleteWebVideoFiles === true) { |
107 | logger.info('Removing web video files of %s now we have a HLS version of it.', video.uuid, this.lTags(video.uuid)) | 107 | logger.info('Removing web video files of %s now we have a HLS version of it.', video.uuid, this.lTags(video.uuid)) |
108 | 108 | ||
109 | await removeAllWebTorrentFiles(video) | 109 | await removeAllWebVideoFiles(video) |
110 | } | 110 | } |
111 | 111 | ||
112 | logger.info('Runner VOD HLS job %s for %s ended.', runnerJob.uuid, video.uuid, this.lTags(runnerJob.uuid, video.uuid)) | 112 | logger.info('Runner VOD HLS job %s for %s ended.', runnerJob.uuid, video.uuid, this.lTags(runnerJob.uuid, video.uuid)) |
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index dc450c338..24d340a73 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts | |||
@@ -23,7 +23,7 @@ import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlayli | |||
23 | import { getOrCreateAPVideo } from '../activitypub/videos' | 23 | import { getOrCreateAPVideo } from '../activitypub/videos' |
24 | import { downloadPlaylistSegments } from '../hls' | 24 | import { downloadPlaylistSegments } from '../hls' |
25 | import { removeVideoRedundancy } from '../redundancy' | 25 | import { removeVideoRedundancy } from '../redundancy' |
26 | import { generateHLSRedundancyUrl, generateWebTorrentRedundancyUrl } from '../video-urls' | 26 | import { generateHLSRedundancyUrl, generateWebVideoRedundancyUrl } from '../video-urls' |
27 | import { AbstractScheduler } from './abstract-scheduler' | 27 | import { AbstractScheduler } from './abstract-scheduler' |
28 | 28 | ||
29 | const lTags = loggerTagsFactory('redundancy') | 29 | const lTags = loggerTagsFactory('redundancy') |
@@ -244,7 +244,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { | |||
244 | const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({ | 244 | const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({ |
245 | expiresOn, | 245 | expiresOn, |
246 | url: getLocalVideoCacheFileActivityPubUrl(file), | 246 | url: getLocalVideoCacheFileActivityPubUrl(file), |
247 | fileUrl: generateWebTorrentRedundancyUrl(file), | 247 | fileUrl: generateWebVideoRedundancyUrl(file), |
248 | strategy, | 248 | strategy, |
249 | videoFileId: file.id, | 249 | videoFileId: file.id, |
250 | actorId: serverActor.id | 250 | actorId: serverActor.id |
diff --git a/server/lib/server-config-manager.ts b/server/lib/server-config-manager.ts index 924adb337..5ce89b16d 100644 --- a/server/lib/server-config-manager.ts +++ b/server/lib/server-config-manager.ts | |||
@@ -132,8 +132,8 @@ class ServerConfigManager { | |||
132 | hls: { | 132 | hls: { |
133 | enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.HLS.ENABLED | 133 | enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.HLS.ENABLED |
134 | }, | 134 | }, |
135 | webtorrent: { | 135 | web_videos: { |
136 | enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.WEBTORRENT.ENABLED | 136 | enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED |
137 | }, | 137 | }, |
138 | enabledResolutions: this.getEnabledResolutions('vod'), | 138 | enabledResolutions: this.getEnabledResolutions('vod'), |
139 | profile: CONFIG.TRANSCODING.PROFILE, | 139 | profile: CONFIG.TRANSCODING.PROFILE, |
diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts index 02b867a91..d95442795 100644 --- a/server/lib/thumbnail.ts +++ b/server/lib/thumbnail.ts | |||
@@ -7,13 +7,12 @@ import { ThumbnailModel } from '../models/video/thumbnail' | |||
7 | import { MVideoFile, MVideoThumbnail, MVideoUUID } from '../types/models' | 7 | import { MVideoFile, MVideoThumbnail, MVideoUUID } from '../types/models' |
8 | import { MThumbnail } from '../types/models/video/thumbnail' | 8 | import { MThumbnail } from '../types/models/video/thumbnail' |
9 | import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist' | 9 | import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist' |
10 | import { downloadImageFromWorker } from './local-actor' | ||
11 | import { VideoPathManager } from './video-path-manager' | 10 | import { VideoPathManager } from './video-path-manager' |
12 | import { processImageFromWorker } from './worker/parent-process' | 11 | import { downloadImageFromWorker, processImageFromWorker } from './worker/parent-process' |
13 | 12 | ||
14 | type ImageSize = { height?: number, width?: number } | 13 | type ImageSize = { height?: number, width?: number } |
15 | 14 | ||
16 | function updatePlaylistMiniatureFromExisting (options: { | 15 | function updateLocalPlaylistMiniatureFromExisting (options: { |
17 | inputPath: string | 16 | inputPath: string |
18 | playlist: MVideoPlaylistThumbnail | 17 | playlist: MVideoPlaylistThumbnail |
19 | automaticallyGenerated: boolean | 18 | automaticallyGenerated: boolean |
@@ -35,11 +34,12 @@ function updatePlaylistMiniatureFromExisting (options: { | |||
35 | width, | 34 | width, |
36 | type, | 35 | type, |
37 | automaticallyGenerated, | 36 | automaticallyGenerated, |
37 | onDisk: true, | ||
38 | existingThumbnail | 38 | existingThumbnail |
39 | }) | 39 | }) |
40 | } | 40 | } |
41 | 41 | ||
42 | function updatePlaylistMiniatureFromUrl (options: { | 42 | function updateRemotePlaylistMiniatureFromUrl (options: { |
43 | downloadUrl: string | 43 | downloadUrl: string |
44 | playlist: MVideoPlaylistThumbnail | 44 | playlist: MVideoPlaylistThumbnail |
45 | size?: ImageSize | 45 | size?: ImageSize |
@@ -57,42 +57,10 @@ function updatePlaylistMiniatureFromUrl (options: { | |||
57 | return downloadImageFromWorker({ url: downloadUrl, destDir: basePath, destName: filename, size: { width, height } }) | 57 | return downloadImageFromWorker({ url: downloadUrl, destDir: basePath, destName: filename, size: { width, height } }) |
58 | } | 58 | } |
59 | 59 | ||
60 | return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl }) | 60 | return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl, onDisk: true }) |
61 | } | 61 | } |
62 | 62 | ||
63 | function updateVideoMiniatureFromUrl (options: { | 63 | function updateLocalVideoMiniatureFromExisting (options: { |
64 | downloadUrl: string | ||
65 | video: MVideoThumbnail | ||
66 | type: ThumbnailType | ||
67 | size?: ImageSize | ||
68 | }) { | ||
69 | const { downloadUrl, video, type, size } = options | ||
70 | const { filename: updatedFilename, basePath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) | ||
71 | |||
72 | // Only save the file URL if it is a remote video | ||
73 | const fileUrl = video.isOwned() | ||
74 | ? null | ||
75 | : downloadUrl | ||
76 | |||
77 | const thumbnailUrlChanged = hasThumbnailUrlChanged(existingThumbnail, downloadUrl, video) | ||
78 | |||
79 | // Do not change the thumbnail filename if the file did not change | ||
80 | const filename = thumbnailUrlChanged | ||
81 | ? updatedFilename | ||
82 | : existingThumbnail.filename | ||
83 | |||
84 | const thumbnailCreator = () => { | ||
85 | if (thumbnailUrlChanged) { | ||
86 | return downloadImageFromWorker({ url: downloadUrl, destDir: basePath, destName: filename, size: { width, height } }) | ||
87 | } | ||
88 | |||
89 | return Promise.resolve() | ||
90 | } | ||
91 | |||
92 | return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl }) | ||
93 | } | ||
94 | |||
95 | function updateVideoMiniatureFromExisting (options: { | ||
96 | inputPath: string | 64 | inputPath: string |
97 | video: MVideoThumbnail | 65 | video: MVideoThumbnail |
98 | type: ThumbnailType | 66 | type: ThumbnailType |
@@ -115,11 +83,12 @@ function updateVideoMiniatureFromExisting (options: { | |||
115 | width, | 83 | width, |
116 | type, | 84 | type, |
117 | automaticallyGenerated, | 85 | automaticallyGenerated, |
118 | existingThumbnail | 86 | existingThumbnail, |
87 | onDisk: true | ||
119 | }) | 88 | }) |
120 | } | 89 | } |
121 | 90 | ||
122 | function generateVideoMiniature (options: { | 91 | function generateLocalVideoMiniature (options: { |
123 | video: MVideoThumbnail | 92 | video: MVideoThumbnail |
124 | videoFile: MVideoFile | 93 | videoFile: MVideoFile |
125 | type: ThumbnailType | 94 | type: ThumbnailType |
@@ -150,34 +119,68 @@ function generateVideoMiniature (options: { | |||
150 | width, | 119 | width, |
151 | type, | 120 | type, |
152 | automaticallyGenerated: true, | 121 | automaticallyGenerated: true, |
122 | onDisk: true, | ||
153 | existingThumbnail | 123 | existingThumbnail |
154 | }) | 124 | }) |
155 | }) | 125 | }) |
156 | } | 126 | } |
157 | 127 | ||
158 | function updatePlaceholderThumbnail (options: { | 128 | // --------------------------------------------------------------------------- |
159 | fileUrl: string | 129 | |
130 | function updateLocalVideoMiniatureFromUrl (options: { | ||
131 | downloadUrl: string | ||
160 | video: MVideoThumbnail | 132 | video: MVideoThumbnail |
161 | type: ThumbnailType | 133 | type: ThumbnailType |
162 | size: ImageSize | 134 | size?: ImageSize |
163 | }) { | 135 | }) { |
164 | const { fileUrl, video, type, size } = options | 136 | const { downloadUrl, video, type, size } = options |
165 | const { filename: updatedFilename, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) | 137 | const { filename: updatedFilename, basePath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) |
166 | 138 | ||
167 | const thumbnailUrlChanged = hasThumbnailUrlChanged(existingThumbnail, fileUrl, video) | 139 | // Only save the file URL if it is a remote video |
140 | const fileUrl = video.isOwned() | ||
141 | ? null | ||
142 | : downloadUrl | ||
168 | 143 | ||
169 | const thumbnail = existingThumbnail || new ThumbnailModel() | 144 | const thumbnailUrlChanged = hasThumbnailUrlChanged(existingThumbnail, downloadUrl, video) |
170 | 145 | ||
171 | // Do not change the thumbnail filename if the file did not change | 146 | // Do not change the thumbnail filename if the file did not change |
172 | const filename = thumbnailUrlChanged | 147 | const filename = thumbnailUrlChanged |
173 | ? updatedFilename | 148 | ? updatedFilename |
174 | : existingThumbnail.filename | 149 | : existingThumbnail.filename |
175 | 150 | ||
176 | thumbnail.filename = filename | 151 | const thumbnailCreator = () => { |
152 | if (thumbnailUrlChanged) { | ||
153 | return downloadImageFromWorker({ url: downloadUrl, destDir: basePath, destName: filename, size: { width, height } }) | ||
154 | } | ||
155 | |||
156 | return Promise.resolve() | ||
157 | } | ||
158 | |||
159 | return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl, onDisk: true }) | ||
160 | } | ||
161 | |||
162 | function updateRemoteVideoThumbnail (options: { | ||
163 | fileUrl: string | ||
164 | video: MVideoThumbnail | ||
165 | type: ThumbnailType | ||
166 | size: ImageSize | ||
167 | onDisk: boolean | ||
168 | }) { | ||
169 | const { fileUrl, video, type, size, onDisk } = options | ||
170 | const { filename: generatedFilename, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) | ||
171 | |||
172 | const thumbnail = existingThumbnail || new ThumbnailModel() | ||
173 | |||
174 | // Do not change the thumbnail filename if the file did not change | ||
175 | if (hasThumbnailUrlChanged(existingThumbnail, fileUrl, video)) { | ||
176 | thumbnail.filename = generatedFilename | ||
177 | } | ||
178 | |||
177 | thumbnail.height = height | 179 | thumbnail.height = height |
178 | thumbnail.width = width | 180 | thumbnail.width = width |
179 | thumbnail.type = type | 181 | thumbnail.type = type |
180 | thumbnail.fileUrl = fileUrl | 182 | thumbnail.fileUrl = fileUrl |
183 | thumbnail.onDisk = onDisk | ||
181 | 184 | ||
182 | return thumbnail | 185 | return thumbnail |
183 | } | 186 | } |
@@ -185,14 +188,18 @@ function updatePlaceholderThumbnail (options: { | |||
185 | // --------------------------------------------------------------------------- | 188 | // --------------------------------------------------------------------------- |
186 | 189 | ||
187 | export { | 190 | export { |
188 | generateVideoMiniature, | 191 | generateLocalVideoMiniature, |
189 | updateVideoMiniatureFromUrl, | 192 | updateLocalVideoMiniatureFromUrl, |
190 | updateVideoMiniatureFromExisting, | 193 | updateLocalVideoMiniatureFromExisting, |
191 | updatePlaceholderThumbnail, | 194 | updateRemoteVideoThumbnail, |
192 | updatePlaylistMiniatureFromUrl, | 195 | updateRemotePlaylistMiniatureFromUrl, |
193 | updatePlaylistMiniatureFromExisting | 196 | updateLocalPlaylistMiniatureFromExisting |
194 | } | 197 | } |
195 | 198 | ||
199 | // --------------------------------------------------------------------------- | ||
200 | // Private | ||
201 | // --------------------------------------------------------------------------- | ||
202 | |||
196 | function hasThumbnailUrlChanged (existingThumbnail: MThumbnail, downloadUrl: string, video: MVideoUUID) { | 203 | function hasThumbnailUrlChanged (existingThumbnail: MThumbnail, downloadUrl: string, video: MVideoUUID) { |
197 | const existingUrl = existingThumbnail | 204 | const existingUrl = existingThumbnail |
198 | ? existingThumbnail.fileUrl | 205 | ? existingThumbnail.fileUrl |
@@ -258,6 +265,7 @@ async function updateThumbnailFromFunction (parameters: { | |||
258 | height: number | 265 | height: number |
259 | width: number | 266 | width: number |
260 | type: ThumbnailType | 267 | type: ThumbnailType |
268 | onDisk: boolean | ||
261 | automaticallyGenerated?: boolean | 269 | automaticallyGenerated?: boolean |
262 | fileUrl?: string | 270 | fileUrl?: string |
263 | existingThumbnail?: MThumbnail | 271 | existingThumbnail?: MThumbnail |
@@ -269,6 +277,7 @@ async function updateThumbnailFromFunction (parameters: { | |||
269 | height, | 277 | height, |
270 | type, | 278 | type, |
271 | existingThumbnail, | 279 | existingThumbnail, |
280 | onDisk, | ||
272 | automaticallyGenerated = null, | 281 | automaticallyGenerated = null, |
273 | fileUrl = null | 282 | fileUrl = null |
274 | } = parameters | 283 | } = parameters |
@@ -285,6 +294,7 @@ async function updateThumbnailFromFunction (parameters: { | |||
285 | thumbnail.type = type | 294 | thumbnail.type = type |
286 | thumbnail.fileUrl = fileUrl | 295 | thumbnail.fileUrl = fileUrl |
287 | thumbnail.automaticallyGenerated = automaticallyGenerated | 296 | thumbnail.automaticallyGenerated = automaticallyGenerated |
297 | thumbnail.onDisk = onDisk | ||
288 | 298 | ||
289 | if (oldFilename) thumbnail.previousThumbnailFilename = oldFilename | 299 | if (oldFilename) thumbnail.previousThumbnailFilename = oldFilename |
290 | 300 | ||
diff --git a/server/lib/transcoding/create-transcoding-job.ts b/server/lib/transcoding/create-transcoding-job.ts index abe32684d..d78e68b87 100644 --- a/server/lib/transcoding/create-transcoding-job.ts +++ b/server/lib/transcoding/create-transcoding-job.ts | |||
@@ -15,7 +15,7 @@ export function createOptimizeOrMergeAudioJobs (options: { | |||
15 | // --------------------------------------------------------------------------- | 15 | // --------------------------------------------------------------------------- |
16 | 16 | ||
17 | export function createTranscodingJobs (options: { | 17 | export function createTranscodingJobs (options: { |
18 | transcodingType: 'hls' | 'webtorrent' | 18 | transcodingType: 'hls' | 'webtorrent' | 'web-video' // TODO: remove webtorrent in v7 |
19 | video: MVideoFullLight | 19 | video: MVideoFullLight |
20 | resolutions: number[] | 20 | resolutions: number[] |
21 | isNewVideo: boolean | 21 | isNewVideo: boolean |
diff --git a/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts b/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts index 80dc05bfb..15fc814ae 100644 --- a/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts +++ b/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts | |||
@@ -12,7 +12,7 @@ export abstract class AbstractJobBuilder { | |||
12 | }): Promise<any> | 12 | }): Promise<any> |
13 | 13 | ||
14 | abstract createTranscodingJobs (options: { | 14 | abstract createTranscodingJobs (options: { |
15 | transcodingType: 'hls' | 'webtorrent' | 15 | transcodingType: 'hls' | 'webtorrent' | 'web-video' // TODO: remove webtorrent in v7 |
16 | video: MVideoFullLight | 16 | video: MVideoFullLight |
17 | resolutions: number[] | 17 | resolutions: number[] |
18 | isNewVideo: boolean | 18 | isNewVideo: boolean |
diff --git a/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts b/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts index 4f802e2a6..0505c2b2f 100644 --- a/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts +++ b/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts | |||
@@ -12,7 +12,7 @@ import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAud | |||
12 | import { | 12 | import { |
13 | HLSTranscodingPayload, | 13 | HLSTranscodingPayload, |
14 | MergeAudioTranscodingPayload, | 14 | MergeAudioTranscodingPayload, |
15 | NewWebTorrentResolutionTranscodingPayload, | 15 | NewWebVideoResolutionTranscodingPayload, |
16 | OptimizeTranscodingPayload, | 16 | OptimizeTranscodingPayload, |
17 | VideoTranscodingPayload | 17 | VideoTranscodingPayload |
18 | } from '@shared/models' | 18 | } from '@shared/models' |
@@ -33,7 +33,7 @@ export class TranscodingJobQueueBuilder extends AbstractJobBuilder { | |||
33 | const { video, videoFile, isNewVideo, user, videoFileAlreadyLocked } = options | 33 | const { video, videoFile, isNewVideo, user, videoFileAlreadyLocked } = options |
34 | 34 | ||
35 | let mergeOrOptimizePayload: MergeAudioTranscodingPayload | OptimizeTranscodingPayload | 35 | let mergeOrOptimizePayload: MergeAudioTranscodingPayload | OptimizeTranscodingPayload |
36 | let nextTranscodingSequentialJobPayloads: (NewWebTorrentResolutionTranscodingPayload | HLSTranscodingPayload)[][] = [] | 36 | let nextTranscodingSequentialJobPayloads: (NewWebVideoResolutionTranscodingPayload | HLSTranscodingPayload)[][] = [] |
37 | 37 | ||
38 | const mutexReleaser = videoFileAlreadyLocked | 38 | const mutexReleaser = videoFileAlreadyLocked |
39 | ? () => {} | 39 | ? () => {} |
@@ -60,7 +60,7 @@ export class TranscodingJobQueueBuilder extends AbstractJobBuilder { | |||
60 | if (CONFIG.TRANSCODING.HLS.ENABLED === true) { | 60 | if (CONFIG.TRANSCODING.HLS.ENABLED === true) { |
61 | nextTranscodingSequentialJobPayloads.push([ | 61 | nextTranscodingSequentialJobPayloads.push([ |
62 | this.buildHLSJobPayload({ | 62 | this.buildHLSJobPayload({ |
63 | deleteWebTorrentFiles: CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false, | 63 | deleteWebVideoFiles: CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED === false, |
64 | 64 | ||
65 | // We had some issues with a web video quick transcoded while producing a HLS version of it | 65 | // We had some issues with a web video quick transcoded while producing a HLS version of it |
66 | copyCodecs: !quickTranscode, | 66 | copyCodecs: !quickTranscode, |
@@ -116,7 +116,7 @@ export class TranscodingJobQueueBuilder extends AbstractJobBuilder { | |||
116 | // --------------------------------------------------------------------------- | 116 | // --------------------------------------------------------------------------- |
117 | 117 | ||
118 | async createTranscodingJobs (options: { | 118 | async createTranscodingJobs (options: { |
119 | transcodingType: 'hls' | 'webtorrent' | 119 | transcodingType: 'hls' | 'webtorrent' | 'web-video' // TODO: remove webtorrent in v7 |
120 | video: MVideoFullLight | 120 | video: MVideoFullLight |
121 | resolutions: number[] | 121 | resolutions: number[] |
122 | isNewVideo: boolean | 122 | isNewVideo: boolean |
@@ -138,8 +138,8 @@ export class TranscodingJobQueueBuilder extends AbstractJobBuilder { | |||
138 | return this.buildHLSJobPayload({ videoUUID: video.uuid, resolution, fps, isNewVideo }) | 138 | return this.buildHLSJobPayload({ videoUUID: video.uuid, resolution, fps, isNewVideo }) |
139 | } | 139 | } |
140 | 140 | ||
141 | if (transcodingType === 'webtorrent') { | 141 | if (transcodingType === 'webtorrent' || transcodingType === 'web-video') { |
142 | return this.buildWebTorrentJobPayload({ videoUUID: video.uuid, resolution, fps, isNewVideo }) | 142 | return this.buildWebVideoJobPayload({ videoUUID: video.uuid, resolution, fps, isNewVideo }) |
143 | } | 143 | } |
144 | 144 | ||
145 | throw new Error('Unknown transcoding type') | 145 | throw new Error('Unknown transcoding type') |
@@ -149,7 +149,7 @@ export class TranscodingJobQueueBuilder extends AbstractJobBuilder { | |||
149 | 149 | ||
150 | const parent = transcodingType === 'hls' | 150 | const parent = transcodingType === 'hls' |
151 | ? this.buildHLSJobPayload({ videoUUID: video.uuid, resolution: maxResolution, fps, isNewVideo }) | 151 | ? this.buildHLSJobPayload({ videoUUID: video.uuid, resolution: maxResolution, fps, isNewVideo }) |
152 | : this.buildWebTorrentJobPayload({ videoUUID: video.uuid, resolution: maxResolution, fps, isNewVideo }) | 152 | : this.buildWebVideoJobPayload({ videoUUID: video.uuid, resolution: maxResolution, fps, isNewVideo }) |
153 | 153 | ||
154 | // Process the last resolution after the other ones to prevent concurrency issue | 154 | // Process the last resolution after the other ones to prevent concurrency issue |
155 | // Because low resolutions use the biggest one as ffmpeg input | 155 | // Because low resolutions use the biggest one as ffmpeg input |
@@ -160,8 +160,8 @@ export class TranscodingJobQueueBuilder extends AbstractJobBuilder { | |||
160 | 160 | ||
161 | private async createTranscodingJobsWithChildren (options: { | 161 | private async createTranscodingJobsWithChildren (options: { |
162 | videoUUID: string | 162 | videoUUID: string |
163 | parent: (HLSTranscodingPayload | NewWebTorrentResolutionTranscodingPayload) | 163 | parent: (HLSTranscodingPayload | NewWebVideoResolutionTranscodingPayload) |
164 | children: (HLSTranscodingPayload | NewWebTorrentResolutionTranscodingPayload)[] | 164 | children: (HLSTranscodingPayload | NewWebVideoResolutionTranscodingPayload)[] |
165 | user: MUserId | null | 165 | user: MUserId | null |
166 | }) { | 166 | }) { |
167 | const { videoUUID, parent, children, user } = options | 167 | const { videoUUID, parent, children, user } = options |
@@ -203,14 +203,14 @@ export class TranscodingJobQueueBuilder extends AbstractJobBuilder { | |||
203 | options | 203 | options |
204 | ) | 204 | ) |
205 | 205 | ||
206 | const sequentialPayloads: (NewWebTorrentResolutionTranscodingPayload | HLSTranscodingPayload)[][] = [] | 206 | const sequentialPayloads: (NewWebVideoResolutionTranscodingPayload | HLSTranscodingPayload)[][] = [] |
207 | 207 | ||
208 | for (const resolution of resolutionsEnabled) { | 208 | for (const resolution of resolutionsEnabled) { |
209 | const fps = computeOutputFPS({ inputFPS: inputVideoFPS, resolution }) | 209 | const fps = computeOutputFPS({ inputFPS: inputVideoFPS, resolution }) |
210 | 210 | ||
211 | if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED) { | 211 | if (CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED) { |
212 | const payloads: (NewWebTorrentResolutionTranscodingPayload | HLSTranscodingPayload)[] = [ | 212 | const payloads: (NewWebVideoResolutionTranscodingPayload | HLSTranscodingPayload)[] = [ |
213 | this.buildWebTorrentJobPayload({ | 213 | this.buildWebVideoJobPayload({ |
214 | videoUUID: video.uuid, | 214 | videoUUID: video.uuid, |
215 | resolution, | 215 | resolution, |
216 | fps, | 216 | fps, |
@@ -253,10 +253,10 @@ export class TranscodingJobQueueBuilder extends AbstractJobBuilder { | |||
253 | resolution: number | 253 | resolution: number |
254 | fps: number | 254 | fps: number |
255 | isNewVideo: boolean | 255 | isNewVideo: boolean |
256 | deleteWebTorrentFiles?: boolean // default false | 256 | deleteWebVideoFiles?: boolean // default false |
257 | copyCodecs?: boolean // default false | 257 | copyCodecs?: boolean // default false |
258 | }): HLSTranscodingPayload { | 258 | }): HLSTranscodingPayload { |
259 | const { videoUUID, resolution, fps, isNewVideo, deleteWebTorrentFiles = false, copyCodecs = false } = options | 259 | const { videoUUID, resolution, fps, isNewVideo, deleteWebVideoFiles = false, copyCodecs = false } = options |
260 | 260 | ||
261 | return { | 261 | return { |
262 | type: 'new-resolution-to-hls', | 262 | type: 'new-resolution-to-hls', |
@@ -265,20 +265,20 @@ export class TranscodingJobQueueBuilder extends AbstractJobBuilder { | |||
265 | fps, | 265 | fps, |
266 | copyCodecs, | 266 | copyCodecs, |
267 | isNewVideo, | 267 | isNewVideo, |
268 | deleteWebTorrentFiles | 268 | deleteWebVideoFiles |
269 | } | 269 | } |
270 | } | 270 | } |
271 | 271 | ||
272 | private buildWebTorrentJobPayload (options: { | 272 | private buildWebVideoJobPayload (options: { |
273 | videoUUID: string | 273 | videoUUID: string |
274 | resolution: number | 274 | resolution: number |
275 | fps: number | 275 | fps: number |
276 | isNewVideo: boolean | 276 | isNewVideo: boolean |
277 | }): NewWebTorrentResolutionTranscodingPayload { | 277 | }): NewWebVideoResolutionTranscodingPayload { |
278 | const { videoUUID, resolution, fps, isNewVideo } = options | 278 | const { videoUUID, resolution, fps, isNewVideo } = options |
279 | 279 | ||
280 | return { | 280 | return { |
281 | type: 'new-resolution-to-webtorrent', | 281 | type: 'new-resolution-to-web-video', |
282 | videoUUID, | 282 | videoUUID, |
283 | isNewVideo, | 283 | isNewVideo, |
284 | resolution, | 284 | resolution, |
@@ -294,7 +294,7 @@ export class TranscodingJobQueueBuilder extends AbstractJobBuilder { | |||
294 | const { videoUUID, isNewVideo, hasChildren } = options | 294 | const { videoUUID, isNewVideo, hasChildren } = options |
295 | 295 | ||
296 | return { | 296 | return { |
297 | type: 'merge-audio-to-webtorrent', | 297 | type: 'merge-audio-to-web-video', |
298 | resolution: DEFAULT_AUDIO_RESOLUTION, | 298 | resolution: DEFAULT_AUDIO_RESOLUTION, |
299 | fps: VIDEO_TRANSCODING_FPS.AUDIO_MERGE, | 299 | fps: VIDEO_TRANSCODING_FPS.AUDIO_MERGE, |
300 | videoUUID, | 300 | videoUUID, |
@@ -312,7 +312,7 @@ export class TranscodingJobQueueBuilder extends AbstractJobBuilder { | |||
312 | const { videoUUID, quickTranscode, isNewVideo, hasChildren } = options | 312 | const { videoUUID, quickTranscode, isNewVideo, hasChildren } = options |
313 | 313 | ||
314 | return { | 314 | return { |
315 | type: 'optimize-to-webtorrent', | 315 | type: 'optimize-to-web-video', |
316 | videoUUID, | 316 | videoUUID, |
317 | isNewVideo, | 317 | isNewVideo, |
318 | hasChildren, | 318 | hasChildren, |
diff --git a/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts b/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts index ba2a46f44..f0671bd7a 100644 --- a/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts +++ b/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts | |||
@@ -62,7 +62,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { | |||
62 | if (CONFIG.TRANSCODING.HLS.ENABLED === true) { | 62 | if (CONFIG.TRANSCODING.HLS.ENABLED === true) { |
63 | await new VODHLSTranscodingJobHandler().create({ | 63 | await new VODHLSTranscodingJobHandler().create({ |
64 | video, | 64 | video, |
65 | deleteWebVideoFiles: CONFIG.TRANSCODING.WEBTORRENT.ENABLED === false, | 65 | deleteWebVideoFiles: CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED === false, |
66 | resolution: maxResolution, | 66 | resolution: maxResolution, |
67 | fps, | 67 | fps, |
68 | isNewVideo, | 68 | isNewVideo, |
@@ -89,7 +89,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { | |||
89 | // --------------------------------------------------------------------------- | 89 | // --------------------------------------------------------------------------- |
90 | 90 | ||
91 | async createTranscodingJobs (options: { | 91 | async createTranscodingJobs (options: { |
92 | transcodingType: 'hls' | 'webtorrent' | 92 | transcodingType: 'hls' | 'webtorrent' | 'web-video' // TODO: remove webtorrent in v7 |
93 | video: MVideoFullLight | 93 | video: MVideoFullLight |
94 | resolutions: number[] | 94 | resolutions: number[] |
95 | isNewVideo: boolean | 95 | isNewVideo: boolean |
@@ -130,7 +130,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { | |||
130 | continue | 130 | continue |
131 | } | 131 | } |
132 | 132 | ||
133 | if (transcodingType === 'webtorrent') { | 133 | if (transcodingType === 'webtorrent' || transcodingType === 'web-video') { |
134 | await new VODWebVideoTranscodingJobHandler().create({ | 134 | await new VODWebVideoTranscodingJobHandler().create({ |
135 | video, | 135 | video, |
136 | resolution, | 136 | resolution, |
@@ -169,7 +169,7 @@ export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { | |||
169 | for (const resolution of resolutionsEnabled) { | 169 | for (const resolution of resolutionsEnabled) { |
170 | const fps = computeOutputFPS({ inputFPS: inputVideoFPS, resolution }) | 170 | const fps = computeOutputFPS({ inputFPS: inputVideoFPS, resolution }) |
171 | 171 | ||
172 | if (CONFIG.TRANSCODING.WEBTORRENT.ENABLED) { | 172 | if (CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED) { |
173 | await new VODWebVideoTranscodingJobHandler().create({ | 173 | await new VODWebVideoTranscodingJobHandler().create({ |
174 | video, | 174 | video, |
175 | resolution, | 175 | resolution, |
diff --git a/server/lib/transcoding/web-transcoding.ts b/server/lib/transcoding/web-transcoding.ts index 7cc8f20bc..f92d457a0 100644 --- a/server/lib/transcoding/web-transcoding.ts +++ b/server/lib/transcoding/web-transcoding.ts | |||
@@ -9,7 +9,8 @@ import { ffprobePromise, getVideoStreamDuration, getVideoStreamFPS, TranscodeVOD | |||
9 | import { VideoResolution, VideoStorage } from '@shared/models' | 9 | import { VideoResolution, VideoStorage } from '@shared/models' |
10 | import { CONFIG } from '../../initializers/config' | 10 | import { CONFIG } from '../../initializers/config' |
11 | import { VideoFileModel } from '../../models/video/video-file' | 11 | import { VideoFileModel } from '../../models/video/video-file' |
12 | import { generateWebTorrentVideoFilename } from '../paths' | 12 | import { JobQueue } from '../job-queue' |
13 | import { generateWebVideoFilename } from '../paths' | ||
13 | import { buildFileMetadata } from '../video-file' | 14 | import { buildFileMetadata } from '../video-file' |
14 | import { VideoPathManager } from '../video-path-manager' | 15 | import { VideoPathManager } from '../video-path-manager' |
15 | import { buildFFmpegVOD } from './shared' | 16 | import { buildFFmpegVOD } from './shared' |
@@ -62,10 +63,10 @@ export async function optimizeOriginalVideofile (options: { | |||
62 | // Important to do this before getVideoFilename() to take in account the new filename | 63 | // Important to do this before getVideoFilename() to take in account the new filename |
63 | inputVideoFile.resolution = resolution | 64 | inputVideoFile.resolution = resolution |
64 | inputVideoFile.extname = newExtname | 65 | inputVideoFile.extname = newExtname |
65 | inputVideoFile.filename = generateWebTorrentVideoFilename(resolution, newExtname) | 66 | inputVideoFile.filename = generateWebVideoFilename(resolution, newExtname) |
66 | inputVideoFile.storage = VideoStorage.FILE_SYSTEM | 67 | inputVideoFile.storage = VideoStorage.FILE_SYSTEM |
67 | 68 | ||
68 | const { videoFile } = await onWebTorrentVideoFileTranscoding({ | 69 | const { videoFile } = await onWebVideoFileTranscoding({ |
69 | video, | 70 | video, |
70 | videoFile: inputVideoFile, | 71 | videoFile: inputVideoFile, |
71 | videoOutputPath | 72 | videoOutputPath |
@@ -82,8 +83,8 @@ export async function optimizeOriginalVideofile (options: { | |||
82 | } | 83 | } |
83 | } | 84 | } |
84 | 85 | ||
85 | // Transcode the original video file to a lower resolution compatible with WebTorrent | 86 | // Transcode the original video file to a lower resolution compatible with web browsers |
86 | export async function transcodeNewWebTorrentResolution (options: { | 87 | export async function transcodeNewWebVideoResolution (options: { |
87 | video: MVideoFullLight | 88 | video: MVideoFullLight |
88 | resolution: VideoResolution | 89 | resolution: VideoResolution |
89 | fps: number | 90 | fps: number |
@@ -104,7 +105,7 @@ export async function transcodeNewWebTorrentResolution (options: { | |||
104 | const newVideoFile = new VideoFileModel({ | 105 | const newVideoFile = new VideoFileModel({ |
105 | resolution, | 106 | resolution, |
106 | extname: newExtname, | 107 | extname: newExtname, |
107 | filename: generateWebTorrentVideoFilename(resolution, newExtname), | 108 | filename: generateWebVideoFilename(resolution, newExtname), |
108 | size: 0, | 109 | size: 0, |
109 | videoId: video.id | 110 | videoId: video.id |
110 | }) | 111 | }) |
@@ -125,7 +126,7 @@ export async function transcodeNewWebTorrentResolution (options: { | |||
125 | 126 | ||
126 | await buildFFmpegVOD(job).transcode(transcodeOptions) | 127 | await buildFFmpegVOD(job).transcode(transcodeOptions) |
127 | 128 | ||
128 | return onWebTorrentVideoFileTranscoding({ video, videoFile: newVideoFile, videoOutputPath }) | 129 | return onWebVideoFileTranscoding({ video, videoFile: newVideoFile, videoOutputPath }) |
129 | }) | 130 | }) |
130 | 131 | ||
131 | return result | 132 | return result |
@@ -188,17 +189,18 @@ export async function mergeAudioVideofile (options: { | |||
188 | // Important to do this before getVideoFilename() to take in account the new file extension | 189 | // Important to do this before getVideoFilename() to take in account the new file extension |
189 | inputVideoFile.extname = newExtname | 190 | inputVideoFile.extname = newExtname |
190 | inputVideoFile.resolution = resolution | 191 | inputVideoFile.resolution = resolution |
191 | inputVideoFile.filename = generateWebTorrentVideoFilename(inputVideoFile.resolution, newExtname) | 192 | inputVideoFile.filename = generateWebVideoFilename(inputVideoFile.resolution, newExtname) |
192 | 193 | ||
193 | // ffmpeg generated a new video file, so update the video duration | 194 | // ffmpeg generated a new video file, so update the video duration |
194 | // See https://trac.ffmpeg.org/ticket/5456 | 195 | // See https://trac.ffmpeg.org/ticket/5456 |
195 | video.duration = await getVideoStreamDuration(videoOutputPath) | 196 | video.duration = await getVideoStreamDuration(videoOutputPath) |
196 | await video.save() | 197 | await video.save() |
197 | 198 | ||
198 | return onWebTorrentVideoFileTranscoding({ | 199 | return onWebVideoFileTranscoding({ |
199 | video, | 200 | video, |
200 | videoFile: inputVideoFile, | 201 | videoFile: inputVideoFile, |
201 | videoOutputPath | 202 | videoOutputPath, |
203 | wasAudioFile: true | ||
202 | }) | 204 | }) |
203 | }) | 205 | }) |
204 | 206 | ||
@@ -208,12 +210,13 @@ export async function mergeAudioVideofile (options: { | |||
208 | } | 210 | } |
209 | } | 211 | } |
210 | 212 | ||
211 | export async function onWebTorrentVideoFileTranscoding (options: { | 213 | export async function onWebVideoFileTranscoding (options: { |
212 | video: MVideoFullLight | 214 | video: MVideoFullLight |
213 | videoFile: MVideoFile | 215 | videoFile: MVideoFile |
214 | videoOutputPath: string | 216 | videoOutputPath: string |
217 | wasAudioFile?: boolean // default false | ||
215 | }) { | 218 | }) { |
216 | const { video, videoFile, videoOutputPath } = options | 219 | const { video, videoFile, videoOutputPath, wasAudioFile } = options |
217 | 220 | ||
218 | const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | 221 | const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) |
219 | 222 | ||
@@ -236,12 +239,23 @@ export async function onWebTorrentVideoFileTranscoding (options: { | |||
236 | 239 | ||
237 | await createTorrentAndSetInfoHash(video, videoFile) | 240 | await createTorrentAndSetInfoHash(video, videoFile) |
238 | 241 | ||
239 | const oldFile = await VideoFileModel.loadWebTorrentFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution }) | 242 | const oldFile = await VideoFileModel.loadWebVideoFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution }) |
240 | if (oldFile) await video.removeWebTorrentFile(oldFile) | 243 | if (oldFile) await video.removeWebVideoFile(oldFile) |
241 | 244 | ||
242 | await VideoFileModel.customUpsert(videoFile, 'video', undefined) | 245 | await VideoFileModel.customUpsert(videoFile, 'video', undefined) |
243 | video.VideoFiles = await video.$get('VideoFiles') | 246 | video.VideoFiles = await video.$get('VideoFiles') |
244 | 247 | ||
248 | if (wasAudioFile) { | ||
249 | await JobQueue.Instance.createJob({ | ||
250 | type: 'generate-video-storyboard' as 'generate-video-storyboard', | ||
251 | payload: { | ||
252 | videoUUID: video.uuid, | ||
253 | // No need to federate, we process these jobs sequentially | ||
254 | federate: false | ||
255 | } | ||
256 | }) | ||
257 | } | ||
258 | |||
245 | return { video, videoFile } | 259 | return { video, videoFile } |
246 | } finally { | 260 | } finally { |
247 | mutexReleaser() | 261 | mutexReleaser() |
diff --git a/server/lib/video-file.ts b/server/lib/video-file.ts index 88d48c945..46af67ccd 100644 --- a/server/lib/video-file.ts +++ b/server/lib/video-file.ts | |||
@@ -7,7 +7,7 @@ import { getFileSize } from '@shared/extra-utils' | |||
7 | import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, isAudioFile } from '@shared/ffmpeg' | 7 | import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, isAudioFile } from '@shared/ffmpeg' |
8 | import { VideoFileMetadata, VideoResolution } from '@shared/models' | 8 | import { VideoFileMetadata, VideoResolution } from '@shared/models' |
9 | import { lTags } from './object-storage/shared' | 9 | import { lTags } from './object-storage/shared' |
10 | import { generateHLSVideoFilename, generateWebTorrentVideoFilename } from './paths' | 10 | import { generateHLSVideoFilename, generateWebVideoFilename } from './paths' |
11 | import { VideoPathManager } from './video-path-manager' | 11 | import { VideoPathManager } from './video-path-manager' |
12 | 12 | ||
13 | async function buildNewFile (options: { | 13 | async function buildNewFile (options: { |
@@ -33,7 +33,7 @@ async function buildNewFile (options: { | |||
33 | } | 33 | } |
34 | 34 | ||
35 | videoFile.filename = mode === 'web-video' | 35 | videoFile.filename = mode === 'web-video' |
36 | ? generateWebTorrentVideoFilename(videoFile.resolution, videoFile.extname) | 36 | ? generateWebVideoFilename(videoFile.resolution, videoFile.extname) |
37 | : generateHLSVideoFilename(videoFile.resolution) | 37 | : generateHLSVideoFilename(videoFile.resolution) |
38 | 38 | ||
39 | return videoFile | 39 | return videoFile |
@@ -85,12 +85,12 @@ async function removeHLSFile (video: MVideoWithAllFiles, fileToDeleteId: number) | |||
85 | 85 | ||
86 | // --------------------------------------------------------------------------- | 86 | // --------------------------------------------------------------------------- |
87 | 87 | ||
88 | async function removeAllWebTorrentFiles (video: MVideoWithAllFiles) { | 88 | async function removeAllWebVideoFiles (video: MVideoWithAllFiles) { |
89 | const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | 89 | const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) |
90 | 90 | ||
91 | try { | 91 | try { |
92 | for (const file of video.VideoFiles) { | 92 | for (const file of video.VideoFiles) { |
93 | await video.removeWebTorrentFile(file) | 93 | await video.removeWebVideoFile(file) |
94 | await file.destroy() | 94 | await file.destroy() |
95 | } | 95 | } |
96 | 96 | ||
@@ -102,17 +102,17 @@ async function removeAllWebTorrentFiles (video: MVideoWithAllFiles) { | |||
102 | return video | 102 | return video |
103 | } | 103 | } |
104 | 104 | ||
105 | async function removeWebTorrentFile (video: MVideoWithAllFiles, fileToDeleteId: number) { | 105 | async function removeWebVideoFile (video: MVideoWithAllFiles, fileToDeleteId: number) { |
106 | const files = video.VideoFiles | 106 | const files = video.VideoFiles |
107 | 107 | ||
108 | if (files.length === 1) { | 108 | if (files.length === 1) { |
109 | return removeAllWebTorrentFiles(video) | 109 | return removeAllWebVideoFiles(video) |
110 | } | 110 | } |
111 | 111 | ||
112 | const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) | 112 | const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) |
113 | try { | 113 | try { |
114 | const toDelete = files.find(f => f.id === fileToDeleteId) | 114 | const toDelete = files.find(f => f.id === fileToDeleteId) |
115 | await video.removeWebTorrentFile(toDelete) | 115 | await video.removeWebVideoFile(toDelete) |
116 | await toDelete.destroy() | 116 | await toDelete.destroy() |
117 | 117 | ||
118 | video.VideoFiles = files.filter(f => f.id !== toDelete.id) | 118 | video.VideoFiles = files.filter(f => f.id !== toDelete.id) |
@@ -138,8 +138,8 @@ export { | |||
138 | 138 | ||
139 | removeHLSPlaylist, | 139 | removeHLSPlaylist, |
140 | removeHLSFile, | 140 | removeHLSFile, |
141 | removeAllWebTorrentFiles, | 141 | removeAllWebVideoFiles, |
142 | removeWebTorrentFile, | 142 | removeWebVideoFile, |
143 | 143 | ||
144 | buildFileMetadata | 144 | buildFileMetadata |
145 | } | 145 | } |
diff --git a/server/lib/video-path-manager.ts b/server/lib/video-path-manager.ts index 9953cae5d..133544bb2 100644 --- a/server/lib/video-path-manager.ts +++ b/server/lib/video-path-manager.ts | |||
@@ -8,7 +8,7 @@ import { DIRECTORIES } from '@server/initializers/constants' | |||
8 | import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '@server/types/models' | 8 | import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '@server/types/models' |
9 | import { buildUUID } from '@shared/extra-utils' | 9 | import { buildUUID } from '@shared/extra-utils' |
10 | import { VideoStorage } from '@shared/models' | 10 | import { VideoStorage } from '@shared/models' |
11 | import { makeHLSFileAvailable, makeWebTorrentFileAvailable } from './object-storage' | 11 | import { makeHLSFileAvailable, makeWebVideoFileAvailable } from './object-storage' |
12 | import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths' | 12 | import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths' |
13 | import { isVideoInPrivateDirectory } from './video-privacy' | 13 | import { isVideoInPrivateDirectory } from './video-privacy' |
14 | 14 | ||
@@ -78,7 +78,7 @@ class VideoPathManager { | |||
78 | } | 78 | } |
79 | 79 | ||
80 | return this.makeAvailableFactory( | 80 | return this.makeAvailableFactory( |
81 | () => makeWebTorrentFileAvailable(videoFile.filename, destination), | 81 | () => makeWebVideoFileAvailable(videoFile.filename, destination), |
82 | true, | 82 | true, |
83 | cb | 83 | cb |
84 | ) | 84 | ) |
diff --git a/server/lib/video-pre-import.ts b/server/lib/video-pre-import.ts index df67dc953..381f1f535 100644 --- a/server/lib/video-pre-import.ts +++ b/server/lib/video-pre-import.ts | |||
@@ -29,7 +29,8 @@ import { | |||
29 | } from '@server/types/models' | 29 | } from '@server/types/models' |
30 | import { ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState, VideoPrivacy, VideoState } from '@shared/models' | 30 | import { ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState, VideoPrivacy, VideoState } from '@shared/models' |
31 | import { getLocalVideoActivityPubUrl } from './activitypub/url' | 31 | import { getLocalVideoActivityPubUrl } from './activitypub/url' |
32 | import { updateVideoMiniatureFromExisting, updateVideoMiniatureFromUrl } from './thumbnail' | 32 | import { updateLocalVideoMiniatureFromExisting, updateLocalVideoMiniatureFromUrl } from './thumbnail' |
33 | import { VideoPasswordModel } from '@server/models/video/video-password' | ||
33 | 34 | ||
34 | class YoutubeDlImportError extends Error { | 35 | class YoutubeDlImportError extends Error { |
35 | code: YoutubeDlImportError.CODE | 36 | code: YoutubeDlImportError.CODE |
@@ -64,8 +65,9 @@ async function insertFromImportIntoDB (parameters: { | |||
64 | tags: string[] | 65 | tags: string[] |
65 | videoImportAttributes: FilteredModelAttributes<VideoImportModel> | 66 | videoImportAttributes: FilteredModelAttributes<VideoImportModel> |
66 | user: MUser | 67 | user: MUser |
68 | videoPasswords?: string[] | ||
67 | }): Promise<MVideoImportFormattable> { | 69 | }): Promise<MVideoImportFormattable> { |
68 | const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user } = parameters | 70 | const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user, videoPasswords } = parameters |
69 | 71 | ||
70 | const videoImport = await sequelizeTypescript.transaction(async t => { | 72 | const videoImport = await sequelizeTypescript.transaction(async t => { |
71 | const sequelizeOptions = { transaction: t } | 73 | const sequelizeOptions = { transaction: t } |
@@ -77,6 +79,10 @@ async function insertFromImportIntoDB (parameters: { | |||
77 | if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t) | 79 | if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t) |
78 | if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t) | 80 | if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t) |
79 | 81 | ||
82 | if (videoCreated.privacy === VideoPrivacy.PASSWORD_PROTECTED) { | ||
83 | await VideoPasswordModel.addPasswords(videoPasswords, video.id, t) | ||
84 | } | ||
85 | |||
80 | await autoBlacklistVideoIfNeeded({ | 86 | await autoBlacklistVideoIfNeeded({ |
81 | video: videoCreated, | 87 | video: videoCreated, |
82 | user, | 88 | user, |
@@ -208,7 +214,8 @@ async function buildYoutubeDLImport (options: { | |||
208 | state: VideoImportState.PENDING, | 214 | state: VideoImportState.PENDING, |
209 | userId: user.id, | 215 | userId: user.id, |
210 | videoChannelSyncId: channelSync?.id | 216 | videoChannelSyncId: channelSync?.id |
211 | } | 217 | }, |
218 | videoPasswords: importDataOverride.videoPasswords | ||
212 | }) | 219 | }) |
213 | 220 | ||
214 | // Get video subtitles | 221 | // Get video subtitles |
@@ -249,19 +256,22 @@ async function forgeThumbnail ({ inputPath, video, downloadUrl, type }: { | |||
249 | type: ThumbnailType | 256 | type: ThumbnailType |
250 | }): Promise<MThumbnail> { | 257 | }): Promise<MThumbnail> { |
251 | if (inputPath) { | 258 | if (inputPath) { |
252 | return updateVideoMiniatureFromExisting({ | 259 | return updateLocalVideoMiniatureFromExisting({ |
253 | inputPath, | 260 | inputPath, |
254 | video, | 261 | video, |
255 | type, | 262 | type, |
256 | automaticallyGenerated: false | 263 | automaticallyGenerated: false |
257 | }) | 264 | }) |
258 | } else if (downloadUrl) { | 265 | } |
266 | |||
267 | if (downloadUrl) { | ||
259 | try { | 268 | try { |
260 | return await updateVideoMiniatureFromUrl({ downloadUrl, video, type }) | 269 | return await updateLocalVideoMiniatureFromUrl({ downloadUrl, video, type }) |
261 | } catch (err) { | 270 | } catch (err) { |
262 | logger.warn('Cannot process thumbnail %s from youtube-dl.', downloadUrl, { err }) | 271 | logger.warn('Cannot process thumbnail %s from youtube-dl.', downloadUrl, { err }) |
263 | } | 272 | } |
264 | } | 273 | } |
274 | |||
265 | return null | 275 | return null |
266 | } | 276 | } |
267 | 277 | ||
diff --git a/server/lib/video-privacy.ts b/server/lib/video-privacy.ts index 41f9d62b3..5dd4d9781 100644 --- a/server/lib/video-privacy.ts +++ b/server/lib/video-privacy.ts | |||
@@ -4,7 +4,13 @@ import { logger } from '@server/helpers/logger' | |||
4 | import { DIRECTORIES } from '@server/initializers/constants' | 4 | import { DIRECTORIES } from '@server/initializers/constants' |
5 | import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' | 5 | import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' |
6 | import { VideoPrivacy, VideoStorage } from '@shared/models' | 6 | import { VideoPrivacy, VideoStorage } from '@shared/models' |
7 | import { updateHLSFilesACL, updateWebTorrentFileACL } from './object-storage' | 7 | import { updateHLSFilesACL, updateWebVideoFileACL } from './object-storage' |
8 | |||
9 | const validPrivacySet = new Set([ | ||
10 | VideoPrivacy.PRIVATE, | ||
11 | VideoPrivacy.INTERNAL, | ||
12 | VideoPrivacy.PASSWORD_PROTECTED | ||
13 | ]) | ||
8 | 14 | ||
9 | function setVideoPrivacy (video: MVideo, newPrivacy: VideoPrivacy) { | 15 | function setVideoPrivacy (video: MVideo, newPrivacy: VideoPrivacy) { |
10 | if (video.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) { | 16 | if (video.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) { |
@@ -14,8 +20,8 @@ function setVideoPrivacy (video: MVideo, newPrivacy: VideoPrivacy) { | |||
14 | video.privacy = newPrivacy | 20 | video.privacy = newPrivacy |
15 | } | 21 | } |
16 | 22 | ||
17 | function isVideoInPrivateDirectory (privacy: VideoPrivacy) { | 23 | function isVideoInPrivateDirectory (privacy) { |
18 | return privacy === VideoPrivacy.PRIVATE || privacy === VideoPrivacy.INTERNAL | 24 | return validPrivacySet.has(privacy) |
19 | } | 25 | } |
20 | 26 | ||
21 | function isVideoInPublicDirectory (privacy: VideoPrivacy) { | 27 | function isVideoInPublicDirectory (privacy: VideoPrivacy) { |
@@ -61,9 +67,9 @@ async function moveFiles (options: { | |||
61 | 67 | ||
62 | for (const file of video.VideoFiles) { | 68 | for (const file of video.VideoFiles) { |
63 | if (file.storage === VideoStorage.FILE_SYSTEM) { | 69 | if (file.storage === VideoStorage.FILE_SYSTEM) { |
64 | await moveWebTorrentFileOnFS(type, video, file) | 70 | await moveWebVideoFileOnFS(type, video, file) |
65 | } else { | 71 | } else { |
66 | await updateWebTorrentFileACL(video, file) | 72 | await updateWebVideoFileACL(video, file) |
67 | } | 73 | } |
68 | } | 74 | } |
69 | 75 | ||
@@ -78,22 +84,22 @@ async function moveFiles (options: { | |||
78 | } | 84 | } |
79 | } | 85 | } |
80 | 86 | ||
81 | async function moveWebTorrentFileOnFS (type: MoveType, video: MVideo, file: MVideoFile) { | 87 | async function moveWebVideoFileOnFS (type: MoveType, video: MVideo, file: MVideoFile) { |
82 | const directories = getWebTorrentDirectories(type) | 88 | const directories = getWebVideoDirectories(type) |
83 | 89 | ||
84 | const source = join(directories.old, file.filename) | 90 | const source = join(directories.old, file.filename) |
85 | const destination = join(directories.new, file.filename) | 91 | const destination = join(directories.new, file.filename) |
86 | 92 | ||
87 | try { | 93 | try { |
88 | logger.info('Moving WebTorrent files of %s after privacy change (%s -> %s).', video.uuid, source, destination) | 94 | logger.info('Moving web video files of %s after privacy change (%s -> %s).', video.uuid, source, destination) |
89 | 95 | ||
90 | await move(source, destination) | 96 | await move(source, destination) |
91 | } catch (err) { | 97 | } catch (err) { |
92 | logger.error('Cannot move webtorrent file %s to %s after privacy change', source, destination, { err }) | 98 | logger.error('Cannot move web video file %s to %s after privacy change', source, destination, { err }) |
93 | } | 99 | } |
94 | } | 100 | } |
95 | 101 | ||
96 | function getWebTorrentDirectories (moveType: MoveType) { | 102 | function getWebVideoDirectories (moveType: MoveType) { |
97 | if (moveType === 'private-to-public') { | 103 | if (moveType === 'private-to-public') { |
98 | return { old: DIRECTORIES.VIDEOS.PRIVATE, new: DIRECTORIES.VIDEOS.PUBLIC } | 104 | return { old: DIRECTORIES.VIDEOS.PRIVATE, new: DIRECTORIES.VIDEOS.PUBLIC } |
99 | } | 105 | } |
diff --git a/server/lib/video-studio.ts b/server/lib/video-studio.ts index 0d3db8f60..f549a7084 100644 --- a/server/lib/video-studio.ts +++ b/server/lib/video-studio.ts | |||
@@ -12,7 +12,7 @@ import { JobQueue } from './job-queue' | |||
12 | import { VideoStudioTranscodingJobHandler } from './runners' | 12 | import { VideoStudioTranscodingJobHandler } from './runners' |
13 | import { createOptimizeOrMergeAudioJobs } from './transcoding/create-transcoding-job' | 13 | import { createOptimizeOrMergeAudioJobs } from './transcoding/create-transcoding-job' |
14 | import { getTranscodingJobPriority } from './transcoding/transcoding-priority' | 14 | import { getTranscodingJobPriority } from './transcoding/transcoding-priority' |
15 | import { buildNewFile, removeHLSPlaylist, removeWebTorrentFile } from './video-file' | 15 | import { buildNewFile, removeHLSPlaylist, removeWebVideoFile } from './video-file' |
16 | import { VideoPathManager } from './video-path-manager' | 16 | import { VideoPathManager } from './video-path-manager' |
17 | 17 | ||
18 | const lTags = loggerTagsFactory('video-studio') | 18 | const lTags = loggerTagsFactory('video-studio') |
@@ -119,12 +119,12 @@ export async function onVideoStudioEnded (options: { | |||
119 | // Private | 119 | // Private |
120 | // --------------------------------------------------------------------------- | 120 | // --------------------------------------------------------------------------- |
121 | 121 | ||
122 | async function removeAllFiles (video: MVideoWithAllFiles, webTorrentFileException: MVideoFile) { | 122 | async function removeAllFiles (video: MVideoWithAllFiles, webVideoFileException: MVideoFile) { |
123 | await removeHLSPlaylist(video) | 123 | await removeHLSPlaylist(video) |
124 | 124 | ||
125 | for (const file of video.VideoFiles) { | 125 | for (const file of video.VideoFiles) { |
126 | if (file.id === webTorrentFileException.id) continue | 126 | if (file.id === webVideoFileException.id) continue |
127 | 127 | ||
128 | await removeWebTorrentFile(video, file.id) | 128 | await removeWebVideoFile(video, file.id) |
129 | } | 129 | } |
130 | } | 130 | } |
diff --git a/server/lib/video-tokens-manager.ts b/server/lib/video-tokens-manager.ts index 660533528..e28e55cf7 100644 --- a/server/lib/video-tokens-manager.ts +++ b/server/lib/video-tokens-manager.ts | |||
@@ -12,26 +12,34 @@ class VideoTokensManager { | |||
12 | 12 | ||
13 | private static instance: VideoTokensManager | 13 | private static instance: VideoTokensManager |
14 | 14 | ||
15 | private readonly lruCache = new LRUCache<string, { videoUUID: string, user: MUserAccountUrl }>({ | 15 | private readonly lruCache = new LRUCache<string, { videoUUID: string, user?: MUserAccountUrl }>({ |
16 | max: LRU_CACHE.VIDEO_TOKENS.MAX_SIZE, | 16 | max: LRU_CACHE.VIDEO_TOKENS.MAX_SIZE, |
17 | ttl: LRU_CACHE.VIDEO_TOKENS.TTL | 17 | ttl: LRU_CACHE.VIDEO_TOKENS.TTL |
18 | }) | 18 | }) |
19 | 19 | ||
20 | private constructor () {} | 20 | private constructor () {} |
21 | 21 | ||
22 | create (options: { | 22 | createForAuthUser (options: { |
23 | user: MUserAccountUrl | 23 | user: MUserAccountUrl |
24 | videoUUID: string | 24 | videoUUID: string |
25 | }) { | 25 | }) { |
26 | const token = buildUUID() | 26 | const { token, expires } = this.generateVideoToken() |
27 | |||
28 | const expires = new Date(new Date().getTime() + LRU_CACHE.VIDEO_TOKENS.TTL) | ||
29 | 27 | ||
30 | this.lruCache.set(token, pick(options, [ 'user', 'videoUUID' ])) | 28 | this.lruCache.set(token, pick(options, [ 'user', 'videoUUID' ])) |
31 | 29 | ||
32 | return { token, expires } | 30 | return { token, expires } |
33 | } | 31 | } |
34 | 32 | ||
33 | createForPasswordProtectedVideo (options: { | ||
34 | videoUUID: string | ||
35 | }) { | ||
36 | const { token, expires } = this.generateVideoToken() | ||
37 | |||
38 | this.lruCache.set(token, pick(options, [ 'videoUUID' ])) | ||
39 | |||
40 | return { token, expires } | ||
41 | } | ||
42 | |||
35 | hasToken (options: { | 43 | hasToken (options: { |
36 | token: string | 44 | token: string |
37 | videoUUID: string | 45 | videoUUID: string |
@@ -54,6 +62,13 @@ class VideoTokensManager { | |||
54 | static get Instance () { | 62 | static get Instance () { |
55 | return this.instance || (this.instance = new this()) | 63 | return this.instance || (this.instance = new this()) |
56 | } | 64 | } |
65 | |||
66 | private generateVideoToken () { | ||
67 | const token = buildUUID() | ||
68 | const expires = new Date(new Date().getTime() + LRU_CACHE.VIDEO_TOKENS.TTL) | ||
69 | |||
70 | return { token, expires } | ||
71 | } | ||
57 | } | 72 | } |
58 | 73 | ||
59 | // --------------------------------------------------------------------------- | 74 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/video-urls.ts b/server/lib/video-urls.ts index 64c2c9bf9..0597488ad 100644 --- a/server/lib/video-urls.ts +++ b/server/lib/video-urls.ts | |||
@@ -9,7 +9,7 @@ function generateHLSRedundancyUrl (video: MVideo, playlist: MStreamingPlaylist) | |||
9 | return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + playlist.getStringType() + '/' + video.uuid | 9 | return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + playlist.getStringType() + '/' + video.uuid |
10 | } | 10 | } |
11 | 11 | ||
12 | function generateWebTorrentRedundancyUrl (file: MVideoFile) { | 12 | function generateWebVideoRedundancyUrl (file: MVideoFile) { |
13 | return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + file.filename | 13 | return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + file.filename |
14 | } | 14 | } |
15 | 15 | ||
@@ -26,6 +26,6 @@ function getLocalVideoFileMetadataUrl (video: MVideoUUID, videoFile: MVideoFile) | |||
26 | export { | 26 | export { |
27 | getLocalVideoFileMetadataUrl, | 27 | getLocalVideoFileMetadataUrl, |
28 | 28 | ||
29 | generateWebTorrentRedundancyUrl, | 29 | generateWebVideoRedundancyUrl, |
30 | generateHLSRedundancyUrl | 30 | generateHLSRedundancyUrl |
31 | } | 31 | } |
diff --git a/server/lib/video.ts b/server/lib/video.ts index 588dc553f..362c861a5 100644 --- a/server/lib/video.ts +++ b/server/lib/video.ts | |||
@@ -10,7 +10,7 @@ import { FilteredModelAttributes } from '@server/types' | |||
10 | import { MThumbnail, MVideoFullLight, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' | 10 | import { MThumbnail, MVideoFullLight, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' |
11 | import { ManageVideoTorrentPayload, ThumbnailType, VideoCreate, VideoPrivacy, VideoState } from '@shared/models' | 11 | import { ManageVideoTorrentPayload, ThumbnailType, VideoCreate, VideoPrivacy, VideoState } from '@shared/models' |
12 | import { CreateJobArgument, JobQueue } from './job-queue/job-queue' | 12 | import { CreateJobArgument, JobQueue } from './job-queue/job-queue' |
13 | import { updateVideoMiniatureFromExisting } from './thumbnail' | 13 | import { updateLocalVideoMiniatureFromExisting } from './thumbnail' |
14 | import { moveFilesIfPrivacyChanged } from './video-privacy' | 14 | import { moveFilesIfPrivacyChanged } from './video-privacy' |
15 | 15 | ||
16 | function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> { | 16 | function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes<VideoModel> { |
@@ -55,7 +55,7 @@ async function buildVideoThumbnailsFromReq (options: { | |||
55 | const fields = files?.[p.fieldName] | 55 | const fields = files?.[p.fieldName] |
56 | 56 | ||
57 | if (fields) { | 57 | if (fields) { |
58 | return updateVideoMiniatureFromExisting({ | 58 | return updateLocalVideoMiniatureFromExisting({ |
59 | inputPath: fields[0].path, | 59 | inputPath: fields[0].path, |
60 | video, | 60 | video, |
61 | type: p.type, | 61 | type: p.type, |
diff --git a/server/lib/worker/workers/image-downloader.ts b/server/lib/worker/workers/image-downloader.ts index 4b32f723e..209594589 100644 --- a/server/lib/worker/workers/image-downloader.ts +++ b/server/lib/worker/workers/image-downloader.ts | |||
@@ -24,6 +24,8 @@ async function downloadImage (options: { | |||
24 | 24 | ||
25 | throw err | 25 | throw err |
26 | } | 26 | } |
27 | |||
28 | return destPath | ||
27 | } | 29 | } |
28 | 30 | ||
29 | module.exports = downloadImage | 31 | module.exports = downloadImage |